862 files changed, 98700 insertions, 19233 deletions

diff --git a/sys/contrib/dev/acpica/changes.txt b/sys/contrib/dev/acpica/changes.txt
index 435540b254f1..4e3cf4f2f41c 100644
--- a/sys/contrib/dev/acpica/changes.txt
+++ b/sys/contrib/dev/acpica/changes.txt
@@ -1,11 +1,29 @@
 ----------------------------------------
+7 August 2025. Summary of changes for version 20250807:
+
+Major changes:
+
+Added option to skip the global lock for SMM - Huacai Chen
+
+Fixed non-NUL terminated string implementations - Ahmed Salem
+
+Fixed CCEL and CDAT templates - Ahmed Salem
+
+Fixed a major Linux kernel bug (UAF) that was triggered by unequal number of method parameters (definition) vs arguments (invocation) in different places - Peter Williams, Hans de Goede, Rafael Wysocki
+
+Define distinct D3 states (D3Hot and D3Cold) that help clarify the device behavior support - Aymeric Wibo
+
+A few cleanups, improvements to existing table supports, small fixes, spelling corrections etc.
+
+
+----------------------------------------
 4 April 2025. Summary of changes for version 20250404:
 
 Major changes:
 
 Update all the copyright continuation year to 2025 in the license header of all files
 
-Add complete support for 3 new ACPI tables ? MRRM,ERDT and RIMT (Tony Luck & V L Sunil)
+Add complete support for 3 new ACPI tables - MRRM,ERDT and RIMT (Tony Luck & V L Sunil)
 
 Add a license file to the project which is a great improvement (Dionna Glaze)
 
@@ -21,11 +39,11 @@ Major changes:
 
 Fix 2 critical CVE addressing memory leaks - Seunghun Han
 
-EINJ V2 updates ? Zaid Alali (Ampere Computing)
+EINJ V2 updates - Zaid Alali (Ampere Computing)
 
-CDAT updates ? Ira Weiny (Intel Corporation)
+CDAT updates - Ira Weiny (Intel Corporation)
 
-Fix mutex handling, don?t release ones that were never acquired ? Daniil Tatianin
+Fix mutex handling, do not release ones that were never acquired - Daniil Tatianin
 
 Experiment with new tag name format Ryyyy_mm_dd to solve chronological sorting problems
 
@@ -39,7 +57,7 @@ Fix the acpixf.h file which caused issues for the last release (before this) 202
 
 Fix the pointer offset for the SLIC table
 
-Verify the local environment and GitHub commits are all in sync which was a problem with the second from last release (before this)20240322 (aka 20240323 – date issue) 
+Verify the local environment and GitHub commits are all in sync which was a problem with the second from last release (before this)20240322 (aka 20240323 - date issue) 
 
 
 
diff --git a/sys/contrib/dev/acpica/common/adisasm.c b/sys/contrib/dev/acpica/common/adisasm.c
index 96cd6c7f5d3c..83125098cbd1 100644
--- a/sys/contrib/dev/acpica/common/adisasm.c
+++ b/sys/contrib/dev/acpica/common/adisasm.c
@@ -481,12 +481,12 @@ AdDisassembleOneTable (
             "FieldName : FieldValue (in hex)\n */\n\n");
 
         AcpiDmDumpDataTable (Table);
-        fprintf (stderr, "Acpi Data Table [%4.4s] decoded\n",
+        fprintf (stdout, "Acpi Data Table [%4.4s] decoded\n",
             AcpiGbl_CDAT ? (char *) AcpiGbl_CDAT : Table->Signature);
 
         if (File)
         {
-            fprintf (stderr, "Formatted output:  %s - %u bytes\n",
+            fprintf (stdout, "Formatted output:  %s - %u bytes\n",
                 DisasmFilename, CmGetFileSize (File));
         }
 
@@ -584,16 +584,16 @@ AdDisassembleOneTable (
 
         AcpiDmDumpDataTable (Table);
 
-        fprintf (stderr, "Disassembly completed\n");
+        fprintf (stdout, "Disassembly completed\n");
         if (File)
         {
-            fprintf (stderr, "ASL Output:    %s - %u bytes\n",
+            fprintf (stdout, "ASL Output:    %s - %u bytes\n",
                 DisasmFilename, CmGetFileSize (File));
         }
 
         if (AslGbl_MapfileFlag)
         {
-            fprintf (stderr, "%14s %s - %u bytes\n",
+            fprintf (stdout, "%14s %s - %u bytes\n",
                 AslGbl_FileDescs[ASL_FILE_MAP_OUTPUT].ShortDescription,
                 AslGbl_Files[ASL_FILE_MAP_OUTPUT].Filename,
                 FlGetFileSize (ASL_FILE_MAP_OUTPUT));
@@ -630,7 +630,7 @@ AdReparseOneTable (
     ACPI_COMMENT_ADDR_NODE  *AddrListHead;
 
 
-    fprintf (stderr,
+    fprintf (stdout,
         "\nFound %u external control methods, "
         "reparsing with new information\n",
         AcpiDmGetUnresolvedExternalMethodCount ());
diff --git a/sys/contrib/dev/acpica/common/ahtable.c b/sys/contrib/dev/acpica/common/ahtable.c
index 898b2d09f609..587bf61016f0 100644
--- a/sys/contrib/dev/acpica/common/ahtable.c
+++ b/sys/contrib/dev/acpica/common/ahtable.c
@@ -265,6 +265,7 @@ const AH_TABLE      AcpiGbl_SupportedTables[] =
     {ACPI_SIG_SSDT, "Secondary System Description Table (AML table)"},
     {ACPI_SIG_STAO, "Status Override Table"},
     {ACPI_SIG_SVKL, "Storage Volume Key Location Table"},
+    {ACPI_SIG_SWFT, "SoundWire File Table"},
     {ACPI_SIG_TCPA, "Trusted Computing Platform Alliance Table"},
     {ACPI_SIG_TDEL, "TD-Event Log Table"},
     {ACPI_SIG_TPM2, "Trusted Platform Module hardware interface Table"},
diff --git a/sys/contrib/dev/acpica/common/dmtable.c b/sys/contrib/dev/acpica/common/dmtable.c
index fcff97a304ae..702f4f7965e4 100644
--- a/sys/contrib/dev/acpica/common/dmtable.c
+++ b/sys/contrib/dev/acpica/common/dmtable.c
@@ -721,6 +721,7 @@ const ACPI_DMTABLE_DATA     AcpiDmTableData[] =
     {ACPI_SIG_SRAT, NULL,                   AcpiDmDumpSrat, DtCompileSrat,  TemplateSrat},
     {ACPI_SIG_STAO, NULL,                   AcpiDmDumpStao, DtCompileStao,  TemplateStao},
     {ACPI_SIG_SVKL, AcpiDmTableInfoSvkl,    AcpiDmDumpSvkl, DtCompileSvkl,  TemplateSvkl},
+    {ACPI_SIG_SWFT, NULL,                   NULL,           NULL,           NULL},
     {ACPI_SIG_TCPA, NULL,                   AcpiDmDumpTcpa, DtCompileTcpa,  TemplateTcpa},
     {ACPI_SIG_TDEL, AcpiDmTableInfoTdel,    NULL,           NULL,           TemplateTdel},
     {ACPI_SIG_TPM2, AcpiDmTableInfoTpm2,    AcpiDmDumpTpm2, DtCompileTpm2,  TemplateTpm2},
diff --git a/sys/contrib/dev/acpica/common/dmtbdump2.c b/sys/contrib/dev/acpica/common/dmtbdump2.c
index 822920d2ea94..d29a60be0f67 100644
--- a/sys/contrib/dev/acpica/common/dmtbdump2.c
+++ b/sys/contrib/dev/acpica/common/dmtbdump2.c
@@ -2637,7 +2637,7 @@ AcpiDmDumpRhct (
                     RhctIsaString, RhctIsaString->IsaLength, AcpiDmTableInfoRhctIsa1);
             if (Subtable->Length > IsaPadOffset)
             {
-                Status = AcpiDmDumpTable (Table->Length, Offset + SubtableOffset,
+                Status = AcpiDmDumpTable (Table->Length, Offset + IsaPadOffset,
                          ACPI_ADD_PTR (UINT8, Subtable, IsaPadOffset),
                          (Subtable->Length - IsaPadOffset), AcpiDmTableInfoRhctIsaPad);
             }
diff --git a/sys/contrib/dev/acpica/common/dmtbinfo2.c b/sys/contrib/dev/acpica/common/dmtbinfo2.c
index 9ecf877fcfb0..b7c6d3b8d536 100644
--- a/sys/contrib/dev/acpica/common/dmtbinfo2.c
+++ b/sys/contrib/dev/acpica/common/dmtbinfo2.c
@@ -2180,7 +2180,7 @@ ACPI_DMTABLE_INFO           AcpiDmTableInfoRhct[] =
 ACPI_DMTABLE_INFO           AcpiDmTableInfoRhctNodeHdr[] =
 {
     {ACPI_DMT_RHCT,      ACPI_RHCTH_OFFSET (Type),            "Subtable Type", 0},
-    {ACPI_DMT_UINT16,    ACPI_RHCTH_OFFSET (Length),          "Length", 0},
+    {ACPI_DMT_UINT16,    ACPI_RHCTH_OFFSET (Length),          "Length", DT_LENGTH},
     {ACPI_DMT_UINT16,    ACPI_RHCTH_OFFSET (Revision),        "Revision", 0},
     ACPI_DMT_TERMINATOR
 };
diff --git a/sys/contrib/dev/acpica/common/dmtbinfo3.c b/sys/contrib/dev/acpica/common/dmtbinfo3.c
index 75b580e0d890..0935fc86aff9 100644
--- a/sys/contrib/dev/acpica/common/dmtbinfo3.c
+++ b/sys/contrib/dev/acpica/common/dmtbinfo3.c
@@ -200,7 +200,7 @@ ACPI_DMTABLE_INFO           AcpiDmTableInfoCcel[] =
 {
     {ACPI_DMT_UINT8,    ACPI_CCEL_OFFSET (CCType),                  "CC Type", 0},
     {ACPI_DMT_UINT8,    ACPI_CCEL_OFFSET (CCSubType),               "CC Sub Type", 0},
-    {ACPI_DMT_UINT32,   ACPI_CCEL_OFFSET (Reserved),                "Reserved", 0},
+    {ACPI_DMT_UINT16,   ACPI_CCEL_OFFSET (Reserved),                "Reserved", 0},
     {ACPI_DMT_UINT64,   ACPI_CCEL_OFFSET (LogAreaMinimumLength),    "Log Area Minimum Length", 0},
     {ACPI_DMT_UINT64,   ACPI_CCEL_OFFSET (LogAreaStartAddress),     "Log Area Start Address", 0},
     ACPI_DMT_TERMINATOR
diff --git a/sys/contrib/dev/acpica/compiler/aslanalyze.c b/sys/contrib/dev/acpica/compiler/aslanalyze.c
index 17e2674817a9..625611a630de 100644
--- a/sys/contrib/dev/acpica/compiler/aslanalyze.c
+++ b/sys/contrib/dev/acpica/compiler/aslanalyze.c
@@ -572,10 +572,22 @@ ApCheckForGpeNameConflict (
     ACPI_PARSE_OBJECT       *NextOp;
     UINT32                  GpeNumber;
     char                    Name[ACPI_NAMESEG_SIZE + 1];
-    char                    Target[ACPI_NAMESEG_SIZE];
+    char                    Target[ACPI_NAMESEG_SIZE] ACPI_NONSTRING;
 
 
-    /* Need a null-terminated string version of NameSeg */
+    /**
+     * Need a null-terminated string version of NameSeg
+     *
+     * NOTE: during a review on Name[ACPI_NAMESEG_SIZE + 1] having an extra
+     *       byte[1], compiler testing exhibited a difference in behavior between
+     *       GCC and Clang[2] (at least; MSVC may also exhibit the same) in
+     *       how optimization is done. The extra byte is needed to ensure
+     *       the signature does not get mangled, subsequently avoiding
+     *       GpeNumber being a completely different return value from strtoul.
+     *
+     *       [1] https://github.com/acpica/acpica/pull/1019#discussion_r2058687704
+     *       [2] https://github.com/acpica/acpica/pull/1019#discussion_r2061953039
+     */
 
     ACPI_MOVE_32_TO_32 (Name, Op->Asl.NameSeg);
     Name[ACPI_NAMESEG_SIZE] = 0;
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2s.c b/sys/contrib/dev/acpica/compiler/aslrestype2s.c
index 096862290384..f47402d4e025 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype2s.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype2s.c
@@ -1469,7 +1469,7 @@ RsDoCsi2SerialBusDescriptor (
 
         case 2: /* Local Port Instance [Integer] (_PRT) */
 
-            RsSetFlagBits16 ((UINT16 *) &Descriptor->Csi2SerialBus.TypeSpecificFlags, InitializerOp, 0, 0);
+            RsSetFlagBits16 ((UINT16 *) &Descriptor->Csi2SerialBus.TypeSpecificFlags, InitializerOp, 2, 0);
             RsCreateMultiBitField (InitializerOp, ACPI_RESTAG_LOCALPORT,
                 CurrentByteOffset + ASL_RESDESC_OFFSET (Csi2SerialBus.TypeSpecificFlags), 2, 6);
             break;
diff --git a/sys/contrib/dev/acpica/compiler/dttable2.c b/sys/contrib/dev/acpica/compiler/dttable2.c
index 6203a382ad62..754880346299 100644
--- a/sys/contrib/dev/acpica/compiler/dttable2.c
+++ b/sys/contrib/dev/acpica/compiler/dttable2.c
@@ -1929,24 +1929,30 @@ DtCompileRhct (
 {
     ACPI_STATUS             Status;
     ACPI_RHCT_NODE_HEADER   *RhctHeader;
-    ACPI_RHCT_HART_INFO     *RhctHartInfo = NULL;
+    ACPI_RHCT_HART_INFO     *RhctHartInfo;
     DT_SUBTABLE             *Subtable;
     DT_SUBTABLE             *ParentTable;
     ACPI_DMTABLE_INFO       *InfoTable;
     DT_FIELD                **PFieldList = (DT_FIELD **) List;
     DT_FIELD                *SubtableStart;
+    ACPI_TABLE_RHCT         *Table;
+    BOOLEAN                 FirstNode = TRUE;
 
 
     /* Compile the main table */
 
+    ParentTable = DtPeekSubtable ();
     Status = DtCompileTable (PFieldList, AcpiDmTableInfoRhct,
         &Subtable);
     if (ACPI_FAILURE (Status))
     {
         return (Status);
     }
+    DtInsertSubtable (ParentTable, Subtable);
+    Table = ACPI_CAST_PTR (ACPI_TABLE_RHCT, ParentTable->Buffer);
+    Table->NodeCount = 0;
+    Table->NodeOffset = sizeof (ACPI_TABLE_RHCT);
 
-    ParentTable = DtPeekSubtable ();
     while (*PFieldList)
     {
         SubtableStart = *PFieldList;
@@ -1961,7 +1967,10 @@ DtCompileRhct (
         }
         DtInsertSubtable (ParentTable, Subtable);
         RhctHeader = ACPI_CAST_PTR (ACPI_RHCT_NODE_HEADER, Subtable->Buffer);
-        RhctHeader->Length = (UINT16)(Subtable->Length);
+
+        DtPushSubtable (Subtable);
+        ParentTable = DtPeekSubtable ();
+        Table->NodeCount++;
 
         switch (RhctHeader->Type)
         {
@@ -1999,37 +2008,54 @@ DtCompileRhct (
             return (Status);
         }
         DtInsertSubtable (ParentTable, Subtable);
-        RhctHeader->Length += (UINT16)(Subtable->Length);
+        if (FirstNode)
+        {
+            Table->NodeOffset = ACPI_PTR_DIFF(ParentTable->Buffer, Table);
+            FirstNode = FALSE;
+        }
 
         /* Compile RHCT subtable additionals */
 
         switch (RhctHeader->Type)
         {
-        case ACPI_RHCT_NODE_TYPE_HART_INFO:
+        case ACPI_RHCT_NODE_TYPE_ISA_STRING:
 
-            RhctHartInfo = ACPI_SUB_PTR (ACPI_RHCT_HART_INFO,
-                Subtable->Buffer, sizeof (ACPI_RHCT_NODE_HEADER));
-            if (RhctHartInfo)
+            /*
+             * Padding - Variable-length data
+             * Optionally allows the padding of the ISA string to be used
+             * for filling this field.
+             */
+            Status = DtCompileTable (PFieldList, AcpiDmTableInfoRhctIsaPad,
+                                     &Subtable);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+            if (Subtable)
             {
+                DtInsertSubtable (ParentTable, Subtable);
+            }
+            break;
 
-                RhctHartInfo->NumOffsets = 0;
-                while (*PFieldList)
-                {
-                    Status = DtCompileTable (PFieldList,
-                        AcpiDmTableInfoRhctHartInfo2, &Subtable);
-                    if (ACPI_FAILURE (Status))
-                    {
-                        return (Status);
-                    }
-                    if (!Subtable)
-                    {
-                        break;
-                    }
+        case ACPI_RHCT_NODE_TYPE_HART_INFO:
 
-                    DtInsertSubtable (ParentTable, Subtable);
-                    RhctHeader->Length += (UINT16)(Subtable->Length);
-                    RhctHartInfo->NumOffsets++;
+            RhctHartInfo = ACPI_CAST_PTR (ACPI_RHCT_HART_INFO,
+                Subtable->Buffer);
+            RhctHartInfo->NumOffsets = 0;
+            while (*PFieldList)
+            {
+                Status = DtCompileTable (PFieldList,
+                    AcpiDmTableInfoRhctHartInfo2, &Subtable);
+                if (ACPI_FAILURE (Status))
+                {
+                    return (Status);
                 }
+                if (!Subtable)
+                {
+                    break;
+                }
+                DtInsertSubtable (ParentTable, Subtable);
+                RhctHartInfo->NumOffsets++;
             }
             break;
 
@@ -2037,6 +2063,9 @@ DtCompileRhct (
 
             break;
         }
+
+        DtPopSubtable ();
+        ParentTable = DtPeekSubtable ();
     }
 
     return (AE_OK);
diff --git a/sys/contrib/dev/acpica/compiler/dttemplate.c b/sys/contrib/dev/acpica/compiler/dttemplate.c
index 67b13bb82d1b..d7140712d4e6 100644
--- a/sys/contrib/dev/acpica/compiler/dttemplate.c
+++ b/sys/contrib/dev/acpica/compiler/dttemplate.c
@@ -255,7 +255,7 @@ DtCreateTemplates (
 
     if (AcpiGbl_Optind < 3)
     {
-        fprintf (stderr, "Creating default template: [DSDT]\n");
+        fprintf (stdout, "Creating default template: [DSDT]\n");
         Status = DtCreateOneTemplateFile (ACPI_SIG_DSDT, 0);
         goto Exit;
     }
@@ -411,7 +411,7 @@ DtCreateAllTemplates (
     ACPI_STATUS             Status;
 
 
-    fprintf (stderr, "Creating all supported Template files\n");
+    fprintf (stdout, "Creating all supported Template files\n");
 
     /* Walk entire ACPI table data structure */
 
@@ -421,8 +421,13 @@ DtCreateAllTemplates (
 
         if (TableData->Template)
         {
-            Status = DtCreateOneTemplate (TableData->Signature,
-                0, TableData);
+	    if (ACPI_COMPARE_NAMESEG (TableData->Signature, ACPI_SIG_CDAT))
+		/* Special handling of CDAT */
+                Status = DtCreateOneTemplate (TableData->Signature,
+                    0, NULL);
+	    else
+                Status = DtCreateOneTemplate (TableData->Signature,
+                    0, TableData);
             if (ACPI_FAILURE (Status))
             {
                 return (Status);
@@ -563,7 +568,7 @@ DtCreateOneTemplate (
     }
     else
     {
-        /* Special ACPI tables - DSDT, SSDT, OSDT, FACS, RSDP */
+        /* Special ACPI tables - DSDT, SSDT, OSDT, FACS, RSDP, CDAT */
 
         AcpiOsPrintf (" (AML byte code table)\n");
         AcpiOsPrintf (" */\n");
@@ -621,6 +626,11 @@ DtCreateOneTemplate (
             AcpiDmDumpDataTable (ACPI_CAST_PTR (ACPI_TABLE_HEADER,
                 TemplateRsdp));
         }
+        else if (ACPI_COMPARE_NAMESEG (Signature, ACPI_SIG_CDAT))
+        {
+            AcpiDmDumpCdat (ACPI_CAST_PTR (ACPI_TABLE_HEADER,
+                TemplateCdat));
+        }
         else
         {
             fprintf (stderr,
@@ -632,14 +642,14 @@ DtCreateOneTemplate (
 
     if (TableCount == 0)
     {
-        fprintf (stderr,
+        fprintf (stdout,
             "Created ACPI table template for [%4.4s], "
             "written to \"%s\"\n",
             Signature, DisasmFilename);
     }
     else
     {
-        fprintf (stderr,
+        fprintf (stdout,
             "Created ACPI table templates for [%4.4s] "
             "and %u [SSDT] in same file, written to \"%s\"\n",
             Signature, TableCount, DisasmFilename);
diff --git a/sys/contrib/dev/acpica/compiler/dttemplate.h b/sys/contrib/dev/acpica/compiler/dttemplate.h
index 0fdd90f73a23..51a34be5c36b 100644
--- a/sys/contrib/dev/acpica/compiler/dttemplate.h
+++ b/sys/contrib/dev/acpica/compiler/dttemplate.h
@@ -389,7 +389,7 @@ const unsigned char TemplateBoot[] =
 const unsigned char TemplateCcel[] =
 {
     0x43,0x43,0x45,0x4C,0x38,0x00,0x00,0x00,  /* 00000000    "CCEL8..." */
-    0x04,0x1C,0x49,0x4E,0x54,0x45,0x4C,0x20,  /* 00000008    "..INTEL " */
+    0x04,0x2E,0x49,0x4E,0x54,0x45,0x4C,0x20,  /* 00000008    "..INTEL " */
     0x54,0x65,0x6D,0x70,0x6C,0x61,0x74,0x65,  /* 00000010    "Template" */
     0x00,0x00,0x00,0x00,0x49,0x4E,0x54,0x4C,  /* 00000018    "....INTL" */
     0x30,0x09,0x21,0x20,0x00,0x00,0x00,0x00,  /* 00000020    "0.! ...." */
@@ -1951,25 +1951,25 @@ const unsigned char TemplateRgrt[] =
 
 const unsigned char TemplateRhct[] =
 {
-    0x52,0x48,0x43,0x54,0x96,0x00,0x00,0x00,  /* 00000000    "RHCT|..." */
-    0x01,0x24,0x4F,0x45,0x4D,0x43,0x41,0x00,  /* 00000008    "..OEMCA." */
+    0x52,0x48,0x43,0x54,0x96,0x00,0x00,0x00,  /* 00000000    "RHCT...." */
+    0x01,0x6D,0x4F,0x45,0x4D,0x43,0x41,0x00,  /* 00000008    ".mOEMCA." */
     0x54,0x45,0x4D,0x50,0x4C,0x41,0x54,0x45,  /* 00000010    "TEMPLATE" */
     0x01,0x00,0x00,0x00,0x49,0x4E,0x54,0x4C,  /* 00000018    "....INTL" */
-    0x28,0x09,0x22,0x20,0x00,0x00,0x00,0x00,  /* 00000020    "... ...." */
+    0x04,0x04,0x25,0x20,0x00,0x00,0x00,0x00,  /* 00000020    "..% ...." */
     0x80,0x96,0x98,0x00,0x00,0x00,0x00,0x00,  /* 00000028    "........" */
-    0x02,0x00,0x00,0x00,0x38,0x00,0x00,0x00,  /* 00000030    "....8..." */
-    0x00,0x00,0x34,0x00,0x01,0x00,0x2B,0x00,  /* 00000038    "..4...*." */
+    0x04,0x00,0x00,0x00,0x38,0x00,0x00,0x00,  /* 00000030    "....8..." */
+    0x00,0x00,0x34,0x00,0x01,0x00,0x2B,0x00,  /* 00000038    "..4...+." */
     0x72,0x76,0x36,0x34,0x69,0x6D,0x61,0x66,  /* 00000040    "rv64imaf" */
     0x64,0x63,0x68,0x5F,0x7A,0x69,0x63,0x73,  /* 00000048    "dch_zics" */
     0x72,0x5F,0x7A,0x69,0x66,0x65,0x6E,0x63,  /* 00000050    "r_zifenc" */
     0x65,0x69,0x5F,0x7A,0x62,0x61,0x5F,0x7A,  /* 00000058    "ei_zba_z" */
     0x62,0x62,0x5F,0x7A,0x62,0x63,0x5F,0x7A,  /* 00000060    "bb_zbc_z" */
-    0x62,0x73,0x00,0x00,0xFF,0xFF,0x18,0x00,  /* 00000068    "bs......" */
-    0x01,0x00,0x03,0x00,0x00,0x00,0x00,0x00,  /* 00000070    "........" */
-    0x38,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,  /* 00000078    "........" */
-    0x8E,0x00,0x00,0x00,0x01,0x00,0x0A,0x00,  /* 00000080    "........" */
-    0x01,0x00,0x00,0x06,0x06,0x06,0x02,0x00,  /* 00000088    "........" */
-    0x08,0x00,0x01,0x00,0x00,0x02             /* 00000090    "........" */
+    0x62,0x73,0x00,0x00,0x01,0x00,0x0A,0x00,  /* 00000068    "bs......" */
+    0x01,0x00,0x00,0x06,0x06,0x06,0x02,0x00,  /* 00000070    "........" */
+    0x08,0x00,0x01,0x00,0x00,0x02,0xFF,0xFF,  /* 00000078    "........" */
+    0x18,0x00,0x01,0x00,0x03,0x00,0x00,0x00,  /* 00000080    "........" */
+    0x00,0x00,0x3B,0x00,0x00,0x00,0x6C,0x00,  /* 00000088    "..;...l." */
+    0x00,0x00,0x76,0x00,0x00,0x00             /* 00000090    "..v..."   */
 };
 
 const unsigned char TemplateRimt[] =
diff --git a/sys/contrib/dev/acpica/compiler/dtutils.c b/sys/contrib/dev/acpica/compiler/dtutils.c
index f2463f74b8fc..18ea18cefdd6 100644
--- a/sys/contrib/dev/acpica/compiler/dtutils.c
+++ b/sys/contrib/dev/acpica/compiler/dtutils.c
@@ -623,6 +623,7 @@ DtGetFieldLength (
     case ACPI_DMT_NFIT:
     case ACPI_DMT_PCI_PATH:
     case ACPI_DMT_PHAT:
+    case ACPI_DMT_RHCT:
 
         ByteLength = 2;
         break;
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c b/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c
index dd8cf4889885..551cf8178d94 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c
@@ -778,7 +778,7 @@ AcpiDmCsi2SerialBusDescriptor (
 
     AcpiOsPrintf (" 0x%2.2X, 0x%2.2X,\n",
         Resource->Csi2SerialBus.TypeSpecificFlags & 0x03,
-        Resource->Csi2SerialBus.TypeSpecificFlags & 0xFC);
+        (Resource->Csi2SerialBus.TypeSpecificFlags & 0xFC) >> 2);
 
     /* ResourceSource is a required field */
 
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c b/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c
index 8b6efc070b1b..becdb95f8b83 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c
@@ -646,8 +646,6 @@ AcpiDsCallControlMethod (
     ACPI_WALK_STATE         *NextWalkState = NULL;
     ACPI_OPERAND_OBJECT     *ObjDesc;
     ACPI_EVALUATE_INFO      *Info;
-    UINT32                  i;
-
 
     ACPI_FUNCTION_TRACE_PTR (DsCallControlMethod, ThisWalkState);
 
@@ -670,6 +668,23 @@ AcpiDsCallControlMethod (
         return_ACPI_STATUS (AE_NULL_OBJECT);
     }
 
+    if (ThisWalkState->NumOperands < ObjDesc->Method.ParamCount)
+    {
+        ACPI_ERROR ((AE_INFO, "Missing argument(s) for method [%4.4s]", 
+            AcpiUtGetNodeName (MethodNode)));
+
+        return_ACPI_STATUS (AE_AML_TOO_FEW_ARGUMENTS);
+    }
+
+    else if (ThisWalkState->NumOperands > ObjDesc->Method.ParamCount)
+    {
+        ACPI_ERROR ((AE_INFO, "Too many arguments for method [%4.4s]",
+            AcpiUtGetNodeName (MethodNode)));
+
+        return_ACPI_STATUS (AE_AML_TOO_MANY_ARGUMENTS);
+    }
+
+
     /* Init for new method, possibly wait on method mutex */
 
     Status = AcpiDsBeginMethodExecution (
@@ -726,15 +741,7 @@ AcpiDsCallControlMethod (
      * Delete the operands on the previous walkstate operand stack
      * (they were copied to new objects)
      */
-    for (i = 0; i < ObjDesc->Method.ParamCount; i++)
-    {
-        AcpiUtRemoveReference (ThisWalkState->Operands [i]);
-        ThisWalkState->Operands [i] = NULL;
-    }
-
-    /* Clear the operand stack */
-
-    ThisWalkState->NumOperands = 0;
+    AcpiDsClearOperands (ThisWalkState);
 
     ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH,
         "**** Begin nested execution of [%4.4s] **** WalkState=%p\n",
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c b/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c
index 42e1aa505d02..2c45e8c91f57 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c
@@ -357,6 +357,7 @@ AcpiDsMethodDataInitArgs (
 
         Index++;
     }
+    AcpiExTraceArgs(Params, Index);
 
     ACPI_DEBUG_PRINT ((ACPI_DB_EXEC, "%u args passed to method\n", Index));
     return_ACPI_STATUS (AE_OK);
diff --git a/sys/contrib/dev/acpica/components/events/evglock.c b/sys/contrib/dev/acpica/components/events/evglock.c
index 872e7b499a8f..395ca14fb315 100644
--- a/sys/contrib/dev/acpica/components/events/evglock.c
+++ b/sys/contrib/dev/acpica/components/events/evglock.c
@@ -195,6 +195,11 @@ AcpiEvInitGlobalLockHandler (
         return_ACPI_STATUS (AE_OK);
     }
 
+    if (!AcpiGbl_UseGlobalLock)
+    {
+        return_ACPI_STATUS (AE_OK);
+    }
+
     /* Attempt installation of the global lock handler */
 
     Status = AcpiInstallFixedEventHandler (ACPI_EVENT_GLOBAL,
diff --git a/sys/contrib/dev/acpica/components/executer/extrace.c b/sys/contrib/dev/acpica/components/executer/extrace.c
index 0eceb0ffccb1..b48a5fcb289b 100644
--- a/sys/contrib/dev/acpica/components/executer/extrace.c
+++ b/sys/contrib/dev/acpica/components/executer/extrace.c
@@ -269,6 +269,68 @@ AcpiExGetTraceEventName (
 
 #endif
 
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiExTraceArgs
+ *
+ * PARAMETERS:  Params            - AML method arguments
+ *              Count             - numer of method arguments
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Trace any arguments
+ *
+ ******************************************************************************/
+
+void
+AcpiExTraceArgs(ACPI_OPERAND_OBJECT **Params, UINT32 Count)
+{
+    UINT32 i;
+
+    ACPI_FUNCTION_NAME(ExTraceArgs);
+
+    for (i = 0; i < Count; i++)
+    {
+        ACPI_OPERAND_OBJECT *obj_desc = Params[i];
+
+        if (!i)
+        {
+            ACPI_DEBUG_PRINT((ACPI_DB_TRACE_POINT, " "));
+        }
+
+        switch (obj_desc->Common.Type)
+        {
+        case ACPI_TYPE_INTEGER:
+            ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "%jx", (uintmax_t)obj_desc->Integer.Value));
+            break;
+
+        case ACPI_TYPE_STRING:
+            if (!obj_desc->String.Length)
+            {
+                ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "NULL"));
+                break;
+            }
+            if (ACPI_IS_DEBUG_ENABLED(ACPI_LV_TRACE_POINT, _COMPONENT))
+            {
+                AcpiUtPrintString(obj_desc->String.Pointer, ACPI_UINT8_MAX);
+            }
+            break;
+
+        default:
+            ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "Unknown"));
+            break;
+        }
+
+        if ((i + 1) == Count)
+        {
+            ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "\n"));
+        }
+        else
+        {
+            ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, ", "));
+        }
+    }
+}
 
 /*******************************************************************************
  *
@@ -299,9 +361,9 @@ AcpiExTracePoint (
     if (Pathname)
     {
         ACPI_DEBUG_PRINT ((ACPI_DB_TRACE_POINT,
-            "%s %s [0x%p:%s] execution.\n",
+            "%s %s [%s] execution.\n",
             AcpiExGetTraceEventName (Type), Begin ? "Begin" : "End",
-            Aml, Pathname));
+            Pathname));
     }
     else
     {
diff --git a/sys/contrib/dev/acpica/components/parser/psopinfo.c b/sys/contrib/dev/acpica/components/parser/psopinfo.c
index 21c2b831ef24..1db32f4e8246 100644
--- a/sys/contrib/dev/acpica/components/parser/psopinfo.c
+++ b/sys/contrib/dev/acpica/components/parser/psopinfo.c
@@ -180,8 +180,8 @@ const ACPI_OPCODE_INFO *
 AcpiPsGetOpcodeInfo (
     UINT16                  Opcode)
 {
-#ifdef ACPI_DEBUG_OUTPUT
-    const char              *OpcodeName = "Unknown AML opcode";
+#if defined ACPI_ASL_COMPILER && defined ACPI_DEBUG_OUTPUT
+    const char 		    *OpcodeName = "Unknown AML opcode";
 #endif
 
     ACPI_FUNCTION_NAME (PsGetOpcodeInfo);
@@ -207,7 +207,7 @@ AcpiPsGetOpcodeInfo (
 
 #if defined ACPI_ASL_COMPILER && defined ACPI_DEBUG_OUTPUT
 #include <contrib/dev/acpica/compiler/asldefine.h>
-
+    
     switch (Opcode)
     {
     case AML_RAW_DATA_BYTE:
@@ -249,12 +249,12 @@ AcpiPsGetOpcodeInfo (
     default:
         break;
     }
-#endif
 
     /* Unknown AML opcode */
 
     ACPI_DEBUG_PRINT ((ACPI_DB_EXEC,
         "%s [%4.4X]\n", OpcodeName, Opcode));
+#endif
 
     return (&AcpiGbl_AmlOpInfo [_UNK]);
 }
diff --git a/sys/contrib/dev/acpica/components/tables/tbprint.c b/sys/contrib/dev/acpica/components/tables/tbprint.c
index 7211673c42a2..8b812a890a07 100644
--- a/sys/contrib/dev/acpica/components/tables/tbprint.c
+++ b/sys/contrib/dev/acpica/components/tables/tbprint.c
@@ -279,6 +279,14 @@ AcpiTbPrintTableHeader (
             ACPI_CAST_PTR (ACPI_TABLE_RSDP, Header)->Revision,
             LocalHeader.OemId));
     }
+    else if (AcpiGbl_CDAT && !AcpiUtValidNameseg (Header->Signature))
+    {
+	/* CDAT does not use the common ACPI table header */
+
+        ACPI_INFO (("%-4.4s 0x%8.8X%8.8X %06X",
+            ACPI_SIG_CDAT, ACPI_FORMAT_UINT64 (Address),
+            ACPI_CAST_PTR (ACPI_TABLE_CDAT, Header)->Length));
+    }
     else
     {
         /* Standard ACPI table with full common header */
diff --git a/sys/contrib/dev/acpica/components/utilities/utnonansi.c b/sys/contrib/dev/acpica/components/utilities/utnonansi.c
index bfbe1194ceae..f8b3a29e3283 100644
--- a/sys/contrib/dev/acpica/components/utilities/utnonansi.c
+++ b/sys/contrib/dev/acpica/components/utilities/utnonansi.c
@@ -353,7 +353,7 @@ AcpiUtSafeStrncpy (
 {
     /* Always terminate destination string */
 
-    memcpy (Dest, Source, DestSize);
+    strncpy (Dest, Source, DestSize);
     Dest[DestSize - 1] = 0;
 }
 
diff --git a/sys/contrib/dev/acpica/include/acdebug.h b/sys/contrib/dev/acpica/include/acdebug.h
index e335752148b9..63f39051a8ac 100644
--- a/sys/contrib/dev/acpica/include/acdebug.h
+++ b/sys/contrib/dev/acpica/include/acdebug.h
@@ -187,7 +187,7 @@ typedef struct acpi_db_execute_walk
 {
     UINT32                  Count;
     UINT32                  MaxCount;
-    char                    NameSeg[ACPI_NAMESEG_SIZE + 1] ACPI_NONSTRING;
+    char                    NameSeg[ACPI_NAMESEG_SIZE + 1];
 
 } ACPI_DB_EXECUTE_WALK;
 
diff --git a/sys/contrib/dev/acpica/include/acexcep.h b/sys/contrib/dev/acpica/include/acexcep.h
index 57f98ab4540f..7216e0d49148 100644
--- a/sys/contrib/dev/acpica/include/acexcep.h
+++ b/sys/contrib/dev/acpica/include/acexcep.h
@@ -322,8 +322,11 @@ typedef struct acpi_exception_info
 #define AE_AML_TARGET_TYPE              EXCEP_AML (0x0023)
 #define AE_AML_PROTOCOL                 EXCEP_AML (0x0024)
 #define AE_AML_BUFFER_LENGTH            EXCEP_AML (0x0025)
+#define AE_AML_TOO_FEW_ARGUMENTS        EXCEP_AML (0x0026)
+#define AE_AML_TOO_MANY_ARGUMENTS       EXCEP_AML (0x0027)
 
-#define AE_CODE_AML_MAX                 0x0025
+
+#define AE_CODE_AML_MAX                 0x0027
 
 
 /*
@@ -456,7 +459,9 @@ static const ACPI_EXCEPTION_INFO    AcpiGbl_ExceptionNames_Aml[] =
     EXCEP_TXT ("AE_AML_UNINITIALIZED_NODE",     "A namespace node is uninitialized or unresolved"),
     EXCEP_TXT ("AE_AML_TARGET_TYPE",            "A target operand of an incorrect type was encountered"),
     EXCEP_TXT ("AE_AML_PROTOCOL",               "Violation of a fixed ACPI protocol"),
-    EXCEP_TXT ("AE_AML_BUFFER_LENGTH",          "The length of the buffer is invalid/incorrect")
+    EXCEP_TXT ("AE_AML_BUFFER_LENGTH",          "The length of the buffer is invalid/incorrect"),
+    EXCEP_TXT ("AE_AML_TOO_FEW_ARGUMENTS",      "There are fewer than expected method arguments"),
+    EXCEP_TXT ("AE_AML_TOO_MANY_ARGUMENTS",     "There are too many arguments for this method")
 };
 
 static const ACPI_EXCEPTION_INFO    AcpiGbl_ExceptionNames_Ctrl[] =
diff --git a/sys/contrib/dev/acpica/include/acinterp.h b/sys/contrib/dev/acpica/include/acinterp.h
index 74166384f172..b7f9e8f615e4 100644
--- a/sys/contrib/dev/acpica/include/acinterp.h
+++ b/sys/contrib/dev/acpica/include/acinterp.h
@@ -280,6 +280,10 @@ AcpiExTracePoint (
     UINT8                   *Aml,
     char                    *Pathname);
 
+void
+AcpiExTraceArgs(
+    ACPI_OPERAND_OBJECT     **Params,
+    UINT32                  Count);
 
 /*
  * exfield - ACPI AML (p-code) execution - field manipulation
diff --git a/sys/contrib/dev/acpica/include/acpixf.h b/sys/contrib/dev/acpica/include/acpixf.h
index 193b0e6a70dc..b5961e21bb9b 100644
--- a/sys/contrib/dev/acpica/include/acpixf.h
+++ b/sys/contrib/dev/acpica/include/acpixf.h
@@ -154,7 +154,7 @@
 
 /* Current ACPICA subsystem version in YYYYMMDD format */
 
-#define ACPI_CA_VERSION                 0x20250404  
+#define ACPI_CA_VERSION                 0x20250807 
 
 #include <contrib/dev/acpica/include/acconfig.h>
 #include <contrib/dev/acpica/include/actypes.h>
@@ -358,6 +358,12 @@ ACPI_INIT_GLOBAL (UINT8,            AcpiGbl_OsiData, 0);
 ACPI_INIT_GLOBAL (BOOLEAN,          AcpiGbl_ReducedHardware, FALSE);
 
 /*
+ * ACPI Global Lock is mainly used for systems with SMM, so no-SMM systems
+ * (such as LoongArch) may not have and not use Global Lock.
+ */
+ACPI_INIT_GLOBAL (BOOLEAN,          AcpiGbl_UseGlobalLock, TRUE);
+
+/*
  * Maximum timeout for While() loop iterations before forced method abort.
  * This mechanism is intended to prevent infinite loops during interpreter
  * execution within a host kernel.
diff --git a/sys/contrib/dev/acpica/include/actbl.h b/sys/contrib/dev/acpica/include/actbl.h
index eafd5d8a0f8b..ae52bd452c90 100644
--- a/sys/contrib/dev/acpica/include/actbl.h
+++ b/sys/contrib/dev/acpica/include/actbl.h
@@ -220,7 +220,7 @@ typedef struct acpi_table_header
     char                    OemId[ACPI_OEM_ID_SIZE] ACPI_NONSTRING;            /* ASCII OEM identification */
     char                    OemTableId[ACPI_OEM_TABLE_ID_SIZE] ACPI_NONSTRING; /* ASCII OEM table identification */
     UINT32                  OemRevision;                                       /* OEM revision number */
-    char                    AslCompilerId[ACPI_NAMESEG_SIZE];                  /* ASCII ASL compiler vendor ID */
+    char                    AslCompilerId[ACPI_NAMESEG_SIZE] ACPI_NONSTRING;   /* ASCII ASL compiler vendor ID */
     UINT32                  AslCompilerRevision;                               /* ASL compiler version */
 
 } ACPI_TABLE_HEADER;
diff --git a/sys/contrib/dev/acpica/include/actbl1.h b/sys/contrib/dev/acpica/include/actbl1.h
index 876b721068c6..ec04f0a0ab9f 100644
--- a/sys/contrib/dev/acpica/include/actbl1.h
+++ b/sys/contrib/dev/acpica/include/actbl1.h
@@ -262,7 +262,7 @@ typedef struct acpi_whea_header
 
 /* Larger subtable header (when Length can exceed 255) */
 
-typedef struct acpi_subtable_header_16
+typedef struct acpi_subtbl_hdr_16
 {
     UINT16                  Type;
     UINT16                  Length;
diff --git a/sys/contrib/dev/acpica/include/actbl2.h b/sys/contrib/dev/acpica/include/actbl2.h
index 4899929b2d45..a74b6d555a3a 100644
--- a/sys/contrib/dev/acpica/include/actbl2.h
+++ b/sys/contrib/dev/acpica/include/actbl2.h
@@ -201,6 +201,7 @@
 #define ACPI_SIG_SDEI           "SDEI"      /* Software Delegated Exception Interface Table */
 #define ACPI_SIG_SDEV           "SDEV"      /* Secure Devices table */
 #define ACPI_SIG_SVKL           "SVKL"      /* Storage Volume Key Location Table */
+#define ACPI_SIG_SWFT           "SWFT"      /* SoundWire File Table */
 #define ACPI_SIG_TDEL           "TDEL"      /* TD Event Log Table */
 
 
@@ -4094,6 +4095,30 @@ enum acpi_svkl_format
     ACPI_SVKL_FORMAT_RESERVED   = 1 /* 1 and greater are reserved */
 };
 
+/*******************************************************************************
+ *
+ * SWFT - SoundWire File Table
+ *        as described in Discovery and Configuration (DisCo) Specification
+ *        for SoundWire®
+ *        Version 1
+ *
+ ******************************************************************************/
+
+typedef struct acpi_table_swft
+{
+    ACPI_TABLE_HEADER       Header; /* Common ACPI table header */
+
+} ACPI_TABLE_SWFT;
+
+typedef struct acpi_swft_file
+{
+    UINT16                  VendorID;
+    UINT32                  FileID;
+    UINT16                  FileVersion;
+    UINT16                  FileLength;
+    UINT8                   FileData[];
+
+} ACPI_SWFT_FILE;
 
 /*******************************************************************************
  *
diff --git a/sys/contrib/dev/iwlwifi/cfg/22000.c b/sys/contrib/dev/iwlwifi/cfg/22000.c
index 2e2fcb3807ef..ca488931a33c 100644
--- a/sys/contrib/dev/iwlwifi/cfg/22000.c
+++ b/sys/contrib/dev/iwlwifi/cfg/22000.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -15,40 +15,16 @@
 /* Lowest firmware API version supported */
 #define IWL_22000_UCODE_API_MIN	77
 
-/* NVM versions */
-#define IWL_22000_NVM_VERSION		0x0a1d
-
 /* Memory offsets and lengths */
-#define IWL_22000_DCCM_OFFSET		0x800000 /* LMAC1 */
-#define IWL_22000_DCCM_LEN		0x10000 /* LMAC1 */
-#define IWL_22000_DCCM2_OFFSET		0x880000
-#define IWL_22000_DCCM2_LEN		0x8000
 #define IWL_22000_SMEM_OFFSET		0x400000
 #define IWL_22000_SMEM_LEN		0xD0000
 
-#define IWL_QU_B_HR_B_FW_PRE		"iwlwifi-Qu-b0-hr-b0"
-#define IWL_QU_C_HR_B_FW_PRE		"iwlwifi-Qu-c0-hr-b0"
-#define IWL_QU_B_JF_B_FW_PRE		"iwlwifi-Qu-b0-jf-b0"
-#define IWL_QU_C_JF_B_FW_PRE		"iwlwifi-Qu-c0-jf-b0"
-#define IWL_QUZ_A_HR_B_FW_PRE		"iwlwifi-QuZ-a0-hr-b0"
-#define IWL_QUZ_A_JF_B_FW_PRE		"iwlwifi-QuZ-a0-jf-b0"
 #define IWL_CC_A_FW_PRE			"iwlwifi-cc-a0"
 
-#define IWL_QU_B_HR_B_MODULE_FIRMWARE(api) \
-	IWL_QU_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_QUZ_A_HR_B_MODULE_FIRMWARE(api) \
-	IWL_QUZ_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_QUZ_A_JF_B_MODULE_FIRMWARE(api) \
-	IWL_QUZ_A_JF_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_QU_C_HR_B_MODULE_FIRMWARE(api) \
-	IWL_QU_C_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_QU_B_JF_B_MODULE_FIRMWARE(api) \
-	IWL_QU_B_JF_B_FW_PRE "-" __stringify(api) ".ucode"
 #define IWL_CC_A_MODULE_FIRMWARE(api)			\
 	IWL_CC_A_FW_PRE "-" __stringify(api) ".ucode"
 
-static const struct iwl_base_params iwl_22000_base_params = {
-	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
+static const struct iwl_family_base_params iwl_22000_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 256,
 	.shadow_ram_support = true,
@@ -57,156 +33,78 @@ static const struct iwl_base_params iwl_22000_base_params = {
 	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-};
-
-const struct iwl_ht_params iwl_22000_ht_params = {
-	.stbc = true,
-	.ldpc = true,
-	.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ) |
-		      BIT(NL80211_BAND_6GHZ),
-};
-
-#define IWL_DEVICE_22000_COMMON						\
-	.ucode_api_min = IWL_22000_UCODE_API_MIN,			\
-	.led_mode = IWL_LED_RF_STATE,					\
-	.nvm_hw_section_num = 10,					\
-	.non_shared_ant = ANT_B,					\
-	.dccm_offset = IWL_22000_DCCM_OFFSET,				\
-	.dccm_len = IWL_22000_DCCM_LEN,					\
-	.dccm2_offset = IWL_22000_DCCM2_OFFSET,				\
-	.dccm2_len = IWL_22000_DCCM2_LEN,				\
-	.smem_offset = IWL_22000_SMEM_OFFSET,				\
-	.smem_len = IWL_22000_SMEM_LEN,					\
-	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,		\
-	.apmg_not_supported = true,					\
-	.trans.mq_rx_supported = true,					\
-	.vht_mu_mimo_supported = true,					\
-	.mac_addr_from_csr = 0x380,					\
-	.ht_params = &iwl_22000_ht_params,				\
-	.nvm_ver = IWL_22000_NVM_VERSION,				\
-	.trans.rf_id = true,						\
-	.trans.gen2 = true,						\
-	.nvm_type = IWL_NVM_EXT,					\
-	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0x400000,				\
-	.d3_debug_data_base_addr = 0x401000,				\
-	.d3_debug_data_length = 60 * 1024,				\
-	.mon_smem_regs = {						\
-		.write_ptr = {						\
-			.addr = LDBG_M2S_BUF_WPTR,			\
-			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,		\
-	},								\
-		.cycle_cnt = {						\
-			.addr = LDBG_M2S_BUF_WRAP_CNT,			\
-			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,		\
-		},							\
-	}
-
-#define IWL_DEVICE_22500						\
-	IWL_DEVICE_22000_COMMON,					\
-	.ucode_api_max = IWL_22000_UCODE_API_MAX,			\
-	.trans.device_family = IWL_DEVICE_FAMILY_22000,			\
-	.trans.base_params = &iwl_22000_base_params,			\
-	.gp2_reg_addr = 0xa02c68,					\
-	.mon_dram_regs = {						\
-		.write_ptr = {						\
-			.addr = MON_BUFF_WRPTR_VER2,			\
-			.mask = 0xffffffff,				\
-		},							\
-		.cycle_cnt = {						\
-			.addr = MON_BUFF_CYCLE_CNT_VER2,		\
-			.mask = 0xffffffff,				\
-		},							\
-	}
-
-const struct iwl_cfg_trans_params iwl_qu_trans_cfg = {
+	.smem_offset = IWL_22000_SMEM_OFFSET,
+	.smem_len = IWL_22000_SMEM_LEN,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.apmg_not_supported = true,
+	.mac_addr_from_csr = 0x380,
+	.min_umac_error_event_table = 0x400000,
+	.d3_debug_data_base_addr = 0x401000,
+	.d3_debug_data_length = 60 * 1024,
+	.mon_smem_regs = {
+		.write_ptr = {
+			.addr = LDBG_M2S_BUF_WPTR,
+			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,
+	},
+		.cycle_cnt = {
+			.addr = LDBG_M2S_BUF_WRAP_CNT,
+			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,
+		},
+	},
+	.gp2_reg_addr = 0xa02c68,
+	.mon_dram_regs = {
+		.write_ptr = {
+			.addr = MON_BUFF_WRPTR_VER2,
+			.mask = 0xffffffff,
+		},
+		.cycle_cnt = {
+			.addr = MON_BUFF_CYCLE_CNT_VER2,
+			.mask = 0xffffffff,
+		},
+	},
+	.ucode_api_min = IWL_22000_UCODE_API_MIN,
+	.ucode_api_max = IWL_22000_UCODE_API_MAX,
+};
+
+const struct iwl_mac_cfg iwl_qu_mac_cfg = {
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.device_family = IWL_DEVICE_FAMILY_22000,
-	.base_params = &iwl_22000_base_params,
+	.base = &iwl_22000_base,
 	.integrated = true,
 	.xtal_latency = 500,
 	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_200US,
 };
 
-const struct iwl_cfg_trans_params iwl_qu_medium_latency_trans_cfg = {
+const struct iwl_mac_cfg iwl_qu_medium_latency_mac_cfg = {
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.device_family = IWL_DEVICE_FAMILY_22000,
-	.base_params = &iwl_22000_base_params,
+	.base = &iwl_22000_base,
 	.integrated = true,
 	.xtal_latency = 1820,
 	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_1820US,
 };
 
-const struct iwl_cfg_trans_params iwl_qu_long_latency_trans_cfg = {
+const struct iwl_mac_cfg iwl_qu_long_latency_mac_cfg = {
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.device_family = IWL_DEVICE_FAMILY_22000,
-	.base_params = &iwl_22000_base_params,
+	.base = &iwl_22000_base,
 	.integrated = true,
 	.xtal_latency = 12000,
 	.low_latency_xtal = true,
 	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_2500US,
 };
 
-/*
- * If the device doesn't support HE, no need to have that many buffers.
- * 22000 devices can split multiple frames into a single RB, so fewer are
- * needed; AX210 cannot (but use smaller RBs by default) - these sizes
- * were picked according to 8 MSDUs inside 256 A-MSDUs in an A-MPDU, with
- * additional overhead to account for processing time.
- */
-#define IWL_NUM_RBDS_NON_HE		512
-#define IWL_NUM_RBDS_22000_HE		2048
-
-/*
- * All JF radio modules are part of the 9000 series, but the MAC part
- * looks more like 22000.  That's why this device is here, but called
- * 9560 nevertheless.
- */
-const struct iwl_cfg iwl9560_qu_b0_jf_b0_cfg = {
-	.fw_name_pre = IWL_QU_B_JF_B_FW_PRE,
-	IWL_DEVICE_22500,
-	.num_rbds = IWL_NUM_RBDS_NON_HE,
-};
-
-const struct iwl_cfg iwl9560_qu_c0_jf_b0_cfg = {
-	.fw_name_pre = IWL_QU_C_JF_B_FW_PRE,
-	IWL_DEVICE_22500,
-	.num_rbds = IWL_NUM_RBDS_NON_HE,
-};
-
-const struct iwl_cfg iwl9560_quz_a0_jf_b0_cfg = {
-	.fw_name_pre = IWL_QUZ_A_JF_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_NON_HE,
-};
-
-const struct iwl_cfg_trans_params iwl_ax200_trans_cfg = {
+const struct iwl_mac_cfg iwl_ax200_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_22000,
-	.base_params = &iwl_22000_base_params,
+	.base = &iwl_22000_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.bisr_workaround = 1,
 };
 
-const char iwl_ax101_name[] = "Intel(R) Wi-Fi 6 AX101";
-const char iwl_ax200_name[] = "Intel(R) Wi-Fi 6 AX200 160MHz";
-const char iwl_ax201_name[] = "Intel(R) Wi-Fi 6 AX201 160MHz";
-const char iwl_ax203_name[] = "Intel(R) Wi-Fi 6 AX203";
-const char iwl_ax204_name[] = "Intel(R) Wi-Fi 6 AX204 160MHz";
-
 const char iwl_ax200_killer_1650w_name[] =
 	"Killer(R) Wi-Fi 6 AX1650w 160MHz Wireless Network Adapter (200D2W)";
 const char iwl_ax200_killer_1650x_name[] =
@@ -216,213 +114,4 @@ const char iwl_ax201_killer_1650s_name[] =
 const char iwl_ax201_killer_1650i_name[] =
 	"Killer(R) Wi-Fi 6 AX1650i 160MHz Wireless Network Adapter (201NGW)";
 
-const struct iwl_cfg iwl_qu_b0_hr1_b0 = {
-	.fw_name_pre = IWL_QU_B_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.tx_with_siso_diversity = true,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_qu_b0_hr_b0 = {
-	.fw_name_pre = IWL_QU_B_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_ax201_cfg_qu_hr = {
-	.name = "Intel(R) Wi-Fi 6 AX201 160MHz",
-	.fw_name_pre = IWL_QU_B_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_qu_c0_hr1_b0 = {
-	.fw_name_pre = IWL_QU_C_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.tx_with_siso_diversity = true,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_qu_c0_hr_b0 = {
-	.fw_name_pre = IWL_QU_C_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_ax201_cfg_qu_c0_hr_b0 = {
-	.name = "Intel(R) Wi-Fi 6 AX201 160MHz",
-	.fw_name_pre = IWL_QU_C_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_quz_a0_hr1_b0 = {
-	.fw_name_pre = IWL_QUZ_A_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.tx_with_siso_diversity = true,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_ax201_cfg_quz_hr = {
-	.name = "Intel(R) Wi-Fi 6 AX201 160MHz",
-	.fw_name_pre = IWL_QUZ_A_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-         * This device doesn't support receiving BlockAck with a large bitmap
-         * so we need to restrict the size of transmitted aggregation to the
-         * HT size; mac80211 would otherwise pick the HE max (256) by default.
-         */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_ax1650s_cfg_quz_hr = {
-	.name = "Killer(R) Wi-Fi 6 AX1650s 160MHz Wireless Network Adapter (201D2W)",
-	.fw_name_pre = IWL_QUZ_A_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-         * This device doesn't support receiving BlockAck with a large bitmap
-         * so we need to restrict the size of transmitted aggregation to the
-         * HT size; mac80211 would otherwise pick the HE max (256) by default.
-         */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_ax1650i_cfg_quz_hr = {
-	.name = "Killer(R) Wi-Fi 6 AX1650i 160MHz Wireless Network Adapter (201NGW)",
-	.fw_name_pre = IWL_QUZ_A_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-         * This device doesn't support receiving BlockAck with a large bitmap
-         * so we need to restrict the size of transmitted aggregation to the
-         * HT size; mac80211 would otherwise pick the HE max (256) by default.
-         */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_ax200_cfg_cc = {
-	.fw_name_pre = IWL_CC_A_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg killer1650s_2ax_cfg_qu_b0_hr_b0 = {
-	.name = "Killer(R) Wi-Fi 6 AX1650s 160MHz Wireless Network Adapter (201NGW)",
-	.fw_name_pre = IWL_QU_B_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg killer1650i_2ax_cfg_qu_b0_hr_b0 = {
-	.name = "Killer(R) Wi-Fi 6 AX1650i 160MHz Wireless Network Adapter (201D2W)",
-	.fw_name_pre = IWL_QU_B_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg killer1650s_2ax_cfg_qu_c0_hr_b0 = {
-	.name = "Killer(R) Wi-Fi 6 AX1650s 160MHz Wireless Network Adapter (201NGW)",
-	.fw_name_pre = IWL_QU_C_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg killer1650i_2ax_cfg_qu_c0_hr_b0 = {
-	.name = "Killer(R) Wi-Fi 6 AX1650i 160MHz Wireless Network Adapter (201D2W)",
-	.fw_name_pre = IWL_QU_C_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-const struct iwl_cfg iwl_cfg_quz_a0_hr_b0 = {
-	.fw_name_pre = IWL_QUZ_A_HR_B_FW_PRE,
-	IWL_DEVICE_22500,
-	/*
-	 * This device doesn't support receiving BlockAck with a large bitmap
-	 * so we need to restrict the size of transmitted aggregation to the
-	 * HT size; mac80211 would otherwise pick the HE max (256) by default.
-	 */
-	.max_tx_agg_size = IEEE80211_MAX_AMPDU_BUF_HT,
-	.num_rbds = IWL_NUM_RBDS_22000_HE,
-};
-
-MODULE_FIRMWARE(IWL_QU_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_QU_C_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_QU_B_JF_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_QUZ_A_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_QUZ_A_JF_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_CC_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
diff --git a/sys/contrib/dev/iwlwifi/cfg/7000.c b/sys/contrib/dev/iwlwifi/cfg/7000.c
index 4e2afdedf4c6..f987ad3192c1 100644
--- a/sys/contrib/dev/iwlwifi/cfg/7000.c
+++ b/sys/contrib/dev/iwlwifi/cfg/7000.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2020, 2023 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2020, 2023, 2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015 Intel Deutschland GmbH
  */
@@ -49,7 +49,7 @@
 #define IWL7265D_FW_PRE "iwlwifi-7265D"
 #define IWL7265D_MODULE_FIRMWARE(api) IWL7265D_FW_PRE "-" __stringify(api) ".ucode"
 
-static const struct iwl_base_params iwl7000_base_params = {
+static const struct iwl_family_base_params iwl7000_base = {
 	.eeprom_size = OTP_LOW_IMAGE_SIZE_16K,
 	.num_of_queues = 31,
 	.max_tfd_queue_size = 256,
@@ -60,6 +60,7 @@ static const struct iwl_base_params iwl7000_base_params = {
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
 	.apmg_wake_up_wa = true,
+	.nvm_hw_section_num = 0,
 };
 
 static const struct iwl_tt_params iwl7000_high_temp_tt_params = {
@@ -84,16 +85,13 @@ static const struct iwl_tt_params iwl7000_high_temp_tt_params = {
 	.support_tx_backoff = true,
 };
 
-static const struct iwl_ht_params iwl7000_ht_params = {
-	.stbc = true,
-	.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+const struct iwl_mac_cfg iwl7000_mac_cfg = {
+	.device_family = IWL_DEVICE_FAMILY_7000,
+	.base = &iwl7000_base,
 };
 
 #define IWL_DEVICE_7000_COMMON					\
-	.trans.device_family = IWL_DEVICE_FAMILY_7000,		\
-	.trans.base_params = &iwl7000_base_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
-	.nvm_hw_section_num = 0,				\
 	.non_shared_ant = ANT_A,				\
 	.dccm_offset = IWL7000_DCCM_OFFSET
 
@@ -117,77 +115,52 @@ static const struct iwl_ht_params iwl7000_ht_params = {
 	.ucode_api_max = IWL7265D_UCODE_API_MAX,		\
 	.ucode_api_min = IWL7265D_UCODE_API_MIN
 
-const struct iwl_cfg iwl7260_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 7260",
+const char iwl7260_2ac_name[] = "Intel(R) Dual Band Wireless AC 7260";
+const char iwl7260_2n_name[] = "Intel(R) Dual Band Wireless N 7260";
+const char iwl7260_n_name[] = "Intel(R) Wireless N 7260";
+const char iwl3160_2ac_name[] = "Intel(R) Dual Band Wireless AC 3160";
+const char iwl3160_2n_name[] = "Intel(R) Dual Band Wireless N 3160";
+const char iwl3160_n_name[] = "Intel(R) Wireless N 3160";
+const char iwl3165_2ac_name[] = "Intel(R) Dual Band Wireless-AC 3165";
+const char iwl3168_2ac_name[] = "Intel(R) Dual Band Wireless-AC 3168";
+const char iwl7265_2ac_name[] = "Intel(R) Dual Band Wireless-AC 7265";
+const char iwl7265_2n_name[] = "Intel(R) Dual Band Wireless-N 7265";
+const char iwl7265_n_name[] = "Intel(R) Wireless-N 7265";
+
+const struct iwl_rf_cfg iwl7260_cfg = {
 	.fw_name_pre = IWL7260_FW_PRE,
 	IWL_DEVICE_7000,
-	.ht_params = &iwl7000_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL7260_NVM_VERSION,
 	.host_interrupt_operation_mode = true,
 	.lp_xtal_workaround = true,
 	.dccm_len = IWL7260_DCCM_LEN,
 };
 
-const struct iwl_cfg iwl7260_2ac_cfg_high_temp = {
-	.name = "Intel(R) Dual Band Wireless AC 7260",
+const struct iwl_rf_cfg iwl7260_high_temp_cfg = {
 	.fw_name_pre = IWL7260_FW_PRE,
 	IWL_DEVICE_7000,
-	.ht_params = &iwl7000_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL7260_NVM_VERSION,
-	.high_temp = true,
 	.host_interrupt_operation_mode = true,
 	.lp_xtal_workaround = true,
 	.dccm_len = IWL7260_DCCM_LEN,
 	.thermal_params = &iwl7000_high_temp_tt_params,
 };
 
-const struct iwl_cfg iwl7260_2n_cfg = {
-	.name = "Intel(R) Dual Band Wireless N 7260",
-	.fw_name_pre = IWL7260_FW_PRE,
-	IWL_DEVICE_7000,
-	.ht_params = &iwl7000_ht_params,
-	.nvm_ver = IWL7260_NVM_VERSION,
-	.host_interrupt_operation_mode = true,
-	.lp_xtal_workaround = true,
-	.dccm_len = IWL7260_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl7260_n_cfg = {
-	.name = "Intel(R) Wireless N 7260",
-	.fw_name_pre = IWL7260_FW_PRE,
-	IWL_DEVICE_7000,
-	.ht_params = &iwl7000_ht_params,
-	.nvm_ver = IWL7260_NVM_VERSION,
-	.host_interrupt_operation_mode = true,
-	.lp_xtal_workaround = true,
-	.dccm_len = IWL7260_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl3160_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 3160",
+const struct iwl_rf_cfg iwl3160_cfg = {
 	.fw_name_pre = IWL3160_FW_PRE,
 	IWL_DEVICE_7000,
-	.ht_params = &iwl7000_ht_params,
-	.nvm_ver = IWL3160_NVM_VERSION,
-	.host_interrupt_operation_mode = true,
-	.dccm_len = IWL3160_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl3160_2n_cfg = {
-	.name = "Intel(R) Dual Band Wireless N 3160",
-	.fw_name_pre = IWL3160_FW_PRE,
-	IWL_DEVICE_7000,
-	.ht_params = &iwl7000_ht_params,
-	.nvm_ver = IWL3160_NVM_VERSION,
-	.host_interrupt_operation_mode = true,
-	.dccm_len = IWL3160_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl3160_n_cfg = {
-	.name = "Intel(R) Wireless N 3160",
-	.fw_name_pre = IWL3160_FW_PRE,
-	IWL_DEVICE_7000,
-	.ht_params = &iwl7000_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL3160_NVM_VERSION,
 	.host_interrupt_operation_mode = true,
 	.dccm_len = IWL3160_DCCM_LEN,
@@ -204,88 +177,52 @@ static const struct iwl_pwr_tx_backoff iwl7265_pwr_tx_backoffs[] = {
 	{0},
 };
 
-static const struct iwl_ht_params iwl7265_ht_params = {
-	.stbc = true,
-	.ldpc = true,
-	.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
-};
-
-const struct iwl_cfg iwl3165_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 3165",
+const struct iwl_rf_cfg iwl3165_2ac_cfg = {
 	.fw_name_pre = IWL7265D_FW_PRE,
 	IWL_DEVICE_7005D,
-	.ht_params = &iwl7000_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL3165_NVM_VERSION,
 	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
 	.dccm_len = IWL7265_DCCM_LEN,
 };
 
-const struct iwl_cfg iwl3168_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 3168",
+const struct iwl_rf_cfg iwl3168_2ac_cfg = {
 	.fw_name_pre = IWL3168_FW_PRE,
 	IWL_DEVICE_3008,
-	.ht_params = &iwl7000_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL3168_NVM_VERSION,
 	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
 	.dccm_len = IWL7265_DCCM_LEN,
 	.nvm_type = IWL_NVM_SDP,
 };
 
-const struct iwl_cfg iwl7265_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 7265",
-	.fw_name_pre = IWL7265_FW_PRE,
-	IWL_DEVICE_7005,
-	.ht_params = &iwl7265_ht_params,
-	.nvm_ver = IWL7265_NVM_VERSION,
-	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
-	.dccm_len = IWL7265_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl7265_2n_cfg = {
-	.name = "Intel(R) Dual Band Wireless N 7265",
-	.fw_name_pre = IWL7265_FW_PRE,
-	IWL_DEVICE_7005,
-	.ht_params = &iwl7265_ht_params,
-	.nvm_ver = IWL7265_NVM_VERSION,
-	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
-	.dccm_len = IWL7265_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl7265_n_cfg = {
-	.name = "Intel(R) Wireless N 7265",
+const struct iwl_rf_cfg iwl7265_cfg = {
 	.fw_name_pre = IWL7265_FW_PRE,
 	IWL_DEVICE_7005,
-	.ht_params = &iwl7265_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ldpc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL7265_NVM_VERSION,
 	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
 	.dccm_len = IWL7265_DCCM_LEN,
 };
 
-const struct iwl_cfg iwl7265d_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 7265",
-	.fw_name_pre = IWL7265D_FW_PRE,
-	IWL_DEVICE_7005D,
-	.ht_params = &iwl7265_ht_params,
-	.nvm_ver = IWL7265D_NVM_VERSION,
-	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
-	.dccm_len = IWL7265_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl7265d_2n_cfg = {
-	.name = "Intel(R) Dual Band Wireless N 7265",
+const struct iwl_rf_cfg iwl7265d_cfg = {
 	.fw_name_pre = IWL7265D_FW_PRE,
 	IWL_DEVICE_7005D,
-	.ht_params = &iwl7265_ht_params,
-	.nvm_ver = IWL7265D_NVM_VERSION,
-	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
-	.dccm_len = IWL7265_DCCM_LEN,
-};
-
-const struct iwl_cfg iwl7265d_n_cfg = {
-	.name = "Intel(R) Wireless N 7265",
-	.fw_name_pre = IWL7265D_FW_PRE,
-	IWL_DEVICE_7005D,
-	.ht_params = &iwl7265_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ldpc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL7265D_NVM_VERSION,
 	.pwr_tx_backoffs = iwl7265_pwr_tx_backoffs,
 	.dccm_len = IWL7265_DCCM_LEN,
diff --git a/sys/contrib/dev/iwlwifi/cfg/8000.c b/sys/contrib/dev/iwlwifi/cfg/8000.c
index d09cf8d7dc01..b56574006ee0 100644
--- a/sys/contrib/dev/iwlwifi/cfg/8000.c
+++ b/sys/contrib/dev/iwlwifi/cfg/8000.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2014, 2018-2020, 2023 Intel Corporation
+ * Copyright (C) 2014, 2018-2020, 2023, 2025 Intel Corporation
  * Copyright (C) 2014-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -35,9 +35,7 @@
 #define IWL8265_MODULE_FIRMWARE(api) \
 	IWL8265_FW_PRE "-" __stringify(api) ".ucode"
 
-#define DEFAULT_NVM_FILE_FAMILY_8000C		"nvmData-8000C"
-
-static const struct iwl_base_params iwl8000_base_params = {
+static const struct iwl_family_base_params iwl8000_base = {
 	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
 	.num_of_queues = 31,
 	.max_tfd_queue_size = 256,
@@ -47,12 +45,12 @@ static const struct iwl_base_params iwl8000_base_params = {
 	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-};
-
-static const struct iwl_ht_params iwl8000_ht_params = {
-	.stbc = true,
-	.ldpc = true,
-	.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	.nvm_hw_section_num = 10,
+	.features = NETIF_F_RXCSUM,
+	.smem_offset = IWL8260_SMEM_OFFSET,
+	.smem_len = IWL8260_SMEM_LEN,
+	.apmg_not_supported = true,
+	.min_umac_error_event_table = 0x800000,
 };
 
 static const struct iwl_tt_params iwl8000_tt_params = {
@@ -76,30 +74,20 @@ static const struct iwl_tt_params iwl8000_tt_params = {
 	.support_tx_backoff = true,
 };
 
+const struct iwl_mac_cfg iwl8000_mac_cfg = {
+	.device_family = IWL_DEVICE_FAMILY_8000,
+	.base = &iwl8000_base,
+};
+
 #define IWL_DEVICE_8000_COMMON						\
-	.trans.device_family = IWL_DEVICE_FAMILY_8000,			\
-	.trans.base_params = &iwl8000_base_params,			\
 	.led_mode = IWL_LED_RF_STATE,					\
-	.nvm_hw_section_num = 10,					\
-	.features = NETIF_F_RXCSUM,					\
 	.non_shared_ant = ANT_A,					\
 	.dccm_offset = IWL8260_DCCM_OFFSET,				\
 	.dccm_len = IWL8260_DCCM_LEN,					\
 	.dccm2_offset = IWL8260_DCCM2_OFFSET,				\
 	.dccm2_len = IWL8260_DCCM2_LEN,					\
-	.smem_offset = IWL8260_SMEM_OFFSET,				\
-	.smem_len = IWL8260_SMEM_LEN,					\
-	.default_nvm_file_C_step = DEFAULT_NVM_FILE_FAMILY_8000C,	\
 	.thermal_params = &iwl8000_tt_params,				\
-	.apmg_not_supported = true,					\
-	.nvm_type = IWL_NVM_EXT,					\
-	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0x800000
-
-#define IWL_DEVICE_8000							\
-	IWL_DEVICE_8000_COMMON,						\
-	.ucode_api_max = IWL8000_UCODE_API_MAX,				\
-	.ucode_api_min = IWL8000_UCODE_API_MIN				\
+	.nvm_type = IWL_NVM_EXT
 
 #define IWL_DEVICE_8260							\
 	IWL_DEVICE_8000_COMMON,						\
@@ -111,47 +99,39 @@ static const struct iwl_tt_params iwl8000_tt_params = {
 	.ucode_api_max = IWL8265_UCODE_API_MAX,				\
 	.ucode_api_min = IWL8265_UCODE_API_MIN				\
 
-const struct iwl_cfg iwl8260_2n_cfg = {
-	.name = "Intel(R) Dual Band Wireless N 8260",
-	.fw_name_pre = IWL8000_FW_PRE,
-	IWL_DEVICE_8260,
-	.ht_params = &iwl8000_ht_params,
-	.nvm_ver = IWL8000_NVM_VERSION,
-};
+const char iwl8260_2n_name[] = "Intel(R) Dual Band Wireless-N 8260";
+const char iwl8260_2ac_name[] = "Intel(R) Dual Band Wireless-AC 8260";
+const char iwl8265_2ac_name[] = "Intel(R) Dual Band Wireless-AC 8265";
+const char iwl8275_2ac_name[] = "Intel(R) Dual Band Wireless-AC 8275";
+const char iwl4165_2ac_name[] = "Intel(R) Dual Band Wireless-AC 4165";
 
-const struct iwl_cfg iwl8260_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 8260",
+const char iwl_killer_1435i_name[] =
+	"Killer(R) Wireless-AC 1435i Wireless Network Adapter (8265D2W)";
+const char iwl_killer_1434_kix_name[] =
+	"Killer(R) Wireless-AC 1435-KIX Wireless Network Adapter (8265NGW)";
+
+const struct iwl_rf_cfg iwl8260_cfg = {
 	.fw_name_pre = IWL8000_FW_PRE,
 	IWL_DEVICE_8260,
-	.ht_params = &iwl8000_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ldpc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL8000_NVM_VERSION,
 };
 
-const struct iwl_cfg iwl8265_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 8265",
+const struct iwl_rf_cfg iwl8265_cfg = {
 	.fw_name_pre = IWL8265_FW_PRE,
 	IWL_DEVICE_8265,
-	.ht_params = &iwl8000_ht_params,
-	.nvm_ver = IWL8000_NVM_VERSION,
-	.vht_mu_mimo_supported = true,
-};
-
-const struct iwl_cfg iwl8275_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 8275",
-	.fw_name_pre = IWL8265_FW_PRE,
-	IWL_DEVICE_8265,
-	.ht_params = &iwl8000_ht_params,
+	.ht_params = {
+		.stbc = true,
+		.ldpc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+	},
 	.nvm_ver = IWL8000_NVM_VERSION,
 	.vht_mu_mimo_supported = true,
 };
 
-const struct iwl_cfg iwl4165_2ac_cfg = {
-	.name = "Intel(R) Dual Band Wireless AC 4165",
-	.fw_name_pre = IWL8000_FW_PRE,
-	IWL_DEVICE_8000,
-	.ht_params = &iwl8000_ht_params,
-	.nvm_ver = IWL8000_NVM_VERSION,
-};
-
 MODULE_FIRMWARE(IWL8000_MODULE_FIRMWARE(IWL8000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL8265_MODULE_FIRMWARE(IWL8265_UCODE_API_MAX));
diff --git a/sys/contrib/dev/iwlwifi/cfg/9000.c b/sys/contrib/dev/iwlwifi/cfg/9000.c
index 0130d9a9b78b..ac1fa291cf2f 100644
--- a/sys/contrib/dev/iwlwifi/cfg/9000.c
+++ b/sys/contrib/dev/iwlwifi/cfg/9000.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2021, 2023 Intel Corporation
+ * Copyright (C) 2018-2021, 2023, 2025 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -15,14 +15,7 @@
 /* Lowest firmware API version supported */
 #define IWL9000_UCODE_API_MIN	30
 
-/* NVM versions */
-#define IWL9000_NVM_VERSION		0x0a1d
-
 /* Memory offsets and lengths */
-#define IWL9000_DCCM_OFFSET		0x800000
-#define IWL9000_DCCM_LEN		0x18000
-#define IWL9000_DCCM2_OFFSET		0x880000
-#define IWL9000_DCCM2_LEN		0x8000
 #define IWL9000_SMEM_OFFSET		0x400000
 #define IWL9000_SMEM_LEN		0x68000
 
@@ -33,7 +26,7 @@
 #define IWL9260_MODULE_FIRMWARE(api) \
 	IWL9260_FW_PRE "-" __stringify(api) ".ucode"
 
-static const struct iwl_base_params iwl9000_base_params = {
+static const struct iwl_family_base_params iwl9000_base = {
 	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
 	.num_of_queues = 31,
 	.max_tfd_queue_size = 256,
@@ -43,150 +36,69 @@ static const struct iwl_base_params iwl9000_base_params = {
 	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-};
-
-static const struct iwl_ht_params iwl9000_ht_params = {
-	.stbc = true,
-	.ldpc = true,
-	.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
-};
-
-static const struct iwl_tt_params iwl9000_tt_params = {
-	.ct_kill_entry = 115,
-	.ct_kill_exit = 93,
-	.ct_kill_duration = 5,
-	.dynamic_smps_entry = 111,
-	.dynamic_smps_exit = 107,
-	.tx_protection_entry = 112,
-	.tx_protection_exit = 105,
-	.tx_backoff = {
-		{.temperature = 110, .backoff = 200},
-		{.temperature = 111, .backoff = 600},
-		{.temperature = 112, .backoff = 1200},
-		{.temperature = 113, .backoff = 2000},
-		{.temperature = 114, .backoff = 4000},
+	.smem_offset = IWL9000_SMEM_OFFSET,
+	.smem_len = IWL9000_SMEM_LEN,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.apmg_not_supported = true,
+	.mac_addr_from_csr = 0x380,
+	.min_umac_error_event_table = 0x800000,
+	.d3_debug_data_base_addr = 0x401000,
+	.d3_debug_data_length = 92 * 1024,
+	.nvm_hw_section_num = 10,
+	.mon_smem_regs = {
+		.write_ptr = {
+			.addr = LDBG_M2S_BUF_WPTR,
+			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,
+		},
+		.cycle_cnt = {
+			.addr = LDBG_M2S_BUF_WRAP_CNT,
+			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,
+		},
+	},
+	.mon_dram_regs = {
+		.write_ptr = {
+			.addr = MON_BUFF_WRPTR_VER2,
+			.mask = 0xffffffff,
+		},
+		.cycle_cnt = {
+			.addr = MON_BUFF_CYCLE_CNT_VER2,
+			.mask = 0xffffffff,
+		},
 	},
-	.support_ct_kill = true,
-	.support_dynamic_smps = true,
-	.support_tx_protection = true,
-	.support_tx_backoff = true,
+	.ucode_api_max = IWL9000_UCODE_API_MAX,
+	.ucode_api_min = IWL9000_UCODE_API_MIN,
 };
 
-#define IWL_DEVICE_9000							\
-	.ucode_api_max = IWL9000_UCODE_API_MAX,				\
-	.ucode_api_min = IWL9000_UCODE_API_MIN,				\
-	.led_mode = IWL_LED_RF_STATE,					\
-	.nvm_hw_section_num = 10,					\
-	.non_shared_ant = ANT_B,					\
-	.dccm_offset = IWL9000_DCCM_OFFSET,				\
-	.dccm_len = IWL9000_DCCM_LEN,					\
-	.dccm2_offset = IWL9000_DCCM2_OFFSET,				\
-	.dccm2_len = IWL9000_DCCM2_LEN,					\
-	.smem_offset = IWL9000_SMEM_OFFSET,				\
-	.smem_len = IWL9000_SMEM_LEN,					\
-	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,		\
-	.thermal_params = &iwl9000_tt_params,				\
-	.apmg_not_supported = true,					\
-	.num_rbds = 512,						\
-	.vht_mu_mimo_supported = true,					\
-	.mac_addr_from_csr = 0x380,					\
-	.nvm_type = IWL_NVM_EXT,					\
-	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0x800000,				\
-	.d3_debug_data_base_addr = 0x401000,				\
-	.d3_debug_data_length = 92 * 1024,				\
-	.ht_params = &iwl9000_ht_params,				\
-	.nvm_ver = IWL9000_NVM_VERSION,					\
-	.mon_smem_regs = {						\
-		.write_ptr = {						\
-			.addr = LDBG_M2S_BUF_WPTR,			\
-			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,		\
-		},							\
-		.cycle_cnt = {						\
-			.addr = LDBG_M2S_BUF_WRAP_CNT,			\
-			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,		\
-		},							\
-	},								\
-	.mon_dram_regs = {						\
-		.write_ptr = {						\
-			.addr = MON_BUFF_WRPTR_VER2,			\
-			.mask = 0xffffffff,				\
-		},							\
-		.cycle_cnt = {						\
-			.addr = MON_BUFF_CYCLE_CNT_VER2,		\
-			.mask = 0xffffffff,				\
-		},							\
-	}
-
-const struct iwl_cfg_trans_params iwl9000_trans_cfg = {
+const struct iwl_mac_cfg iwl9000_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_9000,
-	.base_params = &iwl9000_base_params,
+	.base = &iwl9000_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 };
 
-const struct iwl_cfg_trans_params iwl9560_trans_cfg = {
+const struct iwl_mac_cfg iwl9560_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_9000,
-	.base_params = &iwl9000_base_params,
+	.base = &iwl9000_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.integrated = true,
 	.xtal_latency = 650,
 };
 
-const struct iwl_cfg_trans_params iwl9560_long_latency_trans_cfg = {
+const struct iwl_mac_cfg iwl9560_long_latency_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_9000,
-	.base_params = &iwl9000_base_params,
+	.base = &iwl9000_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.integrated = true,
 	.xtal_latency = 2820,
 };
 
-const struct iwl_cfg_trans_params iwl9560_shared_clk_trans_cfg = {
+const struct iwl_mac_cfg iwl9560_shared_clk_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_9000,
-	.base_params = &iwl9000_base_params,
+	.base = &iwl9000_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.integrated = true,
 	.xtal_latency = 670,
 	.extra_phy_cfg_flags = FW_PHY_CFG_SHARED_CLK
 };
 
-const char iwl9162_name[] = "Intel(R) Wireless-AC 9162";
-const char iwl9260_name[] = "Intel(R) Wireless-AC 9260";
-const char iwl9260_1_name[] = "Intel(R) Wireless-AC 9260-1";
-const char iwl9270_name[] = "Intel(R) Wireless-AC 9270";
-const char iwl9461_name[] = "Intel(R) Wireless-AC 9461";
-const char iwl9462_name[] = "Intel(R) Wireless-AC 9462";
-const char iwl9560_name[] = "Intel(R) Wireless-AC 9560";
-const char iwl9162_160_name[] = "Intel(R) Wireless-AC 9162 160MHz";
-const char iwl9260_160_name[] = "Intel(R) Wireless-AC 9260 160MHz";
-const char iwl9270_160_name[] = "Intel(R) Wireless-AC 9270 160MHz";
-const char iwl9461_160_name[] = "Intel(R) Wireless-AC 9461 160MHz";
-const char iwl9462_160_name[] = "Intel(R) Wireless-AC 9462 160MHz";
-const char iwl9560_160_name[] = "Intel(R) Wireless-AC 9560 160MHz";
-
-const char iwl9260_killer_1550_name[] =
-	"Killer (R) Wireless-AC 1550 Wireless Network Adapter (9260NGW) 160MHz";
-const char iwl9560_killer_1550i_name[] =
-	"Killer (R) Wireless-AC 1550i Wireless Network Adapter (9560NGW)";
-const char iwl9560_killer_1550i_160_name[] =
-	"Killer(R) Wireless-AC 1550i Wireless Network Adapter (9560NGW) 160MHz";
-const char iwl9560_killer_1550s_name[] =
-	"Killer (R) Wireless-AC 1550s Wireless Network Adapter (9560NGW)";
-const char iwl9560_killer_1550s_160_name[] =
-	"Killer(R) Wireless-AC 1550s Wireless Network Adapter (9560D2W) 160MHz";
-
-const struct iwl_cfg iwl9260_2ac_cfg = {
-	.fw_name_pre = IWL9260_FW_PRE,
-	IWL_DEVICE_9000,
-};
-
-const struct iwl_cfg iwl9560_2ac_cfg_soc = {
-	.fw_name_pre = IWL9000_FW_PRE,
-	IWL_DEVICE_9000,
-};
-
 MODULE_FIRMWARE(IWL9000_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL9260_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
diff --git a/sys/contrib/dev/iwlwifi/cfg/ax210.c b/sys/contrib/dev/iwlwifi/cfg/ax210.c
index 975e8aed1526..ddf3d313da5a 100644
--- a/sys/contrib/dev/iwlwifi/cfg/ax210.c
+++ b/sys/contrib/dev/iwlwifi/cfg/ax210.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -13,61 +13,13 @@
 #define IWL_AX210_UCODE_API_MAX	89
 
 /* Lowest firmware API version supported */
-#define IWL_AX210_UCODE_API_MIN	77
-
-/* NVM versions */
-#define IWL_AX210_NVM_VERSION		0x0a1d
+#define IWL_AX210_UCODE_API_MIN	89
 
 /* Memory offsets and lengths */
-#define IWL_AX210_DCCM_OFFSET		0x800000 /* LMAC1 */
-#define IWL_AX210_DCCM_LEN		0x10000 /* LMAC1 */
-#define IWL_AX210_DCCM2_OFFSET		0x880000
-#define IWL_AX210_DCCM2_LEN		0x8000
 #define IWL_AX210_SMEM_OFFSET		0x400000
 #define IWL_AX210_SMEM_LEN		0xD0000
 
-#define IWL_SO_A_JF_B_FW_PRE		"iwlwifi-so-a0-jf-b0"
-#define IWL_SO_A_HR_B_FW_PRE		"iwlwifi-so-a0-hr-b0"
-#define IWL_SO_A_GF_A_FW_PRE		"iwlwifi-so-a0-gf-a0"
-#define IWL_TY_A_GF_A_FW_PRE		"iwlwifi-ty-a0-gf-a0"
-#define IWL_SO_A_GF4_A_FW_PRE		"iwlwifi-so-a0-gf4-a0"
-#define IWL_SO_A_MR_A_FW_PRE		"iwlwifi-so-a0-mr-a0"
-#define IWL_MA_A_HR_B_FW_PRE		"iwlwifi-ma-a0-hr-b0"
-#define IWL_MA_A_GF_A_FW_PRE		"iwlwifi-ma-a0-gf-a0"
-#define IWL_MA_A_GF4_A_FW_PRE		"iwlwifi-ma-a0-gf4-a0"
-#define IWL_MA_A_MR_A_FW_PRE		"iwlwifi-ma-a0-mr-a0"
-#define IWL_MA_B_HR_B_FW_PRE		"iwlwifi-ma-b0-hr-b0"
-#define IWL_MA_B_GF_A_FW_PRE		"iwlwifi-ma-b0-gf-a0"
-#define IWL_MA_B_GF4_A_FW_PRE		"iwlwifi-ma-b0-gf4-a0"
-#define IWL_MA_B_MR_A_FW_PRE		"iwlwifi-ma-b0-mr-a0"
-
-#define IWL_SO_A_JF_B_MODULE_FIRMWARE(api) \
-	IWL_SO_A_JF_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SO_A_HR_B_MODULE_FIRMWARE(api) \
-	IWL_SO_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SO_A_GF_A_MODULE_FIRMWARE(api) \
-	IWL_SO_A_GF_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_TY_A_GF_A_MODULE_FIRMWARE(api) \
-	IWL_TY_A_GF_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_A_HR_B_FW_MODULE_FIRMWARE(api)		\
-	IWL_MA_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_A_GF_A_FW_MODULE_FIRMWARE(api)		\
-	IWL_MA_A_GF_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_A_GF4_A_FW_MODULE_FIRMWARE(api)		\
-	IWL_MA_A_GF4_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_A_MR_A_FW_MODULE_FIRMWARE(api) \
-	IWL_MA_A_MR_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(api)		\
-	IWL_MA_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_B_GF_A_FW_MODULE_FIRMWARE(api)		\
-	IWL_MA_B_GF_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_B_GF4_A_FW_MODULE_FIRMWARE(api)		\
-	IWL_MA_B_GF4_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_B_MR_A_FW_MODULE_FIRMWARE(api) \
-	IWL_MA_B_MR_A_FW_PRE "-" __stringify(api) ".ucode"
-
-static const struct iwl_base_params iwl_ax210_base_params = {
-	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
+static const struct iwl_family_base_params iwl_ax210_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
 	.shadow_ram_support = true,
@@ -76,74 +28,60 @@ static const struct iwl_base_params iwl_ax210_base_params = {
 	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
+	.smem_offset = IWL_AX210_SMEM_OFFSET,
+	.smem_len = IWL_AX210_SMEM_LEN,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.apmg_not_supported = true,
+	.mac_addr_from_csr = 0x380,
+	.min_umac_error_event_table = 0x400000,
+	.d3_debug_data_base_addr = 0x401000,
+	.d3_debug_data_length = 60 * 1024,
+	.mon_smem_regs = {
+		.write_ptr = {
+			.addr = LDBG_M2S_BUF_WPTR,
+			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,
+		},
+		.cycle_cnt = {
+			.addr = LDBG_M2S_BUF_WRAP_CNT,
+			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,
+		},
+	},
+	.min_txq_size = 128,
+	.gp2_reg_addr = 0xd02c68,
+	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_HE,
+	.mon_dram_regs = {
+		.write_ptr = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK,
+		},
+		.cycle_cnt = {
+			.addr = DBGC_DBGBUF_WRAP_AROUND,
+			.mask = 0xffffffff,
+		},
+		.cur_frag = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_IDX_MSK,
+		},
+	},
+	.ucode_api_min = IWL_AX210_UCODE_API_MIN,
+	.ucode_api_max = IWL_AX210_UCODE_API_MAX,
 };
 
-#define IWL_DEVICE_AX210_COMMON						\
-	.ucode_api_min = IWL_AX210_UCODE_API_MIN,			\
-	.led_mode = IWL_LED_RF_STATE,					\
-	.nvm_hw_section_num = 10,					\
-	.non_shared_ant = ANT_B,					\
-	.dccm_offset = IWL_AX210_DCCM_OFFSET,				\
-	.dccm_len = IWL_AX210_DCCM_LEN,					\
-	.dccm2_offset = IWL_AX210_DCCM2_OFFSET,				\
-	.dccm2_len = IWL_AX210_DCCM2_LEN,				\
-	.smem_offset = IWL_AX210_SMEM_OFFSET,				\
-	.smem_len = IWL_AX210_SMEM_LEN,					\
-	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,		\
-	.apmg_not_supported = true,					\
-	.trans.mq_rx_supported = true,					\
-	.vht_mu_mimo_supported = true,					\
-	.mac_addr_from_csr = 0x380,					\
-	.ht_params = &iwl_22000_ht_params,				\
-	.nvm_ver = IWL_AX210_NVM_VERSION,				\
-	.trans.rf_id = true,						\
-	.trans.gen2 = true,						\
-	.nvm_type = IWL_NVM_EXT,					\
-	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0x400000,				\
-	.d3_debug_data_base_addr = 0x401000,				\
-	.d3_debug_data_length = 60 * 1024,				\
-	.mon_smem_regs = {						\
-		.write_ptr = {						\
-			.addr = LDBG_M2S_BUF_WPTR,			\
-			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,		\
-	},								\
-		.cycle_cnt = {						\
-			.addr = LDBG_M2S_BUF_WRAP_CNT,			\
-			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,		\
-		},							\
-	}
-
-#define IWL_DEVICE_AX210						\
-	IWL_DEVICE_AX210_COMMON,					\
-	.ucode_api_max = IWL_AX210_UCODE_API_MAX,			\
-	.trans.umac_prph_offset = 0x300000,				\
-	.trans.device_family = IWL_DEVICE_FAMILY_AX210,			\
-	.trans.base_params = &iwl_ax210_base_params,			\
-	.min_txq_size = 128,						\
-	.gp2_reg_addr = 0xd02c68,					\
-	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_HE,		\
-	.mon_dram_regs = {						\
-		.write_ptr = {						\
-			.addr = DBGC_CUR_DBGBUF_STATUS,			\
-			.mask = DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK,	\
-		},							\
-		.cycle_cnt = {						\
-			.addr = DBGC_DBGBUF_WRAP_AROUND,		\
-			.mask = 0xffffffff,				\
-		},							\
-		.cur_frag = {						\
-			.addr = DBGC_CUR_DBGBUF_STATUS,			\
-			.mask = DBGC_CUR_DBGBUF_STATUS_IDX_MSK,		\
-		},							\
-	}
+const struct iwl_mac_cfg iwl_ty_mac_cfg = {
+	.mq_rx_supported = true,
+	.gen2 = true,
+	.device_family = IWL_DEVICE_FAMILY_AX210,
+	.base = &iwl_ax210_base,
+	.umac_prph_offset = 0x300000,
+	/* TODO: the following values need to be checked */
+	.xtal_latency = 500,
+};
 
-const struct iwl_cfg_trans_params iwl_so_trans_cfg = {
+const struct iwl_mac_cfg iwl_so_mac_cfg = {
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.device_family = IWL_DEVICE_FAMILY_AX210,
-	.base_params = &iwl_ax210_base_params,
+	.base = &iwl_ax210_base,
 	.umac_prph_offset = 0x300000,
 	.integrated = true,
 	/* TODO: the following values need to be checked */
@@ -151,12 +89,11 @@ const struct iwl_cfg_trans_params iwl_so_trans_cfg = {
 	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_200US,
 };
 
-const struct iwl_cfg_trans_params iwl_so_long_latency_trans_cfg = {
+const struct iwl_mac_cfg iwl_so_long_latency_mac_cfg = {
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.device_family = IWL_DEVICE_FAMILY_AX210,
-	.base_params = &iwl_ax210_base_params,
+	.base = &iwl_ax210_base,
 	.umac_prph_offset = 0x300000,
 	.integrated = true,
 	.low_latency_xtal = true,
@@ -164,12 +101,11 @@ const struct iwl_cfg_trans_params iwl_so_long_latency_trans_cfg = {
 	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_2500US,
 };
 
-const struct iwl_cfg_trans_params iwl_so_long_latency_imr_trans_cfg = {
+const struct iwl_mac_cfg iwl_so_long_latency_imr_mac_cfg = {
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.device_family = IWL_DEVICE_FAMILY_AX210,
-	.base_params = &iwl_ax210_base_params,
+	.base = &iwl_ax210_base,
 	.umac_prph_offset = 0x300000,
 	.integrated = true,
 	.low_latency_xtal = true,
@@ -178,130 +114,11 @@ const struct iwl_cfg_trans_params iwl_so_long_latency_imr_trans_cfg = {
 	.imr_enabled = true,
 };
 
-/*
- * If the device doesn't support HE, no need to have that many buffers.
- * AX210 devices can split multiple frames into a single RB, so fewer are
- * needed; AX210 cannot (but use smaller RBs by default) - these sizes
- * were picked according to 8 MSDUs inside 256 A-MSDUs in an A-MPDU, with
- * additional overhead to account for processing time.
- */
-#define IWL_NUM_RBDS_NON_HE		512
-#define IWL_NUM_RBDS_AX210_HE		4096
-
-const struct iwl_cfg_trans_params iwl_ma_trans_cfg = {
+const struct iwl_mac_cfg iwl_ma_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_AX210,
-	.base_params = &iwl_ax210_base_params,
+	.base = &iwl_ax210_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.integrated = true,
 	.umac_prph_offset = 0x300000
 };
-
-const char iwl_ax211_name[] = "Intel(R) Wi-Fi 6E AX211 160MHz";
-const char iwl_ax221_name[] = "Intel(R) Wi-Fi 6E AX221 160MHz";
-const char iwl_ax231_name[] = "Intel(R) Wi-Fi 6E AX231 160MHz";
-const char iwl_ax411_name[] = "Intel(R) Wi-Fi 6E AX411 160MHz";
-
-const char iwl_ax210_killer_1675w_name[] =
-	"Killer(R) Wi-Fi 6E AX1675w 160MHz Wireless Network Adapter (210D2W)";
-const char iwl_ax210_killer_1675x_name[] =
-	"Killer(R) Wi-Fi 6E AX1675x 160MHz Wireless Network Adapter (210NGW)";
-const char iwl_ax211_killer_1675s_name[] =
-	"Killer(R) Wi-Fi 6E AX1675s 160MHz Wireless Network Adapter (211NGW)";
-const char iwl_ax211_killer_1675i_name[] =
-	"Killer(R) Wi-Fi 6E AX1675i 160MHz Wireless Network Adapter (211NGW)";
-const char iwl_ax411_killer_1690s_name[] =
-	"Killer(R) Wi-Fi 6E AX1690s 160MHz Wireless Network Adapter (411D2W)";
-const char iwl_ax411_killer_1690i_name[] =
-	"Killer(R) Wi-Fi 6E AX1690i 160MHz Wireless Network Adapter (411NGW)";
-
-const struct iwl_cfg iwlax210_2ax_cfg_so_jf_b0 = {
-	.name = "Intel(R) Wireless-AC 9560 160MHz",
-	.fw_name_pre = IWL_SO_A_JF_B_FW_PRE,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_NON_HE,
-};
-
-const struct iwl_cfg iwlax211_2ax_cfg_so_gf_a0 = {
-	.name = iwl_ax211_name,
-	.fw_name_pre = IWL_SO_A_GF_A_FW_PRE,
-	.uhb_supported = true,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-};
-
-const struct iwl_cfg iwlax211_2ax_cfg_so_gf_a0_long = {
-	.name = iwl_ax211_name,
-	.fw_name_pre = IWL_SO_A_GF_A_FW_PRE,
-	.uhb_supported = true,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-	.trans.xtal_latency = 12000,
-	.trans.low_latency_xtal = true,
-};
-
-const struct iwl_cfg iwlax210_2ax_cfg_ty_gf_a0 = {
-	.name = "Intel(R) Wi-Fi 6 AX210 160MHz",
-	.fw_name_pre = IWL_TY_A_GF_A_FW_PRE,
-	.uhb_supported = true,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-};
-
-const struct iwl_cfg iwlax411_2ax_cfg_so_gf4_a0 = {
-	.name = iwl_ax411_name,
-	.fw_name_pre = IWL_SO_A_GF4_A_FW_PRE,
-	.uhb_supported = true,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-};
-
-const struct iwl_cfg iwlax411_2ax_cfg_so_gf4_a0_long = {
-	.name = iwl_ax411_name,
-	.fw_name_pre = IWL_SO_A_GF4_A_FW_PRE,
-	.uhb_supported = true,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-	.trans.xtal_latency = 12000,
-	.trans.low_latency_xtal = true,
-};
-
-const struct iwl_cfg iwl_cfg_so_a0_ms_a0 = {
-	.fw_name_pre = IWL_SO_A_MR_A_FW_PRE,
-	.uhb_supported = false,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-};
-
-const struct iwl_cfg iwl_cfg_ma = {
-	.fw_name_mac = "ma",
-	.uhb_supported = true,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-};
-
-const struct iwl_cfg iwl_cfg_so_a0_hr_a0 = {
-	.fw_name_pre = IWL_SO_A_HR_B_FW_PRE,
-	IWL_DEVICE_AX210,
-	.num_rbds = IWL_NUM_RBDS_AX210_HE,
-};
-
-MODULE_FIRMWARE(IWL_SO_A_JF_B_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SO_A_HR_B_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SO_A_GF_A_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_TY_A_GF_A_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_A_HR_B_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_A_GF_A_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_A_GF4_A_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_A_MR_A_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_B_GF_A_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_B_GF4_A_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_B_MR_A_FW_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
-
-MODULE_FIRMWARE("iwlwifi-so-a0-gf-a0.pnvm");
-MODULE_FIRMWARE("iwlwifi-so-a0-gf4-a0.pnvm");
-MODULE_FIRMWARE("iwlwifi-ty-a0-gf-a0.pnvm");
-MODULE_FIRMWARE("iwlwifi-ma-b0-gf-a0.pnvm");
-MODULE_FIRMWARE("iwlwifi-ma-b0-gf4-a0.pnvm");
diff --git a/sys/contrib/dev/iwlwifi/cfg/bz.c b/sys/contrib/dev/iwlwifi/cfg/bz.c
index 3b6b8b410be5..9f543946b285 100644
--- a/sys/contrib/dev/iwlwifi/cfg/bz.c
+++ b/sys/contrib/dev/iwlwifi/cfg/bz.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -10,50 +10,22 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware API version supported */
-#define IWL_BZ_UCODE_API_MAX	92
+#define IWL_BZ_UCODE_API_MAX	102
 
 /* Lowest firmware API version supported */
-#define IWL_BZ_UCODE_API_MIN	90
-
-/* NVM versions */
-#define IWL_BZ_NVM_VERSION		0x0a1d
+#define IWL_BZ_UCODE_API_MIN	98
 
 /* Memory offsets and lengths */
-#define IWL_BZ_DCCM_OFFSET		0x800000 /* LMAC1 */
-#define IWL_BZ_DCCM_LEN			0x10000 /* LMAC1 */
-#define IWL_BZ_DCCM2_OFFSET		0x880000
-#define IWL_BZ_DCCM2_LEN		0x8000
 #define IWL_BZ_SMEM_OFFSET		0x400000
 #define IWL_BZ_SMEM_LEN			0xD0000
 
-#define IWL_BZ_A_HR_B_FW_PRE		"iwlwifi-bz-a0-hr-b0"
-#define IWL_BZ_A_GF_A_FW_PRE		"iwlwifi-bz-a0-gf-a0"
-#define IWL_BZ_A_GF4_A_FW_PRE		"iwlwifi-bz-a0-gf4-a0"
 #define IWL_BZ_A_FM_B_FW_PRE		"iwlwifi-bz-a0-fm-b0"
 #define IWL_BZ_A_FM_C_FW_PRE		"iwlwifi-bz-a0-fm-c0"
 #define IWL_BZ_A_FM4_B_FW_PRE		"iwlwifi-bz-a0-fm4-b0"
 #define IWL_GL_B_FM_B_FW_PRE		"iwlwifi-gl-b0-fm-b0"
 #define IWL_GL_C_FM_C_FW_PRE		"iwlwifi-gl-c0-fm-c0"
 
-#define IWL_BZ_A_HR_B_MODULE_FIRMWARE(api) \
-	IWL_BZ_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_BZ_A_GF_A_MODULE_FIRMWARE(api) \
-	IWL_BZ_A_GF_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_BZ_A_GF4_A_MODULE_FIRMWARE(api) \
-	IWL_BZ_A_GF4_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_BZ_A_FM_B_MODULE_FIRMWARE(api) \
-	IWL_BZ_A_FM_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_BZ_A_FM_C_MODULE_FIRMWARE(api) \
-	IWL_BZ_A_FM_C_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_BZ_A_FM4_B_MODULE_FIRMWARE(api) \
-	IWL_BZ_A_FM4_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_GL_B_FM_B_MODULE_FIRMWARE(api) \
-	IWL_GL_B_FM_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_GL_C_FM_C_MODULE_FIRMWARE(api) \
-	IWL_GL_C_FM_C_FW_PRE "-" __stringify(api) ".ucode"
-
-static const struct iwl_base_params iwl_bz_base_params = {
-	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
+static const struct iwl_family_base_params iwl_bz_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
 	.shadow_ram_support = true,
@@ -62,84 +34,55 @@ static const struct iwl_base_params iwl_bz_base_params = {
 	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
+	.smem_offset = IWL_BZ_SMEM_OFFSET,
+	.smem_len = IWL_BZ_SMEM_LEN,
+	.apmg_not_supported = true,
+	.mac_addr_from_csr = 0x30,
+	.min_umac_error_event_table = 0xD0000,
+	.d3_debug_data_base_addr = 0x401000,
+	.d3_debug_data_length = 60 * 1024,
+	.mon_smem_regs = {
+		.write_ptr = {
+			.addr = LDBG_M2S_BUF_WPTR,
+			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,
+	},
+		.cycle_cnt = {
+			.addr = LDBG_M2S_BUF_WRAP_CNT,
+			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,
+		},
+	},
+	.min_txq_size = 128,
+	.gp2_reg_addr = 0xd02c68,
+	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT,
+	.mon_dram_regs = {
+		.write_ptr = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK,
+		},
+		.cycle_cnt = {
+			.addr = DBGC_DBGBUF_WRAP_AROUND,
+			.mask = 0xffffffff,
+		},
+		.cur_frag = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_IDX_MSK,
+		},
+	},
+	.mon_dbgi_regs = {
+		.write_ptr = {
+			.addr = DBGI_SRAM_FIFO_POINTERS,
+			.mask = DBGI_SRAM_FIFO_POINTERS_WR_PTR_MSK,
+		},
+	},
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.ucode_api_max = IWL_BZ_UCODE_API_MAX,
+	.ucode_api_min = IWL_BZ_UCODE_API_MIN,
 };
 
-#define IWL_DEVICE_BZ_COMMON						\
-	.ucode_api_max = IWL_BZ_UCODE_API_MAX,			\
-	.ucode_api_min = IWL_BZ_UCODE_API_MIN,			\
-	.led_mode = IWL_LED_RF_STATE,					\
-	.nvm_hw_section_num = 10,					\
-	.non_shared_ant = ANT_B,					\
-	.dccm_offset = IWL_BZ_DCCM_OFFSET,				\
-	.dccm_len = IWL_BZ_DCCM_LEN,					\
-	.dccm2_offset = IWL_BZ_DCCM2_OFFSET,				\
-	.dccm2_len = IWL_BZ_DCCM2_LEN,				\
-	.smem_offset = IWL_BZ_SMEM_OFFSET,				\
-	.smem_len = IWL_BZ_SMEM_LEN,					\
-	.apmg_not_supported = true,					\
-	.trans.mq_rx_supported = true,					\
-	.vht_mu_mimo_supported = true,					\
-	.mac_addr_from_csr = 0x30,					\
-	.nvm_ver = IWL_BZ_NVM_VERSION,				\
-	.trans.rf_id = true,						\
-	.trans.gen2 = true,						\
-	.nvm_type = IWL_NVM_EXT,					\
-	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0xD0000,				\
-	.d3_debug_data_base_addr = 0x401000,				\
-	.d3_debug_data_length = 60 * 1024,				\
-	.mon_smem_regs = {						\
-		.write_ptr = {						\
-			.addr = LDBG_M2S_BUF_WPTR,			\
-			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,		\
-	},								\
-		.cycle_cnt = {						\
-			.addr = LDBG_M2S_BUF_WRAP_CNT,			\
-			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,		\
-		},							\
-	},								\
-	.trans.umac_prph_offset = 0x300000,				\
-	.trans.device_family = IWL_DEVICE_FAMILY_BZ,			\
-	.trans.base_params = &iwl_bz_base_params,			\
-	.min_txq_size = 128,						\
-	.gp2_reg_addr = 0xd02c68,					\
-	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT,			\
-	.mon_dram_regs = {						\
-		.write_ptr = {						\
-			.addr = DBGC_CUR_DBGBUF_STATUS,			\
-			.mask = DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK,	\
-		},							\
-		.cycle_cnt = {						\
-			.addr = DBGC_DBGBUF_WRAP_AROUND,		\
-			.mask = 0xffffffff,				\
-		},							\
-		.cur_frag = {						\
-			.addr = DBGC_CUR_DBGBUF_STATUS,			\
-			.mask = DBGC_CUR_DBGBUF_STATUS_IDX_MSK,		\
-		},							\
-	},								\
-	.mon_dbgi_regs = {						\
-		.write_ptr = {						\
-			.addr = DBGI_SRAM_FIFO_POINTERS,		\
-			.mask = DBGI_SRAM_FIFO_POINTERS_WR_PTR_MSK,	\
-		},							\
-	}
-
-#define IWL_DEVICE_BZ							\
-	IWL_DEVICE_BZ_COMMON,						\
-	.ht_params = &iwl_22000_ht_params
-
-/*
- * This size was picked according to 8 MSDUs inside 512 A-MSDUs in an
- * A-MPDU, with additional overhead to account for processing time.
- */
-#define IWL_NUM_RBDS_BZ_EHT		(512 * 16)
-
-const struct iwl_cfg_trans_params iwl_bz_trans_cfg = {
+const struct iwl_mac_cfg iwl_bz_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_BZ,
-	.base_params = &iwl_bz_base_params,
+	.base = &iwl_bz_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.integrated = true,
 	.umac_prph_offset = 0x300000,
@@ -148,35 +91,18 @@ const struct iwl_cfg_trans_params iwl_bz_trans_cfg = {
 	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_2500US,
 };
 
-const char iwl_bz_name[] = "Intel(R) TBD Bz device";
-const char iwl_fm_name[] = "Intel(R) Wi-Fi 7 BE201 320MHz";
-const char iwl_gl_name[] = "Intel(R) Wi-Fi 7 BE200 320MHz";
-const char iwl_mtp_name[] = "Intel(R) Wi-Fi 7 BE202 160MHz";
-
-const struct iwl_cfg iwl_cfg_bz = {
-	.fw_name_mac = "bz",
-	.uhb_supported = true,
-	IWL_DEVICE_BZ,
-	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
-	.num_rbds = IWL_NUM_RBDS_BZ_EHT,
-};
-
-const struct iwl_cfg iwl_cfg_gl = {
-	.fw_name_mac = "gl",
-	.uhb_supported = true,
-	IWL_DEVICE_BZ,
-	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
-	.num_rbds = IWL_NUM_RBDS_BZ_EHT,
+const struct iwl_mac_cfg iwl_gl_mac_cfg = {
+	.device_family = IWL_DEVICE_FAMILY_BZ,
+	.base = &iwl_bz_base,
+	.mq_rx_supported = true,
+	.gen2 = true,
+	.umac_prph_offset = 0x300000,
+	.xtal_latency = 12000,
+	.low_latency_xtal = true,
 };
 
-
-MODULE_FIRMWARE(IWL_BZ_A_HR_B_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_BZ_A_GF_A_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_BZ_A_GF4_A_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_BZ_A_FM_B_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_BZ_A_FM_C_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_BZ_A_FM4_B_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_GL_B_FM_B_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_GL_C_FM_C_MODULE_FIRMWARE(IWL_BZ_UCODE_API_MAX));
-
-MODULE_FIRMWARE("iwlwifi-gl-c0-fm-c0.pnvm");
+IWL_FW_AND_PNVM(IWL_BZ_A_FM_B_FW_PRE, IWL_BZ_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_BZ_A_FM_C_FW_PRE, IWL_BZ_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_BZ_A_FM4_B_FW_PRE, IWL_BZ_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_GL_B_FM_B_FW_PRE, IWL_BZ_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_GL_C_FM_C_FW_PRE, IWL_BZ_UCODE_API_MAX);
diff --git a/sys/contrib/dev/iwlwifi/cfg/dr.c b/sys/contrib/dev/iwlwifi/cfg/dr.c
new file mode 100644
index 000000000000..807f4e29d55a
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/cfg/dr.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <linux/module.h>
+#include <linux/stringify.h>
+#include "iwl-config.h"
+#include "iwl-prph.h"
+#include "fw/api/txq.h"
+
+/* Highest firmware API version supported */
+#define IWL_DR_UCODE_API_MAX	102
+
+/* Lowest firmware API version supported */
+#define IWL_DR_UCODE_API_MIN	98
+
+/* Memory offsets and lengths */
+#define IWL_DR_SMEM_OFFSET		0x400000
+#define IWL_DR_SMEM_LEN			0xD0000
+
+#define IWL_DR_A_PE_A_FW_PRE		"iwlwifi-dr-a0-pe-a0"
+
+#define IWL_DR_A_PE_A_FW_MODULE_FIRMWARE(api) \
+	IWL_DR_A_PE_A_FW_PRE "-" __stringify(api) ".ucode"
+
+static const struct iwl_family_base_params iwl_dr_base = {
+	.num_of_queues = 512,
+	.max_tfd_queue_size = 65536,
+	.shadow_ram_support = true,
+	.led_compensation = 57,
+	.wd_timeout = IWL_LONG_WD_TIMEOUT,
+	.max_event_log_size = 512,
+	.shadow_reg_enable = true,
+	.pcie_l1_allowed = true,
+	.smem_offset = IWL_DR_SMEM_OFFSET,
+	.smem_len = IWL_DR_SMEM_LEN,
+	.apmg_not_supported = true,
+	.mac_addr_from_csr = 0x30,
+	.min_umac_error_event_table = 0xD0000,
+	.d3_debug_data_base_addr = 0x401000,
+	.d3_debug_data_length = 60 * 1024,
+	.mon_smem_regs = {
+		.write_ptr = {
+			.addr = LDBG_M2S_BUF_WPTR,
+			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,
+		},
+		.cycle_cnt = {
+			.addr = LDBG_M2S_BUF_WRAP_CNT,
+			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,
+		},
+	},
+	.min_txq_size = 128,
+	.gp2_reg_addr = 0xd02c68,
+	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT,
+	.mon_dram_regs = {
+		.write_ptr = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK,
+		},
+		.cycle_cnt = {
+			.addr = DBGC_DBGBUF_WRAP_AROUND,
+			.mask = 0xffffffff,
+		},
+		.cur_frag = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_IDX_MSK,
+		},
+	},
+	.mon_dbgi_regs = {
+		.write_ptr = {
+			.addr = DBGI_SRAM_FIFO_POINTERS,
+			.mask = DBGI_SRAM_FIFO_POINTERS_WR_PTR_MSK,
+		},
+	},
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.ucode_api_max = IWL_DR_UCODE_API_MAX,
+	.ucode_api_min = IWL_DR_UCODE_API_MIN,
+};
+
+const struct iwl_mac_cfg iwl_dr_mac_cfg = {
+	.device_family = IWL_DEVICE_FAMILY_DR,
+	.base = &iwl_dr_base,
+	.mq_rx_supported = true,
+	.gen2 = true,
+	.integrated = true,
+	.umac_prph_offset = 0x300000,
+	.xtal_latency = 12000,
+	.low_latency_xtal = true,
+	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_2500US,
+};
+
+MODULE_FIRMWARE(IWL_DR_A_PE_A_FW_MODULE_FIRMWARE(IWL_DR_UCODE_API_MAX));
+
diff --git a/sys/contrib/dev/iwlwifi/cfg/rf-fm.c b/sys/contrib/dev/iwlwifi/cfg/rf-fm.c
new file mode 100644
index 000000000000..456a666c8dfd
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/cfg/rf-fm.c
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2015-2017 Intel Deutschland GmbH
+ * Copyright (C) 2018-2025 Intel Corporation
+ */
+#include "iwl-config.h"
+
+/* NVM versions */
+#define IWL_FM_NVM_VERSION		0x0a1d
+
+#define IWL_DEVICE_FM							\
+	.ht_params = {							\
+		.stbc = true,						\
+		.ldpc = true,						\
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) |			\
+			      BIT(NL80211_BAND_5GHZ),			\
+	},								\
+	.led_mode = IWL_LED_RF_STATE,					\
+	.non_shared_ant = ANT_B,					\
+	.vht_mu_mimo_supported = true,					\
+	.uhb_supported = true,						\
+	.num_rbds = IWL_NUM_RBDS_EHT,					\
+	.nvm_ver = IWL_FM_NVM_VERSION,					\
+	.nvm_type = IWL_NVM_EXT
+
+const struct iwl_rf_cfg iwl_rf_fm = {
+	IWL_DEVICE_FM,
+};
+
+const struct iwl_rf_cfg iwl_rf_fm_160mhz = {
+	IWL_DEVICE_FM,
+	.bw_limit = 160,
+};
+
+const char iwl_killer_be1750s_name[] =
+	"Killer(R) Wi-Fi 7 BE1750s 320MHz Wireless Network Adapter (BE201D2W)";
+const char iwl_killer_be1750i_name[] =
+	"Killer(R) Wi-Fi 7 BE1750i 320MHz Wireless Network Adapter (BE201NGW)";
+const char iwl_killer_be1750w_name[] =
+	"Killer(TM) Wi-Fi 7 BE1750w 320MHz Wireless Network Adapter (BE200D2W)";
+const char iwl_killer_be1750x_name[] =
+	"Killer(TM) Wi-Fi 7 BE1750x 320MHz Wireless Network Adapter (BE200NGW)";
+const char iwl_killer_be1790s_name[] =
+	"Killer(R) Wi-Fi 7 BE1790s 320MHz Wireless Network Adapter (BE401D2W)";
+const char iwl_killer_be1790i_name[] =
+	"Killer(R) Wi-Fi 7 BE1790i 320MHz Wireless Network Adapter (BE401NGW)";
+
+const char iwl_be201_name[] = "Intel(R) Wi-Fi 7 BE201 320MHz";
+const char iwl_be200_name[] = "Intel(R) Wi-Fi 7 BE200 320MHz";
+const char iwl_be202_name[] = "Intel(R) Wi-Fi 7 BE202 160MHz";
+const char iwl_be401_name[] = "Intel(R) Wi-Fi 7 BE401 320MHz";
diff --git a/sys/contrib/dev/iwlwifi/cfg/rf-gf.c b/sys/contrib/dev/iwlwifi/cfg/rf-gf.c
new file mode 100644
index 000000000000..7ff5170faaa9
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/cfg/rf-gf.c
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2015-2017 Intel Deutschland GmbH
+ * Copyright (C) 2018-2025 Intel Corporation
+ */
+#include "iwl-config.h"
+
+/* Highest firmware API version supported */
+#define IWL_GF_UCODE_API_MAX	100
+
+/* Lowest firmware API version supported */
+#define IWL_GF_UCODE_API_MIN	98
+
+#define IWL_SO_A_GF_A_FW_PRE		"iwlwifi-so-a0-gf-a0"
+#define IWL_TY_A_GF_A_FW_PRE		"iwlwifi-ty-a0-gf-a0"
+#define IWL_MA_A_GF_A_FW_PRE		"iwlwifi-ma-a0-gf-a0"
+#define IWL_MA_B_GF_A_FW_PRE		"iwlwifi-ma-b0-gf-a0"
+#define IWL_SO_A_GF4_A_FW_PRE		"iwlwifi-so-a0-gf4-a0"
+#define IWL_MA_A_GF4_A_FW_PRE		"iwlwifi-ma-a0-gf4-a0"
+#define IWL_MA_B_GF4_A_FW_PRE		"iwlwifi-ma-b0-gf4-a0"
+#define IWL_BZ_A_GF_A_FW_PRE		"iwlwifi-bz-a0-gf-a0"
+#define IWL_BZ_A_GF4_A_FW_PRE		"iwlwifi-bz-a0-gf4-a0"
+#define IWL_SC_A_GF_A_FW_PRE		"iwlwifi-sc-a0-gf-a0"
+#define IWL_SC_A_GF4_A_FW_PRE		"iwlwifi-sc-a0-gf4-a0"
+
+/* NVM versions */
+#define IWL_GF_NVM_VERSION		0x0a1d
+
+const struct iwl_rf_cfg iwl_rf_gf = {
+	.uhb_supported = true,
+	.led_mode = IWL_LED_RF_STATE,
+	.non_shared_ant = ANT_B,
+	.vht_mu_mimo_supported = true,
+	.ht_params = {
+		.stbc = true,
+		.ldpc = true,
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) |
+			      BIT(NL80211_BAND_5GHZ),
+	},
+	.nvm_ver = IWL_GF_NVM_VERSION,
+	.nvm_type = IWL_NVM_EXT,
+	.num_rbds = IWL_NUM_RBDS_HE,
+	.ucode_api_min = IWL_GF_UCODE_API_MIN,
+	.ucode_api_max = IWL_GF_UCODE_API_MAX,
+};
+
+const char iwl_ax210_killer_1675w_name[] =
+	"Killer(R) Wi-Fi 6E AX1675w 160MHz Wireless Network Adapter (210D2W)";
+const char iwl_ax210_killer_1675x_name[] =
+	"Killer(R) Wi-Fi 6E AX1675x 160MHz Wireless Network Adapter (210NGW)";
+const char iwl_ax211_killer_1675s_name[] =
+	"Killer(R) Wi-Fi 6E AX1675s 160MHz Wireless Network Adapter (211D2W)";
+const char iwl_ax211_killer_1675i_name[] =
+	"Killer(R) Wi-Fi 6E AX1675i 160MHz Wireless Network Adapter (211NGW)";
+const char iwl_ax411_killer_1690s_name[] =
+	"Killer(R) Wi-Fi 6E AX1690s 160MHz Wireless Network Adapter (411D2W)";
+const char iwl_ax411_killer_1690i_name[] =
+	"Killer(R) Wi-Fi 6E AX1690i 160MHz Wireless Network Adapter (411NGW)";
+
+const char iwl_ax210_name[] = "Intel(R) Wi-Fi 6E AX210 160MHz";
+const char iwl_ax211_name[] = "Intel(R) Wi-Fi 6E AX211 160MHz";
+const char iwl_ax411_name[] = "Intel(R) Wi-Fi 6E AX411 160MHz";
+
+IWL_FW_AND_PNVM(IWL_SO_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_TY_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_B_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_A_GF4_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_B_GF4_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_BZ_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_BZ_A_GF4_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_SC_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_SC_A_GF4_A_FW_PRE, IWL_GF_UCODE_API_MAX);
diff --git a/sys/contrib/dev/iwlwifi/cfg/rf-hr.c b/sys/contrib/dev/iwlwifi/cfg/rf-hr.c
new file mode 100644
index 000000000000..9f408d276ce9
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/cfg/rf-hr.c
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2015-2017 Intel Deutschland GmbH
+ * Copyright (C) 2018-2025 Intel Corporation
+ */
+#include "iwl-config.h"
+
+/* Highest firmware API version supported */
+#define IWL_HR_UCODE_API_MAX	100
+
+/* Lowest firmware API version supported */
+#define IWL_HR_UCODE_API_MIN	98
+
+#define IWL_QU_B_HR_B_FW_PRE		"iwlwifi-Qu-b0-hr-b0"
+#define IWL_QU_C_HR_B_FW_PRE		"iwlwifi-Qu-c0-hr-b0"
+#define IWL_QUZ_A_HR_B_FW_PRE		"iwlwifi-QuZ-a0-hr-b0"
+#define IWL_SO_A_HR_B_FW_PRE		"iwlwifi-so-a0-hr-b0"
+#define IWL_MA_A_HR_B_FW_PRE		"iwlwifi-ma-a0-hr-b0"
+#define IWL_MA_B_HR_B_FW_PRE		"iwlwifi-ma-b0-hr-b0"
+#define IWL_BZ_A_HR_B_FW_PRE		"iwlwifi-bz-a0-hr-b0"
+#define IWL_SC_A_HR_A_FW_PRE		"iwlwifi-sc-a0-hr-b0"
+#define IWL_SC_A_HR_B_FW_PRE		"iwlwifi-sc-a0-hr-b0"
+
+#define IWL_QU_B_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_QU_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_QUZ_A_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_QUZ_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_QU_C_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_QU_C_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_SO_A_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_SO_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_MA_A_HR_B_FW_MODULE_FIRMWARE(api)	\
+	IWL_MA_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(api)	\
+	IWL_MA_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_BZ_A_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_BZ_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_SC_A_HR_A_FW_MODULE_FIRMWARE(api)	\
+	IWL_SC_A_HR_A_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_SC_A_HR_B_FW_MODULE_FIRMWARE(api)	\
+	IWL_SC_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+
+/* NVM versions */
+#define IWL_HR_NVM_VERSION		0x0a1d
+
+#define IWL_DEVICE_HR							\
+	.led_mode = IWL_LED_RF_STATE,					\
+	.non_shared_ant = ANT_B,					\
+	.vht_mu_mimo_supported = true,					\
+	.ht_params = {							\
+		.stbc = true,						\
+		.ldpc = true,						\
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) |			\
+			      BIT(NL80211_BAND_5GHZ),			\
+	},								\
+	.num_rbds = IWL_NUM_RBDS_HE,					\
+	.nvm_ver = IWL_HR_NVM_VERSION,					\
+	.nvm_type = IWL_NVM_EXT,					\
+	.ucode_api_min = IWL_HR_UCODE_API_MIN,				\
+	.ucode_api_max = IWL_HR_UCODE_API_MAX
+
+const struct iwl_rf_cfg iwl_rf_hr1 = {
+	IWL_DEVICE_HR,
+	.tx_with_siso_diversity = true,
+};
+
+const struct iwl_rf_cfg iwl_rf_hr = {
+	IWL_DEVICE_HR,
+};
+
+const struct iwl_rf_cfg iwl_rf_hr_80mhz = {
+	IWL_DEVICE_HR,
+	.bw_limit = 80,
+};
+
+const char iwl_ax101_name[] = "Intel(R) Wi-Fi 6 AX101";
+const char iwl_ax200_name[] = "Intel(R) Wi-Fi 6 AX200 160MHz";
+const char iwl_ax201_name[] = "Intel(R) Wi-Fi 6 AX201 160MHz";
+const char iwl_ax203_name[] = "Intel(R) Wi-Fi 6 AX203";
+
+MODULE_FIRMWARE(IWL_QU_B_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_QU_C_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_QUZ_A_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_SO_A_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_A_HR_B_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BZ_A_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_SC_A_HR_A_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_SC_A_HR_B_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
diff --git a/sys/contrib/dev/iwlwifi/cfg/rf-jf.c b/sys/contrib/dev/iwlwifi/cfg/rf-jf.c
new file mode 100644
index 000000000000..0a074e0a3bc6
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/cfg/rf-jf.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2015-2017 Intel Deutschland GmbH
+ * Copyright (C) 2018-2021, 2023, 2025 Intel Corporation
+ */
+#include "iwl-config.h"
+
+/* Highest firmware API version supported */
+#define IWL_JF_UCODE_API_MAX	77
+
+/* Lowest firmware API version supported */
+#define IWL_JF_UCODE_API_MIN	77
+
+#define IWL_QU_B_JF_B_FW_PRE		"iwlwifi-Qu-b0-jf-b0"
+#define IWL_QU_C_JF_B_FW_PRE		"iwlwifi-Qu-c0-jf-b0"
+#define IWL_QUZ_A_JF_B_FW_PRE		"iwlwifi-QuZ-a0-jf-b0"
+#define IWL_SO_A_JF_B_FW_PRE		"iwlwifi-so-a0-jf-b0"
+
+#define IWL_QUZ_A_JF_B_MODULE_FIRMWARE(api) \
+	IWL_QUZ_A_JF_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_QU_B_JF_B_MODULE_FIRMWARE(api) \
+	IWL_QU_B_JF_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_QU_C_JF_B_MODULE_FIRMWARE(api) \
+	IWL_QU_C_JF_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_SO_A_JF_B_MODULE_FIRMWARE(api) \
+	IWL_SO_A_JF_B_FW_PRE "-" __stringify(api) ".ucode"
+
+/* NVM versions */
+#define IWL_JF_NVM_VERSION		0x0a1d
+
+/* Memory offsets and lengths */
+#define IWL9000_DCCM_OFFSET		0x800000
+#define IWL9000_DCCM_LEN		0x18000
+#define IWL9000_DCCM2_OFFSET		0x880000
+#define IWL9000_DCCM2_LEN		0x8000
+
+static const struct iwl_tt_params iwl_jf_tt_params = {
+	.ct_kill_entry = 115,
+	.ct_kill_exit = 93,
+	.ct_kill_duration = 5,
+	.dynamic_smps_entry = 111,
+	.dynamic_smps_exit = 107,
+	.tx_protection_entry = 112,
+	.tx_protection_exit = 105,
+	.tx_backoff = {
+		{.temperature = 110, .backoff = 200},
+		{.temperature = 111, .backoff = 600},
+		{.temperature = 112, .backoff = 1200},
+		{.temperature = 113, .backoff = 2000},
+		{.temperature = 114, .backoff = 4000},
+	},
+	.support_ct_kill = true,
+	.support_dynamic_smps = true,
+	.support_tx_protection = true,
+	.support_tx_backoff = true,
+};
+
+/* these values are ignored if not with Pu/Th MAC firmware, due to offload */
+#define IWL_DEVICE_JF_PU						\
+	.dccm_offset = IWL9000_DCCM_OFFSET,				\
+	.dccm_len = IWL9000_DCCM_LEN,					\
+	.dccm2_offset = IWL9000_DCCM2_OFFSET,				\
+	.dccm2_len = IWL9000_DCCM2_LEN,					\
+	.thermal_params = &iwl_jf_tt_params
+
+#define IWL_DEVICE_JF							\
+	IWL_DEVICE_JF_PU,						\
+	.led_mode = IWL_LED_RF_STATE,					\
+	.non_shared_ant = ANT_B,					\
+	.num_rbds = IWL_NUM_RBDS_NON_HE,				\
+	.vht_mu_mimo_supported = true,					\
+	.ht_params = {							\
+		.stbc = true,						\
+		.ldpc = true,						\
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) |			\
+			      BIT(NL80211_BAND_5GHZ),			\
+	},								\
+	.nvm_ver = IWL_JF_NVM_VERSION,					\
+	.nvm_type = IWL_NVM_EXT,					\
+	.ucode_api_min = IWL_JF_UCODE_API_MIN,				\
+	.ucode_api_max = IWL_JF_UCODE_API_MAX
+
+const struct iwl_rf_cfg iwl_rf_jf = {
+	IWL_DEVICE_JF,
+};
+
+const struct iwl_rf_cfg iwl_rf_jf_80mhz = {
+	IWL_DEVICE_JF,
+	.bw_limit = 80,
+};
+
+const char iwl9260_name[] = "Intel(R) Wireless-AC 9260";
+const char iwl9461_name[] = "Intel(R) Wireless-AC 9461";
+const char iwl9462_name[] = "Intel(R) Wireless-AC 9462";
+const char iwl9560_name[] = "Intel(R) Wireless-AC 9560";
+const char iwl9260_160_name[] = "Intel(R) Wireless-AC 9260 160MHz";
+const char iwl9461_160_name[] = "Intel(R) Wireless-AC 9461 160MHz";
+const char iwl9462_160_name[] = "Intel(R) Wireless-AC 9462 160MHz";
+const char iwl9560_160_name[] = "Intel(R) Wireless-AC 9560 160MHz";
+
+const char iwl9260_killer_1550_name[] =
+	"Killer(R) Wireless-AC 1550 Wireless Network Adapter (9260NGW) 160MHz";
+const char iwl9560_killer_1550i_name[] =
+	"Killer(R) Wireless-AC 1550i Wireless Network Adapter (9560NGW) 160MHz";
+const char iwl9560_killer_1550s_name[] =
+	"Killer(R) Wireless-AC 1550s Wireless Network Adapter (9560D2W) 160MHz";
+
+MODULE_FIRMWARE(IWL_QU_B_JF_B_MODULE_FIRMWARE(IWL_JF_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_QU_C_JF_B_MODULE_FIRMWARE(IWL_JF_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_QUZ_A_JF_B_MODULE_FIRMWARE(IWL_JF_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_SO_A_JF_B_MODULE_FIRMWARE(IWL_JF_UCODE_API_MAX));
diff --git a/sys/contrib/dev/iwlwifi/cfg/rf-pe.c b/sys/contrib/dev/iwlwifi/cfg/rf-pe.c
new file mode 100644
index 000000000000..483f21659eff
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/cfg/rf-pe.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+#include "iwl-config.h"
+
+/* currently iwl_rf_wh/iwl_rf_wh_160mhz are just defines for the FM ones */
+
+const char iwl_killer_bn1850w2_name[] =
+	"Killer(R) Wi-Fi 8 BN1850w2 320MHz Wireless Network Adapter (BN201.D2W)";
+const char iwl_killer_bn1850i_name[] =
+	"Killer(R) Wi-Fi 8 BN1850i 320MHz Wireless Network Adapter (BN201.NGW)";
+
+const char iwl_bn201_name[] = "Intel(R) Wi-Fi 8 BN201";
+const char iwl_be221_name[] = "Intel(R) Wi-Fi 7 BE221";
+const char iwl_be223_name[] = "Intel(R) Wi-Fi 7 BE223";
diff --git a/sys/contrib/dev/iwlwifi/cfg/rf-wh.c b/sys/contrib/dev/iwlwifi/cfg/rf-wh.c
new file mode 100644
index 000000000000..97735175cb0e
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/cfg/rf-wh.c
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+#include "iwl-config.h"
+
+/* currently iwl_rf_wh/iwl_rf_wh_160mhz are just defines for the FM ones */
+
+const char iwl_killer_be1775s_name[] =
+	"Killer(R) Wi-Fi 7 BE1775s 320MHz Wireless Network Adapter (BE211D2W)";
+const char iwl_killer_be1775i_name[] =
+	"Killer(R) Wi-Fi 7 BE1775i 320MHz Wireless Network Adapter (BE211NGW)";
+
+const char iwl_be211_name[] = "Intel(R) Wi-Fi 7 BE211 320MHz";
+const char iwl_be213_name[] = "Intel(R) Wi-Fi 7 BE213 160MHz";
diff --git a/sys/contrib/dev/iwlwifi/cfg/sc.c b/sys/contrib/dev/iwlwifi/cfg/sc.c
index 4ccb0b7bdc20..6d4a3bce49b9 100644
--- a/sys/contrib/dev/iwlwifi/cfg/sc.c
+++ b/sys/contrib/dev/iwlwifi/cfg/sc.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -10,59 +10,25 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware API version supported */
-#define IWL_SC_UCODE_API_MAX	92
+#define IWL_SC_UCODE_API_MAX	102
 
 /* Lowest firmware API version supported */
-#define IWL_SC_UCODE_API_MIN	90
+#define IWL_SC_UCODE_API_MIN	98
 
 /* NVM versions */
 #define IWL_SC_NVM_VERSION		0x0a1d
 
 /* Memory offsets and lengths */
-#define IWL_SC_DCCM_OFFSET		0x800000 /* LMAC1 */
-#define IWL_SC_DCCM_LEN			0x10000 /* LMAC1 */
-#define IWL_SC_DCCM2_OFFSET		0x880000
-#define IWL_SC_DCCM2_LEN		0x8000
 #define IWL_SC_SMEM_OFFSET		0x400000
 #define IWL_SC_SMEM_LEN			0xD0000
 
 #define IWL_SC_A_FM_B_FW_PRE		"iwlwifi-sc-a0-fm-b0"
 #define IWL_SC_A_FM_C_FW_PRE		"iwlwifi-sc-a0-fm-c0"
-#define IWL_SC_A_HR_A_FW_PRE		"iwlwifi-sc-a0-hr-b0"
-#define IWL_SC_A_HR_B_FW_PRE		"iwlwifi-sc-a0-hr-b0"
-#define IWL_SC_A_GF_A_FW_PRE		"iwlwifi-sc-a0-gf-a0"
-#define IWL_SC_A_GF4_A_FW_PRE		"iwlwifi-sc-a0-gf4-a0"
 #define IWL_SC_A_WH_A_FW_PRE		"iwlwifi-sc-a0-wh-a0"
 #define IWL_SC2_A_FM_C_FW_PRE		"iwlwifi-sc2-a0-fm-c0"
 #define IWL_SC2_A_WH_A_FW_PRE		"iwlwifi-sc2-a0-wh-a0"
-#define IWL_SC2F_A_FM_C_FW_PRE		"iwlwifi-sc2f-a0-fm-c0"
-#define IWL_SC2F_A_WH_A_FW_PRE		"iwlwifi-sc2f-a0-wh-a0"
 
-#define IWL_SC_A_FM_B_FW_MODULE_FIRMWARE(api) \
-	IWL_SC_A_FM_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC_A_FM_C_FW_MODULE_FIRMWARE(api) \
-	IWL_SC_A_FM_C_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC_A_HR_A_FW_MODULE_FIRMWARE(api) \
-	IWL_SC_A_HR_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC_A_HR_B_FW_MODULE_FIRMWARE(api) \
-	IWL_SC_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC_A_GF_A_FW_MODULE_FIRMWARE(api) \
-	IWL_SC_A_GF_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC_A_GF4_A_FW_MODULE_FIRMWARE(api) \
-	IWL_SC_A_GF4_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC_A_WH_A_FW_MODULE_FIRMWARE(api) \
-	IWL_SC_A_WH_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC2_A_FM_C_FW_MODULE_FIRMWARE(api) \
-	IWL_SC2_A_FM_C_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC2_A_WH_A_FW_MODULE_FIRMWARE(api) \
-	IWL_SC2_A_WH_A_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC2F_A_FM_C_FW_MODULE_FIRMWARE(api) \
-	IWL_SC2F_A_FM_C_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SC2F_A_WH_A_FW_MODULE_FIRMWARE(api) \
-	IWL_SC2F_A_WH_A_FW_PRE "-" __stringify(api) ".ucode"
-
-static const struct iwl_base_params iwl_sc_base_params = {
-	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
+static const struct iwl_family_base_params iwl_sc_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
 	.shadow_ram_support = true,
@@ -71,87 +37,55 @@ static const struct iwl_base_params iwl_sc_base_params = {
 	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
+	.smem_offset = IWL_SC_SMEM_OFFSET,
+	.smem_len = IWL_SC_SMEM_LEN,
+	.apmg_not_supported = true,
+	.mac_addr_from_csr = 0x30,
+	.min_umac_error_event_table = 0xD0000,
+	.d3_debug_data_base_addr = 0x401000,
+	.d3_debug_data_length = 60 * 1024,
+	.mon_smem_regs = {
+		.write_ptr = {
+			.addr = LDBG_M2S_BUF_WPTR,
+			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,
+		},
+		.cycle_cnt = {
+			.addr = LDBG_M2S_BUF_WRAP_CNT,
+			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,
+		},
+	},
+	.min_txq_size = 128,
+	.gp2_reg_addr = 0xd02c68,
+	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT,
+	.mon_dram_regs = {
+		.write_ptr = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK,
+		},
+		.cycle_cnt = {
+			.addr = DBGC_DBGBUF_WRAP_AROUND,
+			.mask = 0xffffffff,
+		},
+		.cur_frag = {
+			.addr = DBGC_CUR_DBGBUF_STATUS,
+			.mask = DBGC_CUR_DBGBUF_STATUS_IDX_MSK,
+		},
+	},
+	.mon_dbgi_regs = {
+		.write_ptr = {
+			.addr = DBGI_SRAM_FIFO_POINTERS,
+			.mask = DBGI_SRAM_FIFO_POINTERS_WR_PTR_MSK,
+		},
+	},
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.ucode_api_max = IWL_SC_UCODE_API_MAX,
+	.ucode_api_min = IWL_SC_UCODE_API_MIN,
 };
 
-#define IWL_DEVICE_BZ_COMMON						\
-	.ucode_api_max = IWL_SC_UCODE_API_MAX,			\
-	.ucode_api_min = IWL_SC_UCODE_API_MIN,			\
-	.led_mode = IWL_LED_RF_STATE,					\
-	.nvm_hw_section_num = 10,					\
-	.non_shared_ant = ANT_B,					\
-	.dccm_offset = IWL_SC_DCCM_OFFSET,				\
-	.dccm_len = IWL_SC_DCCM_LEN,					\
-	.dccm2_offset = IWL_SC_DCCM2_OFFSET,				\
-	.dccm2_len = IWL_SC_DCCM2_LEN,				\
-	.smem_offset = IWL_SC_SMEM_OFFSET,				\
-	.smem_len = IWL_SC_SMEM_LEN,					\
-	.apmg_not_supported = true,					\
-	.trans.mq_rx_supported = true,					\
-	.vht_mu_mimo_supported = true,					\
-	.mac_addr_from_csr = 0x30,					\
-	.nvm_ver = IWL_SC_NVM_VERSION,				\
-	.trans.rf_id = true,						\
-	.trans.gen2 = true,						\
-	.nvm_type = IWL_NVM_EXT,					\
-	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0xD0000,				\
-	.d3_debug_data_base_addr = 0x401000,				\
-	.d3_debug_data_length = 60 * 1024,				\
-	.mon_smem_regs = {						\
-		.write_ptr = {						\
-			.addr = LDBG_M2S_BUF_WPTR,			\
-			.mask = LDBG_M2S_BUF_WPTR_VAL_MSK,		\
-	},								\
-		.cycle_cnt = {						\
-			.addr = LDBG_M2S_BUF_WRAP_CNT,			\
-			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,		\
-		},							\
-	},								\
-	.trans.umac_prph_offset = 0x300000,				\
-	.trans.device_family = IWL_DEVICE_FAMILY_SC,			\
-	.trans.base_params = &iwl_sc_base_params,			\
-	.min_txq_size = 128,						\
-	.gp2_reg_addr = 0xd02c68,					\
-	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT,			\
-	.mon_dram_regs = {						\
-		.write_ptr = {						\
-			.addr = DBGC_CUR_DBGBUF_STATUS,			\
-			.mask = DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK,	\
-		},							\
-		.cycle_cnt = {						\
-			.addr = DBGC_DBGBUF_WRAP_AROUND,		\
-			.mask = 0xffffffff,				\
-		},							\
-		.cur_frag = {						\
-			.addr = DBGC_CUR_DBGBUF_STATUS,			\
-			.mask = DBGC_CUR_DBGBUF_STATUS_IDX_MSK,		\
-		},							\
-	},								\
-	.mon_dbgi_regs = {						\
-		.write_ptr = {						\
-			.addr = DBGI_SRAM_FIFO_POINTERS,		\
-			.mask = DBGI_SRAM_FIFO_POINTERS_WR_PTR_MSK,	\
-		},							\
-	}
-
-#define IWL_DEVICE_SC							\
-	IWL_DEVICE_BZ_COMMON,						\
-	.uhb_supported = true,						\
-	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,		\
-	.num_rbds = IWL_NUM_RBDS_SC_EHT,				\
-	.ht_params = &iwl_22000_ht_params
-
-/*
- * This size was picked according to 8 MSDUs inside 512 A-MSDUs in an
- * A-MPDU, with additional overhead to account for processing time.
- */
-#define IWL_NUM_RBDS_SC_EHT		(512 * 16)
-
-const struct iwl_cfg_trans_params iwl_sc_trans_cfg = {
+const struct iwl_mac_cfg iwl_sc_mac_cfg = {
 	.device_family = IWL_DEVICE_FAMILY_SC,
-	.base_params = &iwl_sc_base_params,
+	.base = &iwl_sc_base,
 	.mq_rx_supported = true,
-	.rf_id = true,
 	.gen2 = true,
 	.integrated = true,
 	.umac_prph_offset = 0x300000,
@@ -160,35 +94,8 @@ const struct iwl_cfg_trans_params iwl_sc_trans_cfg = {
 	.ltr_delay = IWL_CFG_TRANS_LTR_DELAY_2500US,
 };
 
-const char iwl_sc_name[] = "Intel(R) TBD Sc device";
-
-const struct iwl_cfg iwl_cfg_sc = {
-	.fw_name_mac = "sc",
-	IWL_DEVICE_SC,
-};
-
-const char iwl_sc2_name[] = "Intel(R) TBD Sc2 device";
-
-const struct iwl_cfg iwl_cfg_sc2 = {
-	.fw_name_mac = "sc2",
-	IWL_DEVICE_SC,
-};
-
-const char iwl_sc2f_name[] = "Intel(R) TBD Sc2f device";
-
-const struct iwl_cfg iwl_cfg_sc2f = {
-	.fw_name_mac = "sc2f",
-	IWL_DEVICE_SC,
-};
-
-MODULE_FIRMWARE(IWL_SC_A_FM_B_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC_A_FM_C_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC_A_HR_A_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC_A_HR_B_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC_A_GF_A_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC_A_GF4_A_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC_A_WH_A_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC2_A_FM_C_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC2_A_WH_A_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC2F_A_FM_C_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SC2F_A_WH_A_FW_MODULE_FIRMWARE(IWL_SC_UCODE_API_MAX));
+IWL_FW_AND_PNVM(IWL_SC_A_FM_B_FW_PRE, IWL_SC_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_SC_A_FM_C_FW_PRE, IWL_SC_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_SC_A_WH_A_FW_PRE, IWL_SC_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_SC2_A_FM_C_FW_PRE, IWL_SC_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_SC2_A_WH_A_FW_PRE, IWL_SC_UCODE_API_MAX);
diff --git a/sys/contrib/dev/iwlwifi/fw/acpi.c b/sys/contrib/dev/iwlwifi/fw/acpi.c
index 8c8880b44827..7ec22738b5d6 100644
--- a/sys/contrib/dev/iwlwifi/fw/acpi.c
+++ b/sys/contrib/dev/iwlwifi/fw/acpi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2019-2024 Intel Corporation
+ * Copyright (C) 2019-2025 Intel Corporation
  */
 #include <linux/uuid.h>
 #include "iwl-drv.h"
@@ -79,9 +79,9 @@ static void *iwl_acpi_get_object(struct device *dev, acpi_string method)
  * method (DSM) interface. The returned acpi object must be freed by calling
  * function.
  */
-static void *iwl_acpi_get_dsm_object(struct device *dev, int rev, int func,
-				     union acpi_object *args,
-				     const guid_t *guid)
+union acpi_object *iwl_acpi_get_dsm_object(struct device *dev, int rev,
+					   int func, union acpi_object *args,
+					   const guid_t *guid)
 {
 	union acpi_object *obj;
 
@@ -108,7 +108,7 @@ static int iwl_acpi_get_dsm_integer(struct device *dev, int rev, int func,
 				    size_t expected_size)
 {
 	union acpi_object *obj;
-	int ret = 0;
+	int ret;
 
 	obj = iwl_acpi_get_dsm_object(dev, rev, func, NULL, guid);
 	if (IS_ERR(obj)) {
@@ -123,8 +123,10 @@ static int iwl_acpi_get_dsm_integer(struct device *dev, int rev, int func,
 	} else if (obj->type == ACPI_TYPE_BUFFER) {
 		__le64 le_value = 0;
 
-		if (WARN_ON_ONCE(expected_size > sizeof(le_value)))
-			return -EINVAL;
+		if (WARN_ON_ONCE(expected_size > sizeof(le_value))) {
+			ret = -EINVAL;
+			goto out;
+		}
 
 		/* if the buffer size doesn't match the expected size */
 		if (obj->buffer.length != expected_size)
@@ -145,8 +147,9 @@ static int iwl_acpi_get_dsm_integer(struct device *dev, int rev, int func,
 	}
 
 	IWL_DEBUG_DEV_RADIO(dev,
-			    "ACPI: DSM method evaluated: func=%d, ret=%d\n",
-			    func, ret);
+			    "ACPI: DSM method evaluated: func=%d, value=%lld\n",
+			    func, *value);
+	ret = 0;
 out:
 	ACPI_FREE(obj);
 	return ret;
@@ -166,7 +169,7 @@ int iwl_acpi_get_dsm(struct iwl_fw_runtime *fwrt,
 
 	BUILD_BUG_ON(ARRAY_SIZE(acpi_dsm_size) != DSM_FUNC_NUM_FUNCS);
 
-	if (WARN_ON(func >= ARRAY_SIZE(acpi_dsm_size)))
+	if (WARN_ON(func >= ARRAY_SIZE(acpi_dsm_size) || !func))
 		return -EINVAL;
 
 	expected_size = acpi_dsm_size[func];
@@ -175,6 +178,29 @@ int iwl_acpi_get_dsm(struct iwl_fw_runtime *fwrt,
 	if (expected_size != sizeof(u8) && expected_size != sizeof(u32))
 		return -EOPNOTSUPP;
 
+	if (!fwrt->acpi_dsm_funcs_valid) {
+		ret = iwl_acpi_get_dsm_integer(fwrt->dev, ACPI_DSM_REV,
+					       DSM_FUNC_QUERY,
+					       &iwl_guid, &tmp,
+					       acpi_dsm_size[DSM_FUNC_QUERY]);
+		if (ret) {
+			/* always indicate BIT(0) to avoid re-reading */
+			fwrt->acpi_dsm_funcs_valid = BIT(0);
+			return ret;
+		}
+
+		IWL_DEBUG_RADIO(fwrt, "ACPI DSM validity bitmap 0x%x\n",
+				(u32)tmp);
+		/* always indicate BIT(0) to avoid re-reading */
+		fwrt->acpi_dsm_funcs_valid = tmp | BIT(0);
+	}
+
+	if (!(fwrt->acpi_dsm_funcs_valid & BIT(func))) {
+		IWL_DEBUG_RADIO(fwrt, "ACPI DSM %d not indicated as valid\n",
+				func);
+		return -ENODATA;
+	}
+
 	ret = iwl_acpi_get_dsm_integer(fwrt->dev, ACPI_DSM_REV, func,
 				       &iwl_guid, &tmp, expected_size);
 	if (ret)
@@ -259,13 +285,14 @@ int iwl_acpi_get_tas_table(struct iwl_fw_runtime *fwrt,
 			   struct iwl_tas_data *tas_data)
 {
 	union acpi_object *wifi_pkg, *data;
-	int ret, tbl_rev, i, block_list_size, enabled;
+	int ret, tbl_rev, block_list_size, enabled;
+	u32 tas_selection;
 
 	data = iwl_acpi_get_object(fwrt->dev, ACPI_WTAS_METHOD);
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	/* try to read wtas table revision 1 or revision 0*/
+	/* try to read wtas table */
 	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
 					 ACPI_WTAS_WIFI_DATA_SIZE,
 					 &tbl_rev);
@@ -274,27 +301,23 @@ int iwl_acpi_get_tas_table(struct iwl_fw_runtime *fwrt,
 		goto out_free;
 	}
 
-	if (tbl_rev == 1 && wifi_pkg->package.elements[1].type ==
-		ACPI_TYPE_INTEGER) {
-		u32 tas_selection =
-			(u32)wifi_pkg->package.elements[1].integer.value;
-
-		enabled = iwl_parse_tas_selection(fwrt, tas_data,
-						  tas_selection);
-
-	} else if (tbl_rev == 0 &&
-		wifi_pkg->package.elements[1].type == ACPI_TYPE_INTEGER) {
-		enabled = !!wifi_pkg->package.elements[1].integer.value;
-	} else {
+	if (tbl_rev < 0 || tbl_rev > 2 ||
+	    wifi_pkg->package.elements[1].type != ACPI_TYPE_INTEGER) {
 		ret = -EINVAL;
 		goto out_free;
 	}
 
-	if (!enabled) {
-		IWL_DEBUG_RADIO(fwrt, "TAS not enabled\n");
-		ret = 0;
-		goto out_free;
-	}
+	tas_selection = (u32)wifi_pkg->package.elements[1].integer.value;
+	enabled = tas_selection & IWL_WTAS_ENABLED_MSK;
+
+	IWL_DEBUG_RADIO(fwrt, "TAS selection as read from BIOS: 0x%x\n",
+			tas_selection);
+	tas_data->table_source = BIOS_SOURCE_ACPI;
+	tas_data->table_revision = tbl_rev;
+	tas_data->tas_selection = tas_selection;
+
+	IWL_DEBUG_RADIO(fwrt, "TAS %s enabled\n",
+			enabled ? "is" : "not");
 
 	IWL_DEBUG_RADIO(fwrt, "Reading TAS table revision %d\n", tbl_rev);
 	if (wifi_pkg->package.elements[2].type != ACPI_TYPE_INTEGER ||
@@ -305,13 +328,14 @@ int iwl_acpi_get_tas_table(struct iwl_fw_runtime *fwrt,
 		ret = -EINVAL;
 		goto out_free;
 	}
+
 	block_list_size = wifi_pkg->package.elements[2].integer.value;
-	tas_data->block_list_size = cpu_to_le32(block_list_size);
+	tas_data->block_list_size = block_list_size;
 
 	IWL_DEBUG_RADIO(fwrt, "TAS array size %u\n", block_list_size);
 
-	for (i = 0; i < block_list_size; i++) {
-		u32 country;
+	for (int i = 0; i < block_list_size; i++) {
+		u16 country;
 
 		if (wifi_pkg->package.elements[3 + i].type !=
 		    ACPI_TYPE_INTEGER) {
@@ -322,11 +346,11 @@ int iwl_acpi_get_tas_table(struct iwl_fw_runtime *fwrt,
 		}
 
 		country = wifi_pkg->package.elements[3 + i].integer.value;
-		tas_data->block_list_array[i] = cpu_to_le32(country);
+		tas_data->block_list_array[i] = country;
 		IWL_DEBUG_RADIO(fwrt, "TAS block list country %d\n", country);
 	}
 
-	ret = 1;
+	ret = enabled;
 out_free:
 	kfree(data);
 	return ret;
@@ -357,6 +381,11 @@ int iwl_acpi_get_mcc(struct iwl_fw_runtime *fwrt, char *mcc)
 	}
 
 	mcc_val = wifi_pkg->package.elements[1].integer.value;
+	if (mcc_val != BIOS_MCC_CHINA) {
+		ret = -EINVAL;
+		IWL_DEBUG_RADIO(fwrt, "ACPI WRDD is supported only for CN\n");
+		goto out_free;
+	}
 
 	mcc[0] = (mcc_val >> 8) & 0xff;
 	mcc[1] = mcc_val & 0xff;
@@ -424,38 +453,28 @@ out_free:
 	return ret;
 }
 
-static int iwl_acpi_sar_set_profile(union acpi_object *table,
-				    struct iwl_sar_profile *profile,
-				    bool enabled, u8 num_chains,
-				    u8 num_sub_bands)
+static int
+iwl_acpi_parse_chains_table(union acpi_object *table,
+			    struct iwl_sar_profile_chain *chains,
+			    u8 num_chains, u8 num_sub_bands)
 {
-	int i, j, idx = 0;
-
-	/*
-	 * The table from ACPI is flat, but we store it in a
-	 * structured array.
-	 */
-	for (i = 0; i < BIOS_SAR_MAX_CHAINS_PER_PROFILE; i++) {
-		for (j = 0; j < BIOS_SAR_MAX_SUB_BANDS_NUM; j++) {
+	for (u8 chain = 0; chain < num_chains; chain++) {
+		for (u8 subband = 0; subband < BIOS_SAR_MAX_SUB_BANDS_NUM;
+		     subband++) {
 			/* if we don't have the values, use the default */
-			if (i >= num_chains || j >= num_sub_bands) {
-				profile->chains[i].subbands[j] = 0;
+			if (subband >= num_sub_bands) {
+				chains[chain].subbands[subband] = 0;
+			} else if (table->type != ACPI_TYPE_INTEGER ||
+				   table->integer.value > U8_MAX) {
+				return -EINVAL;
 			} else {
-				if (table[idx].type != ACPI_TYPE_INTEGER ||
-				    table[idx].integer.value > U8_MAX)
-					return -EINVAL;
-
-				profile->chains[i].subbands[j] =
-					table[idx].integer.value;
-
-				idx++;
+				chains[chain].subbands[subband] =
+					table->integer.value;
+				table++;
 			}
 		}
 	}
 
-	/* Only if all values were valid can the profile be enabled */
-	profile->enabled = enabled;
-
 	return 0;
 }
 
@@ -538,9 +557,11 @@ read_table:
 	/* The profile from WRDS is officially profile 1, but goes
 	 * into sar_profiles[0] (because we don't have a profile 0).
 	 */
-	ret = iwl_acpi_sar_set_profile(table, &fwrt->sar_profiles[0],
-				       flags & IWL_SAR_ENABLE_MSK,
-				       num_chains, num_sub_bands);
+	ret = iwl_acpi_parse_chains_table(table, fwrt->sar_profiles[0].chains,
+					  num_chains, num_sub_bands);
+	if (!ret && flags & IWL_SAR_ENABLE_MSK)
+		fwrt->sar_profiles[0].enabled = true;
+
 out_free:
 	kfree(data);
 	return ret;
@@ -552,7 +573,7 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 	bool enabled;
 	int i, n_profiles, tbl_rev, pos;
 	int ret = 0;
-	u8 num_chains, num_sub_bands;
+	u8 num_sub_bands;
 
 	data = iwl_acpi_get_object(fwrt->dev, ACPI_EWRD_METHOD);
 	if (IS_ERR(data))
@@ -568,7 +589,6 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 			goto out_free;
 		}
 
-		num_chains = ACPI_SAR_NUM_CHAINS_REV2;
 		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV2;
 
 		goto read_table;
@@ -584,7 +604,6 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 			goto out_free;
 		}
 
-		num_chains = ACPI_SAR_NUM_CHAINS_REV1;
 		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV1;
 
 		goto read_table;
@@ -600,7 +619,6 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 			goto out_free;
 		}
 
-		num_chains = ACPI_SAR_NUM_CHAINS_REV0;
 		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV0;
 
 		goto read_table;
@@ -632,23 +650,54 @@ read_table:
 	/* the tables start at element 3 */
 	pos = 3;
 
+	BUILD_BUG_ON(ACPI_SAR_NUM_CHAINS_REV0 != ACPI_SAR_NUM_CHAINS_REV1);
+	BUILD_BUG_ON(ACPI_SAR_NUM_CHAINS_REV2 != 2 * ACPI_SAR_NUM_CHAINS_REV0);
+
+	/* parse non-cdb chains for all profiles */
 	for (i = 0; i < n_profiles; i++) {
 		union acpi_object *table = &wifi_pkg->package.elements[pos];
+
 		/* The EWRD profiles officially go from 2 to 4, but we
 		 * save them in sar_profiles[1-3] (because we don't
 		 * have profile 0).  So in the array we start from 1.
 		 */
-		ret = iwl_acpi_sar_set_profile(table,
-					       &fwrt->sar_profiles[i + 1],
-					       enabled, num_chains,
-					       num_sub_bands);
+		ret = iwl_acpi_parse_chains_table(table,
+						  fwrt->sar_profiles[i + 1].chains,
+						  ACPI_SAR_NUM_CHAINS_REV0,
+						  num_sub_bands);
 		if (ret < 0)
-			break;
+			goto out_free;
 
 		/* go to the next table */
-		pos += num_chains * num_sub_bands;
+		pos += ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands;
 	}
 
+	/* non-cdb table revisions */
+	if (tbl_rev < 2)
+		goto set_enabled;
+
+	/* parse cdb chains for all profiles */
+	for (i = 0; i < n_profiles; i++) {
+		struct iwl_sar_profile_chain *chains;
+		union acpi_object *table;
+
+		table = &wifi_pkg->package.elements[pos];
+		chains = &fwrt->sar_profiles[i + 1].chains[ACPI_SAR_NUM_CHAINS_REV0];
+		ret = iwl_acpi_parse_chains_table(table,
+						  chains,
+						  ACPI_SAR_NUM_CHAINS_REV0,
+						  num_sub_bands);
+		if (ret < 0)
+			goto out_free;
+
+		/* go to the next table */
+		pos += ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands;
+	}
+
+set_enabled:
+	for (i = 0; i < n_profiles; i++)
+		fwrt->sar_profiles[i + 1].enabled = enabled;
+
 out_free:
 	kfree(data);
 	return ret;
@@ -821,12 +870,12 @@ int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	/* try to read ppag table rev 3, 2 or 1 (all have the same data size) */
+	/* try to read ppag table rev 1 to 4 (all have the same data size) */
 	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
 				ACPI_PPAG_WIFI_DATA_SIZE_V2, &tbl_rev);
 
 	if (!IS_ERR(wifi_pkg)) {
-		if (tbl_rev >= 1 && tbl_rev <= 3) {
+		if (tbl_rev >= 1 && tbl_rev <= 4) {
 			num_sub_bands = IWL_NUM_SUB_BANDS_V2;
 			IWL_DEBUG_RADIO(fwrt,
 					"Reading PPAG table (tbl_rev=%d)\n",
@@ -856,7 +905,7 @@ int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	goto out_free;
 
 read_table:
-	fwrt->ppag_ver = tbl_rev;
+	fwrt->ppag_bios_rev = tbl_rev;
 	flags = &wifi_pkg->package.elements[1];
 
 	if (flags->type != ACPI_TYPE_INTEGER) {
@@ -865,7 +914,7 @@ read_table:
 	}
 
 	fwrt->ppag_flags = iwl_bios_get_ppag_flags(flags->integer.value,
-						   fwrt->ppag_ver);
+						   fwrt->ppag_bios_rev);
 
 	/*
 	 * read, verify gain values and save them into the PPAG table.
@@ -886,6 +935,7 @@ read_table:
 		}
 	}
 
+	fwrt->ppag_bios_source = BIOS_SOURCE_ACPI;
 	ret = 0;
 
 out_free:
@@ -893,40 +943,39 @@ out_free:
 	return ret;
 }
 
-void iwl_acpi_get_phy_filters(struct iwl_fw_runtime *fwrt,
-			      struct iwl_phy_specific_cfg *filters)
+int iwl_acpi_get_phy_filters(struct iwl_fw_runtime *fwrt)
 {
+	struct iwl_phy_specific_cfg *filters = &fwrt->phy_filters;
 	struct iwl_phy_specific_cfg tmp = {};
-	union acpi_object *wifi_pkg, *data;
+	union acpi_object *wifi_pkg, *data __free(kfree);
 	int tbl_rev, i;
 
 	data = iwl_acpi_get_object(fwrt->dev, ACPI_WPFC_METHOD);
 	if (IS_ERR(data))
-		return;
+		return PTR_ERR(data);
 
 	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
 					 ACPI_WPFC_WIFI_DATA_SIZE,
 					 &tbl_rev);
 	if (IS_ERR(wifi_pkg))
-		goto out_free;
+		return PTR_ERR(wifi_pkg);
 
 	if (tbl_rev != 0)
-		goto out_free;
+		return -EINVAL;
 
 	BUILD_BUG_ON(ARRAY_SIZE(filters->filter_cfg_chains) !=
 		     ACPI_WPFC_WIFI_DATA_SIZE - 1);
 
 	for (i = 0; i < ARRAY_SIZE(filters->filter_cfg_chains); i++) {
 		if (wifi_pkg->package.elements[i + 1].type != ACPI_TYPE_INTEGER)
-			goto out_free;
+			return -EINVAL;
 		tmp.filter_cfg_chains[i] =
 			cpu_to_le32(wifi_pkg->package.elements[i + 1].integer.value);
 	}
 
 	IWL_DEBUG_RADIO(fwrt, "Loaded WPFC filter config from ACPI\n");
 	*filters = tmp;
-out_free:
-	kfree(data);
+	return 0;
 }
 IWL_EXPORT_SYMBOL(iwl_acpi_get_phy_filters);
 
@@ -998,3 +1047,37 @@ out_free:
 	kfree(data);
 	return ret;
 }
+
+int iwl_acpi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value)
+{
+	union acpi_object *wifi_pkg, *data;
+	int ret = -ENOENT;
+	int tbl_rev;
+
+	data = iwl_acpi_get_object(fwrt->dev, ACPI_DSBR_METHOD);
+	if (IS_ERR(data))
+		return ret;
+
+	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+					 ACPI_DSBR_WIFI_DATA_SIZE,
+					 &tbl_rev);
+	if (IS_ERR(wifi_pkg))
+		goto out_free;
+
+	if (tbl_rev != ACPI_DSBR_WIFI_DATA_REV) {
+		IWL_DEBUG_RADIO(fwrt, "Unsupported ACPI DSBR revision:%d\n",
+				tbl_rev);
+		goto out_free;
+	}
+
+	if (wifi_pkg->package.elements[1].type != ACPI_TYPE_INTEGER)
+		goto out_free;
+
+	*value = wifi_pkg->package.elements[1].integer.value;
+	IWL_DEBUG_RADIO(fwrt, "Loaded DSBR config from ACPI value: 0x%x\n",
+			*value);
+	ret = 0;
+out_free:
+	kfree(data);
+	return ret;
+}
diff --git a/sys/contrib/dev/iwlwifi/fw/acpi.h b/sys/contrib/dev/iwlwifi/fw/acpi.h
index bb88398a6987..68d8fb5f6357 100644
--- a/sys/contrib/dev/iwlwifi/fw/acpi.h
+++ b/sys/contrib/dev/iwlwifi/fw/acpi.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2023 Intel Corporation
+ * Copyright (C) 2018-2023, 2025 Intel Corporation
  */
 #ifndef __iwl_fw_acpi__
 #define __iwl_fw_acpi__
@@ -28,6 +28,7 @@
 #define ACPI_WPFC_METHOD	"WPFC"
 #define ACPI_GLAI_METHOD	"GLAI"
 #define ACPI_WBEM_METHOD	"WBEM"
+#define ACPI_DSBR_METHOD	"DSBR"
 
 #define ACPI_WIFI_DOMAIN	(0x07)
 
@@ -76,6 +77,13 @@
 #define ACPI_WBEM_WIFI_DATA_SIZE	2
 /*
  * One element for domain type,
+ * and one for DSBR response data
+ */
+#define ACPI_DSBR_WIFI_DATA_SIZE	2
+#define ACPI_DSBR_WIFI_DATA_REV		1
+
+/*
+ * One element for domain type,
  * and one for the status
  */
 #define ACPI_GLAI_WIFI_DATA_SIZE	2
@@ -101,6 +109,30 @@
 
 #define ACPI_DSM_REV 0
 
+#define DSM_INTERNAL_FUNC_GET_PLAT_INFO	1
+/* TBD: VPRO is BIT(0) in the result, but what's the result? */
+
+#define DSM_INTERNAL_FUNC_PRODUCT_RESET	2
+
+/* DSM_INTERNAL_FUNC_PRODUCT_RESET - product reset (aka "PLDR") */
+enum iwl_dsm_internal_product_reset_cmds {
+	DSM_INTERNAL_PLDR_CMD_GET_MODE = 1,
+	DSM_INTERNAL_PLDR_CMD_SET_MODE = 2,
+	DSM_INTERNAL_PLDR_CMD_GET_STATUS = 3,
+};
+
+enum iwl_dsm_internal_product_reset_mode {
+	DSM_INTERNAL_PLDR_MODE_EN_PROD_RESET	= BIT(0),
+	DSM_INTERNAL_PLDR_MODE_EN_WIFI_FLR	= BIT(1),
+	DSM_INTERNAL_PLDR_MODE_EN_BT_OFF_ON	= BIT(2),
+};
+
+struct iwl_dsm_internal_product_reset_cmd {
+	/* cmd is from enum iwl_dsm_internal_product_reset_cmds */
+	u16 cmd;
+	u16 value;
+} __packed;
+
 #define IWL_ACPI_WBEM_REV0_MASK (BIT(0) | BIT(1))
 #define IWL_ACPI_WBEM_REVISION 0
 
@@ -110,6 +142,10 @@ struct iwl_fw_runtime;
 
 extern const guid_t iwl_guid;
 
+union acpi_object *iwl_acpi_get_dsm_object(struct device *dev, int rev,
+					   int func, union acpi_object *args,
+					   const guid_t *guid);
+
 /**
  * iwl_acpi_get_mcc - read MCC from ACPI, if available
  *
@@ -144,8 +180,7 @@ int iwl_acpi_get_tas_table(struct iwl_fw_runtime *fwrt,
 
 int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt);
 
-void iwl_acpi_get_phy_filters(struct iwl_fw_runtime *fwrt,
-			      struct iwl_phy_specific_cfg *filters);
+int iwl_acpi_get_phy_filters(struct iwl_fw_runtime *fwrt);
 
 void iwl_acpi_get_guid_lock_status(struct iwl_fw_runtime *fwrt);
 
@@ -153,10 +188,14 @@ int iwl_acpi_get_dsm(struct iwl_fw_runtime *fwrt,
 		     enum iwl_dsm_funcs func, u32 *value);
 
 int iwl_acpi_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value);
+
+int iwl_acpi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value);
+
 #else /* CONFIG_ACPI */
 
-static inline void *iwl_acpi_get_dsm_object(struct device *dev, int rev,
-					    int func, union acpi_object *args)
+static inline union acpi_object *
+iwl_acpi_get_dsm_object(struct device *dev, int rev, int func,
+			union acpi_object *args, const guid_t *guid)
 {
 	return ERR_PTR(-ENOENT);
 }
@@ -204,8 +243,10 @@ static inline int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	return -ENOENT;
 }
 
-/* macro since the second argument doesn't always exist */
-#define iwl_acpi_get_phy_filters(fwrt, filters) do { } while (0)
+static inline int iwl_acpi_get_phy_filters(struct iwl_fw_runtime *fwrt)
+{
+	return -ENOENT;
+}
 
 static inline void iwl_acpi_get_guid_lock_status(struct iwl_fw_runtime *fwrt)
 {
@@ -221,6 +262,11 @@ static inline int iwl_acpi_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value)
 {
 	return -ENOENT;
 }
+
+static inline int iwl_acpi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value)
+{
+	return -ENOENT;
+}
 #endif /* CONFIG_ACPI */
 
 #endif /* __iwl_fw_acpi__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/alive.h b/sys/contrib/dev/iwlwifi/fw/api/alive.h
index ebe85fdf08d3..ad5b95cad0bf 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/alive.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/alive.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018, 2020-2021, 2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018, 2020-2021, 2024-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -82,21 +82,6 @@ struct iwl_alive_ntf_v3 {
 	struct iwl_umac_alive umac_data;
 } __packed; /* UCODE_ALIVE_NTFY_API_S_VER_3 */
 
-struct iwl_alive_ntf_v4 {
-	__le16 status;
-	__le16 flags;
-	struct iwl_lmac_alive lmac_data[2];
-	struct iwl_umac_alive umac_data;
-} __packed; /* UCODE_ALIVE_NTFY_API_S_VER_4 */
-
-struct iwl_alive_ntf_v5 {
-	__le16 status;
-	__le16 flags;
-	struct iwl_lmac_alive lmac_data[2];
-	struct iwl_umac_alive umac_data;
-	struct iwl_sku_id sku_id;
-} __packed; /* UCODE_ALIVE_NTFY_API_S_VER_5 */
-
 struct iwl_imr_alive_info {
 	__le64 base_addr;
 	__le32 size;
@@ -112,12 +97,22 @@ struct iwl_alive_ntf_v6 {
 	struct iwl_imr_alive_info imr;
 } __packed; /* UCODE_ALIVE_NTFY_API_S_VER_6 */
 
+struct iwl_alive_ntf {
+	__le16 status;
+	__le16 flags;
+	struct iwl_lmac_alive lmac_data[2];
+	struct iwl_umac_alive umac_data;
+	struct iwl_sku_id sku_id;
+	struct iwl_imr_alive_info imr;
+	__le64 platform_id;
+} __packed; /* UCODE_ALIVE_NTFY_API_S_VER_8 */
+
 /**
  * enum iwl_extended_cfg_flags - commands driver may send before
  *	finishing init flow
  * @IWL_INIT_DEBUG_CFG: driver is going to send debug config command
  * @IWL_INIT_NVM: driver is going to send NVM_ACCESS commands
- * @IWL_INIT_PHY: driver is going to send PHY_DB commands
+ * @IWL_INIT_PHY: driver is going to send the PHY_CONFIGURATION_CMD
  */
 enum iwl_extended_cfg_flags {
 	IWL_INIT_DEBUG_CFG,
diff --git a/sys/contrib/dev/iwlwifi/fw/api/binding.h b/sys/contrib/dev/iwlwifi/fw/api/binding.h
index 2397fdc37fc5..9b942c4aabd9 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/binding.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/binding.h
@@ -56,8 +56,6 @@ struct iwl_binding_cmd {
 } __packed; /* BINDING_CMD_API_S_VER_2 */
 
 #define IWL_BINDING_CMD_SIZE_V1	sizeof(struct iwl_binding_cmd_v1)
-#define IWL_LMAC_24G_INDEX		0
-#define IWL_LMAC_5G_INDEX		1
 
 /* The maximal number of fragments in the FW's schedule session */
 #define IWL_MVM_MAX_QUOTA 128
diff --git a/sys/contrib/dev/iwlwifi/fw/api/coex.h b/sys/contrib/dev/iwlwifi/fw/api/coex.h
index b97a43353779..ddc84430d895 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/coex.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/coex.h
@@ -95,7 +95,7 @@ enum iwl_bt_ci_compliance {
 }; /* BT_COEX_CI_COMPLIENCE_E_VER_1 */
 
 /**
- * struct iwl_bt_coex_profile_notif - notification about BT coex
+ * struct iwl_bt_coex_prof_old_notif - notification about BT coex
  * @mbox_msg: message from BT to WiFi
  * @msg_idx: the index of the message
  * @bt_ci_compliance: enum %iwl_bt_ci_compliance
@@ -110,7 +110,7 @@ enum iwl_bt_ci_compliance {
  * @wifi_loss_mid_high_rssi: The predicted lost WiFi rate (% of air time that
  *	BT is utilizing) when the RSSI is mid/high (>= -65 dBm)
  */
-struct iwl_bt_coex_profile_notif {
+struct iwl_bt_coex_prof_old_notif {
 	__le32 mbox_msg[4];
 	__le32 msg_idx;
 	__le32 bt_ci_compliance;
@@ -126,4 +126,29 @@ struct iwl_bt_coex_profile_notif {
 	     * BT_COEX_PROFILE_NTFY_API_S_VER_5
 	     */
 
+/**
+ * enum iwl_bt_coex_subcmd_ids - coex configuration command IDs
+ */
+enum iwl_bt_coex_subcmd_ids {
+       /**
+	*@PROFILE_NOTIF: &struct iwl_bt_coex_profile_notif
+	*/
+	PROFILE_NOTIF = 0xFF,
+};
+
+#define COEX_NUM_BAND	3
+#define COEX_NUM_CHAINS	2
+
+/**
+ * struct iwl_bt_coex_profile_notif - notification about BT coex
+ * @wifi_loss_low_rssi: The predicted lost WiFi rate (% of air time that BT is
+ *	utilizing) when the RSSI is low (<= -65 dBm)
+ * @wifi_loss_mid_high_rssi: The predicted lost WiFi rate (% of air time that
+ *	BT is utilizing) when the RSSI is mid/high (>= -65 dBm)
+ */
+struct iwl_bt_coex_profile_notif {
+	u8 wifi_loss_low_rssi[COEX_NUM_BAND][COEX_NUM_CHAINS];
+	u8 wifi_loss_mid_high_rssi[COEX_NUM_BAND][COEX_NUM_CHAINS];
+} __packed; /* BT_COEX_BT_PROFILE_NTF_API_S_VER_1 */
+
 #endif /* __iwl_fw_api_coex_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/commands.h b/sys/contrib/dev/iwlwifi/fw/api/commands.h
index 7544c4cb1a30..997b0c9ce984 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/commands.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/commands.h
@@ -2,7 +2,7 @@
 /*
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2022 Intel Corporation
+ * Copyright (C) 2018-2022, 2024-2025 Intel Corporation
  */
 #ifndef __iwl_fw_api_commands_h__
 #define __iwl_fw_api_commands_h__
@@ -25,6 +25,8 @@
  * @NAN_GROUP: NAN group, uses command IDs from &enum iwl_nan_subcmd_ids
  * @LOCATION_GROUP: location group, uses command IDs from
  *	&enum iwl_location_subcmd_ids
+ * @BT_COEX_GROUP: bt coex group, uses command IDs from
+ *	&enum iwl_bt_coex_subcmd_ids
  * @PROT_OFFLOAD_GROUP: protocol offload group, uses command IDs from
  *	&enum iwl_prot_offload_subcmd_ids
  * @REGULATORY_AND_NVM_GROUP: regulatory/NVM group, uses command IDs from
@@ -43,6 +45,7 @@ enum iwl_mvm_command_groups {
 	SCAN_GROUP = 0x6,
 	NAN_GROUP = 0x7,
 	LOCATION_GROUP = 0x8,
+	BT_COEX_GROUP = 0x9,
 	PROT_OFFLOAD_GROUP = 0xb,
 	REGULATORY_AND_NVM_GROUP = 0xc,
 	DEBUG_GROUP = 0xf,
@@ -142,10 +145,10 @@ enum iwl_legacy_cmds {
 	REMOVE_STA = 0x19,
 
 	/**
-	 * @TX_CMD: uses &struct iwl_tx_cmd or &struct iwl_tx_cmd_gen2 or
-	 *	&struct iwl_tx_cmd_gen3,
-	 *	response in &struct iwl_mvm_tx_resp or
-	 *	&struct iwl_mvm_tx_resp_v3
+	 * @TX_CMD: uses &struct iwl_tx_cmd_v6 or &struct iwl_tx_cmd_v9 or
+	 *	&struct iwl_tx_cmd,
+	 *	response in &struct iwl_tx_resp or
+	 *	&struct iwl_tx_resp_v3
 	 */
 	TX_CMD = 0x1c,
 
@@ -398,7 +401,7 @@ enum iwl_legacy_cmds {
 	REDUCE_TX_POWER_CMD = 0x9f,
 
 	/**
-	 * @MISSED_BEACONS_NOTIFICATION: &struct iwl_missed_beacons_notif
+	 * @MISSED_BEACONS_NOTIFICATION: &struct iwl_missed_beacons_notif_v4
 	 */
 	MISSED_BEACONS_NOTIFICATION = 0xa2,
 
@@ -444,7 +447,7 @@ enum iwl_legacy_cmds {
 
 	/**
 	 * @BA_NOTIF:
-	 * BlockAck notification, uses &struct iwl_mvm_compressed_ba_notif
+	 * BlockAck notification, uses &struct iwl_compressed_ba_notif
 	 * or &struct iwl_mvm_ba_notif depending on the HW
 	 */
 	BA_NOTIF = 0xc5,
@@ -467,7 +470,7 @@ enum iwl_legacy_cmds {
 	MARKER_CMD = 0xcb,
 
 	/**
-	 * @BT_PROFILE_NOTIFICATION: &struct iwl_bt_coex_profile_notif
+	 * @BT_PROFILE_NOTIFICATION: &struct iwl_bt_coex_prof_old_notif
 	 */
 	BT_PROFILE_NOTIFICATION = 0xce,
 
@@ -499,11 +502,16 @@ enum iwl_legacy_cmds {
 	/**
 	 * @DTS_MEASUREMENT_NOTIFICATION:
 	 * &struct iwl_dts_measurement_notif_v1 or
-	 * &struct iwl_dts_measurement_notif_v2
+	 * &struct iwl_dts_measurement_notif
 	 */
 	DTS_MEASUREMENT_NOTIFICATION = 0xdd,
 
 	/**
+	 * @DEBUG_HOST_COMMAND: &struct iwl_dhc_cmd
+	 */
+	DEBUG_HOST_COMMAND = 0xf1,
+
+	/**
 	 * @LDBG_CONFIG_CMD: configure continuous trace recording
 	 */
 	LDBG_CONFIG_CMD = 0xf6,
@@ -565,9 +573,8 @@ enum iwl_legacy_cmds {
 	WOWLAN_KEK_KCK_MATERIAL = 0xe4,
 
 	/**
-	 * @WOWLAN_GET_STATUSES: response in &struct iwl_wowlan_status_v6,
-	 *	&struct iwl_wowlan_status_v7, &struct iwl_wowlan_status_v9 or
-	 *	&struct iwl_wowlan_status_v12
+	 * @WOWLAN_GET_STATUSES: response in &struct iwl_wowlan_status_v6 or
+	 *	&struct iwl_wowlan_status_v7
 	 */
 	WOWLAN_GET_STATUSES = 0xe5,
 
diff --git a/sys/contrib/dev/iwlwifi/fw/api/context.h b/sys/contrib/dev/iwlwifi/fw/api/context.h
index 1fa5678c1cd6..464eed9b5e71 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/context.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/context.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2022 Intel Corporation
+ * Copyright (C) 2012-2014, 2022, 2024 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -14,6 +14,9 @@
  * @FW_CTXT_COLOR_POS: position of the color
  * @FW_CTXT_COLOR_MSK: mask of the color
  * @FW_CTXT_INVALID: value used to indicate unused/invalid
+ * @FW_CTXT_ID_INVALID: value used to indicate unused/invalid. This can be
+ *	used with newer firmware which no longer use the color. Typically,
+ *	firmware versions supported by iwlmld can use this value.
  */
 enum iwl_ctxt_id_and_color {
 	FW_CTXT_ID_POS		= 0,
@@ -21,6 +24,7 @@ enum iwl_ctxt_id_and_color {
 	FW_CTXT_COLOR_POS	= 8,
 	FW_CTXT_COLOR_MSK	= 0xff << FW_CTXT_COLOR_POS,
 	FW_CTXT_INVALID		= 0xffffffff,
+	FW_CTXT_ID_INVALID	= 0xff,
 };
 
 #define FW_CMD_ID_AND_COLOR(_id, _color) (((_id) << FW_CTXT_ID_POS) |\
@@ -40,4 +44,7 @@ enum iwl_ctxt_action {
 	FW_CTXT_ACTION_REMOVE,
 }; /* COMMON_CONTEXT_ACTION_API_E_VER_1 */
 
+#define IWL_LMAC_24G_INDEX		0
+#define IWL_LMAC_5G_INDEX		1
+
 #endif /* __iwl_fw_api_context_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/d3.h b/sys/contrib/dev/iwlwifi/fw/api/d3.h
index ffee7927cf26..53445087e9cb 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/d3.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/d3.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -19,9 +19,11 @@ enum iwl_d0i3_flags {
 /**
  * enum iwl_d3_wakeup_flags - D3 manager wakeup flags
  * @IWL_WAKEUP_D3_CONFIG_FW_ERROR: wake up on firmware sysassert
+ * @IWL_WAKEUP_D3_HOST_TIMER: wake up on host timer expiry
  */
 enum iwl_d3_wakeup_flags {
-	IWL_WAKEUP_D3_CONFIG_FW_ERROR = BIT(0),
+	IWL_WAKEUP_D3_CONFIG_FW_ERROR	= BIT(0),
+	IWL_WAKEUP_D3_HOST_TIMER	= BIT(1),
 }; /* D3_MANAGER_WAKEUP_CONFIG_API_E_VER_3 */
 
 /**
@@ -368,7 +370,7 @@ enum iwl_wowlan_flags {
 };
 
 /**
- * struct iwl_wowlan_config_cmd - WoWLAN configuration (versions 5 and 6)
+ * struct iwl_wowlan_config_cmd_v6 - WoWLAN configuration (versions 5 and 6)
  * @wakeup_filter: filter from &enum iwl_wowlan_wakeup_filters
  * @non_qos_seq: non-QoS sequence counter to use next.
  *               Reserved if the struct has version >= 6.
@@ -380,7 +382,7 @@ enum iwl_wowlan_flags {
  * @sta_id: station ID for wowlan.
  * @reserved: reserved
  */
-struct iwl_wowlan_config_cmd {
+struct iwl_wowlan_config_cmd_v6 {
 	__le32 wakeup_filter;
 	__le16 non_qos_seq;
 	__le16 qos_seq[8];
@@ -390,7 +392,27 @@ struct iwl_wowlan_config_cmd {
 	u8 flags;
 	u8 sta_id;
 	u8 reserved;
-} __packed; /* WOWLAN_CONFIG_API_S_VER_5 */
+} __packed; /* WOWLAN_CONFIG_API_S_VER_6 */
+
+/**
+ * struct iwl_wowlan_config_cmd - WoWLAN configuration
+ * @wakeup_filter: filter from &enum iwl_wowlan_wakeup_filters
+ * @wowlan_ba_teardown_tids: bitmap of BA sessions to tear down
+ * @is_11n_connection: indicates HT connection
+ * @offloading_tid: TID reserved for firmware use
+ * @flags: extra flags, see &enum iwl_wowlan_flags
+ * @sta_id: station ID for wowlan.
+ * @reserved: reserved
+ */
+struct iwl_wowlan_config_cmd {
+	__le32 wakeup_filter;
+	u8 wowlan_ba_teardown_tids;
+	u8 is_11n_connection;
+	u8 offloading_tid;
+	u8 flags;
+	u8 sta_id;
+	u8 reserved[3];
+} __packed; /* WOWLAN_CONFIG_API_S_VER_7 */
 
 #define IWL_NUM_RSC	16
 #define WOWLAN_KEY_MAX_SIZE	32
@@ -434,11 +456,6 @@ struct iwl_wowlan_rsc_tsc_params_cmd_ver_2 {
 	union iwl_all_tsc_rsc all_tsc_rsc;
 } __packed; /* ALL_TSC_RSC_API_S_VER_2 */
 
-struct iwl_wowlan_rsc_tsc_params_cmd_v4 {
-	struct iwl_wowlan_rsc_tsc_params_cmd_ver_2 params;
-	__le32 sta_id;
-} __packed; /* ALL_TSC_RSC_API_S_VER_4 */
-
 struct iwl_wowlan_rsc_tsc_params_cmd {
 	__le64 ucast_rsc[IWL_MAX_TID_COUNT];
 	__le64 mcast_rsc[WOWLAN_GTK_KEYS_NUM][IWL_MAX_TID_COUNT];
@@ -698,82 +715,6 @@ struct iwl_wowlan_status_v7 {
 } __packed; /* WOWLAN_STATUSES_API_S_VER_7 */
 
 /**
- * struct iwl_wowlan_status_v9 - WoWLAN status (versions 9 and 10)
- * @gtk: GTK data
- * @igtk: IGTK data
- * @replay_ctr: GTK rekey replay counter
- * @pattern_number: number of the matched pattern
- * @non_qos_seq_ctr: non-QoS sequence counter to use next.
- *                   Reserved if the struct has version >= 10.
- * @qos_seq_ctr: QoS sequence counters to use next
- * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason
- * @num_of_gtk_rekeys: number of GTK rekeys
- * @transmitted_ndps: number of transmitted neighbor discovery packets
- * @received_beacons: number of received beacons
- * @wake_packet_length: wakeup packet length
- * @wake_packet_bufsize: wakeup packet buffer size
- * @tid_tear_down: bit mask of tids whose BA sessions were closed
- *		   in suspend state
- * @reserved: unused
- * @wake_packet: wakeup packet
- */
-struct iwl_wowlan_status_v9 {
-	struct iwl_wowlan_gtk_status_v2 gtk[WOWLAN_GTK_KEYS_NUM];
-	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
-	__le64 replay_ctr;
-	__le16 pattern_number;
-	__le16 non_qos_seq_ctr;
-	__le16 qos_seq_ctr[8];
-	__le32 wakeup_reasons;
-	__le32 num_of_gtk_rekeys;
-	__le32 transmitted_ndps;
-	__le32 received_beacons;
-	__le32 wake_packet_length;
-	__le32 wake_packet_bufsize;
-	u8 tid_tear_down;
-	u8 reserved[3];
-	u8 wake_packet[]; /* can be truncated from _length to _bufsize */
-} __packed; /* WOWLAN_STATUSES_RSP_API_S_VER_9 */
-
-/**
- * struct iwl_wowlan_status_v12 - WoWLAN status
- * @gtk: GTK data
- * @igtk: IGTK data
- * @replay_ctr: GTK rekey replay counter
- * @pattern_number: number of the matched pattern
- * @non_qos_seq_ctr: non-QoS sequence counter to use next.
- *                   Reserved if the struct has version >= 10.
- * @qos_seq_ctr: QoS sequence counters to use next
- * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason
- * @num_of_gtk_rekeys: number of GTK rekeys
- * @transmitted_ndps: number of transmitted neighbor discovery packets
- * @received_beacons: number of received beacons
- * @wake_packet_length: wakeup packet length
- * @wake_packet_bufsize: wakeup packet buffer size
- * @tid_tear_down: bit mask of tids whose BA sessions were closed
- *		   in suspend state
- * @reserved: unused
- * @wake_packet: wakeup packet
- */
-struct iwl_wowlan_status_v12 {
-	struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM];
-	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
-	__le64 replay_ctr;
-	__le16 pattern_number;
-	__le16 non_qos_seq_ctr;
-	__le16 qos_seq_ctr[8];
-	__le32 wakeup_reasons;
-	__le32 num_of_gtk_rekeys;
-	__le32 transmitted_ndps;
-	__le32 received_beacons;
-	__le32 wake_packet_length;
-	__le32 wake_packet_bufsize;
-	u8 tid_tear_down;
-	u8 reserved[3];
-	u8 wake_packet[]; /* can be truncated from _length to _bufsize */
-} __packed; /* WOWLAN_STATUSES_RSP_API_S_VER_12 */
-
-/**
  * struct iwl_wowlan_info_notif_v1 - WoWLAN information notification
  * @gtk: GTK data
  * @igtk: IGTK data
@@ -810,39 +751,6 @@ struct iwl_wowlan_info_notif_v1 {
 	u8 reserved2[2];
 } __packed; /* WOWLAN_INFO_NTFY_API_S_VER_1 */
 
-/**
- * struct iwl_wowlan_info_notif_v2 - WoWLAN information notification
- * @gtk: GTK data
- * @igtk: IGTK data
- * @replay_ctr: GTK rekey replay counter
- * @pattern_number: number of the matched patterns
- * @reserved1: reserved
- * @qos_seq_ctr: QoS sequence counters to use next
- * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason
- * @num_of_gtk_rekeys: number of GTK rekeys
- * @transmitted_ndps: number of transmitted neighbor discovery packets
- * @received_beacons: number of received beacons
- * @tid_tear_down: bit mask of tids whose BA sessions were closed
- *	in suspend state
- * @station_id: station id
- * @reserved2: reserved
- */
-struct iwl_wowlan_info_notif_v2 {
-	struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM];
-	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
-	__le64 replay_ctr;
-	__le16 pattern_number;
-	__le16 reserved1;
-	__le16 qos_seq_ctr[8];
-	__le32 wakeup_reasons;
-	__le32 num_of_gtk_rekeys;
-	__le32 transmitted_ndps;
-	__le32 received_beacons;
-	u8 tid_tear_down;
-	u8 station_id;
-	u8 reserved2[2];
-} __packed; /* WOWLAN_INFO_NTFY_API_S_VER_2 */
-
 /* MAX MLO keys of non-active links that can arrive in the notification */
 #define WOWLAN_MAX_MLO_KEYS 18
 
@@ -890,7 +798,7 @@ struct iwl_wowlan_mlo_gtk {
 } __packed; /* WOWLAN_MLO_GTK_KEY_API_S_VER_1 */
 
 /**
- * struct iwl_wowlan_info_notif - WoWLAN information notification
+ * struct iwl_wowlan_info_notif_v3 - WoWLAN information notification
  * @gtk: GTK data
  * @igtk: IGTK data
  * @bigtk: BIGTK data
@@ -905,12 +813,9 @@ struct iwl_wowlan_mlo_gtk {
  * @tid_tear_down: bit mask of tids whose BA sessions were closed
  *	in suspend state
  * @station_id: station id
- * @num_mlo_link_keys: number of &struct iwl_wowlan_mlo_gtk structs
- *	following this notif, or reserved in version < 4
  * @reserved2: reserved
- * @mlo_gtks: array of GTKs of size num_mlo_link_keys for version >= 4
  */
-struct iwl_wowlan_info_notif {
+struct iwl_wowlan_info_notif_v3 {
 	struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM];
 	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
 	struct iwl_wowlan_igtk_status bigtk[WOWLAN_BIGTK_KEYS_NUM];
@@ -924,10 +829,47 @@ struct iwl_wowlan_info_notif {
 	__le32 received_beacons;
 	u8 tid_tear_down;
 	u8 station_id;
+	u8 reserved2[2];
+} __packed; /* WOWLAN_INFO_NTFY_API_S_VER_3 */
+
+/**
+ * struct iwl_wowlan_info_notif - WoWLAN information notification
+ * @gtk: GTK data
+ * @igtk: IGTK data
+ * @bigtk: BIGTK data
+ * @replay_ctr: GTK rekey replay counter
+ * @pattern_number: number of the matched patterns
+ * @qos_seq_ctr: QoS sequence counters to use next
+ * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason
+ * @num_of_gtk_rekeys: number of GTK rekeys
+ * @transmitted_ndps: number of transmitted neighbor discovery packets
+ * @received_beacons: number of received beacons
+ * @tid_tear_down: bit mask of tids whose BA sessions were closed
+ *	in suspend state
+ * @station_id: station id
+ * @num_mlo_link_keys: number of &struct iwl_wowlan_mlo_gtk structs
+ *	following this notif
+ * @tid_offloaded_tx: tid used by the firmware to transmit data packets
+ *	while in wowlan
+ * @mlo_gtks: array of GTKs of size num_mlo_link_keys
+ */
+struct iwl_wowlan_info_notif {
+	struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM];
+	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
+	struct iwl_wowlan_igtk_status bigtk[WOWLAN_BIGTK_KEYS_NUM];
+	__le64 replay_ctr;
+	__le16 pattern_number;
+	__le16 qos_seq_ctr;
+	__le32 wakeup_reasons;
+	__le32 num_of_gtk_rekeys;
+	__le32 transmitted_ndps;
+	__le32 received_beacons;
+	u8 tid_tear_down;
+	u8 station_id;
 	u8 num_mlo_link_keys;
-	u8 reserved2;
+	u8 tid_offloaded_tx;
 	struct iwl_wowlan_mlo_gtk mlo_gtks[];
-} __packed; /* WOWLAN_INFO_NTFY_API_S_VER_3, _VER_4 */
+} __packed; /* WOWLAN_INFO_NTFY_API_S_VER_5 */
 
 /**
  * struct iwl_wowlan_wake_pkt_notif - WoWLAN wake packet notification
@@ -947,7 +889,7 @@ struct iwl_wowlan_wake_pkt_notif {
  * struct iwl_mvm_d3_end_notif -  d3 end notification
  * @flags: See &enum iwl_d0i3_flags
  */
-struct iwl_mvm_d3_end_notif {
+struct iwl_d3_end_notif {
 	__le32 flags;
 } __packed;
 
diff --git a/sys/contrib/dev/iwlwifi/fw/api/datapath.h b/sys/contrib/dev/iwlwifi/fw/api/datapath.h
index 2ab38eaeb290..b1c6ee8ae2df 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/datapath.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/datapath.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024-2025 Intel Corporation
  * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
@@ -92,7 +92,7 @@ enum iwl_data_path_subcmd_ids {
 
 	/**
 	 * @ESR_MODE_NOTIF: notification to recommend/force a wanted esr mode,
-	 *	uses &struct iwl_mvm_esr_mode_notif
+	 *	uses &struct iwl_esr_mode_notif or &struct iwl_esr_mode_notif_v1
 	 */
 	ESR_MODE_NOTIF = 0xF3,
 
@@ -119,6 +119,11 @@ enum iwl_data_path_subcmd_ids {
 	TLC_MNG_UPDATE_NOTIF = 0xF7,
 
 	/**
+	 * @BEACON_FILTER_IN_NOTIF: &struct iwl_beacon_filter_notif
+	 */
+	BEACON_FILTER_IN_NOTIF = 0xF8,
+
+	/**
 	 * @STA_PM_NOTIF: &struct iwl_mvm_pm_state_notification
 	 */
 	STA_PM_NOTIF = 0xFD,
@@ -391,7 +396,7 @@ enum iwl_datapath_monitor_notif_type {
 
 struct iwl_datapath_monitor_notif {
 	__le32 type;
-	u8 mac_id;
+	u8 link_id;
 	u8 reserved[3];
 } __packed; /* MONITOR_NTF_API_S_VER_1 */
 
diff --git a/sys/contrib/dev/iwlwifi/fw/api/dbg-tlv.h b/sys/contrib/dev/iwlwifi/fw/api/dbg-tlv.h
index 855cd13a181e..3173fa96cb48 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/dbg-tlv.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/dbg-tlv.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #ifndef __iwl_fw_dbg_tlv_h__
 #define __iwl_fw_dbg_tlv_h__
@@ -476,6 +476,8 @@ enum iwl_fw_ini_region_device_memory_subtype {
  * @IWL_FW_INI_TIME_POINT_PRESET_OVERRIDE_START: start handling override preset
  *	request
  * @IWL_FW_INI_TIME_SCAN_FAILURE: failed scan channel list
+ * @IWL_FW_INI_TIME_ESR_LINK_UP: EMLSR is active (several links are activated)
+ * @IWL_FW_INI_TIME_ESR_LINK_DOWN: EMLSR is inactive (only one active link left)
  * @IWL_FW_INI_TIME_POINT_NUM: number of time points
  */
 enum iwl_fw_ini_time_point {
@@ -509,6 +511,8 @@ enum iwl_fw_ini_time_point {
 	IWL_FW_INI_TIME_POINT_PRESET_OVERRIDE_EXT_REQ,
 	IWL_FW_INI_TIME_POINT_PRESET_OVERRIDE_START,
 	IWL_FW_INI_TIME_SCAN_FAILURE,
+	IWL_FW_INI_TIME_ESR_LINK_UP,
+	IWL_FW_INI_TIME_ESR_LINK_DOWN,
 	IWL_FW_INI_TIME_POINT_NUM,
 }; /* FW_TLV_DEBUG_TIME_POINT_API_E */
 
@@ -523,6 +527,8 @@ enum iwl_fw_ini_time_point {
  * @IWL_FW_INI_APPLY_POLICY_OVERRIDE_DATA: override trigger data.
  *	Append otherwise
  * @IWL_FW_INI_APPLY_POLICY_DUMP_COMPLETE_CMD: send cmd once dump collected
+ * @IWL_FW_INI_APPLY_POLICY_SPLIT_DUMP_RESET: split this dump into regions
+ *	before and after the reset handshake
  */
 enum iwl_fw_ini_trigger_apply_policy {
 	IWL_FW_INI_APPLY_POLICY_MATCH_TIME_POINT	= BIT(0),
@@ -531,6 +537,7 @@ enum iwl_fw_ini_trigger_apply_policy {
 	IWL_FW_INI_APPLY_POLICY_OVERRIDE_CFG		= BIT(9),
 	IWL_FW_INI_APPLY_POLICY_OVERRIDE_DATA		= BIT(10),
 	IWL_FW_INI_APPLY_POLICY_DUMP_COMPLETE_CMD	= BIT(16),
+	IWL_FW_INI_APPLY_POLICY_SPLIT_DUMP_RESET	= BIT(17),
 };
 
 /**
@@ -552,12 +559,14 @@ enum iwl_fw_ini_trigger_reset_fw_policy {
  * @IWL_FW_INI_DEBUG_DUMP_POLICY_NO_LIMIT: OS has no limit of dump size
  * @IWL_FW_INI_DEBUG_DUMP_POLICY_MAX_LIMIT_600KB: mini dump only 600KB region dump
  * @IWL_FW_IWL_DEBUG_DUMP_POLICY_MAX_LIMIT_5MB: mini dump 5MB size dump
+ * @IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET: dump this region before reset
+ *	handshake (if requested by %IWL_FW_INI_APPLY_POLICY_SPLIT_DUMP_RESET)
  */
 enum iwl_fw_ini_dump_policy {
 	IWL_FW_INI_DEBUG_DUMP_POLICY_NO_LIMIT           = BIT(0),
 	IWL_FW_INI_DEBUG_DUMP_POLICY_MAX_LIMIT_600KB    = BIT(1),
 	IWL_FW_IWL_DEBUG_DUMP_POLICY_MAX_LIMIT_5MB      = BIT(2),
-
+	IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET	= BIT(3),
 };
 
 /**
diff --git a/sys/contrib/dev/iwlwifi/fw/api/debug.h b/sys/contrib/dev/iwlwifi/fw/api/debug.h
index bea0f4668cc8..0cf1e5124fba 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/debug.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/debug.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -51,7 +51,7 @@ enum iwl_debug_cmds {
 	/**
 	 * @GET_TAS_STATUS:
 	 * sends command to fw to get TAS status
-	 * the response is &struct iwl_mvm_tas_status_resp
+	 * the response is &struct iwl_tas_status_resp
 	 */
 	GET_TAS_STATUS = 0xA,
 	/**
@@ -439,25 +439,20 @@ struct iwl_dbg_dump_complete_cmd {
 	__le32 tp_data;
 } __packed; /* FW_DUMP_COMPLETE_CMD_API_S_VER_1 */
 
-#define TAS_LMAC_BAND_HB       0
-#define TAS_LMAC_BAND_LB       1
-#define TAS_LMAC_BAND_UHB      2
-#define TAS_LMAC_BAND_INVALID  3
-
 /**
- * struct iwl_mvm_tas_status_per_mac - tas status per lmac
+ * struct iwl_tas_status_per_mac - tas status per lmac
  * @static_status: tas statically enabled or disabled per lmac - TRUE/FALSE
  * @static_dis_reason: TAS static disable reason, uses
- *	&enum iwl_mvm_tas_statically_disabled_reason
+ *	&enum iwl_tas_statically_disabled_reason
  * @dynamic_status: Current TAS  status. uses
- *	&enum iwl_mvm_tas_dyna_status
+ *	&enum iwl_tas_dyna_status
  * @near_disconnection: is TAS currently near disconnection per lmac? - TRUE/FALSE
  * @max_reg_pwr_limit: Regulatory power limits in dBm
  * @sar_limit: SAR limits per lmac in dBm
  * @band: Band per lmac
  * @reserved: reserved
  */
-struct iwl_mvm_tas_status_per_mac {
+struct iwl_tas_status_per_mac {
 	u8 static_status;
 	u8 static_dis_reason;
 	u8 dynamic_status;
@@ -466,31 +461,35 @@ struct iwl_mvm_tas_status_per_mac {
 	__le16 sar_limit;
 	u8 band;
 	u8 reserved[3];
-} __packed; /*DEBUG_GET_TAS_STATUS_PER_MAC_S_VER_1*/
+} __packed; /* DEBUG_GET_TAS_STATUS_PER_MAC_S_VER_1 */
 
 /**
- * struct iwl_mvm_tas_status_resp - Response to GET_TAS_STATUS
+ * struct iwl_tas_status_resp - Response to GET_TAS_STATUS
  * @tas_fw_version: TAS FW version
  * @is_uhb_for_usa_enable: is UHB enabled in USA? - TRUE/FALSE
  * @curr_mcc: current mcc
  * @block_list: country block list
  * @tas_status_mac: TAS status per lmac, uses
- *	&struct iwl_mvm_tas_status_per_mac
+ *	&struct iwl_tas_status_per_mac
  * @in_dual_radio: is TAS in dual radio? - TRUE/FALSE
+ * @uhb_allowed_flags: see &enum iwl_tas_uhb_allowed_flags.
+ *	This member is valid only when fw has
+ *	%IWL_UCODE_TLV_CAPA_UHB_CANADA_TAS_SUPPORT capability.
  * @reserved: reserved
  */
-struct iwl_mvm_tas_status_resp {
+struct iwl_tas_status_resp {
 	u8 tas_fw_version;
 	u8 is_uhb_for_usa_enable;
 	__le16 curr_mcc;
 	__le16 block_list[16];
-	struct iwl_mvm_tas_status_per_mac tas_status_mac[2];
+	struct iwl_tas_status_per_mac tas_status_mac[2];
 	u8 in_dual_radio;
-	u8 reserved[3];
-} __packed; /*DEBUG_GET_TAS_STATUS_RSP_API_S_VER_3*/
+	u8 uhb_allowed_flags;
+	u8 reserved[2];
+} __packed; /* DEBUG_GET_TAS_STATUS_RSP_API_S_VER_3 */
 
 /**
- * enum iwl_mvm_tas_dyna_status - TAS current running status
+ * enum iwl_tas_dyna_status - TAS current running status
  * @TAS_DYNA_INACTIVE: TAS status is inactive
  * @TAS_DYNA_INACTIVE_MVM_MODE: TAS is disabled due because FW is in MVM mode
  *	or is in softap mode.
@@ -503,7 +502,7 @@ struct iwl_mvm_tas_status_resp {
  * @TAS_DYNA_ACTIVE: TAS is currently active
  * @TAS_DYNA_STATUS_MAX: TAS status max value
  */
-enum iwl_mvm_tas_dyna_status {
+enum iwl_tas_dyna_status {
 	TAS_DYNA_INACTIVE,
 	TAS_DYNA_INACTIVE_MVM_MODE,
 	TAS_DYNA_INACTIVE_TRIGGER_MODE,
@@ -512,19 +511,22 @@ enum iwl_mvm_tas_dyna_status {
 	TAS_DYNA_ACTIVE,
 
 	TAS_DYNA_STATUS_MAX,
-}; /*_TAS_DYNA_STATUS_E*/
+};
 
 /**
- * enum iwl_mvm_tas_statically_disabled_reason - TAS statically disabled reason
+ * enum iwl_tas_statically_disabled_reason - TAS statically disabled reason
  * @TAS_DISABLED_DUE_TO_BIOS: TAS is disabled because TAS is disabled in BIOS
  * @TAS_DISABLED_DUE_TO_SAR_6DBM: TAS is disabled because SAR limit is less than 6 Dbm
  * @TAS_DISABLED_REASON_INVALID: TAS disable reason is invalid
+ * @TAS_DISABLED_DUE_TO_TABLE_SOURCE_INVALID: TAS is disabled due to
+ *	table source invalid
  * @TAS_DISABLED_REASON_MAX: TAS disable reason max value
  */
-enum iwl_mvm_tas_statically_disabled_reason {
+enum iwl_tas_statically_disabled_reason {
 	TAS_DISABLED_DUE_TO_BIOS,
 	TAS_DISABLED_DUE_TO_SAR_6DBM,
 	TAS_DISABLED_REASON_INVALID,
+	TAS_DISABLED_DUE_TO_TABLE_SOURCE_INVALID,
 
 	TAS_DISABLED_REASON_MAX,
 }; /*_TAS_STATICALLY_DISABLED_REASON_E*/
diff --git a/sys/contrib/dev/iwlwifi/fw/api/dhc.h b/sys/contrib/dev/iwlwifi/fw/api/dhc.h
new file mode 100644
index 000000000000..ddba2fc9254c
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/fw/api/dhc.h
@@ -0,0 +1,230 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+#ifndef __iwl_fw_api_dhc_h__
+#define __iwl_fw_api_dhc_h__
+
+#define DHC_TABLE_MASK_POS (28)
+
+/**
+ * enum iwl_dhc_table_id - DHC table operations index
+ */
+enum iwl_dhc_table_id {
+	/**
+	 * @DHC_TABLE_INTEGRATION: select the integration table
+	 */
+	DHC_TABLE_INTEGRATION	= 2 << DHC_TABLE_MASK_POS,
+	/**
+	 * @DHC_TABLE_TOOLS: select the tools table
+	 */
+	DHC_TABLE_TOOLS		= 0,
+};
+
+/**
+ * enum iwl_dhc_umac_tools_table - tools operations
+ * @DHC_TOOLS_UMAC_GET_TAS_STATUS: Get TAS status.
+ *	See @struct iwl_dhc_tas_status_resp
+ */
+enum iwl_dhc_umac_tools_table {
+	DHC_TOOLS_UMAC_GET_TAS_STATUS = 0,
+};
+
+/**
+ * enum iwl_dhc_umac_integration_table - integration operations
+ */
+enum iwl_dhc_umac_integration_table {
+	/**
+	 * @DHC_INT_UMAC_TWT_OPERATION: trigger a TWT operation
+	 */
+	DHC_INT_UMAC_TWT_OPERATION = 4,
+	/**
+	 * @DHC_INTEGRATION_TLC_DEBUG_CONFIG: TLC debug
+	 */
+	DHC_INTEGRATION_TLC_DEBUG_CONFIG = 1,
+	/**
+	 * @DHC_INTEGRATION_MAX: Maximum UMAC integration table entries
+	 */
+	DHC_INTEGRATION_MAX
+};
+
+#define DHC_TARGET_UMAC BIT(27)
+
+/**
+ * struct iwl_dhc_cmd - debug host command
+ * @length: length in DWs of the data structure that is concatenated to the end
+ *	of this struct
+ * @index_and_mask: bit 31 is 1 for data set operation else it's 0
+ *	bits 28-30 is the index of the table of the operation -
+ *	&enum iwl_dhc_table_id *
+ *	bit 27 is 0 if the cmd targeted to LMAC and 1 if targeted to UMAC,
+ *	(LMAC is 0 for backward compatibility)
+ *	bit 26 is 0 if the cmd targeted to LMAC0 and 1 if targeted to LMAC1,
+ *	relevant only if bit 27 set to 0
+ *	bits 0-25 is a specific entry index in the table specified in bits 28-30
+ *
+ * @data: the concatenated data.
+ */
+struct iwl_dhc_cmd {
+	__le32 length;
+	__le32 index_and_mask;
+#if defined(__linux__)
+	__le32 data[];
+#elif defined(__FreeBSD__)
+	__le32 data[0];
+#endif
+} __packed; /* DHC_CMD_API_S */
+
+/**
+ * struct iwl_dhc_payload_hdr - DHC payload header
+ * @version: a version of a payload
+ * @reserved: reserved for alignment
+ */
+struct iwl_dhc_payload_hdr {
+	u8 version;
+	u8 reserved[3];
+} __packed; /* DHC_PAYLOAD_HDR_API_S_VER_1 */
+
+/**
+ * struct iwl_dhc_tas_status_per_radio - TAS status per radio
+ * @band: &PHY_BAND_5 for high band, PHY_BAND_24 for low band and
+ *	&PHY_BAND_6 for ultra high band.
+ * @static_status: TAS statically enabled or disabled
+ * @static_disable_reason: TAS static disable reason, uses
+ *	&enum iwl_tas_statically_disabled_reason
+ * @near_disconnection: is TAS currently near disconnection per radio
+ * @dynamic_status_ant_a: Antenna A current TAS status.
+ *	uses &enum iwl_tas_dyna_status
+ * @dynamic_status_ant_b: Antenna B current TAS status.
+ *	uses &enum iwl_tas_dyna_status
+ * @max_reg_pwr_limit_ant_a: Antenna A regulatory power limits in dBm
+ * @max_reg_pwr_limit_ant_b: Antenna B regulatory power limits in dBm
+ * @sar_limit_ant_a: Antenna A SAR limit per radio in dBm
+ * @sar_limit_ant_b: Antenna B SAR limit per radio in dBm
+ * @reserved: reserved for alignment
+ */
+struct iwl_dhc_tas_status_per_radio {
+	u8 band;
+	u8 static_status;
+	u8 static_disable_reason;
+	u8 near_disconnection;
+	u8 dynamic_status_ant_a;
+	u8 dynamic_status_ant_b;
+	__le16 max_reg_pwr_limit_ant_a;
+	__le16 max_reg_pwr_limit_ant_b;
+	__le16 sar_limit_ant_a;
+	__le16 sar_limit_ant_b;
+	u8 reserved[2];
+} __packed; /* DHC_TAS_STATUS_PER_RADIO_S_VER_1 */
+
+/**
+ * struct iwl_dhc_tas_status_resp - Response to DHC_TOOLS_UMAC_GET_TAS_STATUS
+ * @header: DHC payload header, uses &struct iwl_dhc_payload_hdr
+ * @tas_config_info: see @struct bios_value_u32
+ * @mcc_block_list: block listed country codes
+ * @tas_status_radio: TAS status, uses &struct iwl_dhc_tas_status_per_radio
+ * @curr_mcc: current mcc
+ * @valid_radio_mask: represent entry in tas_status_per_radio is valid.
+ * @reserved: reserved for alignment
+ */
+struct iwl_dhc_tas_status_resp {
+	struct iwl_dhc_payload_hdr header;
+	struct bios_value_u32 tas_config_info;
+	__le16 mcc_block_list[IWL_WTAS_BLACK_LIST_MAX];
+	struct iwl_dhc_tas_status_per_radio tas_status_radio[2];
+	__le16 curr_mcc;
+	u8 valid_radio_mask;
+	u8 reserved;
+} __packed; /* DHC_TAS_STATUS_RSP_API_S_VER_1 */
+
+/**
+ * struct iwl_dhc_cmd_resp_v1 - debug host command response
+ * @status: status of the command
+ * @data: the response data
+ */
+struct iwl_dhc_cmd_resp_v1 {
+	__le32 status;
+	__le32 data[];
+} __packed; /* DHC_RESP_API_S_VER_1 */
+
+/**
+ * struct iwl_dhc_cmd_resp - debug host command response
+ * @status: status of the command
+ * @descriptor: command descriptor (index_and_mask) returned
+ * @data: the response data
+ */
+struct iwl_dhc_cmd_resp {
+	__le32 status;
+	__le32 descriptor;
+	__le32 data[];
+} __packed; /* DHC_RESP_API_S_VER_2 and DHC_RESP_API_S_VER_3 */
+
+/**
+ * enum iwl_dhc_twt_operation_type - describes the TWT operation type
+ *
+ * @DHC_TWT_REQUEST: Send a Request TWT command
+ * @DHC_TWT_SUGGEST: Send a Suggest TWT command
+ * @DHC_TWT_DEMAND: Send a Demand TWT command
+ * @DHC_TWT_GROUPING: Send a Grouping TWT command
+ * @DHC_TWT_ACCEPT: Send a Accept TWT command
+ * @DHC_TWT_ALTERNATE: Send a Alternate TWT command
+ * @DHC_TWT_DICTATE: Send a Dictate TWT command
+ * @DHC_TWT_REJECT: Send a Reject TWT command
+ * @DHC_TWT_TEARDOWN: Send a TearDown TWT command
+ */
+enum iwl_dhc_twt_operation_type {
+	DHC_TWT_REQUEST,
+	DHC_TWT_SUGGEST,
+	DHC_TWT_DEMAND,
+	DHC_TWT_GROUPING,
+	DHC_TWT_ACCEPT,
+	DHC_TWT_ALTERNATE,
+	DHC_TWT_DICTATE,
+	DHC_TWT_REJECT,
+	DHC_TWT_TEARDOWN,
+}; /* DHC_TWT_OPERATION_TYPE_E */
+
+/**
+ * struct iwl_dhc_twt_operation - trigger a TWT operation
+ *
+ * @mac_id: the mac Id on which to trigger TWT operation
+ * @twt_operation: see &enum iwl_dhc_twt_operation_type
+ * @target_wake_time: when should we be on channel
+ * @interval_exp: the exponent for the interval
+ * @interval_mantissa: the mantissa for the interval
+ * @min_wake_duration: the minimum duration for the wake period
+ * @trigger: is the TWT triggered or not
+ * @flow_type: is the TWT announced or not
+ * @flow_id: the TWT flow identifier from 0 to 7
+ * @protection: is the TWT protected
+ * @ndo_paging_indicator: is ndo_paging_indicator set
+ * @responder_pm_mode: is responder_pm_mode set
+ * @negotiation_type: if the responder wants to doze outside the TWT SP
+ * @twt_request: 1 for TWT request, 0 otherwise
+ * @implicit: is TWT implicit
+ * @twt_group_assignment: the TWT group assignment
+ * @twt_channel: the TWT channel
+ * @reserved: reserved
+ */
+struct iwl_dhc_twt_operation {
+	__le32 mac_id;
+	__le32 twt_operation;
+	__le64 target_wake_time;
+	__le32 interval_exp;
+	__le32 interval_mantissa;
+	__le32 min_wake_duration;
+	u8 trigger;
+	u8 flow_type;
+	u8 flow_id;
+	u8 protection;
+	u8 ndo_paging_indicator;
+	u8 responder_pm_mode;
+	u8 negotiation_type;
+	u8 twt_request;
+	u8 implicit;
+	u8 twt_group_assignment;
+	u8 twt_channel;
+	u8 reserved;
+}; /* DHC_TWT_OPERATION_API_S */
+
+#endif /* __iwl_fw_api_dhc_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/location.h b/sys/contrib/dev/iwlwifi/fw/api/location.h
index 30a54c7fa001..33541f92c7c7 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/location.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/location.h
@@ -2,10 +2,15 @@
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  * Copyright (C) 2018-2022 Intel Corporation
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024-2025 Intel Corporation
  */
 #ifndef __iwl_fw_api_location_h__
 #define __iwl_fw_api_location_h__
+#include <linux/ieee80211.h>
+#include <linux/if_ether.h>
+#include <linux/types.h>
+#include <linux/bits.h>
+#include "rs.h"
 
 /**
  * enum iwl_location_subcmd_ids - location group command IDs
@@ -616,6 +621,9 @@ struct iwl_tof_range_req_ap_entry_v2 {
  *	continue with the session and will provide the LMR feedback.
  * @IWL_INITIATOR_AP_FLAGS_TEST_INCORRECT_SAC: send an incorrect SAC in the
  *	first NDP exchange. This is used for testing.
+ * @IWL_INITIATOR_AP_FLAGS_TEST_BAD_SLTF: use incorrect secure LTF tx key. This
+ *	is used for testing. Only supported from version 15 of the range request
+ *	command.
  */
 enum iwl_initiator_ap_flags {
 	IWL_INITIATOR_AP_FLAGS_ASAP = BIT(1),
@@ -633,6 +641,7 @@ enum iwl_initiator_ap_flags {
 	IWL_INITIATOR_AP_FLAGS_PMF = BIT(14),
 	IWL_INITIATOR_AP_FLAGS_TERMINATE_ON_LMR_FEEDBACK = BIT(15),
 	IWL_INITIATOR_AP_FLAGS_TEST_INCORRECT_SAC = BIT(16),
+	IWL_INITIATOR_AP_FLAGS_TEST_BAD_SLTF = BIT(17),
 };
 
 /**
@@ -767,7 +776,7 @@ enum iwl_location_cipher {
  * @num_of_bursts: Recommended value to be sent to the AP. 2s Exponent of
  *	the number of measurement iterations (min 2^0 = 1, max 2^14)
  * @sta_id: the station id of the AP. Only relevant when associated to the AP,
- *	otherwise should be set to &IWL_MVM_INVALID_STA.
+ *	otherwise should be set to &IWL_INVALID_STA.
  * @cipher: pairwise cipher suite for secured measurement.
  *          &enum iwl_location_cipher.
  * @hltk: HLTK to be used for secured 11az measurement
@@ -814,7 +823,7 @@ struct iwl_tof_range_req_ap_entry_v6 {
  * @num_of_bursts: Recommended value to be sent to the AP. 2s Exponent of
  *	the number of measurement iterations (min 2^0 = 1, max 2^14)
  * @sta_id: the station id of the AP. Only relevant when associated to the AP,
- *	otherwise should be set to &IWL_MVM_INVALID_STA.
+ *	otherwise should be set to &IWL_INVALID_STA.
  * @cipher: pairwise cipher suite for secured measurement.
  *          &enum iwl_location_cipher.
  * @hltk: HLTK to be used for secured 11az measurement
@@ -827,10 +836,10 @@ struct iwl_tof_range_req_ap_entry_v6 {
  *	&IWL_INITIATOR_AP_FLAGS_TB is set.
  * @rx_pn: the next expected PN for protected management frames Rx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  * @tx_pn: the next PN to use for protected management frames Tx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  */
 struct iwl_tof_range_req_ap_entry_v7 {
 	__le32 initiator_ap_flags;
@@ -872,7 +881,7 @@ struct iwl_tof_range_req_ap_entry_v7 {
  * @num_of_bursts: Recommended value to be sent to the AP. 2s Exponent of
  *	the number of measurement iterations (min 2^0 = 1, max 2^14)
  * @sta_id: the station id of the AP. Only relevant when associated to the AP,
- *	otherwise should be set to &IWL_MVM_INVALID_STA.
+ *	otherwise should be set to &IWL_INVALID_STA.
  * @cipher: pairwise cipher suite for secured measurement.
  *          &enum iwl_location_cipher.
  * @hltk: HLTK to be used for secured 11az measurement
@@ -885,10 +894,10 @@ struct iwl_tof_range_req_ap_entry_v7 {
  *	&IWL_INITIATOR_AP_FLAGS_TB is set.
  * @rx_pn: the next expected PN for protected management frames Rx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  * @tx_pn: the next PN to use for protected management frames Tx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  * @r2i_ndp_params: parameters for R2I NDP ranging negotiation.
  *      bits 0 - 2: max LTF repetitions
  *      bits 3 - 5: max number of spatial streams
@@ -946,7 +955,7 @@ struct iwl_tof_range_req_ap_entry_v8 {
  * @num_of_bursts: Recommended value to be sent to the AP. 2s Exponent of
  *	the number of measurement iterations (min 2^0 = 1, max 2^14)
  * @sta_id: the station id of the AP. Only relevant when associated to the AP,
- *	otherwise should be set to &IWL_MVM_INVALID_STA.
+ *	otherwise should be set to &IWL_INVALID_STA.
  * @cipher: pairwise cipher suite for secured measurement.
  *          &enum iwl_location_cipher.
  * @hltk: HLTK to be used for secured 11az measurement
@@ -961,10 +970,10 @@ struct iwl_tof_range_req_ap_entry_v8 {
  *	&IWL_INITIATOR_AP_FLAGS_TB or &IWL_INITIATOR_AP_FLAGS_NON_TB is set.
  * @rx_pn: the next expected PN for protected management frames Rx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  * @tx_pn: the next PN to use for protected management frames Tx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  * @r2i_ndp_params: parameters for R2I NDP ranging negotiation.
  *      bits 0 - 2: max LTF repetitions
  *      bits 3 - 5: max number of spatial streams
@@ -1011,7 +1020,7 @@ struct iwl_tof_range_req_ap_entry_v9 {
 } __packed; /* LOCATION_RANGE_REQ_AP_ENTRY_CMD_API_S_VER_9 */
 
 /**
- * struct iwl_tof_range_req_ap_entry_v10 - AP configuration parameters
+ * struct iwl_tof_range_req_ap_entry - AP configuration parameters
  * @initiator_ap_flags: see &enum iwl_initiator_ap_flags.
  * @band: 0 for 5.2 GHz, 1 for 2.4 GHz, 2 for 6GHz
  * @channel_num: AP Channel number
@@ -1029,7 +1038,7 @@ struct iwl_tof_range_req_ap_entry_v9 {
  * @num_of_bursts: Recommended value to be sent to the AP. 2s Exponent of
  *	the number of measurement iterations (min 2^0 = 1, max 2^14)
  * @sta_id: the station id of the AP. Only relevant when associated to the AP,
- *	otherwise should be set to &IWL_MVM_INVALID_STA.
+ *	otherwise should be set to &IWL_INVALID_STA.
  * @cipher: pairwise cipher suite for secured measurement.
  *          &enum iwl_location_cipher.
  * @hltk: HLTK to be used for secured 11az measurement
@@ -1042,10 +1051,10 @@ struct iwl_tof_range_req_ap_entry_v9 {
  *	&IWL_INITIATOR_AP_FLAGS_TB is set.
  * @rx_pn: the next expected PN for protected management frames Rx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  * @tx_pn: the next PN to use for protected management frames Tx. LE byte
  *	order. Only valid if &IWL_INITIATOR_AP_FLAGS_SECURED is set and sta_id
- *	is set to &IWL_MVM_INVALID_STA.
+ *	is set to &IWL_INVALID_STA.
  * @r2i_ndp_params: parameters for R2I NDP ranging negotiation.
  *      bits 0 - 2: max LTF repetitions
  *      bits 3 - 5: max number of spatial streams
@@ -1059,7 +1068,7 @@ struct iwl_tof_range_req_ap_entry_v9 {
  * @min_time_between_msr: For non trigger based NDP ranging, the minimum time
  *	between measurements in units of milliseconds
  */
-struct iwl_tof_range_req_ap_entry_v10 {
+struct iwl_tof_range_req_ap_entry {
 	__le32 initiator_ap_flags;
 	u8 band;
 	u8 channel_num;
@@ -1130,7 +1139,7 @@ enum iwl_tof_initiator_flags {
 	IWL_TOF_INITIATOR_FLAGS_NON_ASAP_SUPPORT = BIT(20),
 }; /* LOCATION_RANGE_REQ_CMD_API_S_VER_5 */
 
-#define IWL_MVM_TOF_MAX_APS 5
+#define IWL_TOF_MAX_APS 5
 #define IWL_MVM_TOF_MAX_TWO_SIDED_APS 5
 
 /**
@@ -1149,7 +1158,7 @@ enum iwl_tof_initiator_flags {
  *		   when the session is done (successfully / partially).
  *		   one of iwl_tof_response_mode.
  * @reserved0: reserved
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @macaddr_random: '0' Use default source MAC address (i.e. p2_p),
  *	            '1' Use MAC Address randomization according to the below
  * @range_req_bssid: ranging request BSSID
@@ -1179,7 +1188,7 @@ struct iwl_tof_range_req_cmd_v5 {
 	u8 ftm_tx_chains;
 	__le16 common_calib;
 	__le16 specific_calib;
-	struct iwl_tof_range_req_ap_entry_v2 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_req_ap_entry_v2 ap[IWL_TOF_MAX_APS];
 } __packed;
 /* LOCATION_RANGE_REQ_CMD_API_S_VER_5 */
 
@@ -1188,7 +1197,7 @@ struct iwl_tof_range_req_cmd_v5 {
  * @initiator_flags: see flags @ iwl_tof_initiator_flags
  * @request_id: A Token incremented per request. The same Token will be
  *		sent back in the range response
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @range_req_bssid: ranging request BSSID
  * @macaddr_mask: Bits set to 0 shall be copied from the MAC address template.
  *		  Bits set to 1 shall be randomized by the UMAC
@@ -1212,7 +1221,7 @@ struct iwl_tof_range_req_cmd_v7 {
 	__le32 tsf_mac_id;
 	__le16 common_calib;
 	__le16 specific_calib;
-	struct iwl_tof_range_req_ap_entry_v3 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_req_ap_entry_v3 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_7 */
 
 /**
@@ -1220,7 +1229,7 @@ struct iwl_tof_range_req_cmd_v7 {
  * @initiator_flags: see flags @ iwl_tof_initiator_flags
  * @request_id: A Token incremented per request. The same Token will be
  *		sent back in the range response
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @range_req_bssid: ranging request BSSID
  * @macaddr_mask: Bits set to 0 shall be copied from the MAC address template.
  *		  Bits set to 1 shall be randomized by the UMAC
@@ -1244,7 +1253,7 @@ struct iwl_tof_range_req_cmd_v8 {
 	__le32 tsf_mac_id;
 	__le16 common_calib;
 	__le16 specific_calib;
-	struct iwl_tof_range_req_ap_entry_v4 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_req_ap_entry_v4 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_8 */
 
 /**
@@ -1252,7 +1261,7 @@ struct iwl_tof_range_req_cmd_v8 {
  * @initiator_flags: see flags @ iwl_tof_initiator_flags
  * @request_id: A Token incremented per request. The same Token will be
  *		sent back in the range response
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @range_req_bssid: ranging request BSSID
  * @macaddr_mask: Bits set to 0 shall be copied from the MAC address template.
  *		  Bits set to 1 shall be randomized by the UMAC
@@ -1272,7 +1281,7 @@ struct iwl_tof_range_req_cmd_v9 {
 	u8 macaddr_template[ETH_ALEN];
 	__le32 req_timeout_ms;
 	__le32 tsf_mac_id;
-	struct iwl_tof_range_req_ap_entry_v6 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_req_ap_entry_v6 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_9 */
 
 /**
@@ -1280,7 +1289,7 @@ struct iwl_tof_range_req_cmd_v9 {
  * @initiator_flags: see flags @ iwl_tof_initiator_flags
  * @request_id: A Token incremented per request. The same Token will be
  *		sent back in the range response
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @range_req_bssid: ranging request BSSID
  * @macaddr_mask: Bits set to 0 shall be copied from the MAC address template.
  *		  Bits set to 1 shall be randomized by the UMAC
@@ -1300,7 +1309,7 @@ struct iwl_tof_range_req_cmd_v11 {
 	u8 macaddr_template[ETH_ALEN];
 	__le32 req_timeout_ms;
 	__le32 tsf_mac_id;
-	struct iwl_tof_range_req_ap_entry_v7 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_req_ap_entry_v7 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_11 */
 
 /**
@@ -1308,7 +1317,7 @@ struct iwl_tof_range_req_cmd_v11 {
  * @initiator_flags: see flags @ iwl_tof_initiator_flags
  * @request_id: A Token incremented per request. The same Token will be
  *		sent back in the range response
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @range_req_bssid: ranging request BSSID
  * @macaddr_mask: Bits set to 0 shall be copied from the MAC address template.
  *		  Bits set to 1 shall be randomized by the UMAC
@@ -1328,7 +1337,7 @@ struct iwl_tof_range_req_cmd_v12 {
 	u8 macaddr_template[ETH_ALEN];
 	__le32 req_timeout_ms;
 	__le32 tsf_mac_id;
-	struct iwl_tof_range_req_ap_entry_v8 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_req_ap_entry_v8 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_12 */
 
 /**
@@ -1336,7 +1345,7 @@ struct iwl_tof_range_req_cmd_v12 {
  * @initiator_flags: see flags @ iwl_tof_initiator_flags
  * @request_id: A Token incremented per request. The same Token will be
  *		sent back in the range response
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @range_req_bssid: ranging request BSSID
  * @macaddr_mask: Bits set to 0 shall be copied from the MAC address template.
  *		  Bits set to 1 shall be randomized by the UMAC
@@ -1356,15 +1365,15 @@ struct iwl_tof_range_req_cmd_v13 {
 	u8 macaddr_template[ETH_ALEN];
 	__le32 req_timeout_ms;
 	__le32 tsf_mac_id;
-	struct iwl_tof_range_req_ap_entry_v9 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_req_ap_entry_v9 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_13 */
 
 /**
- * struct iwl_tof_range_req_cmd_v14 - start measurement cmd
+ * struct iwl_tof_range_req_cmd - start measurement cmd
  * @initiator_flags: see flags @ iwl_tof_initiator_flags
  * @request_id: A Token incremented per request. The same Token will be
  *		sent back in the range response
- * @num_of_ap: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_ap: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @range_req_bssid: ranging request BSSID
  * @macaddr_mask: Bits set to 0 shall be copied from the MAC address template.
  *		  Bits set to 1 shall be randomized by the UMAC
@@ -1373,9 +1382,9 @@ struct iwl_tof_range_req_cmd_v13 {
  *	This is the session time for completing the measurement.
  * @tsf_mac_id: report the measurement start time for each ap in terms of the
  *	TSF of this mac id. 0xff to disable TSF reporting.
- * @ap: per-AP request data, see &struct iwl_tof_range_req_ap_entry_v10.
+ * @ap: per-AP request data, see &struct iwl_tof_range_req_ap_entry.
  */
-struct iwl_tof_range_req_cmd_v14 {
+struct iwl_tof_range_req_cmd {
 	__le32 initiator_flags;
 	u8 request_id;
 	u8 num_of_ap;
@@ -1384,8 +1393,8 @@ struct iwl_tof_range_req_cmd_v14 {
 	u8 macaddr_template[ETH_ALEN];
 	__le32 req_timeout_ms;
 	__le32 tsf_mac_id;
-	struct iwl_tof_range_req_ap_entry_v10 ap[IWL_MVM_TOF_MAX_APS];
-} __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_13 */
+	struct iwl_tof_range_req_ap_entry ap[IWL_TOF_MAX_APS];
+} __packed; /* LOCATION_RANGE_REQ_CMD_API_S_VER_15 */
 
 /*
  * enum iwl_tof_range_request_status - status of the sent request
@@ -1605,7 +1614,7 @@ struct iwl_tof_range_rsp_ap_entry_ntfy_v5 {
 } __packed; /* LOCATION_RANGE_RSP_AP_ETRY_NTFY_API_S_VER_5 */
 
 /**
- * struct iwl_tof_range_rsp_ap_entry_ntfy_v6 - AP parameters (response)
+ * struct iwl_tof_range_rsp_ap_entry_ntfy_v7 - AP parameters (response)
  * @bssid: BSSID of the AP
  * @measure_status: current APs measurement status, one of
  *	&enum iwl_tof_entry_status.
@@ -1641,7 +1650,7 @@ struct iwl_tof_range_rsp_ap_entry_ntfy_v5 {
  * @tx_pn: the last PN used for this responder Tx in case PMF is configured in
  *	LE byte order.
  */
-struct iwl_tof_range_rsp_ap_entry_ntfy_v6 {
+struct iwl_tof_range_rsp_ap_entry_ntfy_v7 {
 	u8 bssid[ETH_ALEN];
 	u8 measure_status;
 	u8 measure_bw;
@@ -1668,6 +1677,65 @@ struct iwl_tof_range_rsp_ap_entry_ntfy_v6 {
 } __packed; /* LOCATION_RANGE_RSP_AP_ETRY_NTFY_API_S_VER_6,
 	       LOCATION_RANGE_RSP_AP_ETRY_NTFY_API_S_VER_7 */
 
+/**
+ * struct iwl_tof_range_rsp_ap_entry_ntfy - AP parameters (response)
+ * @bssid: BSSID of the AP
+ * @measure_status: current APs measurement status, one of
+ *	&enum iwl_tof_entry_status.
+ * @measure_bw: Current AP Bandwidth: 0  20MHz, 1  40MHz, 2  80MHz
+ * @rtt: The Round Trip Time that took for the last measurement for
+ *	current AP [pSec]
+ * @rtt_variance: The Variance of the RTT values measured for current AP
+ * @rtt_spread: The Difference between the maximum and the minimum RTT
+ *	values measured for current AP in the current session [pSec]
+ * @rssi: RSSI as uploaded in the Channel Estimation notification
+ * @rssi_spread: The Difference between the maximum and the minimum RSSI values
+ *	measured for current AP in the current session
+ * @last_burst: 1 if no more FTM sessions are scheduled for this responder
+ * @refusal_period: refusal period in case of
+ *	@IWL_TOF_ENTRY_RESPONDER_CANNOT_COLABORATE [sec]
+ * @timestamp: The GP2 Clock [usec] where Channel Estimation notification was
+ *	uploaded by the LMAC
+ * @start_tsf: measurement start time in TSF of the mac specified in the range
+ *	request
+ * @reserved1: reserved, for backwards compatibility
+ * @t2t3_initiator: as calculated from the algo in the initiator
+ * @t1t4_responder: as calculated from the algo in the responder
+ * @common_calib: Calib val that was used in for this AP measurement
+ * @specific_calib: val that was used in for this AP measurement
+ * @papd_calib_output: The result of the tof papd calibration that was injected
+ *	into the algorithm.
+ * @rttConfidence: a value between 0 - 31 that represents the rtt accuracy.
+ * @reserved: for alignment
+ * @rx_pn: the last PN used for this responder Rx in case PMF is configured in
+ *	LE byte order.
+ * @tx_pn: the last PN used for this responder Tx in case PMF is configured in
+ *	LE byte order.
+ */
+struct iwl_tof_range_rsp_ap_entry_ntfy {
+	u8 bssid[ETH_ALEN];
+	u8 measure_status;
+	u8 measure_bw;
+	__le32 rtt;
+	__le32 rtt_variance;
+	__le32 rtt_spread;
+	s8 rssi;
+	u8 rssi_spread;
+	u8 last_burst;
+	u8 refusal_period;
+	__le32 timestamp;
+	__le32 start_tsf;
+	__le32 reserved1[2];
+	__le32 t2t3_initiator;
+	__le32 t1t4_responder;
+	__le16 common_calib;
+	__le16 specific_calib;
+	__le32 papd_calib_output;
+	u8 rttConfidence;
+	u8 reserved[3];
+	u8 rx_pn[IEEE80211_CCMP_PN_LEN];
+	u8 tx_pn[IEEE80211_CCMP_PN_LEN];
+} __packed; /* LOCATION_RANGE_RSP_AP_ETRY_NTFY_API_S_VER_8 */
 
 /**
  * enum iwl_tof_response_status - tof response status
@@ -1691,7 +1759,7 @@ enum iwl_tof_response_status {
  * @request_status: status of current measurement session, one of
  *	&enum iwl_tof_response_status.
  * @last_in_batch: reprot policy (when not all responses are uploaded at once)
- * @num_of_aps: Number of APs to measure (error if > IWL_MVM_TOF_MAX_APS)
+ * @num_of_aps: Number of APs to measure (error if > IWL_TOF_MAX_APS)
  * @ap: per-AP data
  */
 struct iwl_tof_range_rsp_ntfy_v5 {
@@ -1699,7 +1767,7 @@ struct iwl_tof_range_rsp_ntfy_v5 {
 	u8 request_status;
 	u8 last_in_batch;
 	u8 num_of_aps;
-	struct iwl_tof_range_rsp_ap_entry_ntfy_v3 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_rsp_ap_entry_ntfy_v3 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_RSP_NTFY_API_S_VER_5 */
 
 /**
@@ -1715,7 +1783,7 @@ struct iwl_tof_range_rsp_ntfy_v6 {
 	u8 num_of_aps;
 	u8 last_report;
 	u8 reserved;
-	struct iwl_tof_range_rsp_ap_entry_ntfy_v4 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_rsp_ap_entry_ntfy_v4 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_RSP_NTFY_API_S_VER_6 */
 
 /**
@@ -1731,25 +1799,42 @@ struct iwl_tof_range_rsp_ntfy_v7 {
 	u8 num_of_aps;
 	u8 last_report;
 	u8 reserved;
-	struct iwl_tof_range_rsp_ap_entry_ntfy_v5 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_rsp_ap_entry_ntfy_v5 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_RSP_NTFY_API_S_VER_7 */
 
 /**
- * struct iwl_tof_range_rsp_ntfy_v8 - ranging response notification
+ * struct iwl_tof_range_rsp_ntfy_v9 - ranging response notification
  * @request_id: A Token ID of the corresponding Range request
  * @num_of_aps: Number of APs results
  * @last_report: 1 if no more FTM sessions are scheduled, 0 otherwise.
  * @reserved: reserved
  * @ap: per-AP data
  */
-struct iwl_tof_range_rsp_ntfy_v8 {
+struct iwl_tof_range_rsp_ntfy_v9 {
 	u8 request_id;
 	u8 num_of_aps;
 	u8 last_report;
 	u8 reserved;
-	struct iwl_tof_range_rsp_ap_entry_ntfy_v6 ap[IWL_MVM_TOF_MAX_APS];
+	struct iwl_tof_range_rsp_ap_entry_ntfy_v7 ap[IWL_TOF_MAX_APS];
 } __packed; /* LOCATION_RANGE_RSP_NTFY_API_S_VER_8,
-	       LOCATION_RANGE_RSP_NTFY_API_S_VER_9 */
+	     * LOCATION_RANGE_RSP_NTFY_API_S_VER_9
+	     */
+
+/**
+ * struct iwl_tof_range_rsp_ntfy - ranging response notification
+ * @request_id: A Token ID of the corresponding Range request
+ * @num_of_aps: Number of APs results
+ * @last_report: 1 if no more FTM sessions are scheduled, 0 otherwise.
+ * @reserved: reserved
+ * @ap: per-AP data
+ */
+struct iwl_tof_range_rsp_ntfy {
+	u8 request_id;
+	u8 num_of_aps;
+	u8 last_report;
+	u8 reserved;
+	struct iwl_tof_range_rsp_ap_entry_ntfy ap[IWL_TOF_MAX_APS];
+} __packed; /* LOCATION_RANGE_RSP_NTFY_API_S_VER_10 */
 
 #define IWL_MVM_TOF_MCSI_BUF_SIZE  (245)
 /**
diff --git a/sys/contrib/dev/iwlwifi/fw/api/mac-cfg.h b/sys/contrib/dev/iwlwifi/fw/api/mac-cfg.h
index ca6fa66d1917..b9f559dac39f 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/mac-cfg.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/mac-cfg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2019, 2021-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2021-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -42,15 +42,15 @@ enum iwl_mac_conf_subcmd_ids {
 	 */
 	LINK_CONFIG_CMD = 0x9,
 	/**
-	 * @STA_CONFIG_CMD: &struct iwl_mvm_sta_cfg_cmd
+	 * @STA_CONFIG_CMD: &struct iwl_sta_cfg_cmd
 	 */
 	STA_CONFIG_CMD = 0xA,
 	/**
-	 * @AUX_STA_CMD: &struct iwl_mvm_aux_sta_cmd
+	 * @AUX_STA_CMD: &struct iwl_aux_sta_cmd
 	 */
 	AUX_STA_CMD = 0xB,
 	/**
-	 * @STA_REMOVE_CMD: &struct iwl_mvm_remove_sta_cmd
+	 * @STA_REMOVE_CMD: &struct iwl_remove_sta_cmd
 	 */
 	STA_REMOVE_CMD = 0xC,
 	/**
@@ -62,11 +62,23 @@ enum iwl_mac_conf_subcmd_ids {
 	 */
 	ROC_CMD = 0xE,
 	/**
+	 * @TWT_OPERATION_CMD: &struct iwl_twt_operation_cmd
+	 */
+	TWT_OPERATION_CMD = 0x10,
+	/**
+	 * @MISSED_BEACONS_NOTIF: &struct iwl_missed_beacons_notif
+	 */
+	MISSED_BEACONS_NOTIF = 0xF6,
+	/**
+	 * @EMLSR_TRANS_FAIL_NOTIF: &struct iwl_esr_trans_fail_notif
+	 */
+	EMLSR_TRANS_FAIL_NOTIF = 0xF7,
+	/**
 	 * @ROC_NOTIF: &struct iwl_roc_notif
 	 */
 	ROC_NOTIF = 0xF8,
 	/**
-	 * @SESSION_PROTECTION_NOTIF: &struct iwl_mvm_session_prot_notif
+	 * @SESSION_PROTECTION_NOTIF: &struct iwl_session_prot_notif
 	 */
 	SESSION_PROTECTION_NOTIF = 0xFB,
 
@@ -299,8 +311,41 @@ enum iwl_mac_config_filter_flags {
 }; /* MAC_FILTER_FLAGS_MASK_E_VER_1 */
 
 /**
+ * struct iwl_mac_wifi_gen_support_v2 - parameters of iwl_mac_config_cmd
+ *	with support up to eht as in version 2 of the command
+ *
+ * @he_support: does this MAC support HE
+ * @he_ap_support: HE AP enabled, "pseudo HE", no trigger frame handling
+ * @eht_support: does this MAC support EHT. Requires he_support
+ */
+struct iwl_mac_wifi_gen_support_v2 {
+	__le16 he_support;
+	__le16 he_ap_support;
+	__le32 eht_support;
+} __packed;
+
+/**
+ * struct iwl_mac_wifi_gen_support - parameters of iwl_mac_config_cmd
+ *	with support up to uhr as in version 3 of the command
+ * ( MAC_CONTEXT_CONFIG_CMD = 0x8 )
+ *
+ * @he_support: does this MAC support HE
+ * @he_ap_support: HE AP enabled, "pseudo HE", no trigger frame handling
+ * @eht_support: does this MAC support EHT. Requires he_support
+ * @uhr_support: does this MAC support UHR. Requires eht_support
+ * @reserved: reserved for alignment and to match version 2's size
+ */
+struct iwl_mac_wifi_gen_support {
+	u8 he_support;
+	u8 he_ap_support;
+	u8 eht_support;
+	u8 uhr_support;
+	__le32 reserved;
+} __packed;
+
+/**
  * struct iwl_mac_config_cmd - command structure to configure MAC contexts in
- *	MLD API
+ *	MLD API for versions 2 and 3
  * ( MAC_CONTEXT_CONFIG_CMD = 0x8 )
  *
  * @id_and_color: ID and color of the MAC
@@ -309,9 +354,8 @@ enum iwl_mac_config_filter_flags {
  * @local_mld_addr: mld address
  * @reserved_for_local_mld_addr: reserved
  * @filter_flags: combination of &enum iwl_mac_config_filter_flags
- * @he_support: does this MAC support HE
- * @he_ap_support: HE AP enabled, "pseudo HE", no trigger frame handling
- * @eht_support: does this MAC support EHT. Requires he_support
+ * @wifi_gen_v2: he/eht parameters as in cmd version 2
+ * @wifi_gen: he/eht/uhr parameters as in cmd version 3
  * @nic_not_ack_enabled: mark that the NIC doesn't support receiving
  *	ACK-enabled AGG, (i.e. both BACK and non-BACK frames in single AGG).
  *	If the NIC is not ACK_ENABLED it may use the EOF-bit in first non-0
@@ -320,7 +364,6 @@ enum iwl_mac_config_filter_flags {
  * @p2p_dev: mac data for p2p device
  */
 struct iwl_mac_config_cmd {
-	/* COMMON_INDEX_HDR_API_S_VER_1 */
 	__le32 id_and_color;
 	__le32 action;
 	/* MAC_CONTEXT_TYPE_API_E */
@@ -328,16 +371,17 @@ struct iwl_mac_config_cmd {
 	u8 local_mld_addr[6];
 	__le16 reserved_for_local_mld_addr;
 	__le32 filter_flags;
-	__le16 he_support;
-	__le16 he_ap_support;
-	__le32 eht_support;
+	union {
+		struct iwl_mac_wifi_gen_support_v2 wifi_gen_v2;
+		struct iwl_mac_wifi_gen_support wifi_gen;
+	};
 	__le32 nic_not_ack_enabled;
 	/* MAC_CONTEXT_CONFIG_SPECIFIC_DATA_API_U_VER_2 */
 	union {
 		struct iwl_mac_client_data client;
 		struct iwl_mac_p2p_dev_data p2p_dev;
 	};
-} __packed; /* MAC_CONTEXT_CONFIG_CMD_API_S_VER_2 */
+} __packed; /* MAC_CONTEXT_CONFIG_CMD_API_S_VER_2_VER_3 */
 
 /**
  * enum iwl_link_ctx_modify_flags - indicate to the fw what fields are being
@@ -374,6 +418,8 @@ struct iwl_mac_config_cmd {
  * @LINK_CONTEXT_MODIFY_EHT_PARAMS: covers iwl_link_ctx_cfg_cmd::puncture_mask.
  *	This flag can be set only if the MAC that this link relates to has
  *	eht_support set to true. No longer used since _VER_3 of this command.
+ * @LINK_CONTEXT_MODIFY_BANDWIDTH: Covers iwl_link_ctx_cfg_cmd::modify_bandwidth.
+ *	Request RX OMI to the AP to modify bandwidth of this link.
  * @LINK_CONTEXT_MODIFY_ALL: set all above flags
  */
 enum iwl_link_ctx_modify_flags {
@@ -385,6 +431,7 @@ enum iwl_link_ctx_modify_flags {
 	LINK_CONTEXT_MODIFY_HE_PARAMS		= BIT(5),
 	LINK_CONTEXT_MODIFY_BSS_COLOR_DISABLE	= BIT(6),
 	LINK_CONTEXT_MODIFY_EHT_PARAMS		= BIT(7),
+	LINK_CONTEXT_MODIFY_BANDWIDTH		= BIT(8),
 	LINK_CONTEXT_MODIFY_ALL			= 0xff,
 }; /* LINK_CONTEXT_MODIFY_MASK_E_VER_1 */
 
@@ -426,6 +473,40 @@ enum iwl_link_ctx_flags {
 }; /* LINK_CONTEXT_FLAG_E_VER_1 */
 
 /**
+ * enum iwl_link_modify_bandwidth - link modify (RX OMI) bandwidth
+ * @IWL_LINK_MODIFY_BW_20: request 20 MHz
+ * @IWL_LINK_MODIFY_BW_40: request 40 MHz
+ * @IWL_LINK_MODIFY_BW_80: request 80 MHz
+ * @IWL_LINK_MODIFY_BW_160: request 160 MHz
+ * @IWL_LINK_MODIFY_BW_320: request 320 MHz
+ */
+enum iwl_link_modify_bandwidth {
+	IWL_LINK_MODIFY_BW_20,
+	IWL_LINK_MODIFY_BW_40,
+	IWL_LINK_MODIFY_BW_80,
+	IWL_LINK_MODIFY_BW_160,
+	IWL_LINK_MODIFY_BW_320,
+};
+
+/**
+ * struct iwl_npca_params - NPCA parameters (non-primary channel access)
+ *
+ * @switch_delay: after switch, delay TX according to destination AP
+ * @switch_back_delay: switch back to control channel before OBSS frame end
+ * @min_dur_threshold: minimum PPDU time to switch to the non-primary
+ *	NPCA channel
+ * @flags: NPCA flags - bit 0: puncturing allowed, bit 1: new TX allowed
+ * @reserved: reserved for alignment purposes
+ */
+struct iwl_npca_params {
+	u8 switch_delay;
+	u8 switch_back_delay;
+	__le16 min_dur_threshold;
+	__le16 flags;
+	__le16 reserved;
+} __packed; /* NPCA_PARAM_API_S_VER_1 */
+
+/**
  * struct iwl_link_config_cmd - command structure to configure the LINK context
  *	in MLD API
  * ( LINK_CONFIG_CMD =0x9 )
@@ -446,6 +527,11 @@ enum iwl_link_ctx_flags {
  * @listen_lmac: indicates whether the link should be allocated on the Listen
  *	Lmac or on the Main Lmac. Cannot be changed on an active Link.
  *	Relevant only for eSR.
+ * @block_tx: tell the firmware that this link can't Tx. This should be used
+ *	only when a link is de-activated because of CSA with mode = 1.
+ *	Available since version 5.
+ * @modify_bandwidth: bandwidth request value for RX OMI (see also
+ *	%LINK_CONTEXT_MODIFY_BANDWIDTH), from &enum iwl_link_modify_bandwidth.
  * @reserved1: in version 2, listen_lmac became reserved
  * @cck_rates: basic rates available for CCK
  * @ofdm_rates: basic rates available for OFDM
@@ -472,9 +558,13 @@ enum iwl_link_ctx_flags {
  * @bssid_index: index of the associated VAP
  * @bss_color: 11ax AP ID that is used in the HE SIG-A to mark inter BSS frame
  * @spec_link_id: link_id as the AP knows it
- * @reserved2: alignment
+ * @ul_mu_data_disable: OM Control UL MU Data Disable RX Support (bit 44) in
+ *	HE MAC Capabilities information field as defined in figure 9-897 in
+ *	IEEE802.11REVme-D5.0
  * @ibss_bssid_addr: bssid for ibss
  * @reserved_for_ibss_bssid_addr: reserved
+ * @npca_params: NPCA parameters
+ * @prio_edca_params: priority EDCA parameters for enhanced QoS
  * @reserved3: reserved for future use
  */
 struct iwl_link_config_cmd {
@@ -487,8 +577,12 @@ struct iwl_link_config_cmd {
 	__le32 modify_mask;
 	__le32 active;
 	union {
-		__le32 listen_lmac;
-		__le32 reserved1;
+		__le32 listen_lmac; /* only _VER_1 */
+		struct {
+			u8 block_tx; /* since _VER_5 */
+			u8 modify_bandwidth; /* since _VER_6 */
+			u8 reserved1[2];
+		};
 	};
 	__le32 cck_rates;
 	__le32 ofdm_rates;
@@ -508,24 +602,26 @@ struct iwl_link_config_cmd {
 	__le16 puncture_mask; /* removed in _VER_3 */
 	__le16 frame_time_rts_th;
 	__le32 flags;
-	__le32 flags_mask;
+	__le32 flags_mask; /* removed in _VER_6 */
 	/* The below fields are for multi-bssid */
 	u8 ref_bssid_addr[6];
 	__le16 reserved_for_ref_bssid_addr;
 	u8 bssid_index;
 	u8 bss_color;
 	u8 spec_link_id;
-	u8 reserved2;
+	u8 ul_mu_data_disable;
 	u8 ibss_bssid_addr[6];
 	__le16 reserved_for_ibss_bssid_addr;
-	__le32 reserved3[8];
-} __packed; /* LINK_CONTEXT_CONFIG_CMD_API_S_VER_1, _VER_2, _VER_3 */
+	struct iwl_npca_params npca_params; /* since _VER_7 */
+	struct iwl_ac_qos prio_edca_params; /* since _VER_7 */
+	__le32 reserved3[4];
+} __packed; /* LINK_CONTEXT_CONFIG_CMD_API_S_VER_1, _VER_2, _VER_3, _VER_4, _VER_5, _VER_6, _VER_7 */
 
 /* Currently FW supports link ids in the range 0-3 and can have
  * at most two active links for each vif.
  */
-#define IWL_MVM_FW_MAX_ACTIVE_LINKS_NUM 2
-#define IWL_MVM_FW_MAX_LINK_ID 3
+#define IWL_FW_MAX_ACTIVE_LINKS_NUM 2
+#define IWL_FW_MAX_LINK_ID 3
 
 /**
  * enum iwl_fw_sta_type - FW station types
@@ -547,7 +643,7 @@ enum iwl_fw_sta_type {
 }; /* STATION_TYPE_E_VER_1 */
 
 /**
- * struct iwl_mvm_sta_cfg_cmd - cmd structure to add a peer sta to the uCode's
+ * struct iwl_sta_cfg_cmd_v1 - cmd structure to add a peer sta to the uCode's
  *	station table
  * ( STA_CONFIG_CMD = 0xA )
  *
@@ -579,7 +675,7 @@ enum iwl_fw_sta_type {
  *	capa
  * @htc_flags: which features are supported in HTC
  */
-struct iwl_mvm_sta_cfg_cmd {
+struct iwl_sta_cfg_cmd_v1 {
 	__le32 sta_id;
 	__le32 link_id;
 	u8 peer_mld_address[ETH_ALEN];
@@ -603,7 +699,78 @@ struct iwl_mvm_sta_cfg_cmd {
 } __packed; /* STA_CMD_API_S_VER_1 */
 
 /**
- * struct iwl_mvm_aux_sta_cmd - command for AUX STA configuration
+ * struct iwl_sta_cfg_cmd - cmd structure to add a peer sta to the uCode's
+ *	station table
+ * ( STA_CONFIG_CMD = 0xA )
+ *
+ * @sta_id: index of station in uCode's station table
+ * @link_id: the id of the link that is used to communicate with this sta
+ * @peer_mld_address: the peers mld address
+ * @reserved_for_peer_mld_address: reserved
+ * @peer_link_address: the address of the link that is used to communicate
+ *	with this sta
+ * @reserved_for_peer_link_address: reserved
+ * @station_type: type of this station. See &enum iwl_fw_sta_type
+ * @assoc_id: for GO only
+ * @beamform_flags: beam forming controls
+ * @mfp: indicates whether the STA uses management frame protection or not.
+ * @mimo: indicates whether the sta uses mimo or not
+ * @mimo_protection: indicates whether the sta uses mimo protection or not
+ * @ack_enabled: indicates that the AP supports receiving ACK-
+ *	enabled AGG, i.e. both BACK and non-BACK frames in a single AGG
+ * @trig_rnd_alloc: indicates that trigger based random allocation
+ *	is enabled according to UORA element existence
+ * @tx_ampdu_spacing: minimum A-MPDU spacing:
+ *	4 - 2us density, 5 - 4us density, 6 - 8us density, 7 - 16us density
+ * @tx_ampdu_max_size: maximum A-MPDU length: 0 - 8K, 1 - 16K, 2 - 32K,
+ *	3 - 64K, 4 - 128K, 5 - 256K, 6 - 512K, 7 - 1024K.
+ * @sp_length: the size of the SP in actual number of frames
+ * @uapsd_acs:  4 LS bits are trigger enabled ACs, 4 MS bits are the deliver
+ *	enabled ACs.
+ * @pkt_ext: optional, exists according to PPE-present bit in the HE/EHT-PHY
+ *	capa
+ * @htc_flags: which features are supported in HTC
+ * @use_ldpc_x2_cw: Indicates whether to use LDPC with double CW
+ * @use_icf: Indicates whether to use ICF instead of RTS
+ * @dps_pad_time: DPS (Dynamic Power Save) padding delay resolution to ensure
+ *	proper timing alignment
+ * @dps_trans_delay: DPS minimal time that takes the peer to return to low power
+ * @mic_prep_pad_delay: MIC prep time padding
+ * @mic_compute_pad_delay: MIC compute time padding
+ * @reserved: Reserved for alignment
+ */
+struct iwl_sta_cfg_cmd {
+	__le32 sta_id;
+	__le32 link_id;
+	u8 peer_mld_address[ETH_ALEN];
+	__le16 reserved_for_peer_mld_address;
+	u8 peer_link_address[ETH_ALEN];
+	__le16 reserved_for_peer_link_address;
+	__le32 station_type;
+	__le32 assoc_id;
+	__le32 beamform_flags;
+	__le32 mfp;
+	__le32 mimo;
+	__le32 mimo_protection;
+	__le32 ack_enabled;
+	__le32 trig_rnd_alloc;
+	__le32 tx_ampdu_spacing;
+	__le32 tx_ampdu_max_size;
+	__le32 sp_length;
+	__le32 uapsd_acs;
+	struct iwl_he_pkt_ext_v2 pkt_ext;
+	__le32 htc_flags;
+	u8 use_ldpc_x2_cw;
+	u8 use_icf;
+	u8 dps_pad_time;
+	u8 dps_trans_delay;
+	u8 mic_prep_pad_delay;
+	u8 mic_compute_pad_delay;
+	u8 reserved[2];
+} __packed; /* STA_CMD_API_S_VER_2 */
+
+/**
+ * struct iwl_aux_sta_cmd - command for AUX STA configuration
  * ( AUX_STA_CMD = 0xB )
  *
  * @sta_id: index of aux sta to configure
@@ -611,7 +778,7 @@ struct iwl_mvm_sta_cfg_cmd {
  * @mac_addr: mac addr of the auxilary sta
  * @reserved_for_mac_addr: reserved
  */
-struct iwl_mvm_aux_sta_cmd {
+struct iwl_aux_sta_cmd {
 	__le32 sta_id;
 	__le32 lmac_id;
 	u8 mac_addr[ETH_ALEN];
@@ -620,13 +787,13 @@ struct iwl_mvm_aux_sta_cmd {
 } __packed; /* AUX_STA_CMD_API_S_VER_1 */
 
 /**
- * struct iwl_mvm_remove_sta_cmd - a cmd structure to remove a sta added by
+ * struct iwl_remove_sta_cmd - a cmd structure to remove a sta added by
  *	STA_CONFIG_CMD or AUX_STA_CONFIG_CMD
  * ( STA_REMOVE_CMD = 0xC )
  *
  * @sta_id: index of station to remove
  */
-struct iwl_mvm_remove_sta_cmd {
+struct iwl_remove_sta_cmd {
 	__le32 sta_id;
 } __packed; /* REMOVE_STA_API_S_VER_1 */
 
@@ -644,9 +811,9 @@ struct iwl_mvm_sta_disable_tx_cmd {
 
 /**
  * enum iwl_mvm_fw_esr_recommendation - FW recommendation code
- * @ESR_RECOMMEND_LEAVE: recommendation to leave esr
- * @ESR_FORCE_LEAVE: force exiting esr
- * @ESR_RECOMMEND_ENTER: recommendation to enter esr
+ * @ESR_RECOMMEND_LEAVE: recommendation to leave EMLSR
+ * @ESR_FORCE_LEAVE: force exiting EMLSR
+ * @ESR_RECOMMEND_ENTER: recommendation to enter EMLSR
  */
 enum iwl_mvm_fw_esr_recommendation {
 	ESR_RECOMMEND_LEAVE,
@@ -655,12 +822,169 @@ enum iwl_mvm_fw_esr_recommendation {
 }; /* ESR_MODE_RECOMMENDATION_CODE_API_E_VER_1 */
 
 /**
- * struct iwl_mvm_esr_mode_notif - FWs recommendation/force for esr mode
+ * struct iwl_esr_mode_notif_v1 - FW recommendation/force for EMLSR mode
  *
- * @action: the action to apply on esr state. See &iwl_mvm_fw_esr_recommendation
+ * @action: the action to apply on EMLSR state.
+ *	See &iwl_mvm_fw_esr_recommendation
  */
-struct iwl_mvm_esr_mode_notif {
+struct iwl_esr_mode_notif_v1 {
 	__le32 action;
 } __packed; /* ESR_MODE_RECOMMENDATION_NTFY_API_S_VER_1 */
 
+/**
+ * enum iwl_esr_leave_reason - reasons for leaving EMLSR mode
+ *
+ * @ESR_LEAVE_REASON_OMI_MU_UL_DISALLOWED: OMI MU UL disallowed
+ * @ESR_LEAVE_REASON_NO_TRIG_FOR_ESR_STA: No trigger for EMLSR station
+ * @ESR_LEAVE_REASON_NO_ESR_STA_IN_MU_DL: No EMLSR station in MU DL
+ * @ESR_LEAVE_REASON_BAD_ACTIV_FRAME_TH: Bad activation frame threshold
+ * @ESR_LEAVE_REASON_RTS_IN_DUAL_LISTEN: RTS in dual listen
+ */
+enum iwl_esr_leave_reason {
+	ESR_LEAVE_REASON_OMI_MU_UL_DISALLOWED	= BIT(0),
+	ESR_LEAVE_REASON_NO_TRIG_FOR_ESR_STA	= BIT(1),
+	ESR_LEAVE_REASON_NO_ESR_STA_IN_MU_DL	= BIT(2),
+	ESR_LEAVE_REASON_BAD_ACTIV_FRAME_TH	= BIT(3),
+	ESR_LEAVE_REASON_RTS_IN_DUAL_LISTEN	= BIT(4),
+};
+
+/**
+ * struct iwl_esr_mode_notif - FW recommendation/force for EMLSR mode
+ *
+ * @action: the action to apply on EMLSR state.
+ *	See &iwl_mvm_fw_esr_recommendation
+ * @leave_reason_mask: mask for various reasons to leave EMLSR mode.
+ *	See &iwl_esr_leave_reason
+ */
+struct iwl_esr_mode_notif {
+	__le32 action;
+	__le32 leave_reason_mask;
+} __packed; /* ESR_MODE_RECOMMENDATION_NTFY_API_S_VER_2 */
+
+/**
+ * struct iwl_missed_beacons_notif - sent when by the firmware upon beacon loss
+ *  ( MISSED_BEACONS_NOTIF = 0xF6 )
+ * @link_id: fw link ID
+ * @consec_missed_beacons_since_last_rx: number of consecutive missed
+ *	beacons since last RX.
+ * @consec_missed_beacons: number of consecutive missed beacons
+ * @other_link_id: used in EMLSR only. The fw link ID for
+ *	&consec_missed_beacons_other_link. IWL_MVM_FW_LINK_ID_INVALID (0xff) if
+ *	invalid.
+ * @consec_missed_beacons_other_link: number of consecutive missed beacons on
+ *	&other_link_id.
+ */
+struct iwl_missed_beacons_notif {
+	__le32 link_id;
+	__le32 consec_missed_beacons_since_last_rx;
+	__le32 consec_missed_beacons;
+	__le32 other_link_id;
+	__le32 consec_missed_beacons_other_link;
+} __packed; /* MISSED_BEACON_NTFY_API_S_VER_5 */
+
+/*
+ * enum iwl_esr_trans_fail_code: to be used to parse the notif below
+ *
+ * @ESR_TRANS_FAILED_TX_STATUS_ERROR: failed to TX EML OMN frame
+ * @ESR_TRANSITION_FAILED_TX_TIMEOUT: timeout on the EML OMN frame
+ * @ESR_TRANSITION_FAILED_BEACONS_NOT_HEARD: can't get a beacon on the new link
+ */
+enum iwl_esr_trans_fail_code {
+	ESR_TRANS_FAILED_TX_STATUS_ERROR,
+	ESR_TRANSITION_FAILED_TX_TIMEOUT,
+	ESR_TRANSITION_FAILED_BEACONS_NOT_HEARD,
+};
+
+/**
+ * struct iwl_esr_trans_fail_notif - FW reports a failure in EMLSR transition
+ *
+ * @link_id: the link_id that still works after the failure
+ * @activation: true if the link was activated, false otherwise
+ * @err_code: see &enum iwl_esr_trans_fail_code
+ */
+struct iwl_esr_trans_fail_notif {
+	__le32 link_id;
+	__le32 activation;
+	__le32 err_code;
+} __packed; /* ESR_TRANSITION_FAILED_NTFY_API_S_VER_1 */
+
+/*
+ * enum iwl_twt_operation_type: TWT operation in a TWT action frame
+ *
+ * @TWT_OPERATION_REQUEST: TWT Request
+ * @TWT_OPERATION_SUGGEST: TWT Suggest
+ * @TWT_OPERATION_DEMAND: TWT Demand
+ * @TWT_OPERATION_GROUPING: TWT Grouping
+ * @TWT_OPERATION_ACCEPT: TWT Accept
+ * @TWT_OPERATION_ALTERNATE: TWT Alternate
+ * @TWT_OPERATION_DICTATE: TWT Dictate
+ * @TWT_OPERATION_REJECT: TWT Reject
+ * @TWT_OPERATION_TEARDOWN: TWT Teardown
+ * @TWT_OPERATION_UNAVAILABILITY: TWT Unavailability
+ */
+enum iwl_twt_operation_type {
+	TWT_OPERATION_REQUEST,
+	TWT_OPERATION_SUGGEST,
+	TWT_OPERATION_DEMAND,
+	TWT_OPERATION_GROUPING,
+	TWT_OPERATION_ACCEPT,
+	TWT_OPERATION_ALTERNATE,
+	TWT_OPERATION_DICTATE,
+	TWT_OPERATION_REJECT,
+	TWT_OPERATION_TEARDOWN,
+	TWT_OPERATION_UNAVAILABILITY,
+	TWT_OPERATION_MAX,
+}; /* TWT_OPERATION_TYPE_E_VER_1 */
+
+/**
+ * struct iwl_twt_operation_cmd - initiate a TWT session from driver
+ *
+ * @link_id: FW link id to initiate the TWT
+ * @twt_operation: &enum iwl_twt_operation_type
+ * @target_wake_time: TSF time to start the TWT
+ * @interval_exponent: the exponent for the interval
+ * @interval_mantissa: the mantissa for the interval
+ * @minimum_wake_duration: the minimum duration for the wake period
+ * @trigger: is the TWT triggered or not
+ * @flow_type: is the TWT announced (0) or not (1)
+ * @flow_id: the TWT flow identifier 0 - 7
+ * @twt_protection: is the TWT protected
+ * @ndp_paging_indicator: is ndp paging indicator set
+ * @responder_pm_mode: is responder pm mode set
+ * @negotiation_type: if the responder wants to doze outside the TWT SP
+ * @twt_request: 1 for TWT request (STA), 0 for TWT response (AP)
+ * @implicit: is TWT implicit
+ * @twt_group_assignment: the TWT group assignment
+ * @twt_channel: the TWT channel
+ * @restricted_info_present: is this a restricted TWT
+ * @dl_bitmap_valid: is DL (download) bitmap valid (restricted TWT)
+ * @ul_bitmap_valid: is UL (upload) bitmap valid (restricted TWT)
+ * @dl_tid_bitmap: DL TID bitmap (restricted TWT)
+ * @ul_tid_bitmap: UL TID bitmap (restricted TWT)
+ */
+struct iwl_twt_operation_cmd {
+	__le32 link_id;
+	__le32 twt_operation;
+	__le64 target_wake_time;
+	__le32 interval_exponent;
+	__le32 interval_mantissa;
+	__le32 minimum_wake_duration;
+	u8 trigger;
+	u8 flow_type;
+	u8 flow_id;
+	u8 twt_protection;
+	u8 ndp_paging_indicator;
+	u8 responder_pm_mode;
+	u8 negotiation_type;
+	u8 twt_request;
+	u8 implicit;
+	u8 twt_group_assignment;
+	u8 twt_channel;
+	u8 restricted_info_present;
+	u8 dl_bitmap_valid;
+	u8 ul_bitmap_valid;
+	u8 dl_tid_bitmap;
+	u8 ul_tid_bitmap;
+} __packed; /* TWT_OPERATION_API_S_VER_1 */
+
 #endif /* __iwl_fw_api_mac_cfg_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/mac.h b/sys/contrib/dev/iwlwifi/fw/api/mac.h
index bcbbf8c4a297..2a174c00b712 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/mac.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/mac.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2022, 2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022, 2024-2025 Intel Corporation
  * Copyright (C) 2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_fw_api_mac_h__
@@ -16,8 +16,8 @@
 #define NUM_MAC_INDEX		(NUM_MAC_INDEX_DRIVER + 1)
 #define NUM_MAC_INDEX_CDB	(NUM_MAC_INDEX_DRIVER + 2)
 
-#define IWL_MVM_STATION_COUNT_MAX	16
-#define IWL_MVM_INVALID_STA		0xFF
+#define IWL_STATION_COUNT_MAX	16
+#define IWL_INVALID_STA		0xFF
 
 enum iwl_ac {
 	AC_BK,
@@ -287,9 +287,9 @@ struct iwl_ac_qos {
 	__le16 cw_min;
 	__le16 cw_max;
 	u8 aifsn;
-	u8 fifos_mask;
+	u8 fifos_mask; /* not in use since _VER_3 */
 	__le16 edca_txop;
-} __packed; /* AC_QOS_API_S_VER_2 */
+} __packed; /* AC_QOS_API_S_VER_2, _VER_3 */
 
 /**
  * struct iwl_mac_ctx_cmd - command structure to configure MAC contexts
@@ -378,7 +378,7 @@ struct iwl_missed_beacons_notif_ver_3 {
 } __packed; /* MISSED_BEACON_NTFY_API_S_VER_3 */
 
 /**
- * struct iwl_missed_beacons_notif - information on missed beacons
+ * struct iwl_missed_beacons_notif_v4 - information on missed beacons
  * ( MISSED_BEACONS_NOTIFICATION = 0xa2 )
  * @link_id: fw link ID
  * @consec_missed_beacons_since_last_rx: number of consecutive missed
@@ -387,7 +387,7 @@ struct iwl_missed_beacons_notif_ver_3 {
  * @num_expected_beacons: number of expected beacons
  * @num_recvd_beacons: number of received beacons
  */
-struct iwl_missed_beacons_notif {
+struct iwl_missed_beacons_notif_v4 {
 	__le32 link_id;
 	__le32 consec_missed_beacons_since_last_rx;
 	__le32 consec_missed_beacons;
diff --git a/sys/contrib/dev/iwlwifi/fw/api/nvm-reg.h b/sys/contrib/dev/iwlwifi/fw/api/nvm-reg.h
index d424d0126367..e90f3187e55c 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/nvm-reg.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/nvm-reg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -464,21 +464,30 @@ struct iwl_tas_config_cmd_v3 {
 } __packed; /* TAS_CONFIG_CMD_API_S_VER_3 */
 
 /**
+ * enum iwl_tas_uhb_allowed_flags - per country TAS UHB allowed flags.
+ * @TAS_UHB_ALLOWED_CANADA: TAS UHB is allowed in Canada. This flag is valid
+ *	only when fw has %IWL_UCODE_TLV_CAPA_UHB_CANADA_TAS_SUPPORT capability.
+ */
+enum iwl_tas_uhb_allowed_flags {
+	TAS_UHB_ALLOWED_CANADA	= BIT(0),
+};
+
+/**
  * struct iwl_tas_config_cmd_v4 - configures the TAS
  * @override_tas_iec: indicates whether to override default value of IEC regulatory
  * @enable_tas_iec: in case override_tas_iec is set -
  *	indicates whether IEC regulatory is enabled or disabled
  * @usa_tas_uhb_allowed: if set, allow TAS UHB in the USA
- * @reserved: reserved
-*/
+ * @uhb_allowed_flags: see &enum iwl_tas_uhb_allowed_flags.
+ */
 struct iwl_tas_config_cmd_v4 {
 	u8 override_tas_iec;
 	u8 enable_tas_iec;
 	u8 usa_tas_uhb_allowed;
-	u8 reserved;
+	u8 uhb_allowed_flags;
 } __packed; /* TAS_CONFIG_CMD_API_S_VER_4 */
 
-struct iwl_tas_config_cmd {
+struct iwl_tas_config_cmd_v2_v4 {
 	struct iwl_tas_config_cmd_common common;
 	union {
 		struct iwl_tas_config_cmd_v3 v3;
@@ -487,6 +496,46 @@ struct iwl_tas_config_cmd {
 };
 
 /**
+ * enum bios_source - source of bios data
+ * @BIOS_SOURCE_NONE: BIOS source is not defined
+ * @BIOS_SOURCE_ACPI: BIOS source is ACPI
+ * @BIOS_SOURCE_UEFI: BIOS source is UEFI
+ */
+enum bios_source {
+	BIOS_SOURCE_NONE,
+	BIOS_SOURCE_ACPI,
+	BIOS_SOURCE_UEFI,
+};
+
+/**
+ * struct bios_value_u32 - BIOS configuration.
+ * @table_source: see &enum bios_source
+ * @table_revision: table revision.
+ * @reserved: reserved
+ * @value: value in bios.
+ */
+struct bios_value_u32 {
+	u8 table_source;
+	u8 table_revision;
+	u8 reserved[2];
+	__le32 value;
+} __packed; /* BIOS_TABLE_SOURCE_U32_S_VER_1 */
+
+/**
+ * struct iwl_tas_config_cmd - configures the TAS.
+ * @block_list_size: size of relevant field in block_list_array
+ * @block_list_array: list of countries where TAS must be disabled
+ * @reserved: reserved
+ * @tas_config_info: see @struct bios_value_u32
+ */
+struct iwl_tas_config_cmd {
+	__le16 block_list_size;
+	__le16 block_list_array[IWL_WTAS_BLACK_LIST_MAX];
+	u8 reserved[2];
+	struct bios_value_u32 tas_config_info;
+} __packed; /* TAS_CONFIG_CMD_API_S_VER_5 */
+
+/**
  * enum iwl_lari_config_masks - bit masks for the various LARI config operations
  * @LARI_CONFIG_DISABLE_11AC_UKRAINE_MSK: disable 11ac in ukraine
  * @LARI_CONFIG_CHANGE_ETSI_TO_PASSIVE_MSK: ETSI 5.8GHz SRD passive scan
@@ -705,7 +754,7 @@ struct iwl_lari_config_change_cmd_v10 {
  *	according to the BIOS definitions.
  *	For LARI cmd version 11 - bits 0:4 are supported.
  *	For LARI cmd version 12 - bits 0:6 are supported and bits 7:31 are
- *	reserved. No need to mask out the reserved bits.
+ *	reserved.
  * @force_disable_channels_bitmap: Bitmap of disabled bands/channels.
  *	Each bit represents a set of channels in a specific band that should be
  *	disabled
@@ -738,6 +787,7 @@ struct iwl_lari_config_change_cmd {
 /* Activate UNII-1 (5.2GHz) for World Wide */
 #define ACTIVATE_5G2_IN_WW_MASK			BIT(4)
 #define CHAN_STATE_ACTIVE_BITMAP_CMD_V11	0x1F
+#define CHAN_STATE_ACTIVE_BITMAP_CMD_V12	0x7F
 
 /**
  * struct iwl_pnvm_init_complete_ntfy - PNVM initialization complete
diff --git a/sys/contrib/dev/iwlwifi/fw/api/offload.h b/sys/contrib/dev/iwlwifi/fw/api/offload.h
index 6a7bbfd6b2b7..2a1c2b0f19e4 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/offload.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/offload.h
@@ -3,7 +3,7 @@
  * Copyright (C) 2012-2014 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2021-2024 Intel Corporation
+ * Copyright (C) 2021-2025 Intel Corporation
  */
 #ifndef __iwl_fw_api_offload_h__
 #define __iwl_fw_api_offload_h__
@@ -19,7 +19,7 @@ enum iwl_prot_offload_subcmd_ids {
 
 	/**
 	 * @WOWLAN_INFO_NOTIFICATION: Notification in
-	 * &struct iwl_wowlan_info_notif_v1, &struct iwl_wowlan_info_notif_v2,
+	 * &struct iwl_wowlan_info_notif_v1, iwl_wowlan_info_notif_v3,
 	 * or &struct iwl_wowlan_info_notif
 	 */
 	WOWLAN_INFO_NOTIFICATION = 0xFD,
@@ -31,7 +31,7 @@ enum iwl_prot_offload_subcmd_ids {
 
 	/**
 	 * @STORED_BEACON_NTF: &struct iwl_stored_beacon_notif_v2 or
-	 *	&struct iwl_stored_beacon_notif_v3
+	 *	&struct iwl_stored_beacon_notif
 	 */
 	STORED_BEACON_NTF = 0xFF,
 };
@@ -71,18 +71,18 @@ struct iwl_stored_beacon_notif_v2 {
 } __packed; /* WOWLAN_STROED_BEACON_INFO_S_VER_2 */
 
 /**
- * struct iwl_stored_beacon_notif_v3 - Stored beacon notification
+ * struct iwl_stored_beacon_notif - Stored beacon notification
  *
  * @common: fields common for all versions
  * @sta_id: station for which the beacon was received
  * @reserved: reserved for alignment
  * @data: beacon data, length in @byte_count
  */
-struct iwl_stored_beacon_notif_v3 {
+struct iwl_stored_beacon_notif {
 	struct iwl_stored_beacon_notif_common common;
 	u8 sta_id;
 	u8 reserved[3];
 	u8 data[MAX_STORED_BEACON_SIZE];
-} __packed; /* WOWLAN_STROED_BEACON_INFO_S_VER_3 */
+} __packed; /* WOWLAN_STROED_BEACON_INFO_S_VER_3, _VER_4 */
 
 #endif /* __iwl_fw_api_offload_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/phy-ctxt.h b/sys/contrib/dev/iwlwifi/fw/api/phy-ctxt.h
index 4d8a12799c4d..4594a7c94bd6 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/phy-ctxt.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/phy-ctxt.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018, 2020-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018, 2020-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -146,6 +146,7 @@ struct iwl_phy_context_cmd_v1 {
  * @sbb_ctrl_channel_loc: location of the control channel
  * @puncture_mask: bitmap of punctured subchannels
  * @dsp_cfg_flags: set to 0
+ * @secondary_ctrl_chnl_loc: location of secondary control channel
  * @reserved: reserved to align to 64 bit
  */
 struct iwl_phy_context_cmd {
@@ -164,11 +165,13 @@ struct iwl_phy_context_cmd {
 		};
 	};
 	__le32 dsp_cfg_flags;
-	__le32 reserved;
+	u8 secondary_ctrl_chnl_loc;
+	u8 reserved[3];
 } __packed; /* PHY_CONTEXT_CMD_API_VER_3,
 	     * PHY_CONTEXT_CMD_API_VER_4,
 	     * PHY_CONTEXT_CMD_API_VER_5,
-	     * PHY_CONTEXT_CMD_API_VER_6
+	     * PHY_CONTEXT_CMD_API_VER_6,
+	     * PHY_CONTEXT_CMD_API_S_VER_7
 	     */
 
 #endif /* __iwl_fw_api_phy_ctxt_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/phy.h b/sys/contrib/dev/iwlwifi/fw/api/phy.h
index c73d4d597857..4c5221debbcb 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/phy.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/phy.h
@@ -1,11 +1,16 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2019-2022, 2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2019-2022, 2024-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_fw_api_phy_h__
 #define __iwl_fw_api_phy_h__
+#include <linux/types.h>
+#include <linux/bits.h>
+#if defined(__FreeBSD__)
+#include <linux/bitops.h>
+#endif
 
 /**
  * enum iwl_phy_ops_subcmd_ids - PHY group commands
@@ -19,7 +24,7 @@ enum iwl_phy_ops_subcmd_ids {
 	CMD_DTS_MEASUREMENT_TRIGGER_WIDE = 0x0,
 
 	/**
-	 * @CTDP_CONFIG_CMD: &struct iwl_mvm_ctdp_cmd
+	 * @CTDP_CONFIG_CMD: &struct iwl_ctdp_cmd
 	 */
 	CTDP_CONFIG_CMD = 0x03,
 
@@ -55,7 +60,7 @@ enum iwl_phy_ops_subcmd_ids {
 	/**
 	 * @DTS_MEASUREMENT_NOTIF_WIDE:
 	 * &struct iwl_dts_measurement_notif_v1 or
-	 * &struct iwl_dts_measurement_notif_v2
+	 * &struct iwl_dts_measurement_notif
 	 */
 	DTS_MEASUREMENT_NOTIF_WIDE = 0xFF,
 };
@@ -152,13 +157,13 @@ struct iwl_dts_measurement_notif_v1 {
 } __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_1*/
 
 /**
- * struct iwl_dts_measurement_notif_v2 - measurements notification
+ * struct iwl_dts_measurement_notif - measurements notification
  *
  * @temp: the measured temperature
  * @voltage: the measured voltage
  * @threshold_idx: the trip index that was crossed
  */
-struct iwl_dts_measurement_notif_v2 {
+struct iwl_dts_measurement_notif {
 	__le32 temp;
 	__le32 voltage;
 	__le32 threshold_idx;
@@ -195,25 +200,25 @@ struct ct_kill_notif {
 } __packed; /* CT_KILL_NOTIFICATION_API_S_VER_1, CT_KILL_NOTIFICATION_API_S_VER_2 */
 
 /**
-* enum iwl_mvm_ctdp_cmd_operation - CTDP command operations
+* enum iwl_ctdp_cmd_operation - CTDP command operations
 * @CTDP_CMD_OPERATION_START: update the current budget
 * @CTDP_CMD_OPERATION_STOP: stop ctdp
 * @CTDP_CMD_OPERATION_REPORT: get the average budget
 */
-enum iwl_mvm_ctdp_cmd_operation {
+enum iwl_ctdp_cmd_operation {
 	CTDP_CMD_OPERATION_START	= 0x1,
 	CTDP_CMD_OPERATION_STOP		= 0x2,
 	CTDP_CMD_OPERATION_REPORT	= 0x4,
 };/* CTDP_CMD_OPERATION_TYPE_E */
 
 /**
- * struct iwl_mvm_ctdp_cmd - track and manage the FW power consumption budget
+ * struct iwl_ctdp_cmd - track and manage the FW power consumption budget
  *
- * @operation: see &enum iwl_mvm_ctdp_cmd_operation
+ * @operation: see &enum iwl_ctdp_cmd_operation
  * @budget: the budget in milliwatt
  * @window_size: defined in API but not used
  */
-struct iwl_mvm_ctdp_cmd {
+struct iwl_ctdp_cmd {
 	__le32 operation;
 	__le32 budget;
 	__le32 window_size;
diff --git a/sys/contrib/dev/iwlwifi/fw/api/power.h b/sys/contrib/dev/iwlwifi/fw/api/power.h
index 6e6a92d173cc..786b3bf4b448 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/power.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/power.h
@@ -1,12 +1,14 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_fw_api_power_h__
 #define __iwl_fw_api_power_h__
 
+#include "nvm-reg.h"
+
 /* Power Management Commands, Responses, Notifications */
 
 /**
@@ -54,7 +56,7 @@ struct iwl_ltr_config_cmd_v1 {
  * @flags: See &enum iwl_ltr_config_flags
  * @static_long: static LTR Long register value.
  * @static_short: static LTR Short register value.
- * @ltr_cfg_values: LTR parameters table values (in usec) in folowing order:
+ * @ltr_cfg_values: LTR parameters table values (in usec) in following order:
  *	TX, RX, Short Idle, Long Idle. Used only if %LTR_CFG_FLAG_UPDATE_VALUES
  *	is set.
  * @ltr_short_idle_timeout: LTR Short Idle timeout (in usec). Used only if
@@ -89,6 +91,7 @@ struct iwl_ltr_config_cmd {
  * @POWER_FLAGS_LPRX_ENA_MSK: Low Power RX enable.
  * @POWER_FLAGS_UAPSD_MISBEHAVING_ENA_MSK: AP/GO's uAPSD misbehaving
  *		detection enablement
+ * @POWER_FLAGS_ENABLE_SMPS_MSK: SMPS is allowed for this vif
 */
 enum iwl_power_flags {
 	POWER_FLAGS_POWER_SAVE_ENA_MSK		= BIT(0),
@@ -99,6 +102,7 @@ enum iwl_power_flags {
 	POWER_FLAGS_ADVANCE_PM_ENA_MSK		= BIT(9),
 	POWER_FLAGS_LPRX_ENA_MSK		= BIT(11),
 	POWER_FLAGS_UAPSD_MISBEHAVING_ENA_MSK	= BIT(12),
+	POWER_FLAGS_ENABLE_SMPS_MSK		= BIT(14),
 };
 
 #define IWL_POWER_VEC_SIZE 5
@@ -216,7 +220,6 @@ struct iwl_mac_power_cmd {
 	/* CONTEXT_DESC_API_T_VER_1 */
 	__le32 id_and_color;
 
-	/* CLIENT_PM_POWER_TABLE_S_VER_1 */
 	__le16 flags;
 	__le16 keep_alive_seconds;
 	__le32 rx_data_timeout;
@@ -237,7 +240,7 @@ struct iwl_mac_power_cmd {
 	u8 heavy_rx_thld_percentage;
 	u8 limited_ps_threshold;
 	u8 reserved;
-} __packed;
+} __packed; /* CLIENT_PM_POWER_TABLE_S_VER_1, VER_2 */
 
 /*
  * struct iwl_uapsd_misbehaving_ap_notif - FW sends this notification when
@@ -252,21 +255,8 @@ struct iwl_uapsd_misbehaving_ap_notif {
 	u8 reserved[3];
 } __packed;
 
-/**
- * struct iwl_reduce_tx_power_cmd - TX power reduction command
- * REDUCE_TX_POWER_CMD = 0x9f
- * @flags: (reserved for future implementation)
- * @mac_context_id: id of the mac ctx for which we are reducing TX power.
- * @pwr_restriction: TX power restriction in dBms.
- */
-struct iwl_reduce_tx_power_cmd {
-	u8 flags;
-	u8 mac_context_id;
-	__le16 pwr_restriction;
-} __packed; /* TX_REDUCED_POWER_API_S_VER_1 */
-
 enum iwl_dev_tx_power_cmd_mode {
-	IWL_TX_POWER_MODE_SET_MAC = 0,
+	IWL_TX_POWER_MODE_SET_LINK = 0,
 	IWL_TX_POWER_MODE_SET_DEVICE = 1,
 	IWL_TX_POWER_MODE_SET_CHAINS = 2,
 	IWL_TX_POWER_MODE_SET_ACK = 3,
@@ -283,20 +273,16 @@ enum iwl_dev_tx_power_cmd_mode {
 /**
  * struct iwl_dev_tx_power_common - Common part of the TX power reduction cmd
  * @set_mode: see &enum iwl_dev_tx_power_cmd_mode
- * @mac_context_id: id of the mac ctx for which we are reducing TX power.
+ * @link_id: id of the link ctx for which we are reducing TX power.
+ *	For version 9 / 10, this is the link id. For earlier versions, it is
+ *	the mac id.
  * @pwr_restriction: TX power restriction in 1/8 dBms.
- * @dev_24: device TX power restriction in 1/8 dBms
- * @dev_52_low: device TX power restriction upper band - low
- * @dev_52_high: device TX power restriction upper band - high
  */
 struct iwl_dev_tx_power_common {
 	__le32 set_mode;
-	__le32 mac_context_id;
+	__le32 link_id;
 	__le16 pwr_restriction;
-	__le16 dev_24;
-	__le16 dev_52_low;
-	__le16 dev_52_high;
-};
+} __packed;
 
 /**
  * struct iwl_dev_tx_power_cmd_v3 - TX power reduction command version 3
@@ -343,7 +329,7 @@ struct iwl_dev_tx_power_cmd_v5 {
 } __packed; /* TX_REDUCED_POWER_API_S_VER_5 */
 
 /**
- * struct iwl_dev_tx_power_cmd_v6 - TX power reduction command version 6
+ * struct iwl_dev_tx_power_cmd_v8 - TX power reduction command version 8
  * @per_chain: per chain restrictions
  * @enable_ack_reduction: enable or disable close range ack TX power
  *	reduction.
@@ -354,43 +340,74 @@ struct iwl_dev_tx_power_cmd_v5 {
  * @reserved: reserved (padding)
  * @timer_period: timer in milliseconds. if expires FW will change to default
  *	BIOS values. relevant if setMode is IWL_TX_POWER_MODE_SET_SAR_TIMER
+ * @flags: reduce power flags.
+ * @tpc_vlp_backoff_level: user backoff of UNII5,7 VLP channels in USA.
+ *	Not in use.
  */
-struct iwl_dev_tx_power_cmd_v6 {
+struct iwl_dev_tx_power_cmd_v8 {
 	__le16 per_chain[IWL_NUM_CHAIN_TABLES_V2][IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V2];
 	u8 enable_ack_reduction;
 	u8 per_chain_restriction_changed;
 	u8 reserved[2];
 	__le32 timer_period;
-} __packed; /* TX_REDUCED_POWER_API_S_VER_6 */
+	__le32 flags;
+	__le32 tpc_vlp_backoff_level;
+} __packed; /* TX_REDUCED_POWER_API_S_VER_8 */
+
+/*
+ * @dev_24: device TX power restriction in 1/8 dBms
+ * @dev_52_low: device TX power restriction upper band - low
+ * @dev_52_high: device TX power restriction upper band - high
+ */
+struct iwl_dev_tx_power_cmd_per_band {
+	__le16 dev_24;
+	__le16 dev_52_low;
+	__le16 dev_52_high;
+} __packed;
 
 /**
- * struct iwl_dev_tx_power_cmd_v7 - TX power reduction command version 7
+ * struct iwl_dev_tx_power_cmd_v3_v8 - TX power reduction command (multiversion)
+ * @per_band: per band restrictions
+ * @common: common part of the command
+ * @v3: version 3 part of the command
+ * @v4: version 4 part of the command
+ * @v5: version 5 part of the command
+ * @v8: version 8 part of the command
+ */
+struct iwl_dev_tx_power_cmd_v3_v8 {
+	struct iwl_dev_tx_power_common common;
+	struct iwl_dev_tx_power_cmd_per_band per_band;
+	union {
+		struct iwl_dev_tx_power_cmd_v3 v3;
+		struct iwl_dev_tx_power_cmd_v4 v4;
+		struct iwl_dev_tx_power_cmd_v5 v5;
+		struct iwl_dev_tx_power_cmd_v8 v8;
+	};
+};
+
+/**
+ * struct iwl_dev_tx_power_cmd_v9 - TX power reduction cmd
+ * @reserved: reserved (padding)
  * @per_chain: per chain restrictions
- * @enable_ack_reduction: enable or disable close range ack TX power
- *	reduction.
  * @per_chain_restriction_changed: is per_chain_restriction has changed
  *	from last command. used if set_mode is
  *	IWL_TX_POWER_MODE_SET_SAR_TIMER.
  *	note: if not changed, the command is used for keep alive only.
- * @reserved: reserved (padding)
+ * @reserved1: reserved (padding)
  * @timer_period: timer in milliseconds. if expires FW will change to default
  *	BIOS values. relevant if setMode is IWL_TX_POWER_MODE_SET_SAR_TIMER
- * @flags: reduce power flags.
  */
-struct iwl_dev_tx_power_cmd_v7 {
-	__le16 per_chain[IWL_NUM_CHAIN_TABLES_V2][IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V2];
-	u8 enable_ack_reduction;
+struct iwl_dev_tx_power_cmd_v9 {
+	__le16 reserved;
+	__le16 per_chain[IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V1];
 	u8 per_chain_restriction_changed;
-	u8 reserved[2];
+	u8 reserved1[3];
 	__le32 timer_period;
-	__le32 flags;
-} __packed; /* TX_REDUCED_POWER_API_S_VER_7 */
+} __packed; /* TX_REDUCED_POWER_API_S_VER_9 */
 
 /**
- * struct iwl_dev_tx_power_cmd_v8 - TX power reduction command version 8
+ * struct iwl_dev_tx_power_cmd_v10 - TX power reduction cmd
  * @per_chain: per chain restrictions
- * @enable_ack_reduction: enable or disable close range ack TX power
- *	reduction.
  * @per_chain_restriction_changed: is per_chain_restriction has changed
  *	from last command. used if set_mode is
  *	IWL_TX_POWER_MODE_SET_SAR_TIMER.
@@ -399,40 +416,28 @@ struct iwl_dev_tx_power_cmd_v7 {
  * @timer_period: timer in milliseconds. if expires FW will change to default
  *	BIOS values. relevant if setMode is IWL_TX_POWER_MODE_SET_SAR_TIMER
  * @flags: reduce power flags.
- * @tpc_vlp_backoff_level: user backoff of UNII5,7 VLP channels in USA.
- *	Not in use.
  */
-struct iwl_dev_tx_power_cmd_v8 {
+struct iwl_dev_tx_power_cmd_v10 {
 	__le16 per_chain[IWL_NUM_CHAIN_TABLES_V2][IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V2];
-	u8 enable_ack_reduction;
 	u8 per_chain_restriction_changed;
-	u8 reserved[2];
+	u8 reserved;
 	__le32 timer_period;
 	__le32 flags;
-	__le32 tpc_vlp_backoff_level;
-} __packed; /* TX_REDUCED_POWER_API_S_VER_8 */
+} __packed; /* TX_REDUCED_POWER_API_S_VER_10 */
 
-/**
+/*
  * struct iwl_dev_tx_power_cmd - TX power reduction command (multiversion)
  * @common: common part of the command
- * @v3: version 3 part of the command
- * @v4: version 4 part of the command
- * @v5: version 5 part of the command
- * @v6: version 6 part of the command
- * @v7: version 7 part of the command
- * @v8: version 8 part of the command
+ * @v9: version 9 part of the command
+ * @v10: version 10 part of the command
  */
 struct iwl_dev_tx_power_cmd {
 	struct iwl_dev_tx_power_common common;
 	union {
-		struct iwl_dev_tx_power_cmd_v3 v3;
-		struct iwl_dev_tx_power_cmd_v4 v4;
-		struct iwl_dev_tx_power_cmd_v5 v5;
-		struct iwl_dev_tx_power_cmd_v6 v6;
-		struct iwl_dev_tx_power_cmd_v7 v7;
-		struct iwl_dev_tx_power_cmd_v8 v8;
+		struct iwl_dev_tx_power_cmd_v9 v9;
+		struct iwl_dev_tx_power_cmd_v10 v10;
 	};
-};
+} __packed; /* TX_REDUCED_POWER_API_S_VER_9_VER10 */
 
 #define IWL_NUM_GEO_PROFILES		3
 #define IWL_NUM_GEO_PROFILES_V3		8
@@ -565,28 +570,37 @@ enum iwl_ppag_flags {
 
 /**
  * union iwl_ppag_table_cmd - union for all versions of PPAG command
- * @v1: version 1
- * @v2: version 2
- * version 3, 4, 5 and 6 are the same structure as v2,
+ * @v1: command version 1 structure.
+ * @v2: command version from 2 to 6 are same structure as v2.
  *	but has a different format of the flags bitmap
+ * @v3: command version 7 structure.
  * @v1.flags: values from &enum iwl_ppag_flags
  * @v1.gain: table of antenna gain values per chain and sub-band
  * @v1.reserved: reserved
  * @v2.flags: values from &enum iwl_ppag_flags
  * @v2.gain: table of antenna gain values per chain and sub-band
- * @v2.reserved: reserved
+ * @v3.ppag_config_info: see @struct bios_value_u32
+ * @v3.gain: table of antenna gain values per chain and sub-band
+ * @v3.reserved: reserved
  */
 union iwl_ppag_table_cmd {
 	struct {
 		__le32 flags;
 		s8 gain[IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V1];
 		s8 reserved[2];
-	} v1;
+	} __packed v1; /* PER_PLAT_ANTENNA_GAIN_CMD_API_S_VER_1 */
 	struct {
 		__le32 flags;
 		s8 gain[IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V2];
 		s8 reserved[2];
-	} v2;
+	} __packed v2; /* PER_PLAT_ANTENNA_GAIN_CMD_API_S_VER_2, VER3, VER4,
+			* VER5, VER6
+			*/
+	struct {
+		struct bios_value_u32 ppag_config_info;
+		s8 gain[IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V2];
+		s8 reserved[2];
+	} __packed v3; /* PER_PLAT_ANTENNA_GAIN_CMD_API_S_VER_7 */
 } __packed;
 
 #define IWL_PPAG_CMD_V4_MASK (IWL_PPAG_ETSI_MASK | IWL_PPAG_CHINA_MASK)
@@ -594,6 +608,15 @@ union iwl_ppag_table_cmd {
 			      IWL_PPAG_ETSI_LPI_UHB_MASK | \
 			      IWL_PPAG_USA_LPI_UHB_MASK)
 
+#define IWL_PPAG_CMD_V6_MASK (IWL_PPAG_CMD_V5_MASK |		\
+			      IWL_PPAG_ETSI_VLP_UHB_MASK |	\
+			      IWL_PPAG_ETSI_SP_UHB_MASK |	\
+			      IWL_PPAG_USA_VLP_UHB_MASK |	\
+			      IWL_PPAG_USA_SP_UHB_MASK |	\
+			      IWL_PPAG_CANADA_LPI_UHB_MASK |	\
+			      IWL_PPAG_CANADA_VLP_UHB_MASK |	\
+			      IWL_PPAG_CANADA_SP_UHB_MASK)
+
 #define MCC_TO_SAR_OFFSET_TABLE_ROW_SIZE	26
 #define MCC_TO_SAR_OFFSET_TABLE_COL_SIZE	13
 
@@ -627,7 +650,7 @@ struct iwl_sar_offset_mapping_cmd {
  *      Roaming Energy Delta Threshold, otherwise use normal Energy Delta
  *      Threshold. Typical energy threshold is -72dBm.
  * @bf_temp_threshold: This threshold determines the type of temperature
- *	filtering (Slow or Fast) that is selected (Units are in Celsuis):
+ *	filtering (Slow or Fast) that is selected (Units are in Celsius):
  *	If the current temperature is above this threshold - Fast filter
  *	will be used, If the current temperature is below this threshold -
  *	Slow filter will be used.
@@ -635,12 +658,12 @@ struct iwl_sar_offset_mapping_cmd {
  *      calculated for this and the last passed beacon is greater than this
  *      threshold. Zero value means that the temperature change is ignored for
  *      beacon filtering; beacons will not be  forced to be sent to driver
- *      regardless of whether its temerature has been changed.
+ *      regardless of whether its temperature has been changed.
  * @bf_temp_slow_filter: Send Beacon to driver if delta in temperature values
  *      calculated for this and the last passed beacon is greater than this
  *      threshold. Zero value means that the temperature change is ignored for
  *      beacon filtering; beacons will not be forced to be sent to driver
- *      regardless of whether its temerature has been changed.
+ *      regardless of whether its temperature has been changed.
  * @bf_enable_beacon_filter: 1, beacon filtering is enabled; 0, disabled.
  * @bf_debug_flag: beacon filtering debug configuration
  * @bf_escape_timer: Send beacons to to driver if no beacons were passed
diff --git a/sys/contrib/dev/iwlwifi/fw/api/rs.h b/sys/contrib/dev/iwlwifi/fw/api/rs.h
index 1a60f0cdf972..3222cbcbe1ab 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/rs.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/rs.h
@@ -1,11 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2022, 2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022, 2024-2025 Intel Corporation
  * Copyright (C) 2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_fw_api_rs_h__
 #define __iwl_fw_api_rs_h__
-
+#include <linux/bitfield.h>
+#include <linux/types.h>
+#include <linux/bits.h>
 #include "mac.h"
 
 /**
@@ -213,7 +215,8 @@ enum iwl_tlc_update_flags {
  * @sta_id: station id
  * @reserved: reserved
  * @flags: bitmap of notifications reported
- * @rate: current initial rate
+ * @rate: current initial rate, format depends on the notification
+ *	version
  * @amsdu_size: Max AMSDU size, in bytes
  * @amsdu_enabled: bitmap for per-TID AMSDU enablement
  */
@@ -224,8 +227,60 @@ struct iwl_tlc_update_notif {
 	__le32 rate;
 	__le32 amsdu_size;
 	__le32 amsdu_enabled;
-} __packed; /* TLC_MNG_UPDATE_NTFY_API_S_VER_2 */
+} __packed; /* TLC_MNG_UPDATE_NTFY_API_S_VER_2, _VER_3, _VER_4 */
 
+/**
+ * enum iwl_tlc_debug_types - debug options
+ */
+enum iwl_tlc_debug_types {
+	/**
+	 *  @IWL_TLC_DEBUG_FIXED_RATE: set fixed rate for rate scaling
+	 */
+	IWL_TLC_DEBUG_FIXED_RATE,
+	/**
+	 * @IWL_TLC_DEBUG_AGG_DURATION_LIM: time limit for a BA
+	 * session, in usec
+	 */
+	IWL_TLC_DEBUG_AGG_DURATION_LIM,
+	/**
+	 * @IWL_TLC_DEBUG_AGG_FRAME_CNT_LIM: set max number of frames
+	 * in an aggregation
+	 */
+	IWL_TLC_DEBUG_AGG_FRAME_CNT_LIM,
+	/**
+	 * @IWL_TLC_DEBUG_TPC_ENABLED: enable or disable tpc
+	 */
+	IWL_TLC_DEBUG_TPC_ENABLED,
+	/**
+	 * @IWL_TLC_DEBUG_TPC_STATS: get number of frames Tx'ed in each
+	 * tpc step
+	 */
+	IWL_TLC_DEBUG_TPC_STATS,
+	/**
+	 * @IWL_TLC_DEBUG_RTS_DISABLE: disable RTS (bool true/false).
+	 */
+	IWL_TLC_DEBUG_RTS_DISABLE,
+	/**
+	 * @IWL_TLC_DEBUG_PARTIAL_FIXED_RATE: set partial fixed rate to fw
+	 */
+	IWL_TLC_DEBUG_PARTIAL_FIXED_RATE,
+}; /* TLC_MNG_DEBUG_TYPES_API_E */
+
+#define MAX_DATA_IN_DHC_TLC_CMD 10
+
+/**
+ * struct iwl_dhc_tlc_cmd - fixed debug config
+ * @sta_id: bit 0 - enable/disable, bits 1 - 7 hold station id
+ * @reserved1: reserved
+ * @type: type id of %enum iwl_tlc_debug_types
+ * @data: data to send
+ */
+struct iwl_dhc_tlc_cmd {
+	u8 sta_id;
+	u8 reserved1[3];
+	__le32 type;
+	__le32 data[MAX_DATA_IN_DHC_TLC_CMD];
+} __packed; /* TLC_MNG_DEBUG_CMD_S */
 
 #define IWL_MAX_MCS_DISPLAY_SIZE        12
 
@@ -375,6 +430,7 @@ enum {
 
 /* Bit 4-5: (0) SISO, (1) MIMO2 (2) MIMO3 */
 #define RATE_VHT_MCS_RATE_CODE_MSK	0xf
+#define RATE_VHT_MCS_NSS_MSK		0x30
 
 /*
  * Legacy OFDM rate format for bits 7:0
@@ -489,7 +545,7 @@ enum {
 #define RATE_MCS_CTS_REQUIRED_POS  (31)
 #define RATE_MCS_CTS_REQUIRED_MSK  (0x1 << RATE_MCS_CTS_REQUIRED_POS)
 
-/* rate_n_flags bit field version 2
+/* rate_n_flags bit field version 2 and 3
  *
  * The 32-bit value has different layouts in the low 8 bits depending on the
  * format. There are three formats, HT, VHT and legacy (11abg, with subformats
@@ -501,23 +557,25 @@ enum {
  * (0) Legacy CCK (1) Legacy OFDM (2) High-throughput (HT)
  * (3) Very High-throughput (VHT) (4) High-efficiency (HE)
  * (5) Extremely High-throughput (EHT)
+ * (6) Ultra High Reliability (UHR) (v3 rate format only)
  */
 #define RATE_MCS_MOD_TYPE_POS		8
 #define RATE_MCS_MOD_TYPE_MSK		(0x7 << RATE_MCS_MOD_TYPE_POS)
-#define RATE_MCS_CCK_MSK		(0 << RATE_MCS_MOD_TYPE_POS)
-#define RATE_MCS_LEGACY_OFDM_MSK	(1 << RATE_MCS_MOD_TYPE_POS)
-#define RATE_MCS_HT_MSK			(2 << RATE_MCS_MOD_TYPE_POS)
-#define RATE_MCS_VHT_MSK		(3 << RATE_MCS_MOD_TYPE_POS)
-#define RATE_MCS_HE_MSK			(4 << RATE_MCS_MOD_TYPE_POS)
-#define RATE_MCS_EHT_MSK		(5 << RATE_MCS_MOD_TYPE_POS)
+#define RATE_MCS_MOD_TYPE_CCK		(0 << RATE_MCS_MOD_TYPE_POS)
+#define RATE_MCS_MOD_TYPE_LEGACY_OFDM	(1 << RATE_MCS_MOD_TYPE_POS)
+#define RATE_MCS_MOD_TYPE_HT		(2 << RATE_MCS_MOD_TYPE_POS)
+#define RATE_MCS_MOD_TYPE_VHT		(3 << RATE_MCS_MOD_TYPE_POS)
+#define RATE_MCS_MOD_TYPE_HE		(4 << RATE_MCS_MOD_TYPE_POS)
+#define RATE_MCS_MOD_TYPE_EHT		(5 << RATE_MCS_MOD_TYPE_POS)
+#define RATE_MCS_MOD_TYPE_UHR		(6 << RATE_MCS_MOD_TYPE_POS)
 
 /*
  * Legacy CCK rate format for bits 0:3:
  *
- * (0) 0xa - 1 Mbps
- * (1) 0x14 - 2 Mbps
- * (2) 0x37 - 5.5 Mbps
- * (3) 0x6e - 11 nbps
+ * (0) 1 Mbps
+ * (1) 2 Mbps
+ * (2) 5.5 Mbps
+ * (3) 11 Mbps
  *
  * Legacy OFDM rate format for bis 3:0:
  *
@@ -534,15 +592,19 @@ enum {
 #define RATE_LEGACY_RATE_MSK		0x7
 
 /*
- * HT, VHT, HE, EHT rate format for bits 3:0
- * 3-0: MCS
- *
+ * HT, VHT, HE, EHT, UHR rate format
+ * Version 2: (not applicable for UHR)
+ *   3-0: MCS
+ *   4: NSS==2 indicator
+ * Version 3:
+ *   4-0: MCS
+ *   5: NSS==2 indicator
  */
 #define RATE_HT_MCS_CODE_MSK		0x7
-#define RATE_MCS_NSS_POS		4
-#define RATE_MCS_NSS_MSK		(1 << RATE_MCS_NSS_POS)
-#define RATE_MCS_CODE_MSK		0xf
-#define RATE_HT_MCS_INDEX(r)		((((r) & RATE_MCS_NSS_MSK) >> 1) | \
+#define RATE_MCS_NSS_MSK_V2		0x10
+#define RATE_MCS_NSS_MSK		0x20
+#define RATE_MCS_CODE_MSK		0x1f
+#define RATE_HT_MCS_INDEX(r)		((((r) & RATE_MCS_NSS_MSK) >> 2) | \
 					 ((r) & RATE_HT_MCS_CODE_MSK))
 
 /* Bits 7-5: reserved */
@@ -758,11 +820,38 @@ struct iwl_lq_cmd {
 }; /* LINK_QUALITY_CMD_API_S_VER_1 */
 
 u8 iwl_fw_rate_idx_to_plcp(int idx);
-u32 iwl_new_rate_from_v1(u32 rate_v1);
 const struct iwl_rate_mcs_info *iwl_rate_mcs(int idx);
 const char *iwl_rs_pretty_ant(u8 ant);
 const char *iwl_rs_pretty_bw(int bw);
 int rs_pretty_print_rate(char *buf, int bufsz, const u32 rate);
 bool iwl_he_is_sgi(u32 rate_n_flags);
 
+static inline u32 iwl_v3_rate_from_v2_v3(__le32 rate, bool fw_v3)
+{
+	u32 val;
+
+	if (fw_v3)
+		return le32_to_cpu(rate);
+
+	val = le32_to_cpu(rate) & ~RATE_MCS_NSS_MSK_V2;
+	val |= u32_encode_bits(le32_get_bits(rate, RATE_MCS_NSS_MSK_V2),
+			       RATE_MCS_NSS_MSK);
+
+	return val;
+}
+
+static inline __le32 iwl_v3_rate_to_v2_v3(u32 rate, bool fw_v3)
+{
+	__le32 val;
+
+	if (fw_v3)
+		return cpu_to_le32(rate);
+
+	val = cpu_to_le32(rate & ~RATE_MCS_NSS_MSK);
+	val |= le32_encode_bits(u32_get_bits(rate, RATE_MCS_NSS_MSK),
+				RATE_MCS_NSS_MSK_V2);
+
+	return val;
+}
+
 #endif /* __iwl_fw_api_rs_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/rx.h b/sys/contrib/dev/iwlwifi/fw/api/rx.h
index ec89e9576d2e..2024b11c80d0 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/rx.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/rx.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -193,10 +193,13 @@ enum iwl_rx_mpdu_amsdu_info {
 	IWL_RX_MPDU_AMSDU_LAST_SUBFRAME		= 0x80,
 };
 
-#define RX_MPDU_BAND_POS 6
-#define RX_MPDU_BAND_MASK 0xC0
-#define BAND_IN_RX_STATUS(_val) \
-	(((_val) & RX_MPDU_BAND_MASK) >> RX_MPDU_BAND_POS)
+enum iwl_rx_mpdu_mac_phy_band {
+	/* whether or not this is MAC or LINK depends on the API */
+	IWL_RX_MPDU_MAC_PHY_BAND_MAC_MASK	= 0x0f,
+	IWL_RX_MPDU_MAC_PHY_BAND_LINK_MASK	= 0x0f,
+	IWL_RX_MPDU_MAC_PHY_BAND_PHY_MASK	= 0x30,
+	IWL_RX_MPDU_MAC_PHY_BAND_BAND_MASK	= 0xc0,
+};
 
 enum iwl_rx_l3_proto_values {
 	IWL_RX_L3_TYPE_NONE,
@@ -642,7 +645,9 @@ struct iwl_rx_mpdu_desc_v3 {
 	 */
 	__le32 reserved[1];
 } __packed; /* RX_MPDU_RES_START_API_S_VER_3,
-	       RX_MPDU_RES_START_API_S_VER_5 */
+	     * RX_MPDU_RES_START_API_S_VER_5,
+	     * RX_MPDU_RES_START_API_S_VER_6
+	     */
 
 /**
  * struct iwl_rx_mpdu_desc - RX MPDU descriptor
@@ -671,9 +676,10 @@ struct iwl_rx_mpdu_desc {
 	 */
 	__le16 phy_info;
 	/**
-	 * @mac_phy_idx: MAC/PHY index
+	 * @mac_phy_band: MAC/link ID, PHY ID, band;
+	 *	see &enum iwl_rx_mpdu_mac_phy_band
 	 */
-	u8 mac_phy_idx;
+	u8 mac_phy_band;
 	/* DW4 */
 	union {
 		struct {
@@ -725,8 +731,10 @@ struct iwl_rx_mpdu_desc {
 		struct iwl_rx_mpdu_desc_v3 v3;
 	};
 } __packed; /* RX_MPDU_RES_START_API_S_VER_3,
-	       RX_MPDU_RES_START_API_S_VER_4,
-	       RX_MPDU_RES_START_API_S_VER_5 */
+	     * RX_MPDU_RES_START_API_S_VER_4,
+	     * RX_MPDU_RES_START_API_S_VER_5,
+	     * RX_MPDU_RES_START_API_S_VER_6
+	     */
 
 #define IWL_RX_DESC_SIZE_V1 offsetofend(struct iwl_rx_mpdu_desc, v1)
 
@@ -822,7 +830,7 @@ struct iwl_rx_no_data {
  *	15:8 chain-B, measured at FINA time (FINA_ENERGY), 16:23 channel
  * @on_air_rise_time: GP2 during on air rise
  * @fr_time: frame time
- * @rate: rate/mcs of frame
+ * @rate: rate/mcs of frame, format depends on the notification version
  * @phy_info: &enum iwl_rx_phy_eht_data0 and &enum iwl_rx_phy_info_type
  * @rx_vec: DW-12:9 raw RX vectors from DSP according to modulation type.
  *	for VHT: OFDM_RX_VECTOR_SIGA1_OUT, OFDM_RX_VECTOR_SIGA2_OUT
@@ -838,9 +846,7 @@ struct iwl_rx_no_data_ver_3 {
 	__le32 rate;
 	__le32 phy_info[2];
 	__le32 rx_vec[4];
-} __packed; /* RX_NO_DATA_NTFY_API_S_VER_1,
-	       RX_NO_DATA_NTFY_API_S_VER_2
-	       RX_NO_DATA_NTFY_API_S_VER_3 */
+} __packed; /* RX_NO_DATA_NTFY_API_S_VER_3, _VER_4 */
 
 struct iwl_frame_release {
 	u8 baid;
@@ -1030,4 +1036,24 @@ struct iwl_rfh_queue_config {
 #endif
 } __packed; /* RFH_QUEUE_CONFIG_API_S_VER_1 */
 
+/**
+ * struct iwl_beacon_filter_notif_v1 - beacon filter notification
+ * @average_energy: average energy for the received beacon
+ * @mac_id: MAC ID the beacon was received for
+ */
+struct iwl_beacon_filter_notif_v1 {
+	__le32 average_energy;
+	__le32 mac_id;
+} __packed; /* BEACON_FILTER_IN_NTFY_API_S_VER_1 */
+
+/**
+ * struct iwl_beacon_filter_notif - beacon filter notification
+ * @average_energy: average energy for the received beacon
+ * @link_id: link ID the beacon was received for
+ */
+struct iwl_beacon_filter_notif {
+	__le32 average_energy;
+	__le32 link_id;
+} __packed; /* BEACON_FILTER_IN_NTFY_API_S_VER_2 */
+
 #endif /* __iwl_fw_api_rx_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/scan.h b/sys/contrib/dev/iwlwifi/fw/api/scan.h
index 8598031567bb..f486d624500b 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/scan.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/scan.h
@@ -731,39 +731,46 @@ enum iwl_umac_scan_general_params_flags2 {
  * struct iwl_scan_channel_cfg_umac
  * @flags:		bitmap - 0-19:	directed scan to i'th ssid.
  * @channel_num:	channel number 1-13 etc.
- * @band:		band of channel: 0 for 2GHz, 1 for 5GHz
- * @iter_count:		repetition count for the channel.
- * @iter_interval:	interval between two scan iterations on one channel.
+ * @v1:			command version 1
+ * @v1.iter_count:	repetition count for the channel.
+ * @v1.iter_interval:	interval between two scan iterations on one channel.
+ * @v2:			command versions 2-4
+ * @v2.band:		band of channel: 0 for 2GHz, 1 for 5GHz
+ * @v2.iter_count:	repetition count for the channel.
+ * @v2.iter_interval:	interval between two scan iterations on one channel.
+ * @v5:			command versions 5 and up
+ * @v5.iter_count:	repetition count for the channel.
+ * @v5.iter_interval:	interval between two scan iterations on one channel.
+ * @v5.psd_20:		highest PSD value for all APs known so far
+ *			on this channel.
  */
 struct  iwl_scan_channel_cfg_umac {
 #define IWL_CHAN_CFG_FLAGS_BAND_POS 30
 	__le32 flags;
+	u8 channel_num;
 
 	/* All versions are of the same size, so use a union without adjusting
 	 * the command size later
 	 */
 	union {
 		struct {
-			u8 channel_num;
 			u8 iter_count;
 			__le16 iter_interval;
-		} v1;  /* SCAN_CHANNEL_CONFIG_API_S_VER_1 */
+		} __packed v1;  /* SCAN_CHANNEL_CONFIG_API_S_VER_1 */
 		struct {
-			u8 channel_num;
 			u8 band;
 			u8 iter_count;
 			u8 iter_interval;
-		 } v2; /* SCAN_CHANNEL_CONFIG_API_S_VER_2
-			* SCAN_CHANNEL_CONFIG_API_S_VER_3
-			* SCAN_CHANNEL_CONFIG_API_S_VER_4
-			*/
+		} __packed v2; /* SCAN_CHANNEL_CONFIG_API_S_VER_2
+				* SCAN_CHANNEL_CONFIG_API_S_VER_3
+				* SCAN_CHANNEL_CONFIG_API_S_VER_4
+				*/
 		struct {
-			u8 channel_num;
 			u8 psd_20;
 			u8 iter_count;
 			u8 iter_interval;
-		} v5;  /* SCAN_CHANNEL_CONFIG_API_S_VER_5 */
-	};
+		} __packed v5;  /* SCAN_CHANNEL_CONFIG_API_S_VER_5 */
+	} __packed;
 } __packed;
 
 /**
@@ -1133,6 +1140,19 @@ struct iwl_umac_scan_abort {
 } __packed; /* SCAN_ABORT_CMD_UMAC_API_S_VER_1 */
 
 /**
+ * enum iwl_umac_scan_abort_status
+ *
+ * @IWL_UMAC_SCAN_ABORT_STATUS_SUCCESS: scan was successfully aborted
+ * @IWL_UMAC_SCAN_ABORT_STATUS_IN_PROGRESS: scan abort is in progress
+ * @IWL_UMAC_SCAN_ABORT_STATUS_NOT_FOUND: nothing to abort
+ */
+enum iwl_umac_scan_abort_status {
+	IWL_UMAC_SCAN_ABORT_STATUS_SUCCESS = 0,
+	IWL_UMAC_SCAN_ABORT_STATUS_IN_PROGRESS,
+	IWL_UMAC_SCAN_ABORT_STATUS_NOT_FOUND,
+};
+
+/**
  * struct iwl_umac_scan_complete
  * @uid: scan id, &enum iwl_umac_scan_uid_offsets
  * @last_schedule: last scheduling line
diff --git a/sys/contrib/dev/iwlwifi/fw/api/sta.h b/sys/contrib/dev/iwlwifi/fw/api/sta.h
index 893049edb4ae..b2a3fdb07280 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/sta.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/sta.h
@@ -191,6 +191,7 @@ enum iwl_sta_sleep_flag {
 #define STA_KEY_IDX_INVALID (0xff)
 #define STA_KEY_MAX_DATA_KEY_NUM (4)
 #define IWL_MAX_GLOBAL_KEYS (4)
+#define IWL_MAX_NUM_IGTKS 2
 #define STA_KEY_LEN_WEP40 (5)
 #define STA_KEY_LEN_WEP104 (13)
 
diff --git a/sys/contrib/dev/iwlwifi/fw/api/stats.h b/sys/contrib/dev/iwlwifi/fw/api/stats.h
index 2271b19213fa..00713a991879 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/stats.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/stats.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018, 2020 - 2021, 2023 Intel Corporation
+ * Copyright (C) 2012-2014, 2018, 2020-2021, 2023-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -327,14 +327,14 @@ struct mvm_statistics_load {
 	__le32 air_time[MAC_INDEX_AUX];
 	__le32 byte_count[MAC_INDEX_AUX];
 	__le32 pkt_count[MAC_INDEX_AUX];
-	u8 avg_energy[IWL_MVM_STATION_COUNT_MAX];
+	u8 avg_energy[IWL_STATION_COUNT_MAX];
 } __packed; /* STATISTICS_RX_MAC_STATION_S_VER_3 */
 
 struct mvm_statistics_load_v1 {
 	__le32 air_time[NUM_MAC_INDEX];
 	__le32 byte_count[NUM_MAC_INDEX];
 	__le32 pkt_count[NUM_MAC_INDEX];
-	u8 avg_energy[IWL_MVM_STATION_COUNT_MAX];
+	u8 avg_energy[IWL_STATION_COUNT_MAX];
 } __packed; /* STATISTICS_RX_MAC_STATION_S_VER_1 */
 
 struct mvm_statistics_rx {
@@ -584,6 +584,9 @@ struct iwl_stats_ntfy_per_phy {
 	__le32 last_tx_ch_width_indx;
 } __packed; /* STATISTICS_NTFY_PER_PHY_API_S_VER_1 */
 
+/* unknown channel load (due to not being active on channel) */
+#define IWL_STATS_UNKNOWN_CHANNEL_LOAD	0xffffffff
+
 /**
  * struct iwl_stats_ntfy_per_sta
  *
@@ -594,7 +597,7 @@ struct iwl_stats_ntfy_per_sta {
 } __packed; /* STATISTICS_NTFY_PER_STA_API_S_VER_1 */
 
 #define IWL_STATS_MAX_PHY_OPERATIONAL 3
-#define IWL_STATS_MAX_FW_LINKS	(IWL_MVM_FW_MAX_LINK_ID + 1)
+#define IWL_STATS_MAX_FW_LINKS	(IWL_FW_MAX_LINK_ID + 1)
 
 /**
  * struct iwl_system_statistics_notif_oper
@@ -608,7 +611,7 @@ struct iwl_system_statistics_notif_oper {
 	__le32 time_stamp;
 	struct iwl_stats_ntfy_per_link per_link[IWL_STATS_MAX_FW_LINKS];
 	struct iwl_stats_ntfy_per_phy per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
-	struct iwl_stats_ntfy_per_sta per_sta[IWL_MVM_STATION_COUNT_MAX];
+	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
 } __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_3 */
 
 /**
@@ -651,7 +654,7 @@ struct iwl_statistics_operational_ntfy {
 	__le32 flags;
 	struct iwl_stats_ntfy_per_mac per_mac[MAC_INDEX_AUX];
 	struct iwl_stats_ntfy_per_phy per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
-	struct iwl_stats_ntfy_per_sta per_sta[IWL_MVM_STATION_COUNT_MAX];
+	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
 	__le64 rx_time;
 	__le64 tx_time;
 	__le64 on_time_rf;
@@ -699,7 +702,7 @@ struct iwl_statistics_operational_ntfy_ver_14 {
 	__le64 tx_time;
 	__le64 on_time_rf;
 	__le64 on_time_scan;
-	__le32 average_energy[IWL_MVM_STATION_COUNT_MAX];
+	__le32 average_energy[IWL_STATION_COUNT_MAX];
 	__le32 reserved;
 } __packed; /* STATISTICS_OPERATIONAL_NTFY_API_S_VER_14 */
 
diff --git a/sys/contrib/dev/iwlwifi/fw/api/tdls.h b/sys/contrib/dev/iwlwifi/fw/api/tdls.h
index 893438aadab0..08edd1d99992 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/tdls.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/tdls.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018 Intel Corporation
+ * Copyright (C) 2012-2014, 2018, 2024-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -10,7 +10,7 @@
 #include "fw/api/tx.h"
 #include "fw/api/phy-ctxt.h"
 
-#define IWL_MVM_TDLS_STA_COUNT	4
+#define IWL_TDLS_STA_COUNT	4
 
 /* Type of TDLS request */
 enum iwl_tdls_channel_switch_type {
@@ -50,7 +50,7 @@ struct iwl_tdls_channel_switch_timing {
  */
 struct iwl_tdls_channel_switch_frame {
 	__le32 switch_time_offset;
-	struct iwl_tx_cmd tx_cmd;
+	struct iwl_tx_cmd_v6_params tx_cmd;
 	u8 data[IWL_TDLS_CH_SW_FRAME_MAX_SIZE];
 } __packed; /* TDLS_STA_CHANNEL_SWITCH_FRAME_API_S_VER_1 */
 
@@ -128,10 +128,10 @@ struct iwl_tdls_config_cmd {
 	u8 tdls_peer_count;
 	u8 tx_to_ap_tid;
 	__le16 tx_to_ap_ssn;
-	struct iwl_tdls_sta_info sta_info[IWL_MVM_TDLS_STA_COUNT];
+	struct iwl_tdls_sta_info sta_info[IWL_TDLS_STA_COUNT];
 
 	__le32 pti_req_data_offset;
-	struct iwl_tx_cmd pti_req_tx_cmd;
+	struct iwl_tx_cmd_v6_params pti_req_tx_cmd;
 	u8 pti_req_template[];
 } __packed; /* TDLS_CONFIG_CMD_API_S_VER_1 */
 
@@ -155,7 +155,7 @@ struct iwl_tdls_config_sta_info_res {
  */
 struct iwl_tdls_config_res {
 	__le32 tx_to_ap_last_seq;
-	struct iwl_tdls_config_sta_info_res sta_info[IWL_MVM_TDLS_STA_COUNT];
+	struct iwl_tdls_config_sta_info_res sta_info[IWL_TDLS_STA_COUNT];
 } __packed; /* TDLS_CONFIG_RSP_API_S_VER_1 */
 
 #endif /* __iwl_fw_api_tdls_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/api/time-event.h b/sys/contrib/dev/iwlwifi/fw/api/time-event.h
index f4b827b58bd3..46d35ef4751e 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/time-event.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/time-event.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2020, 2022-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2020, 2022-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -351,7 +351,7 @@ enum iwl_roc_activity {
 }; /* ROC_ACTIVITY_API_E_VER_1 */
 
 /*
- * ROC command
+ * ROC command v5
  *
  * Command requests the firmware to remain on a channel for a certain duration.
  *
@@ -366,7 +366,7 @@ enum iwl_roc_activity {
  * @max_delay: max delay the ROC can start in TU
  * @duration: remain on channel duration in TU
  */
-struct iwl_roc_req {
+struct iwl_roc_req_v5 {
 	__le32 action;
 	__le32 activity;
 	__le32 sta_id;
@@ -375,7 +375,41 @@ struct iwl_roc_req {
 	__le16 reserved;
 	__le32 max_delay;
 	__le32 duration;
-} __packed; /* ROC_CMD_API_S_VER_3 */
+} __packed; /* ROC_CMD_API_S_VER_5 */
+
+/*
+ * ROC command
+ *
+ * Command requests the firmware to remain on a channel for a certain duration.
+ *
+ * ( MAC_CONF_GROUP 0x3, ROC_CMD 0xE )
+ *
+ * @action: action to perform, see &enum iwl_ctxt_action
+ * @activity: type of activity, see &enum iwl_roc_activity
+ * @sta_id: station id, resumed during "Remain On Channel" activity.
+ * @channel_info: &struct iwl_fw_channel_info
+ * @node_addr: node MAC address for Rx filtering
+ * @reserved1: align to a dword
+ * @max_delay: max delay the ROC can start in TU
+ * @duration: remain on channel duration in TU
+ * @interval: interval between repetitions (when repetitions > 1).
+ * @repetitions: number of repetitions
+ *	0xFF: infinite repetitions. 0 or 1: single repetition.
+ * @reserved2: align to a dword
+ */
+struct iwl_roc_req {
+	__le32 action;
+	__le32 activity;
+	__le32 sta_id;
+	struct iwl_fw_channel_info channel_info;
+	u8 node_addr[ETH_ALEN];
+	__le16 reserved1;
+	__le32 max_delay;
+	__le32 duration;
+	__le32 interval;
+	u8 repetitions;
+	u8 reserved2[3];
+} __packed; /* ROC_CMD_API_S_VER_6 */
 
 /*
  * ROC notification
@@ -395,7 +429,7 @@ struct iwl_roc_notif {
 } __packed; /* ROC_NOTIF_API_S_VER_1 */
 
 /**
- * enum iwl_mvm_session_prot_conf_id - session protection's configurations
+ * enum iwl_session_prot_conf_id - session protection's configurations
  * @SESSION_PROTECT_CONF_ASSOC: Start a session protection for association.
  *	The firmware will allocate two events.
  *	Valid for BSS_STA and P2P_STA.
@@ -418,13 +452,13 @@ struct iwl_roc_notif {
  *	listen mode. Will be fragmented. Valid only on the P2P Device MAC.
  *	Valid only on the P2P Device MAC. The firmware will take into account
  *	the duration, the interval and the repetition count.
- * @SESSION_PROTECT_CONF_P2P_GO_NEGOTIATION: Schedule the P2P Device to be be
+ * @SESSION_PROTECT_CONF_P2P_GO_NEGOTIATION: Schedule the P2P Device to be
  *	able to run the GO Negotiation. Will not be fragmented and not
  *	repetitive. Valid only on the P2P Device MAC. Only the duration will
  *	be taken into account.
  * @SESSION_PROTECT_CONF_MAX_ID: not used
  */
-enum iwl_mvm_session_prot_conf_id {
+enum iwl_session_prot_conf_id {
 	SESSION_PROTECT_CONF_ASSOC,
 	SESSION_PROTECT_CONF_GO_CLIENT_ASSOC,
 	SESSION_PROTECT_CONF_P2P_DEVICE_DISCOV,
@@ -433,12 +467,12 @@ enum iwl_mvm_session_prot_conf_id {
 }; /* SESSION_PROTECTION_CONF_ID_E_VER_1 */
 
 /**
- * struct iwl_mvm_session_prot_cmd - configure a session protection
+ * struct iwl_session_prot_cmd - configure a session protection
  * @id_and_color: the id and color of the link (or mac, for command version 1)
  *	for which this session protection is sent
  * @action: can be either FW_CTXT_ACTION_ADD or FW_CTXT_ACTION_REMOVE,
  *	see &enum iwl_ctxt_action
- * @conf_id: see &enum iwl_mvm_session_prot_conf_id
+ * @conf_id: see &enum iwl_session_prot_conf_id
  * @duration_tu: the duration of the whole protection in TUs.
  * @repetition_count: not used
  * @interval: not used
@@ -448,7 +482,7 @@ enum iwl_mvm_session_prot_conf_id {
  * The firmware supports only one concurrent session protection per vif.
  * Adding a new session protection will remove any currently running session.
  */
-struct iwl_mvm_session_prot_cmd {
+struct iwl_session_prot_cmd {
 	/* COMMON_INDEX_HDR_API_S_VER_1 hdr */
 	__le32 id_and_color;
 	__le32 action;
@@ -462,17 +496,17 @@ struct iwl_mvm_session_prot_cmd {
  */
 
 /**
- * struct iwl_mvm_session_prot_notif - session protection started / ended
+ * struct iwl_session_prot_notif - session protection started / ended
  * @mac_link_id: the mac id (or link id, for notif ver > 2) for which the
  *	session protection started / ended
  * @status: 1 means success, 0 means failure
  * @start: 1 means the session protection started, 0 means it ended
- * @conf_id: see &enum iwl_mvm_session_prot_conf_id
+ * @conf_id: see &enum iwl_session_prot_conf_id
  *
  * Note that any session protection will always get two notifications: start
  * and end even the firmware could not schedule it.
  */
-struct iwl_mvm_session_prot_notif {
+struct iwl_session_prot_notif {
 	__le32 mac_link_id;
 	__le32 status;
 	__le32 start;
diff --git a/sys/contrib/dev/iwlwifi/fw/api/tx.h b/sys/contrib/dev/iwlwifi/fw/api/tx.h
index c5277e2f8cd4..26d2013905ed 100644
--- a/sys/contrib/dev/iwlwifi/fw/api/tx.h
+++ b/sys/contrib/dev/iwlwifi/fw/api/tx.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_fw_api_tx_h__
@@ -151,7 +151,7 @@ enum iwl_tx_cmd_sec_ctrl {
 #define IWL_LOW_RETRY_LIMIT			7
 
 /**
- * enum iwl_tx_offload_assist_flags_pos -  set %iwl_tx_cmd offload_assist values
+ * enum iwl_tx_offload_assist_flags_pos -  set %iwl_tx_cmd_v6 offload_assist values
  * @TX_CMD_OFFLD_IP_HDR: offset to start of IP header (in words)
  *	from mac header end. For normal case it is 4 words for SNAP.
  *	note: tx_cmd, mac header and pad are not counted in the offset.
@@ -181,8 +181,8 @@ enum iwl_tx_offload_assist_flags_pos {
 
 /* TODO: complete documentation for try_cnt and btkill_cnt */
 /**
- * struct iwl_tx_cmd - TX command struct to FW
- * ( TX_CMD = 0x1c )
+ * struct iwl_tx_cmd_v6_params - parameters of the TX
+ *
  * @len: in bytes of the payload, see below for details
  * @offload_assist: TX offload configuration
  * @tx_flags: combination of TX_CMD_FLG_*, see &enum iwl_tx_flags
@@ -191,7 +191,7 @@ enum iwl_tx_offload_assist_flags_pos {
  *	cleared. Combination of RATE_MCS_*
  * @sta_id: index of destination station in FW station table
  * @sec_ctl: security control, TX_CMD_SEC_*
- * @initial_rate_index: index into the the rate table for initial TX attempt.
+ * @initial_rate_index: index into the rate table for initial TX attempt.
  *	Applied if TX_CMD_FLG_STA_RATE_MSK is set, normally 0 for data frames.
  * @reserved2: reserved
  * @key: security key
@@ -205,8 +205,6 @@ enum iwl_tx_offload_assist_flags_pos {
  * @tid_tspec: TID/tspec
  * @pm_frame_timeout: PM TX frame timeout
  * @reserved4: reserved
- * @payload: payload (same as @hdr)
- * @hdr: 802.11 header (same as @payload)
  *
  * The byte count (both len and next_frame_len) includes MAC header
  * (24/26/30/32 bytes)
@@ -217,11 +215,8 @@ enum iwl_tx_offload_assist_flags_pos {
  * It does not include post-MAC padding, i.e.,
  * MIC (CCM) 8 bytes, ICV (WEP/TKIP/CKIP) 4 bytes, CRC 4 bytes.
  * Range of len: 14-2342 bytes.
- *
- * After the struct fields the MAC header is placed, plus any padding,
- * and then the actial payload.
  */
-struct iwl_tx_cmd {
+struct iwl_tx_cmd_v6_params {
 	__le16 len;
 	__le16 offload_assist;
 	__le32 tx_flags;
@@ -245,10 +240,20 @@ struct iwl_tx_cmd {
 	u8 tid_tspec;
 	__le16 pm_frame_timeout;
 	__le16 reserved4;
-	union {
-		DECLARE_FLEX_ARRAY(u8, payload);
-		DECLARE_FLEX_ARRAY(struct ieee80211_hdr, hdr);
-	};
+} __packed; /* TX_CMD_API_S_VER_6 */
+
+/**
+ * struct iwl_tx_cmd_v6 - TX command struct to FW
+ * ( TX_CMD = 0x1c )
+ * @params: parameters of the TX, see &struct iwl_tx_cmd_v6_tx_params
+ * @hdr: 802.11 header
+ *
+ * After &params, the MAC header is placed, plus any padding,
+ * and then the actual payload.
+ */
+struct iwl_tx_cmd_v6 {
+	struct iwl_tx_cmd_v6_params params;
+	struct ieee80211_hdr hdr[];
 } __packed; /* TX_CMD_API_S_VER_6 */
 
 struct iwl_dram_sec_info {
@@ -258,7 +263,7 @@ struct iwl_dram_sec_info {
 } __packed; /* DRAM_SEC_INFO_API_S_VER_1 */
 
 /**
- * struct iwl_tx_cmd_gen2 - TX command struct to FW for 22000 devices
+ * struct iwl_tx_cmd_v9 - TX command struct to FW for 22000 devices
  * ( TX_CMD = 0x1c )
  * @len: in bytes of the payload, see below for details
  * @offload_assist: TX offload configuration
@@ -268,7 +273,7 @@ struct iwl_dram_sec_info {
  *	cleared. Combination of RATE_MCS_*
  * @hdr: 802.11 header
  */
-struct iwl_tx_cmd_gen2 {
+struct iwl_tx_cmd_v9 {
 	__le16 len;
 	__le16 offload_assist;
 	__le32 flags;
@@ -279,18 +284,18 @@ struct iwl_tx_cmd_gen2 {
 	       TX_CMD_API_S_VER_9 */
 
 /**
- * struct iwl_tx_cmd_gen3 - TX command struct to FW for AX210+ devices
+ * struct iwl_tx_cmd - TX command struct to FW for AX210+ devices
  * ( TX_CMD = 0x1c )
  * @len: in bytes of the payload, see below for details
  * @flags: combination of &enum iwl_tx_cmd_flags
  * @offload_assist: TX offload configuration
  * @dram_info: FW internal DRAM storage
  * @rate_n_flags: rate for *all* Tx attempts, if TX_CMD_FLG_STA_RATE_MSK is
- *	cleared. Combination of RATE_MCS_*
+ *	cleared. Combination of RATE_MCS_*; format depends on version
  * @reserved: reserved
  * @hdr: 802.11 header
  */
-struct iwl_tx_cmd_gen3 {
+struct iwl_tx_cmd {
 	__le16 len;
 	__le16 flags;
 	__le32 offload_assist;
@@ -298,8 +303,9 @@ struct iwl_tx_cmd_gen3 {
 	__le32 rate_n_flags;
 	u8 reserved[8];
 	struct ieee80211_hdr hdr[];
-} __packed; /* TX_CMD_API_S_VER_8,
-	       TX_CMD_API_S_VER_10 */
+} __packed; /* TX_CMD_API_S_VER_10,
+	     * TX_CMD_API_S_VER_11
+	     */
 
 /*
  * TX response related data
@@ -482,11 +488,11 @@ struct agg_tx_status {
 #define TX_RES_RATE_TABLE_COL_GET(_f) (((_f) & TX_RES_RATE_TABLE_COLOR_MSK) >>\
 				       TX_RES_RATE_TABLE_COLOR_POS)
 
-#define IWL_MVM_TX_RES_GET_TID(_ra_tid) ((_ra_tid) & 0x0f)
-#define IWL_MVM_TX_RES_GET_RA(_ra_tid) ((_ra_tid) >> 4)
+#define IWL_TX_RES_GET_TID(_ra_tid) ((_ra_tid) & 0x0f)
+#define IWL_TX_RES_GET_RA(_ra_tid) ((_ra_tid) >> 4)
 
 /**
- * struct iwl_mvm_tx_resp_v3 - notifies that fw is TXing a packet
+ * struct iwl_tx_resp_v3 - notifies that fw is TXing a packet
  * ( REPLY_TX = 0x1c )
  * @frame_count: 1 no aggregation, >1 aggregation
  * @bt_kill_count: num of times blocked by bluetooth (unused for agg)
@@ -517,7 +523,7 @@ struct agg_tx_status {
  * After the array of statuses comes the SSN of the SCD. Look at
  * %iwl_mvm_get_scd_ssn for more details.
  */
-struct iwl_mvm_tx_resp_v3 {
+struct iwl_tx_resp_v3 {
 	u8 frame_count;
 	u8 bt_kill_count;
 	u8 failure_rts;
@@ -543,14 +549,14 @@ struct iwl_mvm_tx_resp_v3 {
 } __packed; /* TX_RSP_API_S_VER_3 */
 
 /**
- * struct iwl_mvm_tx_resp - notifies that fw is TXing a packet
+ * struct iwl_tx_resp - notifies that fw is TXing a packet
  * ( REPLY_TX = 0x1c )
  * @frame_count: 1 no aggregation, >1 aggregation
  * @bt_kill_count: num of times blocked by bluetooth (unused for agg)
  * @failure_rts: num of failures due to unsuccessful RTS
  * @failure_frame: num failures due to no ACK (unused for agg)
  * @initial_rate: for non-agg: rate of the successful Tx. For agg: rate of the
- *	Tx of all the batch. RATE_MCS_*
+ *	Tx of all the batch. RATE_MCS_*; format depends on command version
  * @wireless_media_time: for non-agg: RTS + CTS + frame tx attempts time + ACK.
  *	for agg: RTS + CTS + aggregation tx time + block-ack time.
  *	in usec.
@@ -575,7 +581,7 @@ struct iwl_mvm_tx_resp_v3 {
  * After the array of statuses comes the SSN of the SCD. Look at
  * %iwl_mvm_get_scd_ssn for more details.
  */
-struct iwl_mvm_tx_resp {
+struct iwl_tx_resp {
 	u8 frame_count;
 	u8 bt_kill_count;
 	u8 failure_rts;
@@ -601,7 +607,10 @@ struct iwl_mvm_tx_resp {
 	__le16 reserved2;
 	struct agg_tx_status status;
 } __packed; /* TX_RSP_API_S_VER_6,
-	       TX_RSP_API_S_VER_7 */
+	     * TX_RSP_API_S_VER_7,
+	     * TX_RSP_API_S_VER_8,
+	     * TX_RSP_API_S_VER_9
+	     */
 
 /**
  * struct iwl_mvm_ba_notif - notifies about reception of BA
@@ -638,14 +647,14 @@ struct iwl_mvm_ba_notif {
 } __packed;
 
 /**
- * struct iwl_mvm_compressed_ba_tfd - progress of a TFD queue
+ * struct iwl_compressed_ba_tfd - progress of a TFD queue
  * @q_num: TFD queue number
  * @tfd_index: Index of first un-acked frame in the  TFD queue
  * @scd_queue: For debug only - the physical queue the TFD queue is bound to
  * @tid: TID of the queue (0-7)
  * @reserved: reserved for alignment
  */
-struct iwl_mvm_compressed_ba_tfd {
+struct iwl_compressed_ba_tfd {
 	__le16 q_num;
 	__le16 tfd_index;
 	u8 scd_queue;
@@ -654,12 +663,12 @@ struct iwl_mvm_compressed_ba_tfd {
 } __packed; /* COMPRESSED_BA_TFD_API_S_VER_1 */
 
 /**
- * struct iwl_mvm_compressed_ba_ratid - progress of a RA TID queue
+ * struct iwl_compressed_ba_ratid - progress of a RA TID queue
  * @q_num: RA TID queue number
  * @tid: TID of the queue
  * @ssn: BA window current SSN
  */
-struct iwl_mvm_compressed_ba_ratid {
+struct iwl_compressed_ba_ratid {
 	u8 q_num;
 	u8 tid;
 	__le16 ssn;
@@ -685,7 +694,7 @@ enum iwl_mvm_ba_resp_flags {
 };
 
 /**
- * struct iwl_mvm_compressed_ba_notif - notifies about reception of BA
+ * struct iwl_compressed_ba_notif - notifies about reception of BA
  * ( BA_NOTIF = 0xc5 )
  * @flags: status flag, see the &iwl_mvm_ba_resp_flags
  * @sta_id: Index of recipient (BA-sending) station in fw's station table
@@ -701,15 +710,16 @@ enum iwl_mvm_ba_resp_flags {
  * @rts_retry_cnt: RTS retry count
  * @reserved: reserved (for alignment)
  * @wireless_time: Wireless-media time
- * @tx_rate: the rate the aggregation was sent at
+ * @tx_rate: the rate the aggregation was sent at. Format depends on command
+ *	version.
  * @tfd_cnt: number of TFD-Q elements
  * @ra_tid_cnt: number of RATID-Q elements
- * @tfd: array of TFD queue status updates. See &iwl_mvm_compressed_ba_tfd
+ * @tfd: array of TFD queue status updates. See &iwl_compressed_ba_tfd
  *	for details. Length in @tfd_cnt.
  * @ra_tid: array of RA-TID queue status updates. For debug purposes only. See
- *	&iwl_mvm_compressed_ba_ratid for more details. Length in @ra_tid_cnt.
+ *	&iwl_compressed_ba_ratid for more details. Length in @ra_tid_cnt.
  */
-struct iwl_mvm_compressed_ba_notif {
+struct iwl_compressed_ba_notif {
 	__le32 flags;
 	u8 sta_id;
 	u8 reduced_txp;
@@ -726,11 +736,12 @@ struct iwl_mvm_compressed_ba_notif {
 	__le16 tfd_cnt;
 	__le16 ra_tid_cnt;
 	union {
-		DECLARE_FLEX_ARRAY(struct iwl_mvm_compressed_ba_ratid, ra_tid);
-		DECLARE_FLEX_ARRAY(struct iwl_mvm_compressed_ba_tfd, tfd);
+		DECLARE_FLEX_ARRAY(struct iwl_compressed_ba_ratid, ra_tid);
+		DECLARE_FLEX_ARRAY(struct iwl_compressed_ba_tfd, tfd);
 	};
 } __packed; /* COMPRESSED_BA_RES_API_S_VER_4,
-	       COMPRESSED_BA_RES_API_S_VER_5 */
+	       COMPRESSED_BA_RES_API_S_VER_6,
+	       COMPRESSED_BA_RES_API_S_VER_7 */
 
 /**
  * struct iwl_mac_beacon_cmd_v6 - beacon template command
@@ -742,7 +753,7 @@ struct iwl_mvm_compressed_ba_notif {
  * @frame: the template of the beacon frame
  */
 struct iwl_mac_beacon_cmd_v6 {
-	struct iwl_tx_cmd tx;
+	struct iwl_tx_cmd_v6_params tx;
 	__le32 template_id;
 	__le32 tim_idx;
 	__le32 tim_size;
@@ -761,7 +772,7 @@ struct iwl_mac_beacon_cmd_v6 {
  * @frame: the template of the beacon frame
  */
 struct iwl_mac_beacon_cmd_v7 {
-	struct iwl_tx_cmd tx;
+	struct iwl_tx_cmd_v6_params tx;
 	__le32 template_id;
 	__le32 tim_idx;
 	__le32 tim_size;
@@ -823,7 +834,7 @@ struct iwl_mac_beacon_cmd {
 	     */
 
 struct iwl_beacon_notif {
-	struct iwl_mvm_tx_resp beacon_notify_hdr;
+	struct iwl_tx_resp beacon_notify_hdr;
 	__le64 tsf;
 	__le32 ibss_mgr_status;
 } __packed;
@@ -836,7 +847,7 @@ struct iwl_beacon_notif {
  * @gp2: last beacon time in gp2
  */
 struct iwl_extended_beacon_notif_v5 {
-	struct iwl_mvm_tx_resp beacon_notify_hdr;
+	struct iwl_tx_resp beacon_notify_hdr;
 	__le64 tsf;
 	__le32 ibss_mgr_status;
 	__le32 gp2;
@@ -858,7 +869,7 @@ struct iwl_extended_beacon_notif {
 
 /**
  * enum iwl_dump_control - dump (flush) control flags
- * @DUMP_TX_FIFO_FLUSH: Dump MSDUs until the the FIFO is empty
+ * @DUMP_TX_FIFO_FLUSH: Dump MSDUs until the FIFO is empty
  *	and the TFD queues are empty.
  */
 enum iwl_dump_control {
diff --git a/sys/contrib/dev/iwlwifi/fw/dbg.c b/sys/contrib/dev/iwlwifi/fw/dbg.c
index 57e1cdec6b27..e3d7e484daeb 100644
--- a/sys/contrib/dev/iwlwifi/fw/dbg.c
+++ b/sys/contrib/dev/iwlwifi/fw/dbg.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -190,7 +190,7 @@ static void iwl_fw_dump_rxf(struct iwl_fw_runtime *fwrt,
 		/* Pull RXF2 */
 		iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
 				  RXF_DIFF_FROM_PREV +
-				  fwrt->trans->trans_cfg->umac_prph_offset, 1);
+				  fwrt->trans->mac_cfg->umac_prph_offset, 1);
 		/* Pull LMAC2 RXF1 */
 		if (fwrt->smem_cfg.num_lmacs > 1)
 			iwl_fwrt_dump_rxf(fwrt, dump_data,
@@ -561,41 +561,71 @@ static void iwl_dump_prph(struct iwl_fw_runtime *fwrt,
 }
 
 /*
- * alloc_sgtable - allocates scallerlist table in the given size,
- * fills it with pages and returns it
+ * alloc_sgtable - allocates (chained) scatterlist in the given size,
+ *	fills it with pages and returns it
  * @size: the size (in bytes) of the table
-*/
-static struct scatterlist *alloc_sgtable(int size)
+ */
+static struct scatterlist *alloc_sgtable(ssize_t size)
 {
-	int alloc_size, nents, i;
-	struct page *new_page;
-	struct scatterlist *iter;
-	struct scatterlist *table;
+	struct scatterlist *result = NULL, *prev;
+	int nents, i, n_prev;
 
 	nents = DIV_ROUND_UP(size, PAGE_SIZE);
-	table = kcalloc(nents, sizeof(*table), GFP_KERNEL);
-	if (!table)
-		return NULL;
-	sg_init_table(table, nents);
-	iter = table;
-	for_each_sg(table, iter, sg_nents(table), i) {
-		new_page = alloc_page(GFP_KERNEL);
-		if (!new_page) {
-			/* release all previous allocated pages in the table */
-			iter = table;
-			for_each_sg(table, iter, sg_nents(table), i) {
-				new_page = sg_page(iter);
-				if (new_page)
-					__free_page(new_page);
-			}
-			kfree(table);
+
+#define N_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(*result))
+	/*
+	 * We need an additional entry for table chaining,
+	 * this ensures the loop can finish i.e. we can
+	 * fit at least two entries per page (obviously,
+	 * many more really fit.)
+	 */
+	BUILD_BUG_ON(N_ENTRIES_PER_PAGE < 2);
+
+	while (nents > 0) {
+		struct scatterlist *new, *iter;
+		int n_fill, n_alloc;
+
+		if (nents <= N_ENTRIES_PER_PAGE) {
+			/* last needed table */
+			n_fill = nents;
+			n_alloc = nents;
+			nents = 0;
+		} else {
+			/* fill a page with entries */
+			n_alloc = N_ENTRIES_PER_PAGE;
+			/* reserve one for chaining */
+			n_fill = n_alloc - 1;
+			nents -= n_fill;
+		}
+
+		new = kcalloc(n_alloc, sizeof(*new), GFP_KERNEL);
+		if (!new) {
+			if (result)
+				_devcd_free_sgtable(result);
 			return NULL;
 		}
-		alloc_size = min_t(int, size, PAGE_SIZE);
-		size -= PAGE_SIZE;
-		sg_set_page(iter, new_page, alloc_size, 0);
+		sg_init_table(new, n_alloc);
+
+		if (!result)
+			result = new;
+		else
+			sg_chain(prev, n_prev, new);
+		prev = new;
+		n_prev = n_alloc;
+
+		for_each_sg(new, iter, n_fill, i) {
+			struct page *new_page = alloc_page(GFP_KERNEL);
+
+			if (!new_page) {
+				_devcd_free_sgtable(result);
+				return NULL;
+			}
+
+			sg_set_page(iter, new_page, PAGE_SIZE, 0);
+		}
 	}
-	return table;
+
+	return result;
 }
 
 static void iwl_fw_get_prph_len(struct iwl_fw_runtime *fwrt,
@@ -627,10 +657,10 @@ static void iwl_fw_prph_handler(struct iwl_fw_runtime *fwrt, void *ptr,
 {
 	u32 range_len;
 
-	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		range_len = ARRAY_SIZE(iwl_prph_dump_addr_ax210);
 		handler(fwrt, iwl_prph_dump_addr_ax210, range_len, ptr);
-	} else if (fwrt->trans->trans_cfg->device_family >=
+	} else if (fwrt->trans->mac_cfg->device_family >=
 		   IWL_DEVICE_FAMILY_22000) {
 		range_len = ARRAY_SIZE(iwl_prph_dump_addr_22000);
 		handler(fwrt, iwl_prph_dump_addr_22000, range_len, ptr);
@@ -638,7 +668,7 @@ static void iwl_fw_prph_handler(struct iwl_fw_runtime *fwrt, void *ptr,
 		range_len = ARRAY_SIZE(iwl_prph_dump_addr_comm);
 		handler(fwrt, iwl_prph_dump_addr_comm, range_len, ptr);
 
-		if (fwrt->trans->trans_cfg->mq_rx_supported) {
+		if (fwrt->trans->mac_cfg->mq_rx_supported) {
 			range_len = ARRAY_SIZE(iwl_prph_dump_addr_9000);
 			handler(fwrt, iwl_prph_dump_addr_9000, range_len, ptr);
 		}
@@ -782,13 +812,14 @@ iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
 	const struct iwl_fw_dbg_mem_seg_tlv *fw_mem = fwrt->fw->dbg.mem_tlv;
 	struct iwl_fwrt_shared_mem_cfg *mem_cfg = &fwrt->smem_cfg;
 	u32 file_len, fifo_len = 0, prph_len = 0, radio_len = 0;
-	u32 smem_len = fwrt->fw->dbg.n_mem_tlv ? 0 : fwrt->trans->cfg->smem_len;
+	u32 smem_len = fwrt->fw->dbg.n_mem_tlv ? 0 : fwrt->trans->mac_cfg->base->smem_len;
 	u32 sram2_len = fwrt->fw->dbg.n_mem_tlv ?
 				0 : fwrt->trans->cfg->dccm2_len;
 	int i;
 
 	/* SRAM - include stack CCM if driver knows the values for it */
-	if (!fwrt->trans->cfg->dccm_offset || !fwrt->trans->cfg->dccm_len) {
+	if (!fwrt->trans->cfg->dccm_offset ||
+	    !fwrt->trans->cfg->dccm_len) {
 		const struct fw_img *img;
 
 		if (fwrt->cur_fw_img >= IWL_UCODE_TYPE_MAX)
@@ -811,7 +842,7 @@ iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
 			iwl_fw_prph_handler(fwrt, &prph_len,
 					    iwl_fw_get_prph_len);
 
-		if (fwrt->trans->trans_cfg->device_family ==
+		if (fwrt->trans->mac_cfg->device_family ==
 		    IWL_DEVICE_FAMILY_7000 &&
 		    iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RADIO_REG))
 			radio_len = sizeof(*dump_data) + RADIO_REG_MAX_READ;
@@ -849,7 +880,7 @@ iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
 
 	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
 		file_len += sizeof(*dump_data) +
-			fwrt->trans->cfg->d3_debug_data_length * 2;
+			fwrt->trans->mac_cfg->base->d3_debug_data_length * 2;
 	}
 
 	/* If we only want a monitor dump, reset the file length */
@@ -877,13 +908,14 @@ iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
 		dump_data->len = cpu_to_le32(sizeof(*dump_info));
 		dump_info = (void *)dump_data->data;
 		dump_info->hw_type =
-			cpu_to_le32(CSR_HW_REV_TYPE(fwrt->trans->hw_rev));
+			cpu_to_le32(CSR_HW_REV_TYPE(fwrt->trans->info.hw_rev));
 		dump_info->hw_step =
-			cpu_to_le32(fwrt->trans->hw_rev_step);
+			cpu_to_le32(fwrt->trans->info.hw_rev_step);
 		memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable,
 		       sizeof(dump_info->fw_human_readable));
-		strscpy_pad(dump_info->dev_human_readable, fwrt->trans->name,
-			sizeof(dump_info->dev_human_readable));
+		strscpy_pad(dump_info->dev_human_readable,
+			    fwrt->trans->info.name,
+			    sizeof(dump_info->dev_human_readable));
 #if defined(__linux__)
 		strscpy_pad(dump_info->bus_human_readable, fwrt->dev->bus->name,
 			sizeof(dump_info->bus_human_readable));
@@ -974,7 +1006,7 @@ iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
 		}
 
 		iwl_fw_dump_mem(fwrt, &dump_data, smem_len,
-				fwrt->trans->cfg->smem_offset,
+				fwrt->trans->mac_cfg->base->smem_offset,
 				IWL_FW_ERROR_DUMP_MEM_SMEM);
 
 		iwl_fw_dump_mem(fwrt, &dump_data, sram2_len,
@@ -983,8 +1015,8 @@ iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
 	}
 
 	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
-		u32 addr = fwrt->trans->cfg->d3_debug_data_base_addr;
-		size_t data_size = fwrt->trans->cfg->d3_debug_data_length;
+		u32 addr = fwrt->trans->mac_cfg->base->d3_debug_data_base_addr;
+		size_t data_size = fwrt->trans->mac_cfg->base->d3_debug_data_length;
 
 		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_D3_DEBUG_DATA);
 		dump_data->len = cpu_to_le32(data_size * 2);
@@ -1082,12 +1114,13 @@ static int iwl_dump_ini_prph_phy_iter_common(struct iwl_fw_runtime *fwrt,
 	u32 prph_val;
 	u32 dphy_state;
 	u32 dphy_addr;
+	u32 prph_stts;
 	int i;
 
 	range->internal_base_addr = cpu_to_le32(addr);
 	range->range_data_size = size;
 
-	if (fwrt->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		indirect_wr_addr = WMAL_INDRCT_CMD1;
 
 	indirect_wr_addr += le32_to_cpu(offset);
@@ -1109,6 +1142,21 @@ static int iwl_dump_ini_prph_phy_iter_common(struct iwl_fw_runtime *fwrt,
 
 		iwl_write_prph_no_grab(fwrt->trans, indirect_wr_addr,
 				       WMAL_INDRCT_CMD(addr + i));
+
+		if (fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_JF1 &&
+		    fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_JF2 &&
+		    fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_HR1 &&
+		    fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_HR2) {
+			udelay(2);
+			prph_stts = iwl_read_prph_no_grab(fwrt->trans,
+							  WMAL_MRSPF_STTS);
+
+			/* Abort dump if status is 0xA5A5A5A2 or FIFO1 empty */
+			if (prph_stts == WMAL_TIMEOUT_VAL ||
+			    !WMAL_MRSPF_STTS_IS_FIFO1_NOT_EMPTY(prph_stts))
+				break;
+		}
+
 		prph_val = iwl_read_prph_no_grab(fwrt->trans,
 						 indirect_rd_addr);
 		*val++ = cpu_to_le32(prph_val);
@@ -1244,7 +1292,7 @@ static int iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
 	/* all paged index start from 1 to skip CSS section */
 	idx++;
 
-	if (!fwrt->trans->trans_cfg->gen2)
+	if (!fwrt->trans->mac_cfg->gen2)
 		return _iwl_dump_ini_paging_iter(fwrt, range_ptr, range_len, idx);
 
 	range = range_ptr;
@@ -1771,7 +1819,7 @@ iwl_dump_ini_mon_fill_header(struct iwl_fw_runtime *fwrt, u32 alloc_id,
 
 	data->write_ptr = iwl_get_mon_reg(fwrt, alloc_id,
 					  &addrs->write_ptr);
-	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		u32 wrt_ptr = le32_to_cpu(data->write_ptr);
 
 		data->write_ptr = cpu_to_le32(wrt_ptr >> 2);
@@ -1798,7 +1846,7 @@ iwl_dump_ini_mon_dram_fill_header(struct iwl_fw_runtime *fwrt,
 	u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
 
 	return iwl_dump_ini_mon_fill_header(fwrt, alloc_id, mon_dump,
-					    &fwrt->trans->cfg->mon_dram_regs);
+					    &fwrt->trans->mac_cfg->base->mon_dram_regs);
 }
 
 static void *
@@ -1811,7 +1859,7 @@ iwl_dump_ini_mon_smem_fill_header(struct iwl_fw_runtime *fwrt,
 	u32 alloc_id = le32_to_cpu(reg->internal_buffer.alloc_id);
 
 	return iwl_dump_ini_mon_fill_header(fwrt, alloc_id, mon_dump,
-					    &fwrt->trans->cfg->mon_smem_regs);
+					    &fwrt->trans->mac_cfg->base->mon_smem_regs);
 }
 
 static void *
@@ -1825,7 +1873,7 @@ iwl_dump_ini_mon_dbgi_fill_header(struct iwl_fw_runtime *fwrt,
 					    /* no offset calculation later */
 					    IWL_FW_INI_ALLOCATION_ID_DBGC1,
 					    mon_dump,
-					    &fwrt->trans->cfg->mon_dbgi_regs);
+					    &fwrt->trans->mac_cfg->base->mon_dbgi_regs);
 }
 
 static void *
@@ -1890,7 +1938,7 @@ iwl_dump_ini_mem_block_ranges(struct iwl_fw_runtime *fwrt,
 static u32 iwl_dump_ini_paging_ranges(struct iwl_fw_runtime *fwrt,
 				      struct iwl_dump_ini_region_data *reg_data)
 {
-	if (fwrt->trans->trans_cfg->gen2) {
+	if (fwrt->trans->mac_cfg->gen2) {
 		if (fwrt->trans->init_dram.paging_cnt)
 			return fwrt->trans->init_dram.paging_cnt - 1;
 		else
@@ -2002,7 +2050,7 @@ iwl_dump_ini_paging_get_size(struct iwl_fw_runtime *fwrt,
 	/* start from 1 to skip CSS section */
 	for (i = 1; i <= iwl_dump_ini_paging_ranges(fwrt, reg_data); i++) {
 		size += range_header_len;
-		if (fwrt->trans->trans_cfg->gen2)
+		if (fwrt->trans->mac_cfg->gen2)
 			size += fwrt->trans->init_dram.paging[i].size;
 		else
 			size += fwrt->fw_paging_db[i].fw_paging_size;
@@ -2364,7 +2412,7 @@ static u32 iwl_dump_ini_info(struct iwl_fw_runtime *fwrt,
 	struct iwl_fw_ini_dump_cfg_name *cfg_name;
 	u32 size = sizeof(*tlv) + sizeof(*dump);
 	u32 num_of_cfg_names = 0;
-	u32 hw_type;
+	u32 hw_type, is_cdb, is_jacket;
 
 	list_for_each_entry(node, &fwrt->trans->dbg.debug_info_tlv_list, list) {
 		size += sizeof(*cfg_name);
@@ -2392,33 +2440,24 @@ static u32 iwl_dump_ini_info(struct iwl_fw_runtime *fwrt,
 	dump->ver_type = cpu_to_le32(fwrt->dump.fw_ver.type);
 	dump->ver_subtype = cpu_to_le32(fwrt->dump.fw_ver.subtype);
 
-	dump->hw_step = cpu_to_le32(fwrt->trans->hw_rev_step);
+	dump->hw_step = cpu_to_le32(fwrt->trans->info.hw_rev_step);
 
-	/*
-	 * Several HWs all have type == 0x42, so we'll override this value
-	 * according to the detected HW
-	 */
-	hw_type = CSR_HW_REV_TYPE(fwrt->trans->hw_rev);
-	if (hw_type == IWL_AX210_HW_TYPE) {
-		u32 prph_val = iwl_read_umac_prph(fwrt->trans, WFPM_OTP_CFG1_ADDR);
-		u32 is_jacket = !!(prph_val & WFPM_OTP_CFG1_IS_JACKET_BIT);
-		u32 is_cdb = !!(prph_val & WFPM_OTP_CFG1_IS_CDB_BIT);
-		u32 masked_bits = is_jacket | (is_cdb << 1);
+	hw_type = CSR_HW_REV_TYPE(fwrt->trans->info.hw_rev);
+
+	is_cdb = CSR_HW_RFID_IS_CDB(fwrt->trans->info.hw_rf_id);
+	is_jacket = !!(iwl_read_umac_prph(fwrt->trans, WFPM_OTP_CFG1_ADDR) &
+				WFPM_OTP_CFG1_IS_JACKET_BIT);
+
+	/* Use bits 12 and 13 to indicate jacket/CDB, respectively */
+	hw_type |= (is_jacket | (is_cdb << 1)) << IWL_JACKET_CDB_SHIFT;
 
-		/*
-		 * The HW type depends on certain bits in this case, so add
-		 * these bits to the HW type. We won't have collisions since we
-		 * add these bits after the highest possible bit in the mask.
-		 */
-		hw_type |= masked_bits << IWL_AX210_HW_TYPE_ADDITION_SHIFT;
-	}
 	dump->hw_type = cpu_to_le32(hw_type);
 
 	dump->rf_id_flavor =
-		cpu_to_le32(CSR_HW_RFID_FLAVOR(fwrt->trans->hw_rf_id));
-	dump->rf_id_dash = cpu_to_le32(CSR_HW_RFID_DASH(fwrt->trans->hw_rf_id));
-	dump->rf_id_step = cpu_to_le32(CSR_HW_RFID_STEP(fwrt->trans->hw_rf_id));
-	dump->rf_id_type = cpu_to_le32(CSR_HW_RFID_TYPE(fwrt->trans->hw_rf_id));
+		cpu_to_le32(CSR_HW_RFID_FLAVOR(fwrt->trans->info.hw_rf_id));
+	dump->rf_id_dash = cpu_to_le32(CSR_HW_RFID_DASH(fwrt->trans->info.hw_rf_id));
+	dump->rf_id_step = cpu_to_le32(CSR_HW_RFID_STEP(fwrt->trans->info.hw_rf_id));
+	dump->rf_id_type = cpu_to_le32(CSR_HW_RFID_TYPE(fwrt->trans->info.hw_rf_id));
 
 	dump->lmac_major = cpu_to_le32(fwrt->dump.fw_ver.lmac_major);
 	dump->lmac_minor = cpu_to_le32(fwrt->dump.fw_ver.lmac_minor);
@@ -2607,29 +2646,34 @@ static const struct iwl_dump_ini_mem_ops iwl_dump_ini_region_ops[] = {
 	},
 };
 
-static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
-				struct iwl_fwrt_dump_data *dump_data,
-				struct list_head *list)
+enum iwl_dump_ini_region_selector {
+	IWL_INI_DUMP_ALL_REGIONS,
+	IWL_INI_DUMP_EARLY_REGIONS,
+	IWL_INI_DUMP_LATE_REGIONS,
+};
+
+static bool iwl_dump_due_to_error(enum iwl_fw_ini_time_point tp_id)
 {
-	struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
-	enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trigger->time_point);
-	struct iwl_dump_ini_region_data reg_data = {
-		.dump_data = dump_data,
-	};
-	struct iwl_dump_ini_region_data imr_reg_data = {
-		.dump_data = dump_data,
-	};
-	int i;
-	u32 size = 0;
-	u64 regions_mask = le64_to_cpu(trigger->regions_mask) &
-			   ~(fwrt->trans->dbg.unsupported_region_msk);
+	return tp_id == IWL_FW_INI_TIME_POINT_FW_ASSERT ||
+	       tp_id == IWL_FW_INI_TIME_POINT_FW_HW_ERROR;
+}
 
-	BUILD_BUG_ON(sizeof(trigger->regions_mask) != sizeof(regions_mask));
-	BUILD_BUG_ON((sizeof(trigger->regions_mask) * BITS_PER_BYTE) <
-		     ARRAY_SIZE(fwrt->trans->dbg.active_regions));
+static u32
+iwl_dump_ini_dump_regions(struct iwl_fw_runtime *fwrt,
+			  struct iwl_fwrt_dump_data *dump_data,
+			  struct list_head *list,
+			  enum iwl_fw_ini_time_point tp_id,
+			  u64 regions_mask,
+			  struct iwl_dump_ini_region_data *imr_reg_data,
+			  enum iwl_dump_ini_region_selector which)
+{
+	u32 size = 0;
 
-	for (i = 0; i < ARRAY_SIZE(fwrt->trans->dbg.active_regions); i++) {
-		u32 reg_type;
+	for (int i = 0; i < ARRAY_SIZE(fwrt->trans->dbg.active_regions); i++) {
+		struct iwl_dump_ini_region_data reg_data = {
+			.dump_data = dump_data,
+		};
+		u32 reg_type, dp;
 		struct iwl_fw_ini_region_tlv *reg;
 
 		if (!(BIT_ULL(i) & regions_mask))
@@ -2647,6 +2691,8 @@ static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
 		if (reg_type >= ARRAY_SIZE(iwl_dump_ini_region_ops))
 			continue;
 
+		dp = le32_get_bits(reg->id, IWL_FW_INI_REGION_DUMP_POLICY_MASK);
+
 		if ((reg_type == IWL_FW_INI_REGION_PERIPHERY_PHY ||
 		     reg_type == IWL_FW_INI_REGION_PERIPHERY_PHY_RANGE ||
 		     reg_type == IWL_FW_INI_REGION_PERIPHERY_SNPS_DPHYIP) &&
@@ -2656,6 +2702,20 @@ static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
 				 tp_id);
 			continue;
 		}
+
+		switch (which) {
+		case IWL_INI_DUMP_ALL_REGIONS:
+			break;
+		case IWL_INI_DUMP_EARLY_REGIONS:
+			if (!(dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET))
+				continue;
+			break;
+		case IWL_INI_DUMP_LATE_REGIONS:
+			if (dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET)
+				continue;
+			break;
+		}
+
 		/*
 		 * DRAM_IMR can be collected only for FW/HW error timepoint
 		 * when fw is not alive. In addition, it must be collected
@@ -2663,9 +2723,9 @@ static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
 		 * debug data which also need to be collected.
 		 */
 		if (reg_type == IWL_FW_INI_REGION_DRAM_IMR) {
-			if (tp_id == IWL_FW_INI_TIME_POINT_FW_ASSERT ||
-			    tp_id == IWL_FW_INI_TIME_POINT_FW_HW_ERROR)
-				imr_reg_data.reg_tlv = fwrt->trans->dbg.active_regions[i];
+			if (iwl_dump_due_to_error(tp_id))
+				imr_reg_data->reg_tlv =
+					fwrt->trans->dbg.active_regions[i];
 			else
 				IWL_INFO(fwrt,
 					 "WRT: trying to collect DRAM_IMR at time point: %d, skipping\n",
@@ -2678,9 +2738,48 @@ static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
 		size += iwl_dump_ini_mem(fwrt, list, &reg_data,
 					 &iwl_dump_ini_region_ops[reg_type]);
 	}
+
+	return size;
+}
+
+static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
+				struct iwl_fwrt_dump_data *dump_data,
+				struct list_head *list)
+{
+	struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
+	enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trigger->time_point);
+	struct iwl_dump_ini_region_data imr_reg_data = {
+		.dump_data = dump_data,
+	};
+	u32 size = 0;
+	u64 regions_mask = le64_to_cpu(trigger->regions_mask) &
+			   ~(fwrt->trans->dbg.unsupported_region_msk);
+
+	BUILD_BUG_ON(sizeof(trigger->regions_mask) != sizeof(regions_mask));
+	BUILD_BUG_ON((sizeof(trigger->regions_mask) * BITS_PER_BYTE) <
+		     ARRAY_SIZE(fwrt->trans->dbg.active_regions));
+
+	if (trigger->apply_policy &
+			cpu_to_le32(IWL_FW_INI_APPLY_POLICY_SPLIT_DUMP_RESET)) {
+		size += iwl_dump_ini_dump_regions(fwrt, dump_data, list, tp_id,
+						  regions_mask, &imr_reg_data,
+						  IWL_INI_DUMP_EARLY_REGIONS);
+		iwl_trans_pcie_fw_reset_handshake(fwrt->trans);
+		size += iwl_dump_ini_dump_regions(fwrt, dump_data, list, tp_id,
+						  regions_mask, &imr_reg_data,
+						  IWL_INI_DUMP_LATE_REGIONS);
+	} else {
+		if (fw_has_capa(&fwrt->fw->ucode_capa,
+				IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT) &&
+		    iwl_dump_due_to_error(tp_id))
+			iwl_trans_pcie_fw_reset_handshake(fwrt->trans);
+		size += iwl_dump_ini_dump_regions(fwrt, dump_data, list, tp_id,
+						  regions_mask, &imr_reg_data,
+						  IWL_INI_DUMP_ALL_REGIONS);
+	}
 	/* collect DRAM_IMR region in the last */
 	if (imr_reg_data.reg_tlv)
-		size += iwl_dump_ini_mem(fwrt, list, &reg_data,
+		size += iwl_dump_ini_mem(fwrt, list, &imr_reg_data,
 					 &iwl_dump_ini_region_ops[IWL_FW_INI_REGION_DRAM_IMR]);
 
 	if (size) {
@@ -2902,7 +3001,7 @@ IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_desc);
 int iwl_fw_dbg_error_collect(struct iwl_fw_runtime *fwrt,
 			     enum iwl_fw_dbg_trigger trig_type)
 {
-	if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status))
+	if (!iwl_trans_device_enabled(fwrt->trans))
 		return -EIO;
 
 	if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
@@ -2948,6 +3047,7 @@ int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
 	struct iwl_fw_dump_desc *desc;
 	unsigned int delay = 0;
 	bool monitor_only = false;
+	int ret;
 
 	if (trigger) {
 		u16 occurrences = le16_to_cpu(trigger->occurrences) - 1;
@@ -2978,7 +3078,11 @@ int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
 	desc->trig_desc.type = cpu_to_le32(trig);
 	memcpy(desc->trig_desc.data, str, len);
 
-	return iwl_fw_dbg_collect_desc(fwrt, desc, monitor_only, delay);
+	ret = iwl_fw_dbg_collect_desc(fwrt, desc, monitor_only, delay);
+	if (ret)
+		kfree(desc);
+
+	return ret;
 }
 IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect);
 
@@ -2986,7 +3090,7 @@ int iwl_fw_dbg_collect_trig(struct iwl_fw_runtime *fwrt,
 			    struct iwl_fw_dbg_trigger_tlv *trigger,
 			    const char *fmt, ...)
 {
-	int ret, len = 0;
+	int len = 0;
 	char buf[64];
 
 	if (iwl_trans_dbg_ini_valid(fwrt->trans))
@@ -3008,13 +3112,8 @@ int iwl_fw_dbg_collect_trig(struct iwl_fw_runtime *fwrt,
 		len = strlen(buf) + 1;
 	}
 
-	ret = iwl_fw_dbg_collect(fwrt, le32_to_cpu(trigger->id), buf, len,
-				 trigger);
-
-	if (ret)
-		return ret;
-
-	return 0;
+	return iwl_fw_dbg_collect(fwrt, le32_to_cpu(trigger->id), buf, len,
+				  trigger);
 }
 IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_trig);
 
@@ -3065,9 +3164,8 @@ int iwl_fw_start_dbg_conf(struct iwl_fw_runtime *fwrt, u8 conf_id)
 }
 IWL_EXPORT_SYMBOL(iwl_fw_start_dbg_conf);
 
-void iwl_send_dbg_dump_complete_cmd(struct iwl_fw_runtime *fwrt,
-				    u32 timepoint,
-				    u32 timepoint_data)
+static void iwl_send_dbg_dump_complete_cmd(struct iwl_fw_runtime *fwrt,
+					   u32 timepoint, u32 timepoint_data)
 {
 	struct iwl_dbg_dump_complete_cmd hcmd_data;
 	struct iwl_host_cmd hcmd = {
@@ -3095,6 +3193,7 @@ static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
 	struct iwl_fw_dbg_params params = {0};
 	struct iwl_fwrt_dump_data *dump_data =
 		&fwrt->dump.wks[wk_idx].dump_data;
+
 	if (!test_bit(wk_idx, &fwrt->dump.active_wks))
 		return;
 
@@ -3104,13 +3203,13 @@ static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
 		goto out;
 	}
 
-	if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status)) {
+	if (!iwl_trans_device_enabled(fwrt->trans)) {
 		IWL_ERR(fwrt, "Device is not enabled - cannot dump error\n");
 		goto out;
 	}
 
 	/* there's no point in fw dump if the bus is dead */
-	if (test_bit(STATUS_TRANS_DEAD, &fwrt->trans->status)) {
+	if (iwl_trans_is_dead(fwrt->trans)) {
 		IWL_ERR(fwrt, "Skip fw error dump since bus is dead\n");
 		goto out;
 	}
@@ -3119,9 +3218,9 @@ static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
 
 	IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection start\n");
 	if (iwl_trans_dbg_ini_valid(fwrt->trans))
-		iwl_fw_error_ini_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
+		iwl_fw_error_ini_dump(fwrt, dump_data);
 	else
-		iwl_fw_error_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
+		iwl_fw_error_dump(fwrt, dump_data);
 	IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection done\n");
 
 	iwl_fw_dbg_stop_restart_recording(fwrt, &params, false);
@@ -3138,7 +3237,6 @@ static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
 
 	if (fwrt->trans->dbg.last_tp_resetfw == IWL_FW_INI_RESET_FW_MODE_STOP_FW_ONLY)
 		iwl_force_nmi(fwrt->trans);
-
 out:
 	if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
 		iwl_fw_error_dump_data_free(dump_data);
@@ -3225,13 +3323,13 @@ void iwl_fw_error_dump_wk(struct work_struct *work)
 
 void iwl_fw_dbg_read_d3_debug_data(struct iwl_fw_runtime *fwrt)
 {
-	const struct iwl_cfg *cfg = fwrt->trans->cfg;
+	const struct iwl_mac_cfg *mac_cfg = fwrt->trans->mac_cfg;
 
 	if (!iwl_fw_dbg_is_d3_debug_enabled(fwrt))
 		return;
 
 	if (!fwrt->dump.d3_debug_data) {
-		fwrt->dump.d3_debug_data = kmalloc(cfg->d3_debug_data_length,
+		fwrt->dump.d3_debug_data = kmalloc(mac_cfg->base->d3_debug_data_length,
 						   GFP_KERNEL);
 		if (!fwrt->dump.d3_debug_data) {
 			IWL_ERR(fwrt,
@@ -3241,15 +3339,15 @@ void iwl_fw_dbg_read_d3_debug_data(struct iwl_fw_runtime *fwrt)
 	}
 
 	/* if the buffer holds previous debug data it is overwritten */
-	iwl_trans_read_mem_bytes(fwrt->trans, cfg->d3_debug_data_base_addr,
+	iwl_trans_read_mem_bytes(fwrt->trans, mac_cfg->base->d3_debug_data_base_addr,
 				 fwrt->dump.d3_debug_data,
-				 cfg->d3_debug_data_length);
+				 mac_cfg->base->d3_debug_data_length);
 
 	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
 		fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx,
-					     cfg->d3_debug_data_base_addr,
+					     mac_cfg->base->d3_debug_data_base_addr,
 					     fwrt->dump.d3_debug_data,
-					     cfg->d3_debug_data_length);
+					     mac_cfg->base->d3_debug_data_length);
 }
 IWL_EXPORT_SYMBOL(iwl_fw_dbg_read_d3_debug_data);
 
@@ -3284,7 +3382,7 @@ static int iwl_fw_dbg_suspend_resume_hcmd(struct iwl_trans *trans, bool suspend)
 static void iwl_fw_dbg_stop_recording(struct iwl_trans *trans,
 				      struct iwl_fw_dbg_params *params)
 {
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
 		iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
 		return;
 	}
@@ -3308,7 +3406,7 @@ static int iwl_fw_dbg_restart_recording(struct iwl_trans *trans,
 	if (!params)
 		return -EIO;
 
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
 		iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
 		iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
 		iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
@@ -3410,7 +3508,7 @@ void iwl_fw_disable_dbg_asserts(struct iwl_fw_runtime *fwrt)
 				  GENMASK(31, IWL_FW_DBG_DOMAIN_POS + 1));
 
 	/* supported starting from 9000 devices */
-	if (fwrt->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000)
+	if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
 		return;
 
 	if (fwrt->trans->dbg.yoyo_bin_loaded || (preset && preset != 1))
diff --git a/sys/contrib/dev/iwlwifi/fw/dbg.h b/sys/contrib/dev/iwlwifi/fw/dbg.h
index 7ef061e090c7..87ccfc174fdc 100644
--- a/sys/contrib/dev/iwlwifi/fw/dbg.h
+++ b/sys/contrib/dev/iwlwifi/fw/dbg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2019, 2021-2023 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2019, 2021-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -205,7 +205,7 @@ static inline bool iwl_fw_dbg_is_d3_debug_enabled(struct iwl_fw_runtime *fwrt)
 {
 	return fw_has_capa(&fwrt->fw->ucode_capa,
 			   IWL_UCODE_TLV_CAPA_D3_DEBUG) &&
-		fwrt->trans->cfg->d3_debug_data_length && fwrt->ops &&
+		fwrt->trans->mac_cfg->base->d3_debug_data_length && fwrt->ops &&
 		fwrt->ops->d3_debug_enable &&
 		fwrt->ops->d3_debug_enable(fwrt->ops_ctx) &&
 		iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_D3_DEBUG_DATA);
@@ -214,7 +214,7 @@ static inline bool iwl_fw_dbg_is_d3_debug_enabled(struct iwl_fw_runtime *fwrt)
 static inline bool iwl_fw_dbg_is_paging_enabled(struct iwl_fw_runtime *fwrt)
 {
 	return iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_PAGING) &&
-		!fwrt->trans->trans_cfg->gen2 &&
+		!fwrt->trans->mac_cfg->gen2 &&
 		fwrt->cur_fw_img < IWL_UCODE_TYPE_MAX &&
 		fwrt->fw->img[fwrt->cur_fw_img].paging_mem_size &&
 		fwrt->fw_paging_db[0].fw_paging_block;
@@ -291,7 +291,7 @@ static inline void iwl_fw_umac_set_alive_err_table(struct iwl_trans *trans,
 		trans->dbg.umac_error_event_table = umac_error_event_table;
 }
 
-static inline void iwl_fw_error_collect(struct iwl_fw_runtime *fwrt, bool sync)
+static inline void iwl_fw_error_collect(struct iwl_fw_runtime *fwrt)
 {
 	enum iwl_fw_ini_time_point tp_id;
 
@@ -307,7 +307,7 @@ static inline void iwl_fw_error_collect(struct iwl_fw_runtime *fwrt, bool sync)
 		tp_id = IWL_FW_INI_TIME_POINT_FW_ASSERT;
 	}
 
-	_iwl_dbg_tlv_time_point(fwrt, tp_id, NULL, sync);
+	iwl_dbg_tlv_time_point_sync(fwrt, tp_id, NULL);
 }
 
 static inline void iwl_fwrt_update_fw_versions(struct iwl_fw_runtime *fwrt,
@@ -328,21 +328,19 @@ static inline void iwl_fwrt_update_fw_versions(struct iwl_fw_runtime *fwrt,
 }
 
 void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt);
-void iwl_send_dbg_dump_complete_cmd(struct iwl_fw_runtime *fwrt,
-				    u32 timepoint,
-				    u32 timepoint_data);
+bool iwl_fwrt_read_err_table(struct iwl_trans *trans, u32 base, u32 *err_id);
 void iwl_fw_disable_dbg_asserts(struct iwl_fw_runtime *fwrt);
 void iwl_fw_dbg_clear_monitor_buf(struct iwl_fw_runtime *fwrt);
 
-#define IWL_FW_CHECK_FAILED(_obj, _fmt, ...)				\
-	IWL_ERR_LIMIT(_obj, _fmt, __VA_ARGS__)
+#define IWL_FW_CHECK_FAILED(_obj, ...)					\
+	IWL_ERR_LIMIT(_obj, __VA_ARGS__)
 
 #define IWL_FW_CHECK(_obj, _cond, _fmt, ...)				\
 	({								\
 		bool __cond = (_cond);					\
 									\
 		if (unlikely(__cond))					\
-			IWL_FW_CHECK_FAILED(_obj, _fmt, __VA_ARGS__);	\
+			IWL_FW_CHECK_FAILED(_obj, _fmt, ##__VA_ARGS__);	\
 									\
 		unlikely(__cond);					\
 	})
diff --git a/sys/contrib/dev/iwlwifi/fw/debugfs.c b/sys/contrib/dev/iwlwifi/fw/debugfs.c
index 893b21fcaf87..3b0e8c43ba4a 100644
--- a/sys/contrib/dev/iwlwifi/fw/debugfs.c
+++ b/sys/contrib/dev/iwlwifi/fw/debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2023 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -123,6 +123,24 @@ static const struct file_operations iwl_dbgfs_##name##_ops = {		\
 #define FWRT_DEBUGFS_ADD_FILE(name, parent, mode) \
 	FWRT_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
 
+static ssize_t iwl_dbgfs_fw_dbg_collect_write(struct iwl_fw_runtime *fwrt,
+					      char *buf, size_t count)
+{
+	if (count == 0)
+		return 0;
+
+	if (!iwl_trans_fw_running(fwrt->trans))
+		return count;
+
+	iwl_dbg_tlv_time_point(fwrt, IWL_FW_INI_TIME_POINT_USER_TRIGGER, NULL);
+
+	iwl_fw_dbg_collect(fwrt, FW_DBG_TRIGGER_USER, buf, (count - 1), NULL);
+
+	return count;
+}
+
+FWRT_DEBUGFS_WRITE_FILE_OPS(fw_dbg_collect, 16);
+
 static int iwl_dbgfs_enabled_severities_write(struct iwl_fw_runtime *fwrt,
 					      char *buf, size_t count)
 {
@@ -180,7 +198,7 @@ void iwl_fw_trigger_timestamp(struct iwl_fw_runtime *fwrt, u32 delay)
 
 	iwl_fw_cancel_timestamp(fwrt);
 
-	fwrt->timestamp.delay = msecs_to_jiffies(delay * 1000);
+	fwrt->timestamp.delay = secs_to_jiffies(delay);
 
 	schedule_delayed_work(&fwrt->timestamp.wk,
 			      round_jiffies_relative(fwrt->timestamp.delay));
@@ -282,6 +300,26 @@ static ssize_t iwl_dbgfs_fw_dbg_domain_read(struct iwl_fw_runtime *fwrt,
 
 FWRT_DEBUGFS_READ_FILE_OPS(fw_dbg_domain, 20);
 
+static ssize_t iwl_dbgfs_fw_ver_read(struct iwl_fw_runtime *fwrt,
+				     size_t size, char *buf)
+{
+	char *pos = buf;
+	char *endpos = buf + size;
+
+	pos += scnprintf(pos, endpos - pos, "FW id: %s\n",
+			 fwrt->fw->fw_version);
+	pos += scnprintf(pos, endpos - pos, "FW: %s\n",
+			 fwrt->fw->human_readable);
+	pos += scnprintf(pos, endpos - pos, "Device: %s\n",
+			 fwrt->trans->info.name);
+	pos += scnprintf(pos, endpos - pos, "Bus: %s\n",
+			 fwrt->dev->bus->name);
+
+	return pos - buf;
+}
+
+FWRT_DEBUGFS_READ_FILE_OPS(fw_ver, 1024);
+
 struct iwl_dbgfs_fw_info_priv {
 	struct iwl_fw_runtime *fwrt;
 };
@@ -351,6 +389,12 @@ static int iwl_dbgfs_fw_info_seq_show(struct seq_file *seq, void *v)
 			   "    %d: %d\n",
 			   IWL_UCODE_TLV_CAPA_CHINA_22_REG_SUPPORT,
 			   has_capa);
+		has_capa = fw_has_capa(&fw->ucode_capa,
+				       IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE) ? 1 : 0;
+		seq_printf(seq,
+			   "    %d: %d\n",
+			   IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE,
+			   has_capa);
 		seq_puts(seq, "fw_api_ver:\n");
 	}
 
@@ -403,5 +447,7 @@ void iwl_fwrt_dbgfs_register(struct iwl_fw_runtime *fwrt,
 	FWRT_DEBUGFS_ADD_FILE(fw_info, dbgfs_dir, 0200);
 	FWRT_DEBUGFS_ADD_FILE(send_hcmd, dbgfs_dir, 0200);
 	FWRT_DEBUGFS_ADD_FILE(enabled_severities, dbgfs_dir, 0200);
+	FWRT_DEBUGFS_ADD_FILE(fw_dbg_collect, dbgfs_dir, 0200);
 	FWRT_DEBUGFS_ADD_FILE(fw_dbg_domain, dbgfs_dir, 0400);
+	FWRT_DEBUGFS_ADD_FILE(fw_ver, dbgfs_dir, 0400);
 }
diff --git a/sys/contrib/dev/iwlwifi/fw/dhc-utils.h b/sys/contrib/dev/iwlwifi/fw/dhc-utils.h
new file mode 100644
index 000000000000..983acee5cd7d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/fw/dhc-utils.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2021, 2025 Intel Corporation
+ */
+#ifndef __iwl_fw_dhc_utils_h__
+#define __iwl_fw_dhc_utils_h__
+
+#include <linux/types.h>
+#include "fw/img.h"
+#include "api/commands.h"
+#include "api/dhc.h"
+
+/**
+ * iwl_dhc_resp_status - return status of DHC response
+ * @fw: firwmware image information
+ * @pkt: response packet, must not be %NULL
+ *
+ * Returns: the status value of the DHC command or (u32)-1 if the
+ *	    response was too short.
+ */
+static inline u32 iwl_dhc_resp_status(const struct iwl_fw *fw,
+				      struct iwl_rx_packet *pkt)
+{
+	if (iwl_fw_lookup_notif_ver(fw, IWL_ALWAYS_LONG_GROUP,
+				    DEBUG_HOST_COMMAND, 1) >= 2) {
+		struct iwl_dhc_cmd_resp *resp = (void *)pkt->data;
+
+		if (iwl_rx_packet_payload_len(pkt) < sizeof(*resp))
+			return (u32)-1;
+
+		return le32_to_cpu(resp->status);
+	} else {
+		struct iwl_dhc_cmd_resp_v1 *resp = (void *)pkt->data;
+
+		if (iwl_rx_packet_payload_len(pkt) < sizeof(*resp))
+			return (u32)-1;
+
+		return le32_to_cpu(resp->status);
+	}
+}
+
+/**
+ * iwl_dhc_resp_data - return data pointer of DHC response
+ * @fw: firwmware image information
+ * @pkt: response packet, must not be %NULL
+ * @len: where to store the length
+ *
+ * Returns: The data pointer, or an ERR_PTR() if the data was
+ *	    not valid (too short).
+ */
+static inline void *iwl_dhc_resp_data(const struct iwl_fw *fw,
+				      struct iwl_rx_packet *pkt,
+				      unsigned int *len)
+{
+	if (iwl_fw_lookup_notif_ver(fw, IWL_ALWAYS_LONG_GROUP,
+				    DEBUG_HOST_COMMAND, 1) >= 2) {
+		struct iwl_dhc_cmd_resp *resp = (void *)pkt->data;
+
+		if (iwl_rx_packet_payload_len(pkt) < sizeof(*resp))
+			return ERR_PTR(-EINVAL);
+
+		*len = iwl_rx_packet_payload_len(pkt) - sizeof(*resp);
+		return (void *)&resp->data;
+	} else {
+		struct iwl_dhc_cmd_resp_v1 *resp = (void *)pkt->data;
+
+		if (iwl_rx_packet_payload_len(pkt) < sizeof(*resp))
+			return ERR_PTR(-EINVAL);
+
+		*len = iwl_rx_packet_payload_len(pkt) - sizeof(*resp);
+		return (void *)&resp->data;
+	}
+}
+
+#endif  /* __iwl_fw_dhc_utils_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/dump.c b/sys/contrib/dev/iwlwifi/fw/dump.c
index 8f107ceec407..f633124979ab 100644
--- a/sys/contrib/dev/iwlwifi/fw/dump.c
+++ b/sys/contrib/dev/iwlwifi/fw/dump.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2023 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -199,7 +199,7 @@ static void iwl_fwrt_dump_lmac_error_log(struct iwl_fw_runtime *fwrt, u8 lmac_nu
 		IWL_ERR(trans, "HW error, resetting before reading\n");
 
 		/* reset the device */
-		err = iwl_trans_sw_reset(trans, true);
+		err = iwl_trans_sw_reset(trans);
 		if (err)
 			return;
 
@@ -417,10 +417,10 @@ static void iwl_fwrt_dump_iml_error_log(struct iwl_fw_runtime *fwrt)
 	struct iwl_trans *trans = fwrt->trans;
 	u32 error, data1;
 
-	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
 		error = UMAG_SB_CPU_2_STATUS;
 		data1 = UMAG_SB_CPU_1_STATUS;
-	} else if (fwrt->trans->trans_cfg->device_family >=
+	} else if (fwrt->trans->mac_cfg->device_family >=
 		   IWL_DEVICE_FAMILY_8000) {
 		error = SB_CPU_2_STATUS;
 		data1 = SB_CPU_1_STATUS;
@@ -439,7 +439,7 @@ static void iwl_fwrt_dump_iml_error_log(struct iwl_fw_runtime *fwrt)
 	IWL_ERR(fwrt, "0x%08X | IML/ROM data1\n",
 		iwl_read_umac_prph(trans, data1));
 
-	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000)
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000)
 		IWL_ERR(fwrt, "0x%08X | IML/ROM WFPM_AUTH_KEY_0\n",
 			iwl_read_umac_prph(trans, SB_MODIFY_CFG_FLAG));
 }
@@ -490,7 +490,7 @@ void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt)
 	struct iwl_pc_data *pc_data;
 	u8 count;
 
-	if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status)) {
+	if (!iwl_trans_device_enabled(fwrt->trans)) {
 		IWL_ERR(fwrt,
 			"DEVICE_ENABLED bit is not set. Aborting dump.\n");
 		return;
@@ -508,7 +508,7 @@ void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt)
 		iwl_fwrt_dump_rcm_error_log(fwrt, 1);
 	iwl_fwrt_dump_iml_error_log(fwrt);
 	iwl_fwrt_dump_fseq_regs(fwrt);
-	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
 		pc_data = fwrt->trans->dbg.pc_data;
 
 		if (!iwl_trans_grab_nic_access(fwrt->trans))
@@ -522,7 +522,7 @@ void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt)
 		iwl_trans_release_nic_access(fwrt->trans);
 	}
 
-	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		u32 scratch = iwl_read32(fwrt->trans, CSR_FUNC_SCRATCH);
 
 		IWL_ERR(fwrt, "Function Scratch status:\n");
@@ -530,3 +530,31 @@ void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt)
 	}
 }
 IWL_EXPORT_SYMBOL(iwl_fwrt_dump_error_logs);
+
+bool iwl_fwrt_read_err_table(struct iwl_trans *trans, u32 base, u32 *err_id)
+{
+	struct error_table_start {
+		/* cf. struct iwl_error_event_table */
+		u32 valid;
+		__le32 err_id;
+	} err_info = {};
+	int ret;
+
+	if (err_id)
+		*err_id = 0;
+
+	if (!base)
+		return false;
+
+	ret = iwl_trans_read_mem_bytes(trans, base,
+				       &err_info, sizeof(err_info));
+
+	if (ret)
+		return true;
+
+	if (err_info.valid && err_id)
+		*err_id = le32_to_cpu(err_info.err_id);
+
+	return !!err_info.valid;
+}
+IWL_EXPORT_SYMBOL(iwl_fwrt_read_err_table);
diff --git a/sys/contrib/dev/iwlwifi/fw/error-dump.h b/sys/contrib/dev/iwlwifi/fw/error-dump.h
index e63b08b7d336..cf41021d59ad 100644
--- a/sys/contrib/dev/iwlwifi/fw/error-dump.h
+++ b/sys/contrib/dev/iwlwifi/fw/error-dump.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2014, 2018-2025 Intel Corporation
  * Copyright (C) 2014-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -169,7 +169,7 @@ struct iwl_fw_error_dump_info {
  * @fw_mon_wr_ptr: the position of the write pointer in the cyclic buffer
  * @fw_mon_base_ptr: base pointer of the data
  * @fw_mon_cycle_cnt: number of wraparounds
- * @fw_mon_base_high_ptr: used in AX210 devices, the base adderss is 64 bit
+ * @fw_mon_base_high_ptr: used in AX210 devices, the base address is 64 bit
  *	so fw_mon_base_ptr holds LSB 32 bits and fw_mon_base_high_ptr hold
  *	MSB 32 bits
  * @reserved: for future use
@@ -372,10 +372,7 @@ struct iwl_fw_ini_dump_cfg_name {
 	u8 cfg_name[IWL_FW_INI_MAX_CFG_NAME];
 } __packed;
 
-/* AX210's HW type */
-#define IWL_AX210_HW_TYPE 0x42
-/* How many bits to roll when adding to the HW type of AX210 HW */
-#define IWL_AX210_HW_TYPE_ADDITION_SHIFT 12
+#define IWL_JACKET_CDB_SHIFT 12
 
 /* struct iwl_fw_ini_dump_info - ini dump information
  * @version: dump version
diff --git a/sys/contrib/dev/iwlwifi/fw/file.h b/sys/contrib/dev/iwlwifi/fw/file.h
index ae05227b6153..b7c1ab7a3006 100644
--- a/sys/contrib/dev/iwlwifi/fw/file.h
+++ b/sys/contrib/dev/iwlwifi/fw/file.h
@@ -102,8 +102,13 @@ enum iwl_ucode_tlv_type {
 	IWL_UCODE_TLV_SEC_TABLE_ADDR		= 66,
 	IWL_UCODE_TLV_D3_KEK_KCK_ADDR		= 67,
 	IWL_UCODE_TLV_CURRENT_PC		= 68,
+	IWL_UCODE_TLV_FSEQ_BIN_VERSION		= 72,
+
+	/* contains sub-sections like PNVM file does (did) */
+	IWL_UCODE_TLV_PNVM_DATA			= 74,
 
 	IWL_UCODE_TLV_FW_NUM_STATIONS		= IWL_UCODE_TLV_CONST_BASE + 0,
+	IWL_UCODE_TLV_FW_NUM_LINKS		= IWL_UCODE_TLV_CONST_BASE + 1,
 	IWL_UCODE_TLV_FW_NUM_BEACONS		= IWL_UCODE_TLV_CONST_BASE + 2,
 
 	IWL_UCODE_TLV_TYPE_DEBUG_INFO		= IWL_UCODE_TLV_DEBUG_BASE + 0,
@@ -384,7 +389,8 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
  *	to report the CSI information with (certain) RX frames
  * @IWL_UCODE_TLV_CAPA_FTM_CALIBRATED: has FTM calibrated and thus supports both
  *	initiator and responder
- * @IWL_UCODE_TLV_CAPA_MLME_OFFLOAD: supports MLME offload
+ * @IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_UNII4_US_CA: supports (de)activating UNII-4
+ *	for US/CA/WW from BIOS
  * @IWL_UCODE_TLV_CAPA_PROTECTED_TWT: Supports protection of TWT action frames
  * @IWL_UCODE_TLV_CAPA_FW_RESET_HANDSHAKE: Supports the firmware handshake in
  *	reset flow
@@ -397,6 +403,12 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
  * @IWL_UCODE_TLV_CAPA_SECURE_LTF_SUPPORT: Support secure LTF measurement.
  * @IWL_UCODE_TLV_CAPA_MONITOR_PASSIVE_CHANS: Support monitor mode on otherwise
  *	passive channels
+ * @IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_5G9_FOR_CA: supports (de)activating 5G9
+ *	for CA from BIOS.
+ * @IWL_UCODE_TLV_CAPA_UHB_CANADA_TAS_SUPPORT: supports %TAS_UHB_ALLOWED_CANADA
+ * @IWL_UCODE_TLV_CAPA_EXT_FSEQ_IMAGE_SUPPORT: external FSEQ image support
+ * @IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE: Firmware has capability of
+ *	handling raw DSM table data.
  *
  * @NUM_IWL_UCODE_TLV_CAPA: number of bits used
  */
@@ -474,7 +486,7 @@ enum iwl_ucode_tlv_capa {
 	IWL_UCODE_TLV_CAPA_DBG_BUF_ALLOC_CMD_SUPP	= (__force iwl_ucode_tlv_capa_t)93,
 
 	/* set 3 */
-	IWL_UCODE_TLV_CAPA_MLME_OFFLOAD			= (__force iwl_ucode_tlv_capa_t)96,
+	IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_UNII4_US_CA	= (__force iwl_ucode_tlv_capa_t)96,
 
 	/*
 	 * @IWL_UCODE_TLV_CAPA_PSC_CHAN_SUPPORT: supports PSC channels
@@ -497,6 +509,17 @@ enum iwl_ucode_tlv_capa {
 	IWL_UCODE_TLV_CAPA_CHINA_22_REG_SUPPORT		= (__force iwl_ucode_tlv_capa_t)117,
 	IWL_UCODE_TLV_CAPA_SECURE_LTF_SUPPORT		= (__force iwl_ucode_tlv_capa_t)121,
 	IWL_UCODE_TLV_CAPA_MONITOR_PASSIVE_CHANS	= (__force iwl_ucode_tlv_capa_t)122,
+	IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_5G9_FOR_CA	= (__force iwl_ucode_tlv_capa_t)123,
+	IWL_UCODE_TLV_CAPA_UHB_CANADA_TAS_SUPPORT	= (__force iwl_ucode_tlv_capa_t)124,
+	IWL_UCODE_TLV_CAPA_EXT_FSEQ_IMAGE_SUPPORT	= (__force iwl_ucode_tlv_capa_t)125,
+
+	/* set 4 */
+	/**
+	 * @IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT: FW reset handshake is needed
+	 *	during assert handling even if the dump isn't split
+	 */
+	IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT		= (__force iwl_ucode_tlv_capa_t)(4 * 32 +  0),
+	IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE 	= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 1),
 	NUM_IWL_UCODE_TLV_CAPA
 /*
  * This construction make both sparse (which cannot increment the previous
@@ -987,6 +1010,10 @@ struct iwl_fw_dump_exclude {
 	__le32 addr, size;
 };
 
+struct iwl_fw_fseq_bin_version {
+	__le32 major, minor;
+}; /* FW_TLV_FSEQ_BIN_VERSION_S */
+
 static inline size_t _iwl_tlv_array_len(const struct iwl_ucode_tlv *tlv,
 					size_t fixed_size, size_t var_size)
 {
@@ -1004,4 +1031,18 @@ static inline size_t _iwl_tlv_array_len(const struct iwl_ucode_tlv *tlv,
 
 #define iwl_tlv_array_len_with_size(_tlv_ptr, _struct_ptr, _size)	\
 	_iwl_tlv_array_len((_tlv_ptr), sizeof(*(_struct_ptr)), _size)
+
+/* external FSEQ file */
+#define IWL_FSEQ_FILE	"intel/fseq-%04x-%04x"
+#define IWL_FSEQ_MAGIC	"INTEL-CNV-FSEQ\n\0"
+
+struct iwl_fseq_file {
+	char magic[16];
+	char version[16];
+	__le32 bt_len;
+	__le32 wifi_len;
+	u8 reserved[8];
+	u8 data[];
+} __packed;
+
 #endif  /* __iwl_fw_file_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/img.c b/sys/contrib/dev/iwlwifi/fw/img.c
index b7deca05a953..c2f4fc83a22c 100644
--- a/sys/contrib/dev/iwlwifi/fw/img.c
+++ b/sys/contrib/dev/iwlwifi/fw/img.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright(c) 2019 - 2021 Intel Corporation
+ * Copyright(c) 2024 Intel Corporation
  */
 #include <fw/api/commands.h>
 #include "img.h"
@@ -75,6 +76,7 @@ static const struct {
 	{ "NMI_INTERRUPT_ACTION_PT", 0x7C },
 	{ "NMI_INTERRUPT_UNKNOWN", 0x84 },
 	{ "NMI_INTERRUPT_INST_ACTION_PT", 0x86 },
+	{ "NMI_INTERRUPT_PREG", 0x88 },
 	{ "PNVM_MISSING", FW_SYSASSERT_PNVM_MISSING },
 	{ "ADVANCED_SYSASSERT", 0 },
 };
diff --git a/sys/contrib/dev/iwlwifi/fw/img.h b/sys/contrib/dev/iwlwifi/fw/img.h
index 96bda80632f3..5256f20623e9 100644
--- a/sys/contrib/dev/iwlwifi/fw/img.h
+++ b/sys/contrib/dev/iwlwifi/fw/img.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2023 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -51,9 +51,10 @@ struct iwl_ucode_capabilities {
 	u32 error_log_addr;
 	u32 error_log_size;
 	u32 num_stations;
+	u32 num_links;
 	u32 num_beacons;
-	unsigned long _api[BITS_TO_LONGS(NUM_IWL_UCODE_TLV_API)];
-	unsigned long _capa[BITS_TO_LONGS(NUM_IWL_UCODE_TLV_CAPA)];
+	DECLARE_BITMAP(_api, NUM_IWL_UCODE_TLV_API);
+	DECLARE_BITMAP(_capa, NUM_IWL_UCODE_TLV_CAPA);
 
 	const struct iwl_fw_cmd_version *cmd_versions;
 	u32 n_cmd_versions;
@@ -194,6 +195,8 @@ struct iwl_dump_exclude {
  * @phy_integration_ver_len: length of @phy_integration_ver
  * @dump_excl: image dump exclusion areas for RT image
  * @dump_excl_wowlan: image dump exclusion areas for WoWLAN image
+ * @pnvm_data: PNVM data embedded in the .ucode file, if any
+ * @pnvm_size: size of the embedded PNVM data
  */
 struct iwl_fw {
 	u32 ucode_ver;
@@ -226,6 +229,9 @@ struct iwl_fw {
 	u32 phy_integration_ver_len;
 
 	struct iwl_dump_exclude dump_excl[2], dump_excl_wowlan[2];
+
+	const void *pnvm_data;
+	u32 pnvm_size;
 };
 
 static inline const char *get_fw_dbg_mode_string(int mode)
diff --git a/sys/contrib/dev/iwlwifi/fw/init.c b/sys/contrib/dev/iwlwifi/fw/init.c
index d8b083be5b6b..d1d8058ad29f 100644
--- a/sys/contrib/dev/iwlwifi/fw/init.c
+++ b/sys/contrib/dev/iwlwifi/fw/init.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2019-2021, 2024 Intel Corporation
+ * Copyright (C) 2019-2021, 2024-2025 Intel Corporation
  */
 #include "iwl-drv.h"
 #include "runtime.h"
@@ -39,10 +39,12 @@ void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
 }
 IWL_EXPORT_SYMBOL(iwl_fw_runtime_init);
 
+/* Assumes the appropriate lock is held by the caller */
 void iwl_fw_runtime_suspend(struct iwl_fw_runtime *fwrt)
 {
 	iwl_fw_suspend_timestamp(fwrt);
-	iwl_dbg_tlv_time_point(fwrt, IWL_FW_INI_TIME_POINT_HOST_D3_START, NULL);
+	iwl_dbg_tlv_time_point_sync(fwrt, IWL_FW_INI_TIME_POINT_HOST_D3_START,
+				    NULL);
 }
 IWL_EXPORT_SYMBOL(iwl_fw_runtime_suspend);
 
@@ -70,7 +72,7 @@ int iwl_set_soc_latency(struct iwl_fw_runtime *fwrt)
 	 * values in VER_1, this is backwards-compatible with VER_2,
 	 * as long as we don't set any other bits.
 	 */
-	if (!fwrt->trans->trans_cfg->integrated)
+	if (!fwrt->trans->mac_cfg->integrated)
 		cmd.flags = cpu_to_le32(SOC_CONFIG_CMD_FLAGS_DISCRETE);
 
 	BUILD_BUG_ON(IWL_CFG_TRANS_LTR_DELAY_NONE !=
@@ -82,17 +84,17 @@ int iwl_set_soc_latency(struct iwl_fw_runtime *fwrt)
 	BUILD_BUG_ON(IWL_CFG_TRANS_LTR_DELAY_1820US !=
 		     SOC_FLAGS_LTR_APPLY_DELAY_1820);
 
-	if (fwrt->trans->trans_cfg->ltr_delay != IWL_CFG_TRANS_LTR_DELAY_NONE &&
-	    !WARN_ON(!fwrt->trans->trans_cfg->integrated))
-		cmd.flags |= le32_encode_bits(fwrt->trans->trans_cfg->ltr_delay,
+	if (fwrt->trans->mac_cfg->ltr_delay != IWL_CFG_TRANS_LTR_DELAY_NONE &&
+	    !WARN_ON(!fwrt->trans->mac_cfg->integrated))
+		cmd.flags |= le32_encode_bits(fwrt->trans->mac_cfg->ltr_delay,
 					      SOC_FLAGS_LTR_APPLY_DELAY_MASK);
 
 	if (iwl_fw_lookup_cmd_ver(fwrt->fw, SCAN_REQ_UMAC,
 				  IWL_FW_CMD_VER_UNKNOWN) >= 2 &&
-	    fwrt->trans->trans_cfg->low_latency_xtal)
+	    fwrt->trans->mac_cfg->low_latency_xtal)
 		cmd.flags |= cpu_to_le32(SOC_CONFIG_CMD_FLAGS_LOW_LATENCY);
 
-	cmd.latency = cpu_to_le32(fwrt->trans->trans_cfg->xtal_latency);
+	cmd.latency = cpu_to_le32(fwrt->trans->mac_cfg->xtal_latency);
 
 	ret = iwl_trans_send_cmd(fwrt->trans, &hcmd);
 	if (ret)
@@ -114,14 +116,14 @@ int iwl_configure_rxq(struct iwl_fw_runtime *fwrt)
 	 * The default queue is configured via context info, so if we
 	 * have a single queue, there's nothing to do here.
 	 */
-	if (fwrt->trans->num_rx_queues == 1)
+	if (fwrt->trans->info.num_rxqs == 1)
 		return 0;
 
-	if (fwrt->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_22000)
+	if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_22000)
 		return 0;
 
 	/* skip the default queue */
-	num_queues = fwrt->trans->num_rx_queues - 1;
+	num_queues = fwrt->trans->info.num_rxqs - 1;
 
 	size = struct_size(cmd, data, num_queues);
 
diff --git a/sys/contrib/dev/iwlwifi/fw/paging.c b/sys/contrib/dev/iwlwifi/fw/paging.c
index 945bc4160cc9..826409f6f710 100644
--- a/sys/contrib/dev/iwlwifi/fw/paging.c
+++ b/sys/contrib/dev/iwlwifi/fw/paging.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2019, 2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2021, 2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -249,7 +249,7 @@ static int iwl_send_paging_cmd(struct iwl_fw_runtime *fwrt,
 	};
 	int blk_idx;
 
-	/* loop for for all paging blocks + CSS block */
+	/* loop for all paging blocks + CSS block */
 	for (blk_idx = 0; blk_idx < fwrt->num_of_paging_blk + 1; blk_idx++) {
 		dma_addr_t addr = fwrt->fw_paging_db[blk_idx].fw_paging_phys;
 		__le32 phy_addr;
@@ -267,7 +267,7 @@ int iwl_init_paging(struct iwl_fw_runtime *fwrt, enum iwl_ucode_type type)
 	const struct fw_img *fw = &fwrt->fw->img[type];
 	int ret;
 
-	if (fwrt->trans->trans_cfg->gen2)
+	if (fwrt->trans->mac_cfg->gen2)
 		return 0;
 
 	/*
diff --git a/sys/contrib/dev/iwlwifi/fw/pnvm.c b/sys/contrib/dev/iwlwifi/fw/pnvm.c
index 1195e708caa9..4d91ae065c8d 100644
--- a/sys/contrib/dev/iwlwifi/fw/pnvm.c
+++ b/sys/contrib/dev/iwlwifi/fw/pnvm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright(c) 2020-2024 Intel Corporation
+ * Copyright(c) 2020-2025 Intel Corporation
  */
 
 #include "iwl-drv.h"
@@ -11,6 +11,7 @@
 #include "fw/api/nvm-reg.h"
 #include "fw/api/alive.h"
 #include "fw/uefi.h"
+#include "fw/img.h"
 
 #define IWL_PNVM_REDUCED_CAP_BIT BIT(25)
 
@@ -96,8 +97,8 @@ static int iwl_pnvm_handle_section(struct iwl_trans *trans, const u8 *data,
 				     "Got IWL_UCODE_TLV_HW_TYPE mac_type 0x%0x rf_id 0x%0x\n",
 				     mac_type, rf_id);
 
-			if (mac_type == CSR_HW_REV_TYPE(trans->hw_rev) &&
-			    rf_id == CSR_HW_RFID_TYPE(trans->hw_rf_id))
+			if (mac_type == CSR_HW_REV_TYPE(trans->info.hw_rev) &&
+			    rf_id == CSR_HW_RFID_TYPE(trans->info.hw_rf_id))
 				hw_match = true;
 			break;
 		case IWL_UCODE_TLV_SEC_RT: {
@@ -152,8 +153,8 @@ done:
 	if (!hw_match) {
 		IWL_DEBUG_FW(trans,
 			     "HW mismatch, skipping PNVM section (need mac_type 0x%x rf_id 0x%x)\n",
-			     CSR_HW_REV_TYPE(trans->hw_rev),
-			     CSR_HW_RFID_TYPE(trans->hw_rf_id));
+			     CSR_HW_REV_TYPE(trans->info.hw_rev),
+			     CSR_HW_RFID_TYPE(trans->info.hw_rf_id));
 		return -ENOENT;
 	}
 
@@ -167,7 +168,8 @@ done:
 
 static int iwl_pnvm_parse(struct iwl_trans *trans, const u8 *data,
 			  size_t len,
-			  struct iwl_pnvm_image *pnvm_data)
+			  struct iwl_pnvm_image *pnvm_data,
+			  __le32 sku_id[3])
 {
 	const struct iwl_ucode_tlv *tlv;
 
@@ -190,23 +192,23 @@ static int iwl_pnvm_parse(struct iwl_trans *trans, const u8 *data,
 		}
 
 		if (tlv_type == IWL_UCODE_TLV_PNVM_SKU) {
-			const struct iwl_sku_id *sku_id =
+			const struct iwl_sku_id *tlv_sku_id =
 				(const void *)(data + sizeof(*tlv));
 
 			IWL_DEBUG_FW(trans,
 				     "Got IWL_UCODE_TLV_PNVM_SKU len %d\n",
 				     tlv_len);
 			IWL_DEBUG_FW(trans, "sku_id 0x%0x 0x%0x 0x%0x\n",
-				     le32_to_cpu(sku_id->data[0]),
-				     le32_to_cpu(sku_id->data[1]),
-				     le32_to_cpu(sku_id->data[2]));
+				     le32_to_cpu(tlv_sku_id->data[0]),
+				     le32_to_cpu(tlv_sku_id->data[1]),
+				     le32_to_cpu(tlv_sku_id->data[2]));
 
 			data += sizeof(*tlv) + ALIGN(tlv_len, 4);
 			len -= ALIGN(tlv_len, 4);
 
 			trans->reduced_cap_sku = false;
-			rf_type = CSR_HW_RFID_TYPE(trans->hw_rf_id);
-			if ((trans->sku_id[0] & IWL_PNVM_REDUCED_CAP_BIT) &&
+			rf_type = CSR_HW_RFID_TYPE(trans->info.hw_rf_id);
+			if ((sku_id[0] & cpu_to_le32(IWL_PNVM_REDUCED_CAP_BIT)) &&
 			    rf_type == IWL_CFG_RF_TYPE_FM)
 				trans->reduced_cap_sku = true;
 
@@ -214,9 +216,9 @@ static int iwl_pnvm_parse(struct iwl_trans *trans, const u8 *data,
 				     "Reduced SKU device %d\n",
 				     trans->reduced_cap_sku);
 
-			if (trans->sku_id[0] == le32_to_cpu(sku_id->data[0]) &&
-			    trans->sku_id[1] == le32_to_cpu(sku_id->data[1]) &&
-			    trans->sku_id[2] == le32_to_cpu(sku_id->data[2])) {
+			if (sku_id[0] == tlv_sku_id->data[0] &&
+			    sku_id[1] == tlv_sku_id->data[1] &&
+			    sku_id[2] == tlv_sku_id->data[2]) {
 				int ret;
 
 				ret = iwl_pnvm_handle_section(trans, data, len,
@@ -263,13 +265,14 @@ static int iwl_pnvm_get_from_fs(struct iwl_trans *trans, u8 **data, size_t *len)
 	return 0;
 }
 
-static u8 *iwl_get_pnvm_image(struct iwl_trans *trans_p, size_t *len)
+static const u8 *iwl_get_pnvm_image(struct iwl_trans *trans_p, size_t *len,
+				    __le32 sku_id[3], const struct iwl_fw *fw)
 {
 	struct pnvm_sku_package *package;
 	u8 *image = NULL;
 
 	/* Get PNVM from BIOS for non-Intel SKU */
-	if (trans_p->sku_id[2]) {
+	if (sku_id[2]) {
 		package = iwl_uefi_get_pnvm(trans_p, len);
 		if (!IS_ERR_OR_NULL(package)) {
 			if (*len >= sizeof(*package)) {
@@ -288,17 +291,25 @@ static u8 *iwl_get_pnvm_image(struct iwl_trans *trans_p, size_t *len)
 		}
 	}
 
+	if (fw->pnvm_data) {
+		*len = fw->pnvm_size;
+
+		return fw->pnvm_data;
+	}
+
 	/* If it's not available, or for Intel SKU, try from the filesystem */
 	if (iwl_pnvm_get_from_fs(trans_p, &image, len))
 		return NULL;
 	return image;
 }
 
-static void iwl_pnvm_load_pnvm_to_trans(struct iwl_trans *trans,
-					const struct iwl_ucode_capabilities *capa)
+static void
+iwl_pnvm_load_pnvm_to_trans(struct iwl_trans *trans,
+			    const struct iwl_fw *fw,
+			    __le32 sku_id[3])
 {
 	struct iwl_pnvm_image *pnvm_data = NULL;
-	u8 *data = NULL;
+	const u8 *data = NULL;
 	size_t length;
 	int ret;
 
@@ -309,7 +320,7 @@ static void iwl_pnvm_load_pnvm_to_trans(struct iwl_trans *trans,
 	if (trans->pnvm_loaded)
 		goto set;
 
-	data = iwl_get_pnvm_image(trans, &length);
+	data = iwl_get_pnvm_image(trans, &length, sku_id, fw);
 	if (!data) {
 		trans->fail_to_parse_pnvm_image = true;
 		return;
@@ -319,27 +330,30 @@ static void iwl_pnvm_load_pnvm_to_trans(struct iwl_trans *trans,
 	if (!pnvm_data)
 		goto free;
 
-	ret = iwl_pnvm_parse(trans, data, length, pnvm_data);
+	ret = iwl_pnvm_parse(trans, data, length, pnvm_data, sku_id);
 	if (ret) {
 		trans->fail_to_parse_pnvm_image = true;
 		goto free;
 	}
 
-	ret = iwl_trans_load_pnvm(trans, pnvm_data, capa);
+	ret = iwl_trans_load_pnvm(trans, pnvm_data, &fw->ucode_capa);
 	if (ret)
 		goto free;
-	IWL_INFO(trans, "loaded PNVM version %08x\n", pnvm_data->version);
+	IWL_DEBUG_INFO(trans, "loaded PNVM version %08x\n", pnvm_data->version);
 
 set:
-	iwl_trans_set_pnvm(trans, capa);
+	iwl_trans_set_pnvm(trans, &fw->ucode_capa);
 free:
-	kvfree(data);
+	/* free only if it was allocated, i.e. not just embedded PNVM data */
+	if (data != fw->pnvm_data)
+		kvfree(data);
 	kfree(pnvm_data);
 }
 
 static void
 iwl_pnvm_load_reduce_power_to_trans(struct iwl_trans *trans,
-				    const struct iwl_ucode_capabilities *capa)
+				    const struct iwl_ucode_capabilities *capa,
+				    __le32 sku_id[3])
 {
 	struct iwl_pnvm_image *pnvm_data = NULL;
 	u8 *data = NULL;
@@ -362,7 +376,8 @@ iwl_pnvm_load_reduce_power_to_trans(struct iwl_trans *trans,
 	if (!pnvm_data)
 		goto free;
 
-	ret = iwl_uefi_reduce_power_parse(trans, data, length, pnvm_data);
+	ret = iwl_uefi_reduce_power_parse(trans, data, length, pnvm_data,
+					  sku_id);
 	if (ret) {
 		trans->failed_to_load_reduce_power_image = true;
 		goto free;
@@ -386,18 +401,18 @@ free:
 
 int iwl_pnvm_load(struct iwl_trans *trans,
 		  struct iwl_notif_wait_data *notif_wait,
-		  const struct iwl_ucode_capabilities *capa)
+		  const struct iwl_fw *fw, __le32 sku_id[3])
 {
 	struct iwl_notification_wait pnvm_wait;
 	static const u16 ntf_cmds[] = { WIDE_ID(REGULATORY_AND_NVM_GROUP,
 						PNVM_INIT_COMPLETE_NTFY) };
 
 	/* if the SKU_ID is empty, there's nothing to do */
-	if (!trans->sku_id[0] && !trans->sku_id[1] && !trans->sku_id[2])
+	if (!sku_id[0] && !sku_id[1] && !sku_id[2])
 		return 0;
 
-	iwl_pnvm_load_pnvm_to_trans(trans, capa);
-	iwl_pnvm_load_reduce_power_to_trans(trans, capa);
+	iwl_pnvm_load_pnvm_to_trans(trans, fw, sku_id);
+	iwl_pnvm_load_reduce_power_to_trans(trans, &fw->ucode_capa, sku_id);
 
 	iwl_init_notification_wait(notif_wait, &pnvm_wait,
 				   ntf_cmds, ARRAY_SIZE(ntf_cmds),
diff --git a/sys/contrib/dev/iwlwifi/fw/pnvm.h b/sys/contrib/dev/iwlwifi/fw/pnvm.h
index 1bac3466154c..ad3b7e2423ac 100644
--- a/sys/contrib/dev/iwlwifi/fw/pnvm.h
+++ b/sys/contrib/dev/iwlwifi/fw/pnvm.h
@@ -1,12 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright(c) 2020-2023 Intel Corporation
+ * Copyright(c) 2020-2023, 2025 Intel Corporation
  */
 #ifndef __IWL_PNVM_H__
 #define __IWL_PNVM_H__
 
 #include "iwl-drv.h"
 #include "fw/notif-wait.h"
+#include "fw/img.h"
 
 #define MVM_UCODE_PNVM_TIMEOUT	(HZ / 4)
 
@@ -14,7 +15,7 @@
 
 int iwl_pnvm_load(struct iwl_trans *trans,
 		  struct iwl_notif_wait_data *notif_wait,
-		  const struct iwl_ucode_capabilities *capa);
+		  const struct iwl_fw *fw, __le32 sku_id[3]);
 
 static inline
 void iwl_pnvm_get_fs_name(struct iwl_trans *trans,
diff --git a/sys/contrib/dev/iwlwifi/fw/regulatory.c b/sys/contrib/dev/iwlwifi/fw/regulatory.c
index 777564068166..768350086a06 100644
--- a/sys/contrib/dev/iwlwifi/fw/regulatory.c
+++ b/sys/contrib/dev/iwlwifi/fw/regulatory.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2023 Intel Corporation
+ * Copyright (C) 2023, 2025 Intel Corporation
  */
 #if defined(__FreeBSD__)
 #include <linux/bitfield.h>
@@ -37,11 +37,13 @@ IWL_BIOS_TABLE_LOADER(wrds_table);
 IWL_BIOS_TABLE_LOADER(ewrd_table);
 IWL_BIOS_TABLE_LOADER(wgds_table);
 IWL_BIOS_TABLE_LOADER(ppag_table);
+IWL_BIOS_TABLE_LOADER(phy_filters);
 IWL_BIOS_TABLE_LOADER_DATA(tas_table, struct iwl_tas_data);
 IWL_BIOS_TABLE_LOADER_DATA(pwr_limit, u64);
 IWL_BIOS_TABLE_LOADER_DATA(mcc, char);
 IWL_BIOS_TABLE_LOADER_DATA(eckv, u32);
 IWL_BIOS_TABLE_LOADER_DATA(wbem, u32);
+IWL_BIOS_TABLE_LOADER_DATA(dsbr, u32);
 
 
 static const struct dmi_system_id dmi_ppag_approved_list[] = {
@@ -103,6 +105,11 @@ static const struct dmi_system_id dmi_ppag_approved_list[] = {
 			DMI_MATCH(DMI_SYS_VENDOR, "HONOR"),
 		},
 	},
+	{ .ident = "WIKO",
+	  .matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "WIKO"),
+		},
+	},
 	{}
 };
 
@@ -177,9 +184,9 @@ bool iwl_sar_geo_support(struct iwl_fw_runtime *fwrt)
 	 */
 	return IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) >= 38 ||
 		(IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) == 17 &&
-		 fwrt->trans->hw_rev != CSR_HW_REV_TYPE_3160) ||
+		 fwrt->trans->info.hw_rev != CSR_HW_REV_TYPE_3160) ||
 		(IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) == 29 &&
-		 ((fwrt->trans->hw_rev & CSR_HW_REV_TYPE_MSK) ==
+		 ((fwrt->trans->info.hw_rev & CSR_HW_REV_TYPE_MSK) ==
 		  CSR_HW_REV_TYPE_7265D));
 }
 IWL_EXPORT_SYMBOL(iwl_sar_geo_support);
@@ -310,7 +317,7 @@ int iwl_fill_ppag_table(struct iwl_fw_runtime *fwrt,
 	bool send_ppag_always;
 
 	/* many firmware images for JF lie about this */
-	if (CSR_HW_RFID_TYPE(fwrt->trans->hw_rf_id) ==
+	if (CSR_HW_RFID_TYPE(fwrt->trans->info.hw_rf_id) ==
 	    CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_JF))
 		return -EOPNOTSUPP;
 
@@ -335,31 +342,35 @@ int iwl_fill_ppag_table(struct iwl_fw_runtime *fwrt,
 		return -EINVAL;
 	}
 
-	/* The 'flags' field is the same in v1 and in v2 so we can just
-	 * use v1 to access it.
-	 */
-	cmd->v1.flags = cpu_to_le32(fwrt->ppag_flags);
-
 	IWL_DEBUG_RADIO(fwrt, "PPAG cmd ver is %d\n", cmd_ver);
 	if (cmd_ver == 1) {
 		num_sub_bands = IWL_NUM_SUB_BANDS_V1;
 		gain = cmd->v1.gain[0];
 		*cmd_size = sizeof(cmd->v1);
-		if (fwrt->ppag_ver >= 1) {
+		cmd->v1.flags = cpu_to_le32(fwrt->ppag_flags);
+		if (fwrt->ppag_bios_rev >= 1) {
 			/* in this case FW supports revision 0 */
 			IWL_DEBUG_RADIO(fwrt,
 					"PPAG table rev is %d, send truncated table\n",
-					fwrt->ppag_ver);
+					fwrt->ppag_bios_rev);
 		}
 	} else if (cmd_ver >= 2 && cmd_ver <= 6) {
 		num_sub_bands = IWL_NUM_SUB_BANDS_V2;
 		gain = cmd->v2.gain[0];
 		*cmd_size = sizeof(cmd->v2);
-		if (fwrt->ppag_ver == 0) {
+		cmd->v2.flags = cpu_to_le32(fwrt->ppag_flags);
+		if (fwrt->ppag_bios_rev == 0) {
 			/* in this case FW supports revisions 1,2 or 3 */
 			IWL_DEBUG_RADIO(fwrt,
 					"PPAG table rev is 0, send padded table\n");
 		}
+	} else if (cmd_ver == 7) {
+		num_sub_bands = IWL_NUM_SUB_BANDS_V2;
+		gain = cmd->v3.gain[0];
+		*cmd_size = sizeof(cmd->v3);
+		cmd->v3.ppag_config_info.table_source = fwrt->ppag_bios_source;
+		cmd->v3.ppag_config_info.table_revision = fwrt->ppag_bios_rev;
+		cmd->v3.ppag_config_info.value = cpu_to_le32(fwrt->ppag_flags);
 	} else {
 		IWL_DEBUG_RADIO(fwrt, "Unsupported PPAG command version\n");
 		return -EINVAL;
@@ -368,9 +379,11 @@ int iwl_fill_ppag_table(struct iwl_fw_runtime *fwrt,
 	/* ppag mode */
 	IWL_DEBUG_RADIO(fwrt,
 			"PPAG MODE bits were read from bios: %d\n",
-			le32_to_cpu(cmd->v1.flags));
+			fwrt->ppag_flags);
 
-	if (cmd_ver == 5)
+	if (cmd_ver == 6)
+		cmd->v1.flags &= cpu_to_le32(IWL_PPAG_CMD_V6_MASK);
+	else if (cmd_ver == 5)
 		cmd->v1.flags &= cpu_to_le32(IWL_PPAG_CMD_V5_MASK);
 	else if (cmd_ver < 5)
 		cmd->v1.flags &= cpu_to_le32(IWL_PPAG_CMD_V4_MASK);
@@ -378,16 +391,20 @@ int iwl_fill_ppag_table(struct iwl_fw_runtime *fwrt,
 	if ((cmd_ver == 1 &&
 	     !fw_has_capa(&fwrt->fw->ucode_capa,
 			  IWL_UCODE_TLV_CAPA_PPAG_CHINA_BIOS_SUPPORT)) ||
-	    (cmd_ver == 2 && fwrt->ppag_ver >= 2)) {
+	    (cmd_ver == 2 && fwrt->ppag_bios_rev >= 2)) {
 		cmd->v1.flags &= cpu_to_le32(IWL_PPAG_ETSI_MASK);
 		IWL_DEBUG_RADIO(fwrt, "masking ppag China bit\n");
 	} else {
 		IWL_DEBUG_RADIO(fwrt, "isn't masking ppag China bit\n");
 	}
 
+	/* The 'flags' field is the same in v1 and v2 so we can just
+	 * use v1 to access it.
+	 */
 	IWL_DEBUG_RADIO(fwrt,
 			"PPAG MODE bits going to be sent: %d\n",
-			le32_to_cpu(cmd->v1.flags));
+			(cmd_ver < 7) ? le32_to_cpu(cmd->v1.flags) :
+					le32_to_cpu(cmd->v3.ppag_config_info.value));
 
 	for (i = 0; i < IWL_NUM_CHAIN_LIMITS; i++) {
 		for (j = 0; j < num_sub_bands; j++) {
@@ -427,33 +444,57 @@ bool iwl_is_tas_approved(void)
 }
 IWL_EXPORT_SYMBOL(iwl_is_tas_approved);
 
-int iwl_parse_tas_selection(struct iwl_fw_runtime *fwrt,
-			    struct iwl_tas_data *tas_data,
-			    const u32 tas_selection)
+struct iwl_tas_selection_data
+iwl_parse_tas_selection(const u32 tas_selection_in, const u8 tbl_rev)
 {
-	u8 override_iec = u32_get_bits(tas_selection,
+	struct iwl_tas_selection_data tas_selection_out = {};
+	u8 override_iec = u32_get_bits(tas_selection_in,
 				       IWL_WTAS_OVERRIDE_IEC_MSK);
-	u8 enabled_iec = u32_get_bits(tas_selection, IWL_WTAS_ENABLE_IEC_MSK);
-	u8 usa_tas_uhb = u32_get_bits(tas_selection, IWL_WTAS_USA_UHB_MSK);
-	int enabled = tas_selection & IWL_WTAS_ENABLED_MSK;
+	u8 canada_tas_uhb = u32_get_bits(tas_selection_in,
+					 IWL_WTAS_CANADA_UHB_MSK);
+	u8 enabled_iec = u32_get_bits(tas_selection_in,
+				      IWL_WTAS_ENABLE_IEC_MSK);
+	u8 usa_tas_uhb = u32_get_bits(tas_selection_in,
+				      IWL_WTAS_USA_UHB_MSK);
+
+	if (tbl_rev > 0) {
+		tas_selection_out.usa_tas_uhb_allowed = usa_tas_uhb;
+		tas_selection_out.override_tas_iec = override_iec;
+		tas_selection_out.enable_tas_iec = enabled_iec;
+	}
+
+	if (tbl_rev > 1)
+		tas_selection_out.canada_tas_uhb_allowed = canada_tas_uhb;
+
+	return tas_selection_out;
+}
+IWL_EXPORT_SYMBOL(iwl_parse_tas_selection);
 
-	IWL_DEBUG_RADIO(fwrt, "TAS selection as read from BIOS: 0x%x\n",
-			tas_selection);
+bool iwl_add_mcc_to_tas_block_list(u16 *list, u8 *size, u16 mcc)
+{
+	for (int i = 0; i < *size; i++) {
+		if (list[i] == mcc)
+			return true;
+	}
 
-	tas_data->usa_tas_uhb_allowed = usa_tas_uhb;
-	tas_data->override_tas_iec = override_iec;
-	tas_data->enable_tas_iec = enabled_iec;
+	/* Verify that there is room for another country
+	 * If *size == IWL_WTAS_BLACK_LIST_MAX, then the table is full.
+	 */
+	if (*size >= IWL_WTAS_BLACK_LIST_MAX)
+		return false;
 
-	return enabled;
+	list[*size++] = mcc;
+	return true;
 }
+IWL_EXPORT_SYMBOL(iwl_add_mcc_to_tas_block_list);
 
-static __le32 iwl_get_lari_config_bitmap(struct iwl_fw_runtime *fwrt)
+__le32 iwl_get_lari_config_bitmap(struct iwl_fw_runtime *fwrt)
 {
 	int ret;
 	u32 val;
 	__le32 config_bitmap = 0;
 
-	switch (CSR_HW_RFID_TYPE(fwrt->trans->hw_rf_id)) {
+	switch (CSR_HW_RFID_TYPE(fwrt->trans->info.hw_rf_id)) {
 	case IWL_CFG_RF_TYPE_HR1:
 	case IWL_CFG_RF_TYPE_HR2:
 	case IWL_CFG_RF_TYPE_JF1:
@@ -494,6 +535,7 @@ static __le32 iwl_get_lari_config_bitmap(struct iwl_fw_runtime *fwrt)
 
 	return config_bitmap;
 }
+IWL_EXPORT_SYMBOL(iwl_get_lari_config_bitmap);
 
 static size_t iwl_get_lari_config_cmd_size(u8 cmd_ver)
 {
@@ -540,34 +582,61 @@ int iwl_fill_lari_config(struct iwl_fw_runtime *fwrt,
 {
 	int ret;
 	u32 value;
+	bool has_raw_dsm_capa = fw_has_capa(&fwrt->fw->ucode_capa,
+					    IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE);
 	u8 cmd_ver = iwl_fw_lookup_cmd_ver(fwrt->fw,
 					   WIDE_ID(REGULATORY_AND_NVM_GROUP,
 						   LARI_CONFIG_CHANGE), 1);
 
+	if (WARN_ONCE(cmd_ver > 12,
+		      "Don't add newer versions to this function\n"))
+		return -EINVAL;
+
 	memset(cmd, 0, sizeof(*cmd));
 	*cmd_size = iwl_get_lari_config_cmd_size(cmd_ver);
 
 	cmd->config_bitmap = iwl_get_lari_config_bitmap(fwrt);
 
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_11AX_ENABLEMENT, &value);
-	if (!ret)
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= DSM_11AX_ALLOW_BITMAP;
 		cmd->oem_11ax_allow_bitmap = cpu_to_le32(value);
+	}
 
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ENABLE_UNII4_CHAN, &value);
 	if (!ret) {
-		if (cmd_ver < 9)
-			value &= DSM_UNII4_ALLOW_BITMAP_CMD_V8;
-		else
+		if (!has_raw_dsm_capa)
 			value &= DSM_UNII4_ALLOW_BITMAP;
 
+		/* Since version 9, bits 4 and 5 are supported
+		 * regardless of this capability, By pass this masking
+		 * if firmware has capability of accepting raw DSM table.
+		 */
+		if (!has_raw_dsm_capa && cmd_ver < 9 &&
+		    !fw_has_capa(&fwrt->fw->ucode_capa,
+				 IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_5G9_FOR_CA))
+			value &= ~(DSM_VALUE_UNII4_CANADA_OVERRIDE_MSK |
+				   DSM_VALUE_UNII4_CANADA_EN_MSK);
+
 		cmd->oem_unii4_allow_bitmap = cpu_to_le32(value);
 	}
 
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ACTIVATE_CHANNEL, &value);
 	if (!ret) {
-		if (cmd_ver < 8)
+		if (!has_raw_dsm_capa)
+			value &= CHAN_STATE_ACTIVE_BITMAP_CMD_V12;
+
+		if (!has_raw_dsm_capa && cmd_ver < 8)
 			value &= ~ACTIVATE_5G2_IN_WW_MASK;
-		if (cmd_ver < 12)
+
+		/* Since version 12, bits 5 and 6 are supported
+		 * regardless of this capability, By pass this masking
+		 * if firmware has capability of accepting raw DSM table.
+		 */
+		if (!has_raw_dsm_capa && cmd_ver < 12 &&
+		    !fw_has_capa(&fwrt->fw->ucode_capa,
+				 IWL_UCODE_TLV_CAPA_BIOS_OVERRIDE_UNII4_US_CA))
 			value &= CHAN_STATE_ACTIVE_BITMAP_CMD_V11;
 
 		cmd->chan_state_active_bitmap = cpu_to_le32(value);
@@ -578,13 +647,19 @@ int iwl_fill_lari_config(struct iwl_fw_runtime *fwrt,
 		cmd->oem_uhb_allow_bitmap = cpu_to_le32(value);
 
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_FORCE_DISABLE_CHANNELS, &value);
-	if (!ret)
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= DSM_FORCE_DISABLE_CHANNELS_ALLOWED_BITMAP;
 		cmd->force_disable_channels_bitmap = cpu_to_le32(value);
+	}
 
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ENERGY_DETECTION_THRESHOLD,
 			       &value);
-	if (!ret)
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= DSM_EDT_ALLOWED_BITMAP;
 		cmd->edt_bitmap = cpu_to_le32(value);
+	}
 
 	ret = iwl_bios_get_wbem(fwrt, &value);
 	if (!ret)
@@ -637,3 +712,50 @@ int iwl_bios_get_dsm(struct iwl_fw_runtime *fwrt, enum iwl_dsm_funcs func,
 	GET_BIOS_TABLE(dsm, fwrt, func, value);
 }
 IWL_EXPORT_SYMBOL(iwl_bios_get_dsm);
+
+bool iwl_puncturing_is_allowed_in_bios(u32 puncturing, u16 mcc)
+{
+	/* Some kind of regulatory mess means we need to currently disallow
+	 * puncturing in the US and Canada unless enabled in BIOS.
+	 */
+	switch (mcc) {
+	case IWL_MCC_US:
+		return puncturing & IWL_UEFI_CNV_PUNCTURING_USA_EN_MSK;
+	case IWL_MCC_CANADA:
+		return puncturing & IWL_UEFI_CNV_PUNCTURING_CANADA_EN_MSK;
+	default:
+		return true;
+	}
+}
+IWL_EXPORT_SYMBOL(iwl_puncturing_is_allowed_in_bios);
+
+bool iwl_rfi_is_enabled_in_bios(struct iwl_fw_runtime *fwrt)
+{
+	/* default behaviour is disabled */
+	u32 value = 0;
+	int ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_RFI_CONFIG, &value);
+
+	if (ret < 0) {
+		IWL_DEBUG_RADIO(fwrt, "Failed to get DSM RFI, ret=%d\n", ret);
+		return false;
+	}
+
+	value &= DSM_VALUE_RFI_DISABLE;
+	/* RFI BIOS CONFIG value can be 0 or 3 only.
+	 * i.e 0 means DDR and DLVR enabled. 3 means DDR and DLVR disabled.
+	 * 1 and 2 are invalid BIOS configurations, So, it's not possible to
+	 * disable ddr/dlvr separately.
+	 */
+	if (!value) {
+		IWL_DEBUG_RADIO(fwrt, "DSM RFI is evaluated to enable\n");
+		return true;
+	} else if (value == DSM_VALUE_RFI_DISABLE) {
+		IWL_DEBUG_RADIO(fwrt, "DSM RFI is evaluated to disable\n");
+	} else {
+		IWL_DEBUG_RADIO(fwrt,
+				"DSM RFI got invalid value, value=%d\n", value);
+	}
+
+	return false;
+}
+IWL_EXPORT_SYMBOL(iwl_rfi_is_enabled_in_bios);
diff --git a/sys/contrib/dev/iwlwifi/fw/regulatory.h b/sys/contrib/dev/iwlwifi/fw/regulatory.h
index e2c056f483c1..a07c512b6ed4 100644
--- a/sys/contrib/dev/iwlwifi/fw/regulatory.h
+++ b/sys/contrib/dev/iwlwifi/fw/regulatory.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2023-2024 Intel Corporation
+ * Copyright (C) 2023-2025 Intel Corporation
  */
 
 #ifndef __fw_regulatory_h__
@@ -40,11 +40,21 @@
 #define IWL_PPAG_ETSI_CHINA_MASK	3
 #define IWL_PPAG_REV3_MASK		0x7FF
 
-#define IWL_WTAS_ENABLED_MSK		0x1
-#define IWL_WTAS_OVERRIDE_IEC_MSK	0x2
-#define IWL_WTAS_ENABLE_IEC_MSK	0x4
+#define IWL_WTAS_ENABLED_MSK		BIT(0)
+#define IWL_WTAS_OVERRIDE_IEC_MSK	BIT(1)
+#define IWL_WTAS_ENABLE_IEC_MSK		BIT(2)
+#define IWL_WTAS_CANADA_UHB_MSK		BIT(15)
 #define IWL_WTAS_USA_UHB_MSK		BIT(16)
 
+struct iwl_tas_selection_data {
+	u8 override_tas_iec:1,
+	   enable_tas_iec:1,
+	   usa_tas_uhb_allowed:1,
+	   canada_tas_uhb_allowed:1;
+};
+
+#define BIOS_MCC_CHINA 0x434e
+
 /*
  * The profile for revision 2 is a superset of revision 1, which is in
  * turn a superset of revision 0.  So we can store all revisions
@@ -95,11 +105,11 @@ struct iwl_ppag_chain {
 };
 
 struct iwl_tas_data {
-	__le32 block_list_size;
-	__le32 block_list_array[IWL_WTAS_BLACK_LIST_MAX];
-	u8 override_tas_iec;
-	u8 enable_tas_iec;
-	u8 usa_tas_uhb_allowed;
+	u8 block_list_size;
+	u16 block_list_array[IWL_WTAS_BLACK_LIST_MAX];
+	u8 table_source;
+	u8 table_revision;
+	u32 tas_selection;
 };
 
 /* For DSM revision 0 and 4 */
@@ -142,14 +152,17 @@ enum iwl_dsm_unii4_bitmap {
 	DSM_VALUE_UNII4_CANADA_EN_MSK		= BIT(5),
 };
 
-#define DSM_UNII4_ALLOW_BITMAP_CMD_V8 (DSM_VALUE_UNII4_US_OVERRIDE_MSK | \
-				       DSM_VALUE_UNII4_US_EN_MSK | \
-				       DSM_VALUE_UNII4_ETSI_OVERRIDE_MSK | \
-				       DSM_VALUE_UNII4_ETSI_EN_MSK)
-#define DSM_UNII4_ALLOW_BITMAP (DSM_UNII4_ALLOW_BITMAP_CMD_V8 | \
-				DSM_VALUE_UNII4_CANADA_OVERRIDE_MSK | \
+#define DSM_UNII4_ALLOW_BITMAP (DSM_VALUE_UNII4_US_OVERRIDE_MSK		|\
+				DSM_VALUE_UNII4_US_EN_MSK		|\
+				DSM_VALUE_UNII4_ETSI_OVERRIDE_MSK	|\
+				DSM_VALUE_UNII4_ETSI_EN_MSK		|\
+				DSM_VALUE_UNII4_CANADA_OVERRIDE_MSK	|\
 				DSM_VALUE_UNII4_CANADA_EN_MSK)
 
+#define DSM_11AX_ALLOW_BITMAP				0xF
+#define DSM_EDT_ALLOWED_BITMAP				0x7ffff0
+#define DSM_FORCE_DISABLE_CHANNELS_ALLOWED_BITMAP	0x7FF
+
 enum iwl_dsm_values_rfi {
 	DSM_VALUE_RFI_DLVR_DISABLE	= BIT(0),
 	DSM_VALUE_RFI_DDR_DISABLE	= BIT(1),
@@ -158,6 +171,8 @@ enum iwl_dsm_values_rfi {
 #define DSM_VALUE_RFI_DISABLE	(DSM_VALUE_RFI_DLVR_DISABLE |\
 				 DSM_VALUE_RFI_DDR_DISABLE)
 
+bool iwl_rfi_is_enabled_in_bios(struct iwl_fw_runtime *fwrt);
+
 enum iwl_dsm_masks_reg {
 	DSM_MASK_CHINA_22_REG = BIT(2)
 };
@@ -181,10 +196,10 @@ int iwl_fill_ppag_table(struct iwl_fw_runtime *fwrt,
 bool iwl_is_ppag_approved(struct iwl_fw_runtime *fwrt);
 
 bool iwl_is_tas_approved(void);
+bool iwl_add_mcc_to_tas_block_list(u16 *list, u8 *size, u16 mcc);
 
-int iwl_parse_tas_selection(struct iwl_fw_runtime *fwrt,
-			    struct iwl_tas_data *tas_data,
-			    const u32 tas_selection);
+struct iwl_tas_selection_data
+iwl_parse_tas_selection(const u32 tas_selection, const u8 tbl_rev);
 
 int iwl_bios_get_wrds_table(struct iwl_fw_runtime *fwrt);
 
@@ -204,6 +219,7 @@ int iwl_bios_get_mcc(struct iwl_fw_runtime *fwrt, char *mcc);
 int iwl_bios_get_eckv(struct iwl_fw_runtime *fwrt, u32 *ext_clk);
 int iwl_bios_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value);
 
+__le32 iwl_get_lari_config_bitmap(struct iwl_fw_runtime *fwrt);
 int iwl_fill_lari_config(struct iwl_fw_runtime *fwrt,
 			 struct iwl_lari_config_change_cmd *cmd,
 			 size_t *cmd_size);
@@ -212,9 +228,41 @@ int iwl_bios_get_dsm(struct iwl_fw_runtime *fwrt, enum iwl_dsm_funcs func,
 		     u32 *value);
 
 static inline u32 iwl_bios_get_ppag_flags(const u32 ppag_modes,
-					  const u8 ppag_ver)
+					  const u8 ppag_bios_rev)
+{
+	/* For revision 4 and above driver is pipe */
+	if (ppag_bios_rev >= 4)
+		return ppag_modes;
+
+	return ppag_modes & (ppag_bios_rev < 3 ? IWL_PPAG_ETSI_CHINA_MASK :
+						 IWL_PPAG_REV3_MASK);
+}
+
+bool iwl_puncturing_is_allowed_in_bios(u32 puncturing, u16 mcc);
+
+#define IWL_DSBR_FW_MODIFIED_URM_MASK	BIT(8)
+#define IWL_DSBR_PERMANENT_URM_MASK	BIT(9)
+
+int iwl_bios_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value);
+int iwl_bios_get_phy_filters(struct iwl_fw_runtime *fwrt);
+
+static inline void iwl_bios_setup_step(struct iwl_trans *trans,
+				       struct iwl_fw_runtime *fwrt)
 {
-	return ppag_modes & (ppag_ver < 3 ? IWL_PPAG_ETSI_CHINA_MASK :
-					    IWL_PPAG_REV3_MASK);
+	u32 dsbr;
+
+	if (!trans->mac_cfg->integrated)
+		return;
+
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
+		return;
+
+	if (iwl_bios_get_dsbr(fwrt, &dsbr))
+		dsbr = 0;
+
+	trans->conf.dsbr_urm_fw_dependent =
+		!!(dsbr & IWL_DSBR_FW_MODIFIED_URM_MASK);
+	trans->conf.dsbr_urm_permanent =
+		!!(dsbr & IWL_DSBR_PERMANENT_URM_MASK);
 }
 #endif /* __fw_regulatory_h__ */
diff --git a/sys/contrib/dev/iwlwifi/fw/rs.c b/sys/contrib/dev/iwlwifi/fw/rs.c
index 8f99e501629e..746f2acffb8f 100644
--- a/sys/contrib/dev/iwlwifi/fw/rs.c
+++ b/sys/contrib/dev/iwlwifi/fw/rs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2021-2022 Intel Corporation
+ * Copyright (C) 2021-2022, 2025 Intel Corporation
  */
 
 #include <net/mac80211.h>
@@ -91,104 +91,6 @@ const char *iwl_rs_pretty_bw(int bw)
 }
 IWL_EXPORT_SYMBOL(iwl_rs_pretty_bw);
 
-static u32 iwl_legacy_rate_to_fw_idx(u32 rate_n_flags)
-{
-	int rate = rate_n_flags & RATE_LEGACY_RATE_MSK_V1;
-	int idx;
-	bool ofdm = !(rate_n_flags & RATE_MCS_CCK_MSK_V1);
-	int offset = ofdm ? IWL_FIRST_OFDM_RATE : 0;
-	int last = ofdm ? IWL_RATE_COUNT_LEGACY : IWL_FIRST_OFDM_RATE;
-
-	for (idx = offset; idx < last; idx++)
-		if (iwl_fw_rate_idx_to_plcp(idx) == rate)
-			return idx - offset;
-	return IWL_RATE_INVALID;
-}
-
-u32 iwl_new_rate_from_v1(u32 rate_v1)
-{
-	u32 rate_v2 = 0;
-	u32 dup = 0;
-
-	if (rate_v1 == 0)
-		return rate_v1;
-	/* convert rate */
-	if (rate_v1 & RATE_MCS_HT_MSK_V1) {
-		u32 nss = 0;
-
-		rate_v2 |= RATE_MCS_HT_MSK;
-		rate_v2 |=
-			rate_v1 & RATE_HT_MCS_RATE_CODE_MSK_V1;
-		nss = (rate_v1 & RATE_HT_MCS_MIMO2_MSK) >>
-			RATE_HT_MCS_NSS_POS_V1;
-		rate_v2 |= nss << RATE_MCS_NSS_POS;
-	} else if (rate_v1 & RATE_MCS_VHT_MSK_V1 ||
-		   rate_v1 & RATE_MCS_HE_MSK_V1) {
-		rate_v2 |= rate_v1 & RATE_VHT_MCS_RATE_CODE_MSK;
-
-		rate_v2 |= rate_v1 & RATE_MCS_NSS_MSK;
-
-		if (rate_v1 & RATE_MCS_HE_MSK_V1) {
-			u32 he_type_bits = rate_v1 & RATE_MCS_HE_TYPE_MSK_V1;
-			u32 he_type = he_type_bits >> RATE_MCS_HE_TYPE_POS_V1;
-			u32 he_106t = (rate_v1 & RATE_MCS_HE_106T_MSK_V1) >>
-				RATE_MCS_HE_106T_POS_V1;
-			u32 he_gi_ltf = (rate_v1 & RATE_MCS_HE_GI_LTF_MSK_V1) >>
-				RATE_MCS_HE_GI_LTF_POS;
-
-			if ((he_type_bits == RATE_MCS_HE_TYPE_SU ||
-			     he_type_bits == RATE_MCS_HE_TYPE_EXT_SU) &&
-			    he_gi_ltf == RATE_MCS_HE_SU_4_LTF)
-				/* the new rate have an additional bit to
-				 * represent the value 4 rather then using SGI
-				 * bit for this purpose - as it was done in the old
-				 * rate */
-				he_gi_ltf += (rate_v1 & RATE_MCS_SGI_MSK_V1) >>
-					RATE_MCS_SGI_POS_V1;
-
-			rate_v2 |= he_gi_ltf << RATE_MCS_HE_GI_LTF_POS;
-			rate_v2 |= he_type << RATE_MCS_HE_TYPE_POS;
-			rate_v2 |= he_106t << RATE_MCS_HE_106T_POS;
-			rate_v2 |= rate_v1 & RATE_HE_DUAL_CARRIER_MODE_MSK;
-			rate_v2 |= RATE_MCS_HE_MSK;
-		} else {
-			rate_v2 |= RATE_MCS_VHT_MSK;
-		}
-	/* if legacy format */
-	} else {
-		u32 legacy_rate = iwl_legacy_rate_to_fw_idx(rate_v1);
-
-		if (WARN_ON_ONCE(legacy_rate == IWL_RATE_INVALID))
-			legacy_rate = (rate_v1 & RATE_MCS_CCK_MSK_V1) ?
-				IWL_FIRST_CCK_RATE : IWL_FIRST_OFDM_RATE;
-
-		rate_v2 |= legacy_rate;
-		if (!(rate_v1 & RATE_MCS_CCK_MSK_V1))
-			rate_v2 |= RATE_MCS_LEGACY_OFDM_MSK;
-	}
-
-	/* convert flags */
-	if (rate_v1 & RATE_MCS_LDPC_MSK_V1)
-		rate_v2 |= RATE_MCS_LDPC_MSK;
-	rate_v2 |= (rate_v1 & RATE_MCS_CHAN_WIDTH_MSK_V1) |
-		(rate_v1 & RATE_MCS_ANT_AB_MSK) |
-		(rate_v1 & RATE_MCS_STBC_MSK) |
-		(rate_v1 & RATE_MCS_BF_MSK);
-
-	dup = (rate_v1 & RATE_MCS_DUP_MSK_V1) >> RATE_MCS_DUP_POS_V1;
-	if (dup) {
-		rate_v2 |= RATE_MCS_DUP_MSK;
-		rate_v2 |= dup << RATE_MCS_CHAN_WIDTH_POS;
-	}
-
-	if ((!(rate_v1 & RATE_MCS_HE_MSK_V1)) &&
-	    (rate_v1 & RATE_MCS_SGI_MSK_V1))
-		rate_v2 |= RATE_MCS_SGI_MSK;
-
-	return rate_v2;
-}
-IWL_EXPORT_SYMBOL(iwl_new_rate_from_v1);
-
 int rs_pretty_print_rate(char *buf, int bufsz, const u32 rate)
 {
 	char *type;
@@ -197,37 +99,40 @@ int rs_pretty_print_rate(char *buf, int bufsz, const u32 rate)
 	u32 bw = (rate & RATE_MCS_CHAN_WIDTH_MSK) >>
 		RATE_MCS_CHAN_WIDTH_POS;
 	u32 format = rate & RATE_MCS_MOD_TYPE_MSK;
+	int index = 0;
 	bool sgi;
 
-	if (format == RATE_MCS_CCK_MSK ||
-	    format == RATE_MCS_LEGACY_OFDM_MSK) {
-		int legacy_rate = rate & RATE_LEGACY_RATE_MSK;
-		int index = format == RATE_MCS_CCK_MSK ?
-			legacy_rate :
-			legacy_rate + IWL_FIRST_OFDM_RATE;
+	switch (format) {
+	case RATE_MCS_MOD_TYPE_LEGACY_OFDM:
+		index = IWL_FIRST_OFDM_RATE;
+		fallthrough;
+	case RATE_MCS_MOD_TYPE_CCK:
+		index += rate & RATE_LEGACY_RATE_MSK;
 
 		return scnprintf(buf, bufsz, "Legacy | ANT: %s Rate: %s Mbps",
 				 iwl_rs_pretty_ant(ant),
 				 iwl_rate_mcs(index)->mbps);
-	}
-
-	if (format ==  RATE_MCS_VHT_MSK)
+	case RATE_MCS_MOD_TYPE_VHT:
 		type = "VHT";
-	else if (format ==  RATE_MCS_HT_MSK)
+		break;
+	case RATE_MCS_MOD_TYPE_HT:
 		type = "HT";
-	else if (format == RATE_MCS_HE_MSK)
+		break;
+	case RATE_MCS_MOD_TYPE_HE:
 		type = "HE";
-	else if (format == RATE_MCS_EHT_MSK)
+		break;
+	case RATE_MCS_MOD_TYPE_EHT:
 		type = "EHT";
-	else
+		break;
+	default:
 		type = "Unknown"; /* shouldn't happen */
+	}
 
-	mcs = format == RATE_MCS_HT_MSK ?
+	mcs = format == RATE_MCS_MOD_TYPE_HT ?
 		RATE_HT_MCS_INDEX(rate) :
 		rate & RATE_MCS_CODE_MSK;
-	nss = ((rate & RATE_MCS_NSS_MSK)
-	       >> RATE_MCS_NSS_POS) + 1;
-	sgi = format == RATE_MCS_HE_MSK ?
+	nss = u32_get_bits(rate, RATE_MCS_NSS_MSK);
+	sgi = format == RATE_MCS_MOD_TYPE_HE ?
 		iwl_he_is_sgi(rate) :
 		rate & RATE_MCS_SGI_MSK;
 
diff --git a/sys/contrib/dev/iwlwifi/fw/runtime.h b/sys/contrib/dev/iwlwifi/fw/runtime.h
index 048877fa7c71..bd3bc2846cfa 100644
--- a/sys/contrib/dev/iwlwifi/fw/runtime.h
+++ b/sys/contrib/dev/iwlwifi/fw/runtime.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #ifndef __iwl_fw_runtime_h__
 #define __iwl_fw_runtime_h__
@@ -104,11 +104,19 @@ struct iwl_txf_iter_data {
  *	the driver by calling &iwl_fw_set_current_image()
  * @dump: debug dump data
  * @uats_table: AP type table
+ * @uats_valid: is AP type table valid
  * @uefi_tables_lock_status: The status of the WIFI GUID UEFI variables lock:
  *	0: Unlocked, 1 and 2: Locked.
  *	Only read the UEFI variables if locked.
  * @sar_profiles: sar profiles as read from WRDS/EWRD BIOS tables
  * @geo_profiles: geographic profiles as read from WGDS BIOS table
+ * @phy_filters: specific phy filters as read from WPFC BIOS table
+ * @ppag_bios_rev: PPAG BIOS revision
+ * @ppag_bios_source: see &enum bios_source
+ * @acpi_dsm_funcs_valid: bitmap indicating which DSM values are valid,
+ *	zero (default initialization) means it hasn't been read yet,
+ *	and BIT(0) is set when it has since function 0 also has this
+ *	bitmap and is always supported
  */
 struct iwl_fw_runtime {
 	struct iwl_trans *trans;
@@ -177,11 +185,18 @@ struct iwl_fw_runtime {
 	bool geo_enabled;
 	struct iwl_ppag_chain ppag_chains[IWL_NUM_CHAIN_LIMITS];
 	u32 ppag_flags;
-	u8 ppag_ver;
+	u8 ppag_bios_rev;
+	u8 ppag_bios_source;
 	struct iwl_sar_offset_mapping_cmd sgom_table;
 	bool sgom_enabled;
 	struct iwl_mcc_allowed_ap_type_cmd uats_table;
+	bool uats_valid;
 	u8 uefi_tables_lock_status;
+	struct iwl_phy_specific_cfg phy_filters;
+
+#ifdef CONFIG_ACPI
+	u32 acpi_dsm_funcs_valid;
+#endif
 };
 
 void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
diff --git a/sys/contrib/dev/iwlwifi/fw/smem.c b/sys/contrib/dev/iwlwifi/fw/smem.c
index 3f1272014daf..90fd69b4860c 100644
--- a/sys/contrib/dev/iwlwifi/fw/smem.c
+++ b/sys/contrib/dev/iwlwifi/fw/smem.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2021, 2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -102,7 +102,7 @@ void iwl_get_shared_mem_conf(struct iwl_fw_runtime *fwrt)
 	}
 
 	pkt = cmd.resp_pkt;
-	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000)
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000)
 		iwl_parse_shared_mem_22000(fwrt, pkt);
 	else
 		iwl_parse_shared_mem(fwrt, pkt);
diff --git a/sys/contrib/dev/iwlwifi/fw/uefi.c b/sys/contrib/dev/iwlwifi/fw/uefi.c
index fb982d4fe851..99a17b9323e9 100644
--- a/sys/contrib/dev/iwlwifi/fw/uefi.c
+++ b/sys/contrib/dev/iwlwifi/fw/uefi.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright(c) 2021-2024 Intel Corporation
+ * Copyright(c) 2021-2025 Intel Corporation
  */
 
 #include "iwl-drv.h"
@@ -13,9 +13,12 @@
 #include <linux/efi.h>
 #include "fw/runtime.h"
 
-#define IWL_EFI_VAR_GUID EFI_GUID(0x92daaf2f, 0xc02b, 0x455b,	\
-				  0xb2, 0xec, 0xf5, 0xa3,	\
-				  0x59, 0x4f, 0x4a, 0xea)
+#define IWL_EFI_WIFI_GUID	EFI_GUID(0x92daaf2f, 0xc02b, 0x455b,	\
+					 0xb2, 0xec, 0xf5, 0xa3,	\
+					 0x59, 0x4f, 0x4a, 0xea)
+#define IWL_EFI_WIFI_BT_GUID	EFI_GUID(0xe65d8884, 0xd4af, 0x4b20,	\
+					 0x8d, 0x03, 0x77, 0x2e,	\
+					 0xcc, 0x3d, 0xa5, 0x31)
 
 struct iwl_uefi_pnvm_mem_desc {
 	__le32 addr;
@@ -61,7 +64,7 @@ void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 
 	*len = 0;
 
-	data = iwl_uefi_get_variable(IWL_UEFI_OEM_PNVM_NAME, &IWL_EFI_VAR_GUID,
+	data = iwl_uefi_get_variable(IWL_UEFI_OEM_PNVM_NAME, &IWL_EFI_WIFI_GUID,
 				     &package_size);
 	if (IS_ERR(data)) {
 		IWL_DEBUG_FW(trans,
@@ -76,18 +79,18 @@ void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 	return data;
 }
 
-static
-void *iwl_uefi_get_verified_variable(struct iwl_trans *trans,
-				     efi_char16_t *uefi_var_name,
-				     char *var_name,
-				     unsigned int expected_size,
-				     unsigned long *size)
+static void *
+iwl_uefi_get_verified_variable_guid(struct iwl_trans *trans,
+				    efi_guid_t *guid,
+				    efi_char16_t *uefi_var_name,
+				    char *var_name,
+				    unsigned int expected_size,
+				    unsigned long *size)
 {
 	void *var;
 	unsigned long var_size;
 
-	var = iwl_uefi_get_variable(uefi_var_name, &IWL_EFI_VAR_GUID,
-				    &var_size);
+	var = iwl_uefi_get_variable(uefi_var_name, guid, &var_size);
 
 	if (IS_ERR(var)) {
 		IWL_DEBUG_RADIO(trans,
@@ -112,6 +115,18 @@ void *iwl_uefi_get_verified_variable(struct iwl_trans *trans,
 	return var;
 }
 
+static void *
+iwl_uefi_get_verified_variable(struct iwl_trans *trans,
+			       efi_char16_t *uefi_var_name,
+			       char *var_name,
+			       unsigned int expected_size,
+			       unsigned long *size)
+{
+	return iwl_uefi_get_verified_variable_guid(trans, &IWL_EFI_WIFI_GUID,
+						   uefi_var_name, var_name,
+						   expected_size, size);
+}
+
 int iwl_uefi_handle_tlv_mem_desc(struct iwl_trans *trans, const u8 *data,
 				 u32 tlv_len, struct iwl_pnvm_image *pnvm_data)
 {
@@ -204,7 +219,8 @@ done:
 
 int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 				const u8 *data, size_t len,
-				struct iwl_pnvm_image *pnvm_data)
+				struct iwl_pnvm_image *pnvm_data,
+				__le32 sku_id[3])
 {
 	const struct iwl_ucode_tlv *tlv;
 
@@ -226,23 +242,23 @@ int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 		}
 
 		if (tlv_type == IWL_UCODE_TLV_PNVM_SKU) {
-			const struct iwl_sku_id *sku_id =
+			const struct iwl_sku_id *tlv_sku_id =
 				(const void *)(data + sizeof(*tlv));
 
 			IWL_DEBUG_FW(trans,
 				     "Got IWL_UCODE_TLV_PNVM_SKU len %d\n",
 				     tlv_len);
 			IWL_DEBUG_FW(trans, "sku_id 0x%0x 0x%0x 0x%0x\n",
-				     le32_to_cpu(sku_id->data[0]),
-				     le32_to_cpu(sku_id->data[1]),
-				     le32_to_cpu(sku_id->data[2]));
+				     le32_to_cpu(tlv_sku_id->data[0]),
+				     le32_to_cpu(tlv_sku_id->data[1]),
+				     le32_to_cpu(tlv_sku_id->data[2]));
 
 			data += sizeof(*tlv) + ALIGN(tlv_len, 4);
 			len -= ALIGN(tlv_len, 4);
 
-			if (trans->sku_id[0] == le32_to_cpu(sku_id->data[0]) &&
-			    trans->sku_id[1] == le32_to_cpu(sku_id->data[1]) &&
-			    trans->sku_id[2] == le32_to_cpu(sku_id->data[2])) {
+			if (sku_id[0] == tlv_sku_id->data[0] &&
+			    sku_id[1] == tlv_sku_id->data[1] &&
+			    sku_id[2] == tlv_sku_id->data[2]) {
 				int ret = iwl_uefi_reduce_power_section(trans,
 								    data, len,
 								    pnvm_data);
@@ -295,11 +311,12 @@ static int iwl_uefi_step_parse(struct uefi_cnv_common_step_data *common_step_dat
 	if (common_step_data->revision != 1)
 		return -EINVAL;
 
-	trans->mbx_addr_0_step = (u32)common_step_data->revision |
+	trans->conf.mbx_addr_0_step =
+		(u32)common_step_data->revision |
 		(u32)common_step_data->cnvi_eq_channel << 8 |
 		(u32)common_step_data->cnvr_eq_channel << 16 |
 		(u32)common_step_data->radio1 << 24;
-	trans->mbx_addr_1_step = (u32)common_step_data->radio2;
+	trans->conf.mbx_addr_1_step = (u32)common_step_data->radio2;
 	return 0;
 }
 
@@ -308,11 +325,12 @@ void iwl_uefi_get_step_table(struct iwl_trans *trans)
 	struct uefi_cnv_common_step_data *data;
 	int ret;
 
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		return;
 
-	data = iwl_uefi_get_verified_variable(trans, IWL_UEFI_STEP_NAME,
-					      "STEP", sizeof(*data), NULL);
+	data = iwl_uefi_get_verified_variable_guid(trans, &IWL_EFI_WIFI_BT_GUID,
+						   IWL_UEFI_STEP_NAME,
+						   "STEP", sizeof(*data), NULL);
 	if (IS_ERR(data))
 		return;
 
@@ -386,11 +404,14 @@ static int iwl_uefi_uats_parse(struct uefi_cnv_wlan_uats_data *uats_data,
 
 	memcpy(fwrt->uats_table.offset_map, uats_data->offset_map,
 	       sizeof(fwrt->uats_table.offset_map));
+
+	fwrt->uats_valid = true;
+
 	return 0;
 }
 
-int iwl_uefi_get_uats_table(struct iwl_trans *trans,
-			    struct iwl_fw_runtime *fwrt)
+void iwl_uefi_get_uats_table(struct iwl_trans *trans,
+			     struct iwl_fw_runtime *fwrt)
 {
 	struct uefi_cnv_wlan_uats_data *data;
 	int ret;
@@ -398,17 +419,12 @@ int iwl_uefi_get_uats_table(struct iwl_trans *trans,
 	data = iwl_uefi_get_verified_variable(trans, IWL_UEFI_UATS_NAME,
 					      "UATS", sizeof(*data), NULL);
 	if (IS_ERR(data))
-		return -EINVAL;
+		return;
 
 	ret = iwl_uefi_uats_parse(data, fwrt);
-	if (ret < 0) {
+	if (ret < 0)
 		IWL_DEBUG_FW(trans, "Cannot read UATS table. rev is invalid\n");
-		kfree(data);
-		return ret;
-	}
-
 	kfree(data);
-	return 0;
 }
 IWL_EXPORT_SYMBOL(iwl_uefi_get_uats_table);
 
@@ -538,13 +554,14 @@ int iwl_uefi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 		goto out;
 	}
 
-	fwrt->ppag_ver = data->revision;
+	fwrt->ppag_bios_rev = data->revision;
 	fwrt->ppag_flags = iwl_bios_get_ppag_flags(data->ppag_modes,
-						   fwrt->ppag_ver);
+						   fwrt->ppag_bios_rev);
 
 	BUILD_BUG_ON(sizeof(fwrt->ppag_chains) != sizeof(data->ppag_chains));
 	memcpy(&fwrt->ppag_chains, &data->ppag_chains,
 	       sizeof(data->ppag_chains));
+	fwrt->ppag_bios_source = BIOS_SOURCE_UEFI;
 out:
 	kfree(data);
 	return ret;
@@ -554,27 +571,31 @@ int iwl_uefi_get_tas_table(struct iwl_fw_runtime *fwrt,
 			   struct iwl_tas_data *tas_data)
 {
 	struct uefi_cnv_var_wtas *uefi_tas;
-	int ret = 0, enabled, i;
+	int ret, enabled;
 
 	uefi_tas = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WTAS_NAME,
 						  "WTAS", sizeof(*uefi_tas), NULL);
 	if (IS_ERR(uefi_tas))
 		return -EINVAL;
 
-	if (uefi_tas->revision != IWL_UEFI_WTAS_REVISION) {
+	if (uefi_tas->revision < IWL_UEFI_MIN_WTAS_REVISION ||
+	    uefi_tas->revision > IWL_UEFI_MAX_WTAS_REVISION) {
 		ret = -EINVAL;
 		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI WTAS revision:%d\n",
 				uefi_tas->revision);
 		goto out;
 	}
 
-	enabled = iwl_parse_tas_selection(fwrt, tas_data,
-					  uefi_tas->tas_selection);
-	if (!enabled) {
-		IWL_DEBUG_RADIO(fwrt, "TAS not enabled\n");
-		ret = 0;
-		goto out;
-	}
+	IWL_DEBUG_RADIO(fwrt, "TAS selection as read from BIOS: 0x%x\n",
+			uefi_tas->tas_selection);
+
+	enabled = uefi_tas->tas_selection & IWL_WTAS_ENABLED_MSK;
+	tas_data->table_source = BIOS_SOURCE_UEFI;
+	tas_data->table_revision = uefi_tas->revision;
+	tas_data->tas_selection = uefi_tas->tas_selection;
+
+	IWL_DEBUG_RADIO(fwrt, "TAS %s enabled\n",
+			enabled ? "is" : "not");
 
 	IWL_DEBUG_RADIO(fwrt, "Reading TAS table revision %d\n",
 			uefi_tas->revision);
@@ -584,15 +605,16 @@ int iwl_uefi_get_tas_table(struct iwl_fw_runtime *fwrt,
 		ret = -EINVAL;
 		goto out;
 	}
-	tas_data->block_list_size = cpu_to_le32(uefi_tas->black_list_size);
+
+	tas_data->block_list_size = uefi_tas->black_list_size;
 	IWL_DEBUG_RADIO(fwrt, "TAS array size %u\n", uefi_tas->black_list_size);
 
-	for (i = 0; i < uefi_tas->black_list_size; i++) {
-		tas_data->block_list_array[i] =
-			cpu_to_le32(uefi_tas->black_list[i]);
+	for (u8 i = 0; i < uefi_tas->black_list_size; i++) {
+		tas_data->block_list_array[i] = uefi_tas->black_list[i];
 		IWL_DEBUG_RADIO(fwrt, "TAS block list country %d\n",
 				uefi_tas->black_list[i]);
 	}
+	ret = enabled;
 out:
 	kfree(uefi_tas);
 	return ret;
@@ -638,7 +660,7 @@ int iwl_uefi_get_mcc(struct iwl_fw_runtime *fwrt, char *mcc)
 		goto out;
 	}
 
-	if (data->mcc != UEFI_MCC_CHINA) {
+	if (data->mcc != BIOS_MCC_CHINA) {
 		ret = -EINVAL;
 		IWL_DEBUG_RADIO(fwrt, "UEFI WRDD is supported only for CN\n");
 		goto out;
@@ -657,14 +679,16 @@ int iwl_uefi_get_eckv(struct iwl_fw_runtime *fwrt, u32 *extl_clk)
 	struct uefi_cnv_var_eckv *data;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_ECKV_NAME,
-					      "ECKV", sizeof(*data), NULL);
+	data = iwl_uefi_get_verified_variable_guid(fwrt->trans,
+						   &IWL_EFI_WIFI_BT_GUID,
+						   IWL_UEFI_ECKV_NAME,
+						   "ECKV", sizeof(*data), NULL);
 	if (IS_ERR(data))
 		return -EINVAL;
 
 	if (data->revision != IWL_UEFI_ECKV_REVISION) {
 		ret = -EINVAL;
-		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI WRDD revision:%d\n",
+		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI ECKV revision:%d\n",
 				data->revision);
 		goto out;
 	}
@@ -723,9 +747,102 @@ int iwl_uefi_get_dsm(struct iwl_fw_runtime *fwrt, enum iwl_dsm_funcs func,
 		goto out;
 	}
 
+	if (!(data->functions[DSM_FUNC_QUERY] & BIT(func))) {
+		IWL_DEBUG_RADIO(fwrt, "DSM func %d not in 0x%x\n",
+				func, data->functions[DSM_FUNC_QUERY]);
+		goto out;
+	}
+
 	*value = data->functions[func];
+
+	IWL_DEBUG_RADIO(fwrt,
+			"UEFI: DSM func=%d: value=%d\n", func, *value);
+
 	ret = 0;
 out:
 	kfree(data);
 	return ret;
 }
+
+int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt)
+{
+	struct uefi_cnv_var_puncturing_data *data;
+	/* default value is not enabled if there is any issue in reading
+	 * uefi variable or revision is not supported
+	 */
+	int puncturing = 0;
+
+	data = iwl_uefi_get_verified_variable(fwrt->trans,
+					      IWL_UEFI_PUNCTURING_NAME,
+					      "UefiCnvWlanPuncturing",
+					      sizeof(*data), NULL);
+	if (IS_ERR(data))
+		return puncturing;
+
+	if (data->revision != IWL_UEFI_PUNCTURING_REVISION) {
+		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI PUNCTURING rev:%d\n",
+				data->revision);
+	} else {
+		puncturing = data->puncturing & IWL_UEFI_PUNCTURING_REV0_MASK;
+		IWL_DEBUG_RADIO(fwrt, "Loaded puncturing bits from UEFI: %d\n",
+				puncturing);
+	}
+
+	kfree(data);
+	return puncturing;
+}
+IWL_EXPORT_SYMBOL(iwl_uefi_get_puncturing);
+
+int iwl_uefi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value)
+{
+	struct uefi_cnv_wlan_dsbr_data *data;
+	int ret = 0;
+
+	data = iwl_uefi_get_verified_variable_guid(fwrt->trans,
+						   &IWL_EFI_WIFI_BT_GUID,
+						   IWL_UEFI_DSBR_NAME, "DSBR",
+						   sizeof(*data), NULL);
+	if (IS_ERR(data))
+		return -EINVAL;
+
+	if (data->revision != IWL_UEFI_DSBR_REVISION) {
+		ret = -EINVAL;
+		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI DSBR revision:%d\n",
+				data->revision);
+		goto out;
+	}
+	*value = data->config;
+	IWL_DEBUG_RADIO(fwrt, "Loaded DSBR config from UEFI value: 0x%x\n",
+			*value);
+out:
+	kfree(data);
+	return ret;
+}
+
+int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt)
+{
+	struct uefi_cnv_wpfc_data *data __free(kfree);
+	struct iwl_phy_specific_cfg *filters = &fwrt->phy_filters;
+
+	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WPFC_NAME,
+					      "WPFC", sizeof(*data), NULL);
+	if (IS_ERR(data))
+		return -EINVAL;
+
+	if (data->revision != 0) {
+		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI WPFC revision:%d\n",
+			data->revision);
+		return -EINVAL;
+	}
+
+	BUILD_BUG_ON(ARRAY_SIZE(filters->filter_cfg_chains) !=
+		     ARRAY_SIZE(data->chains));
+
+	for (int i = 0; i < ARRAY_SIZE(filters->filter_cfg_chains); i++) {
+		filters->filter_cfg_chains[i] = cpu_to_le32(data->chains[i]);
+		IWL_DEBUG_RADIO(fwrt, "WPFC: chain %d: %u\n", i, data->chains[i]);
+	}
+
+	IWL_DEBUG_RADIO(fwrt, "Loaded WPFC config from UEFI\n");
+	return 0;
+}
diff --git a/sys/contrib/dev/iwlwifi/fw/uefi.h b/sys/contrib/dev/iwlwifi/fw/uefi.h
index 50327bb8c2d6..da9d242d85b6 100644
--- a/sys/contrib/dev/iwlwifi/fw/uefi.h
+++ b/sys/contrib/dev/iwlwifi/fw/uefi.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright(c) 2021-2024 Intel Corporation
+ * Copyright(c) 2021-2025 Intel Corporation
  */
 #ifndef __iwl_fw_uefi__
 #define __iwl_fw_uefi__
@@ -19,9 +19,12 @@
 #define IWL_UEFI_WTAS_NAME		L"UefiCnvWlanWTAS"
 #define IWL_UEFI_SPLC_NAME		L"UefiCnvWlanSPLC"
 #define IWL_UEFI_WRDD_NAME		L"UefiCnvWlanWRDD"
-#define IWL_UEFI_ECKV_NAME		L"UefiCnvWlanECKV"
+#define IWL_UEFI_ECKV_NAME		L"UefiCnvCommonECKV"
 #define IWL_UEFI_DSM_NAME		L"UefiCnvWlanGeneralCfg"
 #define IWL_UEFI_WBEM_NAME		L"UefiCnvWlanWBEM"
+#define IWL_UEFI_PUNCTURING_NAME	L"UefiCnvWlanPuncturing"
+#define IWL_UEFI_DSBR_NAME		L"UefiCnvCommonDSBR"
+#define IWL_UEFI_WPFC_NAME		L"WPFC"
 
 
 #define IWL_SGOM_MAP_SIZE		339
@@ -31,13 +34,16 @@
 #define IWL_UEFI_EWRD_REVISION		2
 #define IWL_UEFI_WGDS_REVISION		3
 #define IWL_UEFI_MIN_PPAG_REV		1
-#define IWL_UEFI_MAX_PPAG_REV		3
-#define IWL_UEFI_WTAS_REVISION		1
+#define IWL_UEFI_MAX_PPAG_REV		4
+#define IWL_UEFI_MIN_WTAS_REVISION	1
+#define IWL_UEFI_MAX_WTAS_REVISION	2
 #define IWL_UEFI_SPLC_REVISION		0
 #define IWL_UEFI_WRDD_REVISION		0
 #define IWL_UEFI_ECKV_REVISION		0
 #define IWL_UEFI_WBEM_REVISION		0
 #define IWL_UEFI_DSM_REVISION		4
+#define IWL_UEFI_PUNCTURING_REVISION	0
+#define IWL_UEFI_DSBR_REVISION		1
 
 struct pnvm_sku_package {
 	u8 rev;
@@ -149,8 +155,6 @@ struct uefi_cnv_var_splc {
 	u32 default_pwr_limit;
 } __packed;
 
-#define UEFI_MCC_CHINA 0x434e
-
 /* struct uefi_cnv_var_wrdd - WRDD table as defined in UEFI
  * @revision: the revision of the table
  * @mcc: country identifier as defined in ISO/IEC 3166-1 Alpha 2 code
@@ -194,6 +198,51 @@ struct uefi_cnv_wlan_wbem_data {
 	u32 wbem_320mhz_per_mcc;
 } __packed;
 
+enum iwl_uefi_cnv_puncturing_flags {
+	IWL_UEFI_CNV_PUNCTURING_USA_EN_MSK	= BIT(0),
+	IWL_UEFI_CNV_PUNCTURING_CANADA_EN_MSK	= BIT(1),
+};
+
+#define IWL_UEFI_PUNCTURING_REV0_MASK (IWL_UEFI_CNV_PUNCTURING_USA_EN_MSK | \
+				       IWL_UEFI_CNV_PUNCTURING_CANADA_EN_MSK)
+/**
+ * struct uefi_cnv_var_puncturing_data - controlling channel
+ *	puncturing for few countries.
+ * @revision: the revision of the table
+ * @puncturing: enablement of channel puncturing per mcc
+ *	see &enum iwl_uefi_cnv_puncturing_flags.
+ */
+struct uefi_cnv_var_puncturing_data {
+	u8 revision;
+	u32 puncturing;
+} __packed;
+
+/**
+ * struct uefi_cnv_wlan_dsbr_data - BIOS STEP configuration information
+ * @revision: the revision of the table
+ * @config: STEP configuration flags:
+ *	bit 8, switch to URM depending on FW setting
+ *	bit 9, switch to URM
+ *
+ * Platform information for STEP configuration/workarounds.
+ */
+struct uefi_cnv_wlan_dsbr_data {
+	u8 revision;
+	u32 config;
+} __packed;
+
+/**
+ * struct uefi_cnv_wpfc_data - BIOS Wi-Fi PHY filter Configuration
+ * @revision: the revision of the table
+ * @chains: configuration of each of the chains (a-d)
+ *
+ * specific PHY filter configuration
+ */
+struct uefi_cnv_wpfc_data {
+	u8 revision;
+	u32 chains[4];
+} __packed;
+
 /*
  * This is known to be broken on v4.19 and to work on v5.4.  Until we
  * figure out why this is the case and how to make it work, simply
@@ -204,7 +253,8 @@ void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len);
 u8 *iwl_uefi_get_reduced_power(struct iwl_trans *trans, size_t *len);
 int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 				const u8 *data, size_t len,
-				struct iwl_pnvm_image *pnvm_data);
+				struct iwl_pnvm_image *pnvm_data,
+				__le32 sku_id[3]);
 void iwl_uefi_get_step_table(struct iwl_trans *trans);
 int iwl_uefi_handle_tlv_mem_desc(struct iwl_trans *trans, const u8 *data,
 				 u32 tlv_len, struct iwl_pnvm_image *pnvm_data);
@@ -222,8 +272,11 @@ int iwl_uefi_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value);
 int iwl_uefi_get_dsm(struct iwl_fw_runtime *fwrt, enum iwl_dsm_funcs func,
 		     u32 *value);
 void iwl_uefi_get_sgom_table(struct iwl_trans *trans, struct iwl_fw_runtime *fwrt);
-int iwl_uefi_get_uats_table(struct iwl_trans *trans,
-			    struct iwl_fw_runtime *fwrt);
+void iwl_uefi_get_uats_table(struct iwl_trans *trans,
+			     struct iwl_fw_runtime *fwrt);
+int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt);
+int iwl_uefi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value);
+int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt);
 #else /* CONFIG_EFI */
 static inline void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 {
@@ -233,7 +286,8 @@ static inline void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 static inline int
 iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 			    const u8 *data, size_t len,
-			    struct iwl_pnvm_image *pnvm_data)
+			    struct iwl_pnvm_image *pnvm_data,
+			    __le32 sku_id[3])
 {
 	return -EOPNOTSUPP;
 }
@@ -314,9 +368,13 @@ void iwl_uefi_get_sgom_table(struct iwl_trans *trans, struct iwl_fw_runtime *fwr
 {
 }
 
+static inline void
+iwl_uefi_get_uats_table(struct iwl_trans *trans, struct iwl_fw_runtime *fwrt)
+{
+}
+
 static inline
-int iwl_uefi_get_uats_table(struct iwl_trans *trans,
-			    struct iwl_fw_runtime *fwrt)
+int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt)
 {
 #if defined(__linux__)
 	return 0;
@@ -324,5 +382,16 @@ int iwl_uefi_get_uats_table(struct iwl_trans *trans,
 	return -ENOENT;
 #endif
 }
+
+static inline
+int iwl_uefi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value)
+{
+	return -ENOENT;
+}
+
+static inline int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt)
+{
+	return -ENOENT;
+}
 #endif /* CONFIG_EFI */
 #endif /* __iwl_fw_uefi__ */
diff --git a/sys/contrib/dev/iwlwifi/iwl-config.h b/sys/contrib/dev/iwlwifi/iwl-config.h
index 52026554f2a2..d3cc29c878ba 100644
--- a/sys/contrib/dev/iwlwifi/iwl-config.h
+++ b/sys/contrib/dev/iwlwifi/iwl-config.h
@@ -2,7 +2,7 @@
 /*
  * Copyright (C) 2005-2014, 2018-2021 Intel Corporation
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #ifndef __IWL_CONFIG_H__
 #define __IWL_CONFIG_H__
@@ -38,6 +38,7 @@ enum iwl_device_family {
 	IWL_DEVICE_FAMILY_AX210,
 	IWL_DEVICE_FAMILY_BZ,
 	IWL_DEVICE_FAMILY_SC,
+	IWL_DEVICE_FAMILY_DR,
 };
 
 #if defined(__FreeBSD__)
@@ -138,6 +139,10 @@ enum iwl_nvm_type {
 #define ANT_ABC		(ANT_A | ANT_B | ANT_C)
 
 
+#define IWL_FW_AND_PNVM(pfx, api)				\
+	MODULE_FIRMWARE(pfx "-" __stringify(api) ".ucode");	\
+	MODULE_FIRMWARE(pfx ".pnvm")
+
 static inline u8 num_of_ant(u8 mask)
 {
 	return  !!((mask) & ANT_A) +
@@ -146,7 +151,29 @@ static inline u8 num_of_ant(u8 mask)
 }
 
 /**
- * struct iwl_base_params - params not likely to change within a device family
+ * struct iwl_fw_mon_reg - FW monitor register info
+ * @addr: register address
+ * @mask: register mask
+ */
+struct iwl_fw_mon_reg {
+	u32 addr;
+	u32 mask;
+};
+
+/**
+ * struct iwl_fw_mon_regs - FW monitor registers
+ * @write_ptr: write pointer register
+ * @cycle_cnt: cycle count register
+ * @cur_frag: current fragment in use
+ */
+struct iwl_fw_mon_regs {
+	struct iwl_fw_mon_reg write_ptr;
+	struct iwl_fw_mon_reg cycle_cnt;
+	struct iwl_fw_mon_reg cur_frag;
+};
+
+/**
+ * struct iwl_family_base_params - base parameters for an entire family
  * @max_ll_items: max number of OTP blocks
  * @shadow_ram_support: shadow support for OTP memory
  * @led_compensation: compensate on the led on/off time per HW according
@@ -155,12 +182,34 @@ static inline u8 num_of_ant(u8 mask)
  * @wd_timeout: TX queues watchdog timeout
  * @max_event_log_size: size of event log buffer size for ucode event logging
  * @shadow_reg_enable: HW shadow register support
+ * @apmg_not_supported: there's no APMG
  * @apmg_wake_up_wa: should the MAC access REQ be asserted when a command
  *	is in flight. This is due to a HW bug in 7260, 3160 and 7265.
  * @scd_chain_ext_wa: should the chain extension feature in SCD be disabled.
  * @max_tfd_queue_size: max number of entries in tfd queue.
+ * @eeprom_size: EEPROM size
+ * @num_of_queues: number of HW TX queues supported
+ * @pcie_l1_allowed: PCIe L1 state is allowed
+ * @pll_cfg: PLL configuration needed
+ * @nvm_hw_section_num: the ID of the HW NVM section
+ * @features: hw features, any combination of feature_passlist
+ * @smem_offset: offset from which the SMEM begins
+ * @smem_len: the length of SMEM
+ * @mac_addr_from_csr: read HW address from CSR registers at this offset
+ * @d3_debug_data_base_addr: base address where D3 debug data is stored
+ * @d3_debug_data_length: length of the D3 debug data
+ * @min_ba_txq_size: minimum number of slots required in a TX queue used
+ *	for aggregation
+ * @min_txq_size: minimum number of slots required in a TX queue
+ * @gp2_reg_addr: GP2 (timer) register address
+ * @min_umac_error_event_table: minimum SMEM location of UMAC error table
+ * @mon_dbgi_regs: monitor DBGI registers
+ * @mon_dram_regs: monitor DRAM registers
+ * @mon_smem_regs: monitor SMEM registers
+ * @ucode_api_max: Highest version of uCode API supported by driver.
+ * @ucode_api_min: Lowest version of uCode API supported by driver.
  */
-struct iwl_base_params {
+struct iwl_family_base_params {
 	unsigned int wd_timeout;
 
 	u16 eeprom_size;
@@ -171,6 +220,7 @@ struct iwl_base_params {
 	   shadow_reg_enable:1,
 	   pcie_l1_allowed:1,
 	   apmg_wake_up_wa:1,
+	   apmg_not_supported:1,
 	   scd_chain_ext_wa:1;
 
 	u16 num_of_queues;	/* def: HW dependent */
@@ -178,6 +228,22 @@ struct iwl_base_params {
 
 	u8 max_ll_items;
 	u8 led_compensation;
+	u8 ucode_api_max;
+	u8 ucode_api_min;
+	u32 mac_addr_from_csr:10;
+	u8 nvm_hw_section_num;
+	netdev_features_t features;
+	u32 smem_offset;
+	u32 smem_len;
+	u32 min_umac_error_event_table;
+	u32 d3_debug_data_base_addr;
+	u32 d3_debug_data_length;
+	u32 min_txq_size;
+	u32 gp2_reg_addr;
+	u32 min_ba_txq_size;
+	const struct iwl_fw_mon_regs mon_dram_regs;
+	const struct iwl_fw_mon_regs mon_smem_regs;
+	const struct iwl_fw_mon_regs mon_dbgi_regs;
 };
 
 /*
@@ -269,7 +335,7 @@ struct iwl_pwr_tx_backoff {
 	u32 backoff;
 };
 
-enum iwl_cfg_trans_ltr_delay {
+enum iwl_mac_cfg_ltr_delay {
 	IWL_CFG_TRANS_LTR_DELAY_NONE	= 0,
 	IWL_CFG_TRANS_LTR_DELAY_200US	= 1,
 	IWL_CFG_TRANS_LTR_DELAY_2500US	= 2,
@@ -277,35 +343,33 @@ enum iwl_cfg_trans_ltr_delay {
 };
 
 /**
- * struct iwl_cfg_trans_params - information needed to start the trans
+ * struct iwl_mac_cfg - information about the MAC-specific device part
  *
  * These values are specific to the device ID and do not change when
  * multiple configs are used for a single device ID.  They values are
  * used, among other things, to boot the NIC so that the HW REV or
  * RFID can be read before deciding the remaining parameters to use.
  *
- * @base_params: pointer to basic parameters
+ * @base: pointer to basic parameters
  * @device_family: the device family
  * @umac_prph_offset: offset to add to UMAC periphery address
  * @xtal_latency: power up latency to get the xtal stabilized
  * @extra_phy_cfg_flags: extra configuration flags to pass to the PHY
- * @rf_id: need to read rf_id to determine the firmware image
  * @gen2: 22000 and on transport operation
  * @mq_rx_supported: multi-queue rx support
  * @integrated: discrete or integrated
  * @low_latency_xtal: use the low latency xtal if supported
  * @bisr_workaround: BISR hardware workaround (for 22260 series devices)
- * @ltr_delay: LTR delay parameter, &enum iwl_cfg_trans_ltr_delay.
+ * @ltr_delay: LTR delay parameter, &enum iwl_mac_cfg_ltr_delay.
  * @imr_enabled: use the IMR if supported.
  */
-struct iwl_cfg_trans_params {
-	const struct iwl_base_params *base_params;
+struct iwl_mac_cfg {
+	const struct iwl_family_base_params *base;
 	enum iwl_device_family device_family;
 	u32 umac_prph_offset;
 	u32 xtal_latency;
 	u32 extra_phy_cfg_flags;
-	u32 rf_id:1,
-	    gen2:1,
+	u32 gen2:1,
 	    mq_rx_supported:1,
 	    integrated:1,
 	    low_latency_xtal:1,
@@ -314,36 +378,21 @@ struct iwl_cfg_trans_params {
 	    imr_enabled:1;
 };
 
-/**
- * struct iwl_fw_mon_reg - FW monitor register info
- * @addr: register address
- * @mask: register mask
- */
-struct iwl_fw_mon_reg {
-	u32 addr;
-	u32 mask;
-};
-
-/**
- * struct iwl_fw_mon_regs - FW monitor registers
- * @write_ptr: write pointer register
- * @cycle_cnt: cycle count register
- * @cur_frag: current fragment in use
+/*
+ * These sizes were picked according to 8 MSDUs inside 64/256/612 A-MSDUs
+ * in an A-MPDU, with additional overhead to account for processing time.
+ * They will be doubled for MACs starting from So/Ty that don't support
+ * putting multiple frames into a single buffer.
  */
-struct iwl_fw_mon_regs {
-	struct iwl_fw_mon_reg write_ptr;
-	struct iwl_fw_mon_reg cycle_cnt;
-	struct iwl_fw_mon_reg cur_frag;
-};
+#define IWL_NUM_RBDS_NON_HE		(64 * 8)
+#define IWL_NUM_RBDS_HE			(256 * 8)
+#define IWL_NUM_RBDS_EHT		(512 * 8)
 
 /**
- * struct iwl_cfg
- * @trans: the trans-specific configuration part
- * @name: Official name of the device
+ * struct iwl_rf_cfg
  * @fw_name_pre: Firmware filename prefix. The api version and extension
  *	(.ucode) will be added to filename before loading from disk. The
  *	filename is constructed as <fw_name_pre>-<api>.ucode.
- * @fw_name_mac: MAC name for this config, the remaining pieces of the
  *	name will be generated dynamically
  * @ucode_api_max: Highest version of uCode API supported by driver.
  * @ucode_api_min: Lowest version of uCode API supported by driver.
@@ -354,34 +403,25 @@ struct iwl_fw_mon_regs {
  * @non_shared_ant: the antenna that is for WiFi only
  * @nvm_ver: NVM version
  * @nvm_calib_ver: NVM calibration version
+ * @bw_limit: bandwidth limit for this device, if non-zero
  * @ht_params: point to ht parameters
+ * @eeprom_params: EEPROM parameters (old devices)
+ * @thermal_params: Thermal throttling parameters
+ * @lp_xtal_workaround: low-power crystal workaround needed
  * @led_mode: 0=blinking, 1=On(RF On)/Off(RF Off)
  * @rx_with_siso_diversity: 1x1 device with rx antenna diversity
  * @tx_with_siso_diversity: 1x1 device with tx antenna diversity
  * @internal_wimax_coex: internal wifi/wimax combo device
- * @high_temp: Is this NIC is designated to be in high temperature.
  * @host_interrupt_operation_mode: device needs host interrupt operation
  *	mode set
- * @nvm_hw_section_num: the ID of the HW NVM section
- * @mac_addr_from_csr: read HW address from CSR registers at this offset
- * @features: hw features, any combination of feature_passlist
  * @pwr_tx_backoffs: translation table between power limits and backoffs
- * @max_tx_agg_size: max TX aggregation size of the ADDBA request/response
  * @dccm_offset: offset from which DCCM begins
  * @dccm_len: length of DCCM (including runtime stack CCM)
  * @dccm2_offset: offset from which the second DCCM begins
  * @dccm2_len: length of the second DCCM
- * @smem_offset: offset from which the SMEM begins
- * @smem_len: the length of SMEM
  * @vht_mu_mimo_supported: VHT MU-MIMO support
- * @cdb: CDB support
  * @nvm_type: see &enum iwl_nvm_type
- * @d3_debug_data_base_addr: base address where D3 debug data is stored
- * @d3_debug_data_length: length of the D3 debug data
- * @min_txq_size: minimum number of slots required in a TX queue
  * @uhb_supported: ultra high band channels supported
- * @min_ba_txq_size: minimum number of slots required in a TX queue which
- *	based on hardware support (HE - 256, EHT - 1K).
  * @num_rbds: number of receive buffer descriptors to use
  *	(only used for multi-queue capable devices)
  *
@@ -389,60 +429,38 @@ struct iwl_fw_mon_regs {
  * API differences in uCode shouldn't be handled here but through TLVs
  * and/or the uCode API version instead.
  */
-struct iwl_cfg {
-	struct iwl_cfg_trans_params trans;
+struct iwl_rf_cfg {
 	/* params specific to an individual device within a device family */
-	const char *name;
 	const char *fw_name_pre;
-	const char *fw_name_mac;
 	/* params likely to change within a device family */
-	const struct iwl_ht_params *ht_params;
+	const struct iwl_ht_params ht_params;
 	const struct iwl_eeprom_params *eeprom_params;
 	const struct iwl_pwr_tx_backoff *pwr_tx_backoffs;
-	const char *default_nvm_file_C_step;
 	const struct iwl_tt_params *thermal_params;
 	enum iwl_led_mode led_mode;
 	enum iwl_nvm_type nvm_type;
 	u32 max_data_size;
 	u32 max_inst_size;
-	netdev_features_t features;
 	u32 dccm_offset;
 	u32 dccm_len;
 	u32 dccm2_offset;
 	u32 dccm2_len;
-	u32 smem_offset;
-	u32 smem_len;
 	u16 nvm_ver;
 	u16 nvm_calib_ver;
+	u16 bw_limit;
 	u32 rx_with_siso_diversity:1,
 	    tx_with_siso_diversity:1,
 	    internal_wimax_coex:1,
 	    host_interrupt_operation_mode:1,
-	    high_temp:1,
-	    mac_addr_from_csr:10,
 	    lp_xtal_workaround:1,
-	    apmg_not_supported:1,
 	    vht_mu_mimo_supported:1,
-	    cdb:1,
-	    dbgc_supported:1,
 	    uhb_supported:1;
 	u8 valid_tx_ant;
 	u8 valid_rx_ant;
 	u8 non_shared_ant;
-	u8 nvm_hw_section_num;
-	u8 max_tx_agg_size;
 	u8 ucode_api_max;
 	u8 ucode_api_min;
 	u16 num_rbds;
-	u32 min_umac_error_event_table;
-	u32 d3_debug_data_base_addr;
-	u32 d3_debug_data_length;
-	u32 min_txq_size;
-	u32 gp2_reg_addr;
-	u32 min_ba_txq_size;
-	const struct iwl_fw_mon_regs mon_dram_regs;
-	const struct iwl_fw_mon_regs mon_smem_regs;
-	const struct iwl_fw_mon_regs mon_dbgi_regs;
 };
 
 #define IWL_CFG_ANY (~0)
@@ -450,8 +468,10 @@ struct iwl_cfg {
 #define IWL_CFG_MAC_TYPE_PU		0x31
 #define IWL_CFG_MAC_TYPE_TH		0x32
 #define IWL_CFG_MAC_TYPE_QU		0x33
+#define IWL_CFG_MAC_TYPE_CC		0x34
 #define IWL_CFG_MAC_TYPE_QUZ		0x35
 #define IWL_CFG_MAC_TYPE_SO		0x37
+#define IWL_CFG_MAC_TYPE_TY		0x42
 #define IWL_CFG_MAC_TYPE_SOF		0x43
 #define IWL_CFG_MAC_TYPE_MA		0x44
 #define IWL_CFG_MAC_TYPE_BZ		0x46
@@ -460,9 +480,9 @@ struct iwl_cfg {
 #define IWL_CFG_MAC_TYPE_SC2		0x49
 #define IWL_CFG_MAC_TYPE_SC2F		0x4A
 #define IWL_CFG_MAC_TYPE_BZ_W		0x4B
+#define IWL_CFG_MAC_TYPE_BR		0x4C
+#define IWL_CFG_MAC_TYPE_DR		0x4D
 
-#define IWL_CFG_RF_TYPE_TH		0x105
-#define IWL_CFG_RF_TYPE_TH1		0x108
 #define IWL_CFG_RF_TYPE_JF2		0x105
 #define IWL_CFG_RF_TYPE_JF1		0x108
 #define IWL_CFG_RF_TYPE_HR2		0x10A
@@ -470,6 +490,7 @@ struct iwl_cfg {
 #define IWL_CFG_RF_TYPE_GF		0x10D
 #define IWL_CFG_RF_TYPE_FM		0x112
 #define IWL_CFG_RF_TYPE_WH		0x113
+#define IWL_CFG_RF_TYPE_PE		0x114
 
 #define IWL_CFG_RF_ID_TH		0x1
 #define IWL_CFG_RF_ID_TH1		0x1
@@ -479,12 +500,6 @@ struct iwl_cfg {
 #define IWL_CFG_RF_ID_HR		0x7
 #define IWL_CFG_RF_ID_HR1		0x4
 
-#define IWL_CFG_NO_160			0x1
-#define IWL_CFG_160			0x0
-
-#define IWL_CFG_NO_320			0x1
-#define IWL_CFG_320			0x0
-
 #define IWL_CFG_CORES_BT		0x0
 #define IWL_CFG_CORES_BT_GNSS		0x5
 
@@ -495,52 +510,134 @@ struct iwl_cfg {
 #define IWL_CFG_IS_JACKET		0x1
 
 #define IWL_SUBDEVICE_RF_ID(subdevice)	((u16)((subdevice) & 0x00F0) >> 4)
-#define IWL_SUBDEVICE_NO_160(subdevice)	((u16)((subdevice) & 0x0200) >> 9)
+#define IWL_SUBDEVICE_BW_LIM(subdevice)	((u16)((subdevice) & 0x0200) >> 9)
 #define IWL_SUBDEVICE_CORES(subdevice)	((u16)((subdevice) & 0x1C00) >> 10)
 
 struct iwl_dev_info {
+	const struct iwl_rf_cfg *cfg;
+	const char *name;
 	u16 device;
 	u16 subdevice;
-	u16 mac_type;
-	u16 rf_type;
-	u8 mac_step;
-	u8 rf_step;
-	u8 rf_id;
-	u8 no_160;
-	u8 cores;
-	u8 cdb;
-	u8 jacket;
-	const struct iwl_cfg *cfg;
-	const char *name;
+	u32 subdevice_m_l:4,
+	    subdevice_m_h:4,
+	    match_rf_type:1,
+	    rf_type:9,
+	    match_bw_limit:1,
+	    bw_limit:1,
+	    match_discrete:1,
+	    discrete:1,
+	    match_rf_id:1,
+	    rf_id:4,
+	    match_cdb:1,
+	    cdb:1;
 };
 
 #if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
 extern const struct iwl_dev_info iwl_dev_info_table[];
 extern const unsigned int iwl_dev_info_table_size;
-const struct iwl_dev_info *
-iwl_pci_find_dev_info(u16 device, u16 subsystem_device,
-		      u16 mac_type, u8 mac_step, u16 rf_type, u8 cdb,
-		      u8 jacket, u8 rf_id, u8 no_160, u8 cores, u8 rf_step);
 extern const struct pci_device_id iwl_hw_card_ids[];
 #endif
 
+const struct iwl_dev_info *
+iwl_pci_find_dev_info(u16 device, u16 subsystem_device, u16 rf_type, u8 cdb,
+		      u8 rf_id, u8 bw_limit, bool discrete);
+
 /*
  * This list declares the config structures for all devices.
  */
-extern const struct iwl_cfg_trans_params iwl9000_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl9560_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl9560_long_latency_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl9560_shared_clk_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_qu_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_qu_medium_latency_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_qu_long_latency_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_ax200_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_so_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_so_long_latency_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_so_long_latency_imr_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_ma_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_bz_trans_cfg;
-extern const struct iwl_cfg_trans_params iwl_sc_trans_cfg;
+extern const struct iwl_mac_cfg iwl1000_mac_cfg;
+extern const struct iwl_mac_cfg iwl5000_mac_cfg;
+extern const struct iwl_mac_cfg iwl2000_mac_cfg;
+extern const struct iwl_mac_cfg iwl2030_mac_cfg;
+extern const struct iwl_mac_cfg iwl105_mac_cfg;
+extern const struct iwl_mac_cfg iwl135_mac_cfg;
+extern const struct iwl_mac_cfg iwl5150_mac_cfg;
+extern const struct iwl_mac_cfg iwl6005_mac_cfg;
+extern const struct iwl_mac_cfg iwl6030_mac_cfg;
+extern const struct iwl_mac_cfg iwl6000i_mac_cfg;
+extern const struct iwl_mac_cfg iwl6050_mac_cfg;
+extern const struct iwl_mac_cfg iwl6150_mac_cfg;
+extern const struct iwl_mac_cfg iwl6000_mac_cfg;
+extern const struct iwl_mac_cfg iwl7000_mac_cfg;
+extern const struct iwl_mac_cfg iwl8000_mac_cfg;
+extern const struct iwl_mac_cfg iwl9000_mac_cfg;
+extern const struct iwl_mac_cfg iwl9560_mac_cfg;
+extern const struct iwl_mac_cfg iwl9560_long_latency_mac_cfg;
+extern const struct iwl_mac_cfg iwl9560_shared_clk_mac_cfg;
+extern const struct iwl_mac_cfg iwl_qu_mac_cfg;
+extern const struct iwl_mac_cfg iwl_qu_medium_latency_mac_cfg;
+extern const struct iwl_mac_cfg iwl_qu_long_latency_mac_cfg;
+extern const struct iwl_mac_cfg iwl_ax200_mac_cfg;
+extern const struct iwl_mac_cfg iwl_ty_mac_cfg;
+extern const struct iwl_mac_cfg iwl_so_mac_cfg;
+extern const struct iwl_mac_cfg iwl_so_long_latency_mac_cfg;
+extern const struct iwl_mac_cfg iwl_so_long_latency_imr_mac_cfg;
+extern const struct iwl_mac_cfg iwl_ma_mac_cfg;
+extern const struct iwl_mac_cfg iwl_bz_mac_cfg;
+extern const struct iwl_mac_cfg iwl_gl_mac_cfg;
+extern const struct iwl_mac_cfg iwl_sc_mac_cfg;
+extern const struct iwl_mac_cfg iwl_dr_mac_cfg;
+
+extern const char iwl1000_bgn_name[];
+extern const char iwl1000_bg_name[];
+extern const char iwl100_bgn_name[];
+extern const char iwl100_bg_name[];
+extern const char iwl2000_2bgn_name[];
+extern const char iwl2000_2bgn_d_name[];
+extern const char iwl2030_2bgn_name[];
+extern const char iwl105_bgn_name[];
+extern const char iwl105_bgn_d_name[];
+extern const char iwl135_bgn_name[];
+extern const char iwl5300_agn_name[];
+extern const char iwl5100_bgn_name[];
+extern const char iwl5100_abg_name[];
+extern const char iwl5100_agn_name[];
+extern const char iwl5350_agn_name[];
+extern const char iwl5150_agn_name[];
+extern const char iwl5150_abg_name[];
+extern const char iwl6005_2agn_name[];
+extern const char iwl6005_2abg_name[];
+extern const char iwl6005_2bg_name[];
+extern const char iwl6005_2agn_sff_name[];
+extern const char iwl6005_2agn_d_name[];
+extern const char iwl6005_2agn_mow1_name[];
+extern const char iwl6005_2agn_mow2_name[];
+extern const char iwl6030_2agn_name[];
+extern const char iwl6030_2abg_name[];
+extern const char iwl6030_2bgn_name[];
+extern const char iwl6030_2bg_name[];
+extern const char iwl6035_2agn_name[];
+extern const char iwl6035_2agn_sff_name[];
+extern const char iwl1030_bgn_name[];
+extern const char iwl1030_bg_name[];
+extern const char iwl130_bgn_name[];
+extern const char iwl130_bg_name[];
+extern const char iwl6000i_2agn_name[];
+extern const char iwl6000i_2abg_name[];
+extern const char iwl6000i_2bg_name[];
+extern const char iwl6050_2agn_name[];
+extern const char iwl6050_2abg_name[];
+extern const char iwl6150_bgn_name[];
+extern const char iwl6150_bg_name[];
+extern const char iwl6000_3agn_name[];
+extern const char iwl7260_2ac_name[];
+extern const char iwl7260_2n_name[];
+extern const char iwl7260_n_name[];
+extern const char iwl3160_2ac_name[];
+extern const char iwl3160_2n_name[];
+extern const char iwl3160_n_name[];
+extern const char iwl3165_2ac_name[];
+extern const char iwl3168_2ac_name[];
+extern const char iwl7265_2ac_name[];
+extern const char iwl7265_2n_name[];
+extern const char iwl7265_n_name[];
+extern const char iwl8260_2n_name[];
+extern const char iwl8260_2ac_name[];
+extern const char iwl8265_2ac_name[];
+extern const char iwl8275_2ac_name[];
+extern const char iwl4165_2ac_name[];
+extern const char iwl_killer_1435i_name[];
+extern const char iwl_killer_1434_kix_name[];
 extern const char iwl9162_name[];
 extern const char iwl9260_name[];
 extern const char iwl9260_1_name[];
@@ -559,7 +656,6 @@ extern const char iwl9560_killer_1550i_name[];
 extern const char iwl9560_killer_1550s_name[];
 extern const char iwl_ax200_name[];
 extern const char iwl_ax203_name[];
-extern const char iwl_ax204_name[];
 extern const char iwl_ax201_name[];
 extern const char iwl_ax101_name[];
 extern const char iwl_ax200_killer_1650w_name[];
@@ -574,124 +670,84 @@ extern const char iwl_ax211_killer_1675s_name[];
 extern const char iwl_ax211_killer_1675i_name[];
 extern const char iwl_ax411_killer_1690s_name[];
 extern const char iwl_ax411_killer_1690i_name[];
+extern const char iwl_ax210_name[];
 extern const char iwl_ax211_name[];
-extern const char iwl_ax221_name[];
-extern const char iwl_ax231_name[];
 extern const char iwl_ax411_name[];
-extern const char iwl_bz_name[];
-extern const char iwl_fm_name[];
-extern const char iwl_gl_name[];
-extern const char iwl_mtp_name[];
-extern const char iwl_sc_name[];
-extern const char iwl_sc2_name[];
-extern const char iwl_sc2f_name[];
+extern const char iwl_killer_be1750s_name[];
+extern const char iwl_killer_be1750i_name[];
+extern const char iwl_killer_be1750w_name[];
+extern const char iwl_killer_be1750x_name[];
+extern const char iwl_killer_be1790s_name[];
+extern const char iwl_killer_be1790i_name[];
+extern const char iwl_be201_name[];
+extern const char iwl_be200_name[];
+extern const char iwl_be202_name[];
+extern const char iwl_be401_name[];
+extern const char iwl_be213_name[];
+extern const char iwl_killer_be1775s_name[];
+extern const char iwl_killer_be1775i_name[];
+extern const char iwl_be211_name[];
+extern const char iwl_killer_bn1850w2_name[];
+extern const char iwl_killer_bn1850i_name[];
+extern const char iwl_bn201_name[];
+extern const char iwl_be221_name[];
+extern const char iwl_be223_name[];
 #if IS_ENABLED(CONFIG_IWLDVM)
-extern const struct iwl_cfg iwl5300_agn_cfg;
-extern const struct iwl_cfg iwl5100_agn_cfg;
-extern const struct iwl_cfg iwl5350_agn_cfg;
-extern const struct iwl_cfg iwl5100_bgn_cfg;
-extern const struct iwl_cfg iwl5100_abg_cfg;
-extern const struct iwl_cfg iwl5150_agn_cfg;
-extern const struct iwl_cfg iwl5150_abg_cfg;
-extern const struct iwl_cfg iwl6005_2agn_cfg;
-extern const struct iwl_cfg iwl6005_2abg_cfg;
-extern const struct iwl_cfg iwl6005_2bg_cfg;
-extern const struct iwl_cfg iwl6005_2agn_sff_cfg;
-extern const struct iwl_cfg iwl6005_2agn_d_cfg;
-extern const struct iwl_cfg iwl6005_2agn_mow1_cfg;
-extern const struct iwl_cfg iwl6005_2agn_mow2_cfg;
-extern const struct iwl_cfg iwl1030_bgn_cfg;
-extern const struct iwl_cfg iwl1030_bg_cfg;
-extern const struct iwl_cfg iwl6030_2agn_cfg;
-extern const struct iwl_cfg iwl6030_2abg_cfg;
-extern const struct iwl_cfg iwl6030_2bgn_cfg;
-extern const struct iwl_cfg iwl6030_2bg_cfg;
-extern const struct iwl_cfg iwl6000i_2agn_cfg;
-extern const struct iwl_cfg iwl6000i_2abg_cfg;
-extern const struct iwl_cfg iwl6000i_2bg_cfg;
-extern const struct iwl_cfg iwl6000_3agn_cfg;
-extern const struct iwl_cfg iwl6050_2agn_cfg;
-extern const struct iwl_cfg iwl6050_2abg_cfg;
-extern const struct iwl_cfg iwl6150_bgn_cfg;
-extern const struct iwl_cfg iwl6150_bg_cfg;
-extern const struct iwl_cfg iwl1000_bgn_cfg;
-extern const struct iwl_cfg iwl1000_bg_cfg;
-extern const struct iwl_cfg iwl100_bgn_cfg;
-extern const struct iwl_cfg iwl100_bg_cfg;
-extern const struct iwl_cfg iwl130_bgn_cfg;
-extern const struct iwl_cfg iwl130_bg_cfg;
-extern const struct iwl_cfg iwl2000_2bgn_cfg;
-extern const struct iwl_cfg iwl2000_2bgn_d_cfg;
-extern const struct iwl_cfg iwl2030_2bgn_cfg;
-extern const struct iwl_cfg iwl6035_2agn_cfg;
-extern const struct iwl_cfg iwl6035_2agn_sff_cfg;
-extern const struct iwl_cfg iwl105_bgn_cfg;
-extern const struct iwl_cfg iwl105_bgn_d_cfg;
-extern const struct iwl_cfg iwl135_bgn_cfg;
+extern const struct iwl_rf_cfg iwl5300_agn_cfg;
+extern const struct iwl_rf_cfg iwl5350_agn_cfg;
+extern const struct iwl_rf_cfg iwl5100_n_cfg;
+extern const struct iwl_rf_cfg iwl5100_abg_cfg;
+extern const struct iwl_rf_cfg iwl5150_agn_cfg;
+extern const struct iwl_rf_cfg iwl5150_abg_cfg;
+extern const struct iwl_rf_cfg iwl6005_non_n_cfg;
+extern const struct iwl_rf_cfg iwl6005_n_cfg;
+extern const struct iwl_rf_cfg iwl6030_n_cfg;
+extern const struct iwl_rf_cfg iwl6030_non_n_cfg;
+extern const struct iwl_rf_cfg iwl6000i_2agn_cfg;
+extern const struct iwl_rf_cfg iwl6000i_non_n_cfg;
+extern const struct iwl_rf_cfg iwl6000i_non_n_cfg;
+extern const struct iwl_rf_cfg iwl6000_3agn_cfg;
+extern const struct iwl_rf_cfg iwl6050_2agn_cfg;
+extern const struct iwl_rf_cfg iwl6050_2abg_cfg;
+extern const struct iwl_rf_cfg iwl6150_bgn_cfg;
+extern const struct iwl_rf_cfg iwl6150_bg_cfg;
+extern const struct iwl_rf_cfg iwl1000_bgn_cfg;
+extern const struct iwl_rf_cfg iwl1000_bg_cfg;
+extern const struct iwl_rf_cfg iwl100_bgn_cfg;
+extern const struct iwl_rf_cfg iwl100_bg_cfg;
+extern const struct iwl_rf_cfg iwl130_bgn_cfg;
+extern const struct iwl_rf_cfg iwl130_bg_cfg;
+extern const struct iwl_rf_cfg iwl2000_2bgn_cfg;
+extern const struct iwl_rf_cfg iwl2030_2bgn_cfg;
+extern const struct iwl_rf_cfg iwl6035_2agn_cfg;
+extern const struct iwl_rf_cfg iwl105_bgn_cfg;
+extern const struct iwl_rf_cfg iwl135_bgn_cfg;
 #endif /* CONFIG_IWLDVM */
 #if IS_ENABLED(CONFIG_IWLMVM)
-extern const struct iwl_ht_params iwl_22000_ht_params;
-extern const struct iwl_cfg iwl7260_2ac_cfg;
-extern const struct iwl_cfg iwl7260_2ac_cfg_high_temp;
-extern const struct iwl_cfg iwl7260_2n_cfg;
-extern const struct iwl_cfg iwl7260_n_cfg;
-extern const struct iwl_cfg iwl3160_2ac_cfg;
-extern const struct iwl_cfg iwl3160_2n_cfg;
-extern const struct iwl_cfg iwl3160_n_cfg;
-extern const struct iwl_cfg iwl3165_2ac_cfg;
-extern const struct iwl_cfg iwl3168_2ac_cfg;
-extern const struct iwl_cfg iwl7265_2ac_cfg;
-extern const struct iwl_cfg iwl7265_2n_cfg;
-extern const struct iwl_cfg iwl7265_n_cfg;
-extern const struct iwl_cfg iwl7265d_2ac_cfg;
-extern const struct iwl_cfg iwl7265d_2n_cfg;
-extern const struct iwl_cfg iwl7265d_n_cfg;
-extern const struct iwl_cfg iwl8260_2n_cfg;
-extern const struct iwl_cfg iwl8260_2ac_cfg;
-extern const struct iwl_cfg iwl8265_2ac_cfg;
-extern const struct iwl_cfg iwl8275_2ac_cfg;
-extern const struct iwl_cfg iwl4165_2ac_cfg;
-extern const struct iwl_cfg iwl9260_2ac_cfg;
-extern const struct iwl_cfg iwl9560_qu_b0_jf_b0_cfg;
-extern const struct iwl_cfg iwl9560_qu_c0_jf_b0_cfg;
-extern const struct iwl_cfg iwl9560_quz_a0_jf_b0_cfg;
-extern const struct iwl_cfg iwl9560_2ac_cfg_soc;
-extern const struct iwl_cfg iwl_qu_b0_hr1_b0;
-extern const struct iwl_cfg iwl_qu_c0_hr1_b0;
-extern const struct iwl_cfg iwl_quz_a0_hr1_b0;
-extern const struct iwl_cfg iwl_qu_b0_hr_b0;
-extern const struct iwl_cfg iwl_qu_c0_hr_b0;
-extern const struct iwl_cfg iwl_ax200_cfg_cc;
-extern const struct iwl_cfg iwl_ax201_cfg_qu_hr;
-extern const struct iwl_cfg iwl_ax201_cfg_qu_c0_hr_b0;
-extern const struct iwl_cfg iwl_ax201_cfg_quz_hr;
-extern const struct iwl_cfg iwl_ax1650i_cfg_quz_hr;
-extern const struct iwl_cfg iwl_ax1650s_cfg_quz_hr;
-extern const struct iwl_cfg killer1650s_2ax_cfg_qu_b0_hr_b0;
-extern const struct iwl_cfg killer1650i_2ax_cfg_qu_b0_hr_b0;
-extern const struct iwl_cfg killer1650s_2ax_cfg_qu_c0_hr_b0;
-extern const struct iwl_cfg killer1650i_2ax_cfg_qu_c0_hr_b0;
-extern const struct iwl_cfg killer1650x_2ax_cfg;
-extern const struct iwl_cfg killer1650w_2ax_cfg;
-extern const struct iwl_cfg iwlax210_2ax_cfg_so_jf_b0;
-extern const struct iwl_cfg iwlax211_2ax_cfg_so_gf_a0;
-extern const struct iwl_cfg iwlax211_2ax_cfg_so_gf_a0_long;
-extern const struct iwl_cfg iwlax210_2ax_cfg_ty_gf_a0;
-extern const struct iwl_cfg iwlax411_2ax_cfg_so_gf4_a0;
-extern const struct iwl_cfg iwlax411_2ax_cfg_so_gf4_a0_long;
-
-extern const struct iwl_cfg iwl_cfg_ma;
-
-extern const struct iwl_cfg iwl_cfg_so_a0_hr_a0;
-extern const struct iwl_cfg iwl_cfg_so_a0_ms_a0;
-extern const struct iwl_cfg iwl_cfg_quz_a0_hr_b0;
-
-extern const struct iwl_cfg iwl_cfg_bz;
-extern const struct iwl_cfg iwl_cfg_gl;
-
-extern const struct iwl_cfg iwl_cfg_sc;
-extern const struct iwl_cfg iwl_cfg_sc2;
-extern const struct iwl_cfg iwl_cfg_sc2f;
+extern const struct iwl_rf_cfg iwl7260_cfg;
+extern const struct iwl_rf_cfg iwl7260_high_temp_cfg;
+extern const struct iwl_rf_cfg iwl3160_cfg;
+extern const struct iwl_rf_cfg iwl3165_2ac_cfg;
+extern const struct iwl_rf_cfg iwl3168_2ac_cfg;
+extern const struct iwl_rf_cfg iwl7265_cfg;
+extern const struct iwl_rf_cfg iwl7265d_cfg;
+extern const struct iwl_rf_cfg iwl8260_cfg;
+extern const struct iwl_rf_cfg iwl8265_cfg;
+extern const struct iwl_rf_cfg iwl_rf_jf;
+extern const struct iwl_rf_cfg iwl_rf_jf_80mhz;
+extern const struct iwl_rf_cfg iwl_rf_hr1;
+extern const struct iwl_rf_cfg iwl_rf_hr;
+extern const struct iwl_rf_cfg iwl_rf_hr_80mhz;
+
+extern const struct iwl_rf_cfg iwl_rf_gf;
 #endif /* CONFIG_IWLMVM */
 
+#if IS_ENABLED(CONFIG_IWLMLD)
+extern const struct iwl_rf_cfg iwl_rf_fm;
+extern const struct iwl_rf_cfg iwl_rf_fm_160mhz;
+#define iwl_rf_wh iwl_rf_fm
+#define iwl_rf_wh_160mhz iwl_rf_fm_160mhz
+#define iwl_rf_pe iwl_rf_fm
+#endif /* CONFIG_IWLMLD */
+
 #endif /* __IWL_CONFIG_H__ */
diff --git a/sys/contrib/dev/iwlwifi/iwl-csr.h b/sys/contrib/dev/iwlwifi/iwl-csr.h
index 98563757ce2c..f3fa37fee2e4 100644
--- a/sys/contrib/dev/iwlwifi/iwl-csr.h
+++ b/sys/contrib/dev/iwlwifi/iwl-csr.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -107,6 +107,14 @@
 /* GIO Chicken Bits (PCI Express bus link power management) */
 #define CSR_GIO_CHICKEN_BITS    (CSR_BASE+0x100)
 
+#define CSR_IPC_STATE		(CSR_BASE + 0x110)
+#define CSR_IPC_STATE_RESET	0x00000030
+#define CSR_IPC_STATE_RESET_NONE		0
+#define CSR_IPC_STATE_RESET_SW_READY		1
+#define CSR_IPC_STATE_RESET_TOP_READY		2
+#define CSR_IPC_STATE_RESET_TOP_FOLLOWER	3
+#define CSR_IPC_STATE_TOP_RESET_REQ		BIT(6)
+
 #define CSR_IPC_SLEEP_CONTROL	(CSR_BASE + 0x114)
 #define CSR_IPC_SLEEP_CONTROL_SUSPEND	0x3
 #define CSR_IPC_SLEEP_CONTROL_RESUME	0
@@ -148,6 +156,7 @@
  * during a error FW error.
  */
 #define CSR_FUNC_SCRATCH_INIT_VALUE		(0x01010101)
+#define CSR_FUNC_SCRATCH_POWER_OFF_MASK		0xFFFF
 
 /* Bits for CSR_HW_IF_CONFIG_REG */
 #define CSR_HW_IF_CONFIG_REG_MSK_MAC_STEP_DASH	(0x0000000F)
@@ -167,13 +176,15 @@
 #define CSR_HW_IF_CONFIG_REG_POS_PHY_DASH	(12)
 #define CSR_HW_IF_CONFIG_REG_POS_PHY_STEP	(14)
 
-#define CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A	(0x00080000)
-#define CSR_HW_IF_CONFIG_REG_BIT_EEPROM_OWN_SEM	(0x00200000)
-#define CSR_HW_IF_CONFIG_REG_BIT_NIC_READY	(0x00400000) /* PCI_OWN_SEM */
-#define CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE (0x02000000) /* ME_OWN */
-#define CSR_HW_IF_CONFIG_REG_PREPARE		  (0x08000000) /* WAKE_ME */
-#define CSR_HW_IF_CONFIG_REG_ENABLE_PME		  (0x10000000)
-#define CSR_HW_IF_CONFIG_REG_PERSIST_MODE	  (0x40000000) /* PERSISTENCE */
+#define CSR_HW_IF_CONFIG_REG_HAP_WAKE			0x00080000
+/* NOTE: EEPROM_OWN_SEM is no longer defined for new HW */
+#define CSR_HW_IF_CONFIG_REG_EEPROM_OWN_SEM		0x00200000
+#define CSR_HW_IF_CONFIG_REG_PCI_OWN_SET		0x00400000
+#define CSR_HW_IF_CONFIG_REG_IAMT_UP			0x01000000
+#define CSR_HW_IF_CONFIG_REG_ME_OWN			0x02000000
+#define CSR_HW_IF_CONFIG_REG_WAKE_ME			0x08000000
+#define CSR_HW_IF_CONFIG_REG_WAKE_ME_PCIE_OWNER_EN	0x10000000
+#define CSR_HW_IF_CONFIG_REG_PERSISTENCE		0x40000000
 
 #define CSR_MBOX_SET_REG_OS_ALIVE		BIT(5)
 
@@ -191,17 +202,19 @@
 #define CSR_INT_BIT_RF_KILL      (1 << 7)  /* HW RFKILL switch GP_CNTRL[27] toggled */
 #define CSR_INT_BIT_CT_KILL      (1 << 6)  /* Critical temp (chip too hot) rfkill */
 #define CSR_INT_BIT_SW_RX        (1 << 3)  /* Rx, command responses */
+#define CSR_INT_BIT_RESET_DONE   (1 << 2)  /* reset handshake with firmware is done */
 #define CSR_INT_BIT_WAKEUP       (1 << 1)  /* NIC controller waking up (pwr mgmt) */
 #define CSR_INT_BIT_ALIVE        (1 << 0)  /* uCode interrupts once it initializes */
 
-#define CSR_INI_SET_MASK	(CSR_INT_BIT_FH_RX   | \
-				 CSR_INT_BIT_HW_ERR  | \
-				 CSR_INT_BIT_FH_TX   | \
-				 CSR_INT_BIT_SW_ERR  | \
-				 CSR_INT_BIT_RF_KILL | \
-				 CSR_INT_BIT_SW_RX   | \
-				 CSR_INT_BIT_WAKEUP  | \
-				 CSR_INT_BIT_ALIVE   | \
+#define CSR_INI_SET_MASK	(CSR_INT_BIT_FH_RX	| \
+				 CSR_INT_BIT_HW_ERR	| \
+				 CSR_INT_BIT_FH_TX	| \
+				 CSR_INT_BIT_SW_ERR	| \
+				 CSR_INT_BIT_RF_KILL	| \
+				 CSR_INT_BIT_SW_RX	| \
+				 CSR_INT_BIT_WAKEUP	| \
+				 CSR_INT_BIT_RESET_DONE	| \
+				 CSR_INT_BIT_ALIVE	| \
 				 CSR_INT_BIT_RX_PERIODIC)
 
 /* interrupt flags in FH (flow handler) (PCI busmaster DMA) */
@@ -351,7 +364,6 @@ enum {
 #define CSR_HW_RF_ID_TYPE_HRCDB		(0x00109F00)
 #define CSR_HW_RF_ID_TYPE_GF		(0x0010D000)
 #define CSR_HW_RF_ID_TYPE_GF4		(0x0010E000)
-#define CSR_HW_RF_ID_TYPE_MS		(0x00111000)
 #define CSR_HW_RF_ID_TYPE_FM		(0x00112000)
 #define CSR_HW_RF_ID_TYPE_WP		(0x00113000)
 
@@ -639,7 +651,7 @@ enum msix_hw_int_causes {
  *                     HW address related registers                          *
  *****************************************************************************/
 
-#define CSR_ADDR_BASE(trans)			((trans)->cfg->mac_addr_from_csr)
+#define CSR_ADDR_BASE(trans)			((trans)->mac_cfg->base->mac_addr_from_csr)
 #define CSR_MAC_ADDR0_OTP(trans)		(CSR_ADDR_BASE(trans) + 0x00)
 #define CSR_MAC_ADDR1_OTP(trans)		(CSR_ADDR_BASE(trans) + 0x04)
 #define CSR_MAC_ADDR0_STRAP(trans)		(CSR_ADDR_BASE(trans) + 0x08)
diff --git a/sys/contrib/dev/iwlwifi/iwl-dbg-tlv.c b/sys/contrib/dev/iwlwifi/iwl-dbg-tlv.c
index 08d990ba8a79..5240dacf1360 100644
--- a/sys/contrib/dev/iwlwifi/iwl-dbg-tlv.c
+++ b/sys/contrib/dev/iwlwifi/iwl-dbg-tlv.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include <linux/firmware.h>
 #include "iwl-drv.h"
@@ -503,7 +503,7 @@ void iwl_dbg_tlv_load_bin(struct device *dev, struct iwl_trans *trans)
 	int res;
 
 	if (!iwlwifi_mod_params.enable_ini ||
-	    trans->trans_cfg->device_family <= IWL_DEVICE_FAMILY_8000)
+	    trans->mac_cfg->device_family <= IWL_DEVICE_FAMILY_8000)
 		return;
 
 	res = firmware_request_nowarn(&fw, yoyo_bin, dev);
@@ -603,11 +603,11 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt,
 		return 0;
 
 	num_frags = le32_to_cpu(fw_mon_cfg->max_frags_num);
-	if (fwrt->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
+	if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
 		if (alloc_id != IWL_FW_INI_ALLOCATION_ID_DBGC1)
 			return -EIO;
 		num_frags = 1;
-	} else if (fwrt->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ &&
+	} else if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ &&
 			   alloc_id > IWL_FW_INI_ALLOCATION_ID_DBGC3) {
 		return -EIO;
 	}
@@ -949,7 +949,7 @@ static void iwl_dbg_tlv_apply_config(struct iwl_fw_runtime *fwrt,
 static void iwl_dbg_tlv_periodic_trig_handler(struct timer_list *t)
 {
 	struct iwl_dbg_tlv_timer_node *timer_node =
-		from_timer(timer_node, t, timer);
+		timer_container_of(timer_node, t, timer);
 	struct iwl_fwrt_dump_data dump_data = {
 		.trig = (void *)timer_node->tlv->data,
 	};
@@ -1241,7 +1241,7 @@ iwl_dbg_tlv_tp_trigger(struct iwl_fw_runtime *fwrt, bool sync,
 
 		fwrt->trans->dbg.restart_required = false;
 
-		if (fwrt->trans->trans_cfg->device_family ==
+		if (fwrt->trans->mac_cfg->device_family ==
 		    IWL_DEVICE_FAMILY_9000) {
 			fwrt->trans->dbg.restart_required = true;
 		} else if (tp == IWL_FW_INI_TIME_POINT_FW_ASSERT &&
@@ -1372,15 +1372,15 @@ void _iwl_dbg_tlv_time_point(struct iwl_fw_runtime *fwrt,
 	switch (tp_id) {
 	case IWL_FW_INI_TIME_POINT_EARLY:
 		iwl_dbg_tlv_init_cfg(fwrt);
-		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		iwl_dbg_tlv_update_drams(fwrt);
 		iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data, NULL);
+		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		break;
 	case IWL_FW_INI_TIME_POINT_AFTER_ALIVE:
 		iwl_dbg_tlv_apply_buffers(fwrt);
 		iwl_dbg_tlv_send_hcmds(fwrt, hcmd_list);
-		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data, NULL);
+		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		break;
 	case IWL_FW_INI_TIME_POINT_PERIODIC:
 		iwl_dbg_tlv_set_periodic_trigs(fwrt);
@@ -1390,14 +1390,14 @@ void _iwl_dbg_tlv_time_point(struct iwl_fw_runtime *fwrt,
 	case IWL_FW_INI_TIME_POINT_MISSED_BEACONS:
 	case IWL_FW_INI_TIME_POINT_FW_DHC_NOTIFICATION:
 		iwl_dbg_tlv_send_hcmds(fwrt, hcmd_list);
-		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data,
 				       iwl_dbg_tlv_check_fw_pkt);
+		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		break;
 	default:
 		iwl_dbg_tlv_send_hcmds(fwrt, hcmd_list);
-		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data, NULL);
+		iwl_dbg_tlv_apply_config(fwrt, conf_list);
 		break;
 	}
 }
diff --git a/sys/contrib/dev/iwlwifi/iwl-debug.h b/sys/contrib/dev/iwlwifi/iwl-debug.h
index 7b3b402766b4..e0a195371198 100644
--- a/sys/contrib/dev/iwlwifi/iwl-debug.h
+++ b/sys/contrib/dev/iwlwifi/iwl-debug.h
@@ -62,7 +62,7 @@ enum iwl_dl {
 	IWL_DL_RX		= 0x00080000,
 	IWL_DL_SCAN		= 0x00100000,
 	IWL_DL_STATS		= 0x00200000,
-	/*			= 0x00400000, */
+	IWL_DL_EHT		= 0x00400000,
 	IWL_DL_TDLS		= 0x00800000,
 	IWL_DL_TE		= 0x01000000,
 	IWL_DL_TEMP		= 0x02000000,
@@ -71,8 +71,7 @@ enum iwl_dl {
 	IWL_DL_TX_QUEUES	= 0x10000000,
 	IWL_DL_TX_REPLY		= 0x20000000,
 	IWL_DL_WEP		= 0x40000000,
-
-	IWL_DL_PCI_RW		= 0x80000000,
+	IWL_DL_PTP		= 0x80000000,
 
 	IWL_DL_ANY		= 0x7fffffff,
 };
@@ -161,6 +160,8 @@ void __iwl_dbg(struct device *, u32, bool, const char *, const char *fmt, ...);
 	IWL_DPRINTF(_subsys, IWL_DL_MAC80211, _fmt, ##__VA_ARGS__)
 #define	IWL_DEBUG_POWER(_subsys, _fmt, ...)				\
 	IWL_DPRINTF(_subsys, IWL_DL_POWER, _fmt, ##__VA_ARGS__)
+#define	IWL_DEBUG_DEV_POWER(_dev, _fmt, ...)				\
+	IWL_DPRINTF_DEV((_dev), IWL_DL_POWER, _fmt, ##__VA_ARGS__)
 #define	IWL_DEBUG_QUOTA(_subsys, _fmt, ...)				\
 	IWL_DPRINTF(_subsys, IWL_DL_QUOTA, _fmt, ##__VA_ARGS__)
 #define	IWL_DEBUG_RADIO(_subsys, _fmt, ...)				\
@@ -197,8 +198,9 @@ void __iwl_dbg(struct device *, u32, bool, const char *, const char *fmt, ...);
 	IWL_DPRINTF(_subsys, IWL_DL_WOWLAN, _fmt, ##__VA_ARGS__)
 #define	IWL_DEBUG_DEV_RADIO(_dev, _fmt, ...)				\
 	IWL_DPRINTF_DEV((_dev), IWL_DL_DEV_RADIO, _fmt, ##__VA_ARGS__)
-
-#define	IWL_DEBUG_PCI_RW(_subsys, _fmt, ...)				\
-	IWL_DPRINTF(_subsys, IWL_DL_PCI_RW, _fmt, ##__VA_ARGS__)
+#define	IWL_DEBUG_PTP(_subsys, _fmt, ...)				\
+	IWL_DPRINTF(_subsys, IWL_DL_PTP, _fmt, ##__VA_ARGS__)
+#define	IWL_DEBUG_EHT(_subsys, _fmt, ...)				\
+	IWL_DPRINTF(_subsys, IWL_DL_EHT, _fmt, ##__VA_ARGS__)
 
 #endif /* _IWL_DEBUG_H */
diff --git a/sys/contrib/dev/iwlwifi/iwl-drv.c b/sys/contrib/dev/iwlwifi/iwl-drv.c
index b5c5268f8ddd..dcad2bfdd252 100644
--- a/sys/contrib/dev/iwlwifi/iwl-drv.c
+++ b/sys/contrib/dev/iwlwifi/iwl-drv.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -25,6 +25,7 @@
 #include "iwl-modparams.h"
 #include "fw/api/alive.h"
 #include "fw/api/mac.h"
+#include "fw/api/mac-cfg.h"
 
 /******************************************************************************
  *
@@ -92,6 +93,9 @@ struct iwl_drv {
 enum {
 	DVM_OP_MODE,
 	MVM_OP_MODE,
+#if IS_ENABLED(CONFIG_IWLMLD)
+	MLD_OP_MODE,
+#endif
 };
 
 /* Protects the table contents, i.e. the ops pointer & drv list */
@@ -103,6 +107,9 @@ static struct iwlwifi_opmode_table {
 } iwlwifi_opmode_table[] = {		/* ops set when driver is initialized */
 	[DVM_OP_MODE] = { .name = "iwldvm", .ops = NULL },
 	[MVM_OP_MODE] = { .name = "iwlmvm", .ops = NULL },
+#if IS_ENABLED(CONFIG_IWLMLD)
+	[MLD_OP_MODE] = { .name = "iwlmld", .ops = NULL },
+#endif
 };
 
 #define IWL_DEFAULT_SCAN_CHANNELS 40
@@ -147,6 +154,9 @@ static void iwl_dealloc_ucode(struct iwl_drv *drv)
 	kfree(drv->fw.phy_integration_ver);
 	kfree(drv->trans->dbg.pc_data);
 	drv->trans->dbg.pc_data = NULL;
+	kvfree(drv->fw.pnvm_data);
+	drv->fw.pnvm_data = NULL;
+	drv->fw.pnvm_size = 0;
 
 	for (i = 0; i < IWL_UCODE_TYPE_MAX; i++)
 		iwl_free_fw_img(drv, drv->fw.img + i);
@@ -155,8 +165,7 @@ static void iwl_dealloc_ucode(struct iwl_drv *drv)
 	memset(&drv->fw, 0, sizeof(drv->fw));
 }
 
-static int iwl_alloc_fw_desc(struct iwl_drv *drv, struct fw_desc *desc,
-			     struct fw_sec *sec)
+static int iwl_alloc_fw_desc(struct fw_desc *desc, struct fw_sec *sec)
 {
 	void *data;
 
@@ -186,22 +195,87 @@ static inline char iwl_drv_get_step(int step)
 	return 'a' + step;
 }
 
+static bool iwl_drv_is_wifi7_supported(struct iwl_trans *trans)
+{
+	return CSR_HW_RFID_TYPE(trans->info.hw_rf_id) >= IWL_CFG_RF_TYPE_FM;
+}
+
 const char *iwl_drv_get_fwname_pre(struct iwl_trans *trans, char *buf)
 {
 	char mac_step, rf_step;
-	const char *rf, *cdb;
+	const char *mac, *rf, *cdb;
 
 	if (trans->cfg->fw_name_pre)
 		return trans->cfg->fw_name_pre;
 
-	if (WARN_ON(!trans->cfg->fw_name_mac))
-		return "unconfigured";
+	mac_step = iwl_drv_get_step(trans->info.hw_rev_step);
 
-	mac_step = iwl_drv_get_step(trans->hw_rev_step);
+	switch (CSR_HW_REV_TYPE(trans->info.hw_rev)) {
+	case IWL_CFG_MAC_TYPE_PU:
+		mac = "9000-pu";
+		mac_step = 'b';
+		break;
+	case IWL_CFG_MAC_TYPE_TH:
+		mac = "9260-th";
+		mac_step = 'b';
+		break;
+	case IWL_CFG_MAC_TYPE_QU:
+		mac = "Qu";
+		break;
+	case IWL_CFG_MAC_TYPE_CC:
+		/* special case - no RF since it's fixed (discrete) */
+		scnprintf(buf, FW_NAME_PRE_BUFSIZE, "iwlwifi-cc-a0");
+		return buf;
+	case IWL_CFG_MAC_TYPE_QUZ:
+		mac = "QuZ";
+		/* all QuZ use A0 firmware */
+		mac_step = 'a';
+		break;
+	case IWL_CFG_MAC_TYPE_SO:
+	case IWL_CFG_MAC_TYPE_SOF:
+		mac = "so";
+		mac_step = 'a';
+		break;
+	case IWL_CFG_MAC_TYPE_TY:
+		mac = "ty";
+		mac_step = 'a';
+		break;
+	case IWL_CFG_MAC_TYPE_MA:
+		mac = "ma";
+		break;
+	case IWL_CFG_MAC_TYPE_BZ:
+	case IWL_CFG_MAC_TYPE_BZ_W:
+		mac = "bz";
+		break;
+	case IWL_CFG_MAC_TYPE_GL:
+		mac = "gl";
+		break;
+	case IWL_CFG_MAC_TYPE_SC:
+		mac = "sc";
+		break;
+	case IWL_CFG_MAC_TYPE_SC2:
+	/* Uses the same firmware as SC2 */
+	case IWL_CFG_MAC_TYPE_SC2F:
+		mac = "sc2";
+		break;
+	case IWL_CFG_MAC_TYPE_BR:
+		mac = "br";
+		break;
+	case IWL_CFG_MAC_TYPE_DR:
+		mac = "dr";
+		break;
+	default:
+		return "unknown-mac";
+	}
 
-	rf_step = iwl_drv_get_step(CSR_HW_RFID_STEP(trans->hw_rf_id));
+	rf_step = iwl_drv_get_step(CSR_HW_RFID_STEP(trans->info.hw_rf_id));
 
-	switch (CSR_HW_RFID_TYPE(trans->hw_rf_id)) {
+	switch (CSR_HW_RFID_TYPE(trans->info.hw_rf_id)) {
+	case IWL_CFG_RF_TYPE_JF1:
+	case IWL_CFG_RF_TYPE_JF2:
+		rf = "jf";
+		rf_step = 'b';
+		break;
 	case IWL_CFG_RF_TYPE_HR1:
 	case IWL_CFG_RF_TYPE_HR2:
 		rf = "hr";
@@ -209,29 +283,26 @@ const char *iwl_drv_get_fwname_pre(struct iwl_trans *trans, char *buf)
 		break;
 	case IWL_CFG_RF_TYPE_GF:
 		rf = "gf";
+		rf_step = 'a';
 		break;
 	case IWL_CFG_RF_TYPE_FM:
 		rf = "fm";
 		break;
 	case IWL_CFG_RF_TYPE_WH:
-		if (SILICON_Z_STEP ==
-		    CSR_HW_RFID_STEP(trans->hw_rf_id)) {
-			rf = "whtc";
-			rf_step = 'a';
-		} else {
-			rf = "wh";
-		}
+		rf = "wh";
+		break;
+	case IWL_CFG_RF_TYPE_PE:
+		rf = "pe";
 		break;
 	default:
 		return "unknown-rf";
 	}
 
-	cdb = CSR_HW_RFID_IS_CDB(trans->hw_rf_id) ? "4" : "";
+	cdb = CSR_HW_RFID_IS_CDB(trans->info.hw_rf_id) ? "4" : "";
 
 	scnprintf(buf, FW_NAME_PRE_BUFSIZE,
 		  "iwlwifi-%s-%c0-%s%s-%c0",
-		  trans->cfg->fw_name_mac, mac_step,
-		  rf, cdb, rf_step);
+		  mac, mac_step, rf, cdb, rf_step);
 
 	return buf;
 }
@@ -240,39 +311,68 @@ IWL_EXPORT_SYMBOL(iwl_drv_get_fwname_pre);
 static void iwl_req_fw_callback(const struct firmware *ucode_raw,
 				void *context);
 
+static void iwl_get_ucode_api_versions(struct iwl_trans *trans,
+				       unsigned int *api_min,
+				       unsigned int *api_max)
+{
+	const struct iwl_family_base_params *base = trans->mac_cfg->base;
+	const struct iwl_rf_cfg *cfg = trans->cfg;
+
+	/* if the MAC doesn't have range or if its range it higher than the RF's */
+	if (!base->ucode_api_max ||
+	    (cfg->ucode_api_max && base->ucode_api_min > cfg->ucode_api_max)) {
+		*api_min = cfg->ucode_api_min;
+		*api_max = cfg->ucode_api_max;
+		return;
+	}
+
+	/* if the RF doesn't have range or if its range it higher than the MAC's */
+	if (!cfg->ucode_api_max ||
+	    (base->ucode_api_max && cfg->ucode_api_min > base->ucode_api_max)) {
+		*api_min = base->ucode_api_min;
+		*api_max = base->ucode_api_max;
+		return;
+	}
+
+	*api_min = max(cfg->ucode_api_min, base->ucode_api_min);
+	*api_max = min(cfg->ucode_api_max, base->ucode_api_max);
+}
+
 static int iwl_request_firmware(struct iwl_drv *drv, bool first)
 {
-	const struct iwl_cfg *cfg = drv->trans->cfg;
 	char _fw_name_pre[FW_NAME_PRE_BUFSIZE];
+	unsigned int ucode_api_max, ucode_api_min;
 	const char *fw_name_pre;
 
-	if (drv->trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_9000 &&
-	    (drv->trans->hw_rev_step != SILICON_B_STEP &&
-	     drv->trans->hw_rev_step != SILICON_C_STEP)) {
+	iwl_get_ucode_api_versions(drv->trans, &ucode_api_min, &ucode_api_max);
+
+	if (drv->trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_9000 &&
+	    (drv->trans->info.hw_rev_step != SILICON_B_STEP &&
+	     drv->trans->info.hw_rev_step != SILICON_C_STEP)) {
 		IWL_ERR(drv,
 			"Only HW steps B and C are currently supported (0x%0x)\n",
-			drv->trans->hw_rev);
+			drv->trans->info.hw_rev);
 		return -EINVAL;
 	}
 
 	fw_name_pre = iwl_drv_get_fwname_pre(drv->trans, _fw_name_pre);
 
 	if (first)
-		drv->fw_index = cfg->ucode_api_max;
+		drv->fw_index = ucode_api_max;
 	else
 		drv->fw_index--;
 
-	if (drv->fw_index < cfg->ucode_api_min) {
+	if (drv->fw_index < ucode_api_min) {
 		IWL_ERR(drv, "no suitable firmware found!\n");
 
-		if (cfg->ucode_api_min == cfg->ucode_api_max) {
+		if (ucode_api_min == ucode_api_max) {
 			IWL_ERR(drv, "%s-%d is required\n", fw_name_pre,
-				cfg->ucode_api_max);
+				ucode_api_max);
 		} else {
 			IWL_ERR(drv, "minimum version required: %s-%d\n",
-				fw_name_pre, cfg->ucode_api_min);
+				fw_name_pre, ucode_api_min);
 			IWL_ERR(drv, "maximum version supported: %s-%d\n",
-				fw_name_pre, cfg->ucode_api_max);
+				fw_name_pre, ucode_api_max);
 		}
 
 		IWL_ERR(drv,
@@ -337,19 +437,9 @@ struct iwl_firmware_pieces {
 	size_t dbg_trigger_tlv_len[FW_DBG_TRIGGER_MAX];
 	struct iwl_fw_dbg_mem_seg_tlv *dbg_mem_tlv;
 	size_t n_mem_tlv;
+	u32 major;
 };
 
-/*
- * These functions are just to extract uCode section data from the pieces
- * structure.
- */
-static struct fw_sec *get_sec(struct iwl_firmware_pieces *pieces,
-			      enum iwl_ucode_type type,
-			      int  sec)
-{
-	return &pieces->img[type].sec[sec];
-}
-
 static void alloc_sec_data(struct iwl_firmware_pieces *pieces,
 			   enum iwl_ucode_type type,
 			   int sec)
@@ -410,22 +500,18 @@ static void set_sec_offset(struct iwl_firmware_pieces *pieces,
 /*
  * Gets uCode section from tlv.
  */
-static int iwl_store_ucode_sec(struct iwl_firmware_pieces *pieces,
-			       const void *data, enum iwl_ucode_type type,
-			       int size)
+static int iwl_store_ucode_sec(struct fw_img_parsing *img,
+			       const void *data, int size)
 {
-	struct fw_img_parsing *img;
 	struct fw_sec *sec;
 	const struct fw_sec_parsing *sec_parse;
 	size_t alloc_size;
 
-	if (WARN_ON(!pieces || !data || type >= IWL_UCODE_TYPE_MAX))
-		return -1;
+	if (WARN_ON(!img || !data))
+		return -EINVAL;
 
 	sec_parse = (const struct fw_sec_parsing *)data;
 
-	img = &pieces->img[type];
-
 	alloc_size = sizeof(*img->sec) * (img->sec_counter + 1);
 	sec = krealloc(img->sec, alloc_size, GFP_KERNEL);
 	if (!sec)
@@ -921,18 +1007,18 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 					le32_to_cpup((const __le32 *)tlv_data);
 			break;
 		case IWL_UCODE_TLV_SEC_RT:
-			iwl_store_ucode_sec(pieces, tlv_data, IWL_UCODE_REGULAR,
-					    tlv_len);
+			iwl_store_ucode_sec(&pieces->img[IWL_UCODE_REGULAR],
+					    tlv_data, tlv_len);
 			drv->fw.type = IWL_FW_MVM;
 			break;
 		case IWL_UCODE_TLV_SEC_INIT:
-			iwl_store_ucode_sec(pieces, tlv_data, IWL_UCODE_INIT,
-					    tlv_len);
+			iwl_store_ucode_sec(&pieces->img[IWL_UCODE_INIT],
+					    tlv_data, tlv_len);
 			drv->fw.type = IWL_FW_MVM;
 			break;
 		case IWL_UCODE_TLV_SEC_WOWLAN:
-			iwl_store_ucode_sec(pieces, tlv_data, IWL_UCODE_WOWLAN,
-					    tlv_len);
+			iwl_store_ucode_sec(&pieces->img[IWL_UCODE_WOWLAN],
+					    tlv_data, tlv_len);
 			drv->fw.type = IWL_FW_MVM;
 			break;
 		case IWL_UCODE_TLV_DEF_CALIB:
@@ -953,18 +1039,18 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 						FW_PHY_CFG_RX_CHAIN_POS;
 			break;
 		case IWL_UCODE_TLV_SECURE_SEC_RT:
-			iwl_store_ucode_sec(pieces, tlv_data, IWL_UCODE_REGULAR,
-					    tlv_len);
+			iwl_store_ucode_sec(&pieces->img[IWL_UCODE_REGULAR],
+					    tlv_data, tlv_len);
 			drv->fw.type = IWL_FW_MVM;
 			break;
 		case IWL_UCODE_TLV_SECURE_SEC_INIT:
-			iwl_store_ucode_sec(pieces, tlv_data, IWL_UCODE_INIT,
-					    tlv_len);
+			iwl_store_ucode_sec(&pieces->img[IWL_UCODE_INIT],
+					    tlv_data, tlv_len);
 			drv->fw.type = IWL_FW_MVM;
 			break;
 		case IWL_UCODE_TLV_SECURE_SEC_WOWLAN:
-			iwl_store_ucode_sec(pieces, tlv_data, IWL_UCODE_WOWLAN,
-					    tlv_len);
+			iwl_store_ucode_sec(&pieces->img[IWL_UCODE_WOWLAN],
+					    tlv_data, tlv_len);
 			drv->fw.type = IWL_FW_MVM;
 			break;
 		case IWL_UCODE_TLV_NUM_OF_CPU:
@@ -993,19 +1079,19 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			break;
 		case IWL_UCODE_TLV_FW_VERSION: {
 			const __le32 *ptr = (const void *)tlv_data;
-			u32 major, minor;
+			u32 minor;
 			u8 local_comp;
 
 			if (tlv_len != sizeof(u32) * 3)
 				goto invalid_tlv_len;
 
-			major = le32_to_cpup(ptr++);
+			pieces->major = le32_to_cpup(ptr++);
 			minor = le32_to_cpup(ptr++);
 			local_comp = le32_to_cpup(ptr);
 
 			snprintf(drv->fw.fw_version,
 				 sizeof(drv->fw.fw_version),
-				 "%u.%08x.%u %s", major, minor,
+				 "%u.%08x.%u %s", pieces->major, minor,
 				 local_comp, iwl_reduced_fw_name(drv));
 			break;
 			}
@@ -1131,9 +1217,8 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			}
 		case IWL_UCODE_TLV_SEC_RT_USNIFFER:
 			*usniffer_images = true;
-			iwl_store_ucode_sec(pieces, tlv_data,
-					    IWL_UCODE_REGULAR_USNIFFER,
-					    tlv_len);
+			iwl_store_ucode_sec(&pieces->img[IWL_UCODE_REGULAR_USNIFFER],
+					    tlv_data, tlv_len);
 			break;
 		case IWL_UCODE_TLV_PAGING:
 			if (tlv_len != sizeof(u32))
@@ -1218,15 +1303,15 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 
 			if (tlv_len != sizeof(*fseq_ver))
 				goto invalid_tlv_len;
-			IWL_INFO(drv, "TLV_FW_FSEQ_VERSION: %s\n",
-				 fseq_ver->version);
+			IWL_DEBUG_INFO(drv, "TLV_FW_FSEQ_VERSION: %.32s\n",
+				       fseq_ver->version);
 			}
 			break;
 		case IWL_UCODE_TLV_FW_NUM_STATIONS:
 			if (tlv_len != sizeof(u32))
 				goto invalid_tlv_len;
 			if (le32_to_cpup((const __le32 *)tlv_data) >
-			    IWL_MVM_STATION_COUNT_MAX) {
+			    IWL_STATION_COUNT_MAX) {
 				IWL_ERR(drv,
 					"%d is an invalid number of station\n",
 					le32_to_cpup((const __le32 *)tlv_data));
@@ -1235,6 +1320,19 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			capa->num_stations =
 				le32_to_cpup((const __le32 *)tlv_data);
 			break;
+		case IWL_UCODE_TLV_FW_NUM_LINKS:
+			if (tlv_len != sizeof(u32))
+				goto invalid_tlv_len;
+			if (le32_to_cpup((const __le32 *)tlv_data) >
+			    IWL_FW_MAX_LINK_ID + 1) {
+				IWL_ERR(drv,
+					"%d is an invalid number of links\n",
+					le32_to_cpup((const __le32 *)tlv_data));
+				goto tlv_error;
+			}
+			capa->num_links =
+				le32_to_cpup((const __le32 *)tlv_data);
+			break;
 		case IWL_UCODE_TLV_FW_NUM_BEACONS:
 			if (tlv_len != sizeof(u32))
 				goto invalid_tlv_len;
@@ -1247,7 +1345,7 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 
 			if (tlv_len != sizeof(*dbg_ptrs))
 				goto invalid_tlv_len;
-			if (drv->trans->trans_cfg->device_family <
+			if (drv->trans->mac_cfg->device_family <
 			    IWL_DEVICE_FAMILY_22000)
 				break;
 			drv->trans->dbg.umac_error_event_table =
@@ -1263,7 +1361,7 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 
 			if (tlv_len != sizeof(*dbg_ptrs))
 				goto invalid_tlv_len;
-			if (drv->trans->trans_cfg->device_family <
+			if (drv->trans->mac_cfg->device_family <
 			    IWL_DEVICE_FAMILY_22000)
 				break;
 			drv->trans->dbg.lmac_error_event_table[0] =
@@ -1329,6 +1427,15 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			drv->trans->dbg.num_pc =
 				tlv_len / sizeof(struct iwl_pc_data);
 			break;
+		case IWL_UCODE_TLV_PNVM_DATA:
+			if (drv->fw.pnvm_data)
+				break;
+			drv->fw.pnvm_data =
+				kvmemdup(tlv_data, tlv_len, GFP_KERNEL);
+			if (!drv->fw.pnvm_data)
+				return -ENOMEM;
+			drv->fw.pnvm_size = tlv_len;
+			break;
 		default:
 			IWL_DEBUG_INFO(drv, "unknown TLV: %d\n", tlv_type);
 			break;
@@ -1370,29 +1477,34 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 	return -EINVAL;
 }
 
-static int iwl_alloc_ucode(struct iwl_drv *drv,
-			   struct iwl_firmware_pieces *pieces,
-			   enum iwl_ucode_type type)
+static int iwl_alloc_ucode_mem(struct fw_img *out, struct fw_img_parsing *img)
 {
-	int i;
 	struct fw_desc *sec;
 
-	sec = kcalloc(pieces->img[type].sec_counter, sizeof(*sec), GFP_KERNEL);
+	sec = kcalloc(img->sec_counter, sizeof(*sec), GFP_KERNEL);
 	if (!sec)
 		return -ENOMEM;
-	drv->fw.img[type].sec = sec;
-	drv->fw.img[type].num_sec = pieces->img[type].sec_counter;
 
-	for (i = 0; i < pieces->img[type].sec_counter; i++)
-		if (iwl_alloc_fw_desc(drv, &sec[i], get_sec(pieces, type, i)))
+	out->sec = sec;
+	out->num_sec = img->sec_counter;
+
+	for (int i = 0; i < out->num_sec; i++)
+		if (iwl_alloc_fw_desc(&sec[i], &img->sec[i]))
 			return -ENOMEM;
 
 	return 0;
 }
 
+static int iwl_alloc_ucode(struct iwl_drv *drv,
+			   struct iwl_firmware_pieces *pieces,
+			   enum iwl_ucode_type type)
+{
+	return iwl_alloc_ucode_mem(&drv->fw.img[type], &pieces->img[type]);
+}
+
 static int validate_sec_sizes(struct iwl_drv *drv,
 			      struct iwl_firmware_pieces *pieces,
-			      const struct iwl_cfg *cfg)
+			      const struct iwl_rf_cfg *cfg)
 {
 	IWL_DEBUG_INFO(drv, "f/w package hdr runtime inst size = %zd\n",
 		get_sec_size(pieces, IWL_UCODE_REGULAR,
@@ -1446,26 +1558,39 @@ _iwl_op_mode_start(struct iwl_drv *drv, struct iwlwifi_opmode_table *op)
 	const struct iwl_op_mode_ops *ops = op->ops;
 	struct dentry *dbgfs_dir = NULL;
 	struct iwl_op_mode *op_mode = NULL;
+	int retry, max_retry = !!iwlwifi_mod_params.fw_restart * IWL_MAX_INIT_RETRY;
 
 	/* also protects start/stop from racing against each other */
 	lockdep_assert_held(&iwlwifi_opmode_table_mtx);
 
+	for (retry = 0; retry <= max_retry; retry++) {
+
 #ifdef CONFIG_IWLWIFI_DEBUGFS
-	drv->dbgfs_op_mode = debugfs_create_dir(op->name,
-						drv->dbgfs_drv);
-	dbgfs_dir = drv->dbgfs_op_mode;
+		drv->dbgfs_op_mode = debugfs_create_dir(op->name,
+							drv->dbgfs_drv);
+		dbgfs_dir = drv->dbgfs_op_mode;
 #endif
 
-	op_mode = ops->start(drv->trans, drv->trans->cfg,
-			     &drv->fw, dbgfs_dir);
-	if (op_mode)
-		return op_mode;
+		op_mode = ops->start(drv->trans, drv->trans->cfg,
+				     &drv->fw, dbgfs_dir);
+
+		if (!IS_ERR(op_mode))
+			return op_mode;
+
+		if (iwl_trans_is_dead(drv->trans))
+			break;
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
-	debugfs_remove_recursive(drv->dbgfs_op_mode);
-	drv->dbgfs_op_mode = NULL;
+		debugfs_remove_recursive(drv->dbgfs_op_mode);
+		drv->dbgfs_op_mode = NULL;
 #endif
 
+		if (PTR_ERR(op_mode) != -ETIMEDOUT)
+			break;
+
+		IWL_ERR(drv, "retry init count %d\n", retry);
+	}
+
 	return NULL;
 }
 
@@ -1486,6 +1611,8 @@ static void _iwl_op_mode_stop(struct iwl_drv *drv)
 	}
 }
 
+#define IWL_MLD_SUPPORTED_FW_VERSION 97
+
 /*
  * iwl_req_fw_callback - callback when firmware was loaded
  *
@@ -1500,19 +1627,20 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 	struct iwlwifi_opmode_table *op;
 	int err;
 	struct iwl_firmware_pieces *pieces;
-	const unsigned int api_max = drv->trans->cfg->ucode_api_max;
-	const unsigned int api_min = drv->trans->cfg->ucode_api_min;
+	unsigned int api_min, api_max;
 	size_t trigger_tlv_sz[FW_DBG_TRIGGER_MAX];
 	u32 api_ver;
 	int i;
 	bool usniffer_images = false;
 	bool failure = true;
 
+	iwl_get_ucode_api_versions(drv->trans, &api_min, &api_max);
+
 	fw->ucode_capa.max_probe_length = IWL_DEFAULT_MAX_PROBE_LENGTH;
 	fw->ucode_capa.standard_phy_calibration_size =
 			IWL_DEFAULT_STANDARD_PHY_CALIBRATE_TBL_SIZE;
 	fw->ucode_capa.n_scan_channels = IWL_DEFAULT_SCAN_CHANNELS;
-	fw->ucode_capa.num_stations = IWL_MVM_STATION_COUNT_MAX;
+	fw->ucode_capa.num_stations = IWL_STATION_COUNT_MAX;
 	fw->ucode_capa.num_beacons = 1;
 	/* dump all fw memory areas by default */
 	fw->dbg.dump_mask = 0xffffffff;
@@ -1701,14 +1829,14 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 		fw->init_evtlog_size = (pieces->init_evtlog_size - 16)/12;
 	else
 		fw->init_evtlog_size =
-			drv->trans->trans_cfg->base_params->max_event_log_size;
+			drv->trans->mac_cfg->base->max_event_log_size;
 	fw->init_errlog_ptr = pieces->init_errlog_ptr;
 	fw->inst_evtlog_ptr = pieces->inst_evtlog_ptr;
 	if (pieces->inst_evtlog_size)
 		fw->inst_evtlog_size = (pieces->inst_evtlog_size - 16)/12;
 	else
 		fw->inst_evtlog_size =
-			drv->trans->trans_cfg->base_params->max_event_log_size;
+			drv->trans->mac_cfg->base->max_event_log_size;
 	fw->inst_errlog_ptr = pieces->inst_errlog_ptr;
 
 	/*
@@ -1738,6 +1866,20 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 		break;
 	}
 
+#if IS_ENABLED(CONFIG_IWLMLD)
+	if (pieces->major >= IWL_MLD_SUPPORTED_FW_VERSION &&
+	    iwl_drv_is_wifi7_supported(drv->trans))
+		op = &iwlwifi_opmode_table[MLD_OP_MODE];
+#else
+	if (pieces->major >= IWL_MLD_SUPPORTED_FW_VERSION &&
+	    iwl_drv_is_wifi7_supported(drv->trans)) {
+		IWL_ERR(drv,
+			"IWLMLD needs to be compiled to support this firmware\n");
+		mutex_unlock(&iwlwifi_opmode_table_mtx);
+		goto out_unbind;
+	}
+#endif
+
 	IWL_INFO(drv, "loaded firmware version %s op_mode %s\n",
 		 drv->fw.fw_version, op->name);
 
@@ -1980,8 +2122,6 @@ static int __init iwl_drv_init(void)
 	for (i = 0; i < ARRAY_SIZE(iwlwifi_opmode_table); i++)
 		INIT_LIST_HEAD(&iwlwifi_opmode_table[i].drv);
 
-	pr_info(DRV_DESCRIPTION "\n");
-
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	/* Create the root of iwlwifi debugfs subsystem. */
 	iwl_dbgfs_root = debugfs_create_dir(DRV_NAME, NULL);
@@ -2004,6 +2144,7 @@ module_init(iwl_drv_init);
 static void __exit iwl_drv_exit(void)
 {
 	iwl_pci_unregister_driver();
+	iwl_trans_free_restart_list();
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	debugfs_remove_recursive(iwl_dbgfs_root);
diff --git a/sys/contrib/dev/iwlwifi/iwl-drv.h b/sys/contrib/dev/iwlwifi/iwl-drv.h
index 1549ff429549..595300a14639 100644
--- a/sys/contrib/dev/iwlwifi/iwl-drv.h
+++ b/sys/contrib/dev/iwlwifi/iwl-drv.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2020-2021, 2023 Intel Corporation
+ * Copyright (C) 2005-2014, 2020-2021, 2023, 2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  */
 #ifndef __iwl_drv_h__
@@ -53,7 +53,7 @@
 
 struct iwl_drv;
 struct iwl_trans;
-struct iwl_cfg;
+struct iwl_rf_cfg;
 /**
  * iwl_drv_start - start the drv
  *
@@ -85,7 +85,7 @@ void iwl_drv_stop(struct iwl_drv *drv);
  * everything is built-in, then we can avoid that.
  */
 #ifdef CONFIG_IWLWIFI_OPMODE_MODULAR
-#define IWL_EXPORT_SYMBOL(sym)	EXPORT_SYMBOL_NS_GPL(sym, IWLWIFI)
+#define IWL_EXPORT_SYMBOL(sym)	EXPORT_SYMBOL_NS_GPL(sym, "IWLWIFI")
 #else
 #define IWL_EXPORT_SYMBOL(sym)
 #endif
@@ -98,6 +98,9 @@ void iwl_drv_stop(struct iwl_drv *drv);
 #define VISIBLE_IF_IWLWIFI_KUNIT static
 #endif
 
+/* max retry for init flow */
+#define IWL_MAX_INIT_RETRY 2
+
 #define FW_NAME_PRE_BUFSIZE	64
 struct iwl_trans;
 const char *iwl_drv_get_fwname_pre(struct iwl_trans *trans, char *buf);
diff --git a/sys/contrib/dev/iwlwifi/iwl-fh.h b/sys/contrib/dev/iwlwifi/iwl-fh.h
index 5c8f1868db64..0f6de08b7473 100644
--- a/sys/contrib/dev/iwlwifi/iwl-fh.h
+++ b/sys/contrib/dev/iwlwifi/iwl-fh.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2021, 2023-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2021, 2023-2025 Intel Corporation
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_fh_h__
@@ -71,7 +71,7 @@
 static inline unsigned int FH_MEM_CBBC_QUEUE(struct iwl_trans *trans,
 					     unsigned int chnl)
 {
-	if (trans->trans_cfg->gen2) {
+	if (trans->mac_cfg->gen2) {
 		WARN_ON_ONCE(chnl >= 64);
 		return TFH_TFDQ_CBB_TABLE + 8 * chnl;
 	}
@@ -378,14 +378,14 @@ static inline unsigned int FH_MEM_CBBC_QUEUE(struct iwl_trans *trans,
  * Once the RXF-to-DRAM DMA is active, this flag is immediately turned off.
  */
 #define RFH_GEN_STATUS		0xA09808
-#define RFH_GEN_STATUS_GEN3	0xA07824
+#define RFH_GEN_STATUS_AX210	0xA07824
 #define RBD_FETCH_IDLE	BIT(29)
 #define SRAM_DMA_IDLE	BIT(30)
 #define RXF_DMA_IDLE	BIT(31)
 
 /* DMA configuration */
 #define RFH_RXF_DMA_CFG		0xA09820
-#define RFH_RXF_DMA_CFG_GEN3	0xA07880
+#define RFH_RXF_DMA_CFG_AX210	0xA07880
 /* RB size */
 #define RFH_RXF_DMA_RB_SIZE_MASK (0x000F0000) /* bits 16-19 */
 #define RFH_RXF_DMA_RB_SIZE_POS 16
@@ -588,13 +588,12 @@ struct iwl_rb_status {
 
 
 #define TFD_QUEUE_SIZE_MAX      (256)
-#define TFD_QUEUE_SIZE_MAX_GEN3 (65536)
 /* cb size is the exponent - 3 */
 #define TFD_QUEUE_CB_SIZE(x)	(ilog2(x) - 3)
 #define TFD_QUEUE_SIZE_BC_DUP	(64)
 #define TFD_QUEUE_BC_SIZE	(TFD_QUEUE_SIZE_MAX + TFD_QUEUE_SIZE_BC_DUP)
-#define TFD_QUEUE_BC_SIZE_GEN3_AX210	1024
-#define TFD_QUEUE_BC_SIZE_GEN3_BZ	(1024 * 4)
+#define TFD_QUEUE_BC_SIZE_AX210	1024
+#define TFD_QUEUE_BC_SIZE_BZ	(1024 * 4)
 #define IWL_TX_DMA_MASK        DMA_BIT_MASK(36)
 #define IWL_NUM_OF_TBS		20
 #define IWL_TFH_NUM_TBS		25
@@ -717,30 +716,19 @@ struct iwl_tfh_tfd {
 /* Fixed (non-configurable) rx data from phy */
 
 /**
- * struct iwlagn_scd_bc_tbl - scheduler byte count table
+ * struct iwl_bc_tbl_entry - scheduler byte count table entry
  *	base physical address provided by SCD_DRAM_BASE_ADDR
  * For devices up to 22000:
  * @tfd_offset:
  *	For devices up to 22000:
  *		 0-12 - tx command byte count
  *		12-16 - station index
- *	For 22000:
+ *	For 22000 and on:
  *		 0-12 - tx command byte count
  *		12-13 - number of 64 byte chunks
  *		14-16 - reserved
  */
-struct iwlagn_scd_bc_tbl {
-	__le16 tfd_offset[TFD_QUEUE_BC_SIZE];
-} __packed;
-
-/**
- * struct iwl_gen3_bc_tbl_entry - scheduler byte count table entry gen3
- * For AX210 and on:
- * @tfd_offset: 0-12 - tx command byte count
- *		12-13 - number of 64 byte chunks
- *		14-16 - reserved
- */
-struct iwl_gen3_bc_tbl_entry {
+struct iwl_bc_tbl_entry {
 	__le16 tfd_offset;
 } __packed;
 
diff --git a/sys/contrib/dev/iwlwifi/iwl-io.c b/sys/contrib/dev/iwlwifi/iwl-io.c
index 060becfd64f3..5e483a55a4ba 100644
--- a/sys/contrib/dev/iwlwifi/iwl-io.c
+++ b/sys/contrib/dev/iwlwifi/iwl-io.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2003-2014, 2018-2022, 2024 Intel Corporation
+ * Copyright (C) 2003-2014, 2018-2022, 2024-2025 Intel Corporation
  * Copyright (C) 2015-2016 Intel Deutschland GmbH
  */
 #include <linux/delay.h>
@@ -47,21 +47,21 @@ IWL_EXPORT_SYMBOL(iwl_read32);
 
 #define IWL_POLL_INTERVAL 10	/* microseconds */
 
-int iwl_poll_bit(struct iwl_trans *trans, u32 addr,
-		 u32 bits, u32 mask, int timeout)
+int iwl_poll_bits_mask(struct iwl_trans *trans, u32 addr,
+		       u32 bits, u32 mask, int timeout)
 {
 	int t = 0;
 
 	do {
 		if ((iwl_read32(trans, addr) & mask) == (bits & mask))
-			return t;
+			return 0;
 		udelay(IWL_POLL_INTERVAL);
 		t += IWL_POLL_INTERVAL;
 	} while (t < timeout);
 
 	return -ETIMEDOUT;
 }
-IWL_EXPORT_SYMBOL(iwl_poll_bit);
+IWL_EXPORT_SYMBOL(iwl_poll_bits_mask);
 
 u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg)
 {
@@ -75,7 +75,6 @@ u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg)
 	/* return as if we have a HW timeout/failure */
 	return 0x5a5a5a5a;
 }
-IWL_EXPORT_SYMBOL(iwl_read_direct32);
 
 void iwl_write_direct32(struct iwl_trans *trans, u32 reg, u32 value)
 {
@@ -93,7 +92,6 @@ void iwl_write_direct64(struct iwl_trans *trans, u64 reg, u64 value)
 		iwl_trans_release_nic_access(trans);
 	}
 }
-IWL_EXPORT_SYMBOL(iwl_write_direct64);
 
 int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,
 			int timeout)
@@ -109,7 +107,6 @@ int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,
 
 	return -ETIMEDOUT;
 }
-IWL_EXPORT_SYMBOL(iwl_poll_direct_bit);
 
 u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs)
 {
@@ -117,14 +114,12 @@ u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs)
 	trace_iwlwifi_dev_ioread_prph32(trans->dev, ofs, val);
 	return val;
 }
-IWL_EXPORT_SYMBOL(iwl_read_prph_no_grab);
 
 void iwl_write_prph_no_grab(struct iwl_trans *trans, u32 ofs, u32 val)
 {
 	trace_iwlwifi_dev_iowrite_prph32(trans->dev, ofs, val);
 	iwl_trans_write_prph(trans, ofs, val);
 }
-IWL_EXPORT_SYMBOL(iwl_write_prph_no_grab);
 
 void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val)
 {
@@ -132,7 +127,6 @@ void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val)
 	iwl_write_prph_no_grab(trans, ofs, val & 0xffffffff);
 	iwl_write_prph_no_grab(trans, ofs + 4, val >> 32);
 }
-IWL_EXPORT_SYMBOL(iwl_write_prph64_no_grab);
 
 u32 iwl_read_prph(struct iwl_trans *trans, u32 ofs)
 {
@@ -211,13 +205,13 @@ IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);
 
 void iwl_force_nmi(struct iwl_trans *trans)
 {
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
 		iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,
 				     DEVICE_SET_NMI_VAL_DRV, 1);
-	else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
 				UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);
-	else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
+	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
 		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
 				    UREG_DOORBELL_TO_ISR6_NMI_BIT);
 	else
@@ -260,7 +254,7 @@ struct reg {
 static int iwl_dump_rfh(struct iwl_trans *trans, char **buf)
 {
 	int i, q;
-	int num_q = trans->num_rx_queues;
+	int num_q = trans->info.num_rxqs;
 	static const u32 rfh_tbl[] = {
 		RFH_RXF_DMA_CFG,
 		RFH_GEN_CFG,
@@ -368,7 +362,7 @@ int iwl_dump_fh(struct iwl_trans *trans, char **buf)
 		FH_TSSR_TX_ERROR_REG
 	};
 
-	if (trans->trans_cfg->mq_rx_supported)
+	if (trans->mac_cfg->mq_rx_supported)
 		return iwl_dump_rfh(trans, buf);
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
@@ -423,7 +417,7 @@ static void iwl_dump_host_monitor_block(struct iwl_trans *trans,
 
 static void iwl_dump_host_monitor(struct iwl_trans *trans)
 {
-	switch (trans->trans_cfg->device_family) {
+	switch (trans->mac_cfg->device_family) {
 	case IWL_DEVICE_FAMILY_22000:
 	case IWL_DEVICE_FAMILY_AX210:
 		IWL_ERR(trans, "CSR_RESET = 0x%x\n",
@@ -445,11 +439,11 @@ static void iwl_dump_host_monitor(struct iwl_trans *trans)
 
 int iwl_finish_nic_init(struct iwl_trans *trans)
 {
-	const struct iwl_cfg_trans_params *cfg_trans = trans->trans_cfg;
+	const struct iwl_mac_cfg *mac_cfg = trans->mac_cfg;
 	u32 poll_ready;
 	int err;
 
-	if (cfg_trans->bisr_workaround) {
+	if (mac_cfg->bisr_workaround) {
 		/* ensure the TOP FSM isn't still in previous reset */
 		mdelay(2);
 	}
@@ -458,7 +452,7 @@ int iwl_finish_nic_init(struct iwl_trans *trans)
 	 * Set "initialization complete" bit to move adapter from
 	 * D0U* --> D0A* (powered-up active) state.
 	 */
-	if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		iwl_set_bit(trans, CSR_GP_CNTRL,
 			    CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ |
 			    CSR_GP_CNTRL_REG_FLAG_MAC_INIT);
@@ -469,7 +463,7 @@ int iwl_finish_nic_init(struct iwl_trans *trans)
 		poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY;
 	}
 
-	if (cfg_trans->device_family == IWL_DEVICE_FAMILY_8000)
+	if (mac_cfg->device_family == IWL_DEVICE_FAMILY_8000)
 		udelay(2);
 
 	/*
@@ -477,14 +471,14 @@ int iwl_finish_nic_init(struct iwl_trans *trans)
 	 * device-internal resources is supported, e.g. iwl_write_prph()
 	 * and accesses to uCode SRAM.
 	 */
-	err = iwl_poll_bit(trans, CSR_GP_CNTRL, poll_ready, poll_ready, 25000);
+	err = iwl_poll_bits(trans, CSR_GP_CNTRL, poll_ready, 25000);
 	if (err < 0) {
 		IWL_DEBUG_INFO(trans, "Failed to wake NIC\n");
 
 		iwl_dump_host_monitor(trans);
 	}
 
-	if (cfg_trans->bisr_workaround) {
+	if (mac_cfg->bisr_workaround) {
 		/* ensure BISR shift has finished */
 		udelay(200);
 	}
@@ -526,5 +520,5 @@ void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
 	if (interrupts_enabled)
 		iwl_trans_interrupts(trans, true);
 
-	iwl_trans_fw_error(trans, false);
+	iwl_trans_fw_error(trans, IWL_ERR_TYPE_NMI_FORCED);
 }
diff --git a/sys/contrib/dev/iwlwifi/iwl-io.h b/sys/contrib/dev/iwlwifi/iwl-io.h
index 37b3bd62897e..731cda1a4e66 100644
--- a/sys/contrib/dev/iwlwifi/iwl-io.h
+++ b/sys/contrib/dev/iwlwifi/iwl-io.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2018-2021 Intel Corporation
+ * Copyright (C) 2018-2021, 2025 Intel Corporation
  */
 #ifndef __iwl_io_h__
 #define __iwl_io_h__
@@ -23,8 +23,13 @@ static inline void iwl_clear_bit(struct iwl_trans *trans, u32 reg, u32 mask)
 	iwl_trans_set_bits_mask(trans, reg, mask, 0);
 }
 
-int iwl_poll_bit(struct iwl_trans *trans, u32 addr,
-		 u32 bits, u32 mask, int timeout);
+int iwl_poll_bits_mask(struct iwl_trans *trans, u32 addr,
+		       u32 bits, u32 mask, int timeout);
+static inline int iwl_poll_bits(struct iwl_trans *trans, u32 addr, u32 bits,
+				int timeout)
+{
+	return iwl_poll_bits_mask(trans, addr, bits, bits, timeout);
+}
 int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,
 			int timeout);
 
@@ -64,38 +69,38 @@ int iwl_dump_fh(struct iwl_trans *trans, char **buf);
  */
 static inline u32 iwl_umac_prph(struct iwl_trans *trans, u32 ofs)
 {
-	return ofs + trans->trans_cfg->umac_prph_offset;
+	return ofs + trans->mac_cfg->umac_prph_offset;
 }
 
 static inline u32 iwl_read_umac_prph_no_grab(struct iwl_trans *trans, u32 ofs)
 {
 	return iwl_read_prph_no_grab(trans, ofs +
-				     trans->trans_cfg->umac_prph_offset);
+				     trans->mac_cfg->umac_prph_offset);
 }
 
 static inline u32 iwl_read_umac_prph(struct iwl_trans *trans, u32 ofs)
 {
-	return iwl_read_prph(trans, ofs + trans->trans_cfg->umac_prph_offset);
+	return iwl_read_prph(trans, ofs + trans->mac_cfg->umac_prph_offset);
 }
 
 static inline void iwl_write_umac_prph_no_grab(struct iwl_trans *trans, u32 ofs,
 					       u32 val)
 {
-	iwl_write_prph_no_grab(trans,  ofs + trans->trans_cfg->umac_prph_offset,
+	iwl_write_prph_no_grab(trans,  ofs + trans->mac_cfg->umac_prph_offset,
 			       val);
 }
 
 static inline void iwl_write_umac_prph(struct iwl_trans *trans, u32 ofs,
 				       u32 val)
 {
-	iwl_write_prph(trans,  ofs + trans->trans_cfg->umac_prph_offset, val);
+	iwl_write_prph(trans,  ofs + trans->mac_cfg->umac_prph_offset, val);
 }
 
 static inline int iwl_poll_umac_prph_bit(struct iwl_trans *trans, u32 addr,
 					 u32 bits, u32 mask, int timeout)
 {
 	return iwl_poll_prph_bit(trans, addr +
-				 trans->trans_cfg->umac_prph_offset,
+				 trans->mac_cfg->umac_prph_offset,
 				 bits, mask, timeout);
 }
 
diff --git a/sys/contrib/dev/iwlwifi/iwl-nvm-parse.c b/sys/contrib/dev/iwlwifi/iwl-nvm-parse.c
index 384082e20af8..cfcaf12b7b7d 100644
--- a/sys/contrib/dev/iwlwifi/iwl-nvm-parse.c
+++ b/sys/contrib/dev/iwlwifi/iwl-nvm-parse.c
@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2023 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2023, 2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
 #include <linux/types.h>
+#include <linux/fips.h>
 #include <linux/slab.h>
 #include <linux/export.h>
 #include <linux/etherdevice.h>
@@ -141,8 +142,13 @@ static struct ieee80211_rate iwl_cfg80211_rates[] = {
 /**
  * enum iwl_nvm_channel_flags - channel flags in NVM
  * @NVM_CHANNEL_VALID: channel is usable for this SKU/geo
- * @NVM_CHANNEL_IBSS: usable as an IBSS channel
- * @NVM_CHANNEL_ACTIVE: active scanning allowed
+ * @NVM_CHANNEL_IBSS: usable as an IBSS channel and deprecated
+ *	when %IWL_NVM_SBANDS_FLAGS_LAR enabled.
+ * @NVM_CHANNEL_ALLOW_20MHZ_ACTIVITY: active scanning allowed and
+ *	AP allowed only in 20 MHz. Valid only
+ *	when %IWL_NVM_SBANDS_FLAGS_LAR enabled.
+ * @NVM_CHANNEL_ACTIVE: active scanning allowed and allows IBSS
+ *	when %IWL_NVM_SBANDS_FLAGS_LAR enabled.
  * @NVM_CHANNEL_RADAR: radar detection required
  * @NVM_CHANNEL_INDOOR_ONLY: only indoor use is allowed
  * @NVM_CHANNEL_GO_CONCURRENT: GO operation is allowed when connected to BSS
@@ -155,22 +161,26 @@ static struct ieee80211_rate iwl_cfg80211_rates[] = {
  * @NVM_CHANNEL_DC_HIGH: DC HIGH required/allowed (?)
  * @NVM_CHANNEL_VLP: client support connection to UHB VLP AP
  * @NVM_CHANNEL_AFC: client support connection to UHB AFC AP
+ * @NVM_CHANNEL_VLP_AP_NOT_ALLOWED: UHB VLP AP not allowed,
+ *	Valid only when %NVM_CHANNEL_VLP is enabled.
  */
 enum iwl_nvm_channel_flags {
-	NVM_CHANNEL_VALID		= BIT(0),
-	NVM_CHANNEL_IBSS		= BIT(1),
-	NVM_CHANNEL_ACTIVE		= BIT(3),
-	NVM_CHANNEL_RADAR		= BIT(4),
-	NVM_CHANNEL_INDOOR_ONLY		= BIT(5),
-	NVM_CHANNEL_GO_CONCURRENT	= BIT(6),
-	NVM_CHANNEL_UNIFORM		= BIT(7),
-	NVM_CHANNEL_20MHZ		= BIT(8),
-	NVM_CHANNEL_40MHZ		= BIT(9),
-	NVM_CHANNEL_80MHZ		= BIT(10),
-	NVM_CHANNEL_160MHZ		= BIT(11),
-	NVM_CHANNEL_DC_HIGH		= BIT(12),
-	NVM_CHANNEL_VLP			= BIT(13),
-	NVM_CHANNEL_AFC			= BIT(14),
+	NVM_CHANNEL_VALID			= BIT(0),
+	NVM_CHANNEL_IBSS			= BIT(1),
+	NVM_CHANNEL_ALLOW_20MHZ_ACTIVITY	= BIT(2),
+	NVM_CHANNEL_ACTIVE			= BIT(3),
+	NVM_CHANNEL_RADAR			= BIT(4),
+	NVM_CHANNEL_INDOOR_ONLY			= BIT(5),
+	NVM_CHANNEL_GO_CONCURRENT		= BIT(6),
+	NVM_CHANNEL_UNIFORM			= BIT(7),
+	NVM_CHANNEL_20MHZ			= BIT(8),
+	NVM_CHANNEL_40MHZ			= BIT(9),
+	NVM_CHANNEL_80MHZ			= BIT(10),
+	NVM_CHANNEL_160MHZ			= BIT(11),
+	NVM_CHANNEL_DC_HIGH			= BIT(12),
+	NVM_CHANNEL_VLP				= BIT(13),
+	NVM_CHANNEL_AFC				= BIT(14),
+	NVM_CHANNEL_VLP_AP_NOT_ALLOWED		= BIT(15),
 };
 
 /**
@@ -330,7 +340,7 @@ static inline void iwl_nvm_print_channel_flags(struct device *dev, u32 level,
 }
 
 static u32 iwl_get_channel_flags(u8 ch_num, int ch_idx, enum nl80211_band band,
-				 u32 nvm_flags, const struct iwl_cfg *cfg)
+				 u32 nvm_flags, const struct iwl_rf_cfg *cfg)
 {
 	u32 flags = IEEE80211_CHAN_NO_HT40;
 
@@ -397,7 +407,7 @@ static int iwl_init_channel_map(struct iwl_trans *trans,
 				const void * const nvm_ch_flags,
 				u32 sbands_flags, bool v4)
 {
-	const struct iwl_cfg *cfg = trans->cfg;
+	const struct iwl_rf_cfg *cfg = trans->cfg;
 	struct device *dev = trans->dev;
 	int ch_idx;
 	int n_channels = 0;
@@ -498,7 +508,7 @@ static void iwl_init_vht_hw_capab(struct iwl_trans *trans,
 				  struct ieee80211_sta_vht_cap *vht_cap,
 				  u8 tx_chains, u8 rx_chains)
 {
-	const struct iwl_cfg *cfg = trans->cfg;
+	const struct iwl_rf_cfg *cfg = trans->cfg;
 	int num_rx_ants = num_of_ant(rx_chains);
 	int num_tx_ants = num_of_ant(tx_chains);
 
@@ -511,7 +521,7 @@ static void iwl_init_vht_hw_capab(struct iwl_trans *trans,
 		       IEEE80211_VHT_MAX_AMPDU_1024K <<
 		       IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT;
 
-	if (!trans->cfg->ht_params->stbc)
+	if (!trans->cfg->ht_params.stbc)
 		vht_cap->cap &= ~IEEE80211_VHT_CAP_RXSTBC_MASK;
 
 	if (data->vht160_supported)
@@ -521,7 +531,7 @@ static void iwl_init_vht_hw_capab(struct iwl_trans *trans,
 	if (cfg->vht_mu_mimo_supported)
 		vht_cap->cap |= IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE;
 
-	if (cfg->ht_params->ldpc)
+	if (cfg->ht_params.ldpc)
 		vht_cap->cap |= IEEE80211_VHT_CAP_RXLDPC;
 
 	if (data->sku_cap_mimo_disabled) {
@@ -529,21 +539,27 @@ static void iwl_init_vht_hw_capab(struct iwl_trans *trans,
 		num_tx_ants = 1;
 	}
 
-	if (trans->cfg->ht_params->stbc && num_tx_ants > 1)
+	if (trans->cfg->ht_params.stbc && num_tx_ants > 1)
 		vht_cap->cap |= IEEE80211_VHT_CAP_TXSTBC;
 	else
 		vht_cap->cap |= IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN;
 
+	/*
+	 * With fips_enabled crypto is done by software, so the HW cannot
+	 * split up A-MSDUs and the real limit that was set applies.
+	 * Note that EHT doesn't honour this (HE copies the VHT value),
+	 * but EHT is also entirely disabled for fips_enabled.
+	 */
 	switch (iwlwifi_mod_params.amsdu_size) {
 	case IWL_AMSDU_DEF:
-		if (trans->trans_cfg->mq_rx_supported)
+		if (trans->mac_cfg->mq_rx_supported && !fips_enabled)
 			vht_cap->cap |=
 				IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454;
 		else
 			vht_cap->cap |= IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895;
 		break;
 	case IWL_AMSDU_2K:
-		if (trans->trans_cfg->mq_rx_supported)
+		if (trans->mac_cfg->mq_rx_supported && !fips_enabled)
 			vht_cap->cap |=
 				IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454;
 		else
@@ -654,6 +670,8 @@ static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = {
 				.phy_cap_info[9] =
 					IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB |
 					IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB |
+					IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU |
+					IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU |
 					(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED <<
 					IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_POS),
 				.phy_cap_info[10] =
@@ -682,41 +700,26 @@ static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = {
 			.has_eht = true,
 			.eht_cap_elem = {
 				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1 |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2 |
-					IEEE80211_EHT_MAC_CAP0_SCS_TRAFFIC_DESC,
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL,
 				.phy_cap_info[0] =
 					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
 					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
-					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
 					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
 					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
 				.phy_cap_info[1] =
 					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK  |
 					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK,
 				.phy_cap_info[3] =
-					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK,
 
 				.phy_cap_info[4] =
-					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
 					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI,
 				.phy_cap_info[5] =
 					FIELD_PREP_CONST(IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK,
 							 IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_16US) |
-					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
 					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP,
-				.phy_cap_info[6] =
-					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK |
-					IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP,
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF,
 				.phy_cap_info[8] =
 					IEEE80211_EHT_PHY_CAP8_RX_1024QAM_WIDER_BW_DL_OFDMA |
 					IEEE80211_EHT_PHY_CAP8_RX_4096QAM_WIDER_BW_DL_OFDMA,
@@ -784,6 +787,7 @@ static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = {
 					IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI |
 					IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_242,
 				.phy_cap_info[9] =
+					IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU |
 					IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED
 					<< IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_POS,
 			},
@@ -810,9 +814,7 @@ static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = {
 			.has_eht = true,
 			.eht_cap_elem = {
 				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1 |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2,
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL,
 				.phy_cap_info[0] =
 					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
 					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI,
@@ -911,13 +913,12 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 {
 	bool is_ap = iftype_data->types_mask & (BIT(NL80211_IFTYPE_AP) |
 						BIT(NL80211_IFTYPE_P2P_GO));
-	bool no_320;
+	bool slow_pcie = (!trans->mac_cfg->integrated &&
+			  trans->info.pcie_link_speed < PCI_EXP_LNKSTA_CLS_8_0GB);
 
-	no_320 = (!trans->trans_cfg->integrated &&
-		 trans->pcie_link_speed < PCI_EXP_LNKSTA_CLS_8_0GB) ||
-		 trans->reduced_cap_sku;
-
-	if (!data->sku_cap_11be_enable || iwlwifi_mod_params.disable_11be)
+	/* EHT needs WPA3/MFP so cannot do it for fips_enabled */
+	if (!data->sku_cap_11be_enable || iwlwifi_mod_params.disable_11be ||
+	    fips_enabled)
 		iftype_data->eht_cap.has_eht = false;
 
 	/* Advertise an A-MPDU exponent extension based on
@@ -942,7 +943,8 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 				       IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK);
 		break;
 	case NL80211_BAND_6GHZ:
-		if (!no_320) {
+		if (!trans->reduced_cap_sku &&
+		    (!trans->cfg->bw_limit || trans->cfg->bw_limit >= 320)) {
 			iftype_data->eht_cap.eht_cap_elem.phy_cap_info[0] |=
 				IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ;
 			iftype_data->eht_cap.eht_cap_elem.phy_cap_info[1] |=
@@ -984,6 +986,14 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 				iftype_data->eht_cap.eht_cap_elem.phy_cap_info[4] |= 0x10;
 			}
 		}
+
+		if (slow_pcie) {
+			struct ieee80211_eht_mcs_nss_supp *mcs_nss =
+				&iftype_data->eht_cap.eht_mcs_nss_supp;
+
+			mcs_nss->bw._320.rx_tx_mcs11_max_nss = 0;
+			mcs_nss->bw._320.rx_tx_mcs13_max_nss = 0;
+		}
 	} else {
 		struct ieee80211_he_mcs_nss_supp *he_mcs_nss_supp =
 			&iftype_data->he_cap.he_mcs_nss_supp;
@@ -1021,58 +1031,28 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 			cpu_to_le16(IEEE80211_HE_MCS_NOT_SUPPORTED << 2);
 	}
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210 && !is_ap)
+	/* prior RFs don't have HE, HR RF doesn't have this, later have it */
+	if (CSR_HW_RFID_TYPE(trans->info.hw_rf_id) == IWL_CFG_RF_TYPE_HR1 ||
+	    CSR_HW_RFID_TYPE(trans->info.hw_rf_id) == IWL_CFG_RF_TYPE_HR2)
+		iftype_data->he_cap.he_cap_elem.phy_cap_info[9] &=
+			~(IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU |
+			  IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU);
+
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210 && !is_ap)
 		iftype_data->he_cap.he_cap_elem.phy_cap_info[2] |=
 			IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO;
 
-	switch (CSR_HW_RFID_TYPE(trans->hw_rf_id)) {
-	case IWL_CFG_RF_TYPE_GF:
-	case IWL_CFG_RF_TYPE_FM:
-	case IWL_CFG_RF_TYPE_WH:
-		iftype_data->he_cap.he_cap_elem.phy_cap_info[9] |=
-			IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU;
-		if (!is_ap)
-			iftype_data->he_cap.he_cap_elem.phy_cap_info[9] |=
-				IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU;
-		break;
-	}
-
-	if (CSR_HW_REV_TYPE(trans->hw_rev) == IWL_CFG_MAC_TYPE_GL &&
-	    iftype_data->eht_cap.has_eht) {
-		iftype_data->eht_cap.eht_cap_elem.mac_cap_info[0] &=
-			~(IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1 |
-			  IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE2);
-		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[3] &=
-			~(IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
-			  IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-			  IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-			  IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-			  IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-			  IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-			  IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK);
-		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[4] &=
-			~(IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-			  IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP);
-		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[5] &=
-			~IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK;
-		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[6] &=
-			~(IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK |
-			  IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP);
-		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[5] |=
-			IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF;
-	}
-
 	if (fw_has_capa(&fw->ucode_capa, IWL_UCODE_TLV_CAPA_BROADCAST_TWT))
 		iftype_data->he_cap.he_cap_elem.mac_cap_info[2] |=
 			IEEE80211_HE_MAC_CAP2_BCAST_TWT;
 
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000 &&
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_22000 &&
 	    !is_ap) {
 		iftype_data->vendor_elems.data = iwl_vendor_caps;
 		iftype_data->vendor_elems.len = ARRAY_SIZE(iwl_vendor_caps);
 	}
 
-	if (!trans->cfg->ht_params->stbc) {
+	if (!trans->cfg->ht_params.stbc) {
 		iftype_data->he_cap.he_cap_elem.phy_cap_info[2] &=
 			~IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ;
 		iftype_data->he_cap.he_cap_elem.phy_cap_info[7] &=
@@ -1084,19 +1064,23 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 		iftype_data->eht_cap.eht_mcs_nss_supp.bw._320.rx_tx_mcs13_max_nss = 0;
 	}
 
-	if (trans->no_160)
+	if (trans->cfg->bw_limit && trans->cfg->bw_limit < 160)
 		iftype_data->he_cap.he_cap_elem.phy_cap_info[0] &=
 			~IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
 
-	if (trans->reduced_cap_sku) {
+	if ((trans->cfg->bw_limit && trans->cfg->bw_limit < 320) ||
+	    trans->reduced_cap_sku) {
 		memset(&iftype_data->eht_cap.eht_mcs_nss_supp.bw._320, 0,
 		       sizeof(iftype_data->eht_cap.eht_mcs_nss_supp.bw._320));
+		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[2] &=
+			~IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK;
+	}
+
+	if (trans->reduced_cap_sku) {
 		iftype_data->eht_cap.eht_mcs_nss_supp.bw._80.rx_tx_mcs13_max_nss = 0;
 		iftype_data->eht_cap.eht_mcs_nss_supp.bw._160.rx_tx_mcs13_max_nss = 0;
 		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[8] &=
 			~IEEE80211_EHT_PHY_CAP8_RX_4096QAM_WIDER_BW_DL_OFDMA;
-		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[2] &=
-			~IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK;
 	}
 }
 
@@ -1222,17 +1206,25 @@ static void iwl_init_sbands(struct iwl_trans *trans,
 	n_used += iwl_init_sband_channels(data, sband, n_channels,
 					  NL80211_BAND_6GHZ);
 
-	if (data->sku_cap_11ax_enable && !iwlwifi_mod_params.disable_11ax)
+	/*
+	 * 6 GHz requires WPA3 which requires MFP, which FW cannot do
+	 * when fips_enabled, so don't advertise any 6 GHz channels to
+	 * avoid spending time on scanning those channels and perhaps
+	 * even finding APs there that cannot be used.
+	 */
+	if (!fips_enabled && data->sku_cap_11ax_enable &&
+	    !iwlwifi_mod_params.disable_11ax)
 		iwl_init_he_hw_capab(trans, data, sband, tx_chains, rx_chains,
 				     fw);
 	else
 		sband->n_channels = 0;
+
 	if (n_channels != n_used)
 		IWL_ERR_DEV(dev, "NVM: used only %d of %d channels\n",
 			    n_used, n_channels);
 }
 
-static int iwl_get_sku(const struct iwl_cfg *cfg, const __le16 *nvm_sw,
+static int iwl_get_sku(const struct iwl_rf_cfg *cfg, const __le16 *nvm_sw,
 		       const __le16 *phy_sku)
 {
 	if (cfg->nvm_type != IWL_NVM_EXT)
@@ -1241,7 +1233,7 @@ static int iwl_get_sku(const struct iwl_cfg *cfg, const __le16 *nvm_sw,
 	return le32_to_cpup((const __le32 *)(phy_sku + SKU_FAMILY_8000));
 }
 
-static int iwl_get_nvm_version(const struct iwl_cfg *cfg, const __le16 *nvm_sw)
+static int iwl_get_nvm_version(const struct iwl_rf_cfg *cfg, const __le16 *nvm_sw)
 {
 	if (cfg->nvm_type != IWL_NVM_EXT)
 		return le16_to_cpup(nvm_sw + NVM_VERSION);
@@ -1250,7 +1242,7 @@ static int iwl_get_nvm_version(const struct iwl_cfg *cfg, const __le16 *nvm_sw)
 						     NVM_VERSION_EXT_NVM));
 }
 
-static int iwl_get_radio_cfg(const struct iwl_cfg *cfg, const __le16 *nvm_sw,
+static int iwl_get_radio_cfg(const struct iwl_rf_cfg *cfg, const __le16 *nvm_sw,
 			     const __le16 *phy_sku)
 {
 	if (cfg->nvm_type != IWL_NVM_EXT)
@@ -1260,7 +1252,7 @@ static int iwl_get_radio_cfg(const struct iwl_cfg *cfg, const __le16 *nvm_sw,
 
 }
 
-static int iwl_get_n_hw_addrs(const struct iwl_cfg *cfg, const __le16 *nvm_sw)
+static int iwl_get_n_hw_addrs(const struct iwl_rf_cfg *cfg, const __le16 *nvm_sw)
 {
 	int n_hw_addr;
 
@@ -1272,7 +1264,7 @@ static int iwl_get_n_hw_addrs(const struct iwl_cfg *cfg, const __le16 *nvm_sw)
 	return n_hw_addr & N_HW_ADDR_MASK;
 }
 
-static void iwl_set_radio_cfg(const struct iwl_cfg *cfg,
+static void iwl_set_radio_cfg(const struct iwl_rf_cfg *cfg,
 			      struct iwl_nvm_data *data,
 			      u32 radio_cfg)
 {
@@ -1331,7 +1323,7 @@ static void iwl_set_hw_address_from_csr(struct iwl_trans *trans,
 }
 
 static void iwl_set_hw_address_family_8000(struct iwl_trans *trans,
-					   const struct iwl_cfg *cfg,
+					   const struct iwl_rf_cfg *cfg,
 					   struct iwl_nvm_data *data,
 					   const __le16 *mac_override,
 					   const __be16 *nvm_hw)
@@ -1380,11 +1372,12 @@ static void iwl_set_hw_address_family_8000(struct iwl_trans *trans,
 }
 
 static int iwl_set_hw_address(struct iwl_trans *trans,
-			      const struct iwl_cfg *cfg,
+			      const struct iwl_rf_cfg *cfg,
 			      struct iwl_nvm_data *data, const __be16 *nvm_hw,
 			      const __le16 *mac_override)
 {
-	if (cfg->mac_addr_from_csr) {
+	const struct iwl_mac_cfg *mac_cfg = trans->mac_cfg;
+	if (mac_cfg->base->mac_addr_from_csr) {
 		iwl_set_hw_address_from_csr(trans, data);
 	} else if (cfg->nvm_type != IWL_NVM_EXT) {
 		const u8 *hw_addr = (const u8 *)(nvm_hw + HW_ADDR);
@@ -1419,7 +1412,7 @@ static int iwl_set_hw_address(struct iwl_trans *trans,
 }
 
 static bool
-iwl_nvm_no_wide_in_5ghz(struct iwl_trans *trans, const struct iwl_cfg *cfg,
+iwl_nvm_no_wide_in_5ghz(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 			const __be16 *nvm_hw)
 {
 	/*
@@ -1431,7 +1424,7 @@ iwl_nvm_no_wide_in_5ghz(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	 * in 5GHz otherwise the FW will throw a sysassert when we try
 	 * to use them.
 	 */
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
 		/*
 		 * Unlike the other sections in the NVM, the hw
 		 * section uses big-endian.
@@ -1451,7 +1444,7 @@ iwl_nvm_no_wide_in_5ghz(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 }
 
 struct iwl_nvm_data *
-iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
+iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 		       const struct iwl_mei_nvm *mei_nvm,
 		       const struct iwl_fw *fw, u8 tx_ant, u8 rx_ant)
 {
@@ -1515,7 +1508,7 @@ iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 IWL_EXPORT_SYMBOL(iwl_parse_mei_nvm_data);
 
 struct iwl_nvm_data *
-iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
+iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 		   const struct iwl_fw *fw,
 		   const __be16 *nvm_hw, const __le16 *nvm_sw,
 		   const __le16 *nvm_calib, const __le16 *regulatory,
@@ -1614,8 +1607,7 @@ IWL_EXPORT_SYMBOL(iwl_parse_nvm_data);
 
 static u32 iwl_nvm_get_regdom_bw_flags(const u16 *nvm_chan,
 				       int ch_idx, u16 nvm_flags,
-				       struct iwl_reg_capa reg_capa,
-				       const struct iwl_cfg *cfg)
+				       struct iwl_reg_capa reg_capa)
 {
 	u32 flags = NL80211_RRF_NO_HT40;
 
@@ -1650,6 +1642,10 @@ static u32 iwl_nvm_get_regdom_bw_flags(const u16 *nvm_chan,
 	if (nvm_flags & NVM_CHANNEL_INDOOR_ONLY)
 		flags |= NL80211_RRF_NO_OUTDOOR;
 
+	if (nvm_flags & NVM_CHANNEL_ALLOW_20MHZ_ACTIVITY &&
+	    flags & NL80211_RRF_NO_IR)
+		flags |= NL80211_RRF_ALLOW_20MHZ_ACTIVITY;
+
 	/* Set the GO concurrent flag only in case that NO_IR is set.
 	 * Otherwise it is meaningless
 	 */
@@ -1666,10 +1662,12 @@ static u32 iwl_nvm_get_regdom_bw_flags(const u16 *nvm_chan,
 	}
 
 	/* Set the AP type for the UHB case. */
-	if (nvm_flags & NVM_CHANNEL_VLP)
-		flags |= NL80211_RRF_ALLOW_6GHZ_VLP_AP;
-	else
+	if (nvm_flags & NVM_CHANNEL_VLP) {
+		if (!(nvm_flags & NVM_CHANNEL_VLP_AP_NOT_ALLOWED))
+			flags |= NL80211_RRF_ALLOW_6GHZ_VLP_AP;
+	} else {
 		flags |= NL80211_RRF_NO_6GHZ_VLP_CLIENT;
+	}
 
 	if (!(nvm_flags & NVM_CHANNEL_AFC))
 		flags |= NL80211_RRF_NO_6GHZ_AFC_CLIENT;
@@ -1726,10 +1724,12 @@ static struct iwl_reg_capa iwl_get_reg_capa(u32 flags, u8 resp_ver)
 }
 
 struct ieee80211_regdomain *
-iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
+iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 		       int num_of_ch, __le32 *channels, u16 fw_mcc,
 		       u16 geo_info, u32 cap, u8 resp_ver)
 {
+	const struct iwl_rf_cfg *cfg = trans->cfg;
+	struct device *dev = trans->dev;
 	int ch_idx;
 	u16 ch_flags;
 	u32 reg_rule_flags, prev_reg_rule_flags = 0;
@@ -1794,8 +1794,8 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 		}
 
 		reg_rule_flags = iwl_nvm_get_regdom_bw_flags(nvm_chan, ch_idx,
-							     ch_flags, reg_capa,
-							     cfg);
+							     ch_flags,
+							     reg_capa);
 
 		/* we can't continue the same rule */
 		if (ch_idx == 0 || prev_reg_rule_flags != reg_rule_flags ||
@@ -1984,8 +1984,8 @@ int iwl_read_external_nvm(struct iwl_trans *trans,
 			 le32_to_cpu(dword_buff[3]));
 
 		/* nvm file validation, dword_buff[2] holds the file version */
-		if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_8000 &&
-		    trans->hw_rev_step == SILICON_C_STEP &&
+		if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_8000 &&
+		    trans->info.hw_rev_step == SILICON_C_STEP &&
 		    le32_to_cpu(dword_buff[2]) < 0xE4A) {
 			ret = -EFAULT;
 			goto out;
@@ -2052,7 +2052,7 @@ int iwl_read_external_nvm(struct iwl_trans *trans,
 			break;
 		}
 
-		iwl_nvm_fixups(trans->hw_id, section_id, temp, section_size);
+		iwl_nvm_fixups(trans->info.hw_id, section_id, temp, section_size);
 
 		kfree(nvm_sections[section_id].data);
 		nvm_sections[section_id].data = temp;
@@ -2159,7 +2159,7 @@ struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
 		!!(mac_flags & NVM_MAC_SKU_FLAGS_BAND_5_2_ENABLED);
 	nvm->sku_cap_mimo_disabled =
 		!!(mac_flags & NVM_MAC_SKU_FLAGS_MIMO_DISABLED);
-	if (CSR_HW_RFID_TYPE(trans->hw_rf_id) >= IWL_CFG_RF_TYPE_FM)
+	if (CSR_HW_RFID_TYPE(trans->info.hw_rf_id) >= IWL_CFG_RF_TYPE_FM)
 		nvm->sku_cap_11be_enable = true;
 
 	/* Initialize PHY sku data */
diff --git a/sys/contrib/dev/iwlwifi/iwl-nvm-parse.h b/sys/contrib/dev/iwlwifi/iwl-nvm-parse.h
index 0c6c3fb8c6dd..9ce9fa4e78fd 100644
--- a/sys/contrib/dev/iwlwifi/iwl-nvm-parse.h
+++ b/sys/contrib/dev/iwlwifi/iwl-nvm-parse.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2015, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2015, 2018-2025 Intel Corporation
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_nvm_parse_h__
@@ -30,7 +30,7 @@ enum iwl_nvm_sbands_flags {
  * later with iwl_free_nvm_data().
  */
 struct iwl_nvm_data *
-iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
+iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 		   const struct iwl_fw *fw,
 		   const __be16 *nvm_hw, const __le16 *nvm_sw,
 		   const __le16 *nvm_calib, const __le16 *regulatory,
@@ -46,9 +46,17 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
  * accordingly. An ERR_PTR is returned on error.
  * If not given to the regulatory core, the user is responsible for freeing
  * the regdomain returned here with kfree.
+ *
+ * @trans: the transport
+ * @num_of_ch: the number of channels
+ * @channels: channel map
+ * @fw_mcc: firmware country code
+ * @geo_info: geo info value
+ * @cap: capability
+ * @resp_ver: FW response version
  */
 struct ieee80211_regdomain *
-iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
+iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 		       int num_of_ch, __le32 *channels, u16 fw_mcc,
 		       u16 geo_info, u32 cap, u8 resp_ver);
 
@@ -87,7 +95,7 @@ struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
  * iwl_parse_mei_nvm_data - parse the mei_nvm_data and get an iwl_nvm_data
  */
 struct iwl_nvm_data *
-iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
+iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 		       const struct iwl_mei_nvm *mei_nvm,
 		       const struct iwl_fw *fw, u8 set_tx_ant, u8 set_rx_ant);
 
diff --git a/sys/contrib/dev/iwlwifi/iwl-nvm-utils.c b/sys/contrib/dev/iwlwifi/iwl-nvm-utils.c
index b3c25acd3691..ec312c90ff85 100644
--- a/sys/contrib/dev/iwlwifi/iwl-nvm-utils.c
+++ b/sys/contrib/dev/iwlwifi/iwl-nvm-utils.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2021, 2023 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2021, 2023, 2025 Intel Corporation
  * Copyright (C) 2015 Intel Mobile Communications GmbH
  */
 #include <linux/types.h>
@@ -42,7 +42,7 @@ void iwl_init_ht_hw_capab(struct iwl_trans *trans,
 			  enum nl80211_band band,
 			  u8 tx_chains, u8 rx_chains)
 {
-	const struct iwl_cfg *cfg = trans->cfg;
+	const struct iwl_rf_cfg *cfg = trans->cfg;
 	int max_bit_rate = 0;
 
 	tx_chains = hweight8(tx_chains);
@@ -53,7 +53,8 @@ void iwl_init_ht_hw_capab(struct iwl_trans *trans,
 
 	if (!(data->sku_cap_11n_enable) ||
 	    (iwlwifi_mod_params.disable_11n & IWL_DISABLE_HT_ALL) ||
-	    !cfg->ht_params) {
+	    /* there are no devices with HT but without HT40 entirely */
+	    !cfg->ht_params.ht40_bands) {
 		ht_info->ht_supported = false;
 		return;
 	}
@@ -64,17 +65,17 @@ void iwl_init_ht_hw_capab(struct iwl_trans *trans,
 	ht_info->ht_supported = true;
 	ht_info->cap = IEEE80211_HT_CAP_DSSSCCK40;
 
-	if (cfg->ht_params->stbc) {
+	if (cfg->ht_params.stbc) {
 		ht_info->cap |= (1 << IEEE80211_HT_CAP_RX_STBC_SHIFT);
 
 		if (tx_chains > 1)
 			ht_info->cap |= IEEE80211_HT_CAP_TX_STBC;
 	}
 
-	if (cfg->ht_params->ldpc)
+	if (cfg->ht_params.ldpc)
 		ht_info->cap |= IEEE80211_HT_CAP_LDPC_CODING;
 
-	if (trans->trans_cfg->mq_rx_supported ||
+	if (trans->mac_cfg->mq_rx_supported ||
 	    iwlwifi_mod_params.amsdu_size >= IWL_AMSDU_8K)
 		ht_info->cap |= IEEE80211_HT_CAP_MAX_AMSDU;
 
@@ -90,13 +91,13 @@ void iwl_init_ht_hw_capab(struct iwl_trans *trans,
 	if (rx_chains >= 3)
 		ht_info->mcs.rx_mask[2] = 0xFF;
 
-	if (cfg->ht_params->ht_greenfield_support)
+	if (cfg->ht_params.ht_greenfield_support)
 		ht_info->cap |= IEEE80211_HT_CAP_GRN_FLD;
 	ht_info->cap |= IEEE80211_HT_CAP_SGI_20;
 
 	max_bit_rate = MAX_BIT_RATE_20_MHZ;
 
-	if (cfg->ht_params->ht40_bands & BIT(band)) {
+	if (cfg->ht_params.ht40_bands & BIT(band)) {
 		ht_info->cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40;
 		ht_info->cap |= IEEE80211_HT_CAP_SGI_40;
 		max_bit_rate = MAX_BIT_RATE_40_MHZ;
diff --git a/sys/contrib/dev/iwlwifi/iwl-op-mode.h b/sys/contrib/dev/iwlwifi/iwl-op-mode.h
index 8ef5ed2db051..a146d0e399f2 100644
--- a/sys/contrib/dev/iwlwifi/iwl-op-mode.h
+++ b/sys/contrib/dev/iwlwifi/iwl-op-mode.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2021, 2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2021, 2024-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015 Intel Deutschland GmbH
  */
@@ -17,7 +17,7 @@ struct sk_buff;
 struct iwl_device_cmd;
 struct iwl_rx_cmd_buffer;
 struct iwl_fw;
-struct iwl_cfg;
+struct iwl_rf_cfg;
 
 /**
  * DOC: Operational mode - what is it ?
@@ -45,6 +45,63 @@ struct iwl_cfg;
  */
 
 /**
+ * enum iwl_fw_error_type - FW error types/sources
+ * @IWL_ERR_TYPE_IRQ: "normal" FW error through an IRQ
+ * @IWL_ERR_TYPE_NMI_FORCED: NMI was forced by driver
+ * @IWL_ERR_TYPE_RESET_HS_TIMEOUT: reset handshake timed out,
+ *	any debug collection must happen synchronously as
+ *	the device will be shut down
+ * @IWL_ERR_TYPE_CMD_QUEUE_FULL: command queue was full
+ * @IWL_ERR_TYPE_TOP_RESET_BY_BT: TOP reset initiated by BT
+ * @IWL_ERR_TYPE_TOP_FATAL_ERROR: TOP fatal error
+ * @IWL_ERR_TYPE_TOP_RESET_FAILED: TOP reset failed
+ * @IWL_ERR_TYPE_DEBUGFS: error/reset indication from debugfs
+ */
+enum iwl_fw_error_type {
+	IWL_ERR_TYPE_IRQ,
+	IWL_ERR_TYPE_NMI_FORCED,
+	IWL_ERR_TYPE_RESET_HS_TIMEOUT,
+	IWL_ERR_TYPE_CMD_QUEUE_FULL,
+	IWL_ERR_TYPE_TOP_RESET_BY_BT,
+	IWL_ERR_TYPE_TOP_FATAL_ERROR,
+	IWL_ERR_TYPE_TOP_RESET_FAILED,
+	IWL_ERR_TYPE_DEBUGFS,
+};
+
+/**
+ * enum iwl_fw_error_context - error dump context
+ * @IWL_ERR_CONTEXT_WORKER: regular from worker context,
+ *	opmode must acquire locks and must also check
+ *	for @IWL_ERR_CONTEXT_ABORT after acquiring locks
+ * @IWL_ERR_CONTEXT_FROM_OPMODE: context is in a call
+ *	originating from the opmode, e.g. while resetting
+ *	or stopping the device, so opmode must not acquire
+ *	any locks
+ * @IWL_ERR_CONTEXT_ABORT: after lock acquisition, indicates
+ *	that the dump already happened via another callback
+ *	(currently only while stopping the device) via the
+ *	@IWL_ERR_CONTEXT_FROM_OPMODE context, and this call
+ *	must be aborted
+ */
+enum iwl_fw_error_context {
+	IWL_ERR_CONTEXT_WORKER,
+	IWL_ERR_CONTEXT_FROM_OPMODE,
+	IWL_ERR_CONTEXT_ABORT,
+};
+
+/**
+ * struct iwl_fw_error_dump_mode - error dump mode for callback
+ * @type: The reason for the dump, per &enum iwl_fw_error_type.
+ * @context: The context for the dump, may also indicate this
+ *	call needs to be skipped. This MUST be checked before
+ *	and after acquiring any locks in the op-mode!
+ */
+struct iwl_fw_error_dump_mode {
+	enum iwl_fw_error_type type;
+	enum iwl_fw_error_context context;
+};
+
+/**
  * struct iwl_op_mode_ops - op_mode specific operations
  *
  * The op_mode exports its ops so that external components can start it and
@@ -77,10 +134,11 @@ struct iwl_cfg;
  *	reclaimed by the op_mode. This can happen when the driver is freed and
  *	there are Tx packets pending in the transport layer.
  *	Must be atomic
- * @nic_error: error notification. Must be atomic and must be called with BH
- *	disabled, unless the sync parameter is true.
- * @cmd_queue_full: Called when the command queue gets full. Must be atomic and
- *	called with BH disabled.
+ * @nic_error: error notification. Must be atomic, the op mode should handle
+ *	the error (e.g. abort notification waiters) and print the error if
+ *	applicable
+ * @dump_error: NIC error dump collection (can sleep, synchronous)
+ * @sw_reset: (maybe) initiate a software reset, return %true if started
  * @nic_config: configure NIC, called before firmware is started.
  *	May sleep
  * @wimax_active: invoked when WiMax becomes active. May sleep
@@ -89,10 +147,12 @@ struct iwl_cfg;
  *	Op_mode needs to reset its internal state because the device did not
  *	survive the system state transition. The firmware is no longer running,
  *	etc...
+ * @dump: Op_mode needs to collect the firmware dump upon this handler
+ *	being called.
  */
 struct iwl_op_mode_ops {
 	struct iwl_op_mode *(*start)(struct iwl_trans *trans,
-				     const struct iwl_cfg *cfg,
+				     const struct iwl_rf_cfg *cfg,
 				     const struct iwl_fw *fw,
 				     struct dentry *dbgfs_dir);
 	void (*stop)(struct iwl_op_mode *op_mode);
@@ -104,14 +164,19 @@ struct iwl_op_mode_ops {
 	void (*queue_not_full)(struct iwl_op_mode *op_mode, int queue);
 	bool (*hw_rf_kill)(struct iwl_op_mode *op_mode, bool state);
 	void (*free_skb)(struct iwl_op_mode *op_mode, struct sk_buff *skb);
-	void (*nic_error)(struct iwl_op_mode *op_mode, bool sync);
-	void (*cmd_queue_full)(struct iwl_op_mode *op_mode);
+	void (*nic_error)(struct iwl_op_mode *op_mode,
+			  enum iwl_fw_error_type type);
+	void (*dump_error)(struct iwl_op_mode *op_mode,
+			   struct iwl_fw_error_dump_mode *mode);
+	bool (*sw_reset)(struct iwl_op_mode *op_mode,
+			 enum iwl_fw_error_type type);
 	void (*nic_config)(struct iwl_op_mode *op_mode);
 	void (*wimax_active)(struct iwl_op_mode *op_mode);
 	void (*time_point)(struct iwl_op_mode *op_mode,
 			   enum iwl_fw_ini_time_point tp_id,
 			   union iwl_dbg_tlv_tp_data *tp_data);
 	void (*device_powered_off)(struct iwl_op_mode *op_mode);
+	void (*dump)(struct iwl_op_mode *op_mode);
 };
 
 int iwl_opmode_register(const char *name, const struct iwl_op_mode_ops *ops);
@@ -177,14 +242,22 @@ static inline void iwl_op_mode_free_skb(struct iwl_op_mode *op_mode,
 	op_mode->ops->free_skb(op_mode, skb);
 }
 
-static inline void iwl_op_mode_nic_error(struct iwl_op_mode *op_mode, bool sync)
+static inline void iwl_op_mode_nic_error(struct iwl_op_mode *op_mode,
+					 enum iwl_fw_error_type type)
 {
-	op_mode->ops->nic_error(op_mode, sync);
+	op_mode->ops->nic_error(op_mode, type);
 }
 
-static inline void iwl_op_mode_cmd_queue_full(struct iwl_op_mode *op_mode)
+static inline void iwl_op_mode_dump_error(struct iwl_op_mode *op_mode,
+					  struct iwl_fw_error_dump_mode *mode)
 {
-	op_mode->ops->cmd_queue_full(op_mode);
+	might_sleep();
+
+	if (WARN_ON(mode->type == IWL_ERR_TYPE_TOP_RESET_BY_BT))
+		return;
+
+	if (op_mode->ops->dump_error)
+		op_mode->ops->dump_error(op_mode, mode);
 }
 
 static inline void iwl_op_mode_nic_config(struct iwl_op_mode *op_mode)
@@ -216,4 +289,11 @@ static inline void iwl_op_mode_device_powered_off(struct iwl_op_mode *op_mode)
 	op_mode->ops->device_powered_off(op_mode);
 }
 
+static inline void iwl_op_mode_dump(struct iwl_op_mode *op_mode)
+{
+	if (!op_mode || !op_mode->ops || !op_mode->ops->dump)
+		return;
+	op_mode->ops->dump(op_mode);
+}
+
 #endif /* __iwl_op_mode_h__ */
diff --git a/sys/contrib/dev/iwlwifi/iwl-prph.h b/sys/contrib/dev/iwlwifi/iwl-prph.h
index dc171c29eb7b..a7214ddcfaf5 100644
--- a/sys/contrib/dev/iwlwifi/iwl-prph.h
+++ b/sys/contrib/dev/iwlwifi/iwl-prph.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -381,6 +381,10 @@ enum {
 #define CNVR_SCU_SD_REGS_SD_REG_ACTIVE_VDIG_MIRROR	0xA29938
 #define CNVI_SCU_SEQ_DATA_DW9				0xA27488
 
+#define CNVI_SCU_REG_FOR_ECO_1				0xA26EF8
+#define   CNVI_SCU_REG_FOR_ECO_1_WIAMT_KNOWN		BIT(4)
+#define   CNVI_SCU_REG_FOR_ECO_1_WIAMT_PRESENT		BIT(5)
+
 #define CNVI_PMU_STEP_FLOW				0xA2D588
 #define CNVI_PMU_STEP_FLOW_FORCE_URM			BIT(2)
 
@@ -458,6 +462,7 @@ enum {
 #define REG_CRF_ID_TYPE_GF			0x410
 #define REG_CRF_ID_TYPE_FM			0x910
 #define REG_CRF_ID_TYPE_WHP			0xA10
+#define REG_CRF_ID_TYPE_PE			0xA30
 
 #define HPM_DEBUG			0xA03440
 #define PERSISTENCE_BIT			BIT(12)
@@ -509,6 +514,14 @@ enum {
 #define WMAL_INDRCT_CMD(addr) \
 	((WMAL_CMD_READ_BURST_ACCESS << WMAL_INDRCT_RD_CMD1_OPMOD_POS) | \
 	 ((addr) & WMAL_INDRCT_RD_CMD1_BYTE_ADDRESS_MSK))
+#define WMAL_MRSPF_STTS 0xADFC24
+#define WMAL_MRSPF_STTS_FIFO1_NOT_EMPTY_POS 15
+#define WMAL_MRSPF_STTS_FIFO1_NOT_EMPTY_MSK 0x8000
+#define WMAL_TIMEOUT_VAL 0xA5A5A5A2
+#define WMAL_MRSPF_STTS_IS_FIFO1_NOT_EMPTY(val) \
+	(((val) >> (WMAL_MRSPF_STTS_FIFO1_NOT_EMPTY_POS)) & \
+	 ((WMAL_MRSPF_STTS_FIFO1_NOT_EMPTY_MSK) >> \
+	  (WMAL_MRSPF_STTS_FIFO1_NOT_EMPTY_POS)))
 
 #define WFPM_LMAC1_PS_CTL_RW 0xA03380
 #define WFPM_LMAC2_PS_CTL_RW 0xA033C0
diff --git a/sys/contrib/dev/iwlwifi/iwl-trans.c b/sys/contrib/dev/iwlwifi/iwl-trans.c
index 3c9d91496c82..3694b41d6621 100644
--- a/sys/contrib/dev/iwlwifi/iwl-trans.c
+++ b/sys/contrib/dev/iwlwifi/iwl-trans.c
@@ -2,10 +2,11 @@
 /*
  * Copyright (C) 2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2019-2021, 2023-2024 Intel Corporation
+ * Copyright (C) 2019-2021, 2023-2025 Intel Corporation
  */
 #include <linux/kernel.h>
 #include <linux/bsearch.h>
+#include <linux/list.h>
 
 #include "fw/api/tx.h"
 #include "iwl-trans.h"
@@ -13,56 +14,283 @@
 #include "iwl-fh.h"
 #include <linux/dmapool.h>
 #include "fw/api/commands.h"
-#include "pcie/internal.h"
-#include "iwl-context-info-gen3.h"
+#include "pcie/gen1_2/internal.h"
+#include "pcie/iwl-context-info-v2.h"
+
+struct iwl_trans_dev_restart_data {
+	struct list_head list;
+	unsigned int restart_count;
+	time64_t last_error;
+	bool backoff;
+	char name[];
+};
+
+static LIST_HEAD(restart_data_list);
+static DEFINE_SPINLOCK(restart_data_lock);
+
+static struct iwl_trans_dev_restart_data *
+iwl_trans_get_restart_data(struct device *dev)
+{
+	struct iwl_trans_dev_restart_data *tmp, *data = NULL;
+	const char *name = dev_name(dev);
+
+	spin_lock(&restart_data_lock);
+	list_for_each_entry(tmp, &restart_data_list, list) {
+		if (strcmp(tmp->name, name))
+			continue;
+		data = tmp;
+		break;
+	}
+	spin_unlock(&restart_data_lock);
+
+	if (data)
+		return data;
+
+	data = kzalloc(struct_size(data, name, strlen(name) + 1), GFP_ATOMIC);
+	if (!data)
+		return NULL;
+
+	strcpy(data->name, name);
+	spin_lock(&restart_data_lock);
+	list_add_tail(&data->list, &restart_data_list);
+	spin_unlock(&restart_data_lock);
+
+	return data;
+}
+
+static void iwl_trans_inc_restart_count(struct device *dev)
+{
+	struct iwl_trans_dev_restart_data *data;
+
+	data = iwl_trans_get_restart_data(dev);
+	if (data) {
+		data->last_error = ktime_get_boottime_seconds();
+		data->restart_count++;
+	}
+}
+
+void iwl_trans_free_restart_list(void)
+{
+	struct iwl_trans_dev_restart_data *tmp;
+
+	while ((tmp = list_first_entry_or_null(&restart_data_list,
+					       typeof(*tmp), list))) {
+		list_del(&tmp->list);
+		kfree(tmp);
+	}
+}
+
+struct iwl_trans_reprobe {
+	struct device *dev;
+	struct delayed_work work;
+};
+
+static void iwl_trans_reprobe_wk(struct work_struct *wk)
+{
+	struct iwl_trans_reprobe *reprobe;
+
+	reprobe = container_of(wk, typeof(*reprobe), work.work);
+
+	if (device_reprobe(reprobe->dev))
+		dev_err(reprobe->dev, "reprobe failed!\n");
+	put_device(reprobe->dev);
+	kfree(reprobe);
+	module_put(THIS_MODULE);
+}
+
+static void iwl_trans_schedule_reprobe(struct iwl_trans *trans,
+				       unsigned int delay_ms)
+{
+	struct iwl_trans_reprobe *reprobe;
+
+	/*
+	 * get a module reference to avoid doing this while unloading
+	 * anyway and to avoid scheduling a work with code that's
+	 * being removed.
+	 */
+	if (!try_module_get(THIS_MODULE)) {
+		IWL_ERR(trans, "Module is being unloaded - abort\n");
+		return;
+	}
+
+	reprobe = kzalloc(sizeof(*reprobe), GFP_KERNEL);
+	if (!reprobe) {
+		module_put(THIS_MODULE);
+		return;
+	}
+	reprobe->dev = get_device(trans->dev);
+	INIT_DELAYED_WORK(&reprobe->work, iwl_trans_reprobe_wk);
+	schedule_delayed_work(&reprobe->work, msecs_to_jiffies(delay_ms));
+}
+
+#define IWL_TRANS_RESET_OK_TIME	7 /* seconds */
+
+static enum iwl_reset_mode
+iwl_trans_determine_restart_mode(struct iwl_trans *trans)
+{
+	struct iwl_trans_dev_restart_data *data;
+	enum iwl_reset_mode at_least = 0;
+	unsigned int index;
+	static const enum iwl_reset_mode escalation_list_old[] = {
+		IWL_RESET_MODE_SW_RESET,
+		IWL_RESET_MODE_REPROBE,
+		IWL_RESET_MODE_REPROBE,
+		IWL_RESET_MODE_FUNC_RESET,
+		IWL_RESET_MODE_PROD_RESET,
+	};
+	static const enum iwl_reset_mode escalation_list_sc[] = {
+		IWL_RESET_MODE_SW_RESET,
+		IWL_RESET_MODE_REPROBE,
+		IWL_RESET_MODE_REPROBE,
+		IWL_RESET_MODE_FUNC_RESET,
+		IWL_RESET_MODE_TOP_RESET,
+		IWL_RESET_MODE_PROD_RESET,
+		IWL_RESET_MODE_TOP_RESET,
+		IWL_RESET_MODE_PROD_RESET,
+		IWL_RESET_MODE_TOP_RESET,
+		IWL_RESET_MODE_PROD_RESET,
+	};
+	const enum iwl_reset_mode *escalation_list;
+	size_t escalation_list_size;
+
+	/* used by TOP fatal error/TOP reset */
+	if (trans->restart.mode.type == IWL_ERR_TYPE_TOP_RESET_FAILED)
+		return IWL_RESET_MODE_PROD_RESET;
+
+	if (trans->request_top_reset) {
+		trans->request_top_reset = 0;
+		if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC)
+			return IWL_RESET_MODE_TOP_RESET;
+		return IWL_RESET_MODE_PROD_RESET;
+	}
+
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC) {
+		escalation_list = escalation_list_sc;
+		escalation_list_size = ARRAY_SIZE(escalation_list_sc);
+	} else {
+		escalation_list = escalation_list_old;
+		escalation_list_size = ARRAY_SIZE(escalation_list_old);
+	}
+
+	if (trans->restart.during_reset)
+		at_least = IWL_RESET_MODE_REPROBE;
+
+	data = iwl_trans_get_restart_data(trans->dev);
+	if (!data)
+		return at_least;
+
+	if (!data->backoff &&
+	    ktime_get_boottime_seconds() - data->last_error >=
+			IWL_TRANS_RESET_OK_TIME)
+		data->restart_count = 0;
+
+	index = data->restart_count;
+	if (index >= escalation_list_size) {
+		index = escalation_list_size - 1;
+		if (!data->backoff) {
+			data->backoff = true;
+			return IWL_RESET_MODE_BACKOFF;
+		}
+		data->backoff = false;
+	}
+
+	return max(at_least, escalation_list[index]);
+}
+
+#define IWL_TRANS_TOP_FOLLOWER_WAIT	180 /* ms */
+
+#define IWL_TRANS_RESET_DELAY	(HZ * 60)
+
+static void iwl_trans_restart_wk(struct work_struct *wk)
+{
+	struct iwl_trans *trans = container_of(wk, typeof(*trans),
+					       restart.wk.work);
+	enum iwl_reset_mode mode;
+
+	if (trans->restart.mode.type == IWL_ERR_TYPE_TOP_RESET_BY_BT) {
+		iwl_trans_schedule_reprobe(trans, IWL_TRANS_TOP_FOLLOWER_WAIT);
+		return;
+	}
+
+	if (!trans->op_mode)
+		return;
+
+	/* might have been scheduled before marked as dead, re-check */
+	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
+		return;
+
+	iwl_op_mode_dump_error(trans->op_mode, &trans->restart.mode);
+
+	/*
+	 * If the opmode stopped the device while we were trying to dump and
+	 * reset, then we'll have done the dump already (synchronized by the
+	 * opmode lock that it will acquire in iwl_op_mode_dump_error()) and
+	 * managed that via trans->restart.mode.
+	 * Additionally, make sure that in such a case we won't attempt to do
+	 * any resets now, since it's no longer requested.
+	 */
+	if (!test_and_clear_bit(STATUS_RESET_PENDING, &trans->status))
+		return;
+
+	if (!iwlwifi_mod_params.fw_restart)
+		return;
+
+	mode = iwl_trans_determine_restart_mode(trans);
+	if (mode == IWL_RESET_MODE_BACKOFF) {
+		IWL_ERR(trans, "Too many device errors - delay next reset\n");
+		queue_delayed_work(system_unbound_wq, &trans->restart.wk,
+				   IWL_TRANS_RESET_DELAY);
+		return;
+	}
+
+	iwl_trans_inc_restart_count(trans->dev);
+
+	switch (mode) {
+	case IWL_RESET_MODE_TOP_RESET:
+		trans->do_top_reset = 1;
+		IWL_ERR(trans, "Device error - TOP reset\n");
+		fallthrough;
+	case IWL_RESET_MODE_SW_RESET:
+		if (mode == IWL_RESET_MODE_SW_RESET)
+			IWL_ERR(trans, "Device error - SW reset\n");
+		iwl_trans_opmode_sw_reset(trans, trans->restart.mode.type);
+		break;
+	case IWL_RESET_MODE_REPROBE:
+		IWL_ERR(trans, "Device error - reprobe!\n");
+
+		iwl_trans_schedule_reprobe(trans, 0);
+		break;
+	default:
+		iwl_trans_pcie_reset(trans, mode);
+		break;
+	}
+}
 
 struct iwl_trans *iwl_trans_alloc(unsigned int priv_size,
 				  struct device *dev,
-				  const struct iwl_cfg_trans_params *cfg_trans)
+				  const struct iwl_mac_cfg *mac_cfg,
+				  unsigned int txcmd_size,
+				  unsigned int txcmd_align)
 {
 	struct iwl_trans *trans;
 #ifdef CONFIG_LOCKDEP
-	static struct lock_class_key __key;
+	static struct lock_class_key __sync_cmd_key;
 #endif
 
 	trans = devm_kzalloc(dev, sizeof(*trans) + priv_size, GFP_KERNEL);
 	if (!trans)
 		return NULL;
 
-	trans->trans_cfg = cfg_trans;
+	trans->mac_cfg = mac_cfg;
 
 #ifdef CONFIG_LOCKDEP
 	lockdep_init_map(&trans->sync_cmd_lockdep_map, "sync_cmd_lockdep_map",
-			 &__key, 0);
+			 &__sync_cmd_key, 0);
 #endif
 
 	trans->dev = dev;
-	trans->num_rx_queues = 1;
 
-	return trans;
-}
-
-int iwl_trans_init(struct iwl_trans *trans)
-{
-	int txcmd_size, txcmd_align;
-
-	if (!trans->trans_cfg->gen2) {
-		txcmd_size = sizeof(struct iwl_tx_cmd);
-		txcmd_align = sizeof(void *);
-	} else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
-		txcmd_size = sizeof(struct iwl_tx_cmd_gen2);
-		txcmd_align = 64;
-	} else {
-		txcmd_size = sizeof(struct iwl_tx_cmd_gen3);
-		txcmd_align = 128;
-	}
-
-	txcmd_size += sizeof(struct iwl_cmd_header);
-	txcmd_size += 36; /* biggest possible 802.11 header */
-
-	/* Ensure device TX cmd cannot reach/cross a page boundary in gen2 */
-	if (WARN_ON(trans->trans_cfg->gen2 && txcmd_size >= txcmd_align))
-		return -EINVAL;
+	INIT_DELAYED_WORK(&trans->restart.wk, iwl_trans_restart_wk);
 
 	snprintf(trans->dev_cmd_pool_name, sizeof(trans->dev_cmd_pool_name),
 		 "iwl_cmd_pool:%s", dev_name(trans->dev));
@@ -71,16 +299,14 @@ int iwl_trans_init(struct iwl_trans *trans)
 				  txcmd_size, txcmd_align,
 				  SLAB_HWCACHE_ALIGN, NULL);
 	if (!trans->dev_cmd_pool)
-		return -ENOMEM;
-
-	/* Initialize the wait queue for commands */
-	init_waitqueue_head(&trans->wait_command_queue);
+		return NULL;
 
-	return 0;
+	return trans;
 }
 
 void iwl_trans_free(struct iwl_trans *trans)
 {
+	cancel_delayed_work_sync(&trans->restart.wk);
 	kmem_cache_destroy(trans->dev_cmd_pool);
 }
 
@@ -92,17 +318,7 @@ int iwl_trans_send_cmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd)
 		     test_bit(STATUS_RFKILL_OPMODE, &trans->status)))
 		return -ERFKILL;
 
-	/*
-	 * We can't test IWL_MVM_STATUS_IN_D3 in mvm->status because this
-	 * bit is set early in the D3 flow, before we send all the commands
-	 * that configure the firmware for D3 operation (power, patterns, ...)
-	 * and we don't want to flag all those with CMD_SEND_IN_D3.
-	 * So use the system_pm_mode instead. The only command sent after
-	 * we set system_pm_mode is D3_CONFIG_CMD, which we now flag with
-	 * CMD_SEND_IN_D3.
-	 */
-	if (unlikely(trans->system_pm_mode == IWL_PLAT_PM_MODE_D3 &&
-		     !(cmd->flags & CMD_SEND_IN_D3)))
+	if (unlikely(test_bit(STATUS_SUSPENDED, &trans->status)))
 		return -EHOSTDOWN;
 
 	if (unlikely(test_bit(STATUS_FW_ERROR, &trans->status)))
@@ -115,7 +331,7 @@ int iwl_trans_send_cmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd)
 	if (!(cmd->flags & CMD_ASYNC))
 		lock_map_acquire_read(&trans->sync_cmd_lockdep_map);
 
-	if (trans->wide_cmd_header && !iwl_cmd_groupid(cmd->id)) {
+	if (trans->conf.wide_cmd_header && !iwl_cmd_groupid(cmd->id)) {
 		if (cmd->id != REPLY_ERROR)
 			cmd->id = DEF_ID(cmd->id);
 	}
@@ -159,11 +375,12 @@ const char *iwl_get_cmd_string(struct iwl_trans *trans, u32 id)
 	grp = iwl_cmd_groupid(id);
 	cmd = iwl_cmd_opcode(id);
 
-	if (!trans->command_groups || grp >= trans->command_groups_size ||
-	    !trans->command_groups[grp].arr)
+	if (!trans->conf.command_groups ||
+	    grp >= trans->conf.command_groups_size ||
+	    !trans->conf.command_groups[grp].arr)
 		return "UNKNOWN";
 
-	arr = &trans->command_groups[grp];
+	arr = &trans->conf.command_groups[grp];
 	ret = bsearch(&cmd, arr->arr, arr->size, size, iwl_hcmd_names_cmp);
 	if (!ret)
 		return "UNKNOWN";
@@ -171,37 +388,29 @@ const char *iwl_get_cmd_string(struct iwl_trans *trans, u32 id)
 }
 IWL_EXPORT_SYMBOL(iwl_get_cmd_string);
 
-int iwl_cmd_groups_verify_sorted(const struct iwl_trans_config *trans)
+void iwl_trans_op_mode_enter(struct iwl_trans *trans,
+			     struct iwl_op_mode *op_mode)
 {
-	int i, j;
-	const struct iwl_hcmd_arr *arr;
+	trans->op_mode = op_mode;
 
-	for (i = 0; i < trans->command_groups_size; i++) {
-		arr = &trans->command_groups[i];
-		if (!arr->arr)
-			continue;
-		for (j = 0; j < arr->size - 1; j++)
-			if (arr->arr[j].cmd_id > arr->arr[j + 1].cmd_id)
-				return -1;
-	}
-	return 0;
-}
-IWL_EXPORT_SYMBOL(iwl_cmd_groups_verify_sorted);
+	if (WARN_ON(trans->conf.n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
+		trans->conf.n_no_reclaim_cmds =
+			ARRAY_SIZE(trans->conf.no_reclaim_cmds);
 
-void iwl_trans_configure(struct iwl_trans *trans,
-			 const struct iwl_trans_config *trans_cfg)
-{
-	trans->op_mode = trans_cfg->op_mode;
+	WARN_ON_ONCE(!trans->conf.rx_mpdu_cmd);
 
-	iwl_trans_pcie_configure(trans, trans_cfg);
-	WARN_ON(iwl_cmd_groups_verify_sorted(trans_cfg));
+	iwl_trans_pcie_op_mode_enter(trans);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_configure);
+IWL_EXPORT_SYMBOL(iwl_trans_op_mode_enter);
 
 int iwl_trans_start_hw(struct iwl_trans *trans)
 {
 	might_sleep();
 
+	clear_bit(STATUS_TRANS_RESET_IN_PROGRESS, &trans->status);
+	/* opmode may not resume if it detects errors */
+	clear_bit(STATUS_SUSPENDED, &trans->status);
+
 	return iwl_trans_pcie_start_hw(trans);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_start_hw);
@@ -210,9 +419,15 @@ void iwl_trans_op_mode_leave(struct iwl_trans *trans)
 {
 	might_sleep();
 
-	iwl_trans_pcie_op_mode_leave(trans);
+	if (trans->mac_cfg->gen2)
+		iwl_trans_pcie_gen2_op_mode_leave(trans);
+	else
+		iwl_trans_pcie_op_mode_leave(trans);
+
+	cancel_delayed_work_sync(&trans->restart.wk);
 
 	trans->op_mode = NULL;
+	memset(&trans->conf, 0, sizeof(trans->conf));
 
 	trans->state = IWL_TRANS_NO_FW;
 }
@@ -222,31 +437,26 @@ void iwl_trans_write8(struct iwl_trans *trans, u32 ofs, u8 val)
 {
 	iwl_trans_pcie_write8(trans, ofs, val);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_write8);
 
 void iwl_trans_write32(struct iwl_trans *trans, u32 ofs, u32 val)
 {
 	iwl_trans_pcie_write32(trans, ofs, val);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_write32);
 
 u32 iwl_trans_read32(struct iwl_trans *trans, u32 ofs)
 {
 	return iwl_trans_pcie_read32(trans, ofs);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_read32);
 
 u32 iwl_trans_read_prph(struct iwl_trans *trans, u32 ofs)
 {
 	return iwl_trans_pcie_read_prph(trans, ofs);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_read_prph);
 
 void iwl_trans_write_prph(struct iwl_trans *trans, u32 ofs, u32 val)
 {
 	return iwl_trans_pcie_write_prph(trans, ofs, val);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_write_prph);
 
 int iwl_trans_read_mem(struct iwl_trans *trans, u32 addr,
 		       void *buf, int dwords)
@@ -258,7 +468,19 @@ IWL_EXPORT_SYMBOL(iwl_trans_read_mem);
 int iwl_trans_write_mem(struct iwl_trans *trans, u32 addr,
 			const void *buf, int dwords)
 {
-	return iwl_trans_pcie_write_mem(trans, addr, buf, dwords);
+	int offs, ret = 0;
+	const u32 *vals = buf;
+
+	if (iwl_trans_grab_nic_access(trans)) {
+		iwl_write32(trans, HBUS_TARG_MEM_WADDR, addr);
+		for (offs = 0; offs < dwords; offs++)
+			iwl_write32(trans, HBUS_TARG_MEM_WDAT,
+				    vals ? vals[offs] : 0);
+		iwl_trans_release_nic_access(trans);
+	} else {
+		ret = -EBUSY;
+	}
+	return ret;
 }
 IWL_EXPORT_SYMBOL(iwl_trans_write_mem);
 
@@ -271,11 +493,10 @@ void iwl_trans_set_pmi(struct iwl_trans *trans, bool state)
 }
 IWL_EXPORT_SYMBOL(iwl_trans_set_pmi);
 
-int iwl_trans_sw_reset(struct iwl_trans *trans, bool retake_ownership)
+int iwl_trans_sw_reset(struct iwl_trans *trans)
 {
-	return iwl_trans_pcie_sw_reset(trans, retake_ownership);
+	return iwl_trans_pcie_sw_reset(trans, true);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_sw_reset);
 
 struct iwl_trans_dump_data *
 iwl_trans_dump_data(struct iwl_trans *trans, u32 dump_mask,
@@ -285,22 +506,34 @@ iwl_trans_dump_data(struct iwl_trans *trans, u32 dump_mask,
 	return iwl_trans_pcie_dump_data(trans, dump_mask,
 					sanitize_ops, sanitize_ctx);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_dump_data);
 
 int iwl_trans_d3_suspend(struct iwl_trans *trans, bool test, bool reset)
 {
+	int err;
+
 	might_sleep();
 
-	return iwl_trans_pcie_d3_suspend(trans, test, reset);
+	err = iwl_trans_pcie_d3_suspend(trans, test, reset);
+
+	if (!err)
+		set_bit(STATUS_SUSPENDED, &trans->status);
+
+	return err;
 }
 IWL_EXPORT_SYMBOL(iwl_trans_d3_suspend);
 
 int iwl_trans_d3_resume(struct iwl_trans *trans, enum iwl_d3_status *status,
 			bool test, bool reset)
 {
+	int err;
+
 	might_sleep();
 
-	return iwl_trans_pcie_d3_resume(trans, status, test, reset);
+	err = iwl_trans_pcie_d3_resume(trans, status, test, reset);
+
+	clear_bit(STATUS_SUSPENDED, &trans->status);
+
+	return err;
 }
 IWL_EXPORT_SYMBOL(iwl_trans_d3_resume);
 
@@ -308,20 +541,17 @@ void iwl_trans_interrupts(struct iwl_trans *trans, bool enable)
 {
 	iwl_trans_pci_interrupts(trans, enable);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_interrupts);
 
 void iwl_trans_sync_nmi(struct iwl_trans *trans)
 {
 	iwl_trans_pcie_sync_nmi(trans);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_sync_nmi);
 
 int iwl_trans_write_imr_mem(struct iwl_trans *trans, u32 dst_addr,
 			    u64 src_addr, u32 byte_cnt)
 {
 	return iwl_trans_pcie_copy_imr(trans, dst_addr, src_addr, byte_cnt);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_write_imr_mem);
 
 void iwl_trans_set_bits_mask(struct iwl_trans *trans, u32 reg,
 			     u32 mask, u32 value)
@@ -335,7 +565,6 @@ int iwl_trans_read_config32(struct iwl_trans *trans, u32 ofs,
 {
 	return iwl_trans_pcie_read_config32(trans, ofs, val);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_read_config32);
 
 bool _iwl_trans_grab_nic_access(struct iwl_trans *trans)
 {
@@ -347,38 +576,42 @@ void __releases(nic_access)
 iwl_trans_release_nic_access(struct iwl_trans *trans)
 {
 	iwl_trans_pcie_release_nic_access(trans);
-	__release(nic_access);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_release_nic_access);
 
-void iwl_trans_fw_alive(struct iwl_trans *trans, u32 scd_addr)
+void iwl_trans_fw_alive(struct iwl_trans *trans)
 {
 	might_sleep();
 
 	trans->state = IWL_TRANS_FW_ALIVE;
 
-	if (trans->trans_cfg->gen2)
+	if (trans->mac_cfg->gen2)
 		iwl_trans_pcie_gen2_fw_alive(trans);
 	else
-		iwl_trans_pcie_fw_alive(trans, scd_addr);
+		iwl_trans_pcie_fw_alive(trans);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_fw_alive);
 
-int iwl_trans_start_fw(struct iwl_trans *trans, const struct fw_img *fw,
-		       bool run_in_rfkill)
+int iwl_trans_start_fw(struct iwl_trans *trans, const struct iwl_fw *fw,
+		       enum iwl_ucode_type ucode_type, bool run_in_rfkill)
 {
+	const struct fw_img *img;
 	int ret;
 
 	might_sleep();
 
-	WARN_ON_ONCE(!trans->rx_mpdu_cmd);
+	img = iwl_get_ucode_image(fw, ucode_type);
+	if (!img)
+		return -EINVAL;
 
 	clear_bit(STATUS_FW_ERROR, &trans->status);
 
-	if (trans->trans_cfg->gen2)
-		ret = iwl_trans_pcie_gen2_start_fw(trans, fw, run_in_rfkill);
+	if (trans->mac_cfg->gen2)
+		ret = iwl_trans_pcie_gen2_start_fw(trans, fw, img,
+						   run_in_rfkill);
 	else
-		ret = iwl_trans_pcie_start_fw(trans, fw, run_in_rfkill);
+		ret = iwl_trans_pcie_start_fw(trans, fw, img,
+					      run_in_rfkill);
 
 	if (ret == 0)
 		trans->state = IWL_TRANS_FW_STARTED;
@@ -391,7 +624,35 @@ void iwl_trans_stop_device(struct iwl_trans *trans)
 {
 	might_sleep();
 
-	if (trans->trans_cfg->gen2)
+	/*
+	 * See also the comment in iwl_trans_restart_wk().
+	 *
+	 * When the opmode stops the device while a reset is pending, the
+	 * worker (iwl_trans_restart_wk) might not have run yet or, more
+	 * likely, will be blocked on the opmode lock. Due to the locking,
+	 * we can't just flush the worker.
+	 *
+	 * If this is the case, then the test_and_clear_bit() ensures that
+	 * the worker won't attempt to do anything after the stop.
+	 *
+	 * The trans->restart.mode is a handshake with the opmode, we set
+	 * the context there to ABORT so that when the worker can finally
+	 * acquire the lock in the opmode, the code there won't attempt to
+	 * do any dumps. Since we'd really like to have the dump though,
+	 * also do it inline here (with the opmode locks already held),
+	 * but use a separate mode struct to avoid races.
+	 */
+	if (test_and_clear_bit(STATUS_RESET_PENDING, &trans->status)) {
+		struct iwl_fw_error_dump_mode mode;
+
+		mode = trans->restart.mode;
+		mode.context = IWL_ERR_CONTEXT_FROM_OPMODE;
+		trans->restart.mode.context = IWL_ERR_CONTEXT_ABORT;
+
+		iwl_op_mode_dump_error(trans->op_mode, &mode);
+	}
+
+	if (trans->mac_cfg->gen2)
 		iwl_trans_pcie_gen2_stop_device(trans);
 	else
 		iwl_trans_pcie_stop_device(trans);
@@ -410,7 +671,7 @@ int iwl_trans_tx(struct iwl_trans *trans, struct sk_buff *skb,
 		      "bad state = %d\n", trans->state))
 		return -EIO;
 
-	if (trans->trans_cfg->gen2)
+	if (trans->mac_cfg->gen2)
 		return iwl_txq_gen2_tx(trans, skb, dev_cmd, queue);
 
 	return iwl_trans_pcie_tx(trans, skb, dev_cmd, queue);
@@ -420,6 +681,9 @@ IWL_EXPORT_SYMBOL(iwl_trans_tx);
 void iwl_trans_reclaim(struct iwl_trans *trans, int queue, int ssn,
 		       struct sk_buff_head *skbs, bool is_flush)
 {
+	if (unlikely(test_bit(STATUS_FW_ERROR, &trans->status)))
+		return;
+
 	if (WARN_ONCE(trans->state != IWL_TRANS_FW_ALIVE,
 		      "bad state = %d\n", trans->state))
 		return;
@@ -452,6 +716,9 @@ IWL_EXPORT_SYMBOL(iwl_trans_txq_enable_cfg);
 
 int iwl_trans_wait_txq_empty(struct iwl_trans *trans, int queue)
 {
+	if (unlikely(test_bit(STATUS_FW_ERROR, &trans->status)))
+		return -EIO;
+
 	if (WARN_ONCE(trans->state != IWL_TRANS_FW_ALIVE,
 		      "bad state = %d\n", trans->state))
 		return -EIO;
@@ -493,7 +760,6 @@ void iwl_trans_debugfs_cleanup(struct iwl_trans *trans)
 {
 	iwl_trans_pcie_debugfs_cleanup(trans);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_debugfs_cleanup);
 #endif
 
 void iwl_trans_set_q_ptrs(struct iwl_trans *trans, int queue, int ptr)
@@ -531,35 +797,31 @@ int iwl_trans_get_rxq_dma_data(struct iwl_trans *trans, int queue,
 {
 	return iwl_trans_pcie_rxq_dma_data(trans, queue, data);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_get_rxq_dma_data);
 
 int iwl_trans_load_pnvm(struct iwl_trans *trans,
 			const struct iwl_pnvm_image *pnvm_data,
 			const struct iwl_ucode_capabilities *capa)
 {
-	return iwl_trans_pcie_ctx_info_gen3_load_pnvm(trans, pnvm_data, capa);
+	return iwl_trans_pcie_ctx_info_v2_load_pnvm(trans, pnvm_data, capa);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_load_pnvm);
 
 void iwl_trans_set_pnvm(struct iwl_trans *trans,
 			const struct iwl_ucode_capabilities *capa)
 {
-	iwl_trans_pcie_ctx_info_gen3_set_pnvm(trans, capa);
+	iwl_trans_pcie_ctx_info_v2_set_pnvm(trans, capa);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_set_pnvm);
 
 int iwl_trans_load_reduce_power(struct iwl_trans *trans,
 				const struct iwl_pnvm_image *payloads,
 				const struct iwl_ucode_capabilities *capa)
 {
-	return iwl_trans_pcie_ctx_info_gen3_load_reduce_power(trans, payloads,
+	return iwl_trans_pcie_ctx_info_v2_load_reduce_power(trans, payloads,
 							      capa);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_load_reduce_power);
 
 void iwl_trans_set_reduce_power(struct iwl_trans *trans,
 				const struct iwl_ucode_capabilities *capa)
 {
-	iwl_trans_pcie_ctx_info_gen3_set_reduce_power(trans, capa);
+	iwl_trans_pcie_ctx_info_v2_set_reduce_power(trans, capa);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_set_reduce_power);
diff --git a/sys/contrib/dev/iwlwifi/iwl-trans.h b/sys/contrib/dev/iwlwifi/iwl-trans.h
index 7430d1b37541..557e601c0500 100644
--- a/sys/contrib/dev/iwlwifi/iwl-trans.h
+++ b/sys/contrib/dev/iwlwifi/iwl-trans.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2023 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -113,16 +113,12 @@ static inline u32 iwl_rx_packet_payload_len(const struct iwl_rx_packet *pkt)
  *	the response. The caller needs to call iwl_free_resp when done.
  * @CMD_SEND_IN_RFKILL: Send the command even if the NIC is in RF-kill.
  * @CMD_BLOCK_TXQS: Block TXQs while the comment is executing.
- * @CMD_SEND_IN_D3: Allow the command to be sent in D3 mode, relevant to
- *	SUSPEND and RESUME commands. We are in D3 mode when we set
- *	trans->system_pm_mode to IWL_PLAT_PM_MODE_D3.
  */
 enum CMD_MODE {
 	CMD_ASYNC		= BIT(0),
 	CMD_WANT_SKB		= BIT(1),
 	CMD_SEND_IN_RFKILL	= BIT(2),
 	CMD_BLOCK_TXQS		= BIT(3),
-	CMD_SEND_IN_D3          = BIT(4),
 };
 #define CMD_MODE_BITS 5
 
@@ -313,6 +309,14 @@ enum iwl_d3_status {
  * @STATUS_TRANS_DEAD: trans is dead - avoid any read/write operation
  * @STATUS_SUPPRESS_CMD_ERROR_ONCE: suppress "FW error in SYNC CMD" once,
  *	e.g. for testing
+ * @STATUS_IN_SW_RESET: device is undergoing reset, cleared by opmode
+ *	via iwl_trans_finish_sw_reset()
+ * @STATUS_RESET_PENDING: reset worker was scheduled, but didn't dump
+ *	the firmware state yet
+ * @STATUS_TRANS_RESET_IN_PROGRESS: reset is still in progress, don't
+ *	attempt another reset yet
+ * @STATUS_SUSPENDED: device is suspended, don't send commands that
+ *	aren't marked accordingly
  */
 enum iwl_trans_status {
 	STATUS_SYNC_HCMD_ACTIVE,
@@ -324,6 +328,10 @@ enum iwl_trans_status {
 	STATUS_FW_ERROR,
 	STATUS_TRANS_DEAD,
 	STATUS_SUPPRESS_CMD_ERROR_ONCE,
+	STATUS_IN_SW_RESET,
+	STATUS_RESET_PENDING,
+	STATUS_TRANS_RESET_IN_PROGRESS,
+	STATUS_SUSPENDED,
 };
 
 static inline int
@@ -395,11 +403,11 @@ struct iwl_dump_sanitize_ops {
 /**
  * struct iwl_trans_config - transport configuration
  *
- * @op_mode: pointer to the upper layer.
+ * These values should be set before iwl_trans_op_mode_enter().
+ *
  * @cmd_queue: the index of the command queue.
  *	Must be set before start_fw.
  * @cmd_fifo: the fifo for host commands
- * @cmd_q_wdg_timeout: the timeout of the watchdog timer for the command queue.
  * @no_reclaim_cmds: Some devices erroneously don't set the
  *	SEQ_RX_FRAME bit on some notifications, this is the
  *	list of such notifications to filter. Max length is
@@ -407,8 +415,6 @@ struct iwl_dump_sanitize_ops {
  * @n_no_reclaim_cmds: # of commands in list
  * @rx_buf_size: RX buffer size needed for A-MSDUs
  *	if unset 4k will be the RX buffer size
- * @bc_table_dword: set to true if the BC table expects the byte count to be
- *	in DWORD (as opposed to bytes)
  * @scd_set_active: should the transport configure the SCD for HCMD queue
  * @command_groups: array of command groups, each member is an array of the
  *	commands in the group; for debugging only
@@ -419,18 +425,24 @@ struct iwl_dump_sanitize_ops {
  * @queue_alloc_cmd_ver: queue allocation command version, set to 0
  *	for using the older SCD_QUEUE_CFG, set to the version of
  *	SCD_QUEUE_CONFIG_CMD otherwise.
+ * @wide_cmd_header: true when ucode supports wide command header format
+ * @rx_mpdu_cmd: MPDU RX command ID, must be assigned by opmode before
+ *	starting the firmware, used for tracing
+ * @rx_mpdu_cmd_hdr_size: used for tracing, amount of data before the
+ *	start of the 802.11 header in the @rx_mpdu_cmd
+ * @dsbr_urm_fw_dependent: switch to URM based on fw settings
+ * @dsbr_urm_permanent: switch to URM permanently
+ * @mbx_addr_0_step: step address data 0
+ * @mbx_addr_1_step: step address data 1
+ * @ext_32khz_clock_valid: if true, the external 32 KHz clock can be used
  */
 struct iwl_trans_config {
-	struct iwl_op_mode *op_mode;
-
 	u8 cmd_queue;
 	u8 cmd_fifo;
-	unsigned int cmd_q_wdg_timeout;
-	const u8 *no_reclaim_cmds;
-	unsigned int n_no_reclaim_cmds;
+	u8 n_no_reclaim_cmds;
+	u8 no_reclaim_cmds[MAX_NO_RECLAIM_CMDS];
 
 	enum iwl_amsdu_size rx_buf_size;
-	bool bc_table_dword;
 	bool scd_set_active;
 	const struct iwl_hcmd_arr *command_groups;
 	int command_groups_size;
@@ -438,6 +450,16 @@ struct iwl_trans_config {
 	u8 cb_data_offs;
 	bool fw_reset_handshake;
 	u8 queue_alloc_cmd_ver;
+
+	bool wide_cmd_header;
+	u8 rx_mpdu_cmd, rx_mpdu_cmd_hdr_size;
+
+	u8 dsbr_urm_fw_dependent:1,
+	   dsbr_urm_permanent:1,
+	   ext_32khz_clock_valid:1;
+
+	u32 mbx_addr_0_step;
+	u32 mbx_addr_1_step;
 };
 
 struct iwl_trans_dump_data {
@@ -523,23 +545,6 @@ enum iwl_trans_state {
  */
 
 /**
- * enum iwl_plat_pm_mode - platform power management mode
- *
- * This enumeration describes the device's platform power management
- * behavior when in system-wide suspend (i.e WoWLAN).
- *
- * @IWL_PLAT_PM_MODE_DISABLED: power management is disabled for this
- *	device.  In system-wide suspend mode, it means that the all
- *	connections will be closed automatically by mac80211 before
- *	the platform is suspended.
- * @IWL_PLAT_PM_MODE_D3: the device goes into D3 mode (i.e. WoWLAN).
- */
-enum iwl_plat_pm_mode {
-	IWL_PLAT_PM_MODE_DISABLED,
-	IWL_PLAT_PM_MODE_D3,
-};
-
-/**
  * enum iwl_ini_cfg_state
  * @IWL_INI_CFG_STATE_NOT_LOADED: no debug cfg was given
  * @IWL_INI_CFG_STATE_LOADED: debug cfg was found and loaded
@@ -643,8 +648,6 @@ struct iwl_pc_data {
  * @n_dest_reg: num of reg_ops in %dbg_dest_tlv
  * @rec_on: true iff there is a fw debug recording currently active
  * @dest_tlv: points to the destination TLV for debug
- * @conf_tlv: array of pointers to configuration TLVs for debug
- * @trigger_tlv: array of pointers to triggers TLVs for debug
  * @lmac_error_event_table: addrs of lmacs error tables
  * @umac_error_event_table: addr of umac error table
  * @tcm_error_event_table: address(es) of TCM error table(s)
@@ -679,8 +682,6 @@ struct iwl_trans_debug {
 	bool rec_on;
 
 	const struct iwl_fw_dbg_dest_tlv_v1 *dest_tlv;
-	const struct iwl_fw_dbg_conf_tlv *conf_tlv[FW_DBG_CONF_MAX];
-	struct iwl_fw_dbg_trigger_tlv * const *trigger_tlv;
 
 	u32 lmac_error_event_table[2];
 	u32 umac_error_event_table;
@@ -833,91 +834,90 @@ struct iwl_txq {
 };
 
 /**
+ * struct iwl_trans_info - transport info for outside use
+ * @name: the device name
+ * @max_skb_frags: maximum number of fragments an SKB can have when transmitted.
+ *	0 indicates that frag SKBs (NETIF_F_SG) aren't supported.
+ * @hw_rev: the revision data of the HW
+ * @hw_rev_step: The mac step of the HW
+ * @hw_rf_id: the device RF ID
+ * @hw_cnv_id: the device CNV ID
+ * @hw_crf_id: the device CRF ID
+ * @hw_wfpm_id: the device wfpm ID
+ * @hw_id: the ID of the device / sub-device
+ *	Bits 0:15 represent the sub-device ID
+ *	Bits 16:31 represent the device ID.
+ * @pcie_link_speed: current PCIe link speed (%PCI_EXP_LNKSTA_CLS_*),
+ *	only valid for discrete (not integrated) NICs
+ * @num_rxqs: number of RX queues allocated by the transport
+ */
+struct iwl_trans_info {
+	const char *name;
+	u32 max_skb_frags;
+	u32 hw_rev;
+	u32 hw_rev_step;
+	u32 hw_rf_id;
+	u32 hw_crf_id;
+	u32 hw_cnv_id;
+	u32 hw_wfpm_id;
+	u32 hw_id;
+	u8 pcie_link_speed;
+	u8 num_rxqs;
+};
+
+/**
  * struct iwl_trans - transport common data
  *
  * @csme_own: true if we couldn't get ownership on the device
  * @op_mode: pointer to the op_mode
- * @trans_cfg: the trans-specific configuration part
+ * @mac_cfg: the trans-specific configuration part
  * @cfg: pointer to the configuration
  * @drv: pointer to iwl_drv
+ * @conf: configuration set by the opmode before enter
  * @state: current device state
  * @status: a bit-mask of transport status flags
  * @dev: pointer to struct device * that represents the device
- * @max_skb_frags: maximum number of fragments an SKB can have when transmitted.
- *	0 indicates that frag SKBs (NETIF_F_SG) aren't supported.
- * @hw_rf_id: a u32 with the device RF ID
- * @hw_cnv_id: a u32 with the device CNV ID
- * @hw_crf_id: a u32 with the device CRF ID
- * @hw_wfpm_id: a u32 with the device wfpm ID
- * @hw_id: a u32 with the ID of the device / sub-device.
- *	Set during transport allocation.
- * @hw_id_str: a string with info about HW ID. Set during transport allocation.
- * @sku_id: the SKU identifier (for PNVM matching)
+ * @info: device information for use by other layers
  * @pnvm_loaded: indicates PNVM was loaded
- * @hw_rev: the revision data of the HW
- * @hw_rev_step: The mac step of the HW
  * @pm_support: set to true in start_hw if link pm is supported
  * @ltr_enabled: set to true if the LTR is enabled
  * @fail_to_parse_pnvm_image: set to true if pnvm parsing failed
  * @reduce_power_loaded: indicates reduced power section was loaded
  * @failed_to_load_reduce_power_image: set to true if pnvm loading failed
- * @command_groups: pointer to command group name list array
- * @command_groups_size: array size of @command_groups
- * @wide_cmd_header: true when ucode supports wide command header format
- * @wait_command_queue: wait queue for sync commands
- * @num_rx_queues: number of RX queues allocated by the transport;
- *	the transport must set this before calling iwl_drv_start()
- * @iml_len: the length of the image loader
- * @iml: a pointer to the image loader itself
  * @dev_cmd_pool: pool for Tx cmd allocation - for internal use only.
  *	The user should use iwl_trans_{alloc,free}_tx_cmd.
  * @dev_cmd_pool_name: name for the TX command allocation pool
  * @dbgfs_dir: iwlwifi debugfs base dir for this device
  * @sync_cmd_lockdep_map: lockdep map for checking sync commands
- * @rx_mpdu_cmd: MPDU RX command ID, must be assigned by opmode before
- *	starting the firmware, used for tracing
- * @rx_mpdu_cmd_hdr_size: used for tracing, amount of data before the
- *	start of the 802.11 header in the @rx_mpdu_cmd
  * @dbg: additional debug data, see &struct iwl_trans_debug
  * @init_dram: FW initialization DMA data
- * @system_pm_mode: the system-wide power management mode in use.
- *	This mode is set dynamically, depending on the WoWLAN values
- *	configured from the userspace at runtime.
- * @name: the device name
- * @mbx_addr_0_step: step address data 0
- * @mbx_addr_1_step: step address data 1
- * @pcie_link_speed: current PCIe link speed (%PCI_EXP_LNKSTA_CLS_*),
- *	only valid for discrete (not integrated) NICs
- * @invalid_tx_cmd: invalid TX command buffer
  * @reduced_cap_sku: reduced capability supported SKU
- * @no_160: device not supporting 160 MHz
  * @step_urm: STEP is in URM, no support for MCS>9 in 320 MHz
+ * @restart: restart worker data
+ * @restart.wk: restart worker
+ * @restart.mode: reset/restart error mode information
+ * @restart.during_reset: error occurred during previous software reset
  * @trans_specific: data for the specific transport this is allocated for/with
+ * @request_top_reset: TOP reset was requested, used by the reset
+ *	worker that should be scheduled (with appropriate reason)
+ * @do_top_reset: indication to the (PCIe) transport/context-info
+ *	to do the TOP reset
  */
 struct iwl_trans {
 	bool csme_own;
 	struct iwl_op_mode *op_mode;
-	const struct iwl_cfg_trans_params *trans_cfg;
-	const struct iwl_cfg *cfg;
+	const struct iwl_mac_cfg *mac_cfg;
+	const struct iwl_rf_cfg *cfg;
 	struct iwl_drv *drv;
+	struct iwl_trans_config conf;
 	enum iwl_trans_state state;
 	unsigned long status;
 
 	struct device *dev;
-	u32 max_skb_frags;
-	u32 hw_rev;
-	u32 hw_rev_step;
-	u32 hw_rf_id;
-	u32 hw_crf_id;
-	u32 hw_cnv_id;
-	u32 hw_wfpm_id;
-	u32 hw_id;
-	char hw_id_str[52];
-	u32 sku_id[3];
-	bool reduced_cap_sku;
-	u8 no_160:1, step_urm:1;
 
-	u8 rx_mpdu_cmd, rx_mpdu_cmd_hdr_size;
+	const struct iwl_trans_info info;
+	bool reduced_cap_sku;
+	bool step_urm;
 
 	bool pm_support;
 	bool ltr_enabled;
@@ -926,16 +926,6 @@ struct iwl_trans {
 	u8 reduce_power_loaded:1;
 	u8 failed_to_load_reduce_power_image:1;
 
-	const struct iwl_hcmd_arr *command_groups;
-	int command_groups_size;
-	bool wide_cmd_header;
-
-	wait_queue_head_t wait_command_queue;
-	u8 num_rx_queues;
-
-	size_t iml_len;
-	u8 *iml;
-
 	/* The following fields are internal only */
 	struct kmem_cache *dev_cmd_pool;
 	char dev_cmd_pool_name[50];
@@ -949,15 +939,14 @@ struct iwl_trans {
 	struct iwl_trans_debug dbg;
 	struct iwl_self_init_dram init_dram;
 
-	enum iwl_plat_pm_mode system_pm_mode;
-
-	const char *name;
-	u32 mbx_addr_0_step;
-	u32 mbx_addr_1_step;
-
-	u8 pcie_link_speed;
+	struct {
+		struct delayed_work wk;
+		struct iwl_fw_error_dump_mode mode;
+		bool during_reset;
+	} restart;
 
-	struct iwl_dma_ptr invalid_tx_cmd;
+	u8 request_top_reset:1,
+	   do_top_reset:1;
 
 	/* pointer to trans specific struct */
 	/*Ensure that this pointer will always be aligned to sizeof pointer */
@@ -965,19 +954,18 @@ struct iwl_trans {
 };
 
 const char *iwl_get_cmd_string(struct iwl_trans *trans, u32 id);
-int iwl_cmd_groups_verify_sorted(const struct iwl_trans_config *trans);
 
-void iwl_trans_configure(struct iwl_trans *trans,
-			 const struct iwl_trans_config *trans_cfg);
+void iwl_trans_op_mode_enter(struct iwl_trans *trans,
+			     struct iwl_op_mode *op_mode);
 
 int iwl_trans_start_hw(struct iwl_trans *trans);
 
 void iwl_trans_op_mode_leave(struct iwl_trans *trans);
 
-void iwl_trans_fw_alive(struct iwl_trans *trans, u32 scd_addr);
+void iwl_trans_fw_alive(struct iwl_trans *trans);
 
-int iwl_trans_start_fw(struct iwl_trans *trans, const struct fw_img *fw,
-		       bool run_in_rfkill);
+int iwl_trans_start_fw(struct iwl_trans *trans, const struct iwl_fw *fw,
+		       enum iwl_ucode_type ucode_type, bool run_in_rfkill);
 
 void iwl_trans_stop_device(struct iwl_trans *trans);
 
@@ -1089,12 +1077,13 @@ int iwl_trans_read_config32(struct iwl_trans *trans, u32 ofs,
 void iwl_trans_debugfs_cleanup(struct iwl_trans *trans);
 #endif
 
-#define iwl_trans_read_mem_bytes(trans, addr, buf, bufsize)		      \
-	do {								      \
-		if (__builtin_constant_p(bufsize))			      \
-			BUILD_BUG_ON((bufsize) % sizeof(u32));		      \
-		iwl_trans_read_mem(trans, addr, buf, (bufsize) / sizeof(u32));\
-	} while (0)
+#define iwl_trans_read_mem_bytes(trans, addr, buf, bufsize)	\
+	({							\
+		if (__builtin_constant_p(bufsize))		\
+			BUILD_BUG_ON((bufsize) % sizeof(u32));	\
+		iwl_trans_read_mem(trans, addr, buf,		\
+				   (bufsize) / sizeof(u32));	\
+	})
 
 int iwl_trans_write_imr_mem(struct iwl_trans *trans, u32 dst_addr,
 			    u64 src_addr, u32 byte_cnt);
@@ -1120,7 +1109,7 @@ static inline u32 iwl_trans_write_mem32(struct iwl_trans *trans, u32 addr,
 
 void iwl_trans_set_pmi(struct iwl_trans *trans, bool state);
 
-int iwl_trans_sw_reset(struct iwl_trans *trans, bool retake_ownership);
+int iwl_trans_sw_reset(struct iwl_trans *trans);
 
 void iwl_trans_set_bits_mask(struct iwl_trans *trans, u32 reg,
 			     u32 mask, u32 value);
@@ -1134,7 +1123,31 @@ bool _iwl_trans_grab_nic_access(struct iwl_trans *trans);
 void __releases(nic_access)
 iwl_trans_release_nic_access(struct iwl_trans *trans);
 
-static inline void iwl_trans_fw_error(struct iwl_trans *trans, bool sync)
+static inline void iwl_trans_schedule_reset(struct iwl_trans *trans,
+					    enum iwl_fw_error_type type)
+{
+	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
+		return;
+	/* clear this on device init, not cleared on any unbind/reprobe */
+	if (test_and_set_bit(STATUS_TRANS_RESET_IN_PROGRESS, &trans->status))
+		return;
+
+	trans->restart.mode.type = type;
+	trans->restart.mode.context = IWL_ERR_CONTEXT_WORKER;
+
+	set_bit(STATUS_RESET_PENDING, &trans->status);
+
+	/*
+	 * keep track of whether or not this happened while resetting,
+	 * by the timer the worker runs it might have finished
+	 */
+	trans->restart.during_reset = test_bit(STATUS_IN_SW_RESET,
+					       &trans->status);
+	queue_delayed_work(system_unbound_wq, &trans->restart.wk, 0);
+}
+
+static inline void iwl_trans_fw_error(struct iwl_trans *trans,
+				      enum iwl_fw_error_type type)
 {
 	if (WARN_ON_ONCE(!trans->op_mode))
 		return;
@@ -1142,10 +1155,27 @@ static inline void iwl_trans_fw_error(struct iwl_trans *trans, bool sync)
 	/* prevent double restarts due to the same erroneous FW */
 	if (!test_and_set_bit(STATUS_FW_ERROR, &trans->status)) {
 		trans->state = IWL_TRANS_NO_FW;
-		iwl_op_mode_nic_error(trans->op_mode, sync);
+		iwl_op_mode_nic_error(trans->op_mode, type);
+		iwl_trans_schedule_reset(trans, type);
 	}
 }
 
+static inline void iwl_trans_opmode_sw_reset(struct iwl_trans *trans,
+					     enum iwl_fw_error_type type)
+{
+	if (WARN_ON_ONCE(!trans->op_mode))
+		return;
+
+	set_bit(STATUS_IN_SW_RESET, &trans->status);
+
+	if (WARN_ON(type == IWL_ERR_TYPE_TOP_RESET_BY_BT))
+		return;
+
+	if (!trans->op_mode->ops->sw_reset ||
+	    !trans->op_mode->ops->sw_reset(trans->op_mode, type))
+		clear_bit(STATUS_IN_SW_RESET, &trans->status);
+}
+
 static inline bool iwl_trans_fw_running(struct iwl_trans *trans)
 {
 	return trans->state == IWL_TRANS_FW_ALIVE;
@@ -1178,13 +1208,19 @@ static inline bool iwl_trans_dbg_ini_valid(struct iwl_trans *trans)
 
 void iwl_trans_interrupts(struct iwl_trans *trans, bool enable);
 
+static inline void iwl_trans_finish_sw_reset(struct iwl_trans *trans)
+{
+	clear_bit(STATUS_IN_SW_RESET, &trans->status);
+}
+
 /*****************************************************
  * transport helper functions
  *****************************************************/
 struct iwl_trans *iwl_trans_alloc(unsigned int priv_size,
-			  struct device *dev,
-			  const struct iwl_cfg_trans_params *cfg_trans);
-int iwl_trans_init(struct iwl_trans *trans);
+				  struct device *dev,
+				  const struct iwl_mac_cfg *mac_cfg,
+				  unsigned int txcmd_size,
+				  unsigned int txcmd_align);
 void iwl_trans_free(struct iwl_trans *trans);
 
 static inline bool iwl_trans_is_hw_error_value(u32 val)
@@ -1192,14 +1228,78 @@ static inline bool iwl_trans_is_hw_error_value(u32 val)
 	return ((val & ~0xf) == 0xa5a5a5a0) || ((val & ~0xf) == 0x5a5a5a50);
 }
 
+void iwl_trans_free_restart_list(void);
+
+static inline u16 iwl_trans_get_num_rbds(struct iwl_trans *trans)
+{
+	u16 result = trans->cfg->num_rbds;
+
+	/*
+	 * Since AX210 family (So/Ty) the device cannot put mutliple
+	 * frames into the same buffer, so double the value for them.
+	 */
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+		return 2 * result;
+	return result;
+}
+
+static inline void iwl_trans_suppress_cmd_error_once(struct iwl_trans *trans)
+{
+	set_bit(STATUS_SUPPRESS_CMD_ERROR_ONCE, &trans->status);
+}
+
+static inline bool iwl_trans_device_enabled(struct iwl_trans *trans)
+{
+	return test_bit(STATUS_DEVICE_ENABLED, &trans->status);
+}
+
+static inline bool iwl_trans_is_dead(struct iwl_trans *trans)
+{
+	return test_bit(STATUS_TRANS_DEAD, &trans->status);
+}
+
 /*****************************************************
  * PCIe handling
  *****************************************************/
 int __must_check iwl_pci_register_driver(void);
 void iwl_pci_unregister_driver(void);
-void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan);
+
+/* Note: order matters */
+enum iwl_reset_mode {
+	/* upper level modes: */
+	IWL_RESET_MODE_SW_RESET,
+	IWL_RESET_MODE_REPROBE,
+	/* TOP reset doesn't require PCIe remove */
+	IWL_RESET_MODE_TOP_RESET,
+	/* PCIE level modes: */
+	IWL_RESET_MODE_REMOVE_ONLY,
+	IWL_RESET_MODE_RESCAN,
+	IWL_RESET_MODE_FUNC_RESET,
+	IWL_RESET_MODE_PROD_RESET,
+
+	/* keep last - special backoff value */
+	IWL_RESET_MODE_BACKOFF,
+};
+
+void iwl_trans_pcie_reset(struct iwl_trans *trans, enum iwl_reset_mode mode);
+void iwl_trans_pcie_fw_reset_handshake(struct iwl_trans *trans);
 
 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans,
 			     struct iwl_host_cmd *cmd);
 
+/* Internal helper */
+static inline void iwl_trans_set_info(struct iwl_trans *trans,
+				      struct iwl_trans_info *info)
+{
+	struct iwl_trans_info *write;
+
+	write = (void *)(uintptr_t)&trans->info;
+	*write = *info;
+}
+
+static inline u16 iwl_trans_get_device_id(struct iwl_trans *trans)
+{
+	return u32_get_bits(trans->info.hw_id, GENMASK(31, 16));
+}
+
 #endif /* __iwl_trans_h__ */
diff --git a/sys/contrib/dev/iwlwifi/iwl-utils.c b/sys/contrib/dev/iwlwifi/iwl-utils.c
new file mode 100644
index 000000000000..d503544fda40
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/iwl-utils.c
@@ -0,0 +1,195 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <net/gso.h>
+#include <linux/ieee80211.h>
+#include <net/ip.h>
+
+#include "iwl-drv.h"
+#include "iwl-utils.h"
+
+#ifdef CONFIG_INET
+int iwl_tx_tso_segment(struct sk_buff *skb, unsigned int num_subframes,
+		       netdev_features_t netdev_flags,
+		       struct sk_buff_head *mpdus_skbs)
+{
+	struct sk_buff *tmp, *next;
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	char cb[sizeof(skb->cb)];
+	u16 i = 0;
+	unsigned int tcp_payload_len;
+	unsigned int mss = skb_shinfo(skb)->gso_size;
+	bool ipv4 = (skb->protocol == htons(ETH_P_IP));
+	bool qos = ieee80211_is_data_qos(hdr->frame_control);
+	u16 ip_base_id = ipv4 ? ntohs(ip_hdr(skb)->id) : 0;
+
+	skb_shinfo(skb)->gso_size = num_subframes * mss;
+	memcpy(cb, skb->cb, sizeof(cb));
+
+	next = skb_gso_segment(skb, netdev_flags);
+	skb_shinfo(skb)->gso_size = mss;
+	skb_shinfo(skb)->gso_type = ipv4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
+
+	if (IS_ERR(next) && PTR_ERR(next) == -ENOMEM)
+		return -ENOMEM;
+
+	if (WARN_ONCE(IS_ERR(next),
+		      "skb_gso_segment error: %d\n", (int)PTR_ERR(next)))
+		return PTR_ERR(next);
+
+	if (next)
+		consume_skb(skb);
+
+	skb_list_walk_safe(next, tmp, next) {
+		memcpy(tmp->cb, cb, sizeof(tmp->cb));
+		/*
+		 * Compute the length of all the data added for the A-MSDU.
+		 * This will be used to compute the length to write in the TX
+		 * command. We have: SNAP + IP + TCP for n -1 subframes and
+		 * ETH header for n subframes.
+		 */
+		tcp_payload_len = skb_tail_pointer(tmp) -
+			skb_transport_header(tmp) -
+			tcp_hdrlen(tmp) + tmp->data_len;
+
+		if (ipv4)
+			ip_hdr(tmp)->id = htons(ip_base_id + i * num_subframes);
+
+		if (tcp_payload_len > mss) {
+			skb_shinfo(tmp)->gso_size = mss;
+			skb_shinfo(tmp)->gso_type = ipv4 ? SKB_GSO_TCPV4 :
+							   SKB_GSO_TCPV6;
+		} else {
+			if (qos) {
+				u8 *qc;
+
+				if (ipv4)
+					ip_send_check(ip_hdr(tmp));
+
+				qc = ieee80211_get_qos_ctl((void *)tmp->data);
+				*qc &= ~IEEE80211_QOS_CTL_A_MSDU_PRESENT;
+			}
+			skb_shinfo(tmp)->gso_size = 0;
+		}
+
+		skb_mark_not_on_list(tmp);
+		__skb_queue_tail(mpdus_skbs, tmp);
+		i++;
+	}
+
+	return 0;
+}
+IWL_EXPORT_SYMBOL(iwl_tx_tso_segment);
+#endif /* CONFIG_INET */
+
+static u32 iwl_div_by_db(u32 value, u8 db)
+{
+	/*
+	 * 2^32 * 10**(i / 10) for i = [1, 10], skipping 0 and simply stopping
+	 * at 10 dB and looping instead of using a much larger table.
+	 *
+	 * Using 64 bit math is overkill, but means the helper does not require
+	 * a limit on the input range.
+	 */
+	static const u32 db_to_val[] = {
+		0xcb59185e, 0xa1866ba8, 0x804dce7a, 0x65ea59fe, 0x50f44d89,
+		0x404de61f, 0x331426af, 0x2892c18b, 0x203a7e5b, 0x1999999a,
+	};
+
+	while (value && db > 0) {
+		u8 change = min_t(u8, db, ARRAY_SIZE(db_to_val));
+
+		value = (((u64)value) * db_to_val[change - 1]) >> 32;
+
+		db -= change;
+	}
+
+	return value;
+}
+
+s8 iwl_average_neg_dbm(const u8 *neg_dbm_values, u8 len)
+{
+	int average_magnitude;
+	u32 average_factor;
+	int sum_magnitude = -128;
+	u32 sum_factor = 0;
+	int i, count = 0;
+
+	/*
+	 * To properly average the decibel values (signal values given in dBm)
+	 * we need to do the math in linear space.  Doing a linear average of
+	 * dB (dBm) values is a bit annoying though due to the large range of
+	 * at least -10 to -110 dBm that will not fit into a 32 bit integer.
+	 *
+	 * A 64 bit integer should be sufficient, but then we still have the
+	 * problem that there are no directly usable utility functions
+	 * available.
+	 *
+	 * So, lets not deal with that and instead do much of the calculation
+	 * with a 16.16 fixed point integer along with a base in dBm. 16.16 bit
+	 * gives us plenty of head-room for adding up a few values and even
+	 * doing some math on it. And the tail should be accurate enough too
+	 * (1/2^16 is somewhere around -48 dB, so effectively zero).
+	 *
+	 * i.e. the real value of sum is:
+	 *      sum = sum_factor / 2^16 * 10^(sum_magnitude / 10) mW
+	 *
+	 * However, that does mean we need to be able to bring two values to
+	 * a common base, so we need a helper for that.
+	 *
+	 * Note that this function takes an input with unsigned negative dBm
+	 * values but returns a signed dBm (i.e. a negative value).
+	 */
+
+	for (i = 0; i < len; i++) {
+		int val_magnitude;
+		u32 val_factor;
+
+		/* Assume invalid */
+		if (neg_dbm_values[i] == 0xff)
+			continue;
+
+		val_factor = 0x10000;
+		val_magnitude = -neg_dbm_values[i];
+
+		if (val_magnitude <= sum_magnitude) {
+			u8 div_db = sum_magnitude - val_magnitude;
+
+			val_factor = iwl_div_by_db(val_factor, div_db);
+			val_magnitude = sum_magnitude;
+		} else {
+			u8 div_db = val_magnitude - sum_magnitude;
+
+			sum_factor = iwl_div_by_db(sum_factor, div_db);
+			sum_magnitude = val_magnitude;
+		}
+
+		sum_factor += val_factor;
+		count++;
+	}
+
+	/* No valid noise measurement, return a very high noise level */
+	if (count == 0)
+		return 0;
+
+	average_magnitude = sum_magnitude;
+	average_factor = sum_factor / count;
+
+	/*
+	 * average_factor will be a number smaller than 1.0 (0x10000) at this
+	 * point. What we need to do now is to adjust average_magnitude so that
+	 * average_factor is between -0.5 dB and 0.5 dB.
+	 *
+	 * Just do -1 dB steps and find the point where
+	 *   -0.5 dB * -i dB = 0x10000 * 10^(-0.5/10) / i dB
+	 *                   = div_by_db(0xe429, i)
+	 * is smaller than average_factor.
+	 */
+	for (i = 0; average_factor < iwl_div_by_db(0xe429, i); i++) {
+		/* nothing */
+	}
+
+	return clamp(average_magnitude - i, -128, 0);
+}
+IWL_EXPORT_SYMBOL(iwl_average_neg_dbm);
diff --git a/sys/contrib/dev/iwlwifi/iwl-utils.h b/sys/contrib/dev/iwlwifi/iwl-utils.h
new file mode 100644
index 000000000000..5172035e4d26
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/iwl-utils.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_utils_h__
+#define __iwl_utils_h__
+
+#include <net/cfg80211.h>
+
+#ifdef CONFIG_INET
+/**
+ * iwl_tx_tso_segment - Segments a TSO packet into subframes for A-MSDU.
+ * @skb: buffer to segment.
+ * @num_subframes: number of subframes to create.
+ * @netdev_flags: netdev feature flags.
+ * @mpdus_skbs: list to hold the segmented subframes.
+ *
+ * This function segments a large TCP packet into subframes.
+ * subframes are added to the mpdus_skbs list
+ *
+ * Returns: 0 on success and negative value on failure.
+ */
+int iwl_tx_tso_segment(struct sk_buff *skb, unsigned int num_subframes,
+		       netdev_features_t netdev_flags,
+		       struct sk_buff_head *mpdus_skbs);
+#else
+static inline
+int iwl_tx_tso_segment(struct sk_buff *skb, unsigned int num_subframes,
+		       netdev_features_t netdev_flags,
+		       struct sk_buff_head *mpdus_skbs)
+{
+	WARN_ON(1);
+
+	return -1;
+}
+#endif /* CONFIG_INET */
+
+static inline
+u32 iwl_find_ie_offset(u8 *beacon, u8 eid, u32 frame_size)
+{
+	struct ieee80211_mgmt *mgmt = (void *)beacon;
+	const u8 *ie;
+
+	if (WARN_ON_ONCE(frame_size <= (mgmt->u.beacon.variable - beacon)))
+		return 0;
+
+	frame_size -= mgmt->u.beacon.variable - beacon;
+
+	ie = cfg80211_find_ie(eid, mgmt->u.beacon.variable, frame_size);
+	if (!ie)
+		return 0;
+
+	return ie - beacon;
+}
+
+s8 iwl_average_neg_dbm(const u8 *neg_dbm_values, u8 len);
+
+#endif /* __iwl_utils_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/Makefile b/sys/contrib/dev/iwlwifi/mld/Makefile
new file mode 100644
index 000000000000..c966e573f430
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/Makefile
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+obj-$(CONFIG_IWLMLD)   += iwlmld.o
+obj-$(CONFIG_IWLWIFI_KUNIT_TESTS) += tests/
+
+iwlmld-y += mld.o notif.o mac80211.o fw.o power.o iface.o link.o rx.o mcc.o session-protect.o phy.o
+iwlmld-y += scan.o sta.o tx.o coex.o tlc.o agg.o key.o regulatory.o ap.o thermal.o roc.o stats.o
+iwlmld-y += low_latency.o mlo.o ptp.o time_sync.o ftm-initiator.o
+iwlmld-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o
+iwlmld-$(CONFIG_IWLWIFI_LEDS) += led.o
+iwlmld-$(CONFIG_PM_SLEEP) += d3.o
+
+subdir-ccflags-y += -I$(src)/../
diff --git a/sys/contrib/dev/iwlwifi/mld/agg.c b/sys/contrib/dev/iwlwifi/mld/agg.c
new file mode 100644
index 000000000000..3a346bcd6665
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/agg.c
@@ -0,0 +1,680 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include "agg.h"
+#include "sta.h"
+#include "hcmd.h"
+#if defined(__FreeBSD__)
+#include <linux/cache.h>
+#endif
+
+static void
+iwl_mld_reorder_release_frames(struct iwl_mld *mld, struct ieee80211_sta *sta,
+			       struct napi_struct *napi,
+			       struct iwl_mld_baid_data *baid_data,
+			       struct iwl_mld_reorder_buffer *reorder_buf,
+			       u16 nssn)
+{
+	struct iwl_mld_reorder_buf_entry *entries =
+		&baid_data->entries[reorder_buf->queue *
+				    baid_data->entries_per_queue];
+	u16 ssn = reorder_buf->head_sn;
+
+	while (ieee80211_sn_less(ssn, nssn)) {
+		int index = ssn % baid_data->buf_size;
+		struct sk_buff_head *skb_list = &entries[index].frames;
+		struct sk_buff *skb;
+
+		ssn = ieee80211_sn_inc(ssn);
+
+		/* Empty the list. Will have more than one frame for A-MSDU.
+		 * Empty list is valid as well since nssn indicates frames were
+		 * received.
+		 */
+		while ((skb = __skb_dequeue(skb_list))) {
+			iwl_mld_pass_packet_to_mac80211(mld, napi, skb,
+							reorder_buf->queue,
+							sta);
+			reorder_buf->num_stored--;
+		}
+	}
+	reorder_buf->head_sn = nssn;
+}
+
+static void iwl_mld_release_frames_from_notif(struct iwl_mld *mld,
+					      struct napi_struct *napi,
+					      u8 baid, u16 nssn, int queue)
+{
+	struct iwl_mld_reorder_buffer *reorder_buf;
+	struct iwl_mld_baid_data *ba_data;
+	struct ieee80211_link_sta *link_sta;
+	u32 sta_id;
+
+	IWL_DEBUG_HT(mld, "Frame release notification for BAID %u, NSSN %d\n",
+		     baid, nssn);
+
+	if (WARN_ON_ONCE(baid == IWL_RX_REORDER_DATA_INVALID_BAID ||
+			 baid >= ARRAY_SIZE(mld->fw_id_to_ba)))
+		return;
+
+	rcu_read_lock();
+
+	ba_data = rcu_dereference(mld->fw_id_to_ba[baid]);
+	if (!ba_data) {
+		IWL_DEBUG_HT(mld, "BAID %d not found in map\n", baid);
+		goto out_unlock;
+	}
+
+	/* pick any STA ID to find the pointer */
+	sta_id = ffs(ba_data->sta_mask) - 1;
+	link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]);
+	if (WARN_ON_ONCE(IS_ERR_OR_NULL(link_sta) || !link_sta->sta))
+		goto out_unlock;
+
+	reorder_buf = &ba_data->reorder_buf[queue];
+
+	iwl_mld_reorder_release_frames(mld, link_sta->sta, napi, ba_data,
+				       reorder_buf, nssn);
+out_unlock:
+	rcu_read_unlock();
+}
+
+void iwl_mld_handle_frame_release_notif(struct iwl_mld *mld,
+					struct napi_struct *napi,
+					struct iwl_rx_packet *pkt, int queue)
+{
+	struct iwl_frame_release *release = (void *)pkt->data;
+	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
+
+	if (IWL_FW_CHECK(mld, pkt_len < sizeof(*release),
+			 "Unexpected frame release notif size %u (expected %zu)\n",
+			 pkt_len, sizeof(*release)))
+		return;
+
+	iwl_mld_release_frames_from_notif(mld, napi, release->baid,
+					  le16_to_cpu(release->nssn),
+					  queue);
+}
+
+void iwl_mld_handle_bar_frame_release_notif(struct iwl_mld *mld,
+					    struct napi_struct *napi,
+					    struct iwl_rx_packet *pkt,
+					    int queue)
+{
+	struct iwl_bar_frame_release *release = (void *)pkt->data;
+	struct iwl_mld_baid_data *baid_data;
+	unsigned int baid, nssn, sta_id, tid;
+	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
+
+	if (IWL_FW_CHECK(mld, pkt_len < sizeof(*release),
+			 "Unexpected frame release notif size %u (expected %zu)\n",
+			 pkt_len, sizeof(*release)))
+		return;
+
+	baid = le32_get_bits(release->ba_info,
+			     IWL_BAR_FRAME_RELEASE_BAID_MASK);
+	nssn = le32_get_bits(release->ba_info,
+			     IWL_BAR_FRAME_RELEASE_NSSN_MASK);
+	sta_id = le32_get_bits(release->sta_tid,
+			       IWL_BAR_FRAME_RELEASE_STA_MASK);
+	tid = le32_get_bits(release->sta_tid,
+			    IWL_BAR_FRAME_RELEASE_TID_MASK);
+
+	if (IWL_FW_CHECK(mld, baid >= ARRAY_SIZE(mld->fw_id_to_ba),
+			 "BAR release: invalid BAID (%x)\n", baid))
+		return;
+
+	rcu_read_lock();
+	baid_data = rcu_dereference(mld->fw_id_to_ba[baid]);
+	if (!baid_data) {
+		IWL_DEBUG_HT(mld,
+			     "Got valid BAID %d but not allocated\n",
+			     baid);
+		goto out_unlock;
+	}
+
+	if (IWL_FW_CHECK(mld, tid != baid_data->tid ||
+			 sta_id > mld->fw->ucode_capa.num_stations ||
+			 !(baid_data->sta_mask & BIT(sta_id)),
+			 "BAID 0x%x is mapped to sta_mask:0x%x tid:%d, but BAR release received for sta:%d tid:%d\n",
+			 baid, baid_data->sta_mask, baid_data->tid, sta_id,
+			 tid))
+		goto out_unlock;
+
+	IWL_DEBUG_DROP(mld, "Received a BAR, expect packet loss: nssn %d\n",
+		       nssn);
+
+	iwl_mld_release_frames_from_notif(mld, napi, baid, nssn, queue);
+out_unlock:
+	rcu_read_unlock();
+}
+
+void iwl_mld_del_ba(struct iwl_mld *mld, int queue,
+		    struct iwl_mld_delba_data *data)
+{
+	struct iwl_mld_baid_data *ba_data;
+	struct iwl_mld_reorder_buffer *reorder_buf;
+	struct ieee80211_link_sta *link_sta;
+	u8 baid = data->baid;
+	u32 sta_id;
+
+	if (WARN_ONCE(baid >= IWL_MAX_BAID, "invalid BAID: %x\n", baid))
+		return;
+
+	rcu_read_lock();
+
+	ba_data = rcu_dereference(mld->fw_id_to_ba[baid]);
+	if (WARN_ON_ONCE(!ba_data))
+		goto out_unlock;
+
+	/* pick any STA ID to find the pointer */
+	sta_id = ffs(ba_data->sta_mask) - 1;
+	link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]);
+	if (WARN_ON_ONCE(IS_ERR_OR_NULL(link_sta) || !link_sta->sta))
+		goto out_unlock;
+
+	reorder_buf = &ba_data->reorder_buf[queue];
+
+	/* release all frames that are in the reorder buffer to the stack */
+	iwl_mld_reorder_release_frames(mld, link_sta->sta, NULL,
+				       ba_data, reorder_buf,
+				       ieee80211_sn_add(reorder_buf->head_sn,
+							ba_data->buf_size));
+out_unlock:
+	rcu_read_unlock();
+}
+
+/* Returns true if the MPDU was buffered\dropped, false if it should be passed
+ * to upper layer.
+ */
+enum iwl_mld_reorder_result
+iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
+		int queue, struct ieee80211_sta *sta,
+		struct sk_buff *skb, struct iwl_rx_mpdu_desc *desc)
+{
+	struct ieee80211_hdr *hdr = (void *)skb_mac_header(skb);
+	struct iwl_mld_baid_data *baid_data;
+	struct iwl_mld_reorder_buffer *buffer;
+	struct iwl_mld_reorder_buf_entry *entries;
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_link_sta *mld_link_sta;
+	u32 reorder = le32_to_cpu(desc->reorder_data);
+	bool amsdu, last_subframe, is_old_sn, is_dup;
+	u8 tid = ieee80211_get_tid(hdr);
+	u8 baid;
+	u16 nssn, sn;
+	u32 sta_mask = 0;
+	int index;
+	u8 link_id;
+
+	baid = u32_get_bits(reorder, IWL_RX_MPDU_REORDER_BAID_MASK);
+
+	/* This also covers the case of receiving a Block Ack Request
+	 * outside a BA session; we'll pass it to mac80211 and that
+	 * then sends a delBA action frame.
+	 * This also covers pure monitor mode, in which case we won't
+	 * have any BA sessions.
+	 */
+	if (baid == IWL_RX_REORDER_DATA_INVALID_BAID)
+		return IWL_MLD_PASS_SKB;
+
+	/* no sta yet */
+	if (WARN_ONCE(!sta,
+		      "Got valid BAID without a valid station assigned\n"))
+		return IWL_MLD_PASS_SKB;
+
+	/* not a data packet */
+	if (!ieee80211_is_data_qos(hdr->frame_control) ||
+	    is_multicast_ether_addr(hdr->addr1))
+		return IWL_MLD_PASS_SKB;
+
+	if (unlikely(!ieee80211_is_data_present(hdr->frame_control)))
+		return IWL_MLD_PASS_SKB;
+
+	baid_data = rcu_dereference(mld->fw_id_to_ba[baid]);
+	if (!baid_data) {
+		IWL_DEBUG_HT(mld,
+			     "Got valid BAID but no baid allocated, bypass re-ordering (BAID=%d reorder=0x%x)\n",
+			     baid, reorder);
+		return IWL_MLD_PASS_SKB;
+	}
+
+	for_each_mld_link_sta(mld_sta, mld_link_sta, link_id)
+		sta_mask |= BIT(mld_link_sta->fw_id);
+
+	/* verify the BAID is correctly mapped to the sta and tid */
+	if (IWL_FW_CHECK(mld,
+			 tid != baid_data->tid ||
+			 !(sta_mask & baid_data->sta_mask),
+			 "BAID 0x%x is mapped to sta_mask:0x%x tid:%d, but was received for sta_mask:0x%x tid:%d\n",
+			 baid, baid_data->sta_mask, baid_data->tid,
+			 sta_mask, tid))
+		return IWL_MLD_PASS_SKB;
+
+	buffer = &baid_data->reorder_buf[queue];
+	entries = &baid_data->entries[queue * baid_data->entries_per_queue];
+
+	is_old_sn = !!(reorder & IWL_RX_MPDU_REORDER_BA_OLD_SN);
+
+	if (!buffer->valid && is_old_sn)
+		return IWL_MLD_PASS_SKB;
+
+	buffer->valid = true;
+
+	is_dup = !!(desc->status & cpu_to_le32(IWL_RX_MPDU_STATUS_DUPLICATE));
+
+	/* drop any duplicated or outdated packets */
+	if (is_dup || is_old_sn)
+		return IWL_MLD_DROP_SKB;
+
+	sn = u32_get_bits(reorder, IWL_RX_MPDU_REORDER_SN_MASK);
+	nssn = u32_get_bits(reorder, IWL_RX_MPDU_REORDER_NSSN_MASK);
+	amsdu = desc->mac_flags2 & IWL_RX_MPDU_MFLG2_AMSDU;
+	last_subframe = desc->amsdu_info & IWL_RX_MPDU_AMSDU_LAST_SUBFRAME;
+
+	/* release immediately if allowed by nssn and no stored frames */
+	if (!buffer->num_stored && ieee80211_sn_less(sn, nssn)) {
+		if (!amsdu || last_subframe)
+			buffer->head_sn = nssn;
+		return IWL_MLD_PASS_SKB;
+	}
+
+	/* release immediately if there are no stored frames, and the sn is
+	 * equal to the head.
+	 * This can happen due to reorder timer, where NSSN is behind head_sn.
+	 * When we released everything, and we got the next frame in the
+	 * sequence, according to the NSSN we can't release immediately,
+	 * while technically there is no hole and we can move forward.
+	 */
+	if (!buffer->num_stored && sn == buffer->head_sn) {
+		if (!amsdu || last_subframe)
+			buffer->head_sn = ieee80211_sn_inc(buffer->head_sn);
+		return IWL_MLD_PASS_SKB;
+	}
+
+	/* put in reorder buffer */
+	index = sn % baid_data->buf_size;
+	__skb_queue_tail(&entries[index].frames, skb);
+	buffer->num_stored++;
+
+	/* We cannot trust NSSN for AMSDU sub-frames that are not the last. The
+	 * reason is that NSSN advances on the first sub-frame, and may cause
+	 * the reorder buffer to advance before all the sub-frames arrive.
+	 *
+	 * Example: reorder buffer contains SN 0 & 2, and we receive AMSDU with
+	 * SN 1. NSSN for first sub frame will be 3 with the result of driver
+	 * releasing SN 0,1, 2. When sub-frame 1 arrives - reorder buffer is
+	 * already ahead and it will be dropped.
+	 * If the last sub-frame is not on this queue - we will get frame
+	 * release notification with up to date NSSN.
+	 * If this is the first frame that is stored in the buffer, the head_sn
+	 * may be outdated. Update it based on the last NSSN to make sure it
+	 * will be released when the frame release notification arrives.
+	 */
+	if (!amsdu || last_subframe)
+		iwl_mld_reorder_release_frames(mld, sta, napi, baid_data,
+					       buffer, nssn);
+	else if (buffer->num_stored == 1)
+		buffer->head_sn = nssn;
+
+	return IWL_MLD_BUFFERED_SKB;
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_reorder);
+
+static void iwl_mld_rx_agg_session_expired(struct timer_list *t)
+{
+	struct iwl_mld_baid_data *data =
+		timer_container_of(data, t, session_timer);
+	struct iwl_mld_baid_data __rcu **rcu_ptr = data->rcu_ptr;
+	struct iwl_mld_baid_data *ba_data;
+	struct ieee80211_link_sta *link_sta;
+	struct iwl_mld_sta *mld_sta;
+	unsigned long timeout;
+	unsigned int sta_id;
+
+	rcu_read_lock();
+
+	ba_data = rcu_dereference(*rcu_ptr);
+	if (WARN_ON(!ba_data))
+		goto unlock;
+
+	if (WARN_ON(!ba_data->timeout))
+		goto unlock;
+
+	timeout = ba_data->last_rx_timestamp +
+		  TU_TO_JIFFIES(ba_data->timeout * 2);
+	if (time_is_after_jiffies(timeout)) {
+		mod_timer(&ba_data->session_timer, timeout);
+		goto unlock;
+	}
+
+	/* timer expired, pick any STA ID to find the pointer */
+	sta_id = ffs(ba_data->sta_mask) - 1;
+	link_sta = rcu_dereference(ba_data->mld->fw_id_to_link_sta[sta_id]);
+
+	/* sta should be valid unless the following happens:
+	 * The firmware asserts which triggers a reconfig flow, but
+	 * the reconfig fails before we set the pointer to sta into
+	 * the fw_id_to_link_sta pointer table. mac80211 can't stop
+	 * A-MPDU and hence the timer continues to run. Then, the
+	 * timer expires and sta is NULL.
+	 */
+	if (IS_ERR_OR_NULL(link_sta) || WARN_ON(!link_sta->sta))
+		goto unlock;
+
+	mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	ieee80211_rx_ba_timer_expired(mld_sta->vif, link_sta->sta->addr,
+				      ba_data->tid);
+unlock:
+	rcu_read_unlock();
+}
+
+static int
+iwl_mld_stop_ba_in_fw(struct iwl_mld *mld, struct ieee80211_sta *sta, int tid)
+{
+	struct iwl_rx_baid_cfg_cmd cmd = {
+		.action = cpu_to_le32(IWL_RX_BAID_ACTION_REMOVE),
+		.remove.sta_id_mask =
+			cpu_to_le32(iwl_mld_fw_sta_id_mask(mld, sta)),
+		.remove.tid = cpu_to_le32(tid),
+
+	};
+	int ret;
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(DATA_PATH_GROUP,
+					   RX_BAID_ALLOCATION_CONFIG_CMD),
+				   &cmd);
+	if (ret)
+		return ret;
+
+	IWL_DEBUG_HT(mld, "RX BA Session stopped in fw\n");
+
+	return ret;
+}
+
+static int
+iwl_mld_start_ba_in_fw(struct iwl_mld *mld, struct ieee80211_sta *sta,
+		       int tid, u16 ssn, u16 buf_size)
+{
+	struct iwl_rx_baid_cfg_cmd cmd = {
+		.action = cpu_to_le32(IWL_RX_BAID_ACTION_ADD),
+		.alloc.sta_id_mask =
+			cpu_to_le32(iwl_mld_fw_sta_id_mask(mld, sta)),
+		.alloc.tid = tid,
+		.alloc.ssn = cpu_to_le16(ssn),
+		.alloc.win_size = cpu_to_le16(buf_size),
+	};
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(DATA_PATH_GROUP, RX_BAID_ALLOCATION_CONFIG_CMD),
+		.flags = CMD_WANT_SKB,
+		.len[0] = sizeof(cmd),
+		.data[0] = &cmd,
+	};
+	struct iwl_rx_baid_cfg_resp *resp;
+	struct iwl_rx_packet *pkt;
+	u32 resp_len;
+	int ret, baid;
+
+	BUILD_BUG_ON(sizeof(*resp) != sizeof(baid));
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+	if (ret)
+		return ret;
+
+	pkt = hcmd.resp_pkt;
+
+	resp_len = iwl_rx_packet_payload_len(pkt);
+	if (IWL_FW_CHECK(mld, resp_len != sizeof(*resp),
+			 "BAID_ALLOC_CMD: unexpected response length %d\n",
+			 resp_len)) {
+		ret = -EIO;
+		goto out;
+	}
+
+	IWL_DEBUG_HT(mld, "RX BA Session started in fw\n");
+
+	resp = (void *)pkt->data;
+	baid = le32_to_cpu(resp->baid);
+
+	if (IWL_FW_CHECK(mld, baid < 0 || baid >= ARRAY_SIZE(mld->fw_id_to_ba),
+			 "BAID_ALLOC_CMD: invalid BAID response %d\n", baid)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = baid;
+out:
+	iwl_free_resp(&hcmd);
+	return ret;
+}
+
+static void iwl_mld_init_reorder_buffer(struct iwl_mld *mld,
+					struct iwl_mld_baid_data *data,
+					u16 ssn)
+{
+	for (int i = 0; i < mld->trans->info.num_rxqs; i++) {
+		struct iwl_mld_reorder_buffer *reorder_buf =
+			&data->reorder_buf[i];
+		struct iwl_mld_reorder_buf_entry *entries =
+			&data->entries[i * data->entries_per_queue];
+
+		reorder_buf->head_sn = ssn;
+		reorder_buf->queue = i;
+
+		for (int j = 0; j < data->buf_size; j++)
+			__skb_queue_head_init(&entries[j].frames);
+	}
+}
+
+static void iwl_mld_free_reorder_buffer(struct iwl_mld *mld,
+					struct iwl_mld_baid_data *data)
+{
+	struct iwl_mld_delba_data delba_data = {
+		.baid = data->baid,
+	};
+
+	iwl_mld_sync_rx_queues(mld, IWL_MLD_RXQ_NOTIF_DEL_BA,
+			       &delba_data, sizeof(delba_data));
+
+	for (int i = 0; i < mld->trans->info.num_rxqs; i++) {
+		struct iwl_mld_reorder_buffer *reorder_buf =
+			&data->reorder_buf[i];
+		struct iwl_mld_reorder_buf_entry *entries =
+			&data->entries[i * data->entries_per_queue];
+
+		if (likely(!reorder_buf->num_stored))
+			continue;
+
+		/* This shouldn't happen in regular DELBA since the RX queues
+		 * sync internal DELBA notification should trigger a release
+		 * of all frames in the reorder buffer.
+		 */
+		WARN_ON(1);
+
+		for (int j = 0; j < data->buf_size; j++)
+			__skb_queue_purge(&entries[j].frames);
+	}
+}
+
+int iwl_mld_ampdu_rx_start(struct iwl_mld *mld, struct ieee80211_sta *sta,
+			   int tid, u16 ssn, u16 buf_size, u16 timeout)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_baid_data *baid_data = NULL;
+	u32 reorder_buf_size = buf_size * sizeof(baid_data->entries[0]);
+	int ret, baid;
+	u32 sta_mask;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (mld->num_rx_ba_sessions >= IWL_MAX_BAID) {
+		IWL_DEBUG_HT(mld,
+			     "Max num of RX BA sessions reached; blocking new session\n");
+		return -ENOSPC;
+	}
+
+	sta_mask = iwl_mld_fw_sta_id_mask(mld, sta);
+	if (WARN_ON(!sta_mask))
+		return -EINVAL;
+
+	/* sparse doesn't like the __align() so don't check */
+#ifndef __CHECKER__
+	/* The division below will be OK if either the cache line size
+	 * can be divided by the entry size (ALIGN will round up) or if
+	 * the entry size can be divided by the cache line size, in which
+	 * case the ALIGN() will do nothing.
+	 */
+	BUILD_BUG_ON(SMP_CACHE_BYTES % sizeof(baid_data->entries[0]) &&
+		     sizeof(baid_data->entries[0]) % SMP_CACHE_BYTES);
+#endif
+
+	/* Upward align the reorder buffer size to fill an entire cache
+	 * line for each queue, to avoid sharing cache lines between
+	 * different queues.
+	 */
+	reorder_buf_size = ALIGN(reorder_buf_size, SMP_CACHE_BYTES);
+
+	/* Allocate here so if allocation fails we can bail out early
+	 * before starting the BA session in the firmware
+	 */
+	baid_data = kzalloc(sizeof(*baid_data) +
+			    mld->trans->info.num_rxqs * reorder_buf_size,
+			    GFP_KERNEL);
+	if (!baid_data)
+		return -ENOMEM;
+
+	/* This division is why we need the above BUILD_BUG_ON(),
+	 * if that doesn't hold then this will not be right.
+	 */
+	baid_data->entries_per_queue =
+		reorder_buf_size / sizeof(baid_data->entries[0]);
+
+	baid = iwl_mld_start_ba_in_fw(mld, sta, tid, ssn, buf_size);
+	if (baid < 0) {
+		ret = baid;
+		goto out_free;
+	}
+
+	mld->num_rx_ba_sessions++;
+	mld_sta->tid_to_baid[tid] = baid;
+
+	baid_data->baid = baid;
+	baid_data->mld = mld;
+	baid_data->tid = tid;
+	baid_data->buf_size = buf_size;
+	baid_data->sta_mask = sta_mask;
+	baid_data->timeout = timeout;
+	baid_data->last_rx_timestamp = jiffies;
+	baid_data->rcu_ptr = &mld->fw_id_to_ba[baid];
+
+	iwl_mld_init_reorder_buffer(mld, baid_data, ssn);
+
+	timer_setup(&baid_data->session_timer, iwl_mld_rx_agg_session_expired,
+		    0);
+	if (timeout)
+		mod_timer(&baid_data->session_timer,
+			  TU_TO_EXP_TIME(timeout * 2));
+
+	IWL_DEBUG_HT(mld, "STA mask=0x%x (tid=%d) is assigned to BAID %d\n",
+		     baid_data->sta_mask, tid, baid);
+
+	/* protect the BA data with RCU to cover a case where our
+	 * internal RX sync mechanism will timeout (not that it's
+	 * supposed to happen) and we will free the session data while
+	 * RX is being processed in parallel
+	 */
+	WARN_ON(rcu_access_pointer(mld->fw_id_to_ba[baid]));
+	rcu_assign_pointer(mld->fw_id_to_ba[baid], baid_data);
+
+	return 0;
+
+out_free:
+	kfree(baid_data);
+	return ret;
+}
+
+int iwl_mld_ampdu_rx_stop(struct iwl_mld *mld, struct ieee80211_sta *sta,
+			  int tid)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	int baid = mld_sta->tid_to_baid[tid];
+	struct iwl_mld_baid_data *baid_data;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* during firmware restart, do not send the command as the firmware no
+	 * longer recognizes the session. instead, only clear the driver BA
+	 * session data.
+	 */
+	if (!mld->fw_status.in_hw_restart) {
+		ret = iwl_mld_stop_ba_in_fw(mld, sta, tid);
+		if (ret)
+			return ret;
+	}
+
+	if (!WARN_ON(mld->num_rx_ba_sessions == 0))
+		mld->num_rx_ba_sessions--;
+
+	baid_data = wiphy_dereference(mld->wiphy, mld->fw_id_to_ba[baid]);
+	if (WARN_ON(!baid_data))
+		return -EINVAL;
+
+	if (timer_pending(&baid_data->session_timer))
+		timer_shutdown_sync(&baid_data->session_timer);
+
+	iwl_mld_free_reorder_buffer(mld, baid_data);
+
+	RCU_INIT_POINTER(mld->fw_id_to_ba[baid], NULL);
+	kfree_rcu(baid_data, rcu_head);
+
+	IWL_DEBUG_HT(mld, "BAID %d is free\n", baid);
+
+	return 0;
+}
+
+int iwl_mld_update_sta_baids(struct iwl_mld *mld,
+			     u32 old_sta_mask,
+			     u32 new_sta_mask)
+{
+	struct iwl_rx_baid_cfg_cmd cmd = {
+		.action = cpu_to_le32(IWL_RX_BAID_ACTION_MODIFY),
+		.modify.old_sta_id_mask = cpu_to_le32(old_sta_mask),
+		.modify.new_sta_id_mask = cpu_to_le32(new_sta_mask),
+	};
+	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, RX_BAID_ALLOCATION_CONFIG_CMD);
+	int baid;
+
+	/* mac80211 will remove sessions later, but we ignore all that */
+	if (mld->fw_status.in_hw_restart)
+		return 0;
+
+	BUILD_BUG_ON(sizeof(struct iwl_rx_baid_cfg_resp) != sizeof(baid));
+
+	for (baid = 0; baid < ARRAY_SIZE(mld->fw_id_to_ba); baid++) {
+		struct iwl_mld_baid_data *data;
+		int ret;
+
+		data = wiphy_dereference(mld->wiphy, mld->fw_id_to_ba[baid]);
+		if (!data)
+			continue;
+
+		if (!(data->sta_mask & old_sta_mask))
+			continue;
+
+		WARN_ONCE(data->sta_mask != old_sta_mask,
+			  "BAID data for %d corrupted - expected 0x%x found 0x%x\n",
+			  baid, old_sta_mask, data->sta_mask);
+
+		cmd.modify.tid = cpu_to_le32(data->tid);
+
+		ret = iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd);
+		if (ret)
+			return ret;
+		data->sta_mask = new_sta_mask;
+	}
+
+	return 0;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/agg.h b/sys/contrib/dev/iwlwifi/mld/agg.h
new file mode 100644
index 000000000000..651c80d1c7cd
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/agg.h
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_agg_h__
+#define __iwl_agg_h__
+
+#include "mld.h"
+#include "fw/api/rx.h"
+
+/**
+ * struct iwl_mld_reorder_buffer - per ra/tid/queue reorder buffer
+ * @head_sn: reorder window head sequence number
+ * @num_stored: number of MPDUs stored in the buffer
+ * @queue: queue of this reorder buffer
+ * @valid: true if reordering is valid for this queue
+ */
+struct iwl_mld_reorder_buffer {
+	u16 head_sn;
+	u16 num_stored;
+	int queue;
+	bool valid;
+} ____cacheline_aligned_in_smp;
+
+/**
+ * struct iwl_mld_reorder_buf_entry - reorder buffer entry per-queue/per-seqno
+ * @frames: list of skbs stored. a list is necessary because in an A-MSDU,
+ *	all sub-frames share the same sequence number, so they are stored
+ *	together in the same list.
+ */
+struct iwl_mld_reorder_buf_entry {
+	struct sk_buff_head frames;
+}
+#ifndef __CHECKER__
+/* sparse doesn't like this construct: "bad integer constant expression" */
+__aligned(roundup_pow_of_two(sizeof(struct sk_buff_head)))
+#endif
+;
+
+/**
+ * struct iwl_mld_baid_data - Block Ack session data
+ * @rcu_head: RCU head for freeing this data
+ * @sta_mask: station mask for the BAID
+ * @tid: tid of the session
+ * @baid: baid of the session
+ * @buf_size: the reorder buffer size as set by the last ADDBA request
+ * @entries_per_queue: number of buffers per queue, this actually gets
+ *	aligned up to avoid cache line sharing between queues
+ * @timeout: the timeout value specified in the ADDBA request.
+ * @last_rx_timestamp: timestamp of the last received packet (in jiffies). This
+ *	value is updated only when the configured @timeout has passed since
+ *	the last update to minimize cache bouncing between RX queues.
+ * @session_timer: timer is set to expire after 2 * @timeout (since we want
+ *	to minimize the cache bouncing by updating @last_rx_timestamp only once
+ *	after @timeout has passed). If no packets are received within this
+ *	period, it informs mac80211 to initiate delBA flow, terminating the
+ *	BA session.
+ * @rcu_ptr: BA data RCU protected access
+ * @mld: mld pointer, needed for timer context
+ * @reorder_buf: reorder buffer, allocated per queue
+ * @entries: data
+ */
+struct iwl_mld_baid_data {
+	struct rcu_head rcu_head;
+	u32 sta_mask;
+	u8 tid;
+	u8 baid;
+	u16 buf_size;
+	u16 entries_per_queue;
+	u16 timeout;
+	struct timer_list session_timer;
+	unsigned long last_rx_timestamp;
+	struct iwl_mld_baid_data __rcu **rcu_ptr;
+	struct iwl_mld *mld;
+	struct iwl_mld_reorder_buffer reorder_buf[IWL_MAX_RX_HW_QUEUES];
+	struct iwl_mld_reorder_buf_entry entries[] ____cacheline_aligned_in_smp;
+};
+
+/**
+ * struct iwl_mld_delba_data - RX queue sync data for %IWL_MLD_RXQ_NOTIF_DEL_BA
+ *
+ * @baid: Block Ack id, used to identify the BA session to be removed
+ */
+struct iwl_mld_delba_data {
+	u32 baid;
+} __packed;
+
+/**
+ * enum iwl_mld_reorder_result - Possible return values for iwl_mld_reorder()
+ * indicating how the caller should handle the skb based on the result.
+ *
+ * @IWL_MLD_PASS_SKB: skb should be passed to upper layer.
+ * @IWL_MLD_BUFFERED_SKB: skb has been buffered, don't pass it to upper layer.
+ * @IWL_MLD_DROP_SKB: skb should be dropped and freed by the caller.
+ */
+enum iwl_mld_reorder_result {
+	IWL_MLD_PASS_SKB,
+	IWL_MLD_BUFFERED_SKB,
+	IWL_MLD_DROP_SKB
+};
+
+int iwl_mld_ampdu_rx_start(struct iwl_mld *mld, struct ieee80211_sta *sta,
+			   int tid, u16 ssn, u16 buf_size, u16 timeout);
+int iwl_mld_ampdu_rx_stop(struct iwl_mld *mld, struct ieee80211_sta *sta,
+			  int tid);
+
+enum iwl_mld_reorder_result
+iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
+		int queue, struct ieee80211_sta *sta,
+		struct sk_buff *skb, struct iwl_rx_mpdu_desc *desc);
+
+void iwl_mld_handle_frame_release_notif(struct iwl_mld *mld,
+					struct napi_struct *napi,
+					struct iwl_rx_packet *pkt, int queue);
+void iwl_mld_handle_bar_frame_release_notif(struct iwl_mld *mld,
+					    struct napi_struct *napi,
+					    struct iwl_rx_packet *pkt,
+					    int queue);
+
+void iwl_mld_del_ba(struct iwl_mld *mld, int queue,
+		    struct iwl_mld_delba_data *data);
+
+int iwl_mld_update_sta_baids(struct iwl_mld *mld,
+			     u32 old_sta_mask,
+			     u32 new_sta_mask);
+
+#endif /* __iwl_agg_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/ap.c b/sys/contrib/dev/iwlwifi/mld/ap.c
new file mode 100644
index 000000000000..5c59acc8c4c5
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/ap.c
@@ -0,0 +1,363 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#include <linux/crc32.h>
+
+#include <net/mac80211.h>
+
+#include "ap.h"
+#include "hcmd.h"
+#include "tx.h"
+#include "power.h"
+#include "key.h"
+#include "phy.h"
+#include "iwl-utils.h"
+
+#include "fw/api/sta.h"
+
+void iwl_mld_set_tim_idx(struct iwl_mld *mld, __le32 *tim_index,
+			 u8 *beacon, u32 frame_size)
+{
+	u32 tim_idx;
+	struct ieee80211_mgmt *mgmt = (void *)beacon;
+
+	/* The index is relative to frame start but we start looking at the
+	 * variable-length part of the beacon.
+	 */
+	tim_idx = mgmt->u.beacon.variable - beacon;
+
+	/* Parse variable-length elements of beacon to find WLAN_EID_TIM */
+	while ((tim_idx < (frame_size - 2)) &&
+	       (beacon[tim_idx] != WLAN_EID_TIM))
+		tim_idx += beacon[tim_idx + 1] + 2;
+
+	/* If TIM field was found, set variables */
+	if ((tim_idx < (frame_size - 1)) && beacon[tim_idx] == WLAN_EID_TIM)
+		*tim_index = cpu_to_le32(tim_idx);
+	else
+		IWL_WARN(mld, "Unable to find TIM Element in beacon\n");
+}
+
+u8 iwl_mld_get_rate_flags(struct iwl_mld *mld,
+			  struct ieee80211_tx_info *info,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link,
+			  enum nl80211_band band)
+{
+	u32 legacy = link->beacon_tx_rate.control[band].legacy;
+	u32 rate_idx, rate_flags = 0, fw_rate;
+
+	/* if beacon rate was configured try using it */
+	if (hweight32(legacy) == 1) {
+		u32 rate = ffs(legacy) - 1;
+		struct ieee80211_supported_band *sband =
+			mld->hw->wiphy->bands[band];
+
+		rate_idx = sband->bitrates[rate].hw_value;
+	} else {
+		rate_idx = iwl_mld_get_lowest_rate(mld, info, vif);
+	}
+
+	if (rate_idx <= IWL_LAST_CCK_RATE)
+		rate_flags = IWL_MAC_BEACON_CCK;
+
+	/* Legacy rates are indexed as follows:
+	 * 0 - 3 for CCK and 0 - 7 for OFDM.
+	 */
+	fw_rate = (rate_idx >= IWL_FIRST_OFDM_RATE ?
+		     rate_idx - IWL_FIRST_OFDM_RATE : rate_idx);
+
+	return fw_rate | rate_flags;
+}
+
+int iwl_mld_send_beacon_template_cmd(struct iwl_mld *mld,
+				     struct sk_buff *beacon,
+				     struct iwl_mac_beacon_cmd *cmd)
+{
+	struct iwl_host_cmd hcmd = {
+		.id = BEACON_TEMPLATE_CMD,
+	};
+
+	hcmd.len[0] = sizeof(*cmd);
+	hcmd.data[0] = cmd;
+
+	hcmd.len[1] = beacon->len;
+	hcmd.data[1] = beacon->data;
+	hcmd.dataflags[1] = IWL_HCMD_DFL_DUP;
+
+	return iwl_mld_send_cmd(mld, &hcmd);
+}
+
+static int iwl_mld_fill_beacon_template_cmd(struct iwl_mld *mld,
+					    struct ieee80211_vif *vif,
+					    struct sk_buff *beacon,
+					    struct iwl_mac_beacon_cmd *cmd,
+					    struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(beacon);
+	struct ieee80211_chanctx_conf *ctx;
+	bool enable_fils;
+	u16 flags = 0;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld_link))
+		return -EINVAL;
+
+	cmd->link_id = cpu_to_le32(mld_link->fw_id);
+
+	ctx = wiphy_dereference(mld->wiphy, link->chanctx_conf);
+	if (WARN_ON(!ctx || !ctx->def.chan))
+		return -EINVAL;
+
+	enable_fils = cfg80211_channel_is_psc(ctx->def.chan) ||
+		(ctx->def.chan->band == NL80211_BAND_6GHZ &&
+		 ctx->def.width >= NL80211_CHAN_WIDTH_80);
+
+	if (enable_fils) {
+		flags |= IWL_MAC_BEACON_FILS;
+		cmd->short_ssid = cpu_to_le32(~crc32_le(~0, vif->cfg.ssid,
+							vif->cfg.ssid_len));
+	}
+
+	cmd->byte_cnt = cpu_to_le16((u16)beacon->len);
+
+	flags |= iwl_mld_get_rate_flags(mld, info, vif, link,
+					ctx->def.chan->band);
+
+	cmd->flags = cpu_to_le16(flags);
+
+	if (vif->type == NL80211_IFTYPE_AP) {
+		iwl_mld_set_tim_idx(mld, &cmd->tim_idx,
+				    beacon->data, beacon->len);
+
+		cmd->btwt_offset =
+			cpu_to_le32(iwl_find_ie_offset(beacon->data,
+						       WLAN_EID_S1G_TWT,
+						       beacon->len));
+	}
+
+	cmd->csa_offset =
+		cpu_to_le32(iwl_find_ie_offset(beacon->data,
+					       WLAN_EID_CHANNEL_SWITCH,
+					       beacon->len));
+	cmd->ecsa_offset =
+		cpu_to_le32(iwl_find_ie_offset(beacon->data,
+					       WLAN_EID_EXT_CHANSWITCH_ANN,
+					       beacon->len));
+
+	return 0;
+}
+
+/* The beacon template for the AP/GO/IBSS has changed and needs update */
+int iwl_mld_update_beacon_template(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mac_beacon_cmd cmd = {};
+	struct sk_buff *beacon;
+	int ret;
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+#endif
+
+	WARN_ON(vif->type != NL80211_IFTYPE_AP &&
+		vif->type != NL80211_IFTYPE_ADHOC);
+
+	if (IWL_MLD_NON_TRANSMITTING_AP)
+		return 0;
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	if (mld_vif->beacon_inject_active) {
+		IWL_DEBUG_INFO(mld,
+			       "Can't update template, beacon injection's active\n");
+		return -EBUSY;
+	}
+
+#endif
+	beacon = ieee80211_beacon_get_template(mld->hw, vif, NULL,
+					       link_conf->link_id);
+	if (!beacon)
+		return -ENOMEM;
+
+	ret = iwl_mld_fill_beacon_template_cmd(mld, vif, beacon, &cmd,
+					       link_conf);
+
+	if (!ret)
+		ret = iwl_mld_send_beacon_template_cmd(mld, beacon, &cmd);
+
+	dev_kfree_skb(beacon);
+
+	return ret;
+}
+
+void iwl_mld_free_ap_early_key(struct iwl_mld *mld,
+			       struct ieee80211_key_conf *key,
+			       struct iwl_mld_vif *mld_vif)
+{
+	struct iwl_mld_link *link;
+
+	if (WARN_ON(key->link_id < 0))
+		return;
+
+	link = iwl_mld_link_dereference_check(mld_vif, key->link_id);
+	if (WARN_ON(!link))
+		return;
+
+	for (int i = 0; i < ARRAY_SIZE(link->ap_early_keys); i++) {
+		if (link->ap_early_keys[i] != key)
+			continue;
+		/* Those weren't sent to FW, so should be marked as INVALID */
+		if (WARN_ON(key->hw_key_idx != STA_KEY_IDX_INVALID))
+			key->hw_key_idx = STA_KEY_IDX_INVALID;
+		link->ap_early_keys[i] = NULL;
+	}
+}
+
+int iwl_mld_store_ap_early_key(struct iwl_mld *mld,
+			       struct ieee80211_key_conf *key,
+			       struct iwl_mld_vif *mld_vif)
+{
+	struct iwl_mld_link *link;
+
+	if (WARN_ON(key->link_id < 0))
+		return -EINVAL;
+
+	link = iwl_mld_link_dereference_check(mld_vif, key->link_id);
+	if (WARN_ON(!link))
+		return -EINVAL;
+
+	for (int i = 0; i < ARRAY_SIZE(link->ap_early_keys); i++) {
+		if (!link->ap_early_keys[i]) {
+			link->ap_early_keys[i] = key;
+			return 0;
+		}
+	}
+
+	return -ENOSPC;
+}
+
+static int iwl_mld_send_ap_early_keys(struct iwl_mld *mld,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	int ret = 0;
+
+	if (WARN_ON(!link))
+		return -EINVAL;
+
+	for (int i = 0; i < ARRAY_SIZE(mld_link->ap_early_keys); i++) {
+		struct ieee80211_key_conf *key = mld_link->ap_early_keys[i];
+
+		if (!key)
+			continue;
+
+		mld_link->ap_early_keys[i] = NULL;
+
+		ret = iwl_mld_add_key(mld, vif, NULL, key);
+		if (ret)
+			break;
+	}
+	return ret;
+}
+
+int iwl_mld_start_ap_ibss(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_chanctx_conf *ctx;
+	int ret;
+
+	if (vif->type == NL80211_IFTYPE_AP)
+		iwl_mld_send_ap_tx_power_constraint_cmd(mld, vif, link);
+
+	ret = iwl_mld_update_beacon_template(mld, vif, link);
+	if (ret)
+		return ret;
+
+	/* the link should be already activated when assigning chan context,
+	 * and LINK_CONTEXT_MODIFY_EHT_PARAMS is deprecated
+	 */
+	ret = iwl_mld_change_link_in_fw(mld, link,
+					LINK_CONTEXT_MODIFY_ALL &
+					~(LINK_CONTEXT_MODIFY_ACTIVE |
+					  LINK_CONTEXT_MODIFY_EHT_PARAMS));
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_add_mcast_sta(mld, vif, link);
+	if (ret)
+		return ret;
+
+	mld_vif->ap_ibss_active = true;
+
+	if (vif->p2p && mld->p2p_device_vif) {
+		ret = iwl_mld_mac_fw_action(mld, mld->p2p_device_vif,
+					    FW_CTXT_ACTION_MODIFY);
+		if (ret) {
+			mld_vif->ap_ibss_active = false;
+			goto rm_mcast;
+		}
+	}
+
+	ret = iwl_mld_add_bcast_sta(mld, vif, link);
+	if (ret)
+		goto update_p2p_dev;
+
+	/* Those keys were configured by the upper layers before starting the
+	 * AP. Now that it is started and the bcast and mcast sta were added to
+	 * the FW, we can add the keys too.
+	 */
+	ret = iwl_mld_send_ap_early_keys(mld, vif, link);
+	if (ret)
+		goto rm_bcast;
+
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_AP)
+		iwl_mld_vif_update_low_latency(mld, vif, true,
+					       LOW_LATENCY_VIF_TYPE);
+
+	/* When the channel context was added, the link is not yet active, so
+	 * min_def is always used. Update the PHY again here in case def should
+	 * actually be used.
+	 */
+	ctx = wiphy_dereference(mld->wiphy, link->chanctx_conf);
+	iwl_mld_update_phy_chandef(mld, ctx);
+
+	return 0;
+rm_bcast:
+	iwl_mld_remove_bcast_sta(mld, vif, link);
+update_p2p_dev:
+	mld_vif->ap_ibss_active = false;
+	if (vif->p2p && mld->p2p_device_vif)
+		iwl_mld_mac_fw_action(mld, mld->p2p_device_vif,
+				      FW_CTXT_ACTION_MODIFY);
+rm_mcast:
+	iwl_mld_remove_mcast_sta(mld, vif, link);
+	return ret;
+}
+
+void iwl_mld_stop_ap_ibss(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	mld_vif->ap_ibss_active = false;
+
+	if (vif->p2p && mld->p2p_device_vif)
+		iwl_mld_mac_fw_action(mld, mld->p2p_device_vif,
+				      FW_CTXT_ACTION_MODIFY);
+
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_AP)
+		iwl_mld_vif_update_low_latency(mld, vif, false,
+					       LOW_LATENCY_VIF_TYPE);
+
+	iwl_mld_remove_bcast_sta(mld, vif, link);
+
+	iwl_mld_remove_mcast_sta(mld, vif, link);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/ap.h b/sys/contrib/dev/iwlwifi/mld/ap.h
new file mode 100644
index 000000000000..4a6f52b9552d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/ap.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_ap_h__
+#define __iwl_ap_h__
+
+#include "mld.h"
+#include "iface.h"
+
+#include "fw/api/tx.h"
+
+int iwl_mld_update_beacon_template(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_bss_conf *link_conf);
+
+int iwl_mld_start_ap_ibss(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link);
+
+void iwl_mld_stop_ap_ibss(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link);
+
+int iwl_mld_store_ap_early_key(struct iwl_mld *mld,
+			       struct ieee80211_key_conf *key,
+			       struct iwl_mld_vif *mld_vif);
+
+void iwl_mld_free_ap_early_key(struct iwl_mld *mld,
+			       struct ieee80211_key_conf *key,
+			       struct iwl_mld_vif *mld_vif);
+
+u8 iwl_mld_get_rate_flags(struct iwl_mld *mld,
+			  struct ieee80211_tx_info *info,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link,
+			  enum nl80211_band band);
+
+void iwl_mld_set_tim_idx(struct iwl_mld *mld, __le32 *tim_index,
+			 u8 *beacon, u32 frame_size);
+
+int iwl_mld_send_beacon_template_cmd(struct iwl_mld *mld,
+				     struct sk_buff *beacon,
+				     struct iwl_mac_beacon_cmd *cmd);
+
+#endif /* __iwl_ap_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/coex.c b/sys/contrib/dev/iwlwifi/mld/coex.c
new file mode 100644
index 000000000000..5f262bd43f21
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/coex.c
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include "fw/api/coex.h"
+
+#include "coex.h"
+#include "mld.h"
+#include "hcmd.h"
+#include "mlo.h"
+
+int iwl_mld_send_bt_init_conf(struct iwl_mld *mld)
+{
+	struct iwl_bt_coex_cmd cmd = {
+		.mode = cpu_to_le32(BT_COEX_NW),
+		.enabled_modules = cpu_to_le32(BT_COEX_MPLUT_ENABLED |
+					       BT_COEX_HIGH_BAND_RET),
+	};
+
+	return iwl_mld_send_cmd_pdu(mld, BT_CONFIG, &cmd);
+}
+
+void iwl_mld_handle_bt_coex_notif(struct iwl_mld *mld,
+				  struct iwl_rx_packet *pkt)
+{
+	const struct iwl_bt_coex_profile_notif *notif = (void *)pkt->data;
+	const struct iwl_bt_coex_profile_notif zero_notif = {};
+	/* zeroed structure means that BT is OFF */
+	bool bt_is_active = memcmp(notif, &zero_notif, sizeof(*notif));
+
+	if (bt_is_active == mld->bt_is_active)
+		return;
+
+	IWL_DEBUG_INFO(mld, "BT was turned %s\n", bt_is_active ? "ON" : "OFF");
+
+	mld->bt_is_active = bt_is_active;
+
+	iwl_mld_emlsr_check_bt(mld);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/coex.h b/sys/contrib/dev/iwlwifi/mld/coex.h
new file mode 100644
index 000000000000..a77c5dc9613c
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/coex.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_coex_h__
+#define __iwl_mld_coex_h__
+
+#include "mld.h"
+
+int iwl_mld_send_bt_init_conf(struct iwl_mld *mld);
+
+void iwl_mld_handle_bt_coex_notif(struct iwl_mld *mld,
+				  struct iwl_rx_packet *pkt);
+
+#endif /* __iwl_mld_coex_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/constants.h b/sys/contrib/dev/iwlwifi/mld/constants.h
new file mode 100644
index 000000000000..49accf96f44b
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/constants.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_constants_h__
+#define __iwl_mld_constants_h__
+
+#define IWL_MLD_MISSED_BEACONS_SINCE_RX_THOLD			4
+#define IWL_MLD_MISSED_BEACONS_THRESHOLD			8
+#define IWL_MLD_MISSED_BEACONS_THRESHOLD_LONG			19
+#define IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS		5
+#define IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH			15
+#define IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED	11
+#define IWL_MLD_LOW_RSSI_MLO_SCAN_THRESH			-72
+
+#define IWL_MLD_DEFAULT_PS_TX_DATA_TIMEOUT	(100 * USEC_PER_MSEC)
+#define IWL_MLD_DEFAULT_PS_RX_DATA_TIMEOUT	(100 * USEC_PER_MSEC)
+#define IWL_MLD_WOWLAN_PS_TX_DATA_TIMEOUT	(10 * USEC_PER_MSEC)
+#define IWL_MLD_WOWLAN_PS_RX_DATA_TIMEOUT	(10 * USEC_PER_MSEC)
+#define IWL_MLD_SHORT_PS_TX_DATA_TIMEOUT	(2 * 1024) /* defined in TU */
+#define IWL_MLD_SHORT_PS_RX_DATA_TIMEOUT	(40 * 1024) /* defined in TU */
+
+#define IWL_MLD_UAPSD_RX_DATA_TIMEOUT		(50 * USEC_PER_MSEC)
+#define IWL_MLD_UAPSD_TX_DATA_TIMEOUT		(50 * USEC_PER_MSEC)
+
+#define IWL_MLD_PS_SNOOZE_INTERVAL		25
+#define IWL_MLD_PS_SNOOZE_INTERVAL		25
+#define IWL_MLD_PS_SNOOZE_WINDOW		50
+
+#define IWL_MLD_PS_SNOOZE_HEAVY_TX_THLD_PACKETS	30
+#define IWL_MLD_PS_SNOOZE_HEAVY_RX_THLD_PACKETS	20
+
+#define IWL_MLD_PS_HEAVY_TX_THLD_PERCENT	50
+#define IWL_MLD_PS_HEAVY_RX_THLD_PERCENT	50
+#define IWL_MLD_PS_HEAVY_TX_THLD_PACKETS	20
+#define IWL_MLD_PS_HEAVY_RX_THLD_PACKETS	8
+
+#define IWL_MLD_TRIGGER_LINK_SEL_TIME_SEC	30
+#define IWL_MLD_SCAN_EXPIRE_TIME_SEC		20
+
+#define IWL_MLD_TPT_COUNT_WINDOW (5 * HZ)
+
+#define IWL_MLD_DIS_RANDOM_FW_ID                false
+#define IWL_MLD_D3_DEBUG                        false
+#define IWL_MLD_NON_TRANSMITTING_AP	        false
+#define IWL_MLD_6GHZ_PASSIVE_SCAN_TIMEOUT       3000 /* in seconds */
+#define IWL_MLD_6GHZ_PASSIVE_SCAN_ASSOC_TIMEOUT 60   /* in seconds */
+#define IWL_MLD_CONN_LISTEN_INTERVAL		10
+#define IWL_MLD_ADAPTIVE_DWELL_NUM_APS_OVERRIDE 0
+#define IWL_MLD_AUTO_EML_ENABLE			true
+
+#define IWL_MLD_HIGH_RSSI_THRESH_20MHZ		-67
+#define IWL_MLD_LOW_RSSI_THRESH_20MHZ		-72
+#define IWL_MLD_HIGH_RSSI_THRESH_40MHZ		-64
+#define IWL_MLD_LOW_RSSI_THRESH_40MHZ		-72
+#define IWL_MLD_HIGH_RSSI_THRESH_80MHZ		-61
+#define IWL_MLD_LOW_RSSI_THRESH_80MHZ		-72
+#define IWL_MLD_HIGH_RSSI_THRESH_160MHZ		-58
+#define IWL_MLD_LOW_RSSI_THRESH_160MHZ		-72
+
+#define IWL_MLD_ENTER_EMLSR_TPT_THRESH		400
+#define IWL_MLD_EXIT_EMLSR_CHAN_LOAD		2 /* in percentage */
+
+#define IWL_MLD_FTM_INITIATOR_ALGO		IWL_TOF_ALGO_TYPE_MAX_LIKE
+#define IWL_MLD_FTM_INITIATOR_DYNACK		true
+#define IWL_MLD_FTM_LMR_FEEDBACK_TERMINATE	false
+#define IWL_MLD_FTM_TEST_INCORRECT_SAC		false
+#define IWL_MLD_FTM_R2I_MAX_REP			7
+#define IWL_MLD_FTM_I2R_MAX_REP			7
+#define IWL_MLD_FTM_R2I_MAX_STS			1
+#define IWL_MLD_FTM_I2R_MAX_STS			1
+#define IWL_MLD_FTM_R2I_MAX_TOTAL_LTF		3
+#define IWL_MLD_FTM_I2R_MAX_TOTAL_LTF		3
+#define IWL_MLD_FTM_RESP_NDP_SUPPORT		true
+#define IWL_MLD_FTM_RESP_LMR_FEEDBACK_SUPPORT	true
+#define IWL_MLD_FTM_NON_TB_MIN_TIME_BETWEEN_MSR	7
+#define IWL_MLD_FTM_NON_TB_MAX_TIME_BETWEEN_MSR	1000
+
+#endif /* __iwl_mld_constants_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/d3.c b/sys/contrib/dev/iwlwifi/mld/d3.c
new file mode 100644
index 000000000000..ed0a0f76f1c5
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/d3.c
@@ -0,0 +1,1906 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include "mld.h"
+
+#include "d3.h"
+#include "power.h"
+#include "hcmd.h"
+#include "iface.h"
+#include "mcc.h"
+#include "sta.h"
+#include "mlo.h"
+
+#include "fw/api/d3.h"
+#include "fw/api/offload.h"
+#include "fw/api/sta.h"
+#include "fw/dbg.h"
+
+#include <net/ipv6.h>
+#include <net/addrconf.h>
+#include <linux/bitops.h>
+
+/**
+ * enum iwl_mld_d3_notif - d3 notifications
+ * @IWL_D3_NOTIF_WOWLAN_INFO: WOWLAN_INFO_NOTIF is expected/was received
+ * @IWL_D3_NOTIF_WOWLAN_WAKE_PKT: WOWLAN_WAKE_PKT_NOTIF is expected/was received
+ * @IWL_D3_NOTIF_PROT_OFFLOAD: PROT_OFFLOAD_NOTIF is expected/was received
+ * @IWL_D3_ND_MATCH_INFO: OFFLOAD_MATCH_INFO_NOTIF is expected/was received
+ * @IWL_D3_NOTIF_D3_END_NOTIF: D3_END_NOTIF is expected/was received
+ */
+enum iwl_mld_d3_notif {
+	IWL_D3_NOTIF_WOWLAN_INFO =	BIT(0),
+	IWL_D3_NOTIF_WOWLAN_WAKE_PKT =	BIT(1),
+	IWL_D3_NOTIF_PROT_OFFLOAD =	BIT(2),
+	IWL_D3_ND_MATCH_INFO      =     BIT(3),
+	IWL_D3_NOTIF_D3_END_NOTIF =	BIT(4)
+};
+
+struct iwl_mld_resume_key_iter_data {
+	struct iwl_mld *mld;
+	struct iwl_mld_wowlan_status *wowlan_status;
+	u32 num_keys, gtk_cipher, igtk_cipher, bigtk_cipher;
+	bool unhandled_cipher;
+};
+
+struct iwl_mld_suspend_key_iter_data {
+	struct iwl_wowlan_rsc_tsc_params_cmd *rsc;
+	bool have_rsc;
+	int gtks;
+	int found_gtk_idx[4];
+	__le32 gtk_cipher;
+	__le32 igtk_cipher;
+	__le32 bigtk_cipher;
+};
+
+struct iwl_mld_mcast_key_data {
+	u8 key[WOWLAN_KEY_MAX_SIZE];
+	u8 len;
+	u8 flags;
+	u8 id;
+	union {
+		struct {
+			struct ieee80211_key_seq aes_seq[IWL_MAX_TID_COUNT];
+			struct ieee80211_key_seq tkip_seq[IWL_MAX_TID_COUNT];
+		} gtk;
+		struct {
+			struct ieee80211_key_seq cmac_gmac_seq;
+		} igtk_bigtk;
+	};
+
+};
+
+/**
+ * struct iwl_mld_wowlan_status - contains wowlan status data from
+ * all wowlan notifications
+ * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason
+ * @replay_ctr: GTK rekey replay counter
+ * @pattern_number: number of the matched patterns on packets
+ * @last_qos_seq: QoS sequence counter of offloaded tid
+ * @num_of_gtk_rekeys: number of GTK rekeys during D3
+ * @tid_offloaded_tx: tid used by the firmware to transmit data packets
+ *	while in wowlan
+ * @wake_packet: wakeup packet received
+ * @wake_packet_length: wake packet length
+ * @wake_packet_bufsize: wake packet bufsize
+ * @gtk: data of the last two used gtk's by the FW upon resume
+ * @igtk: data of the last used igtk by the FW upon resume
+ * @bigtk: data of the last two used gtk's by the FW upon resume
+ * @ptk: last seq numbers per tid passed by the FW,
+ *	holds both in tkip and aes formats
+ */
+struct iwl_mld_wowlan_status {
+	u32 wakeup_reasons;
+	u64 replay_ctr;
+	u16 pattern_number;
+	u16 last_qos_seq;
+	u32 num_of_gtk_rekeys;
+	u8 tid_offloaded_tx;
+	u8 *wake_packet;
+	u32 wake_packet_length;
+	u32 wake_packet_bufsize;
+	struct iwl_mld_mcast_key_data gtk[WOWLAN_GTK_KEYS_NUM];
+	struct iwl_mld_mcast_key_data igtk;
+	struct iwl_mld_mcast_key_data bigtk[WOWLAN_BIGTK_KEYS_NUM];
+	struct {
+		struct ieee80211_key_seq aes_seq[IWL_MAX_TID_COUNT];
+		struct ieee80211_key_seq tkip_seq[IWL_MAX_TID_COUNT];
+
+	} ptk;
+};
+
+#define NETDETECT_QUERY_BUF_LEN \
+	(sizeof(struct iwl_scan_offload_profile_match) * \
+	 IWL_SCAN_MAX_PROFILES_V2)
+
+/**
+ * struct iwl_mld_netdetect_res - contains netdetect results from
+ * match_info_notif
+ * @matched_profiles: bitmap of matched profiles, referencing the
+ *	matches passed in the scan offload request
+ * @matches: array of match information, one for each match
+ */
+struct iwl_mld_netdetect_res {
+	u32 matched_profiles;
+	u8 matches[NETDETECT_QUERY_BUF_LEN];
+};
+
+/**
+ * struct iwl_mld_resume_data - d3 resume flow data
+ * @notifs_expected: bitmap of expected notifications from fw,
+ *	see &enum iwl_mld_d3_notif
+ * @notifs_received: bitmap of received notifications from fw,
+ *	see &enum iwl_mld_d3_notif
+ * @d3_end_flags: bitmap of flags from d3_end_notif
+ * @notif_handling_err: error handling one of the resume notifications
+ * @wowlan_status: wowlan status data from all wowlan notifications
+ * @netdetect_res: contains netdetect results from match_info_notif
+ */
+struct iwl_mld_resume_data {
+	u32 notifs_expected;
+	u32 notifs_received;
+	u32 d3_end_flags;
+	bool notif_handling_err;
+	struct iwl_mld_wowlan_status *wowlan_status;
+	struct iwl_mld_netdetect_res *netdetect_res;
+};
+
+#define IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT \
+	(IWL_WOWLAN_WAKEUP_BY_MAGIC_PACKET | \
+	IWL_WOWLAN_WAKEUP_BY_PATTERN | \
+	IWL_WAKEUP_BY_PATTERN_IPV4_TCP_SYN |\
+	IWL_WAKEUP_BY_PATTERN_IPV4_TCP_SYN_WILDCARD |\
+	IWL_WAKEUP_BY_PATTERN_IPV6_TCP_SYN |\
+	IWL_WAKEUP_BY_PATTERN_IPV6_TCP_SYN_WILDCARD)
+
+#define IWL_WOWLAN_OFFLOAD_TID 0
+
+void iwl_mld_set_rekey_data(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif,
+			    struct cfg80211_gtk_rekey_data *data)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_wowlan_data *wowlan_data = &mld_vif->wowlan_data;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	wowlan_data->rekey_data.kek_len = data->kek_len;
+	wowlan_data->rekey_data.kck_len = data->kck_len;
+	memcpy(wowlan_data->rekey_data.kek, data->kek, data->kek_len);
+	memcpy(wowlan_data->rekey_data.kck, data->kck, data->kck_len);
+	wowlan_data->rekey_data.akm = data->akm & 0xFF;
+	wowlan_data->rekey_data.replay_ctr =
+		cpu_to_le64(be64_to_cpup((const __be64 *)data->replay_ctr));
+	wowlan_data->rekey_data.valid = true;
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+void iwl_mld_ipv6_addr_change(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct inet6_dev *idev)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_wowlan_data *wowlan_data = &mld_vif->wowlan_data;
+	struct inet6_ifaddr *ifa;
+	int idx = 0;
+
+	memset(wowlan_data->tentative_addrs, 0,
+	       sizeof(wowlan_data->tentative_addrs));
+
+	read_lock_bh(&idev->lock);
+	list_for_each_entry(ifa, &idev->addr_list, if_list) {
+		wowlan_data->target_ipv6_addrs[idx] = ifa->addr;
+		if (ifa->flags & IFA_F_TENTATIVE)
+			__set_bit(idx, wowlan_data->tentative_addrs);
+		idx++;
+		if (idx >= IWL_PROTO_OFFLOAD_NUM_IPV6_ADDRS_MAX)
+			break;
+	}
+	read_unlock_bh(&idev->lock);
+
+	wowlan_data->num_target_ipv6_addrs = idx;
+}
+#endif
+
+static int
+iwl_mld_netdetect_config(struct iwl_mld *mld,
+			 struct ieee80211_vif *vif,
+			 const struct cfg80211_wowlan *wowlan)
+{
+	int ret;
+	struct cfg80211_sched_scan_request *netdetect_cfg =
+		wowlan->nd_config;
+	struct ieee80211_scan_ies ies = {};
+
+	ret = iwl_mld_scan_stop(mld, IWL_MLD_SCAN_SCHED, true);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_sched_scan_start(mld, vif, netdetect_cfg, &ies,
+				       IWL_MLD_SCAN_NETDETECT);
+	return ret;
+}
+
+static void
+iwl_mld_le64_to_tkip_seq(__le64 le_pn, struct ieee80211_key_seq *seq)
+{
+	u64 pn = le64_to_cpu(le_pn);
+
+	seq->tkip.iv16 = (u16)pn;
+	seq->tkip.iv32 = (u32)(pn >> 16);
+}
+
+static void
+iwl_mld_le64_to_aes_seq(__le64 le_pn, struct ieee80211_key_seq *seq)
+{
+	u64 pn = le64_to_cpu(le_pn);
+
+	seq->ccmp.pn[0] = pn >> 40;
+	seq->ccmp.pn[1] = pn >> 32;
+	seq->ccmp.pn[2] = pn >> 24;
+	seq->ccmp.pn[3] = pn >> 16;
+	seq->ccmp.pn[4] = pn >> 8;
+	seq->ccmp.pn[5] = pn;
+}
+
+static void
+iwl_mld_convert_gtk_resume_seq(struct iwl_mld_mcast_key_data *gtk_data,
+			       const struct iwl_wowlan_all_rsc_tsc_v5 *sc,
+			       int rsc_idx)
+{
+	struct ieee80211_key_seq *aes_seq = gtk_data->gtk.aes_seq;
+	struct ieee80211_key_seq *tkip_seq = gtk_data->gtk.tkip_seq;
+
+	if (rsc_idx >= ARRAY_SIZE(sc->mcast_rsc))
+		return;
+
+	/* We store both the TKIP and AES representations coming from the
+	 * FW because we decode the data from there before we iterate
+	 * the keys and know which type is used.
+	 */
+	for (int tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+		iwl_mld_le64_to_tkip_seq(sc->mcast_rsc[rsc_idx][tid],
+					 &tkip_seq[tid]);
+		iwl_mld_le64_to_aes_seq(sc->mcast_rsc[rsc_idx][tid],
+					&aes_seq[tid]);
+	}
+}
+
+static void
+iwl_mld_convert_gtk_resume_data(struct iwl_mld *mld,
+				struct iwl_mld_wowlan_status *wowlan_status,
+				const struct iwl_wowlan_gtk_status_v3 *gtk_data,
+				const struct iwl_wowlan_all_rsc_tsc_v5 *sc)
+{
+	int status_idx = 0;
+
+	BUILD_BUG_ON(sizeof(wowlan_status->gtk[0].key) <
+		     sizeof(gtk_data[0].key));
+	BUILD_BUG_ON(ARRAY_SIZE(wowlan_status->gtk) < WOWLAN_GTK_KEYS_NUM);
+
+	for (int notif_idx = 0; notif_idx < ARRAY_SIZE(wowlan_status->gtk);
+	     notif_idx++) {
+		int rsc_idx;
+
+		if (!(gtk_data[notif_idx].key_len))
+			continue;
+
+		wowlan_status->gtk[status_idx].len =
+			gtk_data[notif_idx].key_len;
+		wowlan_status->gtk[status_idx].flags =
+			gtk_data[notif_idx].key_flags;
+		wowlan_status->gtk[status_idx].id =
+			wowlan_status->gtk[status_idx].flags &
+			IWL_WOWLAN_GTK_IDX_MASK;
+		memcpy(wowlan_status->gtk[status_idx].key,
+		       gtk_data[notif_idx].key,
+		       sizeof(gtk_data[notif_idx].key));
+
+		/* The rsc for both gtk keys are stored in gtk[0]->sc->mcast_rsc
+		 * The gtk ids can be any two numbers between 0 and 3,
+		 * the id_map maps between the key id and the index in sc->mcast
+		 */
+		rsc_idx =
+			sc->mcast_key_id_map[wowlan_status->gtk[status_idx].id];
+		iwl_mld_convert_gtk_resume_seq(&wowlan_status->gtk[status_idx],
+					       sc, rsc_idx);
+
+		/* if it's as long as the TKIP encryption key, copy MIC key */
+		if (wowlan_status->gtk[status_idx].len ==
+		    NL80211_TKIP_DATA_OFFSET_TX_MIC_KEY)
+			memcpy(wowlan_status->gtk[status_idx].key +
+			       NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY,
+			       gtk_data[notif_idx].tkip_mic_key,
+			       sizeof(gtk_data[notif_idx].tkip_mic_key));
+		status_idx++;
+	}
+}
+
+static void
+iwl_mld_convert_ptk_resume_seq(struct iwl_mld *mld,
+			       struct iwl_mld_wowlan_status *wowlan_status,
+			       const struct iwl_wowlan_all_rsc_tsc_v5 *sc)
+{
+	struct ieee80211_key_seq *aes_seq = wowlan_status->ptk.aes_seq;
+	struct ieee80211_key_seq *tkip_seq = wowlan_status->ptk.tkip_seq;
+
+	BUILD_BUG_ON(ARRAY_SIZE(sc->ucast_rsc) != IWL_MAX_TID_COUNT);
+
+	for (int tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+		iwl_mld_le64_to_aes_seq(sc->ucast_rsc[tid], &aes_seq[tid]);
+		iwl_mld_le64_to_tkip_seq(sc->ucast_rsc[tid], &tkip_seq[tid]);
+	}
+}
+
+static void
+iwl_mld_convert_mcast_ipn(struct iwl_mld_mcast_key_data *key_status,
+			  const struct iwl_wowlan_igtk_status *key)
+{
+	struct ieee80211_key_seq *seq =
+		&key_status->igtk_bigtk.cmac_gmac_seq;
+	u8 ipn_len = ARRAY_SIZE(key->ipn);
+
+	BUILD_BUG_ON(ipn_len != ARRAY_SIZE(seq->aes_gmac.pn));
+	BUILD_BUG_ON(ipn_len != ARRAY_SIZE(seq->aes_cmac.pn));
+	BUILD_BUG_ON(offsetof(struct ieee80211_key_seq, aes_gmac) !=
+		     offsetof(struct ieee80211_key_seq, aes_cmac));
+
+	/* mac80211 expects big endian for memcmp() to work, convert.
+	 * We don't have the key cipher yet so copy to both to cmac and gmac
+	 */
+	for (int i = 0; i < ipn_len; i++) {
+		seq->aes_gmac.pn[i] = key->ipn[ipn_len - i - 1];
+		seq->aes_cmac.pn[i] = key->ipn[ipn_len - i - 1];
+	}
+}
+
+static void
+iwl_mld_convert_igtk_resume_data(struct iwl_mld_wowlan_status *wowlan_status,
+				 const struct iwl_wowlan_igtk_status *igtk)
+{
+	BUILD_BUG_ON(sizeof(wowlan_status->igtk.key) < sizeof(igtk->key));
+
+	if (!igtk->key_len)
+		return;
+
+	wowlan_status->igtk.len = igtk->key_len;
+	wowlan_status->igtk.flags = igtk->key_flags;
+	wowlan_status->igtk.id =
+		u32_get_bits(igtk->key_flags,
+			     IWL_WOWLAN_IGTK_BIGTK_IDX_MASK) +
+		WOWLAN_IGTK_MIN_INDEX;
+
+	memcpy(wowlan_status->igtk.key, igtk->key, sizeof(igtk->key));
+	iwl_mld_convert_mcast_ipn(&wowlan_status->igtk, igtk);
+}
+
+static void
+iwl_mld_convert_bigtk_resume_data(struct iwl_mld_wowlan_status *wowlan_status,
+				  const struct iwl_wowlan_igtk_status *bigtk)
+{
+	int status_idx = 0;
+
+	BUILD_BUG_ON(ARRAY_SIZE(wowlan_status->bigtk) < WOWLAN_BIGTK_KEYS_NUM);
+
+	for (int notif_idx = 0; notif_idx < WOWLAN_BIGTK_KEYS_NUM;
+	     notif_idx++) {
+		if (!bigtk[notif_idx].key_len)
+			continue;
+
+		wowlan_status->bigtk[status_idx].len = bigtk[notif_idx].key_len;
+		wowlan_status->bigtk[status_idx].flags =
+			bigtk[notif_idx].key_flags;
+		wowlan_status->bigtk[status_idx].id =
+			u32_get_bits(bigtk[notif_idx].key_flags,
+				     IWL_WOWLAN_IGTK_BIGTK_IDX_MASK)
+			+ WOWLAN_BIGTK_MIN_INDEX;
+
+		BUILD_BUG_ON(sizeof(wowlan_status->bigtk[status_idx].key) <
+			     sizeof(bigtk[notif_idx].key));
+		memcpy(wowlan_status->bigtk[status_idx].key,
+		       bigtk[notif_idx].key, sizeof(bigtk[notif_idx].key));
+		iwl_mld_convert_mcast_ipn(&wowlan_status->bigtk[status_idx],
+					  &bigtk[notif_idx]);
+		status_idx++;
+	}
+}
+
+static bool
+iwl_mld_handle_wowlan_info_notif(struct iwl_mld *mld,
+				 struct iwl_mld_wowlan_status *wowlan_status,
+				 struct iwl_rx_packet *pkt)
+{
+	const struct iwl_wowlan_info_notif *notif = (void *)pkt->data;
+	u32 expected_len, len = iwl_rx_packet_payload_len(pkt);
+
+	expected_len = sizeof(*notif);
+
+	if (IWL_FW_CHECK(mld, len < expected_len,
+			 "Invalid wowlan_info_notif (expected=%ud got=%ud)\n",
+			 expected_len, len))
+		return true;
+
+	if (IWL_FW_CHECK(mld, notif->tid_offloaded_tx != IWL_WOWLAN_OFFLOAD_TID,
+			 "Invalid tid_offloaded_tx %d\n",
+			 wowlan_status->tid_offloaded_tx))
+		return true;
+
+	iwl_mld_convert_gtk_resume_data(mld, wowlan_status, notif->gtk,
+					&notif->gtk[0].sc);
+	iwl_mld_convert_ptk_resume_seq(mld, wowlan_status, &notif->gtk[0].sc);
+	/* only one igtk is passed by FW */
+	iwl_mld_convert_igtk_resume_data(wowlan_status, &notif->igtk[0]);
+	iwl_mld_convert_bigtk_resume_data(wowlan_status, notif->bigtk);
+
+	wowlan_status->replay_ctr = le64_to_cpu(notif->replay_ctr);
+	wowlan_status->pattern_number = le16_to_cpu(notif->pattern_number);
+
+	wowlan_status->tid_offloaded_tx = notif->tid_offloaded_tx;
+	wowlan_status->last_qos_seq = le16_to_cpu(notif->qos_seq_ctr);
+	wowlan_status->num_of_gtk_rekeys =
+		le32_to_cpu(notif->num_of_gtk_rekeys);
+	wowlan_status->wakeup_reasons = le32_to_cpu(notif->wakeup_reasons);
+	return false;
+	/* TODO: mlo_links (task=MLO)*/
+}
+
+static bool
+iwl_mld_handle_wake_pkt_notif(struct iwl_mld *mld,
+			      struct iwl_mld_wowlan_status *wowlan_status,
+			      struct iwl_rx_packet *pkt)
+{
+	const struct iwl_wowlan_wake_pkt_notif *notif = (void *)pkt->data;
+	u32 actual_size, len = iwl_rx_packet_payload_len(pkt);
+	u32 expected_size = le32_to_cpu(notif->wake_packet_length);
+
+	if (IWL_FW_CHECK(mld, len < sizeof(*notif),
+			 "Invalid WoWLAN wake packet notification (expected size=%zu got=%u)\n",
+			 sizeof(*notif), len))
+		return true;
+
+	if (IWL_FW_CHECK(mld, !(wowlan_status->wakeup_reasons &
+				IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT),
+			 "Got wake packet but wakeup reason is %x\n",
+			 wowlan_status->wakeup_reasons))
+		return true;
+
+	actual_size = len - offsetof(struct iwl_wowlan_wake_pkt_notif,
+				     wake_packet);
+
+	/* actual_size got the padding from the notification, remove it. */
+	if (expected_size < actual_size)
+		actual_size = expected_size;
+	wowlan_status->wake_packet = kmemdup(notif->wake_packet, actual_size,
+					     GFP_ATOMIC);
+	if (!wowlan_status->wake_packet)
+		return true;
+
+	wowlan_status->wake_packet_length = expected_size;
+	wowlan_status->wake_packet_bufsize = actual_size;
+
+	return false;
+}
+
+static void
+iwl_mld_set_wake_packet(struct iwl_mld *mld,
+			struct ieee80211_vif *vif,
+			const struct iwl_mld_wowlan_status *wowlan_status,
+			struct cfg80211_wowlan_wakeup *wakeup,
+			struct sk_buff **_pkt)
+{
+	int pkt_bufsize = wowlan_status->wake_packet_bufsize;
+	int expected_pktlen = wowlan_status->wake_packet_length;
+	const u8 *pktdata = wowlan_status->wake_packet;
+	const struct ieee80211_hdr *hdr = (const void *)pktdata;
+	int truncated = expected_pktlen - pkt_bufsize;
+
+	if (ieee80211_is_data(hdr->frame_control)) {
+		int hdrlen = ieee80211_hdrlen(hdr->frame_control);
+		int ivlen = 0, icvlen = 4; /* also FCS */
+
+		struct sk_buff *pkt = alloc_skb(pkt_bufsize, GFP_KERNEL);
+		*_pkt = pkt;
+		if (!pkt)
+			return;
+
+		skb_put_data(pkt, pktdata, hdrlen);
+		pktdata += hdrlen;
+		pkt_bufsize -= hdrlen;
+
+		/* if truncated, FCS/ICV is (partially) gone */
+		if (truncated >= icvlen) {
+			truncated -= icvlen;
+			icvlen = 0;
+		} else {
+			icvlen -= truncated;
+			truncated = 0;
+		}
+
+		pkt_bufsize -= ivlen + icvlen;
+		pktdata += ivlen;
+
+		skb_put_data(pkt, pktdata, pkt_bufsize);
+
+		if (ieee80211_data_to_8023(pkt, vif->addr, vif->type))
+			return;
+		wakeup->packet = pkt->data;
+		wakeup->packet_present_len = pkt->len;
+		wakeup->packet_len = pkt->len - truncated;
+		wakeup->packet_80211 = false;
+	} else {
+		int fcslen = 4;
+
+		if (truncated >= 4) {
+			truncated -= 4;
+			fcslen = 0;
+		} else {
+			fcslen -= truncated;
+			truncated = 0;
+		}
+		pkt_bufsize -= fcslen;
+		wakeup->packet = wowlan_status->wake_packet;
+		wakeup->packet_present_len = pkt_bufsize;
+		wakeup->packet_len = expected_pktlen - truncated;
+		wakeup->packet_80211 = true;
+	}
+}
+
+static void
+iwl_mld_report_wowlan_wakeup(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct iwl_mld_wowlan_status *wowlan_status)
+{
+	struct sk_buff *pkt = NULL;
+	struct cfg80211_wowlan_wakeup wakeup = {
+		.pattern_idx = -1,
+	};
+	u32 reasons = wowlan_status->wakeup_reasons;
+
+	if (reasons == IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS) {
+		ieee80211_report_wowlan_wakeup(vif, NULL, GFP_KERNEL);
+		return;
+	}
+
+	pm_wakeup_event(mld->dev, 0);
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_MAGIC_PACKET)
+		wakeup.magic_pkt = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_PATTERN)
+		wakeup.pattern_idx =
+			wowlan_status->pattern_number;
+
+	if (reasons & (IWL_WOWLAN_WAKEUP_BY_DISCONNECTION_ON_MISSED_BEACON |
+		       IWL_WOWLAN_WAKEUP_BY_DISCONNECTION_ON_DEAUTH |
+		       IWL_WOWLAN_WAKEUP_BY_GTK_REKEY_FAILURE))
+		wakeup.disconnect = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_GTK_REKEY_FAILURE)
+		wakeup.gtk_rekey_failure = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_RFKILL_DEASSERTED)
+		wakeup.rfkill_release = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_EAPOL_REQUEST)
+		wakeup.eap_identity_req = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_FOUR_WAY_HANDSHAKE)
+		wakeup.four_way_handshake = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_REM_WAKE_LINK_LOSS)
+		wakeup.tcp_connlost = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_REM_WAKE_SIGNATURE_TABLE)
+		wakeup.tcp_nomoretokens = true;
+
+	if (reasons & IWL_WOWLAN_WAKEUP_BY_REM_WAKE_WAKEUP_PACKET)
+		wakeup.tcp_match = true;
+
+	if (reasons & IWL_WAKEUP_BY_11W_UNPROTECTED_DEAUTH_OR_DISASSOC)
+		wakeup.unprot_deauth_disassoc = true;
+
+	if (wowlan_status->wake_packet)
+		iwl_mld_set_wake_packet(mld, vif, wowlan_status, &wakeup, &pkt);
+
+	ieee80211_report_wowlan_wakeup(vif, &wakeup, GFP_KERNEL);
+	kfree_skb(pkt);
+}
+
+static void
+iwl_mld_set_key_rx_seq_tids(struct ieee80211_key_conf *key,
+			    struct ieee80211_key_seq *seq)
+{
+	int tid;
+
+	for (tid = 0; tid < IWL_MAX_TID_COUNT; tid++)
+		ieee80211_set_key_rx_seq(key, tid, &seq[tid]);
+}
+
+static void
+iwl_mld_set_key_rx_seq(struct ieee80211_key_conf *key,
+		       struct iwl_mld_mcast_key_data *key_data)
+{
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_CCMP:
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_GCMP_256:
+		iwl_mld_set_key_rx_seq_tids(key,
+					    key_data->gtk.aes_seq);
+		break;
+	case WLAN_CIPHER_SUITE_TKIP:
+		iwl_mld_set_key_rx_seq_tids(key,
+					    key_data->gtk.tkip_seq);
+		break;
+	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+		/* igtk/bigtk ciphers*/
+		ieee80211_set_key_rx_seq(key, 0,
+					 &key_data->igtk_bigtk.cmac_gmac_seq);
+		break;
+	default:
+		WARN_ON(1);
+	}
+}
+
+static void
+iwl_mld_update_ptk_rx_seq(struct iwl_mld *mld,
+			  struct iwl_mld_wowlan_status *wowlan_status,
+			  struct ieee80211_sta *sta,
+			  struct ieee80211_key_conf *key,
+			  bool is_tkip)
+{
+	struct iwl_mld_sta *mld_sta =
+		iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_ptk_pn *mld_ptk_pn =
+		wiphy_dereference(mld->wiphy,
+				  mld_sta->ptk_pn[key->keyidx]);
+
+	iwl_mld_set_key_rx_seq_tids(key, is_tkip ?
+				    wowlan_status->ptk.tkip_seq :
+				    wowlan_status->ptk.aes_seq);
+	if (is_tkip)
+		return;
+
+	if (WARN_ON(!mld_ptk_pn))
+		return;
+
+	for (int tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+		for (int i = 1; i < mld->trans->info.num_rxqs; i++)
+			memcpy(mld_ptk_pn->q[i].pn[tid],
+			       wowlan_status->ptk.aes_seq[tid].ccmp.pn,
+			       IEEE80211_CCMP_PN_LEN);
+	}
+}
+
+static void
+iwl_mld_resume_keys_iter(struct ieee80211_hw *hw,
+			 struct ieee80211_vif *vif,
+			 struct ieee80211_sta *sta,
+			 struct ieee80211_key_conf *key,
+			 void *_data)
+{
+	struct iwl_mld_resume_key_iter_data *data = _data;
+	struct iwl_mld_wowlan_status *wowlan_status = data->wowlan_status;
+	u8 status_idx;
+
+	/* TODO: check key link id (task=MLO) */
+	if (data->unhandled_cipher)
+		return;
+
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_WEP40:
+	case WLAN_CIPHER_SUITE_WEP104:
+		/* ignore WEP completely, nothing to do */
+		return;
+	case WLAN_CIPHER_SUITE_CCMP:
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_GCMP_256:
+	case WLAN_CIPHER_SUITE_TKIP:
+		if (sta) {
+			iwl_mld_update_ptk_rx_seq(data->mld, wowlan_status,
+						  sta, key,
+						  key->cipher ==
+						  WLAN_CIPHER_SUITE_TKIP);
+			return;
+		}
+
+		if (WARN_ON(data->gtk_cipher &&
+			    data->gtk_cipher != key->cipher))
+			return;
+
+		data->gtk_cipher = key->cipher;
+		status_idx = key->keyidx == wowlan_status->gtk[1].id;
+		iwl_mld_set_key_rx_seq(key, &wowlan_status->gtk[status_idx]);
+		break;
+	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+		if (key->keyidx == 4 || key->keyidx == 5) {
+			if (WARN_ON(data->igtk_cipher &&
+				    data->igtk_cipher != key->cipher))
+				return;
+
+			data->igtk_cipher = key->cipher;
+			if (key->keyidx == wowlan_status->igtk.id)
+				iwl_mld_set_key_rx_seq(key, &wowlan_status->igtk);
+		}
+		if (key->keyidx == 6 || key->keyidx == 7) {
+			if (WARN_ON(data->bigtk_cipher &&
+				    data->bigtk_cipher != key->cipher))
+				return;
+
+			data->bigtk_cipher = key->cipher;
+			status_idx = key->keyidx == wowlan_status->bigtk[1].id;
+			iwl_mld_set_key_rx_seq(key, &wowlan_status->bigtk[status_idx]);
+		}
+		break;
+	default:
+		data->unhandled_cipher = true;
+		return;
+	}
+	data->num_keys++;
+}
+
+static void
+iwl_mld_add_mcast_rekey(struct ieee80211_vif *vif,
+			struct iwl_mld *mld,
+			struct iwl_mld_mcast_key_data *key_data,
+			struct ieee80211_bss_conf *link_conf,
+			u32 cipher)
+{
+	struct ieee80211_key_conf *key_config;
+	struct {
+		struct ieee80211_key_conf conf;
+		u8 key[WOWLAN_KEY_MAX_SIZE];
+	} conf = {
+		.conf.cipher = cipher,
+		.conf.keyidx = key_data->id,
+	};
+	int link_id = vif->active_links ? __ffs(vif->active_links) : -1;
+	u8 key[WOWLAN_KEY_MAX_SIZE];
+
+	BUILD_BUG_ON(WLAN_KEY_LEN_CCMP != WLAN_KEY_LEN_GCMP);
+	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_CCMP);
+	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_GCMP_256);
+	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_TKIP);
+	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_BIP_GMAC_128);
+	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_BIP_GMAC_256);
+	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_AES_CMAC);
+	BUILD_BUG_ON(sizeof(conf.key) < sizeof(key_data->key));
+
+	if (!key_data->len)
+		return;
+
+	switch (cipher) {
+	case WLAN_CIPHER_SUITE_CCMP:
+	case WLAN_CIPHER_SUITE_GCMP:
+		conf.conf.keylen = WLAN_KEY_LEN_CCMP;
+		break;
+	case WLAN_CIPHER_SUITE_GCMP_256:
+		conf.conf.keylen = WLAN_KEY_LEN_GCMP_256;
+		break;
+	case WLAN_CIPHER_SUITE_TKIP:
+		conf.conf.keylen = WLAN_KEY_LEN_TKIP;
+		break;
+	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+		conf.conf.keylen = WLAN_KEY_LEN_BIP_GMAC_128;
+		break;
+	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+		conf.conf.keylen = WLAN_KEY_LEN_BIP_GMAC_256;
+		break;
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+		conf.conf.keylen = WLAN_KEY_LEN_AES_CMAC;
+		break;
+	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
+		conf.conf.keylen = WLAN_KEY_LEN_BIP_CMAC_256;
+		break;
+	default:
+		WARN_ON(1);
+	}
+
+	memcpy(conf.conf.key, key_data->key, conf.conf.keylen);
+
+	memcpy(key, key_data->key, sizeof(key_data->key));
+
+	key_config = ieee80211_gtk_rekey_add(vif, key_data->id, key,
+					     sizeof(key), link_id);
+	if (IS_ERR(key_config))
+		return;
+
+	iwl_mld_set_key_rx_seq(key_config, key_data);
+
+	/* The FW holds only one igtk so we keep track of the valid one */
+	if (key_config->keyidx == 4 || key_config->keyidx == 5) {
+		struct iwl_mld_link *mld_link =
+			iwl_mld_link_from_mac80211(link_conf);
+
+		/* If we had more than one rekey, mac80211 will tell us to
+		 * remove the old and add the new so we will update the IGTK in
+		 * drv_set_key
+		 */
+		if (mld_link->igtk && mld_link->igtk != key_config) {
+			/* mark the old IGTK as not in FW */
+			mld_link->igtk->hw_key_idx = STA_KEY_IDX_INVALID;
+			mld_link->igtk = key_config;
+		}
+	}
+
+	/* Also keep track of the new BIGTK */
+	if ((key_config->keyidx == 6 || key_config->keyidx == 7) &&
+	    vif->type == NL80211_IFTYPE_STATION) {
+		struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+		rcu_assign_pointer(mld_vif->bigtks[key_config->keyidx - 6], key_config);
+	}
+}
+
+static void
+iwl_mld_add_all_rekeys(struct ieee80211_vif *vif,
+		       struct iwl_mld_wowlan_status *wowlan_status,
+		       struct iwl_mld_resume_key_iter_data *key_iter_data,
+		       struct ieee80211_bss_conf *link_conf)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(wowlan_status->gtk); i++)
+		iwl_mld_add_mcast_rekey(vif, key_iter_data->mld,
+					&wowlan_status->gtk[i],
+					link_conf,
+					key_iter_data->gtk_cipher);
+
+	iwl_mld_add_mcast_rekey(vif, key_iter_data->mld,
+				&wowlan_status->igtk,
+				link_conf, key_iter_data->igtk_cipher);
+
+	for (i = 0; i < ARRAY_SIZE(wowlan_status->bigtk); i++)
+		iwl_mld_add_mcast_rekey(vif, key_iter_data->mld,
+					&wowlan_status->bigtk[i],
+					link_conf,
+					key_iter_data->bigtk_cipher);
+}
+
+static bool
+iwl_mld_update_sec_keys(struct iwl_mld *mld,
+			struct ieee80211_vif *vif,
+			struct iwl_mld_wowlan_status *wowlan_status)
+{
+	int link_id = vif->active_links ? __ffs(vif->active_links) : 0;
+	struct ieee80211_bss_conf *link_conf =
+		link_conf_dereference_protected(vif, link_id);
+	__be64 replay_ctr = cpu_to_be64(wowlan_status->replay_ctr);
+	struct iwl_mld_resume_key_iter_data key_iter_data = {
+		.mld = mld,
+		.wowlan_status = wowlan_status,
+	};
+
+	if (WARN_ON(!link_conf))
+		return false;
+
+	ieee80211_iter_keys(mld->hw, vif, iwl_mld_resume_keys_iter,
+			    &key_iter_data);
+
+	if (key_iter_data.unhandled_cipher)
+		return false;
+
+	IWL_DEBUG_WOWLAN(mld,
+			 "Number of installed keys: %d, Number of rekeys: %d\n",
+			 key_iter_data.num_keys,
+			 wowlan_status->num_of_gtk_rekeys);
+
+	if (!key_iter_data.num_keys || !wowlan_status->num_of_gtk_rekeys)
+		return true;
+
+	iwl_mld_add_all_rekeys(vif, wowlan_status, &key_iter_data,
+			       link_conf);
+
+	ieee80211_gtk_rekey_notify(vif, link_conf->bssid,
+				   (void *)&replay_ctr, GFP_KERNEL);
+	/* TODO: MLO rekey (task=MLO) */
+	return true;
+}
+
+static bool
+iwl_mld_process_wowlan_status(struct iwl_mld *mld,
+			      struct ieee80211_vif *vif,
+			      struct iwl_mld_wowlan_status *wowlan_status)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_sta *ap_sta = mld_vif->ap_sta;
+	struct iwl_mld_txq *mld_txq;
+
+	iwl_mld_report_wowlan_wakeup(mld, vif, wowlan_status);
+
+	if (WARN_ON(!ap_sta))
+		return false;
+
+	mld_txq =
+		iwl_mld_txq_from_mac80211(ap_sta->txq[wowlan_status->tid_offloaded_tx]);
+
+	/* Update the pointers of the Tx queue that may have moved during
+	 * suspend if the firmware sent frames.
+	 * The firmware stores last-used value, we store next value.
+	 */
+	WARN_ON(!mld_txq->status.allocated);
+	iwl_trans_set_q_ptrs(mld->trans, mld_txq->fw_id,
+			     (wowlan_status->last_qos_seq +
+			     0x10) >> 4);
+
+	if (!iwl_mld_update_sec_keys(mld, vif, wowlan_status))
+		return false;
+
+	if (wowlan_status->wakeup_reasons &
+	    (IWL_WOWLAN_WAKEUP_BY_DISCONNECTION_ON_MISSED_BEACON |
+	     IWL_WOWLAN_WAKEUP_BY_DISCONNECTION_ON_DEAUTH |
+	     IWL_WOWLAN_WAKEUP_BY_GTK_REKEY_FAILURE))
+		return false;
+
+	return true;
+}
+
+static bool
+iwl_mld_netdetect_match_info_handler(struct iwl_mld *mld,
+				     struct iwl_mld_resume_data *resume_data,
+				     struct iwl_rx_packet *pkt)
+{
+	struct iwl_mld_netdetect_res *results = resume_data->netdetect_res;
+	const struct iwl_scan_offload_match_info *notif = (void *)pkt->data;
+	u32 len = iwl_rx_packet_payload_len(pkt);
+
+	if (IWL_FW_CHECK(mld, !mld->netdetect,
+			 "Got scan match info notif when mld->netdetect==%d\n",
+			 mld->netdetect))
+		return true;
+
+	if (IWL_FW_CHECK(mld, len < sizeof(*notif),
+			 "Invalid scan offload match notif of length: %d\n",
+			 len))
+		return true;
+
+	if (IWL_FW_CHECK(mld, resume_data->wowlan_status->wakeup_reasons !=
+			 IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS,
+			 "Ignore scan match info: unexpected wakeup reason (expected=0x%x got=0x%x)\n",
+			 IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS,
+			 resume_data->wowlan_status->wakeup_reasons))
+		return true;
+
+	results->matched_profiles = le32_to_cpu(notif->matched_profiles);
+	IWL_DEBUG_WOWLAN(mld, "number of matched profiles=%u\n",
+			 results->matched_profiles);
+
+	if (results->matched_profiles)
+		memcpy(results->matches, notif->matches,
+		       NETDETECT_QUERY_BUF_LEN);
+
+	/* No scan should be active at this point */
+	mld->scan.status = 0;
+	memset(mld->scan.uid_status, 0, sizeof(mld->scan.uid_status));
+	return false;
+}
+
+static void
+iwl_mld_set_netdetect_info(struct iwl_mld *mld,
+			   const struct cfg80211_sched_scan_request *netdetect_cfg,
+			   struct cfg80211_wowlan_nd_info *netdetect_info,
+			   struct iwl_mld_netdetect_res *netdetect_res,
+			   unsigned long matched_profiles)
+{
+	int i;
+
+	for_each_set_bit(i, &matched_profiles, netdetect_cfg->n_match_sets) {
+		struct cfg80211_wowlan_nd_match *match;
+		int idx, j, n_channels = 0;
+		struct iwl_scan_offload_profile_match *matches =
+			(void *)netdetect_res->matches;
+
+		for (int k = 0; k < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN; k++)
+			n_channels +=
+				hweight8(matches[i].matching_channels[k]);
+		match = kzalloc(struct_size(match, channels, n_channels),
+				GFP_KERNEL);
+		if (!match)
+			return;
+
+		netdetect_info->matches[netdetect_info->n_matches] = match;
+		netdetect_info->n_matches++;
+
+		/* We inverted the order of the SSIDs in the scan
+		 * request, so invert the index here.
+		 */
+		idx = netdetect_cfg->n_match_sets - i - 1;
+		match->ssid.ssid_len =
+			netdetect_cfg->match_sets[idx].ssid.ssid_len;
+		memcpy(match->ssid.ssid,
+		       netdetect_cfg->match_sets[idx].ssid.ssid,
+		       match->ssid.ssid_len);
+
+		if (netdetect_cfg->n_channels < n_channels)
+			continue;
+
+		for_each_set_bit(j,
+				 (unsigned long *)&matches[i].matching_channels[0],
+				 sizeof(matches[i].matching_channels)) {
+			match->channels[match->n_channels] =
+				netdetect_cfg->channels[j]->center_freq;
+			match->n_channels++;
+		}
+	}
+}
+
+static void
+iwl_mld_process_netdetect_res(struct iwl_mld *mld,
+			      struct ieee80211_vif *vif,
+			      struct iwl_mld_resume_data *resume_data)
+{
+	struct cfg80211_wowlan_nd_info *netdetect_info = NULL;
+	const struct cfg80211_sched_scan_request *netdetect_cfg;
+	struct cfg80211_wowlan_wakeup wakeup = {
+		.pattern_idx = -1,
+	};
+	struct cfg80211_wowlan_wakeup *wakeup_report = &wakeup;
+	unsigned long matched_profiles;
+	u32 wakeup_reasons;
+	int n_matches;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld->wiphy->wowlan_config ||
+		    !mld->wiphy->wowlan_config->nd_config)) {
+		IWL_DEBUG_WOWLAN(mld,
+				 "Netdetect isn't configured on resume flow\n");
+		goto out;
+	}
+
+	netdetect_cfg = mld->wiphy->wowlan_config->nd_config;
+	wakeup_reasons = resume_data->wowlan_status->wakeup_reasons;
+
+	if (wakeup_reasons & IWL_WOWLAN_WAKEUP_BY_RFKILL_DEASSERTED)
+		wakeup.rfkill_release = true;
+
+	if (wakeup_reasons != IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS)
+		goto out;
+
+	if (!resume_data->netdetect_res->matched_profiles) {
+		IWL_DEBUG_WOWLAN(mld,
+				 "Netdetect results aren't valid\n");
+		wakeup_report = NULL;
+		goto out;
+	}
+
+	matched_profiles = resume_data->netdetect_res->matched_profiles;
+	if (!netdetect_cfg->n_match_sets) {
+		IWL_DEBUG_WOWLAN(mld,
+				 "No netdetect match sets are configured\n");
+		goto out;
+	}
+	n_matches = hweight_long(matched_profiles);
+	netdetect_info = kzalloc(struct_size(netdetect_info, matches,
+					     n_matches), GFP_KERNEL);
+	if (netdetect_info)
+		iwl_mld_set_netdetect_info(mld, netdetect_cfg, netdetect_info,
+					   resume_data->netdetect_res,
+					   matched_profiles);
+
+	wakeup.net_detect = netdetect_info;
+ out:
+	ieee80211_report_wowlan_wakeup(vif, wakeup_report, GFP_KERNEL);
+	if (netdetect_info) {
+		for (int i = 0; i < netdetect_info->n_matches; i++)
+			kfree(netdetect_info->matches[i]);
+		kfree(netdetect_info);
+	}
+}
+
+static bool iwl_mld_handle_d3_notif(struct iwl_notif_wait_data *notif_wait,
+				    struct iwl_rx_packet *pkt, void *data)
+{
+	struct iwl_mld_resume_data *resume_data = data;
+	struct iwl_mld *mld =
+		container_of(notif_wait, struct iwl_mld, notif_wait);
+
+	switch (WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)) {
+	case WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_INFO_NOTIFICATION): {
+		if (resume_data->notifs_received & IWL_D3_NOTIF_WOWLAN_INFO) {
+			IWL_DEBUG_WOWLAN(mld,
+					 "got additional wowlan_info notif\n");
+			break;
+		}
+		resume_data->notif_handling_err =
+			iwl_mld_handle_wowlan_info_notif(mld,
+							 resume_data->wowlan_status,
+							 pkt);
+		resume_data->notifs_received |= IWL_D3_NOTIF_WOWLAN_INFO;
+
+		if (resume_data->wowlan_status->wakeup_reasons &
+		    IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT)
+			resume_data->notifs_expected |=
+				IWL_D3_NOTIF_WOWLAN_WAKE_PKT;
+		break;
+	}
+	case WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_WAKE_PKT_NOTIFICATION): {
+		if (resume_data->notifs_received &
+		    IWL_D3_NOTIF_WOWLAN_WAKE_PKT) {
+			/* We shouldn't get two wake packet notifications */
+			IWL_DEBUG_WOWLAN(mld,
+					 "Got additional wowlan wake packet notification\n");
+			break;
+		}
+		resume_data->notif_handling_err =
+			iwl_mld_handle_wake_pkt_notif(mld,
+						      resume_data->wowlan_status,
+						      pkt);
+		resume_data->notifs_received |= IWL_D3_NOTIF_WOWLAN_WAKE_PKT;
+		break;
+	}
+	case WIDE_ID(SCAN_GROUP, OFFLOAD_MATCH_INFO_NOTIF): {
+		if (resume_data->notifs_received & IWL_D3_ND_MATCH_INFO) {
+			IWL_ERR(mld,
+				"Got additional netdetect match info\n");
+			break;
+		}
+
+		resume_data->notif_handling_err =
+			iwl_mld_netdetect_match_info_handler(mld, resume_data,
+							     pkt);
+		resume_data->notifs_received |= IWL_D3_ND_MATCH_INFO;
+		break;
+	}
+	case WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION): {
+		struct iwl_d3_end_notif *notif = (void *)pkt->data;
+
+		resume_data->d3_end_flags = le32_to_cpu(notif->flags);
+		resume_data->notifs_received |= IWL_D3_NOTIF_D3_END_NOTIF;
+		break;
+	}
+	default:
+		WARN_ON(1);
+	}
+
+	return resume_data->notifs_received == resume_data->notifs_expected;
+}
+
+#define IWL_MLD_D3_NOTIF_TIMEOUT (HZ / 3)
+
+static int iwl_mld_wait_d3_notif(struct iwl_mld *mld,
+				 struct iwl_mld_resume_data *resume_data,
+				 bool with_wowlan)
+{
+	static const u16 wowlan_resume_notif[] = {
+		WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_INFO_NOTIFICATION),
+		WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_WAKE_PKT_NOTIFICATION),
+		WIDE_ID(SCAN_GROUP, OFFLOAD_MATCH_INFO_NOTIF),
+		WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION)
+	};
+	static const u16 d3_resume_notif[] = {
+		WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION)
+	};
+	struct iwl_notification_wait wait_d3_notif;
+	enum iwl_d3_status d3_status;
+	int ret;
+
+	if (with_wowlan)
+		iwl_init_notification_wait(&mld->notif_wait, &wait_d3_notif,
+					   wowlan_resume_notif,
+					   ARRAY_SIZE(wowlan_resume_notif),
+					   iwl_mld_handle_d3_notif,
+					   resume_data);
+	else
+		iwl_init_notification_wait(&mld->notif_wait, &wait_d3_notif,
+					   d3_resume_notif,
+					   ARRAY_SIZE(d3_resume_notif),
+					   iwl_mld_handle_d3_notif,
+					   resume_data);
+
+	ret = iwl_trans_d3_resume(mld->trans, &d3_status, false, false);
+	if (ret || d3_status != IWL_D3_STATUS_ALIVE) {
+		if (d3_status != IWL_D3_STATUS_ALIVE) {
+			IWL_INFO(mld, "Device was reset during suspend\n");
+			ret = -ENOENT;
+		} else {
+			IWL_ERR(mld, "Transport resume failed\n");
+		}
+		iwl_remove_notification(&mld->notif_wait, &wait_d3_notif);
+		return ret;
+	}
+
+	ret = iwl_wait_notification(&mld->notif_wait, &wait_d3_notif,
+				    IWL_MLD_D3_NOTIF_TIMEOUT);
+	if (ret)
+		IWL_ERR(mld, "Couldn't get the d3 notif %d\n", ret);
+
+	if (resume_data->notif_handling_err)
+		ret = -EIO;
+
+	return ret;
+}
+
+int iwl_mld_no_wowlan_suspend(struct iwl_mld *mld)
+{
+	struct iwl_d3_manager_config d3_cfg_cmd_data = {};
+	int ret;
+
+	if (mld->debug_max_sleep) {
+		d3_cfg_cmd_data.wakeup_host_timer =
+			cpu_to_le32(mld->debug_max_sleep);
+		d3_cfg_cmd_data.wakeup_flags =
+			cpu_to_le32(IWL_WAKEUP_D3_HOST_TIMER);
+	}
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	IWL_DEBUG_WOWLAN(mld, "Starting the no wowlan suspend flow\n");
+
+	iwl_mld_low_latency_stop(mld);
+
+	/* This will happen if iwl_mld_supsend failed with FW error */
+	if (mld->trans->state == IWL_TRANS_NO_FW &&
+	    test_bit(STATUS_FW_ERROR, &mld->trans->status))
+		return -ENODEV;
+
+	ret = iwl_mld_update_device_power(mld, true);
+	if (ret) {
+		IWL_ERR(mld,
+			"d3 suspend: couldn't send power_device %d\n", ret);
+		goto out;
+	}
+
+	ret = iwl_mld_send_cmd_pdu(mld, D3_CONFIG_CMD,
+				   &d3_cfg_cmd_data);
+	if (ret) {
+		IWL_ERR(mld,
+			"d3 suspend: couldn't send D3_CONFIG_CMD %d\n", ret);
+		goto out;
+	}
+
+	ret = iwl_trans_d3_suspend(mld->trans, false, false);
+	if (ret) {
+		IWL_ERR(mld, "d3 suspend: trans_d3_suspend failed %d\n", ret);
+	} else {
+		/* Async notification might send hcmds, which is not allowed in suspend */
+		iwl_mld_cancel_async_notifications(mld);
+		mld->fw_status.in_d3 = true;
+	}
+
+ out:
+	if (ret) {
+		mld->trans->state = IWL_TRANS_NO_FW;
+		set_bit(STATUS_FW_ERROR, &mld->trans->status);
+	}
+
+	return ret;
+}
+
+int iwl_mld_no_wowlan_resume(struct iwl_mld *mld)
+{
+	struct iwl_mld_resume_data resume_data = {
+		.notifs_expected =
+			IWL_D3_NOTIF_D3_END_NOTIF,
+	};
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	IWL_DEBUG_WOWLAN(mld, "Starting the no wowlan resume flow\n");
+
+	mld->fw_status.in_d3 = false;
+	iwl_fw_dbg_read_d3_debug_data(&mld->fwrt);
+
+	ret = iwl_mld_wait_d3_notif(mld, &resume_data, false);
+
+	if (!ret && (resume_data.d3_end_flags & IWL_D0I3_RESET_REQUIRE))
+		return -ENODEV;
+
+	if (ret) {
+		mld->trans->state = IWL_TRANS_NO_FW;
+		set_bit(STATUS_FW_ERROR, &mld->trans->status);
+		return ret;
+	}
+	iwl_mld_low_latency_restart(mld);
+
+	return iwl_mld_update_device_power(mld, false);
+}
+
+static void
+iwl_mld_aes_seq_to_le64_pn(struct ieee80211_key_conf *key,
+			   __le64 *key_rsc)
+{
+	for (int i = 0; i < IWL_MAX_TID_COUNT; i++) {
+		struct ieee80211_key_seq seq;
+		u8 *pn = key->cipher == WLAN_CIPHER_SUITE_CCMP ? seq.ccmp.pn :
+			seq.gcmp.pn;
+
+		ieee80211_get_key_rx_seq(key, i, &seq);
+		key_rsc[i] = cpu_to_le64((u64)pn[5] |
+					 ((u64)pn[4] << 8) |
+					 ((u64)pn[3] << 16) |
+					 ((u64)pn[2] << 24) |
+					 ((u64)pn[1] << 32) |
+					 ((u64)pn[0] << 40));
+	}
+}
+
+static void
+iwl_mld_suspend_set_ucast_pn(struct iwl_mld *mld, struct ieee80211_sta *sta,
+			     struct ieee80211_key_conf *key, __le64 *key_rsc)
+{
+	struct iwl_mld_sta *mld_sta =
+		iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_ptk_pn *mld_ptk_pn;
+
+	if (WARN_ON(key->keyidx >= ARRAY_SIZE(mld_sta->ptk_pn)))
+		return;
+
+	mld_ptk_pn = wiphy_dereference(mld->wiphy,
+				       mld_sta->ptk_pn[key->keyidx]);
+	if (WARN_ON(!mld_ptk_pn))
+		return;
+
+	for (int tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+		struct ieee80211_key_seq seq;
+		u8 *max_pn = seq.ccmp.pn;
+
+		/* get the PN from mac80211, used on the default queue */
+		ieee80211_get_key_rx_seq(key, tid, &seq);
+
+		/* and use the internal data for all queues */
+		for (int que = 1; que < mld->trans->info.num_rxqs; que++) {
+			u8 *cur_pn = mld_ptk_pn->q[que].pn[tid];
+
+			if (memcmp(max_pn, cur_pn, IEEE80211_CCMP_PN_LEN) < 0)
+				max_pn = cur_pn;
+		}
+		key_rsc[tid] = cpu_to_le64((u64)max_pn[5] |
+					   ((u64)max_pn[4] << 8) |
+					   ((u64)max_pn[3] << 16) |
+					   ((u64)max_pn[2] << 24) |
+					   ((u64)max_pn[1] << 32) |
+					   ((u64)max_pn[0] << 40));
+	}
+}
+
+static void
+iwl_mld_suspend_convert_tkip_ipn(struct ieee80211_key_conf *key,
+				 __le64 *rsc)
+{
+	struct ieee80211_key_seq seq;
+
+	for (int i = 0; i < IWL_MAX_TID_COUNT; i++) {
+		ieee80211_get_key_rx_seq(key, i, &seq);
+		rsc[i] =
+			cpu_to_le64(((u64)seq.tkip.iv32 << 16) |
+				    seq.tkip.iv16);
+	}
+}
+
+static void
+iwl_mld_suspend_key_data_iter(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_sta *sta,
+			      struct ieee80211_key_conf *key,
+			      void *_data)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_suspend_key_iter_data *data = _data;
+	__le64 *key_rsc;
+	__le32 cipher = 0;
+
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_CCMP:
+		cipher = cpu_to_le32(STA_KEY_FLG_CCM);
+		fallthrough;
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_GCMP_256:
+		if (!cipher)
+			cipher = cpu_to_le32(STA_KEY_FLG_GCMP);
+		fallthrough;
+	case WLAN_CIPHER_SUITE_TKIP:
+		if (!cipher)
+			cipher = cpu_to_le32(STA_KEY_FLG_TKIP);
+		if (sta) {
+			key_rsc = data->rsc->ucast_rsc;
+			if (key->cipher == WLAN_CIPHER_SUITE_TKIP)
+				iwl_mld_suspend_convert_tkip_ipn(key, key_rsc);
+			else
+				iwl_mld_suspend_set_ucast_pn(mld, sta, key,
+							     key_rsc);
+
+			data->have_rsc = true;
+			return;
+		}
+		/* We're iterating from old to new, there're 4 possible
+		 * gtk ids, and only the last two keys matter
+		 */
+		if (WARN_ON(data->gtks >=
+				ARRAY_SIZE(data->found_gtk_idx)))
+			return;
+
+		if (WARN_ON(key->keyidx >=
+				ARRAY_SIZE(data->rsc->mcast_key_id_map)))
+			return;
+		data->gtk_cipher = cipher;
+		data->found_gtk_idx[data->gtks] = key->keyidx;
+		key_rsc = data->rsc->mcast_rsc[data->gtks % 2];
+		data->rsc->mcast_key_id_map[key->keyidx] =
+			data->gtks % 2;
+
+		if (data->gtks >= 2) {
+			int prev = data->gtks % 2;
+			int prev_idx = data->found_gtk_idx[prev];
+
+			data->rsc->mcast_key_id_map[prev_idx] =
+				IWL_MCAST_KEY_MAP_INVALID;
+		}
+
+		if (key->cipher == WLAN_CIPHER_SUITE_TKIP)
+			iwl_mld_suspend_convert_tkip_ipn(key, key_rsc);
+		else
+			iwl_mld_aes_seq_to_le64_pn(key, key_rsc);
+
+		data->gtks++;
+		data->have_rsc = true;
+		break;
+	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+		cipher = cpu_to_le32(STA_KEY_FLG_GCMP);
+		fallthrough;
+	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+		if (!cipher)
+			cipher = cpu_to_le32(STA_KEY_FLG_CCM);
+		if (key->keyidx == 4 || key->keyidx == 5)
+			data->igtk_cipher = cipher;
+
+		if (key->keyidx == 6 || key->keyidx == 7)
+			data->bigtk_cipher = cipher;
+
+		break;
+	}
+}
+
+static int
+iwl_mld_send_kek_kck_cmd(struct iwl_mld *mld,
+			 struct iwl_mld_vif *mld_vif,
+			 struct iwl_mld_suspend_key_iter_data data,
+			 int ap_sta_id)
+{
+	struct iwl_wowlan_kek_kck_material_cmd_v4 kek_kck_cmd = {};
+	struct iwl_mld_rekey_data *rekey_data =
+		&mld_vif->wowlan_data.rekey_data;
+
+	memcpy(kek_kck_cmd.kck, rekey_data->kck,
+	       rekey_data->kck_len);
+	kek_kck_cmd.kck_len = cpu_to_le16(rekey_data->kck_len);
+	memcpy(kek_kck_cmd.kek, rekey_data->kek,
+	       rekey_data->kek_len);
+	kek_kck_cmd.kek_len = cpu_to_le16(rekey_data->kek_len);
+	kek_kck_cmd.replay_ctr = rekey_data->replay_ctr;
+	kek_kck_cmd.akm = cpu_to_le32(rekey_data->akm);
+	kek_kck_cmd.sta_id = cpu_to_le32(ap_sta_id);
+	kek_kck_cmd.gtk_cipher = data.gtk_cipher;
+	kek_kck_cmd.igtk_cipher = data.igtk_cipher;
+	kek_kck_cmd.bigtk_cipher = data.bigtk_cipher;
+
+	IWL_DEBUG_WOWLAN(mld, "setting akm %d\n",
+			 rekey_data->akm);
+
+	return iwl_mld_send_cmd_pdu(mld, WOWLAN_KEK_KCK_MATERIAL,
+				    &kek_kck_cmd);
+}
+
+static int
+iwl_mld_suspend_send_security_cmds(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct iwl_mld_vif *mld_vif,
+				   int ap_sta_id)
+{
+	struct iwl_mld_suspend_key_iter_data data = {};
+	int ret;
+
+	data.rsc = kzalloc(sizeof(*data.rsc), GFP_KERNEL);
+	if (!data.rsc)
+		return -ENOMEM;
+
+	memset(data.rsc->mcast_key_id_map, IWL_MCAST_KEY_MAP_INVALID,
+	       ARRAY_SIZE(data.rsc->mcast_key_id_map));
+
+	data.rsc->sta_id = cpu_to_le32(ap_sta_id);
+	ieee80211_iter_keys(mld->hw, vif,
+			    iwl_mld_suspend_key_data_iter,
+			    &data);
+
+	if (data.have_rsc)
+		ret = iwl_mld_send_cmd_pdu(mld, WOWLAN_TSC_RSC_PARAM,
+					   data.rsc);
+	else
+		ret = 0;
+
+	if (!ret && mld_vif->wowlan_data.rekey_data.valid)
+		ret = iwl_mld_send_kek_kck_cmd(mld, mld_vif, data, ap_sta_id);
+
+	kfree(data.rsc);
+
+	return ret;
+}
+
+static void
+iwl_mld_set_wowlan_config_cmd(struct iwl_mld *mld,
+			      struct cfg80211_wowlan *wowlan,
+			      struct iwl_wowlan_config_cmd *wowlan_config_cmd,
+			      struct ieee80211_sta *ap_sta)
+{
+	wowlan_config_cmd->is_11n_connection =
+					ap_sta->deflink.ht_cap.ht_supported;
+	wowlan_config_cmd->flags = ENABLE_L3_FILTERING |
+		ENABLE_NBNS_FILTERING | ENABLE_DHCP_FILTERING;
+
+	if (ap_sta->mfp)
+		wowlan_config_cmd->flags |= IS_11W_ASSOC;
+
+	if (wowlan->disconnect)
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_BEACON_MISS |
+				    IWL_WOWLAN_WAKEUP_LINK_CHANGE);
+	if (wowlan->magic_pkt)
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_MAGIC_PACKET);
+	if (wowlan->gtk_rekey_failure)
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_GTK_REKEY_FAIL);
+	if (wowlan->eap_identity_req)
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_EAP_IDENT_REQ);
+	if (wowlan->four_way_handshake)
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_4WAY_HANDSHAKE);
+	if (wowlan->n_patterns)
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_PATTERN_MATCH);
+
+	if (wowlan->rfkill_release)
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_RF_KILL_DEASSERT);
+
+	if (wowlan->any) {
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_BEACON_MISS |
+				    IWL_WOWLAN_WAKEUP_LINK_CHANGE |
+				    IWL_WOWLAN_WAKEUP_RX_FRAME |
+				    IWL_WOWLAN_WAKEUP_BCN_FILTERING);
+	}
+}
+
+static int iwl_mld_send_patterns(struct iwl_mld *mld,
+				 struct cfg80211_wowlan *wowlan,
+				 int ap_sta_id)
+{
+	struct iwl_wowlan_patterns_cmd *pattern_cmd;
+	struct iwl_host_cmd cmd = {
+		.id = WOWLAN_PATTERNS,
+		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
+	};
+	int ret;
+
+	if (!wowlan->n_patterns)
+		return 0;
+
+	cmd.len[0] = struct_size(pattern_cmd, patterns, wowlan->n_patterns);
+
+	pattern_cmd = kzalloc(cmd.len[0], GFP_KERNEL);
+	if (!pattern_cmd)
+		return -ENOMEM;
+
+	pattern_cmd->n_patterns = wowlan->n_patterns;
+	pattern_cmd->sta_id = ap_sta_id;
+
+	for (int i = 0; i < wowlan->n_patterns; i++) {
+		int mask_len = DIV_ROUND_UP(wowlan->patterns[i].pattern_len, 8);
+
+		pattern_cmd->patterns[i].pattern_type =
+			WOWLAN_PATTERN_TYPE_BITMASK;
+
+		memcpy(&pattern_cmd->patterns[i].u.bitmask.mask,
+		       wowlan->patterns[i].mask, mask_len);
+		memcpy(&pattern_cmd->patterns[i].u.bitmask.pattern,
+		       wowlan->patterns[i].pattern,
+		       wowlan->patterns[i].pattern_len);
+		pattern_cmd->patterns[i].u.bitmask.mask_size = mask_len;
+		pattern_cmd->patterns[i].u.bitmask.pattern_size =
+			wowlan->patterns[i].pattern_len;
+	}
+
+	cmd.data[0] = pattern_cmd;
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	kfree(pattern_cmd);
+	return ret;
+}
+
+static int
+iwl_mld_send_proto_offload(struct iwl_mld *mld,
+			   struct ieee80211_vif *vif,
+			   u8 ap_sta_id)
+{
+	struct iwl_proto_offload_cmd_v4 *cmd __free(kfree);
+	struct iwl_host_cmd hcmd = {
+		.id = PROT_OFFLOAD_CONFIG_CMD,
+		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
+		.len[0] = sizeof(*cmd),
+	};
+	u32 enabled = 0;
+
+	cmd = kzalloc(hcmd.len[0], GFP_KERNEL);
+
+#if IS_ENABLED(CONFIG_IPV6)
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_wowlan_data *wowlan_data = &mld_vif->wowlan_data;
+	struct iwl_ns_config *nsc;
+	struct iwl_targ_addr *addrs;
+	int n_nsc, n_addrs;
+	int i, c;
+	int num_skipped = 0;
+
+	nsc = cmd->ns_config;
+	n_nsc = IWL_PROTO_OFFLOAD_NUM_NS_CONFIG_V3L;
+	addrs = cmd->targ_addrs;
+	n_addrs = IWL_PROTO_OFFLOAD_NUM_IPV6_ADDRS_V3L;
+
+	/* For each address we have (and that will fit) fill a target
+	 * address struct and combine for NS offload structs with the
+	 * solicited node addresses.
+	 */
+	for (i = 0, c = 0;
+		i < wowlan_data->num_target_ipv6_addrs &&
+		i < n_addrs && c < n_nsc; i++) {
+		int j;
+		struct in6_addr solicited_addr;
+
+		/* Because ns is offloaded skip tentative address to avoid
+		 * violating RFC4862.
+		 */
+		if (test_bit(i, wowlan_data->tentative_addrs)) {
+			num_skipped++;
+			continue;
+		}
+
+		addrconf_addr_solict_mult(&wowlan_data->target_ipv6_addrs[i],
+					  &solicited_addr);
+		for (j = 0; j < n_nsc && j < c; j++)
+			if (ipv6_addr_cmp(&nsc[j].dest_ipv6_addr,
+					  &solicited_addr) == 0)
+				break;
+		if (j == c)
+			c++;
+		addrs[i].addr = wowlan_data->target_ipv6_addrs[i];
+		addrs[i].config_num = cpu_to_le32(j);
+		nsc[j].dest_ipv6_addr = solicited_addr;
+		memcpy(nsc[j].target_mac_addr, vif->addr, ETH_ALEN);
+	}
+
+	if (wowlan_data->num_target_ipv6_addrs - num_skipped)
+		enabled |= IWL_D3_PROTO_IPV6_VALID;
+
+	cmd->num_valid_ipv6_addrs = cpu_to_le32(i - num_skipped);
+	if (enabled & IWL_D3_PROTO_IPV6_VALID)
+		enabled |= IWL_D3_PROTO_OFFLOAD_NS;
+#endif
+
+	if (vif->cfg.arp_addr_cnt) {
+		enabled |= IWL_D3_PROTO_OFFLOAD_ARP | IWL_D3_PROTO_IPV4_VALID;
+		cmd->common.host_ipv4_addr = vif->cfg.arp_addr_list[0];
+		ether_addr_copy(cmd->common.arp_mac_addr, vif->addr);
+	}
+
+	enabled |= IWL_D3_PROTO_OFFLOAD_BTM;
+	cmd->common.enabled = cpu_to_le32(enabled);
+	cmd->sta_id = cpu_to_le32(ap_sta_id);
+	hcmd.data[0] = cmd;
+	return iwl_mld_send_cmd(mld, &hcmd);
+}
+
+static int
+iwl_mld_wowlan_config(struct iwl_mld *mld, struct ieee80211_vif *bss_vif,
+		      struct cfg80211_wowlan *wowlan)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(bss_vif);
+	struct ieee80211_sta *ap_sta = mld_vif->ap_sta;
+	struct iwl_wowlan_config_cmd wowlan_config_cmd = {
+			.offloading_tid = IWL_WOWLAN_OFFLOAD_TID,
+	};
+	u32 sta_id_mask;
+	int ap_sta_id, ret;
+	int link_id = iwl_mld_get_primary_link(bss_vif);
+	struct ieee80211_bss_conf *link_conf;
+
+	ret = iwl_mld_block_emlsr_sync(mld, bss_vif,
+				       IWL_MLD_EMLSR_BLOCKED_WOWLAN, link_id);
+	if (ret)
+		return ret;
+
+	link_conf = link_conf_dereference_protected(bss_vif, link_id);
+
+	if (WARN_ON(!ap_sta || !link_conf))
+		return -EINVAL;
+
+	sta_id_mask = iwl_mld_fw_sta_id_mask(mld, ap_sta);
+	if (WARN_ON(hweight32(sta_id_mask) != 1))
+		return -EINVAL;
+
+	ap_sta_id = __ffs(sta_id_mask);
+	wowlan_config_cmd.sta_id = ap_sta_id;
+
+	ret = iwl_mld_ensure_queue(mld,
+				   ap_sta->txq[wowlan_config_cmd.offloading_tid]);
+	if (ret)
+		return ret;
+
+	iwl_mld_set_wowlan_config_cmd(mld, wowlan,
+				      &wowlan_config_cmd, ap_sta);
+	ret = iwl_mld_send_cmd_pdu(mld, WOWLAN_CONFIGURATION,
+				   &wowlan_config_cmd);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_suspend_send_security_cmds(mld, bss_vif, mld_vif,
+						 ap_sta_id);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_send_patterns(mld, wowlan, ap_sta_id);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_send_proto_offload(mld, bss_vif, ap_sta_id);
+	if (ret)
+		return ret;
+
+	iwl_mld_enable_beacon_filter(mld, link_conf, true);
+	return iwl_mld_update_mac_power(mld, bss_vif, true);
+}
+
+int iwl_mld_wowlan_suspend(struct iwl_mld *mld, struct cfg80211_wowlan *wowlan)
+{
+	struct ieee80211_vif *bss_vif;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!wowlan))
+		return 1;
+
+	IWL_DEBUG_WOWLAN(mld, "Starting the wowlan suspend flow\n");
+
+	bss_vif = iwl_mld_get_bss_vif(mld);
+	if (WARN_ON(!bss_vif))
+		return 1;
+
+	if (!bss_vif->cfg.assoc) {
+		int ret;
+		/* If we're not associated, this must be netdetect */
+		if (WARN_ON(!wowlan->nd_config))
+			return 1;
+
+		ret = iwl_mld_netdetect_config(mld, bss_vif, wowlan);
+		if (!ret)
+			mld->netdetect = true;
+
+		return ret;
+	}
+
+	return iwl_mld_wowlan_config(mld, bss_vif, wowlan);
+}
+
+/* Returns 0 on success, 1 if an error occurred in firmware during d3,
+ * A negative value is expected only in unrecovreable cases.
+ */
+int iwl_mld_wowlan_resume(struct iwl_mld *mld)
+{
+	struct ieee80211_vif *bss_vif;
+	struct ieee80211_bss_conf *link_conf;
+	struct iwl_mld_netdetect_res netdetect_res;
+	struct iwl_mld_resume_data resume_data = {
+		.notifs_expected =
+			IWL_D3_NOTIF_WOWLAN_INFO |
+			IWL_D3_NOTIF_D3_END_NOTIF,
+		.netdetect_res = &netdetect_res,
+	};
+	int link_id;
+	int ret;
+	bool fw_err = false;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	IWL_DEBUG_WOWLAN(mld, "Starting the wowlan resume flow\n");
+
+	if (!mld->fw_status.in_d3) {
+		IWL_DEBUG_WOWLAN(mld,
+				 "Device_powered_off() was called during wowlan\n");
+		goto err;
+	}
+
+	mld->fw_status.resuming = true;
+	mld->fw_status.in_d3 = false;
+	mld->scan.last_start_time_jiffies = jiffies;
+
+	bss_vif = iwl_mld_get_bss_vif(mld);
+	if (WARN_ON(!bss_vif))
+		goto err;
+
+	/* We can't have several links upon wowlan entry,
+	 * this is enforced in the suspend flow.
+	 */
+	WARN_ON(hweight16(bss_vif->active_links) > 1);
+	link_id = bss_vif->active_links ? __ffs(bss_vif->active_links) : 0;
+	link_conf = link_conf_dereference_protected(bss_vif, link_id);
+
+	if (WARN_ON(!link_conf))
+		goto err;
+
+	iwl_fw_dbg_read_d3_debug_data(&mld->fwrt);
+
+	resume_data.wowlan_status = kzalloc(sizeof(*resume_data.wowlan_status),
+					    GFP_KERNEL);
+	if (!resume_data.wowlan_status)
+		return -ENOMEM;
+
+	if (mld->netdetect)
+		resume_data.notifs_expected |= IWL_D3_ND_MATCH_INFO;
+
+	ret = iwl_mld_wait_d3_notif(mld, &resume_data, true);
+	if (ret) {
+		IWL_ERR(mld, "Couldn't get the d3 notifs %d\n", ret);
+		fw_err = true;
+		goto err;
+	}
+
+	if (resume_data.d3_end_flags & IWL_D0I3_RESET_REQUIRE) {
+		mld->fw_status.in_hw_restart = true;
+		goto process_wakeup_results;
+	}
+
+	iwl_mld_update_changed_regdomain(mld);
+	iwl_mld_update_mac_power(mld, bss_vif, false);
+	iwl_mld_enable_beacon_filter(mld, link_conf, false);
+	iwl_mld_update_device_power(mld, false);
+
+	if (mld->netdetect)
+		ret = iwl_mld_scan_stop(mld, IWL_MLD_SCAN_NETDETECT, false);
+
+ process_wakeup_results:
+	if (mld->netdetect) {
+		iwl_mld_process_netdetect_res(mld, bss_vif, &resume_data);
+		mld->netdetect = false;
+	} else {
+		bool keep_connection =
+			iwl_mld_process_wowlan_status(mld, bss_vif,
+						      resume_data.wowlan_status);
+
+		/* EMLSR state will be cleared if the connection is not kept */
+		if (keep_connection)
+			iwl_mld_unblock_emlsr(mld, bss_vif,
+					      IWL_MLD_EMLSR_BLOCKED_WOWLAN);
+		else
+			ieee80211_resume_disconnect(bss_vif);
+	}
+
+	goto out;
+
+ err:
+	if (fw_err) {
+		mld->trans->state = IWL_TRANS_NO_FW;
+		set_bit(STATUS_FW_ERROR, &mld->trans->status);
+	}
+
+	mld->fw_status.in_hw_restart = true;
+	ret = 1;
+ out:
+	mld->fw_status.resuming = false;
+
+	if (resume_data.wowlan_status) {
+		kfree(resume_data.wowlan_status->wake_packet);
+		kfree(resume_data.wowlan_status);
+	}
+
+	return ret;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/d3.h b/sys/contrib/dev/iwlwifi/mld/d3.h
new file mode 100644
index 000000000000..618d6fb3c796
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/d3.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_d3_h__
+#define __iwl_mld_d3_h__
+
+#include "fw/api/d3.h"
+
+struct iwl_mld_rekey_data {
+	bool valid;
+	u8 kck[NL80211_KCK_EXT_LEN];
+	u8 kek[NL80211_KEK_EXT_LEN];
+	size_t kck_len;
+	size_t kek_len;
+	__le64 replay_ctr;
+	u32 akm;
+};
+
+/**
+ * struct iwl_mld_wowlan_data - data used by the wowlan suspend flow
+ *
+ * @target_ipv6_addrs: IPv6 addresses on this interface for offload
+ * @tentative_addrs: bitmap of tentative IPv6 addresses in @target_ipv6_addrs
+ * @num_target_ipv6_addrs: number of @target_ipv6_addrs
+ * @rekey_data: security key data used for rekeying during D3
+ */
+struct iwl_mld_wowlan_data {
+#if IS_ENABLED(CONFIG_IPV6)
+	struct in6_addr target_ipv6_addrs[IWL_PROTO_OFFLOAD_NUM_IPV6_ADDRS_MAX];
+	unsigned long tentative_addrs[BITS_TO_LONGS(IWL_PROTO_OFFLOAD_NUM_IPV6_ADDRS_MAX)];
+	int num_target_ipv6_addrs;
+#endif
+	struct iwl_mld_rekey_data rekey_data;
+};
+
+int iwl_mld_no_wowlan_resume(struct iwl_mld *mld);
+int iwl_mld_no_wowlan_suspend(struct iwl_mld *mld);
+int iwl_mld_wowlan_suspend(struct iwl_mld *mld,
+			   struct cfg80211_wowlan *wowlan);
+int iwl_mld_wowlan_resume(struct iwl_mld *mld);
+void iwl_mld_set_rekey_data(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif,
+			    struct cfg80211_gtk_rekey_data *data);
+#if IS_ENABLED(CONFIG_IPV6)
+void iwl_mld_ipv6_addr_change(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct inet6_dev *idev);
+#endif
+
+#endif /* __iwl_mld_d3_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/debugfs.c b/sys/contrib/dev/iwlwifi/mld/debugfs.c
new file mode 100644
index 000000000000..cc052b0aa53f
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/debugfs.c
@@ -0,0 +1,1109 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include "mld.h"
+#include "debugfs.h"
+#include "iwl-io.h"
+#include "hcmd.h"
+#include "iface.h"
+#include "sta.h"
+#include "tlc.h"
+#include "power.h"
+#include "notif.h"
+#include "ap.h"
+#include "iwl-utils.h"
+#include "scan.h"
+#ifdef CONFIG_THERMAL
+#include "thermal.h"
+#endif
+
+#include "fw/api/rs.h"
+#include "fw/api/dhc.h"
+#include "fw/api/rfi.h"
+#include "fw/dhc-utils.h"
+#include <linux/dmi.h>
+
+#define MLD_DEBUGFS_READ_FILE_OPS(name, bufsz)				\
+	_MLD_DEBUGFS_READ_FILE_OPS(name, bufsz, struct iwl_mld)
+
+#define MLD_DEBUGFS_ADD_FILE_ALIAS(alias, name, parent, mode)		\
+	debugfs_create_file(alias, mode, parent, mld,			\
+			    &iwl_dbgfs_##name##_ops)
+#define MLD_DEBUGFS_ADD_FILE(name, parent, mode)			\
+	MLD_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
+
+static bool iwl_mld_dbgfs_fw_cmd_disabled(struct iwl_mld *mld)
+{
+#ifdef CONFIG_PM_SLEEP
+	return !mld->fw_status.running || mld->fw_status.in_d3;
+#else
+	return !mld->fw_status.running;
+#endif /* CONFIG_PM_SLEEP */
+}
+
+static ssize_t iwl_dbgfs_fw_dbg_clear_write(struct iwl_mld *mld,
+					    char *buf, size_t count)
+{
+	/* If the firmware is not running, silently succeed since there is
+	 * no data to clear.
+	 */
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return 0;
+
+	iwl_fw_dbg_clear_monitor_buf(&mld->fwrt);
+
+	return count;
+}
+
+static ssize_t iwl_dbgfs_fw_nmi_write(struct iwl_mld *mld, char *buf,
+				      size_t count)
+{
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	IWL_ERR(mld, "Triggering an NMI from debugfs\n");
+
+	if (count == 6 && !strcmp(buf, "nolog\n"))
+		mld->fw_status.do_not_dump_once = true;
+
+	iwl_force_nmi(mld->trans);
+
+	return count;
+}
+
+static ssize_t iwl_dbgfs_fw_restart_write(struct iwl_mld *mld, char *buf,
+					  size_t count)
+{
+	int __maybe_unused ret;
+
+	if (!iwlwifi_mod_params.fw_restart)
+		return -EPERM;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	if (count == 6 && !strcmp(buf, "nolog\n")) {
+		mld->fw_status.do_not_dump_once = true;
+		iwl_trans_suppress_cmd_error_once(mld->trans);
+	}
+
+	/* take the return value to make compiler happy - it will
+	 * fail anyway
+	 */
+	ret = iwl_mld_send_cmd_empty(mld, WIDE_ID(LONG_GROUP, REPLY_ERROR));
+
+	return count;
+}
+
+static ssize_t iwl_dbgfs_send_echo_cmd_write(struct iwl_mld *mld, char *buf,
+					     size_t count)
+{
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	return iwl_mld_send_cmd_empty(mld, ECHO_CMD) ?: count;
+}
+
+struct iwl_mld_sniffer_apply {
+	struct iwl_mld *mld;
+	const u8 *bssid;
+	u16 aid;
+};
+
+static bool iwl_mld_sniffer_apply(struct iwl_notif_wait_data *notif_data,
+				  struct iwl_rx_packet *pkt, void *data)
+{
+	struct iwl_mld_sniffer_apply *apply = data;
+
+	apply->mld->monitor.cur_aid = cpu_to_le16(apply->aid);
+	memcpy(apply->mld->monitor.cur_bssid, apply->bssid,
+	       sizeof(apply->mld->monitor.cur_bssid));
+
+	return true;
+}
+
+static ssize_t
+iwl_dbgfs_he_sniffer_params_write(struct iwl_mld *mld, char *buf,
+				  size_t count)
+{
+	struct iwl_notification_wait wait;
+	struct iwl_he_monitor_cmd he_mon_cmd = {};
+	struct iwl_mld_sniffer_apply apply = {
+		.mld = mld,
+	};
+	u16 wait_cmds[] = {
+		WIDE_ID(DATA_PATH_GROUP, HE_AIR_SNIFFER_CONFIG_CMD),
+	};
+	u32 aid;
+	int ret;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	if (!mld->monitor.on)
+		return -ENODEV;
+
+	ret = sscanf(buf, "%x %2hhx:%2hhx:%2hhx:%2hhx:%2hhx:%2hhx", &aid,
+		     &he_mon_cmd.bssid[0], &he_mon_cmd.bssid[1],
+		     &he_mon_cmd.bssid[2], &he_mon_cmd.bssid[3],
+		     &he_mon_cmd.bssid[4], &he_mon_cmd.bssid[5]);
+	if (ret != 7)
+		return -EINVAL;
+
+	he_mon_cmd.aid = cpu_to_le16(aid);
+
+	apply.aid = aid;
+	apply.bssid = (void *)he_mon_cmd.bssid;
+
+	/* Use the notification waiter to get our function triggered
+	 * in sequence with other RX. This ensures that frames we get
+	 * on the RX queue _before_ the new configuration is applied
+	 * still have mld->cur_aid pointing to the old AID, and that
+	 * frames on the RX queue _after_ the firmware processed the
+	 * new configuration (and sent the response, synchronously)
+	 * get mld->cur_aid correctly set to the new AID.
+	 */
+	iwl_init_notification_wait(&mld->notif_wait, &wait,
+				   wait_cmds, ARRAY_SIZE(wait_cmds),
+				   iwl_mld_sniffer_apply, &apply);
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(DATA_PATH_GROUP,
+					   HE_AIR_SNIFFER_CONFIG_CMD),
+				   &he_mon_cmd);
+
+	/* no need to really wait, we already did anyway */
+	iwl_remove_notification(&mld->notif_wait, &wait);
+
+	return ret ?: count;
+}
+
+static ssize_t
+iwl_dbgfs_he_sniffer_params_read(struct iwl_mld *mld, char *buf, size_t count)
+{
+	return scnprintf(buf, count,
+			 "%d %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx\n",
+			 le16_to_cpu(mld->monitor.cur_aid),
+			 mld->monitor.cur_bssid[0], mld->monitor.cur_bssid[1],
+			 mld->monitor.cur_bssid[2], mld->monitor.cur_bssid[3],
+			 mld->monitor.cur_bssid[4], mld->monitor.cur_bssid[5]);
+}
+
+/* The size computation is as follows:
+ * each number needs at most 3 characters, number of rows is the size of
+ * the table; So, need 5 chars for the "freq: " part and each tuple afterwards
+ * needs 6 characters for numbers and 5 for the punctuation around. 32 bytes
+ * for feature support message.
+ */
+#define IWL_RFI_DDR_BUF_SIZE (IWL_RFI_DDR_LUT_INSTALLED_SIZE *\
+				(5 + IWL_RFI_DDR_LUT_ENTRY_CHANNELS_NUM *\
+					(6 + 5)) + 32)
+#define IWL_RFI_DLVR_BUF_SIZE (IWL_RFI_DLVR_LUT_INSTALLED_SIZE *\
+				(5 + IWL_RFI_DLVR_LUT_ENTRY_CHANNELS_NUM *\
+					(6 + 5)) + 32)
+#define IWL_RFI_DESENSE_BUF_SIZE IWL_RFI_DDR_BUF_SIZE
+
+/* Extra 32 for "DDR and DLVR table" message */
+#define IWL_RFI_BUF_SIZE (IWL_RFI_DDR_BUF_SIZE + IWL_RFI_DLVR_BUF_SIZE +\
+				IWL_RFI_DESENSE_BUF_SIZE + 32)
+
+static size_t iwl_mld_dump_tas_resp(struct iwl_dhc_tas_status_resp *resp,
+				    size_t count, u8 *buf)
+{
+	const char * const tas_dis_reason[TAS_DISABLED_REASON_MAX] = {
+		[TAS_DISABLED_DUE_TO_BIOS] =
+			"Due To BIOS",
+		[TAS_DISABLED_DUE_TO_SAR_6DBM] =
+			"Due To SAR Limit Less Than 6 dBm",
+		[TAS_DISABLED_REASON_INVALID] =
+			"N/A",
+		[TAS_DISABLED_DUE_TO_TABLE_SOURCE_INVALID] =
+			"Due to table source invalid"
+	};
+	const char * const tas_current_status[TAS_DYNA_STATUS_MAX] = {
+		[TAS_DYNA_INACTIVE] = "INACTIVE",
+		[TAS_DYNA_INACTIVE_MVM_MODE] =
+			"inactive due to mvm mode",
+		[TAS_DYNA_INACTIVE_TRIGGER_MODE] =
+			"inactive due to trigger mode",
+		[TAS_DYNA_INACTIVE_BLOCK_LISTED] =
+			"inactive due to block listed",
+		[TAS_DYNA_INACTIVE_UHB_NON_US] =
+			"inactive due to uhb non US",
+		[TAS_DYNA_ACTIVE] = "ACTIVE",
+	};
+	ssize_t pos = 0;
+
+	if (resp->header.version != 1) {
+		pos += scnprintf(buf + pos, count - pos,
+				 "Unsupported TAS response version:%d",
+				 resp->header.version);
+		return pos;
+	}
+
+	pos += scnprintf(buf + pos, count - pos, "TAS Report\n");
+	switch (resp->tas_config_info.table_source) {
+	case BIOS_SOURCE_NONE:
+		pos += scnprintf(buf + pos, count - pos,
+				 "BIOS SOURCE NONE ");
+		break;
+	case BIOS_SOURCE_ACPI:
+		pos += scnprintf(buf + pos, count - pos,
+				 "BIOS SOURCE ACPI ");
+		break;
+	case BIOS_SOURCE_UEFI:
+		pos += scnprintf(buf + pos, count - pos,
+				 "BIOS SOURCE UEFI ");
+		break;
+	default:
+		pos += scnprintf(buf + pos, count - pos,
+				 "BIOS SOURCE UNKNOWN (%d) ",
+				 resp->tas_config_info.table_source);
+		break;
+	}
+
+	pos += scnprintf(buf + pos, count - pos,
+			 "revision is: %d data is: 0x%08x\n",
+			 resp->tas_config_info.table_revision,
+			 resp->tas_config_info.value);
+	pos += scnprintf(buf + pos, count - pos, "Current MCC: 0x%x\n",
+			 le16_to_cpu(resp->curr_mcc));
+
+	pos += scnprintf(buf + pos, count - pos, "Block list entries:");
+	for (int i = 0; i < ARRAY_SIZE(resp->mcc_block_list); i++)
+		pos += scnprintf(buf + pos, count - pos, " 0x%x",
+				 le16_to_cpu(resp->mcc_block_list[i]));
+
+	pos += scnprintf(buf + pos, count - pos,
+			 "\nDo TAS Support Dual Radio?: %s\n",
+			 hweight8(resp->valid_radio_mask) > 1 ?
+			 "TRUE" : "FALSE");
+
+	for (int i = 0; i < ARRAY_SIZE(resp->tas_status_radio); i++) {
+		int tmp;
+		unsigned long dynamic_status;
+
+		if (!(resp->valid_radio_mask & BIT(i)))
+			continue;
+
+		pos += scnprintf(buf + pos, count - pos,
+				 "TAS report for radio:%d\n", i + 1);
+		pos += scnprintf(buf + pos, count - pos,
+				 "Static status: %sabled\n",
+				 resp->tas_status_radio[i].static_status ?
+				 "En" : "Dis");
+		if (!resp->tas_status_radio[i].static_status) {
+			u8 static_disable_reason =
+				resp->tas_status_radio[i].static_disable_reason;
+
+			pos += scnprintf(buf + pos, count - pos,
+					 "\tStatic Disabled Reason: ");
+			if (static_disable_reason >= TAS_DISABLED_REASON_MAX) {
+				pos += scnprintf(buf + pos, count - pos,
+						 "unsupported value (%d)\n",
+						 static_disable_reason);
+				continue;
+			}
+
+			pos += scnprintf(buf + pos, count - pos,
+					 "%s (%d)\n",
+					 tas_dis_reason[static_disable_reason],
+					 static_disable_reason);
+			continue;
+		}
+
+		pos += scnprintf(buf + pos, count - pos, "\tANT A %s and ",
+				 (resp->tas_status_radio[i].dynamic_status_ant_a
+				  & BIT(TAS_DYNA_ACTIVE)) ? "ON" : "OFF");
+
+		pos += scnprintf(buf + pos, count - pos, "ANT B %s for ",
+				 (resp->tas_status_radio[i].dynamic_status_ant_b
+				  & BIT(TAS_DYNA_ACTIVE)) ? "ON" : "OFF");
+
+		switch (resp->tas_status_radio[i].band) {
+		case PHY_BAND_5:
+			pos += scnprintf(buf + pos, count - pos, "HB\n");
+			break;
+		case PHY_BAND_24:
+			pos += scnprintf(buf + pos, count - pos, "LB\n");
+			break;
+		case PHY_BAND_6:
+			pos += scnprintf(buf + pos, count - pos, "UHB\n");
+			break;
+		default:
+			pos += scnprintf(buf + pos, count - pos,
+					 "Unsupported band (%d)\n",
+					 resp->tas_status_radio[i].band);
+			break;
+		}
+
+		pos += scnprintf(buf + pos, count - pos,
+				 "Is near disconnection?: %s\n",
+				 resp->tas_status_radio[i].near_disconnection ?
+				 "True" : "False");
+
+		pos += scnprintf(buf + pos, count - pos,
+				 "Dynamic status antenna A:\n");
+		dynamic_status = resp->tas_status_radio[i].dynamic_status_ant_a;
+		for_each_set_bit(tmp, &dynamic_status, TAS_DYNA_STATUS_MAX) {
+			pos += scnprintf(buf + pos, count - pos, "\t%s (%d)\n",
+					 tas_current_status[tmp], tmp);
+		}
+		pos += scnprintf(buf + pos, count - pos,
+				 "\nDynamic status antenna B:\n");
+		dynamic_status = resp->tas_status_radio[i].dynamic_status_ant_b;
+		for_each_set_bit(tmp, &dynamic_status, TAS_DYNA_STATUS_MAX) {
+			pos += scnprintf(buf + pos, count - pos, "\t%s (%d)\n",
+					 tas_current_status[tmp], tmp);
+		}
+
+		tmp = le16_to_cpu(resp->tas_status_radio[i].max_reg_pwr_limit_ant_a);
+		pos += scnprintf(buf + pos, count - pos,
+				 "Max antenna A regulatory pwr limit (dBm): %d.%03d\n",
+				 tmp / 8, 125 * (tmp % 8));
+		tmp = le16_to_cpu(resp->tas_status_radio[i].max_reg_pwr_limit_ant_b);
+		pos += scnprintf(buf + pos, count - pos,
+				 "Max antenna B regulatory pwr limit (dBm): %d.%03d\n",
+				 tmp / 8, 125 * (tmp % 8));
+
+		tmp = le16_to_cpu(resp->tas_status_radio[i].sar_limit_ant_a);
+		pos += scnprintf(buf + pos, count - pos,
+				 "Antenna A SAR limit (dBm): %d.%03d\n",
+				 tmp / 8, 125 * (tmp % 8));
+		tmp = le16_to_cpu(resp->tas_status_radio[i].sar_limit_ant_b);
+		pos += scnprintf(buf + pos, count - pos,
+				 "Antenna B SAR limit (dBm): %d.%03d\n",
+				 tmp / 8, 125 * (tmp % 8));
+	}
+
+	return pos;
+}
+
+static ssize_t iwl_dbgfs_tas_get_status_read(struct iwl_mld *mld, char *buf,
+					     size_t count)
+{
+	struct iwl_dhc_cmd cmd = {
+		.index_and_mask = cpu_to_le32(DHC_TABLE_TOOLS |
+					      DHC_TARGET_UMAC |
+					      DHC_TOOLS_UMAC_GET_TAS_STATUS),
+	};
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(LEGACY_GROUP, DEBUG_HOST_COMMAND),
+		.flags = CMD_WANT_SKB,
+		.len[0] = sizeof(cmd),
+		.data[0] = &cmd,
+	};
+	struct iwl_dhc_tas_status_resp *resp = NULL;
+	u32 resp_len = 0;
+	ssize_t pos = 0;
+	u32 status;
+	int ret;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+	if (ret)
+		return ret;
+
+	pos += scnprintf(buf + pos, count - pos, "\nOEM name: %s\n",
+			 dmi_get_system_info(DMI_SYS_VENDOR) ?: "<unknown>");
+	pos += scnprintf(buf + pos, count - pos,
+			 "\tVendor In Approved List: %s\n",
+			 iwl_is_tas_approved() ? "YES" : "NO");
+
+	status = iwl_dhc_resp_status(mld->fwrt.fw, hcmd.resp_pkt);
+	if (status != 1) {
+		pos += scnprintf(buf + pos, count - pos,
+				 "response status is not success: %d\n",
+				 status);
+		goto out;
+	}
+
+	resp = iwl_dhc_resp_data(mld->fwrt.fw, hcmd.resp_pkt, &resp_len);
+	if (IS_ERR(resp) || resp_len != sizeof(*resp)) {
+		pos += scnprintf(buf + pos, count - pos,
+			"Invalid size for TAS response (%u instead of %zd)\n",
+			resp_len, sizeof(*resp));
+		goto out;
+	}
+
+	pos += iwl_mld_dump_tas_resp(resp, count - pos, buf + pos);
+
+out:
+	iwl_free_resp(&hcmd);
+	return pos;
+}
+
+WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(fw_nmi, 10);
+WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(fw_restart, 10);
+WIPHY_DEBUGFS_READ_WRITE_FILE_OPS_MLD(he_sniffer_params, 32);
+WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(fw_dbg_clear, 10);
+WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(send_echo_cmd, 8);
+WIPHY_DEBUGFS_READ_FILE_OPS_MLD(tas_get_status, 2048);
+
+static ssize_t iwl_dbgfs_wifi_6e_enable_read(struct iwl_mld *mld,
+					     size_t count, u8 *buf)
+{
+	int err;
+	u32 value;
+
+	err = iwl_bios_get_dsm(&mld->fwrt, DSM_FUNC_ENABLE_6E, &value);
+	if (err)
+		return err;
+
+	return scnprintf(buf, count, "0x%08x\n", value);
+}
+
+MLD_DEBUGFS_READ_FILE_OPS(wifi_6e_enable, 64);
+
+static ssize_t iwl_dbgfs_inject_packet_write(struct iwl_mld *mld,
+					     char *buf, size_t count)
+{
+	struct iwl_op_mode *opmode = container_of((void *)mld,
+						  struct iwl_op_mode,
+						  op_mode_specific);
+	struct iwl_rx_cmd_buffer rxb = {};
+	struct iwl_rx_packet *pkt;
+	int n_bytes = count / 2;
+	int ret = -EINVAL;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	rxb._page = alloc_pages(GFP_KERNEL, 0);
+	if (!rxb._page)
+		return -ENOMEM;
+	pkt = rxb_addr(&rxb);
+
+	ret = hex2bin(page_address(rxb._page), buf, n_bytes);
+	if (ret)
+		goto out;
+
+	/* avoid invalid memory access and malformed packet */
+	if (n_bytes < sizeof(*pkt) ||
+	    n_bytes != sizeof(*pkt) + iwl_rx_packet_payload_len(pkt))
+		goto out;
+
+	local_bh_disable();
+	iwl_mld_rx(opmode, NULL, &rxb);
+	local_bh_enable();
+	ret = 0;
+
+out:
+	iwl_free_rxb(&rxb);
+
+	return ret ?: count;
+}
+
+WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(inject_packet, 512);
+
+#ifdef CONFIG_THERMAL
+
+static ssize_t iwl_dbgfs_stop_ctdp_write(struct iwl_mld *mld,
+					 char *buf, size_t count)
+{
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	return iwl_mld_config_ctdp(mld, mld->cooling_dev.cur_state,
+				   CTDP_CMD_OPERATION_STOP) ? : count;
+}
+
+WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(stop_ctdp, 8);
+
+static ssize_t iwl_dbgfs_start_ctdp_write(struct iwl_mld *mld,
+					  char *buf, size_t count)
+{
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	return iwl_mld_config_ctdp(mld, mld->cooling_dev.cur_state,
+				   CTDP_CMD_OPERATION_START) ? : count;
+}
+
+WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(start_ctdp, 8);
+
+#endif /* CONFIG_THERMAL */
+
+void
+iwl_mld_add_debugfs_files(struct iwl_mld *mld, struct dentry *debugfs_dir)
+{
+	/* Add debugfs files here */
+
+	MLD_DEBUGFS_ADD_FILE(fw_nmi, debugfs_dir, 0200);
+	MLD_DEBUGFS_ADD_FILE(fw_restart, debugfs_dir, 0200);
+	MLD_DEBUGFS_ADD_FILE(wifi_6e_enable, debugfs_dir, 0400);
+	MLD_DEBUGFS_ADD_FILE(he_sniffer_params, debugfs_dir, 0600);
+	MLD_DEBUGFS_ADD_FILE(fw_dbg_clear, debugfs_dir, 0200);
+	MLD_DEBUGFS_ADD_FILE(send_echo_cmd, debugfs_dir, 0200);
+	MLD_DEBUGFS_ADD_FILE(tas_get_status, debugfs_dir, 0400);
+#ifdef CONFIG_THERMAL
+	MLD_DEBUGFS_ADD_FILE(start_ctdp, debugfs_dir, 0200);
+	MLD_DEBUGFS_ADD_FILE(stop_ctdp, debugfs_dir, 0200);
+#endif
+	MLD_DEBUGFS_ADD_FILE(inject_packet, debugfs_dir, 0200);
+
+#ifdef CONFIG_PM_SLEEP
+	debugfs_create_u32("max_sleep", 0600, debugfs_dir,
+			   &mld->debug_max_sleep);
+#endif
+
+	debugfs_create_bool("rx_ts_ptp", 0600, debugfs_dir,
+			    &mld->monitor.ptp_time);
+
+	/* Create a symlink with mac80211. It will be removed when mac80211
+	 * exits (before the opmode exits which removes the target.)
+	 */
+	if (!IS_ERR(debugfs_dir)) {
+		char buf[100];
+
+		snprintf(buf, 100, "../../%pd2", debugfs_dir->d_parent);
+		debugfs_create_symlink("iwlwifi", mld->wiphy->debugfsdir,
+				       buf);
+	}
+}
+
+#define VIF_DEBUGFS_WRITE_FILE_OPS(name, bufsz)			\
+	WIPHY_DEBUGFS_WRITE_FILE_OPS(vif_##name, bufsz, vif)
+
+#define VIF_DEBUGFS_READ_WRITE_FILE_OPS(name, bufsz)			    \
+	IEEE80211_WIPHY_DEBUGFS_READ_WRITE_FILE_OPS(vif_##name, bufsz, vif) \
+
+#define VIF_DEBUGFS_ADD_FILE_ALIAS(alias, name, parent, mode)	\
+	debugfs_create_file(alias, mode, parent, vif,		\
+			    &iwl_dbgfs_vif_##name##_ops)
+#define VIF_DEBUGFS_ADD_FILE(name, parent, mode)		\
+	VIF_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
+
+static ssize_t iwl_dbgfs_vif_bf_params_write(struct iwl_mld *mld, char *buf,
+					     size_t count, void *data)
+{
+	struct ieee80211_vif *vif = data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int link_id = vif->active_links ? __ffs(vif->active_links) : 0;
+	struct ieee80211_bss_conf *link_conf;
+	int val;
+
+	if (!strncmp("bf_enable_beacon_filter=", buf, 24)) {
+		if (sscanf(buf + 24, "%d", &val) != 1)
+			return -EINVAL;
+	} else {
+		return -EINVAL;
+	}
+
+	if (val != 0 && val != 1)
+		return -EINVAL;
+
+	link_conf = link_conf_dereference_protected(vif, link_id);
+	if (WARN_ON(!link_conf))
+		return -ENODEV;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	mld_vif->disable_bf = !val;
+
+	if (val)
+		return iwl_mld_enable_beacon_filter(mld, link_conf,
+						    false) ?: count;
+	else
+		return iwl_mld_disable_beacon_filter(mld, vif) ?: count;
+}
+
+static ssize_t iwl_dbgfs_vif_pm_params_write(struct iwl_mld *mld,
+					     char *buf,
+					     size_t count, void *data)
+{
+	struct ieee80211_vif *vif = data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int val;
+
+	if (!strncmp("use_ps_poll=", buf, 12)) {
+		if (sscanf(buf + 12, "%d", &val) != 1)
+			return -EINVAL;
+	} else {
+		return -EINVAL;
+	}
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	mld_vif->use_ps_poll = val;
+
+	return iwl_mld_update_mac_power(mld, vif, false) ?: count;
+}
+
+static ssize_t iwl_dbgfs_vif_low_latency_write(struct iwl_mld *mld,
+					       char *buf, size_t count,
+					       void *data)
+{
+	struct ieee80211_vif *vif = data;
+	u8 value;
+	int ret;
+
+	ret = kstrtou8(buf, 0, &value);
+	if (ret)
+		return ret;
+
+	if (value > 1)
+		return -EINVAL;
+
+	iwl_mld_vif_update_low_latency(mld, vif, value, LOW_LATENCY_DEBUGFS);
+
+	return count;
+}
+
+static ssize_t iwl_dbgfs_vif_low_latency_read(struct ieee80211_vif *vif,
+					      size_t count, char *buf)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	char format[] = "traffic=%d\ndbgfs=%d\nvif_type=%d\nactual=%d\n";
+	u8 ll_causes;
+
+	if (WARN_ON(count < sizeof(format)))
+		return -EINVAL;
+
+	ll_causes = READ_ONCE(mld_vif->low_latency_causes);
+
+	/* all values in format are boolean so the size of format is enough
+	 * for holding the result string
+	 */
+	return scnprintf(buf, count, format,
+			 !!(ll_causes & LOW_LATENCY_TRAFFIC),
+			 !!(ll_causes & LOW_LATENCY_DEBUGFS),
+			 !!(ll_causes & LOW_LATENCY_VIF_TYPE),
+			 !!(ll_causes));
+}
+
+VIF_DEBUGFS_WRITE_FILE_OPS(pm_params, 32);
+VIF_DEBUGFS_WRITE_FILE_OPS(bf_params, 32);
+VIF_DEBUGFS_READ_WRITE_FILE_OPS(low_latency, 45);
+
+static int
+_iwl_dbgfs_inject_beacon_ie(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			    char *bin, ssize_t len,
+			    bool restore)
+{
+	struct iwl_mld_vif *mld_vif;
+	struct iwl_mld_link *mld_link;
+	struct iwl_mac_beacon_cmd beacon_cmd = {};
+	int n_bytes = len / 2;
+
+	/* Element len should be represented by u8 */
+	if (n_bytes >= U8_MAX)
+		return -EINVAL;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	if (!vif)
+		return -EINVAL;
+
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+	mld_vif->beacon_inject_active = true;
+	mld->hw->extra_beacon_tailroom = n_bytes;
+
+	for_each_mld_vif_valid_link(mld_vif, mld_link) {
+		u32 offset;
+		struct ieee80211_tx_info *info;
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, link_id);
+		struct ieee80211_chanctx_conf *ctx =
+			wiphy_dereference(mld->wiphy, link_conf->chanctx_conf);
+		struct sk_buff *beacon =
+			ieee80211_beacon_get_template(mld->hw, vif,
+						      NULL, link_id);
+
+		if (!beacon)
+			return -EINVAL;
+
+		if (!restore && (WARN_ON(!n_bytes || !bin) ||
+				 hex2bin(skb_put_zero(beacon, n_bytes),
+					 bin, n_bytes))) {
+			dev_kfree_skb(beacon);
+			return -EINVAL;
+		}
+
+		info = IEEE80211_SKB_CB(beacon);
+
+		beacon_cmd.flags =
+			cpu_to_le16(iwl_mld_get_rate_flags(mld, info, vif,
+							   link_conf,
+							   ctx->def.chan->band));
+		beacon_cmd.byte_cnt = cpu_to_le16((u16)beacon->len);
+		beacon_cmd.link_id =
+			cpu_to_le32(mld_link->fw_id);
+
+		iwl_mld_set_tim_idx(mld, &beacon_cmd.tim_idx,
+				    beacon->data, beacon->len);
+
+		offset = iwl_find_ie_offset(beacon->data,
+					    WLAN_EID_S1G_TWT,
+					    beacon->len);
+
+		beacon_cmd.btwt_offset = cpu_to_le32(offset);
+
+		iwl_mld_send_beacon_template_cmd(mld, beacon, &beacon_cmd);
+		dev_kfree_skb(beacon);
+	}
+
+	if (restore)
+		mld_vif->beacon_inject_active = false;
+
+	return 0;
+}
+
+static ssize_t
+iwl_dbgfs_vif_inject_beacon_ie_write(struct iwl_mld *mld,
+				     char *buf, size_t count,
+				     void *data)
+{
+	struct ieee80211_vif *vif = data;
+	int ret = _iwl_dbgfs_inject_beacon_ie(mld, vif, buf,
+					      count, false);
+
+	mld->hw->extra_beacon_tailroom = 0;
+	return ret ?: count;
+}
+
+VIF_DEBUGFS_WRITE_FILE_OPS(inject_beacon_ie, 512);
+
+static ssize_t
+iwl_dbgfs_vif_inject_beacon_ie_restore_write(struct iwl_mld *mld,
+					     char *buf,
+					     size_t count,
+					     void *data)
+{
+	struct ieee80211_vif *vif = data;
+	int ret = _iwl_dbgfs_inject_beacon_ie(mld, vif, NULL,
+					      0, true);
+
+	mld->hw->extra_beacon_tailroom = 0;
+	return ret ?: count;
+}
+
+VIF_DEBUGFS_WRITE_FILE_OPS(inject_beacon_ie_restore, 512);
+
+static ssize_t
+iwl_dbgfs_vif_twt_setup_write(struct iwl_mld *mld, char *buf, size_t count,
+			      void *data)
+{
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(IWL_ALWAYS_LONG_GROUP, DEBUG_HOST_COMMAND),
+	};
+	struct ieee80211_vif *vif = data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_dhc_cmd *cmd __free(kfree) = NULL;
+	struct iwl_dhc_twt_operation *dhc_twt_cmd;
+	u64 target_wake_time;
+	u32 twt_operation, interval_exp, interval_mantissa, min_wake_duration;
+	u8 trigger, flow_type, flow_id, protection, tenth_param;
+	u8 twt_request = 1, broadcast = 0;
+	int ret;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	ret = sscanf(buf, "%u %llu %u %u %u %hhu %hhu %hhu %hhu %hhu",
+		     &twt_operation, &target_wake_time, &interval_exp,
+		     &interval_mantissa, &min_wake_duration, &trigger,
+		     &flow_type, &flow_id, &protection, &tenth_param);
+
+	/* the new twt_request parameter is optional for station */
+	if ((ret != 9 && ret != 10) ||
+	    (ret == 10 && vif->type != NL80211_IFTYPE_STATION &&
+	     tenth_param == 1))
+		return -EINVAL;
+
+	/* The 10th parameter:
+	 * In STA mode - the TWT type (broadcast or individual)
+	 * In AP mode - the role (0 responder, 2 unsolicited)
+	 */
+	if (ret == 10) {
+		if (vif->type == NL80211_IFTYPE_STATION)
+			broadcast = tenth_param;
+		else
+			twt_request = tenth_param;
+	}
+
+	cmd = kzalloc(sizeof(*cmd) + sizeof(*dhc_twt_cmd), GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	dhc_twt_cmd = (void *)cmd->data;
+	dhc_twt_cmd->mac_id = cpu_to_le32(mld_vif->fw_id);
+	dhc_twt_cmd->twt_operation = cpu_to_le32(twt_operation);
+	dhc_twt_cmd->target_wake_time = cpu_to_le64(target_wake_time);
+	dhc_twt_cmd->interval_exp = cpu_to_le32(interval_exp);
+	dhc_twt_cmd->interval_mantissa = cpu_to_le32(interval_mantissa);
+	dhc_twt_cmd->min_wake_duration = cpu_to_le32(min_wake_duration);
+	dhc_twt_cmd->trigger = trigger;
+	dhc_twt_cmd->flow_type = flow_type;
+	dhc_twt_cmd->flow_id = flow_id;
+	dhc_twt_cmd->protection = protection;
+	dhc_twt_cmd->twt_request = twt_request;
+	dhc_twt_cmd->negotiation_type = broadcast ? 3 : 0;
+
+	cmd->length = cpu_to_le32(sizeof(*dhc_twt_cmd) >> 2);
+	cmd->index_and_mask =
+		cpu_to_le32(DHC_TABLE_INTEGRATION | DHC_TARGET_UMAC |
+			    DHC_INT_UMAC_TWT_OPERATION);
+
+	hcmd.len[0] = sizeof(*cmd) + sizeof(*dhc_twt_cmd);
+	hcmd.data[0] = cmd;
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+
+	return ret ?: count;
+}
+
+VIF_DEBUGFS_WRITE_FILE_OPS(twt_setup, 256);
+
+static ssize_t
+iwl_dbgfs_vif_twt_operation_write(struct iwl_mld *mld, char *buf, size_t count,
+				  void *data)
+{
+	struct ieee80211_vif *vif = data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_twt_operation_cmd twt_cmd = {};
+	int link_id = vif->active_links ? __ffs(vif->active_links) : 0;
+	struct iwl_mld_link *mld_link = iwl_mld_link_dereference_check(mld_vif,
+								       link_id);
+	int ret;
+
+	if (WARN_ON(!mld_link))
+		return -ENODEV;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	if (hweight16(vif->active_links) > 1)
+		return -EOPNOTSUPP;
+
+	ret = sscanf(buf,
+		     "%u %llu %u %u %u %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu %hhu",
+		     &twt_cmd.twt_operation, &twt_cmd.target_wake_time,
+		     &twt_cmd.interval_exponent, &twt_cmd.interval_mantissa,
+		     &twt_cmd.minimum_wake_duration, &twt_cmd.trigger,
+		     &twt_cmd.flow_type, &twt_cmd.flow_id,
+		     &twt_cmd.twt_protection, &twt_cmd.ndp_paging_indicator,
+		     &twt_cmd.responder_pm_mode, &twt_cmd.negotiation_type,
+		     &twt_cmd.twt_request, &twt_cmd.implicit,
+		     &twt_cmd.twt_group_assignment, &twt_cmd.twt_channel,
+		     &twt_cmd.restricted_info_present, &twt_cmd.dl_bitmap_valid,
+		     &twt_cmd.ul_bitmap_valid, &twt_cmd.dl_tid_bitmap,
+		     &twt_cmd.ul_tid_bitmap);
+
+	if (ret != 21)
+		return -EINVAL;
+
+	twt_cmd.link_id = cpu_to_le32(mld_link->fw_id);
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(MAC_CONF_GROUP, TWT_OPERATION_CMD),
+				   &twt_cmd);
+	return ret ?: count;
+}
+
+VIF_DEBUGFS_WRITE_FILE_OPS(twt_operation, 256);
+
+static ssize_t iwl_dbgfs_vif_int_mlo_scan_write(struct iwl_mld *mld, char *buf,
+						size_t count, void *data)
+{
+	struct ieee80211_vif *vif = data;
+	u32 action;
+	int ret;
+
+	if (!vif->cfg.assoc || !ieee80211_vif_is_mld(vif))
+		return -EINVAL;
+
+	if (kstrtou32(buf, 0, &action))
+		return -EINVAL;
+
+	if (action == 0) {
+		ret = iwl_mld_scan_stop(mld, IWL_MLD_SCAN_INT_MLO, false);
+	} else if (action == 1) {
+		iwl_mld_int_mlo_scan(mld, vif);
+		ret = 0;
+	} else {
+		ret = -EINVAL;
+	}
+
+	return ret ?: count;
+}
+
+VIF_DEBUGFS_WRITE_FILE_OPS(int_mlo_scan, 32);
+
+void iwl_mld_add_vif_debugfs(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif)
+{
+	struct dentry *mld_vif_dbgfs =
+		debugfs_create_dir("iwlmld", vif->debugfs_dir);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	char target[3 * 3 + 11 + (NL80211_WIPHY_NAME_MAXLEN + 1) +
+		    (7 + IFNAMSIZ + 1) + 6 + 1];
+	char name[7 + IFNAMSIZ + 1];
+
+	/* Create symlink for convenience pointing to interface specific
+	 * debugfs entries for the driver. For example, under
+	 * /sys/kernel/debug/iwlwifi/0000\:02\:00.0/iwlmld/
+	 * find
+	 * netdev:wlan0 -> ../../../ieee80211/phy0/netdev:wlan0/iwlmld/
+	 */
+	snprintf(name, sizeof(name), "%pd", vif->debugfs_dir);
+	snprintf(target, sizeof(target), "../../../%pd3/iwlmld",
+		 vif->debugfs_dir);
+	if (!mld_vif->dbgfs_slink)
+		mld_vif->dbgfs_slink =
+			debugfs_create_symlink(name, mld->debugfs_dir, target);
+
+	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM &&
+	    vif->type == NL80211_IFTYPE_STATION) {
+		VIF_DEBUGFS_ADD_FILE(pm_params, mld_vif_dbgfs, 0200);
+		VIF_DEBUGFS_ADD_FILE(bf_params, mld_vif_dbgfs, 0200);
+	}
+
+	if (vif->type == NL80211_IFTYPE_AP) {
+		VIF_DEBUGFS_ADD_FILE(inject_beacon_ie, mld_vif_dbgfs, 0200);
+		VIF_DEBUGFS_ADD_FILE(inject_beacon_ie_restore,
+				     mld_vif_dbgfs, 0200);
+	}
+
+	VIF_DEBUGFS_ADD_FILE(low_latency, mld_vif_dbgfs, 0600);
+	VIF_DEBUGFS_ADD_FILE(twt_setup, mld_vif_dbgfs, 0200);
+	VIF_DEBUGFS_ADD_FILE(twt_operation, mld_vif_dbgfs, 0200);
+	VIF_DEBUGFS_ADD_FILE(int_mlo_scan, mld_vif_dbgfs, 0200);
+}
+#define LINK_DEBUGFS_WRITE_FILE_OPS(name, bufsz)			\
+	WIPHY_DEBUGFS_WRITE_FILE_OPS(link_##name, bufsz, bss_conf)
+
+#define LINK_DEBUGFS_ADD_FILE_ALIAS(alias, name, parent, mode)		\
+	debugfs_create_file(alias, mode, parent, link_conf,		\
+			    &iwl_dbgfs_link_##name##_ops)
+#define LINK_DEBUGFS_ADD_FILE(name, parent, mode)			\
+	LINK_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
+
+void iwl_mld_add_link_debugfs(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf,
+			      struct dentry *dir)
+{
+	struct dentry *mld_link_dir;
+
+	mld_link_dir = debugfs_lookup("iwlmld", dir);
+
+	/* For non-MLO vifs, the dir of deflink is the same as the vif's one.
+	 * so if iwlmld dir already exists, this means that this is deflink.
+	 * If not, this is a per-link dir of a MLO vif, add in it the iwlmld
+	 * dir.
+	 */
+	if (!mld_link_dir)
+		mld_link_dir = debugfs_create_dir("iwlmld", dir);
+}
+
+static ssize_t _iwl_dbgfs_fixed_rate_write(struct iwl_mld *mld, char *buf,
+					   size_t count, void *data, bool v3)
+{
+	struct ieee80211_link_sta *link_sta = data;
+	struct iwl_mld_link_sta *mld_link_sta;
+	u32 rate;
+	u32 partial = false;
+	char pretty_rate[100];
+	int ret;
+	u8 fw_sta_id;
+
+	mld_link_sta = iwl_mld_link_sta_from_mac80211(link_sta);
+	if (WARN_ON(!mld_link_sta))
+		return -EINVAL;
+
+	fw_sta_id = mld_link_sta->fw_id;
+
+	if (sscanf(buf, "%i %i", &rate, &partial) == 0)
+		return -EINVAL;
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	/* input is in FW format (v2 or v3) so convert to v3 */
+	rate = iwl_v3_rate_from_v2_v3(cpu_to_le32(rate), v3);
+	rate = le32_to_cpu(iwl_v3_rate_to_v2_v3(rate, mld->fw_rates_ver_3));
+
+	ret = iwl_mld_send_tlc_dhc(mld, fw_sta_id,
+				   partial ? IWL_TLC_DEBUG_PARTIAL_FIXED_RATE :
+					     IWL_TLC_DEBUG_FIXED_RATE,
+				   rate);
+
+	rs_pretty_print_rate(pretty_rate, sizeof(pretty_rate), rate);
+
+	IWL_DEBUG_RATE(mld, "sta_id %d rate %s partial: %d, ret:%d\n",
+		       fw_sta_id, pretty_rate, partial, ret);
+
+	return ret ? : count;
+}
+
+static ssize_t iwl_dbgfs_fixed_rate_write(struct iwl_mld *mld, char *buf,
+					  size_t count, void *data)
+{
+	return _iwl_dbgfs_fixed_rate_write(mld, buf, count, data, false);
+}
+
+static ssize_t iwl_dbgfs_fixed_rate_v3_write(struct iwl_mld *mld, char *buf,
+					     size_t count, void *data)
+{
+	return _iwl_dbgfs_fixed_rate_write(mld, buf, count, data, true);
+}
+
+static ssize_t iwl_dbgfs_tlc_dhc_write(struct iwl_mld *mld, char *buf,
+				       size_t count, void *data)
+{
+	struct ieee80211_link_sta *link_sta = data;
+	struct iwl_mld_link_sta *mld_link_sta;
+	u32 type, value;
+	int ret;
+	u8 fw_sta_id;
+
+	mld_link_sta = iwl_mld_link_sta_from_mac80211(link_sta);
+	if (WARN_ON(!mld_link_sta))
+		return -EINVAL;
+
+	fw_sta_id = mld_link_sta->fw_id;
+
+	if (sscanf(buf, "%i %i", &type, &value) != 2) {
+		IWL_DEBUG_RATE(mld, "usage <type> <value>\n");
+		return -EINVAL;
+	}
+
+	if (iwl_mld_dbgfs_fw_cmd_disabled(mld))
+		return -EIO;
+
+	ret = iwl_mld_send_tlc_dhc(mld, fw_sta_id, type, value);
+
+	return ret ? : count;
+}
+
+#define LINK_STA_DEBUGFS_ADD_FILE_ALIAS(alias, name, parent, mode)	\
+	debugfs_create_file(alias, mode, parent, link_sta,		\
+			    &iwl_dbgfs_##name##_ops)
+#define LINK_STA_DEBUGFS_ADD_FILE(name, parent, mode)			\
+	LINK_STA_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
+
+#define LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(name, bufsz)			\
+	WIPHY_DEBUGFS_WRITE_FILE_OPS(name, bufsz, link_sta)
+
+LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(tlc_dhc, 64);
+LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(fixed_rate, 64);
+LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(fixed_rate_v3, 64);
+
+void iwl_mld_add_link_sta_debugfs(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_link_sta *link_sta,
+				  struct dentry *dir)
+{
+	LINK_STA_DEBUGFS_ADD_FILE(fixed_rate, dir, 0200);
+	LINK_STA_DEBUGFS_ADD_FILE(fixed_rate_v3, dir, 0200);
+	LINK_STA_DEBUGFS_ADD_FILE(tlc_dhc, dir, 0200);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/debugfs.h b/sys/contrib/dev/iwlwifi/mld/debugfs.h
new file mode 100644
index 000000000000..eeba35342ba1
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/debugfs.h
@@ -0,0 +1,244 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include "iface.h"
+#include "sta.h"
+
+#define MLD_DEBUGFS_OPEN_WRAPPER(name, buflen, argtype)			\
+struct dbgfs_##name##_data {						\
+	argtype *arg;							\
+	bool read_done;							\
+	ssize_t rlen;							\
+	char buf[buflen];						\
+};									\
+static int _iwl_dbgfs_##name##_open(struct inode *inode,		\
+				    struct file *file)			\
+{									\
+	struct dbgfs_##name##_data *data;				\
+									\
+	if ((file->f_flags & O_ACCMODE) == O_RDWR)			\
+		return -EOPNOTSUPP;					\
+									\
+	data = kzalloc(sizeof(*data), GFP_KERNEL);			\
+	if (!data)							\
+		return -ENOMEM;						\
+									\
+	data->read_done = false;					\
+	data->arg = inode->i_private;					\
+	file->private_data = data;					\
+									\
+	return 0;							\
+}
+
+#define MLD_DEBUGFS_READ_WRAPPER(name)					\
+static ssize_t _iwl_dbgfs_##name##_read(struct file *file,		\
+					char __user *user_buf,		\
+					size_t count, loff_t *ppos)	\
+{									\
+	struct dbgfs_##name##_data *data = file->private_data;		\
+									\
+	if (!data->read_done) {						\
+		data->read_done = true;					\
+		data->rlen = iwl_dbgfs_##name##_read(data->arg,		\
+						     sizeof(data->buf),\
+						     data->buf);	\
+	}								\
+									\
+	if (data->rlen < 0)						\
+		return data->rlen;					\
+	return simple_read_from_buffer(user_buf, count, ppos,		\
+				       data->buf, data->rlen);		\
+}
+
+static int _iwl_dbgfs_release(struct inode *inode, struct file *file)
+{
+	kfree(file->private_data);
+	return 0;
+}
+
+#define _MLD_DEBUGFS_READ_FILE_OPS(name, buflen, argtype)		\
+MLD_DEBUGFS_OPEN_WRAPPER(name, buflen, argtype)				\
+MLD_DEBUGFS_READ_WRAPPER(name)						\
+static const struct file_operations iwl_dbgfs_##name##_ops = {		\
+	.read = _iwl_dbgfs_##name##_read,				\
+	.open = _iwl_dbgfs_##name##_open,				\
+	.llseek = generic_file_llseek,					\
+	.release = _iwl_dbgfs_release,					\
+}
+
+#define WIPHY_DEBUGFS_WRITE_HANDLER_WRAPPER(name)			\
+static ssize_t iwl_dbgfs_##name##_write_handler(struct wiphy *wiphy,	\
+				       struct file *file, char *buf,	\
+				       size_t count, void *data)	\
+{									\
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);		\
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);			\
+	return iwl_dbgfs_##name##_write(mld, buf, count, data);		\
+}
+
+static inline struct iwl_mld *
+iwl_mld_from_link_sta(struct ieee80211_link_sta *link_sta)
+{
+	struct ieee80211_vif *vif =
+		iwl_mld_sta_from_mac80211(link_sta->sta)->vif;
+	return iwl_mld_vif_from_mac80211(vif)->mld;
+}
+
+static inline struct iwl_mld *
+iwl_mld_from_bss_conf(struct ieee80211_bss_conf *link)
+{
+	return iwl_mld_vif_from_mac80211(link->vif)->mld;
+}
+
+static inline struct iwl_mld *iwl_mld_from_vif(struct ieee80211_vif *vif)
+{
+	return iwl_mld_vif_from_mac80211(vif)->mld;
+}
+
+#define WIPHY_DEBUGFS_WRITE_WRAPPER(name, bufsz, objtype)		\
+WIPHY_DEBUGFS_WRITE_HANDLER_WRAPPER(name)				\
+static ssize_t __iwl_dbgfs_##name##_write(struct file *file,		\
+					  const char __user *user_buf,	\
+					  size_t count, loff_t *ppos)	\
+{									\
+	struct ieee80211_##objtype *arg = file->private_data;		\
+	struct iwl_mld *mld = iwl_mld_from_##objtype(arg);		\
+	char buf[bufsz] = {};						\
+									\
+	return wiphy_locked_debugfs_write(mld->wiphy, file,		\
+				buf, sizeof(buf),			\
+				user_buf, count,			\
+				iwl_dbgfs_##name##_write_handler,	\
+				arg);					\
+}
+
+#define WIPHY_DEBUGFS_WRITE_FILE_OPS(name, bufsz, objtype)		\
+	WIPHY_DEBUGFS_WRITE_WRAPPER(name, bufsz, objtype)		\
+	static const struct file_operations iwl_dbgfs_##name##_ops = {	\
+		.write = __iwl_dbgfs_##name##_write,			\
+		.open = simple_open,					\
+		.llseek = generic_file_llseek,				\
+	}
+
+#define WIPHY_DEBUGFS_READ_HANDLER_WRAPPER_MLD(name)			\
+static ssize_t iwl_dbgfs_##name##_read_handler(struct wiphy *wiphy,	\
+				       struct file *file, char *buf,	\
+				       size_t count, void *data)	\
+{									\
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);		\
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);			\
+	return iwl_dbgfs_##name##_read(mld, buf, count);		\
+}
+
+#define WIPHY_DEBUGFS_WRITE_HANDLER_WRAPPER_MLD(name)			\
+static ssize_t iwl_dbgfs_##name##_write_handler(struct wiphy *wiphy,	\
+				       struct file *file, char *buf,	\
+				       size_t count, void *data)	\
+{									\
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);		\
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);			\
+	return iwl_dbgfs_##name##_write(mld, buf, count);		\
+}
+
+#define WIPHY_DEBUGFS_WRITE_WRAPPER_MLD(name)				\
+WIPHY_DEBUGFS_WRITE_HANDLER_WRAPPER_MLD(name)				\
+static ssize_t __iwl_dbgfs_##name##_write(struct file *file,		\
+					  const char __user *user_buf,	\
+					  size_t count, loff_t *ppos)	\
+{									\
+	struct dbgfs_##name##_data *data = file->private_data;		\
+	struct iwl_mld *mld = data->arg;				\
+									\
+	return wiphy_locked_debugfs_write(mld->wiphy, file,		\
+				data->buf, sizeof(data->buf),		\
+				user_buf, count,			\
+				iwl_dbgfs_##name##_write_handler,	\
+				NULL);					\
+}
+
+#define WIPHY_DEBUGFS_READ_WRAPPER_MLD(name)				\
+WIPHY_DEBUGFS_READ_HANDLER_WRAPPER_MLD(name)				\
+static ssize_t __iwl_dbgfs_##name##_read(struct file *file,		\
+					char __user *user_buf,		\
+					size_t count, loff_t *ppos)	\
+{									\
+	struct dbgfs_##name##_data *data = file->private_data;		\
+	struct iwl_mld *mld = data->arg;				\
+									\
+	if (!data->read_done) {						\
+		data->read_done = true;					\
+		data->rlen = wiphy_locked_debugfs_read(mld->wiphy,	\
+				file, data->buf, sizeof(data->buf),	\
+				user_buf, count, ppos,			\
+				iwl_dbgfs_##name##_read_handler, NULL);	\
+		return data->rlen;					\
+	}								\
+									\
+	if (data->rlen < 0)						\
+		return data->rlen;					\
+	return simple_read_from_buffer(user_buf, count, ppos,		\
+				       data->buf, data->rlen);		\
+}
+
+#define WIPHY_DEBUGFS_READ_FILE_OPS_MLD(name, bufsz)			\
+	MLD_DEBUGFS_OPEN_WRAPPER(name, bufsz, struct iwl_mld)		\
+	WIPHY_DEBUGFS_READ_WRAPPER_MLD(name)				\
+	static const struct file_operations iwl_dbgfs_##name##_ops = {	\
+		.read = __iwl_dbgfs_##name##_read,			\
+		.open = _iwl_dbgfs_##name##_open,			\
+		.llseek = generic_file_llseek,				\
+		.release = _iwl_dbgfs_release,				\
+	}
+
+#define WIPHY_DEBUGFS_WRITE_FILE_OPS_MLD(name, bufsz)			\
+	MLD_DEBUGFS_OPEN_WRAPPER(name, bufsz, struct iwl_mld)		\
+	WIPHY_DEBUGFS_WRITE_WRAPPER_MLD(name)				\
+	static const struct file_operations iwl_dbgfs_##name##_ops = {	\
+		.write = __iwl_dbgfs_##name##_write,			\
+		.open = _iwl_dbgfs_##name##_open,			\
+		.llseek = generic_file_llseek,				\
+		.release = _iwl_dbgfs_release,				\
+	}
+
+#define WIPHY_DEBUGFS_READ_WRITE_FILE_OPS_MLD(name, bufsz)		\
+	MLD_DEBUGFS_OPEN_WRAPPER(name, bufsz, struct iwl_mld)		\
+	WIPHY_DEBUGFS_WRITE_WRAPPER_MLD(name)				\
+	WIPHY_DEBUGFS_READ_WRAPPER_MLD(name)				\
+	static const struct file_operations iwl_dbgfs_##name##_ops = {	\
+		.write = __iwl_dbgfs_##name##_write,			\
+		.read = __iwl_dbgfs_##name##_read,			\
+		.open = _iwl_dbgfs_##name##_open,			\
+		.llseek = generic_file_llseek,				\
+		.release = _iwl_dbgfs_release,				\
+	}
+
+#define WIPHY_DEBUGFS_WRITE_WRAPPER_IEEE80211(name, bufsz, objtype)	\
+WIPHY_DEBUGFS_WRITE_HANDLER_WRAPPER(name)				\
+static ssize_t _iwl_dbgfs_##name##_write(struct file *file,		\
+					  const char __user *user_buf,	\
+					  size_t count, loff_t *ppos)	\
+{									\
+	struct dbgfs_##name##_data *data = file->private_data;		\
+	struct ieee80211_##objtype *arg = data->arg;			\
+	struct iwl_mld *mld = iwl_mld_from_##objtype(arg);		\
+	char buf[bufsz] = {};						\
+									\
+	return wiphy_locked_debugfs_write(mld->wiphy, file,		\
+				buf, sizeof(buf),			\
+				user_buf, count,			\
+				iwl_dbgfs_##name##_write_handler,	\
+				arg);					\
+}
+
+#define IEEE80211_WIPHY_DEBUGFS_READ_WRITE_FILE_OPS(name, bufsz, objtype) \
+	MLD_DEBUGFS_OPEN_WRAPPER(name, bufsz, struct ieee80211_##objtype) \
+	WIPHY_DEBUGFS_WRITE_WRAPPER_IEEE80211(name, bufsz, objtype)	  \
+	MLD_DEBUGFS_READ_WRAPPER(name)					  \
+	static const struct file_operations iwl_dbgfs_##name##_ops = {	  \
+		.write = _iwl_dbgfs_##name##_write,			  \
+		.read = _iwl_dbgfs_##name##_read,			  \
+		.open = _iwl_dbgfs_##name##_open,			  \
+		.llseek = generic_file_llseek,				  \
+		.release = _iwl_dbgfs_release,				  \
+	}
diff --git a/sys/contrib/dev/iwlwifi/mld/ftm-initiator.c b/sys/contrib/dev/iwlwifi/mld/ftm-initiator.c
new file mode 100644
index 000000000000..3464b3268712
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/ftm-initiator.c
@@ -0,0 +1,451 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+#include <linux/etherdevice.h>
+#include <linux/math64.h>
+#include <net/cfg80211.h>
+#include "mld.h"
+#include "iface.h"
+#include "phy.h"
+#include "iwl-io.h"
+#include "iwl-prph.h"
+#include "constants.h"
+#include "fw/api/location.h"
+#include "ftm-initiator.h"
+
+static void iwl_mld_ftm_cmd_common(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct iwl_tof_range_req_cmd *cmd,
+				   struct cfg80211_pmsr_request *req)
+{
+	int i;
+
+	cmd->initiator_flags =
+		cpu_to_le32(IWL_TOF_INITIATOR_FLAGS_MACADDR_RANDOM |
+			    IWL_TOF_INITIATOR_FLAGS_NON_ASAP_SUPPORT);
+	cmd->request_id = req->cookie;
+	cmd->num_of_ap = req->n_peers;
+
+	/* Use a large value for "no timeout". Don't use the maximum value
+	 * because of fw limitations.
+	 */
+	if (req->timeout)
+		cmd->req_timeout_ms = cpu_to_le32(min(req->timeout, 0xfffff));
+	else
+		cmd->req_timeout_ms = cpu_to_le32(0xfffff);
+
+	memcpy(cmd->macaddr_template, req->mac_addr, ETH_ALEN);
+	for (i = 0; i < ETH_ALEN; i++)
+		cmd->macaddr_mask[i] = ~req->mac_addr_mask[i];
+
+	if (vif->cfg.assoc) {
+		memcpy(cmd->range_req_bssid, vif->bss_conf.bssid, ETH_ALEN);
+
+		/* AP's TSF is only relevant if associated */
+		for (i = 0; i < req->n_peers; i++) {
+			if (req->peers[i].report_ap_tsf) {
+				struct iwl_mld_vif *mld_vif =
+					iwl_mld_vif_from_mac80211(vif);
+
+				cmd->tsf_mac_id = cpu_to_le32(mld_vif->fw_id);
+				return;
+			}
+		}
+	} else {
+		eth_broadcast_addr(cmd->range_req_bssid);
+	}
+
+	/* Don't report AP's TSF */
+	cmd->tsf_mac_id = cpu_to_le32(0xff);
+}
+
+static int
+iwl_mld_ftm_set_target_chandef(struct iwl_mld *mld,
+			       struct cfg80211_pmsr_request_peer *peer,
+			       struct iwl_tof_range_req_ap_entry *target)
+{
+	u32 freq = peer->chandef.chan->center_freq;
+
+	target->channel_num = ieee80211_frequency_to_channel(freq);
+
+	switch (peer->chandef.width) {
+	case NL80211_CHAN_WIDTH_20_NOHT:
+		target->format_bw = IWL_LOCATION_FRAME_FORMAT_LEGACY;
+		target->format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		break;
+	case NL80211_CHAN_WIDTH_20:
+		target->format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
+		target->format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		break;
+	case NL80211_CHAN_WIDTH_40:
+		target->format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
+		target->format_bw |= IWL_LOCATION_BW_40MHZ << LOCATION_BW_POS;
+		break;
+	case NL80211_CHAN_WIDTH_80:
+		target->format_bw = IWL_LOCATION_FRAME_FORMAT_VHT;
+		target->format_bw |= IWL_LOCATION_BW_80MHZ << LOCATION_BW_POS;
+		break;
+	case NL80211_CHAN_WIDTH_160:
+		target->format_bw = IWL_LOCATION_FRAME_FORMAT_HE;
+		target->format_bw |= IWL_LOCATION_BW_160MHZ << LOCATION_BW_POS;
+		break;
+	default:
+		IWL_ERR(mld, "Unsupported BW in FTM request (%d)\n",
+			peer->chandef.width);
+		return -EINVAL;
+	}
+
+	/* non EDCA based measurement must use HE preamble */
+	if (peer->ftm.trigger_based || peer->ftm.non_trigger_based)
+		target->format_bw |= IWL_LOCATION_FRAME_FORMAT_HE;
+
+	target->ctrl_ch_position =
+		(peer->chandef.width > NL80211_CHAN_WIDTH_20) ?
+		iwl_mld_get_fw_ctrl_pos(&peer->chandef) : 0;
+
+	target->band = iwl_mld_nl80211_band_to_fw(peer->chandef.chan->band);
+	return 0;
+}
+
+#define FTM_SET_FLAG(flag) (target->initiator_ap_flags |= \
+			    cpu_to_le32(IWL_INITIATOR_AP_FLAGS_##flag))
+
+static void
+iwl_mld_ftm_set_target_flags(struct iwl_mld *mld,
+			     struct cfg80211_pmsr_request_peer *peer,
+			     struct iwl_tof_range_req_ap_entry *target)
+{
+	target->initiator_ap_flags = cpu_to_le32(0);
+
+	if (peer->ftm.asap)
+		FTM_SET_FLAG(ASAP);
+
+	if (peer->ftm.request_lci)
+		FTM_SET_FLAG(LCI_REQUEST);
+
+	if (peer->ftm.request_civicloc)
+		FTM_SET_FLAG(CIVIC_REQUEST);
+
+	if (IWL_MLD_FTM_INITIATOR_DYNACK)
+		FTM_SET_FLAG(DYN_ACK);
+
+	if (IWL_MLD_FTM_INITIATOR_ALGO == IWL_TOF_ALGO_TYPE_LINEAR_REG)
+		FTM_SET_FLAG(ALGO_LR);
+	else if (IWL_MLD_FTM_INITIATOR_ALGO == IWL_TOF_ALGO_TYPE_FFT)
+		FTM_SET_FLAG(ALGO_FFT);
+
+	if (peer->ftm.trigger_based)
+		FTM_SET_FLAG(TB);
+	else if (peer->ftm.non_trigger_based)
+		FTM_SET_FLAG(NON_TB);
+
+	if ((peer->ftm.trigger_based || peer->ftm.non_trigger_based) &&
+	    peer->ftm.lmr_feedback)
+		FTM_SET_FLAG(LMR_FEEDBACK);
+}
+
+static void iwl_mld_ftm_set_sta(struct iwl_mld *mld, struct ieee80211_vif *vif,
+				struct cfg80211_pmsr_request_peer *peer,
+				struct iwl_tof_range_req_ap_entry *target)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	u32 sta_id_mask;
+
+	target->sta_id = IWL_INVALID_STA;
+
+	/* TODO: add ftm_unprotected debugfs support */
+
+	if (!vif->cfg.assoc || !mld_vif->ap_sta)
+		return;
+
+	sta_id_mask = iwl_mld_fw_sta_id_mask(mld, mld_vif->ap_sta);
+	if (WARN_ON(hweight32(sta_id_mask) != 1))
+		return;
+
+	target->sta_id = __ffs(sta_id_mask);
+
+	if (mld_vif->ap_sta->mfp &&
+	    (peer->ftm.trigger_based || peer->ftm.non_trigger_based))
+		FTM_SET_FLAG(PMF);
+}
+
+static int
+iwl_mld_ftm_set_target(struct iwl_mld *mld, struct ieee80211_vif *vif,
+		       struct cfg80211_pmsr_request_peer *peer,
+		       struct iwl_tof_range_req_ap_entry *target)
+{
+	u32 i2r_max_sts;
+	int ret;
+
+	ret = iwl_mld_ftm_set_target_chandef(mld, peer, target);
+	if (ret)
+		return ret;
+
+	memcpy(target->bssid, peer->addr, ETH_ALEN);
+	target->burst_period = cpu_to_le16(peer->ftm.burst_period);
+	target->samples_per_burst = peer->ftm.ftms_per_burst;
+	target->num_of_bursts = peer->ftm.num_bursts_exp;
+	iwl_mld_ftm_set_target_flags(mld, peer, target);
+	iwl_mld_ftm_set_sta(mld, vif, peer, target);
+
+	/* TODO: add secured ranging support */
+
+	i2r_max_sts = IWL_MLD_FTM_I2R_MAX_STS > 1 ? 1 :
+		IWL_MLD_FTM_I2R_MAX_STS;
+
+	target->r2i_ndp_params = IWL_MLD_FTM_R2I_MAX_REP |
+		(IWL_MLD_FTM_R2I_MAX_STS << IWL_LOCATION_MAX_STS_POS) |
+		(IWL_MLD_FTM_R2I_MAX_TOTAL_LTF << IWL_LOCATION_TOTAL_LTF_POS);
+	target->i2r_ndp_params = IWL_MLD_FTM_I2R_MAX_REP |
+		(i2r_max_sts << IWL_LOCATION_MAX_STS_POS) |
+		(IWL_MLD_FTM_I2R_MAX_TOTAL_LTF << IWL_LOCATION_TOTAL_LTF_POS);
+
+	if (peer->ftm.non_trigger_based) {
+		target->min_time_between_msr =
+			cpu_to_le16(IWL_MLD_FTM_NON_TB_MIN_TIME_BETWEEN_MSR);
+		target->burst_period =
+			cpu_to_le16(IWL_MLD_FTM_NON_TB_MAX_TIME_BETWEEN_MSR);
+	} else {
+		target->min_time_between_msr = cpu_to_le16(0);
+	}
+
+	/* TODO: Beacon interval is currently unknown, so use the common value
+	 * of 100 TUs.
+	 */
+	target->beacon_interval = cpu_to_le16(100);
+
+	return 0;
+}
+
+int iwl_mld_ftm_start(struct iwl_mld *mld, struct ieee80211_vif *vif,
+		      struct cfg80211_pmsr_request *req)
+{
+	struct iwl_tof_range_req_cmd cmd;
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(LOCATION_GROUP, TOF_RANGE_REQ_CMD),
+		.dataflags[0] = IWL_HCMD_DFL_DUP,
+		.data[0] = &cmd,
+		.len[0] = sizeof(cmd),
+	};
+	u8 i;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (mld->ftm_initiator.req)
+		return -EBUSY;
+
+	if (req->n_peers > ARRAY_SIZE(cmd.ap))
+		return -EINVAL;
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	iwl_mld_ftm_cmd_common(mld, vif, (void *)&cmd, req);
+
+	for (i = 0; i < cmd.num_of_ap; i++) {
+		struct cfg80211_pmsr_request_peer *peer = &req->peers[i];
+		struct iwl_tof_range_req_ap_entry *target = &cmd.ap[i];
+
+		ret = iwl_mld_ftm_set_target(mld, vif, peer, target);
+		if (ret)
+			return ret;
+	}
+
+	/* TODO: get the status from the response*/
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+	if (!ret) {
+		mld->ftm_initiator.req = req;
+		mld->ftm_initiator.req_wdev = ieee80211_vif_to_wdev(vif);
+	}
+
+	return ret;
+}
+
+static void iwl_mld_ftm_reset(struct iwl_mld *mld)
+{
+	lockdep_assert_wiphy(mld->wiphy);
+
+	mld->ftm_initiator.req = NULL;
+	mld->ftm_initiator.req_wdev = NULL;
+	memset(mld->ftm_initiator.responses, 0,
+	       sizeof(mld->ftm_initiator.responses));
+}
+
+static int iwl_mld_ftm_range_resp_valid(struct iwl_mld *mld, u8 request_id,
+					u8 num_of_aps)
+{
+	if (IWL_FW_CHECK(mld, request_id != (u8)mld->ftm_initiator.req->cookie,
+			 "Request ID mismatch, got %u, active %u\n",
+			 request_id, (u8)mld->ftm_initiator.req->cookie))
+		return -EINVAL;
+
+	if (IWL_FW_CHECK(mld, num_of_aps > mld->ftm_initiator.req->n_peers ||
+			 num_of_aps > IWL_TOF_MAX_APS,
+			 "FTM range response: invalid num of APs (%u)\n",
+			 num_of_aps))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int iwl_mld_ftm_find_peer(struct cfg80211_pmsr_request *req,
+				 const u8 *addr)
+{
+	for (int i = 0; i < req->n_peers; i++) {
+		struct cfg80211_pmsr_request_peer *peer = &req->peers[i];
+
+		if (ether_addr_equal_unaligned(peer->addr, addr))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+static void iwl_mld_debug_range_resp(struct iwl_mld *mld, u8 index,
+				     struct cfg80211_pmsr_result *res)
+{
+	s64 rtt_avg = div_s64(res->ftm.rtt_avg * 100, 6666);
+
+	IWL_DEBUG_INFO(mld, "entry %d\n", index);
+	IWL_DEBUG_INFO(mld, "\tstatus: %d\n", res->status);
+	IWL_DEBUG_INFO(mld, "\tBSSID: %pM\n", res->addr);
+	IWL_DEBUG_INFO(mld, "\thost time: %llu\n", res->host_time);
+	IWL_DEBUG_INFO(mld, "\tburst index: %d\n", res->ftm.burst_index);
+	IWL_DEBUG_INFO(mld, "\tsuccess num: %u\n", res->ftm.num_ftmr_successes);
+	IWL_DEBUG_INFO(mld, "\trssi: %d\n", res->ftm.rssi_avg);
+	IWL_DEBUG_INFO(mld, "\trssi spread: %d\n", res->ftm.rssi_spread);
+	IWL_DEBUG_INFO(mld, "\trtt: %lld\n", res->ftm.rtt_avg);
+	IWL_DEBUG_INFO(mld, "\trtt var: %llu\n", res->ftm.rtt_variance);
+	IWL_DEBUG_INFO(mld, "\trtt spread: %llu\n", res->ftm.rtt_spread);
+	IWL_DEBUG_INFO(mld, "\tdistance: %lld\n", rtt_avg);
+}
+
+void iwl_mld_handle_ftm_resp_notif(struct iwl_mld *mld,
+				   struct iwl_rx_packet *pkt)
+{
+	struct iwl_tof_range_rsp_ntfy *fw_resp = (void *)pkt->data;
+	u8 num_of_aps, last_in_batch;
+
+	if (IWL_FW_CHECK(mld, !mld->ftm_initiator.req,
+			 "FTM response without a pending request\n"))
+		return;
+
+	if (iwl_mld_ftm_range_resp_valid(mld, fw_resp->request_id,
+					 fw_resp->num_of_aps))
+		return;
+
+	num_of_aps = fw_resp->num_of_aps;
+	last_in_batch = fw_resp->last_report;
+
+	IWL_DEBUG_INFO(mld, "Range response received\n");
+	IWL_DEBUG_INFO(mld, "request id: %llu, num of entries: %u\n",
+		       mld->ftm_initiator.req->cookie, num_of_aps);
+
+	for (int i = 0; i < num_of_aps; i++) {
+		struct cfg80211_pmsr_result result = {};
+		struct iwl_tof_range_rsp_ap_entry_ntfy *fw_ap;
+		int peer_idx;
+
+		fw_ap = &fw_resp->ap[i];
+		result.final = fw_ap->last_burst;
+		result.ap_tsf = le32_to_cpu(fw_ap->start_tsf);
+		result.ap_tsf_valid = 1;
+
+		peer_idx = iwl_mld_ftm_find_peer(mld->ftm_initiator.req,
+						 fw_ap->bssid);
+		if (peer_idx < 0) {
+			IWL_WARN(mld,
+				 "Unknown address (%pM, target #%d) in FTM response\n",
+				 fw_ap->bssid, i);
+			continue;
+		}
+
+		switch (fw_ap->measure_status) {
+		case IWL_TOF_ENTRY_SUCCESS:
+			result.status = NL80211_PMSR_STATUS_SUCCESS;
+			break;
+		case IWL_TOF_ENTRY_TIMING_MEASURE_TIMEOUT:
+			result.status = NL80211_PMSR_STATUS_TIMEOUT;
+			break;
+		case IWL_TOF_ENTRY_NO_RESPONSE:
+			result.status = NL80211_PMSR_STATUS_FAILURE;
+			result.ftm.failure_reason =
+				NL80211_PMSR_FTM_FAILURE_NO_RESPONSE;
+			break;
+		case IWL_TOF_ENTRY_REQUEST_REJECTED:
+			result.status = NL80211_PMSR_STATUS_FAILURE;
+			result.ftm.failure_reason =
+				NL80211_PMSR_FTM_FAILURE_PEER_BUSY;
+			result.ftm.busy_retry_time = fw_ap->refusal_period;
+			break;
+		default:
+			result.status = NL80211_PMSR_STATUS_FAILURE;
+			result.ftm.failure_reason =
+				NL80211_PMSR_FTM_FAILURE_UNSPECIFIED;
+			break;
+		}
+		memcpy(result.addr, fw_ap->bssid, ETH_ALEN);
+
+		/* TODO: convert the timestamp from the result to systime */
+		result.host_time = ktime_get_boottime_ns();
+
+		result.type = NL80211_PMSR_TYPE_FTM;
+		result.ftm.burst_index = mld->ftm_initiator.responses[peer_idx];
+		mld->ftm_initiator.responses[peer_idx]++;
+		result.ftm.rssi_avg = fw_ap->rssi;
+		result.ftm.rssi_avg_valid = 1;
+		result.ftm.rssi_spread = fw_ap->rssi_spread;
+		result.ftm.rssi_spread_valid = 1;
+		result.ftm.rtt_avg = (s32)le32_to_cpu(fw_ap->rtt);
+		result.ftm.rtt_avg_valid = 1;
+		result.ftm.rtt_variance = le32_to_cpu(fw_ap->rtt_variance);
+		result.ftm.rtt_variance_valid = 1;
+		result.ftm.rtt_spread = le32_to_cpu(fw_ap->rtt_spread);
+		result.ftm.rtt_spread_valid = 1;
+
+		cfg80211_pmsr_report(mld->ftm_initiator.req_wdev,
+				     mld->ftm_initiator.req,
+				     &result, GFP_KERNEL);
+
+		if (fw_has_api(&mld->fw->ucode_capa,
+			       IWL_UCODE_TLV_API_FTM_RTT_ACCURACY))
+			IWL_DEBUG_INFO(mld, "RTT confidence: %u\n",
+				       fw_ap->rttConfidence);
+
+		iwl_mld_debug_range_resp(mld, i, &result);
+	}
+
+	if (last_in_batch) {
+		cfg80211_pmsr_complete(mld->ftm_initiator.req_wdev,
+				       mld->ftm_initiator.req,
+				       GFP_KERNEL);
+		iwl_mld_ftm_reset(mld);
+	}
+}
+
+void iwl_mld_ftm_restart_cleanup(struct iwl_mld *mld)
+{
+	struct cfg80211_pmsr_result result = {
+		.status = NL80211_PMSR_STATUS_FAILURE,
+		.final = 1,
+		.host_time = ktime_get_boottime_ns(),
+		.type = NL80211_PMSR_TYPE_FTM,
+	};
+
+	if (!mld->ftm_initiator.req)
+		return;
+
+	for (int i = 0; i < mld->ftm_initiator.req->n_peers; i++) {
+		memcpy(result.addr, mld->ftm_initiator.req->peers[i].addr,
+		       ETH_ALEN);
+
+		cfg80211_pmsr_report(mld->ftm_initiator.req_wdev,
+				     mld->ftm_initiator.req,
+				     &result, GFP_KERNEL);
+	}
+
+	cfg80211_pmsr_complete(mld->ftm_initiator.req_wdev,
+			       mld->ftm_initiator.req, GFP_KERNEL);
+	iwl_mld_ftm_reset(mld);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/ftm-initiator.h b/sys/contrib/dev/iwlwifi/mld/ftm-initiator.h
new file mode 100644
index 000000000000..3fab25a52508
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/ftm-initiator.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+#ifndef __iwl_mld_ftm_initiator_h__
+#define __iwl_mld_ftm_initiator_h__
+
+/**
+ * struct ftm_initiator_data - FTM initiator data
+ *
+ * @req: a pointer to cfg80211 FTM request
+ * @req_wdev: a pointer to the wdev that requested the current FTM request
+ * @responses: the number of responses received for the current FTM session.
+ *	Used for tracking the burst index in a periodic request.
+ */
+struct ftm_initiator_data {
+		struct cfg80211_pmsr_request *req;
+		struct wireless_dev *req_wdev;
+		int responses[IWL_TOF_MAX_APS];
+};
+
+int iwl_mld_ftm_start(struct iwl_mld *mld, struct ieee80211_vif *vif,
+		      struct cfg80211_pmsr_request *req);
+
+void iwl_mld_handle_ftm_resp_notif(struct iwl_mld *mld,
+				   struct iwl_rx_packet *pkt);
+void iwl_mld_ftm_restart_cleanup(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_ftm_initiator_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/fw.c b/sys/contrib/dev/iwlwifi/mld/fw.c
new file mode 100644
index 000000000000..b372173c4a79
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/fw.c
@@ -0,0 +1,554 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include "mld.h"
+
+#include "fw/api/alive.h"
+#include "fw/api/scan.h"
+#include "fw/api/rx.h"
+#include "phy.h"
+#include "fw/dbg.h"
+#include "fw/pnvm.h"
+#include "hcmd.h"
+#include "power.h"
+#include "mcc.h"
+#include "led.h"
+#include "coex.h"
+#include "regulatory.h"
+#include "thermal.h"
+
+static int iwl_mld_send_tx_ant_cfg(struct iwl_mld *mld)
+{
+	struct iwl_tx_ant_cfg_cmd cmd;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	cmd.valid = cpu_to_le32(iwl_mld_get_valid_tx_ant(mld));
+
+	IWL_DEBUG_FW(mld, "select valid tx ant: %u\n", cmd.valid);
+
+	return iwl_mld_send_cmd_pdu(mld, TX_ANT_CONFIGURATION_CMD, &cmd);
+}
+
+static int iwl_mld_send_rss_cfg_cmd(struct iwl_mld *mld)
+{
+	struct iwl_rss_config_cmd cmd = {
+		.flags = cpu_to_le32(IWL_RSS_ENABLE),
+		.hash_mask = BIT(IWL_RSS_HASH_TYPE_IPV4_TCP) |
+			     BIT(IWL_RSS_HASH_TYPE_IPV4_UDP) |
+			     BIT(IWL_RSS_HASH_TYPE_IPV4_PAYLOAD) |
+			     BIT(IWL_RSS_HASH_TYPE_IPV6_TCP) |
+			     BIT(IWL_RSS_HASH_TYPE_IPV6_UDP) |
+			     BIT(IWL_RSS_HASH_TYPE_IPV6_PAYLOAD),
+	};
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* Do not direct RSS traffic to Q 0 which is our fallback queue */
+	for (int i = 0; i < ARRAY_SIZE(cmd.indirection_table); i++)
+		cmd.indirection_table[i] =
+			1 + (i % (mld->trans->info.num_rxqs - 1));
+	netdev_rss_key_fill(cmd.secret_key, sizeof(cmd.secret_key));
+
+	return iwl_mld_send_cmd_pdu(mld, RSS_CONFIG_CMD, &cmd);
+}
+
+static int iwl_mld_config_scan(struct iwl_mld *mld)
+{
+	struct iwl_scan_config cmd = {
+		.tx_chains = cpu_to_le32(iwl_mld_get_valid_tx_ant(mld)),
+		.rx_chains = cpu_to_le32(iwl_mld_get_valid_rx_ant(mld))
+	};
+
+	return iwl_mld_send_cmd_pdu(mld, WIDE_ID(LONG_GROUP, SCAN_CFG_CMD),
+				    &cmd);
+}
+
+static void iwl_mld_alive_imr_data(struct iwl_trans *trans,
+				   const struct iwl_imr_alive_info *imr_info)
+{
+	struct iwl_imr_data *imr_data = &trans->dbg.imr_data;
+
+	imr_data->imr_enable = le32_to_cpu(imr_info->enabled);
+	imr_data->imr_size = le32_to_cpu(imr_info->size);
+	imr_data->imr2sram_remainbyte = imr_data->imr_size;
+	imr_data->imr_base_addr = imr_info->base_addr;
+	imr_data->imr_curr_addr = le64_to_cpu(imr_data->imr_base_addr);
+
+	if (imr_data->imr_enable)
+		return;
+
+	for (int i = 0; i < ARRAY_SIZE(trans->dbg.active_regions); i++) {
+		struct iwl_fw_ini_region_tlv *reg;
+
+		if (!trans->dbg.active_regions[i])
+			continue;
+
+		reg = (void *)trans->dbg.active_regions[i]->data;
+
+		/* We have only one DRAM IMR region, so we
+		 * can break as soon as we find the first
+		 * one.
+		 */
+		if (reg->type == IWL_FW_INI_REGION_DRAM_IMR) {
+			trans->dbg.unsupported_region_msk |= BIT(i);
+			break;
+		}
+	}
+}
+
+struct iwl_mld_alive_data {
+	__le32 sku_id[3];
+	bool valid;
+};
+
+static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
+			 struct iwl_rx_packet *pkt, void *data)
+{
+	unsigned int pkt_len = iwl_rx_packet_payload_len(pkt);
+	unsigned int expected_sz;
+	struct iwl_mld *mld =
+		container_of(notif_wait, struct iwl_mld, notif_wait);
+	struct iwl_trans *trans = mld->trans;
+	u32 version = iwl_fw_lookup_notif_ver(mld->fw, LEGACY_GROUP,
+					      UCODE_ALIVE_NTFY, 0);
+	struct iwl_mld_alive_data *alive_data = data;
+	struct iwl_alive_ntf *palive;
+	struct iwl_umac_alive *umac;
+	struct iwl_lmac_alive *lmac1;
+	struct iwl_lmac_alive *lmac2 = NULL;
+	u32 lmac_error_event_table;
+	u32 umac_error_table;
+	u16 status;
+
+	switch (version) {
+	case 6:
+	case 7:
+		expected_sz = sizeof(struct iwl_alive_ntf_v6);
+		break;
+	case 8:
+		expected_sz = sizeof(struct iwl_alive_ntf);
+		break;
+	default:
+		return false;
+	}
+
+	if (pkt_len != expected_sz)
+		return false;
+
+	palive = (void *)pkt->data;
+
+	iwl_mld_alive_imr_data(trans, &palive->imr);
+
+	umac = &palive->umac_data;
+	lmac1 = &palive->lmac_data[0];
+	lmac2 = &palive->lmac_data[1];
+	status = le16_to_cpu(palive->status);
+
+	BUILD_BUG_ON(sizeof(alive_data->sku_id) !=
+		     sizeof(palive->sku_id.data));
+	memcpy(alive_data->sku_id, palive->sku_id.data,
+	       sizeof(palive->sku_id.data));
+
+	IWL_DEBUG_FW(mld, "Got sku_id: 0x0%x 0x0%x 0x0%x\n",
+		     le32_to_cpu(alive_data->sku_id[0]),
+		     le32_to_cpu(alive_data->sku_id[1]),
+		     le32_to_cpu(alive_data->sku_id[2]));
+
+	lmac_error_event_table =
+		le32_to_cpu(lmac1->dbg_ptrs.error_event_table_ptr);
+	iwl_fw_lmac1_set_alive_err_table(trans, lmac_error_event_table);
+
+	if (lmac2)
+		trans->dbg.lmac_error_event_table[1] =
+			le32_to_cpu(lmac2->dbg_ptrs.error_event_table_ptr);
+
+	umac_error_table = le32_to_cpu(umac->dbg_ptrs.error_info_addr) &
+		~FW_ADDR_CACHE_CONTROL;
+
+	if (umac_error_table >= trans->mac_cfg->base->min_umac_error_event_table)
+		iwl_fw_umac_set_alive_err_table(trans, umac_error_table);
+	else
+		IWL_ERR(mld, "Not valid error log pointer 0x%08X\n",
+			umac_error_table);
+
+	alive_data->valid = status == IWL_ALIVE_STATUS_OK;
+
+	IWL_DEBUG_FW(mld,
+		     "Alive ucode status 0x%04x revision 0x%01X 0x%01X\n",
+		     status, lmac1->ver_type, lmac1->ver_subtype);
+
+	if (lmac2)
+		IWL_DEBUG_FW(mld, "Alive ucode CDB\n");
+
+	IWL_DEBUG_FW(mld,
+		     "UMAC version: Major - 0x%x, Minor - 0x%x\n",
+		     le32_to_cpu(umac->umac_major),
+		     le32_to_cpu(umac->umac_minor));
+
+	if (version >= 7)
+		IWL_DEBUG_FW(mld, "FW alive flags 0x%x\n",
+			     le16_to_cpu(palive->flags));
+
+	if (version >= 8)
+		IWL_DEBUG_FW(mld, "platform_id 0x%llx\n",
+			     le64_to_cpu(palive->platform_id));
+
+	iwl_fwrt_update_fw_versions(&mld->fwrt, lmac1, umac);
+
+	return true;
+}
+
+#define MLD_ALIVE_TIMEOUT		(2 * HZ)
+#define MLD_INIT_COMPLETE_TIMEOUT	(2 * HZ)
+
+static void iwl_mld_print_alive_notif_timeout(struct iwl_mld *mld)
+{
+	struct iwl_trans *trans = mld->trans;
+	struct iwl_pc_data *pc_data;
+	u8 count;
+
+	IWL_ERR(mld,
+		"SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
+		iwl_read_umac_prph(trans, UMAG_SB_CPU_1_STATUS),
+		iwl_read_umac_prph(trans,
+				   UMAG_SB_CPU_2_STATUS));
+#define IWL_FW_PRINT_REG_INFO(reg_name) \
+	IWL_ERR(mld, #reg_name ": 0x%x\n", iwl_read_umac_prph(trans, reg_name))
+
+	IWL_FW_PRINT_REG_INFO(WFPM_LMAC1_PD_NOTIFICATION);
+
+	IWL_FW_PRINT_REG_INFO(HPM_SECONDARY_DEVICE_STATE);
+
+	/* print OTP info */
+	IWL_FW_PRINT_REG_INFO(WFPM_MAC_OTP_CFG7_ADDR);
+	IWL_FW_PRINT_REG_INFO(WFPM_MAC_OTP_CFG7_DATA);
+#undef IWL_FW_PRINT_REG_INFO
+
+	pc_data = trans->dbg.pc_data;
+	for (count = 0; count < trans->dbg.num_pc; count++, pc_data++)
+		IWL_ERR(mld, "%s: 0x%x\n", pc_data->pc_name,
+			pc_data->pc_address);
+}
+
+static int iwl_mld_load_fw_wait_alive(struct iwl_mld *mld,
+				      struct iwl_mld_alive_data *alive_data)
+{
+	static const u16 alive_cmd[] = { UCODE_ALIVE_NTFY };
+	struct iwl_notification_wait alive_wait;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	iwl_init_notification_wait(&mld->notif_wait, &alive_wait,
+				   alive_cmd, ARRAY_SIZE(alive_cmd),
+				   iwl_alive_fn, alive_data);
+
+	iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_POINT_EARLY, NULL);
+
+	ret = iwl_trans_start_fw(mld->trans, mld->fw, IWL_UCODE_REGULAR, true);
+	if (ret) {
+		iwl_remove_notification(&mld->notif_wait, &alive_wait);
+		return ret;
+	}
+
+	ret = iwl_wait_notification(&mld->notif_wait, &alive_wait,
+				    MLD_ALIVE_TIMEOUT);
+
+	if (ret) {
+		if (ret == -ETIMEDOUT)
+			iwl_fw_dbg_error_collect(&mld->fwrt,
+						 FW_DBG_TRIGGER_ALIVE_TIMEOUT);
+		iwl_mld_print_alive_notif_timeout(mld);
+		return ret;
+	}
+
+	if (!alive_data->valid) {
+		IWL_ERR(mld, "Loaded firmware is not valid!\n");
+		return -EIO;
+	}
+
+	iwl_trans_fw_alive(mld->trans);
+
+	return 0;
+}
+
+static int iwl_mld_run_fw_init_sequence(struct iwl_mld *mld)
+{
+	struct iwl_notification_wait init_wait;
+	struct iwl_init_extended_cfg_cmd init_cfg = {
+		.init_flags = cpu_to_le32(BIT(IWL_INIT_PHY)),
+	};
+	struct iwl_mld_alive_data alive_data = {};
+	static const u16 init_complete[] = {
+		INIT_COMPLETE_NOTIF,
+	};
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_load_fw_wait_alive(mld, &alive_data);
+	if (ret)
+		return ret;
+
+	ret = iwl_pnvm_load(mld->trans, &mld->notif_wait,
+			    mld->fw, alive_data.sku_id);
+	if (ret) {
+		IWL_ERR(mld, "Timeout waiting for PNVM load %d\n", ret);
+		return ret;
+	}
+
+	iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_POINT_AFTER_ALIVE,
+			       NULL);
+
+	iwl_init_notification_wait(&mld->notif_wait,
+				   &init_wait,
+				   init_complete,
+				   ARRAY_SIZE(init_complete),
+				   NULL, NULL);
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(SYSTEM_GROUP, INIT_EXTENDED_CFG_CMD),
+				   &init_cfg);
+	if (ret) {
+		IWL_ERR(mld, "Failed to send init config command: %d\n", ret);
+		iwl_remove_notification(&mld->notif_wait, &init_wait);
+		return ret;
+	}
+
+	ret = iwl_mld_send_phy_cfg_cmd(mld);
+	if (ret) {
+		IWL_ERR(mld, "Failed to send PHY config command: %d\n", ret);
+		iwl_remove_notification(&mld->notif_wait, &init_wait);
+		return ret;
+	}
+
+	ret = iwl_wait_notification(&mld->notif_wait, &init_wait,
+				    MLD_INIT_COMPLETE_TIMEOUT);
+	if (ret) {
+		IWL_ERR(mld, "Failed to get INIT_COMPLETE %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int iwl_mld_load_fw(struct iwl_mld *mld)
+{
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_trans_start_hw(mld->trans);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_run_fw_init_sequence(mld);
+	if (ret)
+		goto err;
+
+	mld->fw_status.running = true;
+
+	return 0;
+err:
+	iwl_mld_stop_fw(mld);
+	return ret;
+}
+
+void iwl_mld_stop_fw(struct iwl_mld *mld)
+{
+	lockdep_assert_wiphy(mld->wiphy);
+
+	iwl_abort_notification_waits(&mld->notif_wait);
+
+	iwl_fw_dbg_stop_sync(&mld->fwrt);
+
+	iwl_trans_stop_device(mld->trans);
+
+	/* HW is stopped, no more coming RX. Cancel all notifications in
+	 * case they were sent just before stopping the HW.
+	 */
+	iwl_mld_cancel_async_notifications(mld);
+
+	mld->fw_status.running = false;
+}
+
+static void iwl_mld_restart_disconnect_iter(void *data, u8 *mac,
+					    struct ieee80211_vif *vif)
+{
+	if (vif->type == NL80211_IFTYPE_STATION)
+		ieee80211_hw_restart_disconnect(vif);
+}
+
+void iwl_mld_send_recovery_cmd(struct iwl_mld *mld, u32 flags)
+{
+	u32 error_log_size = mld->fw->ucode_capa.error_log_size;
+	struct iwl_fw_error_recovery_cmd recovery_cmd = {
+		.flags = cpu_to_le32(flags),
+	};
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(SYSTEM_GROUP, FW_ERROR_RECOVERY_CMD),
+		.flags = CMD_WANT_SKB,
+		.data = {&recovery_cmd, },
+		.len = {sizeof(recovery_cmd), },
+	};
+	int ret;
+
+	/* no error log was defined in TLV */
+	if (!error_log_size)
+		return;
+
+	if (flags & ERROR_RECOVERY_UPDATE_DB) {
+		/* no buf was allocated upon NIC error */
+		if (!mld->error_recovery_buf)
+			return;
+
+		cmd.data[1] = mld->error_recovery_buf;
+		cmd.len[1] =  error_log_size;
+		cmd.dataflags[1] = IWL_HCMD_DFL_NOCOPY;
+		recovery_cmd.buf_size = cpu_to_le32(error_log_size);
+	}
+
+	ret = iwl_mld_send_cmd(mld, &cmd);
+
+	/* we no longer need the recovery buffer */
+	kfree(mld->error_recovery_buf);
+	mld->error_recovery_buf = NULL;
+
+	if (ret) {
+		IWL_ERR(mld, "Failed to send recovery cmd %d\n", ret);
+		return;
+	}
+
+	if (flags & ERROR_RECOVERY_UPDATE_DB) {
+		struct iwl_rx_packet *pkt = cmd.resp_pkt;
+		u32 pkt_len = iwl_rx_packet_payload_len(pkt);
+		u32 resp;
+
+		if (IWL_FW_CHECK(mld, pkt_len != sizeof(resp),
+				 "Unexpected recovery cmd response size %u (expected %zu)\n",
+				 pkt_len, sizeof(resp)))
+			goto out;
+
+		resp = le32_to_cpup((__le32 *)cmd.resp_pkt->data);
+		if (!resp)
+			goto out;
+
+		IWL_ERR(mld,
+			"Failed to send recovery cmd blob was invalid %d\n",
+			resp);
+
+		ieee80211_iterate_interfaces(mld->hw, 0,
+					     iwl_mld_restart_disconnect_iter,
+					     NULL);
+	}
+
+out:
+	iwl_free_resp(&cmd);
+}
+
+static int iwl_mld_config_fw(struct iwl_mld *mld)
+{
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	iwl_fw_disable_dbg_asserts(&mld->fwrt);
+	iwl_get_shared_mem_conf(&mld->fwrt);
+
+	ret = iwl_mld_send_tx_ant_cfg(mld);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_send_bt_init_conf(mld);
+	if (ret)
+		return ret;
+
+	ret = iwl_set_soc_latency(&mld->fwrt);
+	if (ret)
+		return ret;
+
+	iwl_mld_configure_lari(mld);
+
+	ret = iwl_mld_config_temp_report_ths(mld);
+	if (ret)
+		return ret;
+
+#ifdef CONFIG_THERMAL
+	ret = iwl_mld_config_ctdp(mld, mld->cooling_dev.cur_state,
+				  CTDP_CMD_OPERATION_START);
+	if (ret)
+		return ret;
+#endif
+
+	ret = iwl_configure_rxq(&mld->fwrt);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_send_rss_cfg_cmd(mld);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_config_scan(mld);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_update_device_power(mld, false);
+	if (ret)
+		return ret;
+
+	if (mld->fw_status.in_hw_restart) {
+		iwl_mld_send_recovery_cmd(mld, ERROR_RECOVERY_UPDATE_DB);
+		iwl_mld_time_sync_fw_config(mld);
+	}
+
+	iwl_mld_led_config_fw(mld);
+
+	ret = iwl_mld_init_ppag(mld);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_init_sar(mld);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_init_sgom(mld);
+	if (ret)
+		return ret;
+
+	iwl_mld_init_tas(mld);
+	iwl_mld_init_uats(mld);
+
+	return 0;
+}
+
+int iwl_mld_start_fw(struct iwl_mld *mld)
+{
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_load_fw(mld);
+	if (IWL_FW_CHECK(mld, ret, "Failed to start firmware %d\n", ret)) {
+		iwl_fw_dbg_error_collect(&mld->fwrt, FW_DBG_TRIGGER_DRIVER);
+		return ret;
+	}
+
+	IWL_DEBUG_INFO(mld, "uCode started.\n");
+
+	ret = iwl_mld_config_fw(mld);
+	if (ret)
+		goto error;
+
+	ret = iwl_mld_init_mcc(mld);
+	if (ret)
+		goto error;
+
+	return 0;
+
+error:
+	iwl_mld_stop_fw(mld);
+	return ret;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/hcmd.h b/sys/contrib/dev/iwlwifi/mld/hcmd.h
new file mode 100644
index 000000000000..64a8d4248324
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/hcmd.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_hcmd_h__
+#define __iwl_mld_hcmd_h__
+
+static inline int iwl_mld_send_cmd(struct iwl_mld *mld, struct iwl_host_cmd *cmd)
+{
+	/* No commands, including the d3 related commands, should be sent
+	 * after entering d3
+	 */
+#ifdef CONFIG_PM_SLEEP
+	if (WARN_ON(mld->fw_status.in_d3))
+		return -EIO;
+#endif
+
+	if (!(cmd->flags & CMD_ASYNC))
+		lockdep_assert_wiphy(mld->wiphy);
+
+	/* Devices that need to shutdown immediately on rfkill are not
+	 * supported, so we can send all the cmds in rfkill
+	 */
+	cmd->flags |= CMD_SEND_IN_RFKILL;
+
+	return iwl_trans_send_cmd(mld->trans, cmd);
+}
+
+static inline int
+__iwl_mld_send_cmd_with_flags_pdu(struct iwl_mld *mld, u32 id,
+				  u32 flags, const void *data, u16 len)
+{
+	struct iwl_host_cmd cmd = {
+		.id = id,
+		.len = { data ? len : 0, },
+		.data = { data, },
+		.flags = flags,
+	};
+
+	return iwl_mld_send_cmd(mld, &cmd);
+}
+
+#define _iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, len,	\
+					 ignored...)			\
+	__iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, len)
+#define iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, len...)	\
+	_iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, ##len,	\
+					 sizeof(*(data)))
+
+#define iwl_mld_send_cmd_pdu(mld, id, ...)				\
+	iwl_mld_send_cmd_with_flags_pdu(mld, id, 0, __VA_ARGS__)
+
+#define iwl_mld_send_cmd_empty(mld, id)					\
+	iwl_mld_send_cmd_with_flags_pdu(mld, id, 0, NULL, 0)
+
+#endif /* __iwl_mld_hcmd_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/iface.c b/sys/contrib/dev/iwlwifi/mld/iface.c
new file mode 100644
index 000000000000..38993d65c052
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/iface.c
@@ -0,0 +1,707 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <net/cfg80211.h>
+
+#include "iface.h"
+#include "hcmd.h"
+#include "key.h"
+#include "mlo.h"
+#include "mac80211.h"
+
+#include "fw/api/context.h"
+#include "fw/api/mac.h"
+#include "fw/api/time-event.h"
+#include "fw/api/datapath.h"
+
+/* Cleanup function for struct iwl_mld_vif, will be called in restart */
+void iwl_mld_cleanup_vif(void *data, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld *mld = mld_vif->mld;
+	struct iwl_mld_link *link;
+
+	mld_vif->emlsr.blocked_reasons &= ~IWL_MLD_EMLSR_BLOCKED_ROC;
+
+	if (mld_vif->aux_sta.sta_id != IWL_INVALID_STA)
+		iwl_mld_free_internal_sta(mld, &mld_vif->aux_sta);
+
+	/* EMLSR is turned back on during recovery */
+	vif->driver_flags &= ~IEEE80211_VIF_EML_ACTIVE;
+
+	if (mld_vif->roc_activity != ROC_NUM_ACTIVITIES)
+		ieee80211_remain_on_channel_expired(mld->hw);
+
+	mld_vif->roc_activity = ROC_NUM_ACTIVITIES;
+
+	for_each_mld_vif_valid_link(mld_vif, link) {
+		iwl_mld_cleanup_link(mld_vif->mld, link);
+
+		/* Correctly allocated primary link in non-MLO mode */
+		if (!ieee80211_vif_is_mld(vif) &&
+		    link_id == 0 && link == &mld_vif->deflink)
+			continue;
+
+		if (vif->active_links & BIT(link_id))
+			continue;
+
+		/* Should not happen as link removal should always succeed */
+		WARN_ON(1);
+		if (link != &mld_vif->deflink)
+			kfree_rcu(link, rcu_head);
+		RCU_INIT_POINTER(mld_vif->link[link_id], NULL);
+	}
+
+	ieee80211_iter_keys(mld->hw, vif, iwl_mld_cleanup_keys_iter, NULL);
+
+	wiphy_delayed_work_cancel(mld->wiphy, &mld_vif->mlo_scan_start_wk);
+
+	CLEANUP_STRUCT(mld_vif);
+}
+
+static int iwl_mld_send_mac_cmd(struct iwl_mld *mld,
+				struct iwl_mac_config_cmd *cmd)
+{
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(MAC_CONF_GROUP, MAC_CONFIG_CMD),
+				   cmd);
+	if (ret)
+		IWL_ERR(mld, "Failed to send MAC_CONFIG_CMD ret = %d\n", ret);
+
+	return ret;
+}
+
+int iwl_mld_mac80211_iftype_to_fw(const struct ieee80211_vif *vif)
+{
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		return vif->p2p ? FW_MAC_TYPE_P2P_STA : FW_MAC_TYPE_BSS_STA;
+	case NL80211_IFTYPE_AP:
+		return FW_MAC_TYPE_GO;
+	case NL80211_IFTYPE_MONITOR:
+		return FW_MAC_TYPE_LISTENER;
+	case NL80211_IFTYPE_P2P_DEVICE:
+		return FW_MAC_TYPE_P2P_DEVICE;
+	case NL80211_IFTYPE_ADHOC:
+		return FW_MAC_TYPE_IBSS;
+	default:
+		WARN_ON_ONCE(1);
+	}
+	return FW_MAC_TYPE_BSS_STA;
+}
+
+static bool iwl_mld_is_nic_ack_enabled(struct iwl_mld *mld,
+				       struct ieee80211_vif *vif)
+{
+	const struct ieee80211_supported_band *sband;
+	const struct ieee80211_sta_he_cap *own_he_cap;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* This capability is the same for all bands,
+	 * so take it from one of them.
+	 */
+	sband = mld->hw->wiphy->bands[NL80211_BAND_2GHZ];
+	own_he_cap = ieee80211_get_he_iftype_cap_vif(sband, vif);
+
+	return own_he_cap && (own_he_cap->he_cap_elem.mac_cap_info[2] &
+			       IEEE80211_HE_MAC_CAP2_ACK_EN);
+}
+
+static void iwl_mld_set_he_support(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct iwl_mac_config_cmd *cmd,
+				   int cmd_ver)
+{
+	if (vif->type == NL80211_IFTYPE_AP) {
+		if (cmd_ver == 2)
+			cmd->wifi_gen_v2.he_ap_support = cpu_to_le16(1);
+		else
+			cmd->wifi_gen.he_ap_support = 1;
+	} else {
+		if (cmd_ver == 2)
+			cmd->wifi_gen_v2.he_support = cpu_to_le16(1);
+		else
+			cmd->wifi_gen.he_support = 1;
+	}
+}
+
+/* fill the common part for all interface types */
+static void iwl_mld_mac_cmd_fill_common(struct iwl_mld *mld,
+					struct ieee80211_vif *vif,
+					struct iwl_mac_config_cmd *cmd,
+					u32 action)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_bss_conf *link_conf;
+	unsigned int link_id;
+	int cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw,
+					    WIDE_ID(MAC_CONF_GROUP,
+						    MAC_CONFIG_CMD), 0);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	cmd->id_and_color = cpu_to_le32(mld_vif->fw_id);
+	cmd->action = cpu_to_le32(action);
+
+	cmd->mac_type =
+		cpu_to_le32(iwl_mld_mac80211_iftype_to_fw(vif));
+
+	memcpy(cmd->local_mld_addr, vif->addr, ETH_ALEN);
+
+	if (iwlwifi_mod_params.disable_11ax)
+		return;
+
+	cmd->nic_not_ack_enabled =
+		cpu_to_le32(!iwl_mld_is_nic_ack_enabled(mld, vif));
+
+	/* If we have MLO enabled, then the firmware needs to enable
+	 * address translation for the station(s) we add. That depends
+	 * on having EHT enabled in firmware, which in turn depends on
+	 * mac80211 in the code below.
+	 * However, mac80211 doesn't enable HE/EHT until it has parsed
+	 * the association response successfully, so just skip all that
+	 * and enable both when we have MLO.
+	 */
+	if (ieee80211_vif_is_mld(vif)) {
+		iwl_mld_set_he_support(mld, vif, cmd, cmd_ver);
+		if (cmd_ver == 2)
+			cmd->wifi_gen_v2.eht_support = cpu_to_le32(1);
+		else
+			cmd->wifi_gen.eht_support = 1;
+		return;
+	}
+
+	for_each_vif_active_link(vif, link_conf, link_id) {
+		if (!link_conf->he_support)
+			continue;
+
+		iwl_mld_set_he_support(mld, vif, cmd, cmd_ver);
+
+		/* EHT, if supported, was already set above */
+		break;
+	}
+}
+
+static void iwl_mld_fill_mac_cmd_sta(struct iwl_mld *mld,
+				     struct ieee80211_vif *vif, u32 action,
+				     struct iwl_mac_config_cmd *cmd)
+{
+	struct ieee80211_bss_conf *link;
+	u32 twt_policy = 0;
+	int link_id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	WARN_ON(vif->type != NL80211_IFTYPE_STATION);
+
+	/* We always want to hear MCAST frames, if we're not authorized yet,
+	 * we'll drop them.
+	 */
+	cmd->filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_GRP);
+
+	/* Adding a MAC ctxt with is_assoc set is not allowed in fw
+	 * (and shouldn't happen)
+	 */
+	if (vif->cfg.assoc && action != FW_CTXT_ACTION_ADD) {
+		cmd->client.is_assoc = 1;
+
+		if (!iwl_mld_vif_from_mac80211(vif)->authorized)
+			cmd->client.data_policy |=
+				cpu_to_le16(COEX_HIGH_PRIORITY_ENABLE);
+	} else {
+		/* Allow beacons to pass through as long as we are not
+		 * associated
+		 */
+		cmd->filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_BEACON);
+	}
+
+	cmd->client.assoc_id = cpu_to_le16(vif->cfg.aid);
+
+	if (ieee80211_vif_is_mld(vif)) {
+		u16 esr_transition_timeout =
+			u16_get_bits(vif->cfg.eml_cap,
+				     IEEE80211_EML_CAP_TRANSITION_TIMEOUT);
+
+		cmd->client.esr_transition_timeout =
+			min_t(u16, IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU,
+			      esr_transition_timeout);
+		cmd->client.medium_sync_delay =
+			cpu_to_le16(vif->cfg.eml_med_sync_delay);
+	}
+
+	for_each_vif_active_link(vif, link, link_id) {
+		if (!link->he_support)
+			continue;
+
+		if (link->twt_requester)
+			twt_policy |= TWT_SUPPORTED;
+		if (link->twt_protected)
+			twt_policy |= PROTECTED_TWT_SUPPORTED;
+		if (link->twt_broadcast)
+			twt_policy |= BROADCAST_TWT_SUPPORTED;
+	}
+
+	if (!iwlwifi_mod_params.disable_11ax)
+		cmd->client.data_policy |= cpu_to_le16(twt_policy);
+
+	if (vif->probe_req_reg && vif->cfg.assoc && vif->p2p)
+		cmd->filter_flags |=
+			cpu_to_le32(MAC_CFG_FILTER_ACCEPT_PROBE_REQ);
+}
+
+static void iwl_mld_fill_mac_cmd_ap(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    struct iwl_mac_config_cmd *cmd)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	WARN_ON(vif->type != NL80211_IFTYPE_AP);
+
+	cmd->filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_PROBE_REQ);
+
+	/* in AP mode, pass beacons from other APs (needed for ht protection).
+	 * When there're no any associated station, which means that we are not
+	 * TXing anyway, don't ask FW to pass beacons to prevent unnecessary
+	 * wake-ups.
+	 */
+	if (mld_vif->num_associated_stas)
+		cmd->filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_BEACON);
+}
+
+static void iwl_mld_go_iterator(void *_data, u8 *mac, struct ieee80211_vif *vif)
+{
+	bool *go_active = _data;
+
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_P2P_GO &&
+	    iwl_mld_vif_from_mac80211(vif)->ap_ibss_active)
+		*go_active = true;
+}
+
+static bool iwl_mld_p2p_dev_has_extended_disc(struct iwl_mld *mld)
+{
+	bool go_active = false;
+
+	/* This flag should be set to true when the P2P Device is
+	 * discoverable and there is at least a P2P GO. Setting
+	 * this flag will allow the P2P Device to be discoverable on other
+	 * channels in addition to its listen channel.
+	 * Note that this flag should not be set in other cases as it opens the
+	 * Rx filters on all MAC and increases the number of interrupts.
+	 */
+	ieee80211_iterate_active_interfaces(mld->hw,
+					    IEEE80211_IFACE_ITER_RESUME_ALL,
+					    iwl_mld_go_iterator, &go_active);
+
+	return go_active;
+}
+
+static void iwl_mld_fill_mac_cmd_p2p_dev(struct iwl_mld *mld,
+					 struct ieee80211_vif *vif,
+					 struct iwl_mac_config_cmd *cmd)
+{
+	bool ext_disc = iwl_mld_p2p_dev_has_extended_disc(mld);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* Override the filter flags to accept all management frames. This is
+	 * needed to support both P2P device discovery using probe requests and
+	 * P2P service discovery using action frames
+	 */
+	cmd->filter_flags = cpu_to_le32(MAC_CFG_FILTER_ACCEPT_CONTROL_AND_MGMT);
+
+	if (ext_disc)
+		cmd->p2p_dev.is_disc_extended = cpu_to_le32(1);
+}
+
+static void iwl_mld_fill_mac_cmd_ibss(struct iwl_mld *mld,
+				      struct ieee80211_vif *vif,
+				      struct iwl_mac_config_cmd *cmd)
+{
+	lockdep_assert_wiphy(mld->wiphy);
+
+	WARN_ON(vif->type != NL80211_IFTYPE_ADHOC);
+
+	cmd->filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_BEACON |
+					 MAC_CFG_FILTER_ACCEPT_PROBE_REQ |
+					 MAC_CFG_FILTER_ACCEPT_GRP);
+}
+
+static int
+iwl_mld_rm_mac_from_fw(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mac_config_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
+		.id_and_color = cpu_to_le32(mld_vif->fw_id),
+	};
+
+	return iwl_mld_send_mac_cmd(mld, &cmd);
+}
+
+int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			  u32 action)
+{
+	struct iwl_mac_config_cmd cmd = {};
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (action == FW_CTXT_ACTION_REMOVE)
+		return iwl_mld_rm_mac_from_fw(mld, vif);
+
+	iwl_mld_mac_cmd_fill_common(mld, vif, &cmd, action);
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		iwl_mld_fill_mac_cmd_sta(mld, vif, action, &cmd);
+		break;
+	case NL80211_IFTYPE_AP:
+		iwl_mld_fill_mac_cmd_ap(mld, vif, &cmd);
+		break;
+	case NL80211_IFTYPE_MONITOR:
+		cmd.filter_flags =
+			cpu_to_le32(MAC_CFG_FILTER_PROMISC |
+				    MAC_CFG_FILTER_ACCEPT_CONTROL_AND_MGMT |
+				    MAC_CFG_FILTER_ACCEPT_BEACON |
+				    MAC_CFG_FILTER_ACCEPT_PROBE_REQ |
+				    MAC_CFG_FILTER_ACCEPT_GRP);
+		break;
+	case NL80211_IFTYPE_P2P_DEVICE:
+		iwl_mld_fill_mac_cmd_p2p_dev(mld, vif, &cmd);
+		break;
+	case NL80211_IFTYPE_ADHOC:
+		iwl_mld_fill_mac_cmd_ibss(mld, vif, &cmd);
+		break;
+	default:
+		WARN(1, "not supported yet\n");
+		return -EOPNOTSUPP;
+	}
+
+	return iwl_mld_send_mac_cmd(mld, &cmd);
+}
+
+static void iwl_mld_mlo_scan_start_wk(struct wiphy *wiphy,
+				      struct wiphy_work *wk)
+{
+	struct iwl_mld_vif *mld_vif = container_of(wk, struct iwl_mld_vif,
+						   mlo_scan_start_wk.work);
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	iwl_mld_int_mlo_scan(mld, iwl_mld_vif_to_mac80211(mld_vif));
+}
+
+IWL_MLD_ALLOC_FN(vif, vif)
+
+/* Constructor function for struct iwl_mld_vif */
+static int
+iwl_mld_init_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	mld_vif->mld = mld;
+	mld_vif->roc_activity = ROC_NUM_ACTIVITIES;
+
+	ret = iwl_mld_allocate_vif_fw_id(mld, &mld_vif->fw_id, vif);
+	if (ret)
+		return ret;
+
+	if (!mld->fw_status.in_hw_restart) {
+		wiphy_work_init(&mld_vif->emlsr.unblock_tpt_wk,
+				iwl_mld_emlsr_unblock_tpt_wk);
+		wiphy_delayed_work_init(&mld_vif->emlsr.check_tpt_wk,
+					iwl_mld_emlsr_check_tpt);
+		wiphy_delayed_work_init(&mld_vif->emlsr.prevent_done_wk,
+					iwl_mld_emlsr_prevent_done_wk);
+		wiphy_delayed_work_init(&mld_vif->emlsr.tmp_non_bss_done_wk,
+					iwl_mld_emlsr_tmp_non_bss_done_wk);
+		wiphy_delayed_work_init(&mld_vif->mlo_scan_start_wk,
+					iwl_mld_mlo_scan_start_wk);
+	}
+	iwl_mld_init_internal_sta(&mld_vif->aux_sta);
+
+	return 0;
+}
+
+int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_init_vif(mld, vif);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_ADD);
+	if (ret)
+		RCU_INIT_POINTER(mld->fw_id_to_vif[mld_vif->fw_id], NULL);
+
+	return ret;
+}
+
+int iwl_mld_rm_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_REMOVE);
+
+	if (WARN_ON(mld_vif->fw_id >= ARRAY_SIZE(mld->fw_id_to_vif)))
+		return -EINVAL;
+
+	RCU_INIT_POINTER(mld->fw_id_to_vif[mld_vif->fw_id], NULL);
+
+	iwl_mld_cancel_notifications_of_object(mld, IWL_MLD_OBJECT_TYPE_VIF,
+					       mld_vif->fw_id);
+
+	return ret;
+}
+
+void iwl_mld_set_vif_associated(struct iwl_mld *mld,
+				struct ieee80211_vif *vif)
+{
+	struct ieee80211_bss_conf *link;
+	unsigned int link_id;
+
+	for_each_vif_active_link(vif, link, link_id) {
+		if (iwl_mld_link_set_associated(mld, vif, link))
+			IWL_ERR(mld, "failed to update link %d\n", link_id);
+	}
+
+	iwl_mld_recalc_multicast_filter(mld);
+}
+
+static void iwl_mld_get_fw_id_bss_bitmap_iter(void *_data, u8 *mac,
+					      struct ieee80211_vif *vif)
+{
+	u8 *fw_id_bitmap = _data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION)
+		return;
+
+	*fw_id_bitmap |= BIT(mld_vif->fw_id);
+}
+
+u8 iwl_mld_get_fw_bss_vifs_ids(struct iwl_mld *mld)
+{
+	u8 fw_id_bitmap = 0;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER,
+						iwl_mld_get_fw_id_bss_bitmap_iter,
+						&fw_id_bitmap);
+
+	return fw_id_bitmap;
+}
+
+void iwl_mld_handle_probe_resp_data_notif(struct iwl_mld *mld,
+					  struct iwl_rx_packet *pkt)
+{
+	const struct iwl_probe_resp_data_notif *notif = (void *)pkt->data;
+	struct iwl_probe_resp_data *old_data, *new_data;
+	struct ieee80211_vif *vif;
+	struct iwl_mld_link *mld_link;
+
+	IWL_DEBUG_INFO(mld, "Probe response data notif: noa %d, csa %d\n",
+		       notif->noa_active, notif->csa_counter);
+
+	if (IWL_FW_CHECK(mld, le32_to_cpu(notif->mac_id) >=
+			 ARRAY_SIZE(mld->fw_id_to_vif),
+			 "mac id is invalid: %d\n",
+			 le32_to_cpu(notif->mac_id)))
+		return;
+
+	vif = wiphy_dereference(mld->wiphy,
+				mld->fw_id_to_vif[le32_to_cpu(notif->mac_id)]);
+
+	/* the firmware gives us the mac_id (and not the link_id), mac80211
+	 * gets a vif and not a link, bottom line, this flow is not MLD ready
+	 * yet.
+	 */
+	if (WARN_ON(!vif) || ieee80211_vif_is_mld(vif))
+		return;
+
+	if (notif->csa_counter != IWL_PROBE_RESP_DATA_NO_CSA &&
+	    notif->csa_counter >= 1)
+		ieee80211_beacon_set_cntdwn(vif, notif->csa_counter);
+
+	if (!vif->p2p)
+		return;
+
+	mld_link = &iwl_mld_vif_from_mac80211(vif)->deflink;
+
+	new_data = kzalloc(sizeof(*new_data), GFP_KERNEL);
+	if (!new_data)
+		return;
+
+	memcpy(&new_data->notif, notif, sizeof(new_data->notif));
+
+	/* noa_attr contains 1 reserved byte, need to substruct it */
+	new_data->noa_len = sizeof(struct ieee80211_vendor_ie) +
+			    sizeof(new_data->notif.noa_attr) - 1;
+
+	/*
+	 * If it's a one time NoA, only one descriptor is needed,
+	 * adjust the length according to len_low.
+	 */
+	if (new_data->notif.noa_attr.len_low ==
+	    sizeof(struct ieee80211_p2p_noa_desc) + 2)
+		new_data->noa_len -= sizeof(struct ieee80211_p2p_noa_desc);
+
+	old_data = wiphy_dereference(mld->wiphy, mld_link->probe_resp_data);
+	rcu_assign_pointer(mld_link->probe_resp_data, new_data);
+
+	if (old_data)
+		kfree_rcu(old_data, rcu_head);
+}
+
+void iwl_mld_handle_uapsd_misbehaving_ap_notif(struct iwl_mld *mld,
+					       struct iwl_rx_packet *pkt)
+{
+	struct iwl_uapsd_misbehaving_ap_notif *notif = (void *)pkt->data;
+	struct ieee80211_vif *vif;
+
+	if (IWL_FW_CHECK(mld, notif->mac_id >= ARRAY_SIZE(mld->fw_id_to_vif),
+			 "mac id is invalid: %d\n", notif->mac_id))
+		return;
+
+	vif = wiphy_dereference(mld->wiphy, mld->fw_id_to_vif[notif->mac_id]);
+
+	if (WARN_ON(!vif) || ieee80211_vif_is_mld(vif))
+		return;
+
+	IWL_WARN(mld, "uapsd misbehaving AP: %pM\n", vif->bss_conf.bssid);
+}
+
+void iwl_mld_handle_datapath_monitor_notif(struct iwl_mld *mld,
+					   struct iwl_rx_packet *pkt)
+{
+	struct iwl_datapath_monitor_notif *notif = (void *)pkt->data;
+	struct ieee80211_bss_conf *link;
+	struct ieee80211_supported_band *sband;
+	const struct ieee80211_sta_he_cap *he_cap;
+	struct ieee80211_vif *vif;
+	struct iwl_mld_vif *mld_vif;
+
+	if (notif->type != cpu_to_le32(IWL_DP_MON_NOTIF_TYPE_EXT_CCA))
+		return;
+
+	link = iwl_mld_fw_id_to_link_conf(mld, notif->link_id);
+	if (WARN_ON(!link))
+		return;
+
+	vif = link->vif;
+	if (WARN_ON(!vif) || vif->type != NL80211_IFTYPE_STATION ||
+	    !vif->cfg.assoc)
+		return;
+
+	if (!link->chanreq.oper.chan ||
+	    link->chanreq.oper.chan->band != NL80211_BAND_2GHZ ||
+	    link->chanreq.oper.width < NL80211_CHAN_WIDTH_40)
+		return;
+
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	/* this shouldn't happen *again*, ignore it */
+	if (mld_vif->cca_40mhz_workaround != CCA_40_MHZ_WA_NONE)
+		return;
+
+	mld_vif->cca_40mhz_workaround = CCA_40_MHZ_WA_RECONNECT;
+
+	/*
+	 * This capability manipulation isn't really ideal, but it's the
+	 * easiest choice - otherwise we'd have to do some major changes
+	 * in mac80211 to support this, which isn't worth it. This does
+	 * mean that userspace may have outdated information, but that's
+	 * actually not an issue at all.
+	 */
+	sband = mld->wiphy->bands[NL80211_BAND_2GHZ];
+
+	WARN_ON(!sband->ht_cap.ht_supported);
+	WARN_ON(!(sband->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40));
+	sband->ht_cap.cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40;
+
+	he_cap = ieee80211_get_he_iftype_cap_vif(sband, vif);
+
+	if (he_cap) {
+		/* we know that ours is writable */
+		struct ieee80211_sta_he_cap *he = (void *)(uintptr_t)he_cap;
+
+		WARN_ON(!he->has_he);
+		WARN_ON(!(he->he_cap_elem.phy_cap_info[0] &
+			  IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G));
+		he->he_cap_elem.phy_cap_info[0] &=
+			~IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G;
+	}
+
+	ieee80211_disconnect(vif, true);
+}
+
+void iwl_mld_reset_cca_40mhz_workaround(struct iwl_mld *mld,
+					struct ieee80211_vif *vif)
+{
+	struct ieee80211_supported_band *sband;
+	const struct ieee80211_sta_he_cap *he_cap;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	if (mld_vif->cca_40mhz_workaround == CCA_40_MHZ_WA_NONE)
+		return;
+
+	/* Now we are just reconnecting with the new capabilities,
+	 * but remember to reset the capabilities when we disconnect for real
+	 */
+	if (mld_vif->cca_40mhz_workaround == CCA_40_MHZ_WA_RECONNECT) {
+		mld_vif->cca_40mhz_workaround = CCA_40_MHZ_WA_RESET;
+		return;
+	}
+
+	/* Now cca_40mhz_workaround == CCA_40_MHZ_WA_RESET */
+
+	sband = mld->wiphy->bands[NL80211_BAND_2GHZ];
+
+	sband->ht_cap.cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40;
+
+	he_cap = ieee80211_get_he_iftype_cap_vif(sband, vif);
+
+	if (he_cap) {
+		/* we know that ours is writable */
+		struct ieee80211_sta_he_cap *he = (void *)(uintptr_t)he_cap;
+
+		he->he_cap_elem.phy_cap_info[0] |=
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G;
+	}
+
+	mld_vif->cca_40mhz_workaround = CCA_40_MHZ_WA_NONE;
+}
+
+struct ieee80211_vif *iwl_mld_get_bss_vif(struct iwl_mld *mld)
+{
+	unsigned long fw_id_bitmap = iwl_mld_get_fw_bss_vifs_ids(mld);
+	int fw_id;
+
+	if (hweight8(fw_id_bitmap) != 1)
+		return NULL;
+
+	fw_id = __ffs(fw_id_bitmap);
+
+	return wiphy_dereference(mld->wiphy,
+				 mld->fw_id_to_vif[fw_id]);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/iface.h b/sys/contrib/dev/iwlwifi/mld/iface.h
new file mode 100644
index 000000000000..05dcb63701b1
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/iface.h
@@ -0,0 +1,253 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_iface_h__
+#define __iwl_mld_iface_h__
+
+#include <net/mac80211.h>
+
+#include "link.h"
+#include "session-protect.h"
+#include "d3.h"
+#include "fw/api/time-event.h"
+
+enum iwl_mld_cca_40mhz_wa_status {
+	CCA_40_MHZ_WA_NONE,
+	CCA_40_MHZ_WA_RESET,
+	CCA_40_MHZ_WA_RECONNECT,
+};
+
+/**
+ * enum iwl_mld_emlsr_blocked - defines reasons for which EMLSR is blocked
+ *
+ * These blocks are applied/stored per-VIF.
+ *
+ * @IWL_MLD_EMLSR_BLOCKED_PREVENTION: Prevent repeated EMLSR enter/exit
+ * @IWL_MLD_EMLSR_BLOCKED_WOWLAN: WOWLAN is preventing EMLSR
+ * @IWL_MLD_EMLSR_BLOCKED_ROC: remain-on-channel is preventing EMLSR
+ * @IWL_MLD_EMLSR_BLOCKED_NON_BSS: An active non-BSS interface's link is
+ *      preventing EMLSR
+ * @IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS: An expected active non-BSS interface's
+ *      link is preventing EMLSR. This is a temporary blocking that is set when
+ *      there is an indication that a non-BSS interface is to be added.
+ * @IWL_MLD_EMLSR_BLOCKED_TPT: throughput is too low to make EMLSR worthwhile
+ */
+enum iwl_mld_emlsr_blocked {
+	IWL_MLD_EMLSR_BLOCKED_PREVENTION	= 0x1,
+	IWL_MLD_EMLSR_BLOCKED_WOWLAN		= 0x2,
+	IWL_MLD_EMLSR_BLOCKED_ROC		= 0x4,
+	IWL_MLD_EMLSR_BLOCKED_NON_BSS		= 0x8,
+	IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS	= 0x10,
+	IWL_MLD_EMLSR_BLOCKED_TPT		= 0x20,
+};
+
+/**
+ * enum iwl_mld_emlsr_exit - defines reasons for exiting EMLSR
+ *
+ * Reasons to exit EMLSR may be either link specific or even specific to a
+ * combination of links.
+ *
+ * @IWL_MLD_EMLSR_EXIT_BLOCK: Exit due to a block reason being set
+ * @IWL_MLD_EMLSR_EXIT_MISSED_BEACON: Exit due to missed beacons
+ * @IWL_MLD_EMLSR_EXIT_FAIL_ENTRY: FW failed to enter EMLSR
+ * @IWL_MLD_EMLSR_EXIT_CSA: EMLSR prevented due to channel switch on link
+ * @IWL_MLD_EMLSR_EXIT_EQUAL_BAND: EMLSR prevented as both links share the band
+ * @IWL_MLD_EMLSR_EXIT_LOW_RSSI: Link RSSI is unsuitable for EMLSR
+ * @IWL_MLD_EMLSR_EXIT_LINK_USAGE: Exit EMLSR due to low TPT on secondary link
+ * @IWL_MLD_EMLSR_EXIT_BT_COEX: Exit EMLSR due to BT coexistence
+ * @IWL_MLD_EMLSR_EXIT_CHAN_LOAD: Exit EMLSR because the primary channel is not
+ *	loaded enough to justify EMLSR.
+ * @IWL_MLD_EMLSR_EXIT_RFI: Exit EMLSR due to RFI
+ * @IWL_MLD_EMLSR_EXIT_FW_REQUEST: Exit EMLSR because the FW requested it
+ * @IWL_MLD_EMLSR_EXIT_INVALID: internal exit reason due to invalid data
+ */
+enum iwl_mld_emlsr_exit {
+	IWL_MLD_EMLSR_EXIT_BLOCK		= 0x1,
+	IWL_MLD_EMLSR_EXIT_MISSED_BEACON	= 0x2,
+	IWL_MLD_EMLSR_EXIT_FAIL_ENTRY		= 0x4,
+	IWL_MLD_EMLSR_EXIT_CSA			= 0x8,
+	IWL_MLD_EMLSR_EXIT_EQUAL_BAND		= 0x10,
+	IWL_MLD_EMLSR_EXIT_LOW_RSSI		= 0x20,
+	IWL_MLD_EMLSR_EXIT_LINK_USAGE		= 0x40,
+	IWL_MLD_EMLSR_EXIT_BT_COEX		= 0x80,
+	IWL_MLD_EMLSR_EXIT_CHAN_LOAD		= 0x100,
+	IWL_MLD_EMLSR_EXIT_RFI			= 0x200,
+	IWL_MLD_EMLSR_EXIT_FW_REQUEST		= 0x400,
+	IWL_MLD_EMLSR_EXIT_INVALID		= 0x800,
+};
+
+/**
+ * struct iwl_mld_emlsr - per-VIF data about EMLSR operation
+ *
+ * @primary: The current primary link
+ * @selected_primary: Primary link as selected during the last link selection
+ * @selected_links: Links as selected during the last link selection
+ * @blocked_reasons: Reasons preventing EMLSR from being enabled
+ * @last_exit_reason: Reason for the last EMLSR exit
+ * @last_exit_ts: Time of the last EMLSR exit (if @last_exit_reason is non-zero)
+ * @exit_repeat_count: Number of times EMLSR was exited for the same reason
+ * @last_entry_ts: the time of the last EMLSR entry (if iwl_mld_emlsr_active()
+ *	is true)
+ * @unblock_tpt_wk: Unblock EMLSR because the throughput limit was reached
+ * @check_tpt_wk: a worker to check if IWL_MLD_EMLSR_BLOCKED_TPT should be
+ *	added, for example if there is no longer enough traffic.
+ * @prevent_done_wk: Worker to remove %IWL_MLD_EMLSR_BLOCKED_PREVENTION
+ * @tmp_non_bss_done_wk: Worker to remove %IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS
+ */
+struct iwl_mld_emlsr {
+	struct_group(zeroed_on_not_authorized,
+		u8 primary;
+
+		u8 selected_primary;
+		u16 selected_links;
+
+		enum iwl_mld_emlsr_blocked blocked_reasons;
+
+		enum iwl_mld_emlsr_exit last_exit_reason;
+		unsigned long last_exit_ts;
+		u8 exit_repeat_count;
+		unsigned long last_entry_ts;
+	);
+
+	struct wiphy_work unblock_tpt_wk;
+	struct wiphy_delayed_work check_tpt_wk;
+
+	struct wiphy_delayed_work prevent_done_wk;
+	struct wiphy_delayed_work tmp_non_bss_done_wk;
+};
+
+/**
+ * struct iwl_mld_vif - virtual interface (MAC context) configuration parameters
+ *
+ * @fw_id: fw id of the mac context.
+ * @session_protect: session protection parameters
+ * @ap_sta: pointer to AP sta, for easier access to it.
+ *	Relevant only for STA vifs.
+ * @authorized: indicates the AP station was set to authorized
+ * @bigtks: BIGTKs of the AP, for beacon protection.
+ *	Only valid for STA. (FIXME: needs to be per link)
+ * @num_associated_stas: number of associated STAs. Relevant only for AP mode.
+ * @ap_ibss_active: whether the AP/IBSS was started
+ * @cca_40mhz_workaround: When we are connected in 2.4 GHz and 40 MHz, and the
+ *	environment is too loaded, we work around this by reconnecting to the
+ *	same AP with 20 MHz. This manages the status of the workaround.
+ * @beacon_inject_active: indicates an active debugfs beacon ie injection
+ * @low_latency_causes: bit flags, indicating the causes for low-latency,
+ *	see @iwl_mld_low_latency_cause.
+ * @ps_disabled: indicates that PS is disabled for this interface
+ * @last_link_activation_time: last time a link was activated, for
+ *	deferring MLO scans (to make them more reliable)
+ * @mld: pointer to the mld structure.
+ * @deflink: default link data, for use in non-MLO,
+ * @link: reference to link data for each valid link, for use in MLO.
+ * @emlsr: information related to EMLSR
+ * @wowlan_data: data used by the wowlan suspend flow
+ * @use_ps_poll: use ps_poll frames
+ * @disable_bf: disable beacon filter
+ * @dbgfs_slink: debugfs symlink for this interface
+ * @roc_activity: the id of the roc_activity running. Relevant for STA and
+ *	p2p device only. Set to %ROC_NUM_ACTIVITIES when not in use.
+ * @aux_sta: station used for remain on channel. Used in P2P device.
+ * @mlo_scan_start_wk: worker to start a deferred MLO scan
+ */
+struct iwl_mld_vif {
+	/* Add here fields that need clean up on restart */
+	struct_group(zeroed_on_hw_restart,
+		u8 fw_id;
+		struct iwl_mld_session_protect session_protect;
+		struct ieee80211_sta *ap_sta;
+		bool authorized;
+		struct ieee80211_key_conf __rcu *bigtks[2];
+		u8 num_associated_stas;
+		bool ap_ibss_active;
+		enum iwl_mld_cca_40mhz_wa_status cca_40mhz_workaround;
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+		bool beacon_inject_active;
+#endif
+		u8 low_latency_causes;
+		bool ps_disabled;
+		time64_t last_link_activation_time;
+	);
+	/* And here fields that survive a fw restart */
+	struct iwl_mld *mld;
+	struct iwl_mld_link deflink;
+	struct iwl_mld_link __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
+
+	struct iwl_mld_emlsr emlsr;
+
+#ifdef CONFIG_PM_SLEEP
+	struct iwl_mld_wowlan_data wowlan_data;
+#endif
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	bool use_ps_poll;
+	bool disable_bf;
+	struct dentry *dbgfs_slink;
+#endif
+	enum iwl_roc_activity roc_activity;
+	struct iwl_mld_int_sta aux_sta;
+
+	struct wiphy_delayed_work mlo_scan_start_wk;
+};
+
+static inline struct iwl_mld_vif *
+iwl_mld_vif_from_mac80211(struct ieee80211_vif *vif)
+{
+	return (void *)vif->drv_priv;
+}
+
+static inline struct ieee80211_vif *
+iwl_mld_vif_to_mac80211(struct iwl_mld_vif *mld_vif)
+{
+	return container_of((void *)mld_vif, struct ieee80211_vif, drv_priv);
+}
+
+#define iwl_mld_link_dereference_check(mld_vif, link_id)		\
+	rcu_dereference_check((mld_vif)->link[link_id],			\
+			      lockdep_is_held(&mld_vif->mld->wiphy->mtx))
+
+#define for_each_mld_vif_valid_link(mld_vif, mld_link)			\
+	for (int link_id = 0; link_id < ARRAY_SIZE((mld_vif)->link);	\
+	     link_id++)							\
+		if ((mld_link = iwl_mld_link_dereference_check(mld_vif, link_id)))
+
+/* Retrieve pointer to mld link from mac80211 structures */
+static inline struct iwl_mld_link *
+iwl_mld_link_from_mac80211(struct ieee80211_bss_conf *bss_conf)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(bss_conf->vif);
+
+	return iwl_mld_link_dereference_check(mld_vif, bss_conf->link_id);
+}
+
+int iwl_mld_mac80211_iftype_to_fw(const struct ieee80211_vif *vif);
+
+/* Cleanup function for struct iwl_mld_vif, will be called in restart */
+void iwl_mld_cleanup_vif(void *data, u8 *mac, struct ieee80211_vif *vif);
+int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			  u32 action);
+int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif);
+int iwl_mld_rm_vif(struct iwl_mld *mld, struct ieee80211_vif *vif);
+void iwl_mld_set_vif_associated(struct iwl_mld *mld,
+				struct ieee80211_vif *vif);
+u8 iwl_mld_get_fw_bss_vifs_ids(struct iwl_mld *mld);
+void iwl_mld_handle_probe_resp_data_notif(struct iwl_mld *mld,
+					  struct iwl_rx_packet *pkt);
+
+void iwl_mld_handle_datapath_monitor_notif(struct iwl_mld *mld,
+					   struct iwl_rx_packet *pkt);
+
+void iwl_mld_handle_uapsd_misbehaving_ap_notif(struct iwl_mld *mld,
+					       struct iwl_rx_packet *pkt);
+
+void iwl_mld_reset_cca_40mhz_workaround(struct iwl_mld *mld,
+					struct ieee80211_vif *vif);
+
+static inline bool iwl_mld_vif_low_latency(const struct iwl_mld_vif *mld_vif)
+{
+	return !!mld_vif->low_latency_causes;
+}
+
+struct ieee80211_vif *iwl_mld_get_bss_vif(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_iface_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/key.c b/sys/contrib/dev/iwlwifi/mld/key.c
new file mode 100644
index 000000000000..13462a5ad79a
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/key.c
@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#include "key.h"
+#include "iface.h"
+#include "sta.h"
+#include "fw/api/datapath.h"
+
+static u32 iwl_mld_get_key_flags(struct iwl_mld *mld,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_sta *sta,
+				 struct ieee80211_key_conf *key)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	bool pairwise = key->flags & IEEE80211_KEY_FLAG_PAIRWISE;
+	bool igtk = key->keyidx == 4 || key->keyidx == 5;
+	u32 flags = 0;
+
+	if (!pairwise)
+		flags |= IWL_SEC_KEY_FLAG_MCAST_KEY;
+
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_TKIP:
+		flags |= IWL_SEC_KEY_FLAG_CIPHER_TKIP;
+		break;
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+	case WLAN_CIPHER_SUITE_CCMP:
+		flags |= IWL_SEC_KEY_FLAG_CIPHER_CCMP;
+		break;
+	case WLAN_CIPHER_SUITE_GCMP_256:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+		flags |= IWL_SEC_KEY_FLAG_KEY_SIZE;
+		fallthrough;
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+		flags |= IWL_SEC_KEY_FLAG_CIPHER_GCMP;
+		break;
+	}
+
+	if (!sta && vif->type == NL80211_IFTYPE_STATION)
+		sta = mld_vif->ap_sta;
+
+	/* If we are installing an iGTK (in AP or STA mode), we need to tell
+	 * the firmware this key will en/decrypt MGMT frames.
+	 * Same goes if we are installing a pairwise key for an MFP station.
+	 * In case we're installing a groupwise key (which is not an iGTK),
+	 * then, we will not use this key for MGMT frames.
+	 */
+	if ((sta && sta->mfp && pairwise) || igtk)
+		flags |= IWL_SEC_KEY_FLAG_MFP;
+
+	if (key->flags & IEEE80211_KEY_FLAG_SPP_AMSDU)
+		flags |= IWL_SEC_KEY_FLAG_SPP_AMSDU;
+
+	return flags;
+}
+
+static u32 iwl_mld_get_key_sta_mask(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_sta *sta,
+				    struct ieee80211_key_conf *key)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_link_sta *link_sta;
+	int sta_id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* AP group keys are per link and should be on the mcast/bcast STA */
+	if (vif->type == NL80211_IFTYPE_AP &&
+	    !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) {
+		struct iwl_mld_link *link = NULL;
+
+		if (key->link_id >= 0)
+			link = iwl_mld_link_dereference_check(mld_vif,
+							      key->link_id);
+
+		if (WARN_ON(!link))
+			return 0;
+
+		/* In this stage we should have both the bcast and mcast STAs */
+		if (WARN_ON(link->bcast_sta.sta_id == IWL_INVALID_STA ||
+			    link->mcast_sta.sta_id == IWL_INVALID_STA))
+			return 0;
+
+		/* IGTK/BIGTK to bcast STA */
+		if (key->keyidx >= 4)
+			return BIT(link->bcast_sta.sta_id);
+
+		/* GTK for data to mcast STA */
+		return BIT(link->mcast_sta.sta_id);
+	}
+
+	/* for client mode use the AP STA also for group keys */
+	if (!sta && vif->type == NL80211_IFTYPE_STATION)
+		sta = mld_vif->ap_sta;
+
+	/* STA should be non-NULL now */
+	if (WARN_ON(!sta))
+		return 0;
+
+	/* Key is not per-link, get the full sta mask */
+	if (key->link_id < 0)
+		return iwl_mld_fw_sta_id_mask(mld, sta);
+
+	/* The link_sta shouldn't be NULL now, but this is checked in
+	 * iwl_mld_fw_sta_id_mask
+	 */
+	link_sta = link_sta_dereference_check(sta, key->link_id);
+
+	sta_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
+	if (sta_id < 0)
+		return 0;
+
+	return BIT(sta_id);
+}
+
+static int iwl_mld_add_key_to_fw(struct iwl_mld *mld, u32 sta_mask,
+				 u32 key_flags, struct ieee80211_key_conf *key)
+{
+	struct iwl_sec_key_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_ADD),
+		.u.add.sta_mask = cpu_to_le32(sta_mask),
+		.u.add.key_id = cpu_to_le32(key->keyidx),
+		.u.add.key_flags = cpu_to_le32(key_flags),
+		.u.add.tx_seq = cpu_to_le64(atomic64_read(&key->tx_pn)),
+	};
+	bool tkip = key->cipher == WLAN_CIPHER_SUITE_TKIP;
+	int max_key_len = sizeof(cmd.u.add.key);
+
+#ifdef CONFIG_PM_SLEEP
+	/* If there was a rekey in wowlan, FW already has the key */
+	if (mld->fw_status.resuming)
+		return 0;
+#endif
+
+	if (WARN_ON(!sta_mask))
+		return -EINVAL;
+
+	if (WARN_ON(key->keylen > max_key_len))
+		return -EINVAL;
+
+	memcpy(cmd.u.add.key, key->key, key->keylen);
+
+	if (tkip) {
+		memcpy(cmd.u.add.tkip_mic_rx_key,
+		       key->key + NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY,
+		       8);
+		memcpy(cmd.u.add.tkip_mic_tx_key,
+		       key->key + NL80211_TKIP_DATA_OFFSET_TX_MIC_KEY,
+		       8);
+	}
+
+	return iwl_mld_send_cmd_pdu(mld, WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD),
+				    &cmd);
+}
+
+static void iwl_mld_remove_key_from_fw(struct iwl_mld *mld, u32 sta_mask,
+				       u32 key_flags, u32 keyidx)
+{
+	struct iwl_sec_key_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
+		.u.remove.sta_mask = cpu_to_le32(sta_mask),
+		.u.remove.key_id = cpu_to_le32(keyidx),
+		.u.remove.key_flags = cpu_to_le32(key_flags),
+	};
+
+#ifdef CONFIG_PM_SLEEP
+	/* If there was a rekey in wowlan, FW already removed the key */
+	if (mld->fw_status.resuming)
+		return;
+#endif
+
+	if (WARN_ON(!sta_mask))
+		return;
+
+	iwl_mld_send_cmd_pdu(mld, WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD), &cmd);
+}
+
+void iwl_mld_remove_key(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			struct ieee80211_sta *sta,
+			struct ieee80211_key_conf *key)
+{
+	u32 sta_mask = iwl_mld_get_key_sta_mask(mld, vif, sta, key);
+	u32 key_flags = iwl_mld_get_key_flags(mld, vif, sta, key);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (!sta_mask)
+		return;
+
+	if (key->keyidx == 4 || key->keyidx == 5) {
+		struct iwl_mld_link *mld_link;
+		unsigned int link_id = 0;
+
+		/* set to -1 for non-MLO right now */
+		if (key->link_id >= 0)
+			link_id = key->link_id;
+
+		mld_link = iwl_mld_link_dereference_check(mld_vif, link_id);
+		if (WARN_ON(!mld_link))
+			return;
+
+		if (mld_link->igtk == key)
+			mld_link->igtk = NULL;
+
+		mld->num_igtks--;
+	}
+
+	iwl_mld_remove_key_from_fw(mld, sta_mask, key_flags, key->keyidx);
+
+	/* no longer in HW */
+	key->hw_key_idx = STA_KEY_IDX_INVALID;
+}
+
+int iwl_mld_add_key(struct iwl_mld *mld,
+		    struct ieee80211_vif *vif,
+		    struct ieee80211_sta *sta,
+		    struct ieee80211_key_conf *key)
+{
+	u32 sta_mask = iwl_mld_get_key_sta_mask(mld, vif, sta, key);
+	u32 key_flags = iwl_mld_get_key_flags(mld, vif, sta, key);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *mld_link = NULL;
+	bool igtk = key->keyidx == 4 || key->keyidx == 5;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (!sta_mask)
+		return -EINVAL;
+
+	if (igtk) {
+		if (mld->num_igtks == IWL_MAX_NUM_IGTKS)
+			return -EOPNOTSUPP;
+
+		u8 link_id = 0;
+
+		/* set to -1 for non-MLO right now */
+		if (key->link_id >= 0)
+			link_id = key->link_id;
+
+		mld_link = iwl_mld_link_dereference_check(mld_vif, link_id);
+
+		if (WARN_ON(!mld_link))
+			return -EINVAL;
+
+		if (mld_link->igtk) {
+			IWL_DEBUG_MAC80211(mld, "remove old IGTK %d\n",
+					   mld_link->igtk->keyidx);
+			iwl_mld_remove_key(mld, vif, sta, mld_link->igtk);
+		}
+
+		WARN_ON(mld_link->igtk);
+	}
+
+	ret = iwl_mld_add_key_to_fw(mld, sta_mask, key_flags, key);
+	if (ret)
+		return ret;
+
+	if (mld_link) {
+		mld_link->igtk = key;
+		mld->num_igtks++;
+	}
+
+	/* We don't really need this, but need it to be not invalid,
+	 * so we will know if the key is in fw.
+	 */
+	key->hw_key_idx = 0;
+
+	return 0;
+}
+
+struct remove_ap_keys_iter_data {
+	u8 link_id;
+	struct ieee80211_sta *sta;
+};
+
+static void iwl_mld_remove_ap_keys_iter(struct ieee80211_hw *hw,
+					struct ieee80211_vif *vif,
+					struct ieee80211_sta *sta,
+					struct ieee80211_key_conf *key,
+					void *_data)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct remove_ap_keys_iter_data *data = _data;
+
+	if (key->hw_key_idx == STA_KEY_IDX_INVALID)
+		return;
+
+	/* All the pairwise keys should have been removed by now */
+	if (WARN_ON(sta))
+		return;
+
+	if (key->link_id >= 0 && key->link_id != data->link_id)
+		return;
+
+	iwl_mld_remove_key(mld, vif, data->sta, key);
+}
+
+void iwl_mld_remove_ap_keys(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta, unsigned int link_id)
+{
+	struct remove_ap_keys_iter_data iter_data = {
+		.link_id = link_id,
+		.sta = sta,
+	};
+
+	if (WARN_ON_ONCE(vif->type != NL80211_IFTYPE_STATION))
+		return;
+
+	ieee80211_iter_keys(mld->hw, vif,
+			    iwl_mld_remove_ap_keys_iter,
+			    &iter_data);
+}
+
+struct iwl_mvm_sta_key_update_data {
+	struct ieee80211_sta *sta;
+	u32 old_sta_mask;
+	u32 new_sta_mask;
+	int err;
+};
+
+static void iwl_mld_update_sta_key_iter(struct ieee80211_hw *hw,
+					struct ieee80211_vif *vif,
+					struct ieee80211_sta *sta,
+					struct ieee80211_key_conf *key,
+					void *_data)
+{
+	struct iwl_mvm_sta_key_update_data *data = _data;
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_sec_key_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_MODIFY),
+		.u.modify.old_sta_mask = cpu_to_le32(data->old_sta_mask),
+		.u.modify.new_sta_mask = cpu_to_le32(data->new_sta_mask),
+		.u.modify.key_id = cpu_to_le32(key->keyidx),
+		.u.modify.key_flags =
+			cpu_to_le32(iwl_mld_get_key_flags(mld, vif, sta, key)),
+	};
+	int err;
+
+	/* only need to do this for pairwise keys (link_id == -1) */
+	if (sta != data->sta || key->link_id >= 0)
+		return;
+
+	err = iwl_mld_send_cmd_pdu(mld, WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD),
+				   &cmd);
+
+	if (err)
+		data->err = err;
+}
+
+int iwl_mld_update_sta_keys(struct iwl_mld *mld,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta,
+			    u32 old_sta_mask,
+			    u32 new_sta_mask)
+{
+	struct iwl_mvm_sta_key_update_data data = {
+		.sta = sta,
+		.old_sta_mask = old_sta_mask,
+		.new_sta_mask = new_sta_mask,
+	};
+
+	ieee80211_iter_keys(mld->hw, vif, iwl_mld_update_sta_key_iter,
+			    &data);
+	return data.err;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/key.h b/sys/contrib/dev/iwlwifi/mld/key.h
new file mode 100644
index 000000000000..a68ea48913be
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/key.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_key_h__
+#define __iwl_mld_key_h__
+
+#include "mld.h"
+#include <net/mac80211.h>
+#include "fw/api/sta.h"
+
+void iwl_mld_remove_key(struct iwl_mld *mld,
+			struct ieee80211_vif *vif,
+			struct ieee80211_sta *sta,
+			struct ieee80211_key_conf *key);
+int iwl_mld_add_key(struct iwl_mld *mld,
+		    struct ieee80211_vif *vif,
+		    struct ieee80211_sta *sta,
+		    struct ieee80211_key_conf *key);
+void iwl_mld_remove_ap_keys(struct iwl_mld *mld,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta,
+			    unsigned int link_id);
+
+int iwl_mld_update_sta_keys(struct iwl_mld *mld,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta,
+			    u32 old_sta_mask,
+			    u32 new_sta_mask);
+
+static inline void
+iwl_mld_cleanup_keys_iter(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			  struct ieee80211_sta *sta,
+			  struct ieee80211_key_conf *key, void *data)
+{
+	key->hw_key_idx = STA_KEY_IDX_INVALID;
+}
+
+#endif /* __iwl_mld_key_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/led.c b/sys/contrib/dev/iwlwifi/mld/led.c
new file mode 100644
index 000000000000..a37b32cbc6e6
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/led.c
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#include <linux/leds.h>
+#include <net/mac80211.h>
+
+#include "fw/api/led.h"
+#include "mld.h"
+#include "led.h"
+#include "hcmd.h"
+
+static void iwl_mld_send_led_fw_cmd(struct iwl_mld *mld, bool on)
+{
+	struct iwl_led_cmd led_cmd = {
+		.status = cpu_to_le32(on),
+	};
+	int err;
+
+	if (WARN_ON(!mld->fw_status.running))
+		return;
+
+	err = iwl_mld_send_cmd_with_flags_pdu(mld, WIDE_ID(LONG_GROUP,
+							   LEDS_CMD),
+					      CMD_ASYNC, &led_cmd);
+
+	if (err)
+		IWL_WARN(mld, "LED command failed: %d\n", err);
+}
+
+static void iwl_led_brightness_set(struct led_classdev *led_cdev,
+				   enum led_brightness brightness)
+{
+	struct iwl_mld *mld = container_of(led_cdev, struct iwl_mld, led);
+
+	if (!mld->fw_status.running)
+		return;
+
+	iwl_mld_send_led_fw_cmd(mld, brightness > 0);
+}
+
+int iwl_mld_leds_init(struct iwl_mld *mld)
+{
+	int mode = iwlwifi_mod_params.led_mode;
+	int ret;
+
+	switch (mode) {
+	case IWL_LED_BLINK:
+		IWL_ERR(mld, "Blink led mode not supported, used default\n");
+		fallthrough;
+	case IWL_LED_DEFAULT:
+	case IWL_LED_RF_STATE:
+		mode = IWL_LED_RF_STATE;
+		break;
+	case IWL_LED_DISABLE:
+		IWL_INFO(mld, "Led disabled\n");
+		return 0;
+	default:
+		return -EINVAL;
+	}
+
+	mld->led.name = kasprintf(GFP_KERNEL, "%s-led",
+				  wiphy_name(mld->hw->wiphy));
+	if (!mld->led.name)
+		return -ENOMEM;
+
+	mld->led.brightness_set = iwl_led_brightness_set;
+	mld->led.max_brightness = 1;
+
+	if (mode == IWL_LED_RF_STATE)
+		mld->led.default_trigger =
+			ieee80211_get_radio_led_name(mld->hw);
+
+	ret = led_classdev_register(mld->trans->dev, &mld->led);
+	if (ret) {
+		kfree(mld->led.name);
+		mld->led.name = NULL;
+		IWL_INFO(mld, "Failed to enable led\n");
+	}
+
+	return ret;
+}
+
+void iwl_mld_led_config_fw(struct iwl_mld *mld)
+{
+	if (!mld->led.name)
+		return;
+
+	iwl_mld_send_led_fw_cmd(mld, mld->led.brightness > 0);
+}
+
+void iwl_mld_leds_exit(struct iwl_mld *mld)
+{
+	if (!mld->led.name)
+		return;
+
+	led_classdev_unregister(&mld->led);
+	kfree(mld->led.name);
+	mld->led.name = NULL;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/led.h b/sys/contrib/dev/iwlwifi/mld/led.h
new file mode 100644
index 000000000000..0954e214e73d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/led.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_led_h__
+#define __iwl_mld_led_h__
+
+#include "mld.h"
+
+#ifdef CONFIG_IWLWIFI_LEDS
+int iwl_mld_leds_init(struct iwl_mld *mld);
+void iwl_mld_leds_exit(struct iwl_mld *mld);
+void iwl_mld_led_config_fw(struct iwl_mld *mld);
+#else
+static inline int iwl_mld_leds_init(struct iwl_mld *mld)
+{
+	return 0;
+}
+
+static inline void iwl_mld_leds_exit(struct iwl_mld *mld)
+{
+}
+
+static inline void iwl_mld_led_config_fw(struct iwl_mld *mld)
+{
+}
+#endif
+
+#endif /* __iwl_mld_led_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/link.c b/sys/contrib/dev/iwlwifi/mld/link.c
new file mode 100644
index 000000000000..782fc41aa1c3
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/link.c
@@ -0,0 +1,895 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include "constants.h"
+#include "link.h"
+#include "iface.h"
+#include "mlo.h"
+#include "hcmd.h"
+#include "phy.h"
+#include "fw/api/rs.h"
+#include "fw/api/txq.h"
+#include "fw/api/mac.h"
+
+#include "fw/api/context.h"
+#include "fw/dbg.h"
+
+static int iwl_mld_send_link_cmd(struct iwl_mld *mld,
+				 struct iwl_link_config_cmd *cmd,
+				 enum iwl_ctxt_action action)
+{
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	cmd->action = cpu_to_le32(action);
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(MAC_CONF_GROUP, LINK_CONFIG_CMD),
+				   cmd);
+	if (ret)
+		IWL_ERR(mld, "Failed to send LINK_CONFIG_CMD (action:%d): %d\n",
+			action, ret);
+	return ret;
+}
+
+static int iwl_mld_add_link_to_fw(struct iwl_mld *mld,
+				  struct ieee80211_bss_conf *link_conf)
+{
+	struct ieee80211_vif *vif = link_conf->vif;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *link = iwl_mld_link_from_mac80211(link_conf);
+	struct iwl_link_config_cmd cmd = {};
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!link))
+		return -EINVAL;
+
+	cmd.link_id = cpu_to_le32(link->fw_id);
+	cmd.mac_id = cpu_to_le32(mld_vif->fw_id);
+	cmd.spec_link_id = link_conf->link_id;
+	cmd.phy_id = cpu_to_le32(FW_CTXT_ID_INVALID);
+
+	ether_addr_copy(cmd.local_link_addr, link_conf->addr);
+
+	if (vif->type == NL80211_IFTYPE_ADHOC && link_conf->bssid)
+		ether_addr_copy(cmd.ibss_bssid_addr, link_conf->bssid);
+
+	return iwl_mld_send_link_cmd(mld, &cmd, FW_CTXT_ACTION_ADD);
+}
+
+/* Get the basic rates of the used band and add the mandatory ones */
+static void iwl_mld_fill_rates(struct iwl_mld *mld,
+			       struct ieee80211_bss_conf *link,
+			       struct ieee80211_chanctx_conf *chan_ctx,
+			       __le32 *cck_rates, __le32 *ofdm_rates)
+{
+	struct cfg80211_chan_def *chandef =
+		iwl_mld_get_chandef_from_chanctx(mld, chan_ctx);
+	struct ieee80211_supported_band *sband =
+		mld->hw->wiphy->bands[chandef->chan->band];
+	unsigned long basic = link->basic_rates;
+	int lowest_present_ofdm = 100;
+	int lowest_present_cck = 100;
+	u32 cck = 0;
+	u32 ofdm = 0;
+	int i;
+
+	for_each_set_bit(i, &basic, BITS_PER_LONG) {
+		int hw = sband->bitrates[i].hw_value;
+
+		if (hw >= IWL_FIRST_OFDM_RATE) {
+			ofdm |= BIT(hw - IWL_FIRST_OFDM_RATE);
+			if (lowest_present_ofdm > hw)
+				lowest_present_ofdm = hw;
+		} else {
+			BUILD_BUG_ON(IWL_FIRST_CCK_RATE != 0);
+
+			cck |= BIT(hw);
+			if (lowest_present_cck > hw)
+				lowest_present_cck = hw;
+		}
+	}
+
+	/* Now we've got the basic rates as bitmaps in the ofdm and cck
+	 * variables. This isn't sufficient though, as there might not
+	 * be all the right rates in the bitmap. E.g. if the only basic
+	 * rates are 5.5 Mbps and 11 Mbps, we still need to add 1 Mbps
+	 * and 6 Mbps because the 802.11-2007 standard says in 9.6:
+	 *
+	 *    [...] a STA responding to a received frame shall transmit
+	 *    its Control Response frame [...] at the highest rate in the
+	 *    BSSBasicRateSet parameter that is less than or equal to the
+	 *    rate of the immediately previous frame in the frame exchange
+	 *    sequence ([...]) and that is of the same modulation class
+	 *    ([...]) as the received frame. If no rate contained in the
+	 *    BSSBasicRateSet parameter meets these conditions, then the
+	 *    control frame sent in response to a received frame shall be
+	 *    transmitted at the highest mandatory rate of the PHY that is
+	 *    less than or equal to the rate of the received frame, and
+	 *    that is of the same modulation class as the received frame.
+	 *
+	 * As a consequence, we need to add all mandatory rates that are
+	 * lower than all of the basic rates to these bitmaps.
+	 */
+
+	if (lowest_present_ofdm > IWL_RATE_24M_INDEX)
+		ofdm |= IWL_RATE_BIT_MSK(24) >> IWL_FIRST_OFDM_RATE;
+	if (lowest_present_ofdm > IWL_RATE_12M_INDEX)
+		ofdm |= IWL_RATE_BIT_MSK(12) >> IWL_FIRST_OFDM_RATE;
+	/* 6M already there or needed so always add */
+	ofdm |= IWL_RATE_BIT_MSK(6) >> IWL_FIRST_OFDM_RATE;
+
+	/* CCK is a bit more complex with DSSS vs. HR/DSSS vs. ERP.
+	 * Note, however:
+	 *  - if no CCK rates are basic, it must be ERP since there must
+	 *    be some basic rates at all, so they're OFDM => ERP PHY
+	 *    (or we're in 5 GHz, and the cck bitmap will never be used)
+	 *  - if 11M is a basic rate, it must be ERP as well, so add 5.5M
+	 *  - if 5.5M is basic, 1M and 2M are mandatory
+	 *  - if 2M is basic, 1M is mandatory
+	 *  - if 1M is basic, that's the only valid ACK rate.
+	 * As a consequence, it's not as complicated as it sounds, just add
+	 * any lower rates to the ACK rate bitmap.
+	 */
+	if (lowest_present_cck > IWL_RATE_11M_INDEX)
+		cck |= IWL_RATE_BIT_MSK(11) >> IWL_FIRST_CCK_RATE;
+	if (lowest_present_cck > IWL_RATE_5M_INDEX)
+		cck |= IWL_RATE_BIT_MSK(5) >> IWL_FIRST_CCK_RATE;
+	if (lowest_present_cck > IWL_RATE_2M_INDEX)
+		cck |= IWL_RATE_BIT_MSK(2) >> IWL_FIRST_CCK_RATE;
+	/* 1M already there or needed so always add */
+	cck |= IWL_RATE_BIT_MSK(1) >> IWL_FIRST_CCK_RATE;
+
+	*cck_rates = cpu_to_le32((u32)cck);
+	*ofdm_rates = cpu_to_le32((u32)ofdm);
+}
+
+static void iwl_mld_fill_protection_flags(struct iwl_mld *mld,
+					  struct ieee80211_bss_conf *link,
+					  __le32 *protection_flags)
+{
+	u8 protection_mode = link->ht_operation_mode &
+				IEEE80211_HT_OP_MODE_PROTECTION;
+	u8 ht_flag = LINK_PROT_FLG_HT_PROT | LINK_PROT_FLG_FAT_PROT;
+
+	IWL_DEBUG_RATE(mld, "HT protection mode: %d\n", protection_mode);
+
+	if (link->use_cts_prot)
+		*protection_flags |= cpu_to_le32(LINK_PROT_FLG_TGG_PROTECT);
+
+	/* See section 9.23.3.1 of IEEE 80211-2012.
+	 * Nongreenfield HT STAs Present is not supported.
+	 */
+	switch (protection_mode) {
+	case IEEE80211_HT_OP_MODE_PROTECTION_NONE:
+		break;
+	case IEEE80211_HT_OP_MODE_PROTECTION_NONMEMBER:
+	case IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED:
+		*protection_flags |= cpu_to_le32(ht_flag);
+		break;
+	case IEEE80211_HT_OP_MODE_PROTECTION_20MHZ:
+		/* Protect when channel wider than 20MHz */
+		if (link->chanreq.oper.width > NL80211_CHAN_WIDTH_20)
+			*protection_flags |= cpu_to_le32(ht_flag);
+		break;
+	}
+}
+
+static u8 iwl_mld_mac80211_ac_to_fw_ac(enum ieee80211_ac_numbers ac)
+{
+	static const u8 mac80211_ac_to_fw[] = {
+		AC_VO,
+		AC_VI,
+		AC_BE,
+		AC_BK
+	};
+
+	return mac80211_ac_to_fw[ac];
+}
+
+static void iwl_mld_fill_qos_params(struct ieee80211_bss_conf *link,
+				    struct iwl_ac_qos *ac, __le32 *qos_flags)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+
+	/* no need to check mld_link since it is done in the caller */
+
+	for (int mac_ac = 0; mac_ac < IEEE80211_NUM_ACS; mac_ac++) {
+		u8 txf = iwl_mld_mac80211_ac_to_fw_tx_fifo(mac_ac);
+		u8 fw_ac = iwl_mld_mac80211_ac_to_fw_ac(mac_ac);
+
+		ac[fw_ac].cw_min =
+			cpu_to_le16(mld_link->queue_params[mac_ac].cw_min);
+		ac[fw_ac].cw_max =
+			cpu_to_le16(mld_link->queue_params[mac_ac].cw_max);
+		ac[fw_ac].edca_txop =
+			cpu_to_le16(mld_link->queue_params[mac_ac].txop * 32);
+		ac[fw_ac].aifsn = mld_link->queue_params[mac_ac].aifs;
+		ac[fw_ac].fifos_mask = BIT(txf);
+	}
+
+	if (link->qos)
+		*qos_flags |= cpu_to_le32(MAC_QOS_FLG_UPDATE_EDCA);
+
+	if (link->chanreq.oper.width != NL80211_CHAN_WIDTH_20_NOHT)
+		*qos_flags |= cpu_to_le32(MAC_QOS_FLG_TGN);
+}
+
+static bool iwl_mld_fill_mu_edca(struct iwl_mld *mld,
+				 const struct iwl_mld_link *mld_link,
+				 struct iwl_he_backoff_conf *trig_based_txf)
+{
+	for (int mac_ac = 0; mac_ac < IEEE80211_NUM_ACS; mac_ac++) {
+		const struct ieee80211_he_mu_edca_param_ac_rec *mu_edca =
+			&mld_link->queue_params[mac_ac].mu_edca_param_rec;
+		u8 fw_ac = iwl_mld_mac80211_ac_to_fw_ac(mac_ac);
+
+		if (!mld_link->queue_params[mac_ac].mu_edca)
+			return false;
+
+		trig_based_txf[fw_ac].cwmin =
+			cpu_to_le16(mu_edca->ecw_min_max & 0xf);
+		trig_based_txf[fw_ac].cwmax =
+			cpu_to_le16((mu_edca->ecw_min_max & 0xf0) >> 4);
+		trig_based_txf[fw_ac].aifsn =
+			cpu_to_le16(mu_edca->aifsn & 0xf);
+		trig_based_txf[fw_ac].mu_time =
+			cpu_to_le16(mu_edca->mu_edca_timer);
+	}
+	return true;
+}
+
+int
+iwl_mld_change_link_in_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link,
+			  u32 changes)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	struct ieee80211_vif *vif = link->vif;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_chanctx_conf *chan_ctx;
+	struct iwl_link_config_cmd cmd = {};
+	u32 flags = 0;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld_link))
+		return -EINVAL;
+
+	cmd.link_id = cpu_to_le32(mld_link->fw_id);
+	cmd.spec_link_id = link->link_id;
+	cmd.mac_id = cpu_to_le32(mld_vif->fw_id);
+
+	chan_ctx = wiphy_dereference(mld->wiphy, mld_link->chan_ctx);
+
+	cmd.phy_id = cpu_to_le32(chan_ctx ?
+		iwl_mld_phy_from_mac80211(chan_ctx)->fw_id :
+		FW_CTXT_ID_INVALID);
+
+	ether_addr_copy(cmd.local_link_addr, link->addr);
+
+	cmd.active = cpu_to_le32(mld_link->active);
+
+	if ((changes & LINK_CONTEXT_MODIFY_ACTIVE) && !mld_link->active &&
+	    mld_link->silent_deactivation) {
+		/* We are de-activating a link that is having CSA with
+		 * immediate quiet in EMLSR. Tell the firmware not to send any
+		 * frame.
+		 */
+		cmd.block_tx = 1;
+		mld_link->silent_deactivation = false;
+	}
+
+	if (vif->type == NL80211_IFTYPE_ADHOC && link->bssid)
+		ether_addr_copy(cmd.ibss_bssid_addr, link->bssid);
+
+	/* Channel context is needed to get the rates */
+	if (chan_ctx)
+		iwl_mld_fill_rates(mld, link, chan_ctx, &cmd.cck_rates,
+				   &cmd.ofdm_rates);
+
+	cmd.cck_short_preamble = cpu_to_le32(link->use_short_preamble);
+	cmd.short_slot = cpu_to_le32(link->use_short_slot);
+
+	iwl_mld_fill_protection_flags(mld, link, &cmd.protection_flags);
+
+	iwl_mld_fill_qos_params(link, cmd.ac, &cmd.qos_flags);
+
+	cmd.bi = cpu_to_le32(link->beacon_int);
+	cmd.dtim_interval = cpu_to_le32(link->beacon_int * link->dtim_period);
+
+	/* Configure HE parameters only if HE is supported, and only after
+	 * the parameters are set in mac80211 (meaning after assoc)
+	 */
+	if (!link->he_support || iwlwifi_mod_params.disable_11ax ||
+	    (vif->type == NL80211_IFTYPE_STATION && !vif->cfg.assoc)) {
+		changes &= ~LINK_CONTEXT_MODIFY_HE_PARAMS;
+		goto send_cmd;
+	}
+
+	/* ap_sta may be NULL if we're disconnecting */
+	if (mld_vif->ap_sta) {
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_check(mld_vif->ap_sta,
+						   link->link_id);
+
+		if (!WARN_ON(!link_sta) && link_sta->he_cap.has_he &&
+		    link_sta->he_cap.he_cap_elem.mac_cap_info[5] &
+		    IEEE80211_HE_MAC_CAP5_OM_CTRL_UL_MU_DATA_DIS_RX)
+			cmd.ul_mu_data_disable = 1;
+	}
+
+	cmd.htc_trig_based_pkt_ext = link->htc_trig_based_pkt_ext;
+
+	if (link->uora_exists) {
+		cmd.rand_alloc_ecwmin = link->uora_ocw_range & 0x7;
+		cmd.rand_alloc_ecwmax = (link->uora_ocw_range >> 3) & 0x7;
+	}
+
+	if (iwl_mld_fill_mu_edca(mld, mld_link, cmd.trig_based_txf))
+		flags |= LINK_FLG_MU_EDCA_CW;
+
+	cmd.bss_color = link->he_bss_color.color;
+
+	if (!link->he_bss_color.enabled)
+		flags |= LINK_FLG_BSS_COLOR_DIS;
+
+	cmd.frame_time_rts_th = cpu_to_le16(link->frame_time_rts_th);
+
+	/* Block 26-tone RU OFDMA transmissions */
+	if (mld_link->he_ru_2mhz_block)
+		flags |= LINK_FLG_RU_2MHZ_BLOCK;
+
+	if (link->nontransmitted) {
+		ether_addr_copy(cmd.ref_bssid_addr, link->transmitter_bssid);
+		cmd.bssid_index = link->bssid_index;
+	}
+
+	/* The only EHT parameter is puncturing, and starting from PHY cmd
+	 * version 6 - it is sent there. For older versions of the PHY cmd,
+	 * puncturing is not needed at all.
+	 */
+	if (WARN_ON(changes & LINK_CONTEXT_MODIFY_EHT_PARAMS))
+		changes &= ~LINK_CONTEXT_MODIFY_EHT_PARAMS;
+
+send_cmd:
+	cmd.modify_mask = cpu_to_le32(changes);
+	cmd.flags = cpu_to_le32(flags);
+
+	return iwl_mld_send_link_cmd(mld, &cmd, FW_CTXT_ACTION_MODIFY);
+}
+
+int iwl_mld_activate_link(struct iwl_mld *mld,
+			  struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(link->vif);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld_link || mld_link->active))
+		return -EINVAL;
+
+	mld_link->active = true;
+
+	ret = iwl_mld_change_link_in_fw(mld, link,
+					LINK_CONTEXT_MODIFY_ACTIVE);
+	if (ret)
+		mld_link->active = false;
+	else
+		mld_vif->last_link_activation_time =
+			ktime_get_boottime_seconds();
+
+	return ret;
+}
+
+void iwl_mld_deactivate_link(struct iwl_mld *mld,
+			     struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	struct iwl_probe_resp_data *probe_data;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld_link || !mld_link->active))
+		return;
+
+	iwl_mld_cancel_session_protection(mld, link->vif, link->link_id);
+
+	/* If we deactivate the link, we will probably remove it, or switch
+	 * channel. In both cases, the CSA or Notice of Absence information is
+	 * now irrelevant. Remove the data here.
+	 */
+	probe_data = wiphy_dereference(mld->wiphy, mld_link->probe_resp_data);
+	RCU_INIT_POINTER(mld_link->probe_resp_data, NULL);
+	if (probe_data)
+		kfree_rcu(probe_data, rcu_head);
+
+	mld_link->active = false;
+
+	iwl_mld_change_link_in_fw(mld, link, LINK_CONTEXT_MODIFY_ACTIVE);
+
+	/* Now that the link is not active in FW, we don't expect any new
+	 * notifications for it. Cancel the ones that are already pending
+	 */
+	iwl_mld_cancel_notifications_of_object(mld, IWL_MLD_OBJECT_TYPE_LINK,
+					       mld_link->fw_id);
+}
+
+static void
+iwl_mld_rm_link_from_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	struct iwl_link_config_cmd cmd = {};
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld_link))
+		return;
+
+	cmd.link_id = cpu_to_le32(mld_link->fw_id);
+	cmd.spec_link_id = link->link_id;
+	cmd.phy_id = cpu_to_le32(FW_CTXT_ID_INVALID);
+
+	iwl_mld_send_link_cmd(mld, &cmd, FW_CTXT_ACTION_REMOVE);
+}
+
+IWL_MLD_ALLOC_FN(link, bss_conf)
+
+/* Constructor function for struct iwl_mld_link */
+static int
+iwl_mld_init_link(struct iwl_mld *mld, struct ieee80211_bss_conf *link,
+		  struct iwl_mld_link *mld_link)
+{
+	mld_link->average_beacon_energy = 0;
+
+	iwl_mld_init_internal_sta(&mld_link->bcast_sta);
+	iwl_mld_init_internal_sta(&mld_link->mcast_sta);
+	iwl_mld_init_internal_sta(&mld_link->mon_sta);
+
+	return iwl_mld_allocate_link_fw_id(mld, &mld_link->fw_id, link);
+}
+
+/* Initializes the link structure, maps fw id to the ieee80211_bss_conf, and
+ * adds a link to the fw
+ */
+int iwl_mld_add_link(struct iwl_mld *mld,
+		     struct ieee80211_bss_conf *bss_conf)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(bss_conf->vif);
+	struct iwl_mld_link *link = iwl_mld_link_from_mac80211(bss_conf);
+	bool is_deflink = bss_conf == &bss_conf->vif->bss_conf;
+	int ret;
+
+	if (!link) {
+		if (is_deflink)
+			link = &mld_vif->deflink;
+		else
+			link = kzalloc(sizeof(*link), GFP_KERNEL);
+	} else {
+		WARN_ON(!mld->fw_status.in_hw_restart);
+	}
+
+	ret = iwl_mld_init_link(mld, bss_conf, link);
+	if (ret)
+		goto free;
+
+	rcu_assign_pointer(mld_vif->link[bss_conf->link_id], link);
+
+	ret = iwl_mld_add_link_to_fw(mld, bss_conf);
+	if (ret) {
+		RCU_INIT_POINTER(mld->fw_id_to_bss_conf[link->fw_id], NULL);
+		RCU_INIT_POINTER(mld_vif->link[bss_conf->link_id], NULL);
+		goto free;
+	}
+
+	return ret;
+
+free:
+	if (!is_deflink)
+		kfree(link);
+	return ret;
+}
+
+/* Remove link from fw, unmap the bss_conf, and destroy the link structure */
+void iwl_mld_remove_link(struct iwl_mld *mld,
+			 struct ieee80211_bss_conf *bss_conf)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(bss_conf->vif);
+	struct iwl_mld_link *link = iwl_mld_link_from_mac80211(bss_conf);
+	bool is_deflink = link == &mld_vif->deflink;
+
+	if (WARN_ON(!link || link->active))
+		return;
+
+	iwl_mld_rm_link_from_fw(mld, bss_conf);
+	/* Continue cleanup on failure */
+
+	if (!is_deflink)
+		kfree_rcu(link, rcu_head);
+
+	RCU_INIT_POINTER(mld_vif->link[bss_conf->link_id], NULL);
+
+	if (WARN_ON(link->fw_id >= mld->fw->ucode_capa.num_links))
+		return;
+
+	RCU_INIT_POINTER(mld->fw_id_to_bss_conf[link->fw_id], NULL);
+}
+
+void iwl_mld_handle_missed_beacon_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt)
+{
+	const struct iwl_missed_beacons_notif *notif = (const void *)pkt->data;
+	union iwl_dbg_tlv_tp_data tp_data = { .fw_pkt = pkt };
+	u32 fw_link_id = le32_to_cpu(notif->link_id);
+	u32 missed_bcon = le32_to_cpu(notif->consec_missed_beacons);
+	u32 missed_bcon_since_rx =
+		le32_to_cpu(notif->consec_missed_beacons_since_last_rx);
+	u32 scnd_lnk_bcn_lost =
+		le32_to_cpu(notif->consec_missed_beacons_other_link);
+	struct ieee80211_bss_conf *link_conf =
+		iwl_mld_fw_id_to_link_conf(mld, fw_link_id);
+	u32 bss_param_ch_cnt_link_id;
+	struct ieee80211_vif *vif;
+	u8 link_id;
+
+	if (WARN_ON(!link_conf))
+		return;
+
+	vif = link_conf->vif;
+	link_id = link_conf->link_id;
+	bss_param_ch_cnt_link_id = link_conf->bss_param_ch_cnt_link_id;
+
+	IWL_DEBUG_INFO(mld,
+		       "missed bcn link_id=%u, %u consecutive=%u\n",
+		       link_id, missed_bcon, missed_bcon_since_rx);
+
+	if (WARN_ON(!vif))
+		return;
+
+	mld->trans->dbg.dump_file_name_ext_valid = true;
+	snprintf(mld->trans->dbg.dump_file_name_ext, IWL_FW_INI_MAX_NAME,
+		 "LinkId_%d_MacType_%d", fw_link_id,
+		 iwl_mld_mac80211_iftype_to_fw(vif));
+
+	iwl_dbg_tlv_time_point(&mld->fwrt,
+			       IWL_FW_INI_TIME_POINT_MISSED_BEACONS, &tp_data);
+
+	if (missed_bcon >= IWL_MLD_MISSED_BEACONS_THRESHOLD_LONG) {
+		if (missed_bcon_since_rx >=
+		    IWL_MLD_MISSED_BEACONS_SINCE_RX_THOLD) {
+			ieee80211_connection_loss(vif);
+			return;
+		}
+		IWL_WARN(mld,
+			 "missed beacons exceeds threshold, but receiving data. Stay connected, Expect bugs.\n");
+		return;
+	}
+
+	if (missed_bcon_since_rx > IWL_MLD_MISSED_BEACONS_THRESHOLD) {
+		ieee80211_cqm_beacon_loss_notify(vif, GFP_ATOMIC);
+
+		/* try to switch links, no-op if we don't have MLO */
+		iwl_mld_int_mlo_scan(mld, vif);
+	}
+
+	/* no more logic if we're not in EMLSR */
+	if (hweight16(vif->active_links) <= 1)
+		return;
+
+	/* We are processing a notification before link activation */
+	if (le32_to_cpu(notif->other_link_id) == FW_CTXT_ID_INVALID)
+		return;
+
+	/* Exit EMLSR if we lost more than
+	 * IWL_MLD_MISSED_BEACONS_EXIT_ESR_THRESH beacons on boths links
+	 * OR more than IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH on current link.
+	 * OR more than IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED
+	 * on current link and the link's bss_param_ch_count has changed on
+	 * the other link's beacon.
+	 */
+	if ((missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS &&
+	     scnd_lnk_bcn_lost >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS) ||
+	    missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH ||
+	    (bss_param_ch_cnt_link_id != link_id &&
+	     missed_bcon >=
+	     IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED)) {
+		iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_MISSED_BEACON,
+				   iwl_mld_get_primary_link(vif));
+	}
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_handle_missed_beacon_notif);
+
+bool iwl_mld_cancel_missed_beacon_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt,
+					u32 removed_link_id)
+{
+	struct iwl_missed_beacons_notif *notif = (void *)pkt->data;
+
+	if (le32_to_cpu(notif->other_link_id) == removed_link_id) {
+		/* Second link is being removed. Don't cancel the notification,
+		 * but mark second link as invalid.
+		 */
+		notif->other_link_id = cpu_to_le32(FW_CTXT_ID_INVALID);
+	}
+
+	/* If the primary link is removed, cancel the notification */
+	return le32_to_cpu(notif->link_id) == removed_link_id;
+}
+
+int iwl_mld_link_set_associated(struct iwl_mld *mld, struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link)
+{
+	return iwl_mld_change_link_in_fw(mld, link, LINK_CONTEXT_MODIFY_ALL &
+					 ~(LINK_CONTEXT_MODIFY_ACTIVE |
+					   LINK_CONTEXT_MODIFY_EHT_PARAMS));
+}
+
+struct iwl_mld_rssi_to_grade {
+	s8 rssi[2];
+	u16 grade;
+};
+
+#define RSSI_TO_GRADE_LINE(_lb, _hb_uhb, _grade) \
+	{ \
+		.rssi = {_lb, _hb_uhb}, \
+		.grade = _grade \
+	}
+
+/*
+ * This array must be sorted by increasing RSSI for proper functionality.
+ * The grades are actually estimated throughput, represented as fixed-point
+ * with a scale factor of 1/10.
+ */
+static const struct iwl_mld_rssi_to_grade rssi_to_grade_map[] = {
+	RSSI_TO_GRADE_LINE(-85, -89, 172),
+	RSSI_TO_GRADE_LINE(-83, -86, 344),
+	RSSI_TO_GRADE_LINE(-82, -85, 516),
+	RSSI_TO_GRADE_LINE(-80, -83, 688),
+	RSSI_TO_GRADE_LINE(-77, -79, 1032),
+	RSSI_TO_GRADE_LINE(-73, -76, 1376),
+	RSSI_TO_GRADE_LINE(-70, -74, 1548),
+	RSSI_TO_GRADE_LINE(-69, -72, 1720),
+	RSSI_TO_GRADE_LINE(-65, -68, 2064),
+	RSSI_TO_GRADE_LINE(-61, -66, 2294),
+	RSSI_TO_GRADE_LINE(-58, -61, 2580),
+	RSSI_TO_GRADE_LINE(-55, -58, 2868),
+	RSSI_TO_GRADE_LINE(-46, -55, 3098),
+	RSSI_TO_GRADE_LINE(-43, -54, 3442)
+};
+
+#define MAX_GRADE (rssi_to_grade_map[ARRAY_SIZE(rssi_to_grade_map) - 1].grade)
+
+#define DEFAULT_CHAN_LOAD_2GHZ	30
+#define DEFAULT_CHAN_LOAD_5GHZ	15
+#define DEFAULT_CHAN_LOAD_6GHZ	0
+
+/* Factors calculation is done with fixed-point with a scaling factor of 1/256 */
+#define SCALE_FACTOR 256
+#define MAX_CHAN_LOAD 256
+
+static unsigned int
+iwl_mld_get_n_subchannels(const struct ieee80211_bss_conf *link_conf)
+{
+	enum nl80211_chan_width chan_width =
+		link_conf->chanreq.oper.width;
+	int mhz = nl80211_chan_width_to_mhz(chan_width);
+	unsigned int n_subchannels;
+
+	if (WARN_ONCE(mhz < 20 || mhz > 320,
+		      "Invalid channel width : (%d)\n", mhz))
+		return 1;
+
+	/* total number of subchannels */
+	n_subchannels = mhz / 20;
+
+	/* No puncturing if less than 80 MHz */
+	if (mhz >= 80)
+		n_subchannels -= hweight16(link_conf->chanreq.oper.punctured);
+
+	return n_subchannels;
+}
+
+static int
+iwl_mld_get_chan_load_from_element(struct iwl_mld *mld,
+				   struct ieee80211_bss_conf *link_conf)
+{
+	struct ieee80211_vif *vif = link_conf->vif;
+	const struct cfg80211_bss_ies *ies;
+	const struct element *bss_load_elem = NULL;
+	const struct ieee80211_bss_load_elem *bss_load;
+
+	guard(rcu)();
+
+	if (ieee80211_vif_link_active(vif, link_conf->link_id))
+		ies = rcu_dereference(link_conf->bss->beacon_ies);
+	else
+		ies = rcu_dereference(link_conf->bss->ies);
+
+	if (ies)
+		bss_load_elem = cfg80211_find_elem(WLAN_EID_QBSS_LOAD,
+						   ies->data, ies->len);
+
+	if (!bss_load_elem ||
+	    bss_load_elem->datalen != sizeof(*bss_load))
+		return -EINVAL;
+
+	bss_load = (const void *)bss_load_elem->data;
+
+	return bss_load->channel_util;
+}
+
+static unsigned int
+iwl_mld_get_chan_load_by_us(struct iwl_mld *mld,
+			    struct ieee80211_bss_conf *link_conf,
+			    bool expect_active_link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link_conf);
+	struct ieee80211_chanctx_conf *chan_ctx;
+	struct iwl_mld_phy *phy;
+
+	if (!mld_link || !mld_link->active) {
+		WARN_ON(expect_active_link);
+		return 0;
+	}
+
+	if (WARN_ONCE(!rcu_access_pointer(mld_link->chan_ctx),
+		      "Active link (%u) without channel ctxt assigned!\n",
+		      link_conf->link_id))
+		return 0;
+
+	chan_ctx = wiphy_dereference(mld->wiphy, mld_link->chan_ctx);
+	phy = iwl_mld_phy_from_mac80211(chan_ctx);
+
+	return phy->channel_load_by_us;
+}
+
+/* Returns error if the channel utilization element is invalid/unavailable */
+int iwl_mld_get_chan_load_by_others(struct iwl_mld *mld,
+				    struct ieee80211_bss_conf *link_conf,
+				    bool expect_active_link)
+{
+	int chan_load;
+	unsigned int chan_load_by_us;
+
+	/* get overall load */
+	chan_load = iwl_mld_get_chan_load_from_element(mld, link_conf);
+	if (chan_load < 0)
+		return chan_load;
+
+	chan_load_by_us = iwl_mld_get_chan_load_by_us(mld, link_conf,
+						      expect_active_link);
+
+	/* channel load by us is given in percentage */
+	chan_load_by_us =
+		NORMALIZE_PERCENT_TO_255(chan_load_by_us);
+
+	/* Use only values that firmware sends that can possibly be valid */
+	if (chan_load_by_us <= chan_load)
+		chan_load -= chan_load_by_us;
+
+	return chan_load;
+}
+
+static unsigned int
+iwl_mld_get_default_chan_load(struct ieee80211_bss_conf *link_conf)
+{
+	enum nl80211_band band = link_conf->chanreq.oper.chan->band;
+
+	switch (band) {
+	case NL80211_BAND_2GHZ:
+		return DEFAULT_CHAN_LOAD_2GHZ;
+	case NL80211_BAND_5GHZ:
+		return DEFAULT_CHAN_LOAD_5GHZ;
+	case NL80211_BAND_6GHZ:
+		return DEFAULT_CHAN_LOAD_6GHZ;
+	default:
+		WARN_ON(1);
+		return 0;
+	}
+}
+
+unsigned int iwl_mld_get_chan_load(struct iwl_mld *mld,
+				   struct ieee80211_bss_conf *link_conf)
+{
+	int chan_load;
+
+	chan_load = iwl_mld_get_chan_load_by_others(mld, link_conf, false);
+	if (chan_load >= 0)
+		return chan_load;
+
+	/* No information from the element, take the defaults */
+	chan_load = iwl_mld_get_default_chan_load(link_conf);
+
+	/* The defaults are given in percentage */
+	return NORMALIZE_PERCENT_TO_255(chan_load);
+}
+
+static unsigned int
+iwl_mld_get_avail_chan_load(struct iwl_mld *mld,
+			    struct ieee80211_bss_conf *link_conf)
+{
+	return MAX_CHAN_LOAD - iwl_mld_get_chan_load(mld, link_conf);
+}
+
+/* This function calculates the grade of a link. Returns 0 in error case */
+unsigned int iwl_mld_get_link_grade(struct iwl_mld *mld,
+				    struct ieee80211_bss_conf *link_conf)
+{
+	enum nl80211_band band;
+	int rssi_idx;
+	s32 link_rssi;
+	unsigned int grade = MAX_GRADE;
+
+	if (WARN_ON_ONCE(!link_conf))
+		return 0;
+
+	band = link_conf->chanreq.oper.chan->band;
+	if (WARN_ONCE(band != NL80211_BAND_2GHZ &&
+		      band != NL80211_BAND_5GHZ &&
+		      band != NL80211_BAND_6GHZ,
+		      "Invalid band (%u)\n", band))
+		return 0;
+
+	link_rssi = MBM_TO_DBM(link_conf->bss->signal);
+	/*
+	 * For 6 GHz the RSSI of the beacons is lower than
+	 * the RSSI of the data.
+	 */
+	if (band == NL80211_BAND_6GHZ && link_rssi)
+		link_rssi += 4;
+
+	rssi_idx = band == NL80211_BAND_2GHZ ? 0 : 1;
+
+	/* No valid RSSI - take the lowest grade */
+	if (!link_rssi)
+		link_rssi = rssi_to_grade_map[0].rssi[rssi_idx];
+
+	IWL_DEBUG_EHT(mld,
+		      "Calculating grade of link %d: band = %d, bandwidth = %d, punctured subchannels =0x%x RSSI = %d\n",
+		      link_conf->link_id, band,
+		      link_conf->chanreq.oper.width,
+		      link_conf->chanreq.oper.punctured, link_rssi);
+
+	/* Get grade based on RSSI */
+	for (int i = 0; i < ARRAY_SIZE(rssi_to_grade_map); i++) {
+		const struct iwl_mld_rssi_to_grade *line =
+			&rssi_to_grade_map[i];
+
+		if (link_rssi > line->rssi[rssi_idx])
+			continue;
+		grade = line->grade;
+		break;
+	}
+
+	/* Apply the channel load and puncturing factors */
+	grade = grade * iwl_mld_get_avail_chan_load(mld, link_conf) / SCALE_FACTOR;
+	grade = grade * iwl_mld_get_n_subchannels(link_conf);
+
+	IWL_DEBUG_EHT(mld, "Link %d's grade: %d\n", link_conf->link_id, grade);
+
+	return grade;
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_get_link_grade);
+
+void iwl_mld_handle_beacon_filter_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt)
+{
+	const struct iwl_beacon_filter_notif *notif = (const void *)pkt->data;
+	u32 link_id = le32_to_cpu(notif->link_id);
+	struct ieee80211_bss_conf *link_conf =
+		iwl_mld_fw_id_to_link_conf(mld, link_id);
+	struct iwl_mld_link *mld_link;
+
+	if (IWL_FW_CHECK(mld, !link_conf, "invalid link ID %d\n", link_id))
+		return;
+
+	mld_link = iwl_mld_link_from_mac80211(link_conf);
+	if (WARN_ON_ONCE(!mld_link))
+		return;
+
+	mld_link->average_beacon_energy = le32_to_cpu(notif->average_energy);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/link.h b/sys/contrib/dev/iwlwifi/mld/link.h
new file mode 100644
index 000000000000..cad2c9426349
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/link.h
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_link_h__
+#define __iwl_mld_link_h__
+
+#include <net/mac80211.h>
+
+#include "mld.h"
+#include "sta.h"
+
+/**
+ * struct iwl_probe_resp_data - data for NoA/CSA updates
+ * @rcu_head: used for freeing the data on update
+ * @notif: notification data
+ * @noa_len: length of NoA attribute, calculated from the notification
+ */
+struct iwl_probe_resp_data {
+	struct rcu_head rcu_head;
+	struct iwl_probe_resp_data_notif notif;
+	int noa_len;
+};
+
+/**
+ * struct iwl_mld_link - link configuration parameters
+ *
+ * @rcu_head: RCU head for freeing this data.
+ * @fw_id: the fw id of the link.
+ * @active: if the link is active or not.
+ * @queue_params: QoS data from mac80211. This is updated with a call to
+ *	drv_conf_tx per each AC, and then notified once with BSS_CHANGED_QOS.
+ *	So we store it here and then send one link cmd for all the ACs.
+ * @chan_ctx: pointer to the channel context assigned to the link. If a link
+ *	has an assigned channel context it means that it is active.
+ * @he_ru_2mhz_block: 26-tone RU OFDMA transmissions should be blocked.
+ * @igtk: fw can only have one IGTK at a time, whereas mac80211 can have two.
+ *	This tracks the one IGTK that currently exists in FW.
+ * @bcast_sta: station used for broadcast packets. Used in AP, GO and IBSS.
+ * @mcast_sta: station used for multicast packets. Used in AP, GO and IBSS.
+ * @mon_sta: station used for TX injection in monitor interface.
+ * @average_beacon_energy: average beacon energy for beacons received during
+ *	client connections
+ * @ap_early_keys: The firmware cannot install keys before bcast/mcast STAs,
+ *	but higher layers work differently, so we store the keys here for
+ *	later installation.
+ * @silent_deactivation: next deactivation needs to be silent.
+ * @probe_resp_data: data from FW notification to store NOA related data to be
+ *	inserted into probe response.
+ */
+struct iwl_mld_link {
+	struct rcu_head rcu_head;
+
+	/* Add here fields that need clean up on restart */
+	struct_group(zeroed_on_hw_restart,
+		u8 fw_id;
+		bool active;
+		struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS];
+		struct ieee80211_chanctx_conf __rcu *chan_ctx;
+		bool he_ru_2mhz_block;
+		struct ieee80211_key_conf *igtk;
+	);
+	/* And here fields that survive a fw restart */
+	struct iwl_mld_int_sta bcast_sta;
+	struct iwl_mld_int_sta mcast_sta;
+	struct iwl_mld_int_sta mon_sta;
+
+	/* we can only have 2 GTK + 2 IGTK + 2 BIGTK active at a time */
+	struct ieee80211_key_conf *ap_early_keys[6];
+	u32 average_beacon_energy;
+	bool silent_deactivation;
+	struct iwl_probe_resp_data __rcu *probe_resp_data;
+};
+
+/* Cleanup function for struct iwl_mld_link, will be called in restart */
+static inline void
+iwl_mld_cleanup_link(struct iwl_mld *mld, struct iwl_mld_link *link)
+{
+	struct iwl_probe_resp_data *probe_data;
+
+	probe_data = wiphy_dereference(mld->wiphy, link->probe_resp_data);
+	RCU_INIT_POINTER(link->probe_resp_data, NULL);
+	if (probe_data)
+		kfree_rcu(probe_data, rcu_head);
+
+	CLEANUP_STRUCT(link);
+	if (link->bcast_sta.sta_id != IWL_INVALID_STA)
+		iwl_mld_free_internal_sta(mld, &link->bcast_sta);
+	if (link->mcast_sta.sta_id != IWL_INVALID_STA)
+		iwl_mld_free_internal_sta(mld, &link->mcast_sta);
+	if (link->mon_sta.sta_id != IWL_INVALID_STA)
+		iwl_mld_free_internal_sta(mld, &link->mon_sta);
+}
+
+/* Convert a percentage from [0,100] to [0,255] */
+#define NORMALIZE_PERCENT_TO_255(percentage) ((percentage) * 256 / 100)
+
+int iwl_mld_add_link(struct iwl_mld *mld,
+		     struct ieee80211_bss_conf *bss_conf);
+void iwl_mld_remove_link(struct iwl_mld *mld,
+			 struct ieee80211_bss_conf *bss_conf);
+int iwl_mld_activate_link(struct iwl_mld *mld,
+			  struct ieee80211_bss_conf *link);
+void iwl_mld_deactivate_link(struct iwl_mld *mld,
+			     struct ieee80211_bss_conf *link);
+int iwl_mld_change_link_in_fw(struct iwl_mld *mld,
+			      struct ieee80211_bss_conf *link, u32 changes);
+void iwl_mld_handle_missed_beacon_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt);
+bool iwl_mld_cancel_missed_beacon_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt,
+					u32 removed_link_id);
+int iwl_mld_link_set_associated(struct iwl_mld *mld, struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link);
+
+unsigned int iwl_mld_get_link_grade(struct iwl_mld *mld,
+				    struct ieee80211_bss_conf *link_conf);
+
+unsigned int iwl_mld_get_chan_load(struct iwl_mld *mld,
+				   struct ieee80211_bss_conf *link_conf);
+
+int iwl_mld_get_chan_load_by_others(struct iwl_mld *mld,
+				    struct ieee80211_bss_conf *link_conf,
+				    bool expect_active_link);
+
+void iwl_mld_handle_beacon_filter_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt);
+
+#endif /* __iwl_mld_link_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/low_latency.c b/sys/contrib/dev/iwlwifi/mld/low_latency.c
new file mode 100644
index 000000000000..23362867b400
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/low_latency.c
@@ -0,0 +1,336 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include "mld.h"
+#include "iface.h"
+#include "low_latency.h"
+#include "hcmd.h"
+#include "power.h"
+#include "mlo.h"
+
+#define MLD_LL_WK_INTERVAL_MSEC 500
+#define MLD_LL_PERIOD (HZ * MLD_LL_WK_INTERVAL_MSEC / 1000)
+#define MLD_LL_ACTIVE_WK_PERIOD (HZ * 10)
+
+/* packets/MLD_LL_WK_PERIOD seconds */
+#define MLD_LL_ENABLE_THRESH 100
+
+static bool iwl_mld_calc_low_latency(struct iwl_mld *mld,
+				     unsigned long timestamp)
+{
+	struct iwl_mld_low_latency *ll = &mld->low_latency;
+	bool global_low_latency = false;
+	u8 num_rx_q = mld->trans->info.num_rxqs;
+
+	for (int mac_id = 0; mac_id < NUM_MAC_INDEX_DRIVER; mac_id++) {
+		u32 total_vo_vi_pkts = 0;
+		bool ll_period_expired;
+
+		/* If it's not initialized yet, it means we have not yet
+		 * received/transmitted any vo/vi packet on this MAC.
+		 */
+		if (!ll->window_start[mac_id])
+			continue;
+
+		ll_period_expired =
+			time_after(timestamp, ll->window_start[mac_id] +
+				   MLD_LL_ACTIVE_WK_PERIOD);
+
+		if (ll_period_expired)
+			ll->window_start[mac_id] = timestamp;
+
+		for (int q = 0; q < num_rx_q; q++) {
+			struct iwl_mld_low_latency_packets_counters *counters =
+				&mld->low_latency.pkts_counters[q];
+
+			spin_lock_bh(&counters->lock);
+
+			total_vo_vi_pkts += counters->vo_vi[mac_id];
+
+			if (ll_period_expired)
+				counters->vo_vi[mac_id] = 0;
+
+			spin_unlock_bh(&counters->lock);
+		}
+
+		/* enable immediately with enough packets but defer
+		 * disabling only if the low-latency period expired and
+		 * below threshold.
+		 */
+		if (total_vo_vi_pkts > MLD_LL_ENABLE_THRESH)
+			mld->low_latency.result[mac_id] = true;
+		else if (ll_period_expired)
+			mld->low_latency.result[mac_id] = false;
+
+		global_low_latency |= mld->low_latency.result[mac_id];
+	}
+
+	return global_low_latency;
+}
+
+static void iwl_mld_low_latency_iter(void *_data, u8 *mac,
+				     struct ieee80211_vif *vif)
+{
+	struct iwl_mld *mld = _data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	bool prev = mld_vif->low_latency_causes & LOW_LATENCY_TRAFFIC;
+	bool low_latency;
+
+	if (WARN_ON(mld_vif->fw_id >= ARRAY_SIZE(mld->low_latency.result)))
+		return;
+
+	low_latency = mld->low_latency.result[mld_vif->fw_id];
+
+	if (prev != low_latency)
+		iwl_mld_vif_update_low_latency(mld, vif, low_latency,
+					       LOW_LATENCY_TRAFFIC);
+}
+
+static void iwl_mld_low_latency_wk(struct wiphy *wiphy, struct wiphy_work *wk)
+{
+	struct iwl_mld *mld = container_of(wk, struct iwl_mld,
+					   low_latency.work.work);
+	unsigned long timestamp = jiffies;
+	bool low_latency_active;
+
+	if (mld->fw_status.in_hw_restart)
+		return;
+
+	/* It is assumed that the work was scheduled only after checking
+	 * at least MLD_LL_PERIOD has passed since the last update.
+	 */
+
+	low_latency_active = iwl_mld_calc_low_latency(mld, timestamp);
+
+	/* Update the timestamp now after the low-latency calculation */
+	mld->low_latency.timestamp = timestamp;
+
+	/* If low-latency is active we need to force re-evaluation after
+	 * 10 seconds, so that we can disable low-latency when
+	 * the low-latency traffic ends.
+	 *
+	 * Otherwise, we don't need to run the work because there is nothing to
+	 * disable.
+	 *
+	 * Note that this has no impact on the regular scheduling of the
+	 * updates triggered by traffic - those happen whenever the
+	 * MLD_LL_PERIOD timeout expire.
+	 */
+	if (low_latency_active)
+		wiphy_delayed_work_queue(mld->wiphy, &mld->low_latency.work,
+					 MLD_LL_ACTIVE_WK_PERIOD);
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_low_latency_iter, mld);
+}
+
+int iwl_mld_low_latency_init(struct iwl_mld *mld)
+{
+	struct iwl_mld_low_latency *ll = &mld->low_latency;
+	unsigned long ts = jiffies;
+
+	ll->pkts_counters = kcalloc(mld->trans->info.num_rxqs,
+				    sizeof(*ll->pkts_counters), GFP_KERNEL);
+	if (!ll->pkts_counters)
+		return -ENOMEM;
+
+	for (int q = 0; q < mld->trans->info.num_rxqs; q++)
+		spin_lock_init(&ll->pkts_counters[q].lock);
+
+	wiphy_delayed_work_init(&ll->work, iwl_mld_low_latency_wk);
+
+	ll->timestamp = ts;
+
+	/* The low-latency window_start will be initialized per-MAC on
+	 * the first vo/vi packet received/transmitted.
+	 */
+
+	return 0;
+}
+
+void iwl_mld_low_latency_free(struct iwl_mld *mld)
+{
+	struct iwl_mld_low_latency *ll = &mld->low_latency;
+
+	kfree(ll->pkts_counters);
+	ll->pkts_counters = NULL;
+}
+
+void iwl_mld_low_latency_restart_cleanup(struct iwl_mld *mld)
+{
+	struct iwl_mld_low_latency *ll = &mld->low_latency;
+
+	ll->timestamp = jiffies;
+
+	memset(ll->window_start, 0, sizeof(ll->window_start));
+	memset(ll->result, 0, sizeof(ll->result));
+
+	for (int q = 0; q < mld->trans->info.num_rxqs; q++)
+		memset(ll->pkts_counters[q].vo_vi, 0,
+		       sizeof(ll->pkts_counters[q].vo_vi));
+}
+
+static int iwl_mld_send_low_latency_cmd(struct iwl_mld *mld, bool low_latency,
+					u16 mac_id)
+{
+	struct iwl_mac_low_latency_cmd cmd = {
+		.mac_id = cpu_to_le32(mac_id)
+	};
+	u16 cmd_id = WIDE_ID(MAC_CONF_GROUP, LOW_LATENCY_CMD);
+	int ret;
+
+	if (low_latency) {
+		/* Currently we don't care about the direction */
+		cmd.low_latency_rx = 1;
+		cmd.low_latency_tx = 1;
+	}
+
+	ret = iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd);
+	if (ret)
+		IWL_ERR(mld, "Failed to send low latency command\n");
+
+	return ret;
+}
+
+static void iwl_mld_vif_set_low_latency(struct iwl_mld_vif *mld_vif, bool set,
+					enum iwl_mld_low_latency_cause cause)
+{
+	if (set)
+		mld_vif->low_latency_causes |= cause;
+	else
+		mld_vif->low_latency_causes &= ~cause;
+}
+
+void iwl_mld_vif_update_low_latency(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    bool low_latency,
+				    enum iwl_mld_low_latency_cause cause)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	bool prev;
+
+	prev = iwl_mld_vif_low_latency(mld_vif);
+	iwl_mld_vif_set_low_latency(mld_vif, low_latency, cause);
+
+	low_latency = iwl_mld_vif_low_latency(mld_vif);
+	if (low_latency == prev)
+		return;
+
+	if (iwl_mld_send_low_latency_cmd(mld, low_latency, mld_vif->fw_id)) {
+		/* revert to previous low-latency state */
+		iwl_mld_vif_set_low_latency(mld_vif, prev, cause);
+		return;
+	}
+
+	if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_P2P_CLIENT)
+		return;
+
+	iwl_mld_update_mac_power(mld, vif, false);
+
+	if (low_latency)
+		iwl_mld_retry_emlsr(mld, vif);
+}
+
+static bool iwl_mld_is_vo_vi_pkt(struct ieee80211_hdr *hdr)
+{
+	u8 tid;
+	static const u8 tid_to_mac80211_ac[] = {
+		IEEE80211_AC_BE,
+		IEEE80211_AC_BK,
+		IEEE80211_AC_BK,
+		IEEE80211_AC_BE,
+		IEEE80211_AC_VI,
+		IEEE80211_AC_VI,
+		IEEE80211_AC_VO,
+		IEEE80211_AC_VO,
+	};
+
+	if (!hdr || !ieee80211_is_data_qos(hdr->frame_control))
+		return false;
+
+	tid = ieee80211_get_tid(hdr);
+	if (tid >= IWL_MAX_TID_COUNT)
+		return false;
+
+	return tid_to_mac80211_ac[tid] < IEEE80211_AC_VI;
+}
+
+void iwl_mld_low_latency_update_counters(struct iwl_mld *mld,
+					 struct ieee80211_hdr *hdr,
+					 struct ieee80211_sta *sta,
+					 u8 queue)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(mld_sta->vif);
+	struct iwl_mld_low_latency_packets_counters *counters;
+	unsigned long ts = jiffies ? jiffies : 1;
+	u8 fw_id = mld_vif->fw_id;
+
+	/* we should have failed op mode init if NULL */
+	if (WARN_ON_ONCE(!mld->low_latency.pkts_counters))
+		return;
+
+	if (WARN_ON_ONCE(fw_id >= ARRAY_SIZE(counters->vo_vi) ||
+			 queue >= mld->trans->info.num_rxqs))
+		return;
+
+	if (mld->low_latency.stopped)
+		return;
+
+	if (!iwl_mld_is_vo_vi_pkt(hdr))
+		return;
+
+	counters = &mld->low_latency.pkts_counters[queue];
+
+	spin_lock_bh(&counters->lock);
+	counters->vo_vi[fw_id]++;
+	spin_unlock_bh(&counters->lock);
+
+	/* Initialize the window_start on the first vo/vi packet */
+	if (!mld->low_latency.window_start[fw_id])
+		mld->low_latency.window_start[fw_id] = ts;
+
+	if (time_is_before_jiffies(mld->low_latency.timestamp + MLD_LL_PERIOD))
+		wiphy_delayed_work_queue(mld->wiphy, &mld->low_latency.work,
+					 0);
+}
+
+void iwl_mld_low_latency_stop(struct iwl_mld *mld)
+{
+	lockdep_assert_wiphy(mld->wiphy);
+
+	mld->low_latency.stopped = true;
+
+	wiphy_delayed_work_cancel(mld->wiphy, &mld->low_latency.work);
+}
+
+void iwl_mld_low_latency_restart(struct iwl_mld *mld)
+{
+	struct iwl_mld_low_latency *ll = &mld->low_latency;
+	bool low_latency = false;
+	unsigned long ts = jiffies;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ll->timestamp = ts;
+	mld->low_latency.stopped = false;
+
+	for (int mac = 0; mac < NUM_MAC_INDEX_DRIVER; mac++) {
+		ll->window_start[mac] = 0;
+		low_latency |= ll->result[mac];
+
+		for (int q = 0; q < mld->trans->info.num_rxqs; q++) {
+			spin_lock_bh(&ll->pkts_counters[q].lock);
+			ll->pkts_counters[q].vo_vi[mac] = 0;
+			spin_unlock_bh(&ll->pkts_counters[q].lock);
+		}
+	}
+
+	/* if low latency is active, force re-evaluation to cover the case of
+	 * no traffic.
+	 */
+	if (low_latency)
+		wiphy_delayed_work_queue(mld->wiphy, &ll->work, MLD_LL_PERIOD);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/low_latency.h b/sys/contrib/dev/iwlwifi/mld/low_latency.h
new file mode 100644
index 000000000000..f59684d235af
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/low_latency.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_low_latency_h__
+#define __iwl_mld_low_latency_h__
+
+/**
+ * struct iwl_mld_low_latency_packets_counters - Packets counters
+ * @lock: synchronize the counting in data path against the worker
+ * @vo_vi: per-mac, counts the number of TX and RX voice and video packets
+ */
+struct iwl_mld_low_latency_packets_counters {
+	spinlock_t lock;
+	u32 vo_vi[NUM_MAC_INDEX_DRIVER];
+} ____cacheline_aligned_in_smp;
+
+/**
+ * enum iwl_mld_low_latency_cause - low-latency set causes
+ *
+ * @LOW_LATENCY_TRAFFIC: indicates low-latency traffic was detected
+ * @LOW_LATENCY_DEBUGFS: low-latency mode set from debugfs
+ * @LOW_LATENCY_VIF_TYPE: low-latency mode set because of vif type (AP)
+ */
+enum iwl_mld_low_latency_cause {
+	LOW_LATENCY_TRAFFIC	= BIT(0),
+	LOW_LATENCY_DEBUGFS	= BIT(1),
+	LOW_LATENCY_VIF_TYPE	= BIT(2),
+};
+
+/**
+ * struct iwl_mld_low_latency - Manage low-latency detection and activation.
+ * @work: this work is used to detect low-latency by monitoring the number of
+ *	voice and video packets transmitted in a period of time. If the
+ *	threshold is reached, low-latency is activated. When active,
+ *	it is deactivated if the threshold is not reached within a
+ *	10-second period.
+ * @timestamp: timestamp of the last update.
+ * @window_start: per-mac, timestamp of the start of the current window. when
+ *	the window is over, the counters are reset.
+ * @pkts_counters: per-queue array voice/video packet counters
+ * @result: per-mac latest low-latency result
+ * @stopped: if true, ignore the requests to update the counters
+ */
+struct iwl_mld_low_latency {
+	struct wiphy_delayed_work work;
+	unsigned long timestamp;
+	unsigned long window_start[NUM_MAC_INDEX_DRIVER];
+	struct iwl_mld_low_latency_packets_counters *pkts_counters;
+	bool result[NUM_MAC_INDEX_DRIVER];
+	bool stopped;
+};
+
+int iwl_mld_low_latency_init(struct iwl_mld *mld);
+void iwl_mld_low_latency_free(struct iwl_mld *mld);
+void iwl_mld_low_latency_restart_cleanup(struct iwl_mld *mld);
+void iwl_mld_vif_update_low_latency(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    bool low_latency,
+				    enum iwl_mld_low_latency_cause cause);
+void iwl_mld_low_latency_update_counters(struct iwl_mld *mld,
+					 struct ieee80211_hdr *hdr,
+					 struct ieee80211_sta *sta,
+					 u8 queue);
+void iwl_mld_low_latency_stop(struct iwl_mld *mld);
+void iwl_mld_low_latency_restart(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_low_latency_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/mac80211.c b/sys/contrib/dev/iwlwifi/mld/mac80211.c
new file mode 100644
index 000000000000..b0bd01914a91
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mac80211.c
@@ -0,0 +1,2672 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include <net/mac80211.h>
+#include <linux/fips.h>
+#include <linux/ip.h>
+
+#include "mld.h"
+#include "mac80211.h"
+#include "phy.h"
+#include "iface.h"
+#include "power.h"
+#include "sta.h"
+#include "agg.h"
+#include "scan.h"
+#include "d3.h"
+#include "tlc.h"
+#include "key.h"
+#include "ap.h"
+#include "tx.h"
+#include "roc.h"
+#include "mlo.h"
+#include "stats.h"
+#include "ftm-initiator.h"
+#include "low_latency.h"
+#include "fw/api/scan.h"
+#include "fw/api/context.h"
+#include "fw/api/filter.h"
+#include "fw/api/sta.h"
+#include "fw/api/tdls.h"
+#ifdef CONFIG_PM_SLEEP
+#include "fw/api/d3.h"
+#endif /* CONFIG_PM_SLEEP */
+#include "iwl-trans.h"
+
+#define IWL_MLD_LIMITS(ap)					\
+	{							\
+		.max = 2,					\
+		.types = BIT(NL80211_IFTYPE_STATION),		\
+	},							\
+	{							\
+		.max = 1,					\
+		.types = ap |					\
+			 BIT(NL80211_IFTYPE_P2P_CLIENT) |	\
+			 BIT(NL80211_IFTYPE_P2P_GO),		\
+	},							\
+	{							\
+		.max = 1,					\
+		.types = BIT(NL80211_IFTYPE_P2P_DEVICE),	\
+	}
+
+static const struct ieee80211_iface_limit iwl_mld_limits[] = {
+	IWL_MLD_LIMITS(0)
+};
+
+static const struct ieee80211_iface_limit iwl_mld_limits_ap[] = {
+	IWL_MLD_LIMITS(BIT(NL80211_IFTYPE_AP))
+};
+
+static const struct ieee80211_iface_combination
+iwl_mld_iface_combinations[] = {
+	{
+		.num_different_channels = 2,
+		.max_interfaces = 4,
+		.limits = iwl_mld_limits,
+		.n_limits = ARRAY_SIZE(iwl_mld_limits),
+	},
+	{
+		.num_different_channels = 1,
+		.max_interfaces = 4,
+		.limits = iwl_mld_limits_ap,
+		.n_limits = ARRAY_SIZE(iwl_mld_limits_ap),
+	},
+};
+
+static const u8 if_types_ext_capa_sta[] = {
+	 [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
+	 [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
+	 [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF |
+	       WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB,
+	 [8] = WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB,
+	 [9] = WLAN_EXT_CAPA10_TWT_REQUESTER_SUPPORT,
+};
+
+#define IWL_MLD_EMLSR_CAPA	(IEEE80211_EML_CAP_EMLSR_SUPP | \
+				 IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US << \
+					__bf_shf(IEEE80211_EML_CAP_EMLSR_PADDING_DELAY) | \
+				 IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US << \
+					__bf_shf(IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY))
+#define IWL_MLD_CAPA_OPS (FIELD_PREP_CONST( \
+			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP, \
+			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME) | \
+			IEEE80211_MLD_CAP_OP_LINK_RECONF_SUPPORT)
+
+static const struct wiphy_iftype_ext_capab iftypes_ext_capa[] = {
+	{
+		.iftype = NL80211_IFTYPE_STATION,
+		.extended_capabilities = if_types_ext_capa_sta,
+		.extended_capabilities_mask = if_types_ext_capa_sta,
+		.extended_capabilities_len = sizeof(if_types_ext_capa_sta),
+		/* relevant only if EHT is supported */
+		.eml_capabilities = IWL_MLD_EMLSR_CAPA,
+		.mld_capa_and_ops = IWL_MLD_CAPA_OPS,
+	},
+};
+
+static void iwl_mld_hw_set_addresses(struct iwl_mld *mld)
+{
+	struct wiphy *wiphy = mld->wiphy;
+	int num_addrs = 1;
+
+	/* Extract MAC address */
+	memcpy(mld->addresses[0].addr, mld->nvm_data->hw_addr, ETH_ALEN);
+	wiphy->addresses = mld->addresses;
+	wiphy->n_addresses = 1;
+
+	/* Extract additional MAC addresses if available */
+	if (mld->nvm_data->n_hw_addrs > 1)
+		num_addrs = min(mld->nvm_data->n_hw_addrs,
+				IWL_MLD_MAX_ADDRESSES);
+
+	for (int i = 1; i < num_addrs; i++) {
+		memcpy(mld->addresses[i].addr,
+		       mld->addresses[i - 1].addr,
+		       ETH_ALEN);
+		mld->addresses[i].addr[ETH_ALEN - 1]++;
+		wiphy->n_addresses++;
+	}
+}
+
+static void iwl_mld_hw_set_channels(struct iwl_mld *mld)
+{
+	struct wiphy *wiphy = mld->wiphy;
+	struct ieee80211_supported_band *bands = mld->nvm_data->bands;
+
+	wiphy->bands[NL80211_BAND_2GHZ] = &bands[NL80211_BAND_2GHZ];
+	wiphy->bands[NL80211_BAND_5GHZ] = &bands[NL80211_BAND_5GHZ];
+
+	if (bands[NL80211_BAND_6GHZ].n_channels)
+		wiphy->bands[NL80211_BAND_6GHZ] = &bands[NL80211_BAND_6GHZ];
+}
+
+static void iwl_mld_hw_set_security(struct iwl_mld *mld)
+{
+	struct ieee80211_hw *hw = mld->hw;
+	static const u32 mld_ciphers[] = {
+		WLAN_CIPHER_SUITE_WEP40,
+		WLAN_CIPHER_SUITE_WEP104,
+		WLAN_CIPHER_SUITE_TKIP,
+		WLAN_CIPHER_SUITE_CCMP,
+		WLAN_CIPHER_SUITE_GCMP,
+		WLAN_CIPHER_SUITE_GCMP_256,
+		WLAN_CIPHER_SUITE_AES_CMAC,
+		WLAN_CIPHER_SUITE_BIP_GMAC_128,
+		WLAN_CIPHER_SUITE_BIP_GMAC_256
+	};
+
+	if (fips_enabled)
+		return;
+
+	hw->wiphy->n_cipher_suites = ARRAY_SIZE(mld_ciphers);
+	hw->wiphy->cipher_suites = mld_ciphers;
+
+	ieee80211_hw_set(hw, MFP_CAPABLE);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_BEACON_PROTECTION);
+}
+
+static void iwl_mld_hw_set_antennas(struct iwl_mld *mld)
+{
+	struct wiphy *wiphy = mld->wiphy;
+
+	wiphy->available_antennas_tx = iwl_mld_get_valid_tx_ant(mld);
+	wiphy->available_antennas_rx = iwl_mld_get_valid_rx_ant(mld);
+}
+
+static void iwl_mld_hw_set_pm(struct iwl_mld *mld)
+{
+#ifdef CONFIG_PM_SLEEP
+	struct wiphy *wiphy = mld->wiphy;
+
+	if (!device_can_wakeup(mld->trans->dev))
+		return;
+
+	if (fips_enabled)
+		return;
+
+	mld->wowlan.flags |= WIPHY_WOWLAN_MAGIC_PKT |
+			     WIPHY_WOWLAN_DISCONNECT |
+			     WIPHY_WOWLAN_EAP_IDENTITY_REQ |
+			     WIPHY_WOWLAN_RFKILL_RELEASE |
+			     WIPHY_WOWLAN_NET_DETECT |
+			     WIPHY_WOWLAN_SUPPORTS_GTK_REKEY |
+			     WIPHY_WOWLAN_GTK_REKEY_FAILURE |
+			     WIPHY_WOWLAN_4WAY_HANDSHAKE;
+
+	mld->wowlan.n_patterns = IWL_WOWLAN_MAX_PATTERNS;
+	mld->wowlan.pattern_min_len = IWL_WOWLAN_MIN_PATTERN_LEN;
+	mld->wowlan.pattern_max_len = IWL_WOWLAN_MAX_PATTERN_LEN;
+	mld->wowlan.max_nd_match_sets = IWL_SCAN_MAX_PROFILES_V2;
+
+	wiphy->wowlan = &mld->wowlan;
+#endif /* CONFIG_PM_SLEEP */
+}
+
+static void iwl_mac_hw_set_radiotap(struct iwl_mld *mld)
+{
+	struct ieee80211_hw *hw = mld->hw;
+
+	hw->radiotap_mcs_details |= IEEE80211_RADIOTAP_MCS_HAVE_FEC |
+				    IEEE80211_RADIOTAP_MCS_HAVE_STBC;
+
+	hw->radiotap_vht_details |= IEEE80211_RADIOTAP_VHT_KNOWN_STBC |
+				    IEEE80211_RADIOTAP_VHT_KNOWN_BEAMFORMED;
+
+	hw->radiotap_timestamp.units_pos =
+		IEEE80211_RADIOTAP_TIMESTAMP_UNIT_US |
+		IEEE80211_RADIOTAP_TIMESTAMP_SPOS_PLCP_SIG_ACQ;
+
+	/* this is the case for CCK frames, it's better (only 8) for OFDM */
+	hw->radiotap_timestamp.accuracy = 22;
+}
+
+static void iwl_mac_hw_set_flags(struct iwl_mld *mld)
+{
+	struct ieee80211_hw *hw = mld->hw;
+
+	ieee80211_hw_set(hw, USES_RSS);
+	ieee80211_hw_set(hw, HANDLES_QUIET_CSA);
+	ieee80211_hw_set(hw, AP_LINK_PS);
+	ieee80211_hw_set(hw, SIGNAL_DBM);
+	ieee80211_hw_set(hw, SPECTRUM_MGMT);
+	ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS);
+	ieee80211_hw_set(hw, WANT_MONITOR_VIF);
+	ieee80211_hw_set(hw, SUPPORTS_PS);
+	ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
+	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
+	ieee80211_hw_set(hw, CONNECTION_MONITOR);
+	ieee80211_hw_set(hw, CHANCTX_STA_CSA);
+	ieee80211_hw_set(hw, SUPPORT_FAST_XMIT);
+	ieee80211_hw_set(hw, SUPPORTS_CLONED_SKBS);
+	ieee80211_hw_set(hw, NEEDS_UNIQUE_STA_ADDR);
+	ieee80211_hw_set(hw, SUPPORTS_VHT_EXT_NSS_BW);
+	ieee80211_hw_set(hw, BUFF_MMPDU_TXQ);
+	ieee80211_hw_set(hw, STA_MMPDU_TXQ);
+	ieee80211_hw_set(hw, TX_AMSDU);
+	ieee80211_hw_set(hw, TX_FRAG_LIST);
+	ieee80211_hw_set(hw, TX_AMPDU_SETUP_IN_HW);
+	ieee80211_hw_set(hw, HAS_RATE_CONTROL);
+	ieee80211_hw_set(hw, SUPPORTS_REORDERING_BUFFER);
+	ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS);
+	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
+	ieee80211_hw_set(hw, TDLS_WIDER_BW);
+}
+
+static void iwl_mac_hw_set_wiphy(struct iwl_mld *mld)
+{
+	struct ieee80211_hw *hw = mld->hw;
+	struct wiphy *wiphy = hw->wiphy;
+	const struct iwl_ucode_capabilities *ucode_capa = &mld->fw->ucode_capa;
+
+	snprintf(wiphy->fw_version,
+		 sizeof(wiphy->fw_version),
+		 "%.31s", mld->fw->fw_version);
+
+	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
+				 BIT(NL80211_IFTYPE_P2P_CLIENT) |
+				 BIT(NL80211_IFTYPE_AP) |
+				 BIT(NL80211_IFTYPE_P2P_GO) |
+				 BIT(NL80211_IFTYPE_P2P_DEVICE) |
+				 BIT(NL80211_IFTYPE_ADHOC);
+
+	wiphy->features |= NL80211_FEATURE_SCHED_SCAN_RANDOM_MAC_ADDR |
+			   NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR |
+			   NL80211_FEATURE_ND_RANDOM_MAC_ADDR |
+			   NL80211_FEATURE_HT_IBSS |
+			   NL80211_FEATURE_P2P_GO_CTWIN |
+			   NL80211_FEATURE_LOW_PRIORITY_SCAN |
+			   NL80211_FEATURE_P2P_GO_OPPPS |
+			   NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE |
+			   NL80211_FEATURE_SUPPORTS_WMM_ADMISSION |
+			   NL80211_FEATURE_TX_POWER_INSERTION |
+			   NL80211_FEATURE_DS_PARAM_SET_IE_IN_PROBES;
+
+	wiphy->flags |= WIPHY_FLAG_IBSS_RSN |
+			WIPHY_FLAG_AP_UAPSD |
+			WIPHY_FLAG_HAS_CHANNEL_SWITCH |
+			WIPHY_FLAG_SPLIT_SCAN_6GHZ |
+			WIPHY_FLAG_SUPPORTS_TDLS |
+			WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK;
+
+	/* For fips_enabled, don't support WiFi7 due to WPA3/MFP requirements */
+	if (mld->nvm_data->sku_cap_11be_enable &&
+	    !iwlwifi_mod_params.disable_11ax &&
+	    !iwlwifi_mod_params.disable_11be &&
+	    !fips_enabled)
+		wiphy->flags |= WIPHY_FLAG_SUPPORTS_MLO;
+
+	/* the firmware uses u8 for num of iterations, but 0xff is saved for
+	 * infinite loop, so the maximum number of iterations is actually 254.
+	 */
+	wiphy->max_sched_scan_plan_iterations = 254;
+	wiphy->max_sched_scan_ie_len = iwl_mld_scan_max_template_size();
+	wiphy->max_scan_ie_len = iwl_mld_scan_max_template_size();
+	wiphy->max_sched_scan_ssids = PROBE_OPTION_MAX;
+	wiphy->max_scan_ssids = PROBE_OPTION_MAX;
+	wiphy->max_sched_scan_plans = IWL_MAX_SCHED_SCAN_PLANS;
+	wiphy->max_sched_scan_reqs = 1;
+	wiphy->max_sched_scan_plan_interval = U16_MAX;
+	wiphy->max_match_sets = IWL_SCAN_MAX_PROFILES_V2;
+
+	wiphy->max_remain_on_channel_duration = 10000;
+
+	wiphy->hw_version = mld->trans->info.hw_id;
+
+	wiphy->hw_timestamp_max_peers = 1;
+
+	wiphy->iface_combinations = iwl_mld_iface_combinations;
+	wiphy->n_iface_combinations = ARRAY_SIZE(iwl_mld_iface_combinations);
+
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_DFS_CONCURRENT);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_LEGACY);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SCAN_START_TIME);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BSS_PARENT_TSF);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_ACCEPT_BCAST_PROBE_RESP);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_MAX_CHANNEL_TIME);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_OCE_PROBE_REQ_HIGH_TX_RATE);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_MU_MIMO_AIR_SNIFFER);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SPP_AMSDU_SUPPORT);
+
+	if (fw_has_capa(ucode_capa, IWL_UCODE_TLV_CAPA_PROTECTED_TWT))
+		wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_PROTECTED_TWT);
+
+	wiphy->iftype_ext_capab = NULL;
+	wiphy->num_iftype_ext_capab = 0;
+
+	if (!iwlwifi_mod_params.disable_11ax) {
+		wiphy->iftype_ext_capab = iftypes_ext_capa;
+		wiphy->num_iftype_ext_capab = ARRAY_SIZE(iftypes_ext_capa);
+
+		ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID);
+		ieee80211_hw_set(hw, SUPPORTS_ONLY_HE_MULTI_BSSID);
+	}
+
+	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM)
+		wiphy->flags |= WIPHY_FLAG_PS_ON_BY_DEFAULT;
+	else
+		wiphy->flags &= ~WIPHY_FLAG_PS_ON_BY_DEFAULT;
+}
+
+static void iwl_mac_hw_set_misc(struct iwl_mld *mld)
+{
+	struct ieee80211_hw *hw = mld->hw;
+
+	hw->queues = IEEE80211_NUM_ACS;
+
+	hw->netdev_features = NETIF_F_HIGHDMA | NETIF_F_SG;
+	hw->netdev_features |= mld->trans->mac_cfg->base->features;
+
+	hw->max_tx_fragments = mld->trans->info.max_skb_frags;
+	hw->max_listen_interval = IWL_MLD_CONN_LISTEN_INTERVAL;
+
+	hw->uapsd_max_sp_len = IEEE80211_WMM_IE_STA_QOSINFO_SP_ALL;
+	hw->uapsd_queues = IEEE80211_WMM_IE_STA_QOSINFO_AC_VO |
+			   IEEE80211_WMM_IE_STA_QOSINFO_AC_VI |
+			   IEEE80211_WMM_IE_STA_QOSINFO_AC_BK |
+			   IEEE80211_WMM_IE_STA_QOSINFO_AC_BE;
+
+	hw->chanctx_data_size = sizeof(struct iwl_mld_phy);
+	hw->vif_data_size = sizeof(struct iwl_mld_vif);
+	hw->sta_data_size = sizeof(struct iwl_mld_sta);
+	hw->txq_data_size = sizeof(struct iwl_mld_txq);
+
+	/* TODO: Remove this division when IEEE80211_MAX_AMPDU_BUF_EHT size
+	 * is supported.
+	 * Note: ensure that IWL_DEFAULT_QUEUE_SIZE_EHT is updated accordingly.
+	 */
+	hw->max_rx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_EHT / 2;
+}
+
+static int iwl_mld_hw_verify_preconditions(struct iwl_mld *mld)
+{
+	int ratecheck;
+
+	/* check for rates version 3 */
+	ratecheck =
+		(iwl_fw_lookup_cmd_ver(mld->fw, TX_CMD, 0) >= 11) +
+		(iwl_fw_lookup_notif_ver(mld->fw, DATA_PATH_GROUP,
+					 TLC_MNG_UPDATE_NOTIF, 0) >= 4) +
+		(iwl_fw_lookup_notif_ver(mld->fw, LEGACY_GROUP,
+					 REPLY_RX_MPDU_CMD, 0) >= 6) +
+		(iwl_fw_lookup_notif_ver(mld->fw, DATA_PATH_GROUP,
+					 RX_NO_DATA_NOTIF, 0) >= 4) +
+		(iwl_fw_lookup_notif_ver(mld->fw, LONG_GROUP, TX_CMD, 0) >= 9);
+
+	if (ratecheck != 0 && ratecheck != 5) {
+		IWL_ERR(mld, "Firmware has inconsistent rates\n");
+		return -EINVAL;
+	}
+
+	/* 11ax is expected to be enabled for all supported devices */
+	if (WARN_ON(!mld->nvm_data->sku_cap_11ax_enable))
+		return -EINVAL;
+
+	/* LAR is expected to be enabled for all supported devices */
+	if (WARN_ON(!mld->nvm_data->lar_enabled))
+		return -EINVAL;
+
+	/* All supported devices are currently using version 3 of the cmd.
+	 * Since version 3, IWL_SCAN_MAX_PROFILES_V2 shall be used where
+	 * necessary.
+	 */
+	if (WARN_ON(iwl_fw_lookup_cmd_ver(mld->fw,
+					  SCAN_OFFLOAD_UPDATE_PROFILES_CMD,
+					  IWL_FW_CMD_VER_UNKNOWN) != 3))
+		return -EINVAL;
+
+	return 0;
+}
+
+int iwl_mld_register_hw(struct iwl_mld *mld)
+{
+	/* verify once essential preconditions required for setting
+	 * the hw capabilities
+	 */
+	if (iwl_mld_hw_verify_preconditions(mld))
+		return -EINVAL;
+
+	iwl_mld_hw_set_addresses(mld);
+	iwl_mld_hw_set_channels(mld);
+	iwl_mld_hw_set_security(mld);
+	iwl_mld_hw_set_pm(mld);
+	iwl_mld_hw_set_antennas(mld);
+	iwl_mac_hw_set_radiotap(mld);
+	iwl_mac_hw_set_flags(mld);
+	iwl_mac_hw_set_wiphy(mld);
+	iwl_mac_hw_set_misc(mld);
+
+	SET_IEEE80211_DEV(mld->hw, mld->trans->dev);
+
+	return ieee80211_register_hw(mld->hw);
+}
+
+static void
+iwl_mld_mac80211_tx(struct ieee80211_hw *hw,
+		    struct ieee80211_tx_control *control, struct sk_buff *skb)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct ieee80211_sta *sta = control->sta;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	u32 link_id = u32_get_bits(info->control.flags,
+				   IEEE80211_TX_CTRL_MLO_LINK);
+
+	/* In AP mode, mgmt frames are sent on the bcast station,
+	 * so the FW can't translate the MLD addr to the link addr. Do it here
+	 */
+	if (ieee80211_is_mgmt(hdr->frame_control) && sta &&
+	    link_id != IEEE80211_LINK_UNSPECIFIED &&
+	    !ieee80211_is_probe_resp(hdr->frame_control)) {
+		/* translate MLD addresses to LINK addresses */
+		struct ieee80211_link_sta *link_sta =
+			rcu_dereference(sta->link[link_id]);
+		struct ieee80211_bss_conf *link_conf =
+			rcu_dereference(info->control.vif->link_conf[link_id]);
+		struct ieee80211_mgmt *mgmt;
+
+		if (WARN_ON(!link_sta || !link_conf)) {
+			ieee80211_free_txskb(hw, skb);
+			return;
+		}
+
+		mgmt = (void *)hdr;
+		memcpy(mgmt->da, link_sta->addr, ETH_ALEN);
+		memcpy(mgmt->sa, link_conf->addr, ETH_ALEN);
+		memcpy(mgmt->bssid, link_conf->bssid, ETH_ALEN);
+	}
+
+	iwl_mld_tx_skb(mld, skb, NULL);
+}
+
+static void
+iwl_mld_restart_cleanup(struct iwl_mld *mld)
+{
+	iwl_cleanup_mld(mld);
+
+	ieee80211_iterate_interfaces(mld->hw, IEEE80211_IFACE_ITER_ACTIVE,
+				     iwl_mld_cleanup_vif, NULL);
+
+	ieee80211_iterate_stations_atomic(mld->hw,
+					  iwl_mld_cleanup_sta, NULL);
+
+	iwl_mld_ftm_restart_cleanup(mld);
+}
+
+static
+int iwl_mld_mac80211_start(struct ieee80211_hw *hw)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	bool in_d3 = false;
+	int ret = 0;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+#ifdef CONFIG_PM_SLEEP
+	/* Unless the host goes into hibernate the FW always stays on and
+	 * the d3_resume flow is used. When wowlan is configured, mac80211
+	 * would call it's resume callback and the wowlan_resume flow
+	 * would be used.
+	 */
+
+	in_d3 = mld->fw_status.in_d3;
+	if (in_d3) {
+		/* mac80211 already cleaned up the state, no need for cleanup */
+		ret = iwl_mld_no_wowlan_resume(mld);
+		if (ret) {
+			iwl_mld_stop_fw(mld);
+			/* We're not really restarting in the sense of
+			 * in_hw_restart even if we got an error during
+			 * this. We'll just start again below and have
+			 * nothing to recover, mac80211 will do anyway.
+			 */
+			mld->fw_status.in_hw_restart = false;
+		}
+	}
+#endif /* CONFIG_PM_SLEEP */
+
+	if (mld->fw_status.in_hw_restart) {
+		iwl_mld_stop_fw(mld);
+		iwl_mld_restart_cleanup(mld);
+	}
+
+	if (!in_d3 || ret) {
+		ret = iwl_mld_start_fw(mld);
+		if (ret)
+			goto error;
+	}
+
+	mld->scan.last_start_time_jiffies = jiffies;
+
+	iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_POINT_POST_INIT,
+			       NULL);
+	iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_POINT_PERIODIC,
+			       NULL);
+
+	return 0;
+
+error:
+	/* If we failed to restart the hw, there is nothing useful
+	 * we can do but indicate we are no longer in restart.
+	 */
+	mld->fw_status.in_hw_restart = false;
+
+	return ret;
+}
+
+static
+void iwl_mld_mac80211_stop(struct ieee80211_hw *hw, bool suspend)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	wiphy_work_cancel(mld->wiphy, &mld->add_txqs_wk);
+
+	/* if the suspend flow fails the fw is in error. Stop it here, and it
+	 * will be started upon wakeup
+	 */
+	if (!suspend ||
+	    (IS_ENABLED(CONFIG_PM_SLEEP) && iwl_mld_no_wowlan_suspend(mld)))
+		iwl_mld_stop_fw(mld);
+
+	/* Clear in_hw_restart flag when stopping the hw, as mac80211 won't
+	 * execute the restart.
+	 */
+	mld->fw_status.in_hw_restart = false;
+
+	/* We shouldn't have any UIDs still set. Loop over all the UIDs to
+	 * make sure there's nothing left there and warn if any is found.
+	 */
+	for (int i = 0; i < ARRAY_SIZE(mld->scan.uid_status); i++)
+		if (WARN_ONCE(mld->scan.uid_status[i],
+			      "UMAC scan UID %d status was not cleaned (0x%x 0x%x)\n",
+			      i, mld->scan.uid_status[i], mld->scan.status))
+			mld->scan.uid_status[i] = 0;
+}
+
+static
+int iwl_mld_mac80211_config(struct ieee80211_hw *hw, int radio_idx,
+			    u32 changed)
+{
+	return 0;
+}
+
+static
+int iwl_mld_mac80211_add_interface(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* Construct mld_vif, add it to fw, and map its ID to ieee80211_vif */
+	ret = iwl_mld_add_vif(mld, vif);
+	if (ret)
+		return ret;
+
+	/*
+	 * Add the default link, but not if this is an MLD vif as that implies
+	 * the HW is restarting and it will be configured by change_vif_links.
+	 */
+	if (!ieee80211_vif_is_mld(vif))
+		ret = iwl_mld_add_link(mld, &vif->bss_conf);
+	if (ret)
+		goto err;
+
+	if (vif->type == NL80211_IFTYPE_STATION) {
+		vif->driver_flags |= IEEE80211_VIF_REMOVE_AP_AFTER_DISASSOC;
+		if (!vif->p2p)
+			vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER |
+					     IEEE80211_VIF_SUPPORTS_CQM_RSSI;
+	}
+
+	if (vif->p2p || iwl_fw_lookup_cmd_ver(mld->fw, PHY_CONTEXT_CMD, 0) < 5)
+		vif->driver_flags |= IEEE80211_VIF_IGNORE_OFDMA_WIDER_BW;
+
+	/*
+	 * For an MLD vif (in restart) we may not have a link; delay the call
+	 * the initial change_vif_links.
+	 */
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    !ieee80211_vif_is_mld(vif))
+		iwl_mld_update_mac_power(mld, vif, false);
+
+	if (vif->type == NL80211_IFTYPE_MONITOR) {
+		mld->monitor.on = true;
+		ieee80211_hw_set(mld->hw, RX_INCLUDES_FCS);
+	}
+
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
+		mld->p2p_device_vif = vif;
+
+	return 0;
+
+err:
+	iwl_mld_rm_vif(mld, vif);
+	return ret;
+}
+
+static
+void iwl_mld_mac80211_remove_interface(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_STATION)
+		vif->driver_flags &= ~(IEEE80211_VIF_BEACON_FILTER |
+				       IEEE80211_VIF_SUPPORTS_CQM_RSSI);
+
+	if (vif->type == NL80211_IFTYPE_MONITOR) {
+		__clear_bit(IEEE80211_HW_RX_INCLUDES_FCS, mld->hw->flags);
+		mld->monitor.on = false;
+	}
+
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
+		mld->p2p_device_vif = NULL;
+
+	iwl_mld_remove_link(mld, &vif->bss_conf);
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	debugfs_remove(iwl_mld_vif_from_mac80211(vif)->dbgfs_slink);
+	iwl_mld_vif_from_mac80211(vif)->dbgfs_slink = NULL;
+#endif
+
+	iwl_mld_rm_vif(mld, vif);
+}
+
+struct iwl_mld_mc_iter_data {
+	struct iwl_mld *mld;
+	int port_id;
+};
+
+static void iwl_mld_mc_iface_iterator(void *data, u8 *mac,
+				      struct ieee80211_vif *vif)
+{
+	struct iwl_mld_mc_iter_data *mc_data = data;
+	struct iwl_mld *mld = mc_data->mld;
+	struct iwl_mcast_filter_cmd *cmd = mld->mcast_filter_cmd;
+	struct iwl_host_cmd hcmd = {
+		.id = MCAST_FILTER_CMD,
+		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
+	};
+	int ret, len;
+
+	/* If we don't have free ports, mcast frames will be dropped */
+	if (WARN_ON_ONCE(mc_data->port_id >= MAX_PORT_ID_NUM))
+		return;
+
+	if (vif->type != NL80211_IFTYPE_STATION || !vif->cfg.assoc)
+		return;
+
+	cmd->port_id = mc_data->port_id++;
+	ether_addr_copy(cmd->bssid, vif->bss_conf.bssid);
+	len = roundup(sizeof(*cmd) + cmd->count * ETH_ALEN, 4);
+
+	hcmd.len[0] = len;
+	hcmd.data[0] = cmd;
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+	if (ret)
+		IWL_ERR(mld, "mcast filter cmd error. ret=%d\n", ret);
+}
+
+void iwl_mld_recalc_multicast_filter(struct iwl_mld *mld)
+{
+	struct iwl_mld_mc_iter_data iter_data = {
+		.mld = mld,
+	};
+
+	if (WARN_ON_ONCE(!mld->mcast_filter_cmd))
+		return;
+
+	ieee80211_iterate_active_interfaces(mld->hw,
+					    IEEE80211_IFACE_ITER_NORMAL,
+					    iwl_mld_mc_iface_iterator,
+					    &iter_data);
+}
+
+static u64
+iwl_mld_mac80211_prepare_multicast(struct ieee80211_hw *hw,
+				   struct netdev_hw_addr_list *mc_list)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mcast_filter_cmd *cmd;
+	struct netdev_hw_addr *addr;
+	int addr_count = netdev_hw_addr_list_count(mc_list);
+	bool pass_all = addr_count > MAX_MCAST_FILTERING_ADDRESSES;
+	int len;
+
+	if (pass_all)
+		addr_count = 0;
+
+	/* len must be a multiple of 4 */
+	len = roundup(sizeof(*cmd) + addr_count * ETH_ALEN, 4);
+	cmd = kzalloc(len, GFP_ATOMIC);
+	if (!cmd)
+		return 0;
+
+	if (pass_all) {
+		cmd->pass_all = 1;
+		goto out;
+	}
+
+	netdev_hw_addr_list_for_each(addr, mc_list) {
+		IWL_DEBUG_MAC80211(mld, "mcast addr (%d): %pM\n",
+				   cmd->count, addr->addr);
+		ether_addr_copy(&cmd->addr_list[cmd->count * ETH_ALEN],
+				addr->addr);
+		cmd->count++;
+	}
+
+out:
+	return (u64)(unsigned long)cmd;
+}
+
+static
+void iwl_mld_mac80211_configure_filter(struct ieee80211_hw *hw,
+				       unsigned int changed_flags,
+				       unsigned int *total_flags,
+				       u64 multicast)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mcast_filter_cmd *cmd = (void *)(unsigned long)multicast;
+
+	/* Replace previous configuration */
+	kfree(mld->mcast_filter_cmd);
+	mld->mcast_filter_cmd = cmd;
+
+	if (!cmd)
+		goto out;
+
+	if (changed_flags & FIF_ALLMULTI)
+		cmd->pass_all = !!(*total_flags & FIF_ALLMULTI);
+
+	if (cmd->pass_all)
+		cmd->count = 0;
+
+	iwl_mld_recalc_multicast_filter(mld);
+out:
+	*total_flags = 0;
+}
+
+static
+void iwl_mld_mac80211_wake_tx_queue(struct ieee80211_hw *hw,
+				    struct ieee80211_txq *txq)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_txq *mld_txq = iwl_mld_txq_from_mac80211(txq);
+
+	if (likely(mld_txq->status.allocated) || !txq->sta) {
+		iwl_mld_tx_from_txq(mld, txq);
+		return;
+	}
+
+	/* We don't support TSPEC tids. %IEEE80211_NUM_TIDS is for mgmt */
+	if (txq->tid != IEEE80211_NUM_TIDS && txq->tid >= IWL_MAX_TID_COUNT) {
+		IWL_DEBUG_MAC80211(mld, "TID %d is not supported\n", txq->tid);
+		return;
+	}
+
+	/* The worker will handle any packets we leave on the txq now */
+
+	spin_lock_bh(&mld->add_txqs_lock);
+	/* The list is being deleted only after the queue is fully allocated. */
+	if (list_empty(&mld_txq->list) &&
+	    /* recheck under lock, otherwise it can be added twice */
+	    !mld_txq->status.allocated) {
+		list_add_tail(&mld_txq->list, &mld->txqs_to_add);
+		wiphy_work_queue(mld->wiphy, &mld->add_txqs_wk);
+	}
+	spin_unlock_bh(&mld->add_txqs_lock);
+}
+
+static void iwl_mld_teardown_tdls_peers(struct iwl_mld *mld)
+{
+	lockdep_assert_wiphy(mld->wiphy);
+
+	for (int i = 0; i < mld->fw->ucode_capa.num_stations; i++) {
+		struct ieee80211_link_sta *link_sta;
+		struct iwl_mld_sta *mld_sta;
+
+		link_sta = wiphy_dereference(mld->wiphy,
+					     mld->fw_id_to_link_sta[i]);
+		if (IS_ERR_OR_NULL(link_sta))
+			continue;
+
+		if (!link_sta->sta->tdls)
+			continue;
+
+		mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+
+		ieee80211_tdls_oper_request(mld_sta->vif, link_sta->addr,
+					    NL80211_TDLS_TEARDOWN,
+					    WLAN_REASON_TDLS_TEARDOWN_UNSPECIFIED,
+					    GFP_KERNEL);
+	}
+}
+
+static
+int iwl_mld_add_chanctx(struct ieee80211_hw *hw,
+			struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(ctx);
+	int fw_id = iwl_mld_allocate_fw_phy_id(mld);
+	int ret;
+
+	if (fw_id < 0)
+		return fw_id;
+
+	phy->mld = mld;
+	phy->fw_id = fw_id;
+	phy->chandef = *iwl_mld_get_chandef_from_chanctx(mld, ctx);
+
+	ret = iwl_mld_phy_fw_action(mld, ctx, FW_CTXT_ACTION_ADD);
+	if (ret) {
+		mld->used_phy_ids &= ~BIT(phy->fw_id);
+		return ret;
+	}
+
+	if (hweight8(mld->used_phy_ids) > 1)
+		iwl_mld_teardown_tdls_peers(mld);
+
+	return 0;
+}
+
+static
+void iwl_mld_remove_chanctx(struct ieee80211_hw *hw,
+			    struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(ctx);
+
+	iwl_mld_phy_fw_action(mld, ctx, FW_CTXT_ACTION_REMOVE);
+	mld->used_phy_ids &= ~BIT(phy->fw_id);
+}
+
+static
+void iwl_mld_change_chanctx(struct ieee80211_hw *hw,
+			    struct ieee80211_chanctx_conf *ctx, u32 changed)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(ctx);
+	struct cfg80211_chan_def *chandef =
+		iwl_mld_get_chandef_from_chanctx(mld, ctx);
+
+	/* We don't care about these */
+	if (!(changed & ~(IEEE80211_CHANCTX_CHANGE_RX_CHAINS |
+			  IEEE80211_CHANCTX_CHANGE_RADAR |
+			  IEEE80211_CHANCTX_CHANGE_CHANNEL)))
+		return;
+
+	/* Check if a FW update is required */
+
+	if (changed & IEEE80211_CHANCTX_CHANGE_AP)
+		goto update;
+
+	if (chandef->chan == phy->chandef.chan &&
+	    chandef->center_freq1 == phy->chandef.center_freq1 &&
+	    chandef->punctured == phy->chandef.punctured) {
+		/* Check if we are toggling between HT and non-HT, no-op */
+		if (phy->chandef.width == chandef->width ||
+		    (phy->chandef.width <= NL80211_CHAN_WIDTH_20 &&
+		     chandef->width <= NL80211_CHAN_WIDTH_20))
+			return;
+	}
+update:
+
+	iwl_mld_update_phy_chandef(mld, ctx);
+}
+
+static u8
+iwl_mld_chandef_get_primary_80(struct cfg80211_chan_def *chandef)
+{
+	int data_start;
+	int control_start;
+	int bw;
+
+	if (chandef->width == NL80211_CHAN_WIDTH_320)
+		bw = 320;
+	else if (chandef->width == NL80211_CHAN_WIDTH_160)
+		bw = 160;
+	else
+		return 0;
+
+	/* data is bw wide so the start is half the width */
+	data_start = chandef->center_freq1 - bw / 2;
+	/* control is 20Mhz width */
+	control_start = chandef->chan->center_freq - 10;
+
+	return (control_start - data_start) / 80;
+}
+
+static bool iwl_mld_can_activate_link(struct iwl_mld *mld,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_link_sta *link_sta;
+
+	/* In association, we activate the assoc link before adding the STA. */
+	if (!mld_vif->ap_sta || !vif->cfg.assoc)
+		return true;
+
+	mld_sta = iwl_mld_sta_from_mac80211(mld_vif->ap_sta);
+
+	/* When switching links, we need to wait with the activation until the
+	 * STA was added to the FW. It'll be activated in
+	 * iwl_mld_update_link_stas
+	 */
+	link_sta = wiphy_dereference(mld->wiphy, mld_sta->link[link->link_id]);
+
+	/* In restart we can have a link_sta that doesn't exist in FW yet */
+	return link_sta && link_sta->in_fw;
+}
+
+static
+int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_bss_conf *link,
+			       struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	unsigned int n_active = iwl_mld_count_active_links(mld, vif);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld_link))
+		return -EINVAL;
+
+	/* if the assigned one was not counted yet, count it now */
+	if (!rcu_access_pointer(mld_link->chan_ctx)) {
+		n_active++;
+
+		/* Track addition of non-BSS link */
+		if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION) {
+			ret = iwl_mld_emlsr_check_non_bss_block(mld, 1);
+			if (ret)
+				return ret;
+		}
+	}
+
+	/* for AP, mac parameters such as HE support are updated at this stage. */
+	if (vif->type == NL80211_IFTYPE_AP) {
+		ret = iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY);
+
+		if (ret) {
+			IWL_ERR(mld, "failed to update MAC %pM\n", vif->addr);
+			return -EINVAL;
+		}
+	}
+
+	rcu_assign_pointer(mld_link->chan_ctx, ctx);
+
+	if (n_active > 1) {
+		struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+		/* Indicate to mac80211 that EML is enabled */
+		vif->driver_flags |= IEEE80211_VIF_EML_ACTIVE;
+		mld_vif->emlsr.last_entry_ts = jiffies;
+
+		if (vif->active_links & BIT(mld_vif->emlsr.selected_links))
+			mld_vif->emlsr.primary = mld_vif->emlsr.selected_primary;
+		else
+			mld_vif->emlsr.primary = __ffs(vif->active_links);
+
+		iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_ESR_LINK_UP,
+				       NULL);
+	}
+
+	/* First send the link command with the phy context ID.
+	 * Now that we have the phy, we know the band so also the rates
+	 */
+	ret = iwl_mld_change_link_in_fw(mld, link,
+					LINK_CONTEXT_MODIFY_RATES_INFO);
+	if (ret)
+		goto err;
+
+	/* TODO: Initialize rate control for the AP station, since we might be
+	 * doing a link switch here - we cannot initialize it before since
+	 * this needs the phy context assigned (and in FW?), and we cannot
+	 * do it later because it needs to be initialized as soon as we're
+	 * able to TX on the link, i.e. when active. (task=link-switch)
+	 */
+
+	/* Now activate the link */
+	if (iwl_mld_can_activate_link(mld, vif, link)) {
+		ret = iwl_mld_activate_link(mld, link);
+		if (ret)
+			goto err;
+	}
+
+	if (vif->type == NL80211_IFTYPE_STATION)
+		iwl_mld_send_ap_tx_power_constraint_cmd(mld, vif, link);
+
+	if (vif->type == NL80211_IFTYPE_MONITOR) {
+		ret = iwl_mld_add_mon_sta(mld, vif, link);
+		if (ret)
+			goto deactivate_link;
+
+		mld->monitor.p80 =
+			iwl_mld_chandef_get_primary_80(&vif->bss_conf.chanreq.oper);
+	}
+
+	return 0;
+
+deactivate_link:
+	if (mld_link->active)
+		iwl_mld_deactivate_link(mld, link);
+err:
+	RCU_INIT_POINTER(mld_link->chan_ctx, NULL);
+	return ret;
+}
+
+static
+void iwl_mld_unassign_vif_chanctx(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_bss_conf *link,
+				  struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	unsigned int n_active = iwl_mld_count_active_links(mld, vif);
+
+	if (WARN_ON(!mld_link))
+		return;
+
+	/* Track removal of non-BSS link */
+	if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION)
+		iwl_mld_emlsr_check_non_bss_block(mld, -1);
+
+	iwl_mld_deactivate_link(mld, link);
+
+	if (vif->type == NL80211_IFTYPE_MONITOR)
+		iwl_mld_remove_mon_sta(mld, vif, link);
+
+	if (n_active > 1) {
+		/* Indicate to mac80211 that EML is disabled */
+		vif->driver_flags &= ~IEEE80211_VIF_EML_ACTIVE;
+
+		iwl_dbg_tlv_time_point(&mld->fwrt,
+				       IWL_FW_INI_TIME_ESR_LINK_DOWN,
+				       NULL);
+	}
+
+	RCU_INIT_POINTER(mld_link->chan_ctx, NULL);
+
+	/* in the non-MLO case, remove/re-add the link to clean up FW state.
+	 * In MLO, it'll be done in drv_change_vif_link
+	 */
+	if (!ieee80211_vif_is_mld(vif) && !mld_vif->ap_sta &&
+	    !WARN_ON_ONCE(vif->cfg.assoc) &&
+	    vif->type != NL80211_IFTYPE_AP && !mld->fw_status.in_hw_restart) {
+		iwl_mld_remove_link(mld, link);
+		iwl_mld_add_link(mld, link);
+	}
+}
+
+static
+int iwl_mld_mac80211_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				       u32 value)
+{
+	return 0;
+}
+
+static void
+iwl_mld_link_info_changed_ap_ibss(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_bss_conf *link,
+				  u64 changes)
+{
+	u32 link_changes = 0;
+
+	if (changes & BSS_CHANGED_ERP_SLOT)
+		link_changes |= LINK_CONTEXT_MODIFY_RATES_INFO;
+
+	if (changes & (BSS_CHANGED_ERP_CTS_PROT | BSS_CHANGED_HT))
+		link_changes |= LINK_CONTEXT_MODIFY_PROTECT_FLAGS;
+
+	if (changes & (BSS_CHANGED_QOS | BSS_CHANGED_BANDWIDTH))
+		link_changes |= LINK_CONTEXT_MODIFY_QOS_PARAMS;
+
+	if (changes & BSS_CHANGED_HE_BSS_COLOR)
+		link_changes |= LINK_CONTEXT_MODIFY_HE_PARAMS;
+
+	if (link_changes)
+		iwl_mld_change_link_in_fw(mld, link, link_changes);
+
+	if (changes & BSS_CHANGED_BEACON)
+		iwl_mld_update_beacon_template(mld, vif, link);
+}
+
+static
+u32 iwl_mld_link_changed_mapping(struct iwl_mld *mld,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_bss_conf *link_conf,
+				 u64 changes)
+{
+	u32 link_changes = 0;
+	bool has_he, has_eht;
+
+	if (changes & BSS_CHANGED_QOS && vif->cfg.assoc && link_conf->qos)
+		link_changes |= LINK_CONTEXT_MODIFY_QOS_PARAMS;
+
+	if (changes & (BSS_CHANGED_ERP_PREAMBLE | BSS_CHANGED_BASIC_RATES |
+		       BSS_CHANGED_ERP_SLOT))
+		link_changes |= LINK_CONTEXT_MODIFY_RATES_INFO;
+
+	if (changes & (BSS_CHANGED_HT | BSS_CHANGED_ERP_CTS_PROT))
+		link_changes |= LINK_CONTEXT_MODIFY_PROTECT_FLAGS;
+
+	/* TODO: task=MLO check mac80211's HE flags and if command is needed
+	 * every time there's a link change. Currently used flags are
+	 * BSS_CHANGED_HE_OBSS_PD and BSS_CHANGED_HE_BSS_COLOR.
+	 */
+	has_he = link_conf->he_support && !iwlwifi_mod_params.disable_11ax;
+	has_eht = link_conf->eht_support && !iwlwifi_mod_params.disable_11be;
+
+	if (vif->cfg.assoc && (has_he || has_eht)) {
+		IWL_DEBUG_MAC80211(mld, "Associated in HE mode\n");
+		link_changes |= LINK_CONTEXT_MODIFY_HE_PARAMS;
+	}
+
+	return link_changes;
+}
+
+static void
+iwl_mld_mac80211_link_info_changed_sta(struct iwl_mld *mld,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_bss_conf *link_conf,
+				       u64 changes)
+{
+	u32 link_changes = iwl_mld_link_changed_mapping(mld, vif, link_conf,
+							changes);
+
+	if (link_changes)
+		iwl_mld_change_link_in_fw(mld, link_conf, link_changes);
+
+	if (changes & BSS_CHANGED_TPE)
+		iwl_mld_send_ap_tx_power_constraint_cmd(mld, vif, link_conf);
+
+	if (changes & BSS_CHANGED_BEACON_INFO)
+		iwl_mld_update_mac_power(mld, vif, false);
+
+	/* The firmware will wait quite a while after association before it
+	 * starts filtering the beacons. We can safely enable beacon filtering
+	 * upon CQM configuration, even if we didn't get a beacon yet.
+	 */
+	if (changes & (BSS_CHANGED_CQM | BSS_CHANGED_BEACON_INFO))
+		iwl_mld_enable_beacon_filter(mld, link_conf, false);
+
+	if (changes & BSS_CHANGED_BANDWIDTH)
+		iwl_mld_retry_emlsr(mld, vif);
+}
+
+static int iwl_mld_update_mu_groups(struct iwl_mld *mld,
+				    struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mu_group_mgmt_cmd cmd = {};
+
+	BUILD_BUG_ON(sizeof(cmd.membership_status) !=
+		     sizeof(link_conf->mu_group.membership));
+	BUILD_BUG_ON(sizeof(cmd.user_position) !=
+		     sizeof(link_conf->mu_group.position));
+
+	memcpy(cmd.membership_status, link_conf->mu_group.membership,
+	       WLAN_MEMBERSHIP_LEN);
+	memcpy(cmd.user_position, link_conf->mu_group.position,
+	       WLAN_USER_POSITION_LEN);
+
+	return iwl_mld_send_cmd_pdu(mld,
+				    WIDE_ID(DATA_PATH_GROUP,
+					    UPDATE_MU_GROUPS_CMD),
+				    &cmd);
+}
+
+static void
+iwl_mld_mac80211_link_info_changed(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_bss_conf *link_conf,
+				   u64 changes)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		iwl_mld_mac80211_link_info_changed_sta(mld, vif, link_conf,
+						       changes);
+		break;
+	case NL80211_IFTYPE_AP:
+	case NL80211_IFTYPE_ADHOC:
+		iwl_mld_link_info_changed_ap_ibss(mld, vif, link_conf,
+						  changes);
+		break;
+	case NL80211_IFTYPE_MONITOR:
+		/* The firmware tracks this on its own in STATION mode, but
+		 * obviously not in sniffer mode.
+		 */
+		if (changes & BSS_CHANGED_MU_GROUPS)
+			iwl_mld_update_mu_groups(mld, link_conf);
+		break;
+	default:
+		/* shouldn't happen */
+		WARN_ON_ONCE(1);
+	}
+
+	/* We now know our BSSID, we can configure the MAC context with
+	 * eht_support if needed.
+	 */
+	if (changes & BSS_CHANGED_BSSID)
+		iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY);
+
+	if (changes & BSS_CHANGED_TXPOWER)
+		iwl_mld_set_tx_power(mld, link_conf, link_conf->txpower);
+}
+
+static void
+iwl_mld_smps_workaround(struct iwl_mld *mld, struct ieee80211_vif *vif, bool enable)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	bool workaround_required =
+		iwl_fw_lookup_cmd_ver(mld->fw, MAC_PM_POWER_TABLE, 0) < 2;
+
+	if (!workaround_required)
+		return;
+
+	/* Send the device-level power commands since the
+	 * firmware checks the POWER_TABLE_CMD's POWER_SAVE_EN bit to
+	 * determine SMPS mode.
+	 */
+	if (mld_vif->ps_disabled == !enable)
+		return;
+
+	mld_vif->ps_disabled = !enable;
+
+	iwl_mld_update_device_power(mld, false);
+}
+
+static
+void iwl_mld_mac80211_vif_cfg_changed(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      u64 changes)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	if (changes & BSS_CHANGED_ASSOC) {
+		ret = iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY);
+		if (ret)
+			IWL_ERR(mld, "failed to update context\n");
+
+		if (vif->cfg.assoc) {
+			/* Clear statistics to get clean beacon counter, and
+			 * ask for periodic statistics, as they are needed for
+			 * link selection and RX OMI decisions.
+			 */
+			iwl_mld_clear_stats_in_fw(mld);
+			iwl_mld_request_periodic_fw_stats(mld, true);
+
+			iwl_mld_set_vif_associated(mld, vif);
+		} else {
+			iwl_mld_request_periodic_fw_stats(mld, false);
+		}
+	}
+
+	if (changes & BSS_CHANGED_PS) {
+		iwl_mld_smps_workaround(mld, vif, vif->cfg.ps);
+		iwl_mld_update_mac_power(mld, vif, false);
+	}
+
+	/* TODO: task=MLO BSS_CHANGED_MLD_VALID_LINKS/CHANGED_MLD_TTLM */
+}
+
+static int
+iwl_mld_mac80211_hw_scan(struct ieee80211_hw *hw,
+			 struct ieee80211_vif *vif,
+			 struct ieee80211_scan_request *hw_req)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	if (WARN_ON(!hw_req->req.n_channels ||
+		    hw_req->req.n_channels >
+		    mld->fw->ucode_capa.n_scan_channels))
+		return -EINVAL;
+
+	ret = iwl_mld_regular_scan_start(mld, vif, &hw_req->req, &hw_req->ies);
+	if (!ret) {
+		/* We will be busy with scanning, so the counters may not reflect the
+		 * reality. Stop checking the counters until the scan ends
+		 */
+		iwl_mld_start_ignoring_tpt_updates(mld);
+	}
+
+	return ret;
+}
+
+static void
+iwl_mld_mac80211_cancel_hw_scan(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	/* Due to a race condition, it's possible that mac80211 asks
+	 * us to stop a hw_scan when it's already stopped. This can
+	 * happen, for instance, if we stopped the scan ourselves,
+	 * called ieee80211_scan_completed() and the userspace called
+	 * cancel scan before ieee80211_scan_work() could run.
+	 * To handle that, simply return if the scan is not running.
+	 */
+	if (mld->scan.status & IWL_MLD_SCAN_REGULAR) {
+		iwl_mld_scan_stop(mld, IWL_MLD_SCAN_REGULAR, true);
+		/* Scan is over, we can check again the tpt counters */
+		iwl_mld_stop_ignoring_tpt_updates(mld);
+	}
+}
+
+static int
+iwl_mld_mac80211_sched_scan_start(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct cfg80211_sched_scan_request *req,
+				  struct ieee80211_scan_ies *ies)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	return iwl_mld_sched_scan_start(mld, vif, req, ies, IWL_MLD_SCAN_SCHED);
+}
+
+static int
+iwl_mld_mac80211_sched_scan_stop(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	/* Due to a race condition, it's possible that mac80211 asks
+	 * us to stop a sched_scan when it's already stopped. This
+	 * can happen, for instance, if we stopped the scan ourselves,
+	 * called ieee80211_sched_scan_stopped() and the userspace called
+	 * stop sched scan before ieee80211_sched_scan_stopped_work()
+	 * could run. To handle this, simply return if the scan is
+	 * not running.
+	 */
+	if (!(mld->scan.status & IWL_MLD_SCAN_SCHED))
+		return 0;
+
+	return iwl_mld_scan_stop(mld, IWL_MLD_SCAN_SCHED, false);
+}
+
+static void
+iwl_mld_mac80211_reconfig_complete(struct ieee80211_hw *hw,
+				   enum ieee80211_reconfig_type reconfig_type)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	switch (reconfig_type) {
+	case IEEE80211_RECONFIG_TYPE_RESTART:
+		mld->fw_status.in_hw_restart = false;
+		iwl_mld_send_recovery_cmd(mld, ERROR_RECOVERY_END_OF_RECOVERY);
+		iwl_trans_finish_sw_reset(mld->trans);
+		/* no need to lock, adding in parallel would schedule too */
+		if (!list_empty(&mld->txqs_to_add))
+			wiphy_work_queue(mld->wiphy, &mld->add_txqs_wk);
+
+		IWL_INFO(mld, "restart completed\n");
+		break;
+	case IEEE80211_RECONFIG_TYPE_SUSPEND:
+		break;
+	}
+}
+
+static
+void iwl_mld_mac80211_mgd_prepare_tx(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_prep_tx_info *info)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	u32 duration = IWL_MLD_SESSION_PROTECTION_ASSOC_TIME_MS;
+
+	/* After a successful association the connection is established
+	 * and we can rely on the quota to send the disassociation frame.
+	 */
+	if (info->was_assoc)
+		return;
+
+	if (info->duration > duration)
+		duration = info->duration;
+
+	iwl_mld_schedule_session_protection(mld, vif, duration,
+					    IWL_MLD_SESSION_PROTECTION_MIN_TIME_MS,
+					    info->link_id);
+}
+
+static
+void iwl_mld_mac_mgd_complete_tx(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_prep_tx_info *info)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	/* Successful authentication is the only case that requires to let
+	 * the session protection go. We'll need it for the upcoming
+	 * association. For all the other cases, we need to cancel the session
+	 * protection.
+	 * After successful association the connection is established and
+	 * further mgd tx can rely on the quota.
+	 */
+	if (info->success && info->subtype == IEEE80211_STYPE_AUTH)
+		return;
+
+	/* The firmware will be on medium after we configure the vif as
+	 * associated. Removing the session protection allows the firmware
+	 * to stop being on medium. In order to ensure the continuity of our
+	 * presence on medium, we need first to configure the vif as associated
+	 * and only then, remove the session protection.
+	 * Currently, mac80211 calls vif_cfg_changed() first and then,
+	 * drv_mgd_complete_tx(). Ensure that this assumption stays true by
+	 * a warning.
+	 */
+	WARN_ON(info->success &&
+		(info->subtype == IEEE80211_STYPE_ASSOC_REQ ||
+		 info->subtype == IEEE80211_STYPE_REASSOC_REQ) &&
+		!vif->cfg.assoc);
+
+	iwl_mld_cancel_session_protection(mld, vif, info->link_id);
+}
+
+static int
+iwl_mld_mac80211_conf_tx(struct ieee80211_hw *hw,
+			 struct ieee80211_vif *vif,
+			 unsigned int link_id, u16 ac,
+			 const struct ieee80211_tx_queue_params *params)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *link;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	link = iwl_mld_link_dereference_check(mld_vif, link_id);
+	if (!link)
+		return -EINVAL;
+
+	link->queue_params[ac] = *params;
+
+	/* No need to update right away, we'll get BSS_CHANGED_QOS
+	 * The exception is P2P_DEVICE interface which needs immediate update.
+	 */
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
+		iwl_mld_change_link_in_fw(mld, &vif->bss_conf,
+					  LINK_CONTEXT_MODIFY_QOS_PARAMS);
+
+	return 0;
+}
+
+static void iwl_mld_set_uapsd(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	vif->driver_flags &= ~IEEE80211_VIF_SUPPORTS_UAPSD;
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	if (vif->p2p &&
+	    !(iwlwifi_mod_params.uapsd_disable & IWL_DISABLE_UAPSD_P2P_CLIENT))
+		vif->driver_flags |= IEEE80211_VIF_SUPPORTS_UAPSD;
+
+	if (!vif->p2p &&
+	    !(iwlwifi_mod_params.uapsd_disable & IWL_DISABLE_UAPSD_BSS))
+		vif->driver_flags |= IEEE80211_VIF_SUPPORTS_UAPSD;
+}
+
+int iwl_mld_tdls_sta_count(struct iwl_mld *mld)
+{
+	int count = 0;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	for (int i = 0; i < mld->fw->ucode_capa.num_stations; i++) {
+		struct ieee80211_link_sta *link_sta;
+
+		link_sta = wiphy_dereference(mld->wiphy,
+					     mld->fw_id_to_link_sta[i]);
+		if (IS_ERR_OR_NULL(link_sta))
+			continue;
+
+		if (!link_sta->sta->tdls)
+			continue;
+
+		count++;
+	}
+
+	return count;
+}
+
+static void iwl_mld_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy,
+						    struct cfg80211_bss *bss,
+						    void *_data)
+{
+	bool *tolerated = _data;
+	const struct cfg80211_bss_ies *ies;
+	const struct element *elem;
+
+	rcu_read_lock();
+	ies = rcu_dereference(bss->ies);
+	elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data,
+				  ies->len);
+
+	if (!elem || elem->datalen < 10 ||
+	    !(elem->data[10] &
+	      WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) {
+		*tolerated = false;
+	}
+	rcu_read_unlock();
+}
+
+static void
+iwl_mld_check_he_obss_narrow_bw_ru(struct iwl_mld *mld,
+				   struct iwl_mld_link *mld_link,
+				   struct ieee80211_bss_conf *link_conf)
+{
+	bool tolerated = true;
+
+	if (WARN_ON_ONCE(!link_conf->chanreq.oper.chan))
+		return;
+
+	if (!(link_conf->chanreq.oper.chan->flags & IEEE80211_CHAN_RADAR)) {
+		mld_link->he_ru_2mhz_block = false;
+		return;
+	}
+
+	cfg80211_bss_iter(mld->wiphy, &link_conf->chanreq.oper,
+			  iwl_mld_check_he_obss_narrow_bw_ru_iter, &tolerated);
+
+	/* If there is at least one AP on radar channel that cannot
+	 * tolerate 26-tone RU UL OFDMA transmissions using HE TB PPDU.
+	 */
+	mld_link->he_ru_2mhz_block = !tolerated;
+}
+
+static void iwl_mld_link_set_2mhz_block(struct iwl_mld *mld,
+					struct ieee80211_vif *vif,
+					struct ieee80211_sta *sta)
+{
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, link_id);
+		struct iwl_mld_link *mld_link =
+			iwl_mld_link_from_mac80211(link_conf);
+
+		if (WARN_ON(!link_conf || !mld_link))
+			continue;
+
+		if (link_sta->he_cap.has_he)
+			iwl_mld_check_he_obss_narrow_bw_ru(mld, mld_link,
+							   link_conf);
+	}
+}
+
+static int iwl_mld_move_sta_state_up(struct iwl_mld *mld,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_sta *sta,
+				     enum ieee80211_sta_state old_state,
+				     enum ieee80211_sta_state new_state)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int tdls_count = 0;
+	int ret;
+
+	if (old_state == IEEE80211_STA_NOTEXIST &&
+	    new_state == IEEE80211_STA_NONE) {
+		if (sta->tdls) {
+			if (vif->p2p || hweight8(mld->used_phy_ids) != 1)
+				return -EBUSY;
+
+			tdls_count = iwl_mld_tdls_sta_count(mld);
+			if (tdls_count >= IWL_TDLS_STA_COUNT)
+				return -EBUSY;
+		}
+
+		ret = iwl_mld_add_sta(mld, sta, vif, STATION_TYPE_PEER);
+		if (ret)
+			return ret;
+
+		/* just added first TDLS STA, so disable PM */
+		if (sta->tdls && tdls_count == 0)
+			iwl_mld_update_mac_power(mld, vif, false);
+
+		if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls)
+			mld_vif->ap_sta = sta;
+
+		/* Initialize TLC here already - this really tells
+		 * the firmware only what the supported legacy rates are
+		 * (may be) since it's initialized already from what the
+		 * AP advertised in the beacon/probe response. This will
+		 * allow the firmware to send auth/assoc frames with one
+		 * of the supported rates already, rather than having to
+		 * use a mandatory rate.
+		 * If we're the AP, we'll just assume mandatory rates at
+		 * this point, but we know nothing about the STA anyway.
+		 */
+		iwl_mld_config_tlc(mld, vif, sta);
+
+		return ret;
+	} else if (old_state == IEEE80211_STA_NONE &&
+		   new_state == IEEE80211_STA_AUTH) {
+		iwl_mld_set_uapsd(mld, vif);
+		return 0;
+	} else if (old_state == IEEE80211_STA_AUTH &&
+		   new_state == IEEE80211_STA_ASSOC) {
+		ret = iwl_mld_update_all_link_stations(mld, sta);
+
+		if (vif->type == NL80211_IFTYPE_STATION)
+			iwl_mld_link_set_2mhz_block(mld, vif, sta);
+		/* Now the link_sta's capabilities are set, update the FW */
+		iwl_mld_config_tlc(mld, vif, sta);
+
+		if (vif->type == NL80211_IFTYPE_AP) {
+			/* Update MAC_CFG_FILTER_ACCEPT_BEACON if at least
+			 * one sta is associated
+			 */
+			if (++mld_vif->num_associated_stas == 1)
+				ret = iwl_mld_mac_fw_action(mld, vif,
+							    FW_CTXT_ACTION_MODIFY);
+		}
+
+		return ret;
+	} else if (old_state == IEEE80211_STA_ASSOC &&
+		   new_state == IEEE80211_STA_AUTHORIZED) {
+		ret = 0;
+
+		if (!sta->tdls) {
+			mld_vif->authorized = true;
+
+			/* Ensure any block due to a non-BSS link is synced */
+			iwl_mld_emlsr_check_non_bss_block(mld, 0);
+
+			/* Block EMLSR until a certain throughput it reached */
+			if (!mld->fw_status.in_hw_restart &&
+			    IWL_MLD_ENTER_EMLSR_TPT_THRESH > 0)
+				iwl_mld_block_emlsr(mld_vif->mld, vif,
+						    IWL_MLD_EMLSR_BLOCKED_TPT,
+						    0);
+
+			/* clear COEX_HIGH_PRIORITY_ENABLE */
+			ret = iwl_mld_mac_fw_action(mld, vif,
+						    FW_CTXT_ACTION_MODIFY);
+			if (ret)
+				return ret;
+			iwl_mld_smps_workaround(mld, vif, vif->cfg.ps);
+		}
+
+		/* MFP is set by default before the station is authorized.
+		 * Clear it here in case it's not used.
+		 */
+		if (!sta->mfp)
+			ret = iwl_mld_update_all_link_stations(mld, sta);
+
+		/* We can use wide bandwidth now, not only 20 MHz */
+		iwl_mld_config_tlc(mld, vif, sta);
+
+		return ret;
+	} else {
+		return -EINVAL;
+	}
+}
+
+static int iwl_mld_move_sta_state_down(struct iwl_mld *mld,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_sta *sta,
+				       enum ieee80211_sta_state old_state,
+				       enum ieee80211_sta_state new_state)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (old_state == IEEE80211_STA_AUTHORIZED &&
+	    new_state == IEEE80211_STA_ASSOC) {
+		if (!sta->tdls) {
+			mld_vif->authorized = false;
+
+			memset(&mld_vif->emlsr.zeroed_on_not_authorized, 0,
+			       sizeof(mld_vif->emlsr.zeroed_on_not_authorized));
+
+			wiphy_delayed_work_cancel(mld->wiphy,
+						  &mld_vif->emlsr.prevent_done_wk);
+			wiphy_delayed_work_cancel(mld->wiphy,
+						  &mld_vif->emlsr.tmp_non_bss_done_wk);
+			wiphy_work_cancel(mld->wiphy, &mld_vif->emlsr.unblock_tpt_wk);
+			wiphy_delayed_work_cancel(mld->wiphy,
+						  &mld_vif->emlsr.check_tpt_wk);
+
+			iwl_mld_reset_cca_40mhz_workaround(mld, vif);
+			iwl_mld_smps_workaround(mld, vif, true);
+		}
+
+		/* once we move into assoc state, need to update the FW to
+		 * stop using wide bandwidth
+		 */
+		iwl_mld_config_tlc(mld, vif, sta);
+	} else if (old_state == IEEE80211_STA_ASSOC &&
+		   new_state == IEEE80211_STA_AUTH) {
+		if (vif->type == NL80211_IFTYPE_AP &&
+		    !WARN_ON(!mld_vif->num_associated_stas)) {
+			/* Update MAC_CFG_FILTER_ACCEPT_BEACON if the last sta
+			 * is disassociating
+			 */
+			if (--mld_vif->num_associated_stas == 0)
+				iwl_mld_mac_fw_action(mld, vif,
+						      FW_CTXT_ACTION_MODIFY);
+		}
+	} else if (old_state == IEEE80211_STA_AUTH &&
+		   new_state == IEEE80211_STA_NONE) {
+		/* nothing */
+	} else if (old_state == IEEE80211_STA_NONE &&
+		   new_state == IEEE80211_STA_NOTEXIST) {
+		iwl_mld_remove_sta(mld, sta);
+
+		if (sta->tdls && iwl_mld_tdls_sta_count(mld) == 0) {
+			/* just removed last TDLS STA, so enable PM */
+			iwl_mld_update_mac_power(mld, vif, false);
+		}
+	} else {
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int iwl_mld_mac80211_sta_state(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_sta *sta,
+				      enum ieee80211_sta_state old_state,
+				      enum ieee80211_sta_state new_state)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+	IWL_DEBUG_MAC80211(mld, "station %pM state change %d->%d\n",
+			   sta->addr, old_state, new_state);
+
+	mld_sta->sta_state = new_state;
+
+	if (old_state < new_state)
+		return iwl_mld_move_sta_state_up(mld, vif, sta, old_state,
+						 new_state);
+	else
+		return iwl_mld_move_sta_state_down(mld, vif, sta, old_state,
+						   new_state);
+}
+
+static void iwl_mld_mac80211_flush(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   u32 queues, bool drop)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	/* Make sure we're done with the deferred traffic before flushing */
+	iwl_mld_add_txq_list(mld);
+
+	for (int i = 0; i < mld->fw->ucode_capa.num_stations; i++) {
+		struct ieee80211_link_sta *link_sta =
+			wiphy_dereference(mld->wiphy,
+					  mld->fw_id_to_link_sta[i]);
+
+		if (IS_ERR_OR_NULL(link_sta))
+			continue;
+
+		/* Check that the sta belongs to the given vif */
+		if (vif && vif != iwl_mld_sta_from_mac80211(link_sta->sta)->vif)
+			continue;
+
+		if (drop)
+			iwl_mld_flush_sta_txqs(mld, link_sta->sta);
+		else
+			iwl_mld_wait_sta_txqs_empty(mld, link_sta->sta);
+	}
+}
+
+static void iwl_mld_mac80211_flush_sta(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_sta *sta)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	iwl_mld_flush_sta_txqs(mld, sta);
+}
+
+static int
+iwl_mld_mac80211_ampdu_action(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_ampdu_params *params)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct ieee80211_sta *sta = params->sta;
+	enum ieee80211_ampdu_mlme_action action = params->action;
+	u16 tid = params->tid;
+	u16 ssn = params->ssn;
+	u16 buf_size = params->buf_size;
+	u16 timeout = params->timeout;
+	int ret;
+
+	IWL_DEBUG_HT(mld, "A-MPDU action on addr %pM tid: %d action: %d\n",
+		     sta->addr, tid, action);
+
+	switch (action) {
+	case IEEE80211_AMPDU_RX_START:
+		if (!iwl_enable_rx_ampdu()) {
+			ret = -EINVAL;
+			break;
+		}
+		ret = iwl_mld_ampdu_rx_start(mld, sta, tid, ssn, buf_size,
+					     timeout);
+		break;
+	case IEEE80211_AMPDU_RX_STOP:
+		ret = iwl_mld_ampdu_rx_stop(mld, sta, tid);
+		break;
+	default:
+		/* The mac80211 TX_AMPDU_SETUP_IN_HW flag is set for all
+		 * devices, since all support TX A-MPDU offload in hardware.
+		 * Therefore, no TX action should be requested here.
+		 */
+		WARN_ON_ONCE(1);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static bool iwl_mld_can_hw_csum(struct sk_buff *skb)
+{
+	u8 protocol = ip_hdr(skb)->protocol;
+
+	return protocol == IPPROTO_TCP || protocol == IPPROTO_UDP;
+}
+
+static bool iwl_mld_mac80211_can_aggregate(struct ieee80211_hw *hw,
+					   struct sk_buff *head,
+					   struct sk_buff *skb)
+{
+	if (!IS_ENABLED(CONFIG_INET))
+		return false;
+
+	/* For now don't aggregate IPv6 in AMSDU */
+	if (skb->protocol != htons(ETH_P_IP))
+		return false;
+
+	/* Allow aggregation only if both frames have the same HW csum offload
+	 * capability, ensuring consistent HW or SW csum handling in A-MSDU.
+	 */
+	return iwl_mld_can_hw_csum(skb) == iwl_mld_can_hw_csum(head);
+}
+
+static void iwl_mld_mac80211_sync_rx_queues(struct ieee80211_hw *hw)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	iwl_mld_sync_rx_queues(mld, IWL_MLD_RXQ_EMPTY, NULL, 0);
+}
+
+static void iwl_mld_sta_rc_update(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_link_sta *link_sta,
+				  u32 changed)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	if (changed & (IEEE80211_RC_BW_CHANGED |
+		       IEEE80211_RC_SUPP_RATES_CHANGED |
+		       IEEE80211_RC_NSS_CHANGED)) {
+		struct ieee80211_bss_conf *link =
+			link_conf_dereference_check(vif, link_sta->link_id);
+
+		if (WARN_ON(!link))
+			return;
+
+		iwl_mld_config_tlc_link(mld, vif, link, link_sta);
+	}
+}
+
+#ifdef CONFIG_PM_SLEEP
+static void iwl_mld_set_wakeup(struct ieee80211_hw *hw, bool enabled)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	device_set_wakeup_enable(mld->trans->dev, enabled);
+}
+
+/* Returns 0 on success. 1 if failed to suspend with wowlan:
+ * If the circumstances didn't satisfy the conditions for suspension
+ * with wowlan, mac80211 would use the no_wowlan flow.
+ * If an error had occurred we update the trans status and state here
+ * and the result will be stopping the FW.
+ */
+static int
+iwl_mld_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	iwl_fw_runtime_suspend(&mld->fwrt);
+
+	ret = iwl_mld_wowlan_suspend(mld, wowlan);
+	if (ret) {
+		if (ret < 0) {
+			mld->trans->state = IWL_TRANS_NO_FW;
+			set_bit(STATUS_FW_ERROR, &mld->trans->status);
+		}
+		return 1;
+	}
+
+	if (iwl_mld_no_wowlan_suspend(mld))
+		return 1;
+
+	return 0;
+}
+
+static int iwl_mld_resume(struct ieee80211_hw *hw)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	ret = iwl_mld_wowlan_resume(mld);
+	if (ret)
+		return ret;
+
+	iwl_fw_runtime_resume(&mld->fwrt);
+
+	iwl_mld_low_latency_restart(mld);
+
+	return 0;
+}
+#endif
+
+static int iwl_mld_alloc_ptk_pn(struct iwl_mld *mld,
+				struct iwl_mld_sta *mld_sta,
+				struct ieee80211_key_conf *key,
+				struct iwl_mld_ptk_pn **ptk_pn)
+{
+	u8 num_rx_queues = mld->trans->info.num_rxqs;
+	int keyidx = key->keyidx;
+	struct ieee80211_key_seq seq;
+
+	if (WARN_ON(keyidx >= ARRAY_SIZE(mld_sta->ptk_pn)))
+		return -EINVAL;
+
+	WARN_ON(rcu_access_pointer(mld_sta->ptk_pn[keyidx]));
+	*ptk_pn = kzalloc(struct_size(*ptk_pn, q, num_rx_queues),
+			  GFP_KERNEL);
+	if (!*ptk_pn)
+		return -ENOMEM;
+
+	for (u8 tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+		ieee80211_get_key_rx_seq(key, tid, &seq);
+		for (u8 q = 0; q < num_rx_queues; q++)
+			memcpy((*ptk_pn)->q[q].pn[tid], seq.ccmp.pn,
+			       IEEE80211_CCMP_PN_LEN);
+	}
+
+	rcu_assign_pointer(mld_sta->ptk_pn[keyidx], *ptk_pn);
+
+	return 0;
+}
+
+static int iwl_mld_set_key_add(struct iwl_mld *mld,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_sta *sta,
+			       struct ieee80211_key_conf *key)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_sta *mld_sta =
+		sta ? iwl_mld_sta_from_mac80211(sta) : NULL;
+	struct iwl_mld_ptk_pn *ptk_pn = NULL;
+	int keyidx = key->keyidx;
+	int ret;
+
+	/* Will be set to 0 if added successfully */
+	key->hw_key_idx = STA_KEY_IDX_INVALID;
+
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_WEP40:
+	case WLAN_CIPHER_SUITE_WEP104:
+		IWL_DEBUG_MAC80211(mld, "Use SW encryption for WEP\n");
+		return -EOPNOTSUPP;
+	case WLAN_CIPHER_SUITE_TKIP:
+		if (vif->type == NL80211_IFTYPE_STATION) {
+			key->flags |= IEEE80211_KEY_FLAG_PUT_MIC_SPACE;
+			break;
+		}
+		IWL_DEBUG_MAC80211(mld, "Use SW encryption for TKIP\n");
+		return -EOPNOTSUPP;
+	case WLAN_CIPHER_SUITE_CCMP:
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_GCMP_256:
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    (keyidx == 6 || keyidx == 7))
+		rcu_assign_pointer(mld_vif->bigtks[keyidx - 6], key);
+
+	/* After exiting from RFKILL, hostapd configures GTK/ITGK before the
+	 * AP is started, but those keys can't be sent to the FW before the
+	 * MCAST/BCAST STAs are added to it (which happens upon AP start).
+	 * Store it here to be sent later when the AP is started.
+	 */
+	if ((vif->type == NL80211_IFTYPE_ADHOC ||
+	     vif->type == NL80211_IFTYPE_AP) && !sta &&
+	     !mld_vif->ap_ibss_active)
+		return iwl_mld_store_ap_early_key(mld, key, mld_vif);
+
+	if (!mld->fw_status.in_hw_restart && mld_sta &&
+	    key->flags & IEEE80211_KEY_FLAG_PAIRWISE &&
+	    (key->cipher == WLAN_CIPHER_SUITE_CCMP ||
+	     key->cipher == WLAN_CIPHER_SUITE_GCMP ||
+	     key->cipher == WLAN_CIPHER_SUITE_GCMP_256)) {
+		ret = iwl_mld_alloc_ptk_pn(mld, mld_sta, key, &ptk_pn);
+		if (ret)
+			return ret;
+	}
+
+	IWL_DEBUG_MAC80211(mld, "set hwcrypto key (sta:%pM, id:%d)\n",
+			   sta ? sta->addr : NULL, keyidx);
+
+	ret = iwl_mld_add_key(mld, vif, sta, key);
+	if (ret) {
+		IWL_WARN(mld, "set key failed (%d)\n", ret);
+		if (ptk_pn) {
+			RCU_INIT_POINTER(mld_sta->ptk_pn[keyidx], NULL);
+			kfree(ptk_pn);
+		}
+
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static void iwl_mld_set_key_remove(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_sta *sta,
+				   struct ieee80211_key_conf *key)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_sta *mld_sta =
+		sta ? iwl_mld_sta_from_mac80211(sta) : NULL;
+	int keyidx = key->keyidx;
+
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    (keyidx == 6 || keyidx == 7))
+		RCU_INIT_POINTER(mld_vif->bigtks[keyidx - 6], NULL);
+
+	if (mld_sta && key->flags & IEEE80211_KEY_FLAG_PAIRWISE &&
+	    (key->cipher == WLAN_CIPHER_SUITE_CCMP ||
+	     key->cipher == WLAN_CIPHER_SUITE_GCMP ||
+	     key->cipher == WLAN_CIPHER_SUITE_GCMP_256)) {
+		struct iwl_mld_ptk_pn *ptk_pn;
+
+		if (WARN_ON(keyidx >= ARRAY_SIZE(mld_sta->ptk_pn)))
+			return;
+
+		ptk_pn = wiphy_dereference(mld->wiphy,
+					   mld_sta->ptk_pn[keyidx]);
+		RCU_INIT_POINTER(mld_sta->ptk_pn[keyidx], NULL);
+		if (!WARN_ON(!ptk_pn))
+			kfree_rcu(ptk_pn, rcu_head);
+	}
+
+	/* if this key was stored to be added later to the FW - free it here */
+	if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
+		iwl_mld_free_ap_early_key(mld, key, mld_vif);
+
+	/* We already removed it */
+	if (key->hw_key_idx == STA_KEY_IDX_INVALID)
+		return;
+
+	IWL_DEBUG_MAC80211(mld, "disable hwcrypto key\n");
+
+	iwl_mld_remove_key(mld, vif, sta, key);
+}
+
+static int iwl_mld_mac80211_set_key(struct ieee80211_hw *hw,
+				    enum set_key_cmd cmd,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_sta *sta,
+				    struct ieee80211_key_conf *key)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	switch (cmd) {
+	case SET_KEY:
+		ret = iwl_mld_set_key_add(mld, vif, sta, key);
+		if (ret)
+			ret = -EOPNOTSUPP;
+		break;
+	case DISABLE_KEY:
+		iwl_mld_set_key_remove(mld, vif, sta, key);
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int
+iwl_mld_pre_channel_switch(struct ieee80211_hw *hw,
+			   struct ieee80211_vif *vif,
+			   struct ieee80211_channel_switch *chsw)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *mld_link =
+		iwl_mld_link_dereference_check(mld_vif, chsw->link_id);
+	u8 primary;
+	int selected;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(!mld_link))
+		return -EINVAL;
+
+	IWL_DEBUG_MAC80211(mld, "pre CSA to freq %d\n",
+			   chsw->chandef.center_freq1);
+
+	if (!iwl_mld_emlsr_active(vif))
+		return 0;
+
+	primary = iwl_mld_get_primary_link(vif);
+
+	/* stay on the primary link unless it is undergoing a CSA with quiet */
+	if (chsw->link_id == primary && chsw->block_tx)
+		selected = iwl_mld_get_other_link(vif, primary);
+	else
+		selected = primary;
+
+	/* Remember to tell the firmware that this link can't tx
+	 * Note that this logic seems to be unrelated to emlsr, but it
+	 * really is needed only when emlsr is active. When we have a
+	 * single link, the firmware will handle all this on its own.
+	 * In multi-link scenarios, we can learn about the CSA from
+	 * another link and this logic is too complex for the firmware
+	 * to track.
+	 * Since we want to de-activate the link that got a CSA with mode=1,
+	 * we need to tell the firmware not to send any frame on that link
+	 * as the firmware may not be aware that link is under a CSA
+	 * with mode=1 (no Tx allowed).
+	 */
+	mld_link->silent_deactivation = chsw->block_tx;
+	iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_CSA, selected);
+
+	return 0;
+}
+
+static void
+iwl_mld_channel_switch(struct ieee80211_hw *hw,
+		       struct ieee80211_vif *vif,
+		       struct ieee80211_channel_switch *chsw)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	/* By implementing this operation, we prevent mac80211 from
+	 * starting its own channel switch timer, so that we can call
+	 * ieee80211_chswitch_done() ourselves at the right time
+	 * (Upon receiving the channel_switch_start notification from the fw)
+	 */
+	IWL_DEBUG_MAC80211(mld,
+			   "dummy channel switch op\n");
+}
+
+static int
+iwl_mld_post_channel_switch(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link_conf);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	WARN_ON(mld_link->silent_deactivation);
+
+	return 0;
+}
+
+static void
+iwl_mld_abort_channel_switch(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link_conf);
+
+	IWL_DEBUG_MAC80211(mld,
+			   "abort channel switch op\n");
+	mld_link->silent_deactivation = false;
+}
+
+static int
+iwl_mld_switch_vif_chanctx_swap(struct ieee80211_hw *hw,
+				struct ieee80211_vif_chanctx_switch *vifs)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	iwl_mld_unassign_vif_chanctx(hw, vifs[0].vif, vifs[0].link_conf,
+				     vifs[0].old_ctx);
+	iwl_mld_remove_chanctx(hw, vifs[0].old_ctx);
+
+	ret = iwl_mld_add_chanctx(hw, vifs[0].new_ctx);
+	if (ret) {
+		IWL_ERR(mld, "failed to add new_ctx during channel switch\n");
+		goto out_reassign;
+	}
+
+	ret = iwl_mld_assign_vif_chanctx(hw, vifs[0].vif, vifs[0].link_conf,
+					 vifs[0].new_ctx);
+	if (ret) {
+		IWL_ERR(mld,
+			"failed to assign new_ctx during channel switch\n");
+		goto out_remove;
+	}
+
+	return 0;
+
+ out_remove:
+	iwl_mld_remove_chanctx(hw, vifs[0].new_ctx);
+ out_reassign:
+	if (iwl_mld_add_chanctx(hw, vifs[0].old_ctx)) {
+		IWL_ERR(mld, "failed to add old_ctx after failure\n");
+		return ret;
+	}
+
+	if (iwl_mld_assign_vif_chanctx(hw, vifs[0].vif, vifs[0].link_conf,
+				       vifs[0].old_ctx))
+		IWL_ERR(mld, "failed to reassign old_ctx after failure\n");
+
+	return ret;
+}
+
+static int
+iwl_mld_switch_vif_chanctx_reassign(struct ieee80211_hw *hw,
+				    struct ieee80211_vif_chanctx_switch *vifs)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int ret;
+
+	iwl_mld_unassign_vif_chanctx(hw, vifs[0].vif, vifs[0].link_conf,
+				     vifs[0].old_ctx);
+	ret = iwl_mld_assign_vif_chanctx(hw, vifs[0].vif, vifs[0].link_conf,
+					 vifs[0].new_ctx);
+	if (ret) {
+		IWL_ERR(mld,
+			"failed to assign new_ctx during channel switch\n");
+		goto out_reassign;
+	}
+
+	return 0;
+
+out_reassign:
+	if (iwl_mld_assign_vif_chanctx(hw, vifs[0].vif, vifs[0].link_conf,
+				       vifs[0].old_ctx))
+		IWL_ERR(mld, "failed to reassign old_ctx after failure\n");
+
+	return ret;
+}
+
+static int
+iwl_mld_switch_vif_chanctx(struct ieee80211_hw *hw,
+			   struct ieee80211_vif_chanctx_switch *vifs,
+			   int n_vifs,
+			   enum ieee80211_chanctx_switch_mode mode)
+{
+	int ret;
+
+	/* we only support a single-vif right now */
+	if (n_vifs > 1)
+		return -EOPNOTSUPP;
+
+	switch (mode) {
+	case CHANCTX_SWMODE_SWAP_CONTEXTS:
+		ret = iwl_mld_switch_vif_chanctx_swap(hw, vifs);
+		break;
+	case CHANCTX_SWMODE_REASSIGN_VIF:
+		ret = iwl_mld_switch_vif_chanctx_reassign(hw, vifs);
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+
+	return ret;
+}
+
+static void iwl_mld_sta_pre_rcu_remove(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_sta *sta)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link_sta *mld_link_sta;
+	u8 link_id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* This is called before mac80211 does RCU synchronisation,
+	 * so here we already invalidate our internal RCU-protected
+	 * station pointer. The rest of the code will thus no longer
+	 * be able to find the station this way, and we don't rely
+	 * on further RCU synchronisation after the sta_state()
+	 * callback deleted the station.
+	 */
+	for_each_mld_link_sta(mld_sta, mld_link_sta, link_id)
+		RCU_INIT_POINTER(mld->fw_id_to_link_sta[mld_link_sta->fw_id],
+				 NULL);
+
+	if (sta == mld_vif->ap_sta)
+		mld_vif->ap_sta = NULL;
+}
+
+static void
+iwl_mld_mac80211_mgd_protect_tdls_discover(struct ieee80211_hw *hw,
+					   struct ieee80211_vif *vif,
+					   unsigned int link_id)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct ieee80211_bss_conf *link_conf;
+	u32 duration;
+	int ret;
+
+	link_conf = wiphy_dereference(hw->wiphy, vif->link_conf[link_id]);
+	if (WARN_ON_ONCE(!link_conf))
+		return;
+
+	/* Protect the session to hear the TDLS setup response on the channel */
+
+	duration = 2 * link_conf->dtim_period * link_conf->beacon_int;
+
+	ret = iwl_mld_start_session_protection(mld, vif, duration, duration,
+					       link_id, HZ / 5);
+	if (ret)
+		IWL_ERR(mld,
+			"Failed to start session protection for TDLS: %d\n",
+			ret);
+}
+
+static bool iwl_mld_can_activate_links(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       u16 desired_links)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int n_links = hweight16(desired_links);
+
+	/* Check if HW supports the wanted number of links */
+	return n_links <= iwl_mld_max_active_links(mld, vif);
+}
+
+static int
+iwl_mld_change_vif_links(struct ieee80211_hw *hw,
+			 struct ieee80211_vif *vif,
+			 u16 old_links, u16 new_links,
+			 struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct ieee80211_bss_conf *link_conf;
+	u16 removed = old_links & ~new_links;
+	u16 added = new_links & ~old_links;
+	int err;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/*
+	 * No bits designate non-MLO mode. We can handle MLO exit/enter by
+	 * simply mapping that to link ID zero internally.
+	 * Note that mac80211 does such a non-MLO to MLO switch during restart
+	 * if it was in MLO before. In that case, we do not have a link to
+	 * remove.
+	 */
+	if (old_links == 0 && !mld->fw_status.in_hw_restart)
+		removed |= BIT(0);
+
+	if (new_links == 0)
+		added |= BIT(0);
+
+	for (int i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+		if (removed & BIT(i) && !WARN_ON(!old[i]))
+			iwl_mld_remove_link(mld, old[i]);
+	}
+
+	for (int i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+		if (added & BIT(i)) {
+			link_conf = link_conf_dereference_protected(vif, i);
+			if (!link_conf) {
+				err = -EINVAL;
+				goto remove_added_links;
+			}
+
+			err = iwl_mld_add_link(mld, link_conf);
+			if (err)
+				goto remove_added_links;
+		}
+	}
+
+	/*
+	 * Ensure we always have a valid primary_link. When using multiple
+	 * links the proper value is set in assign_vif_chanctx.
+	 */
+	mld_vif->emlsr.primary = new_links ? __ffs(new_links) : 0;
+
+	/*
+	 * Special MLO restart case. We did not have a link when the interface
+	 * was added, so do the power configuration now.
+	 */
+	if (old_links == 0 && mld->fw_status.in_hw_restart)
+		iwl_mld_update_mac_power(mld, vif, false);
+
+	return 0;
+
+remove_added_links:
+	for (int i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+		if (!(added & BIT(i)))
+			continue;
+
+		link_conf = link_conf_dereference_protected(vif, i);
+		if (!link_conf || !iwl_mld_link_from_mac80211(link_conf))
+			continue;
+
+		iwl_mld_remove_link(mld, link_conf);
+	}
+
+	if (WARN_ON(!iwl_mld_error_before_recovery(mld)))
+		return err;
+
+	/* reconfig will fix us anyway */
+	return 0;
+}
+
+static int iwl_mld_change_sta_links(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_sta *sta,
+				    u16 old_links, u16 new_links)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	return iwl_mld_update_link_stas(mld, vif, sta, old_links, new_links);
+}
+
+static int iwl_mld_mac80211_join_ibss(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif)
+{
+	return iwl_mld_start_ap_ibss(hw, vif, &vif->bss_conf);
+}
+
+static void iwl_mld_mac80211_leave_ibss(struct ieee80211_hw *hw,
+					struct ieee80211_vif *vif)
+{
+	return iwl_mld_stop_ap_ibss(hw, vif, &vif->bss_conf);
+}
+
+static int iwl_mld_mac80211_tx_last_beacon(struct ieee80211_hw *hw)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	return mld->ibss_manager;
+}
+
+static void iwl_mld_prep_add_interface(struct ieee80211_hw *hw,
+				       enum nl80211_iftype type)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	IWL_DEBUG_MAC80211(mld, "prep_add_interface: type=%u\n", type);
+
+	if (!(type == NL80211_IFTYPE_AP ||
+	      type == NL80211_IFTYPE_P2P_GO ||
+	      type == NL80211_IFTYPE_P2P_CLIENT))
+		return;
+
+	iwl_mld_emlsr_block_tmp_non_bss(mld);
+}
+
+static int iwl_mld_set_hw_timestamp(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct cfg80211_set_hw_timestamp *hwts)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	u32 protocols = 0;
+
+	/* HW timestamping is only supported for a specific station */
+	if (!hwts->macaddr)
+		return -EOPNOTSUPP;
+
+	if (hwts->enable)
+		protocols =
+			IWL_TIME_SYNC_PROTOCOL_TM | IWL_TIME_SYNC_PROTOCOL_FTM;
+
+	return iwl_mld_time_sync_config(mld, hwts->macaddr, protocols);
+}
+
+static int iwl_mld_start_pmsr(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct cfg80211_pmsr_request *request)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	return iwl_mld_ftm_start(mld, vif, request);
+}
+
+static enum ieee80211_neg_ttlm_res
+iwl_mld_can_neg_ttlm(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		     struct ieee80211_neg_ttlm *neg_ttlm)
+{
+	u16 map;
+
+	/* Verify all TIDs are mapped to the same links set */
+	map = neg_ttlm->downlink[0];
+	for (int i = 0; i < IEEE80211_TTLM_NUM_TIDS; i++) {
+		if (neg_ttlm->downlink[i] != neg_ttlm->uplink[i] ||
+		    neg_ttlm->uplink[i] != map)
+			return NEG_TTLM_RES_REJECT;
+	}
+
+	return NEG_TTLM_RES_ACCEPT;
+}
+
+const struct ieee80211_ops iwl_mld_hw_ops = {
+	.tx = iwl_mld_mac80211_tx,
+	.start = iwl_mld_mac80211_start,
+	.stop = iwl_mld_mac80211_stop,
+	.config = iwl_mld_mac80211_config,
+	.add_interface = iwl_mld_mac80211_add_interface,
+	.remove_interface = iwl_mld_mac80211_remove_interface,
+	.conf_tx = iwl_mld_mac80211_conf_tx,
+	.prepare_multicast = iwl_mld_mac80211_prepare_multicast,
+	.configure_filter = iwl_mld_mac80211_configure_filter,
+	.reconfig_complete = iwl_mld_mac80211_reconfig_complete,
+	.wake_tx_queue = iwl_mld_mac80211_wake_tx_queue,
+	.add_chanctx = iwl_mld_add_chanctx,
+	.remove_chanctx = iwl_mld_remove_chanctx,
+	.change_chanctx = iwl_mld_change_chanctx,
+	.assign_vif_chanctx = iwl_mld_assign_vif_chanctx,
+	.unassign_vif_chanctx = iwl_mld_unassign_vif_chanctx,
+	.set_rts_threshold = iwl_mld_mac80211_set_rts_threshold,
+	.link_info_changed = iwl_mld_mac80211_link_info_changed,
+	.vif_cfg_changed = iwl_mld_mac80211_vif_cfg_changed,
+	.set_key = iwl_mld_mac80211_set_key,
+	.hw_scan = iwl_mld_mac80211_hw_scan,
+	.cancel_hw_scan = iwl_mld_mac80211_cancel_hw_scan,
+	.sched_scan_start = iwl_mld_mac80211_sched_scan_start,
+	.sched_scan_stop = iwl_mld_mac80211_sched_scan_stop,
+	.mgd_prepare_tx = iwl_mld_mac80211_mgd_prepare_tx,
+	.mgd_complete_tx = iwl_mld_mac_mgd_complete_tx,
+	.sta_state = iwl_mld_mac80211_sta_state,
+	.sta_statistics = iwl_mld_mac80211_sta_statistics,
+	.get_survey = iwl_mld_mac80211_get_survey,
+	.flush = iwl_mld_mac80211_flush,
+	.flush_sta = iwl_mld_mac80211_flush_sta,
+	.ampdu_action = iwl_mld_mac80211_ampdu_action,
+	.can_aggregate_in_amsdu = iwl_mld_mac80211_can_aggregate,
+	.sync_rx_queues = iwl_mld_mac80211_sync_rx_queues,
+	.link_sta_rc_update = iwl_mld_sta_rc_update,
+	.start_ap = iwl_mld_start_ap_ibss,
+	.stop_ap = iwl_mld_stop_ap_ibss,
+	.pre_channel_switch = iwl_mld_pre_channel_switch,
+	.channel_switch = iwl_mld_channel_switch,
+	.post_channel_switch = iwl_mld_post_channel_switch,
+	.abort_channel_switch = iwl_mld_abort_channel_switch,
+	.switch_vif_chanctx = iwl_mld_switch_vif_chanctx,
+	.sta_pre_rcu_remove = iwl_mld_sta_pre_rcu_remove,
+	.remain_on_channel = iwl_mld_start_roc,
+	.cancel_remain_on_channel = iwl_mld_cancel_roc,
+	.can_activate_links = iwl_mld_can_activate_links,
+	.change_vif_links = iwl_mld_change_vif_links,
+	.change_sta_links = iwl_mld_change_sta_links,
+#ifdef CONFIG_PM_SLEEP
+	.suspend = iwl_mld_suspend,
+	.resume = iwl_mld_resume,
+	.set_wakeup = iwl_mld_set_wakeup,
+	.set_rekey_data = iwl_mld_set_rekey_data,
+#if IS_ENABLED(CONFIG_IPV6)
+	.ipv6_addr_change = iwl_mld_ipv6_addr_change,
+#endif /* IS_ENABLED(CONFIG_IPV6) */
+#endif /* CONFIG_PM_SLEEP */
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	.vif_add_debugfs = iwl_mld_add_vif_debugfs,
+	.link_add_debugfs = iwl_mld_add_link_debugfs,
+	.link_sta_add_debugfs = iwl_mld_add_link_sta_debugfs,
+#endif
+	.mgd_protect_tdls_discover = iwl_mld_mac80211_mgd_protect_tdls_discover,
+	.join_ibss = iwl_mld_mac80211_join_ibss,
+	.leave_ibss = iwl_mld_mac80211_leave_ibss,
+	.tx_last_beacon = iwl_mld_mac80211_tx_last_beacon,
+	.prep_add_interface = iwl_mld_prep_add_interface,
+	.set_hw_timestamp = iwl_mld_set_hw_timestamp,
+	.start_pmsr = iwl_mld_start_pmsr,
+	.can_neg_ttlm = iwl_mld_can_neg_ttlm,
+};
diff --git a/sys/contrib/dev/iwlwifi/mld/mac80211.h b/sys/contrib/dev/iwlwifi/mld/mac80211.h
new file mode 100644
index 000000000000..aad04d7b2617
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mac80211.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_mac80211_h__
+#define __iwl_mld_mac80211_h__
+
+#include "mld.h"
+
+int iwl_mld_register_hw(struct iwl_mld *mld);
+void iwl_mld_recalc_multicast_filter(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_mac80211_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/mcc.c b/sys/contrib/dev/iwlwifi/mld/mcc.c
new file mode 100644
index 000000000000..16bb1b4904f9
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mcc.c
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include <net/cfg80211.h>
+#include <net/mac80211.h>
+
+#include <fw/dbg.h>
+#include <iwl-nvm-parse.h>
+
+#include "mld.h"
+#include "hcmd.h"
+#include "mcc.h"
+
+/* It is the caller's responsibility to free the pointer returned here */
+static struct iwl_mcc_update_resp_v8 *
+iwl_mld_copy_mcc_resp(const struct iwl_rx_packet *pkt)
+{
+	const struct iwl_mcc_update_resp_v8 *mcc_resp_v8 = (const void *)pkt->data;
+	int n_channels = __le32_to_cpu(mcc_resp_v8->n_channels);
+	struct iwl_mcc_update_resp_v8 *resp_cp;
+	int notif_len = struct_size(resp_cp, channels, n_channels);
+
+	if (iwl_rx_packet_payload_len(pkt) != notif_len)
+		return ERR_PTR(-EINVAL);
+
+	resp_cp = kmemdup(mcc_resp_v8, notif_len, GFP_KERNEL);
+	if (!resp_cp)
+		return ERR_PTR(-ENOMEM);
+
+	return resp_cp;
+}
+
+/* It is the caller's responsibility to free the pointer returned here */
+static struct iwl_mcc_update_resp_v8 *
+iwl_mld_update_mcc(struct iwl_mld *mld, const char *alpha2,
+		   enum iwl_mcc_source src_id)
+{
+	struct iwl_mcc_update_cmd mcc_update_cmd = {
+		.mcc = cpu_to_le16(alpha2[0] << 8 | alpha2[1]),
+		.source_id = (u8)src_id,
+	};
+	struct iwl_mcc_update_resp_v8 *resp_cp;
+	struct iwl_rx_packet *pkt;
+	struct iwl_host_cmd cmd = {
+		.id = MCC_UPDATE_CMD,
+		.flags = CMD_WANT_SKB,
+		.data = { &mcc_update_cmd },
+		.len[0] = sizeof(mcc_update_cmd),
+	};
+	int ret;
+	u16 mcc;
+
+	IWL_DEBUG_LAR(mld, "send MCC update to FW with '%c%c' src = %d\n",
+		      alpha2[0], alpha2[1], src_id);
+
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	if (ret)
+		return ERR_PTR(ret);
+
+	pkt = cmd.resp_pkt;
+
+	resp_cp = iwl_mld_copy_mcc_resp(pkt);
+	if (IS_ERR(resp_cp))
+		goto exit;
+
+	mcc = le16_to_cpu(resp_cp->mcc);
+
+	IWL_FW_CHECK(mld, !mcc, "mcc can't be 0: %d\n", mcc);
+
+	IWL_DEBUG_LAR(mld,
+		      "MCC response status: 0x%x. new MCC: 0x%x ('%c%c')\n",
+		      le32_to_cpu(resp_cp->status), mcc, mcc >> 8, mcc & 0xff);
+
+exit:
+	iwl_free_resp(&cmd);
+	return resp_cp;
+}
+
+/* It is the caller's responsibility to free the pointer returned here */
+struct ieee80211_regdomain *
+iwl_mld_get_regdomain(struct iwl_mld *mld,
+		      const char *alpha2,
+		      enum iwl_mcc_source src_id,
+		      bool *changed)
+{
+	struct ieee80211_regdomain *regd = NULL;
+	struct iwl_mcc_update_resp_v8 *resp;
+	u8 resp_ver = iwl_fw_lookup_notif_ver(mld->fw, IWL_ALWAYS_LONG_GROUP,
+					      MCC_UPDATE_CMD, 0);
+
+	IWL_DEBUG_LAR(mld, "Getting regdomain data for %s from FW\n", alpha2);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	resp = iwl_mld_update_mcc(mld, alpha2, src_id);
+	if (IS_ERR(resp)) {
+		IWL_DEBUG_LAR(mld, "Could not get update from FW %ld\n",
+			      PTR_ERR(resp));
+		resp = NULL;
+		goto out;
+	}
+
+	if (changed) {
+		u32 status = le32_to_cpu(resp->status);
+
+		*changed = (status == MCC_RESP_NEW_CHAN_PROFILE ||
+			    status == MCC_RESP_ILLEGAL);
+	}
+	IWL_DEBUG_LAR(mld, "MCC update response version: %d\n", resp_ver);
+
+	regd = iwl_parse_nvm_mcc_info(mld->trans,
+				      __le32_to_cpu(resp->n_channels),
+				      resp->channels,
+				      __le16_to_cpu(resp->mcc),
+				      __le16_to_cpu(resp->geo_info),
+				      le32_to_cpu(resp->cap), resp_ver);
+
+	if (IS_ERR(regd)) {
+		IWL_DEBUG_LAR(mld, "Could not get parse update from FW %ld\n",
+			      PTR_ERR(regd));
+		goto out;
+	}
+
+	IWL_DEBUG_LAR(mld, "setting alpha2 from FW to %s (0x%x, 0x%x) src=%d\n",
+		      regd->alpha2, regd->alpha2[0],
+		      regd->alpha2[1], resp->source_id);
+
+	mld->mcc_src = resp->source_id;
+
+	/* FM is the earliest supported and later always do puncturing */
+	if (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id) == IWL_CFG_RF_TYPE_FM) {
+		if (!iwl_puncturing_is_allowed_in_bios(mld->bios_enable_puncturing,
+						       le16_to_cpu(resp->mcc)))
+			ieee80211_hw_set(mld->hw, DISALLOW_PUNCTURING);
+		else
+			__clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING,
+				    mld->hw->flags);
+	}
+
+out:
+	kfree(resp);
+	return regd;
+}
+
+/* It is the caller's responsibility to free the pointer returned here */
+static struct ieee80211_regdomain *
+iwl_mld_get_current_regdomain(struct iwl_mld *mld,
+			      bool *changed)
+{
+	return iwl_mld_get_regdomain(mld, "ZZ",
+				     MCC_SOURCE_GET_CURRENT, changed);
+}
+
+void iwl_mld_update_changed_regdomain(struct iwl_mld *mld)
+{
+	struct ieee80211_regdomain *regd;
+	bool changed;
+
+	regd = iwl_mld_get_current_regdomain(mld, &changed);
+
+	if (IS_ERR_OR_NULL(regd))
+		return;
+
+	if (changed)
+		regulatory_set_wiphy_regd(mld->wiphy, regd);
+	kfree(regd);
+}
+
+static int iwl_mld_apply_last_mcc(struct iwl_mld *mld,
+				  const char *alpha2)
+{
+	struct ieee80211_regdomain *regd;
+	u32 used_src;
+	bool changed;
+	int ret;
+
+	/* save the last source in case we overwrite it below */
+	used_src = mld->mcc_src;
+
+	/* Notify the firmware we support wifi location updates */
+	regd = iwl_mld_get_current_regdomain(mld, NULL);
+	if (!IS_ERR_OR_NULL(regd))
+		kfree(regd);
+
+	/* Now set our last stored MCC and source */
+	regd = iwl_mld_get_regdomain(mld, alpha2, used_src,
+				     &changed);
+	if (IS_ERR_OR_NULL(regd))
+		return -EIO;
+
+	/* update cfg80211 if the regdomain was changed */
+	if (changed)
+		ret = regulatory_set_wiphy_regd_sync(mld->wiphy, regd);
+	else
+		ret = 0;
+
+	kfree(regd);
+	return ret;
+}
+
+int iwl_mld_init_mcc(struct iwl_mld *mld)
+{
+	const struct ieee80211_regdomain *r;
+	struct ieee80211_regdomain *regd;
+	char mcc[3];
+	int retval;
+
+	/* try to replay the last set MCC to FW */
+	r = wiphy_dereference(mld->wiphy, mld->wiphy->regd);
+
+	if (r)
+		return iwl_mld_apply_last_mcc(mld, r->alpha2);
+
+	regd = iwl_mld_get_current_regdomain(mld, NULL);
+	if (IS_ERR_OR_NULL(regd))
+		return -EIO;
+
+	if (!iwl_bios_get_mcc(&mld->fwrt, mcc)) {
+		kfree(regd);
+		regd = iwl_mld_get_regdomain(mld, mcc, MCC_SOURCE_BIOS, NULL);
+		if (IS_ERR_OR_NULL(regd))
+			return -EIO;
+	}
+
+	retval = regulatory_set_wiphy_regd_sync(mld->wiphy, regd);
+
+	kfree(regd);
+	return retval;
+}
+
+static void iwl_mld_find_assoc_vif_iterator(void *data, u8 *mac,
+					    struct ieee80211_vif *vif)
+{
+	bool *assoc = data;
+
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    vif->cfg.assoc)
+		*assoc = true;
+}
+
+static bool iwl_mld_is_a_vif_assoc(struct iwl_mld *mld)
+{
+	bool assoc = false;
+
+	ieee80211_iterate_active_interfaces_atomic(mld->hw,
+						   IEEE80211_IFACE_ITER_NORMAL,
+						   iwl_mld_find_assoc_vif_iterator,
+						   &assoc);
+	return assoc;
+}
+
+void iwl_mld_handle_update_mcc(struct iwl_mld *mld, struct iwl_rx_packet *pkt)
+{
+	struct iwl_mcc_chub_notif *notif = (void *)pkt->data;
+	enum iwl_mcc_source src;
+	char mcc[3];
+	struct ieee80211_regdomain *regd;
+	bool changed;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (iwl_mld_is_a_vif_assoc(mld) &&
+	    notif->source_id == MCC_SOURCE_WIFI) {
+		IWL_DEBUG_LAR(mld, "Ignore mcc update while associated\n");
+		return;
+	}
+
+	mcc[0] = le16_to_cpu(notif->mcc) >> 8;
+	mcc[1] = le16_to_cpu(notif->mcc) & 0xff;
+	mcc[2] = '\0';
+	src = notif->source_id;
+
+	IWL_DEBUG_LAR(mld,
+		      "RX: received chub update mcc cmd (mcc '%s' src %d)\n",
+		      mcc, src);
+	regd = iwl_mld_get_regdomain(mld, mcc, src, &changed);
+	if (IS_ERR_OR_NULL(regd))
+		return;
+
+	if (changed)
+		regulatory_set_wiphy_regd(mld->hw->wiphy, regd);
+	kfree(regd);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/mcc.h b/sys/contrib/dev/iwlwifi/mld/mcc.h
new file mode 100644
index 000000000000..2b31e5b5e2ed
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mcc.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_mcc_h__
+#define __iwl_mld_mcc_h__
+
+int iwl_mld_init_mcc(struct iwl_mld *mld);
+void iwl_mld_handle_update_mcc(struct iwl_mld *mld, struct iwl_rx_packet *pkt);
+void iwl_mld_update_changed_regdomain(struct iwl_mld *mld);
+struct ieee80211_regdomain *
+iwl_mld_get_regdomain(struct iwl_mld *mld,
+		      const char *alpha2,
+		      enum iwl_mcc_source src_id,
+		      bool *changed);
+
+#endif /* __iwl_mld_mcc_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/mld.c b/sys/contrib/dev/iwlwifi/mld/mld.c
new file mode 100644
index 000000000000..7b46ccc306ab
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mld.c
@@ -0,0 +1,771 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <linux/rtnetlink.h>
+#include <net/mac80211.h>
+
+#include "fw/api/rx.h"
+#include "fw/api/datapath.h"
+#include "fw/api/commands.h"
+#include "fw/api/offload.h"
+#include "fw/api/coex.h"
+#include "fw/dbg.h"
+#include "fw/uefi.h"
+
+#include "mld.h"
+#include "mlo.h"
+#include "mac80211.h"
+#include "led.h"
+#include "scan.h"
+#include "tx.h"
+#include "sta.h"
+#include "regulatory.h"
+#include "thermal.h"
+#include "low_latency.h"
+#include "hcmd.h"
+#include "fw/api/location.h"
+
+#include "iwl-nvm-parse.h"
+
+#define DRV_DESCRIPTION "Intel(R) MLD wireless driver for Linux"
+MODULE_DESCRIPTION(DRV_DESCRIPTION);
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IWLWIFI");
+
+static const struct iwl_op_mode_ops iwl_mld_ops;
+
+static int __init iwl_mld_init(void)
+{
+	int ret = iwl_opmode_register("iwlmld", &iwl_mld_ops);
+
+	if (ret)
+		pr_err("Unable to register MLD op_mode: %d\n", ret);
+
+	return ret;
+}
+module_init(iwl_mld_init);
+
+static void __exit iwl_mld_exit(void)
+{
+	iwl_opmode_deregister("iwlmld");
+}
+module_exit(iwl_mld_exit);
+
+static void iwl_mld_hw_set_regulatory(struct iwl_mld *mld)
+{
+	struct wiphy *wiphy = mld->wiphy;
+
+	wiphy->regulatory_flags |= REGULATORY_WIPHY_SELF_MANAGED;
+	wiphy->regulatory_flags |= REGULATORY_ENABLE_RELAX_NO_IR;
+}
+
+VISIBLE_IF_IWLWIFI_KUNIT
+void iwl_construct_mld(struct iwl_mld *mld, struct iwl_trans *trans,
+		       const struct iwl_rf_cfg *cfg, const struct iwl_fw *fw,
+		       struct ieee80211_hw *hw, struct dentry *dbgfs_dir)
+{
+	mld->dev = trans->dev;
+	mld->trans = trans;
+	mld->cfg = cfg;
+	mld->fw = fw;
+	mld->hw = hw;
+	mld->wiphy = hw->wiphy;
+	mld->debugfs_dir = dbgfs_dir;
+
+	iwl_notification_wait_init(&mld->notif_wait);
+
+	/* Setup async RX handling */
+	spin_lock_init(&mld->async_handlers_lock);
+	INIT_LIST_HEAD(&mld->async_handlers_list);
+	wiphy_work_init(&mld->async_handlers_wk,
+			iwl_mld_async_handlers_wk);
+
+	/* Dynamic Queue Allocation */
+	spin_lock_init(&mld->add_txqs_lock);
+	INIT_LIST_HEAD(&mld->txqs_to_add);
+	wiphy_work_init(&mld->add_txqs_wk, iwl_mld_add_txqs_wk);
+
+	/* Setup RX queues sync wait queue */
+	init_waitqueue_head(&mld->rxq_sync.waitq);
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_construct_mld);
+
+static void __acquires(&mld->wiphy->mtx)
+iwl_mld_fwrt_dump_start(void *ctx)
+{
+	struct iwl_mld *mld = ctx;
+
+	wiphy_lock(mld->wiphy);
+}
+
+static void __releases(&mld->wiphy->mtx)
+iwl_mld_fwrt_dump_end(void *ctx)
+{
+	struct iwl_mld *mld = ctx;
+
+	wiphy_unlock(mld->wiphy);
+}
+
+static bool iwl_mld_d3_debug_enable(void *ctx)
+{
+	return IWL_MLD_D3_DEBUG;
+}
+
+static int iwl_mld_fwrt_send_hcmd(void *ctx, struct iwl_host_cmd *host_cmd)
+{
+	struct iwl_mld *mld = (struct iwl_mld *)ctx;
+	int ret;
+
+	wiphy_lock(mld->wiphy);
+	ret = iwl_mld_send_cmd(mld, host_cmd);
+	wiphy_unlock(mld->wiphy);
+
+	return ret;
+}
+
+static const struct iwl_fw_runtime_ops iwl_mld_fwrt_ops = {
+	.dump_start = iwl_mld_fwrt_dump_start,
+	.dump_end = iwl_mld_fwrt_dump_end,
+	.send_hcmd = iwl_mld_fwrt_send_hcmd,
+	.d3_debug_enable = iwl_mld_d3_debug_enable,
+};
+
+static void
+iwl_mld_construct_fw_runtime(struct iwl_mld *mld, struct iwl_trans *trans,
+			     const struct iwl_fw *fw,
+			     struct dentry *debugfs_dir)
+{
+	iwl_fw_runtime_init(&mld->fwrt, trans, fw, &iwl_mld_fwrt_ops, mld,
+			    NULL, NULL, debugfs_dir);
+
+	iwl_fw_set_current_image(&mld->fwrt, IWL_UCODE_REGULAR);
+}
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_legacy_names[] = {
+	HCMD_NAME(UCODE_ALIVE_NTFY),
+	HCMD_NAME(INIT_COMPLETE_NOTIF),
+	HCMD_NAME(PHY_CONTEXT_CMD),
+	HCMD_NAME(SCAN_CFG_CMD),
+	HCMD_NAME(SCAN_REQ_UMAC),
+	HCMD_NAME(SCAN_ABORT_UMAC),
+	HCMD_NAME(SCAN_COMPLETE_UMAC),
+	HCMD_NAME(TX_CMD),
+	HCMD_NAME(TXPATH_FLUSH),
+	HCMD_NAME(LEDS_CMD),
+	HCMD_NAME(WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION),
+	HCMD_NAME(WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION),
+	HCMD_NAME(SCAN_OFFLOAD_UPDATE_PROFILES_CMD),
+	HCMD_NAME(POWER_TABLE_CMD),
+	HCMD_NAME(PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION),
+	HCMD_NAME(BEACON_NOTIFICATION),
+	HCMD_NAME(BEACON_TEMPLATE_CMD),
+	HCMD_NAME(TX_ANT_CONFIGURATION_CMD),
+	HCMD_NAME(REDUCE_TX_POWER_CMD),
+	HCMD_NAME(MISSED_BEACONS_NOTIFICATION),
+	HCMD_NAME(MAC_PM_POWER_TABLE),
+	HCMD_NAME(MFUART_LOAD_NOTIFICATION),
+	HCMD_NAME(RSS_CONFIG_CMD),
+	HCMD_NAME(SCAN_ITERATION_COMPLETE_UMAC),
+	HCMD_NAME(REPLY_RX_MPDU_CMD),
+	HCMD_NAME(BA_NOTIF),
+	HCMD_NAME(MCC_UPDATE_CMD),
+	HCMD_NAME(MCC_CHUB_UPDATE_CMD),
+	HCMD_NAME(MCAST_FILTER_CMD),
+	HCMD_NAME(REPLY_BEACON_FILTERING_CMD),
+	HCMD_NAME(PROT_OFFLOAD_CONFIG_CMD),
+	HCMD_NAME(MATCH_FOUND_NOTIFICATION),
+	HCMD_NAME(WOWLAN_PATTERNS),
+	HCMD_NAME(WOWLAN_CONFIGURATION),
+	HCMD_NAME(WOWLAN_TSC_RSC_PARAM),
+	HCMD_NAME(WOWLAN_KEK_KCK_MATERIAL),
+	HCMD_NAME(DEBUG_HOST_COMMAND),
+	HCMD_NAME(LDBG_CONFIG_CMD),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_system_names[] = {
+	HCMD_NAME(SHARED_MEM_CFG_CMD),
+	HCMD_NAME(SOC_CONFIGURATION_CMD),
+	HCMD_NAME(INIT_EXTENDED_CFG_CMD),
+	HCMD_NAME(FW_ERROR_RECOVERY_CMD),
+	HCMD_NAME(RFI_CONFIG_CMD),
+	HCMD_NAME(RFI_GET_FREQ_TABLE_CMD),
+	HCMD_NAME(SYSTEM_STATISTICS_CMD),
+	HCMD_NAME(SYSTEM_STATISTICS_END_NOTIF),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_reg_and_nvm_names[] = {
+	HCMD_NAME(LARI_CONFIG_CHANGE),
+	HCMD_NAME(NVM_GET_INFO),
+	HCMD_NAME(TAS_CONFIG),
+	HCMD_NAME(SAR_OFFSET_MAPPING_TABLE_CMD),
+	HCMD_NAME(MCC_ALLOWED_AP_TYPE_CMD),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_debug_names[] = {
+	HCMD_NAME(HOST_EVENT_CFG),
+	HCMD_NAME(DBGC_SUSPEND_RESUME),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_mac_conf_names[] = {
+	HCMD_NAME(LOW_LATENCY_CMD),
+	HCMD_NAME(SESSION_PROTECTION_CMD),
+	HCMD_NAME(MAC_CONFIG_CMD),
+	HCMD_NAME(LINK_CONFIG_CMD),
+	HCMD_NAME(STA_CONFIG_CMD),
+	HCMD_NAME(AUX_STA_CMD),
+	HCMD_NAME(STA_REMOVE_CMD),
+	HCMD_NAME(ROC_CMD),
+	HCMD_NAME(MISSED_BEACONS_NOTIF),
+	HCMD_NAME(EMLSR_TRANS_FAIL_NOTIF),
+	HCMD_NAME(ROC_NOTIF),
+	HCMD_NAME(CHANNEL_SWITCH_ERROR_NOTIF),
+	HCMD_NAME(SESSION_PROTECTION_NOTIF),
+	HCMD_NAME(PROBE_RESPONSE_DATA_NOTIF),
+	HCMD_NAME(CHANNEL_SWITCH_START_NOTIF),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_data_path_names[] = {
+	HCMD_NAME(TRIGGER_RX_QUEUES_NOTIF_CMD),
+	HCMD_NAME(WNM_PLATFORM_PTM_REQUEST_CMD),
+	HCMD_NAME(WNM_80211V_TIMING_MEASUREMENT_CONFIG_CMD),
+	HCMD_NAME(RFH_QUEUE_CONFIG_CMD),
+	HCMD_NAME(TLC_MNG_CONFIG_CMD),
+	HCMD_NAME(RX_BAID_ALLOCATION_CONFIG_CMD),
+	HCMD_NAME(SCD_QUEUE_CONFIG_CMD),
+	HCMD_NAME(ESR_MODE_NOTIF),
+	HCMD_NAME(MONITOR_NOTIF),
+	HCMD_NAME(TLC_MNG_UPDATE_NOTIF),
+	HCMD_NAME(BEACON_FILTER_IN_NOTIF),
+	HCMD_NAME(MU_GROUP_MGMT_NOTIF),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_scan_names[] = {
+	HCMD_NAME(CHANNEL_SURVEY_NOTIF),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_location_names[] = {
+	HCMD_NAME(TOF_RANGE_REQ_CMD),
+	HCMD_NAME(TOF_RANGE_RESPONSE_NOTIF),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_phy_names[] = {
+	HCMD_NAME(CMD_DTS_MEASUREMENT_TRIGGER_WIDE),
+	HCMD_NAME(CTDP_CONFIG_CMD),
+	HCMD_NAME(TEMP_REPORTING_THRESHOLDS_CMD),
+	HCMD_NAME(PER_CHAIN_LIMIT_OFFSET_CMD),
+	HCMD_NAME(CT_KILL_NOTIFICATION),
+	HCMD_NAME(DTS_MEASUREMENT_NOTIF_WIDE),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_statistics_names[] = {
+	HCMD_NAME(STATISTICS_OPER_NOTIF),
+	HCMD_NAME(STATISTICS_OPER_PART1_NOTIF),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_prot_offload_names[] = {
+	HCMD_NAME(WOWLAN_WAKE_PKT_NOTIFICATION),
+	HCMD_NAME(WOWLAN_INFO_NOTIFICATION),
+	HCMD_NAME(D3_END_NOTIFICATION),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mld_coex_names[] = {
+	HCMD_NAME(PROFILE_NOTIF),
+};
+
+VISIBLE_IF_IWLWIFI_KUNIT
+const struct iwl_hcmd_arr iwl_mld_groups[] = {
+	[LEGACY_GROUP] = HCMD_ARR(iwl_mld_legacy_names),
+	[LONG_GROUP] = HCMD_ARR(iwl_mld_legacy_names),
+	[SYSTEM_GROUP] = HCMD_ARR(iwl_mld_system_names),
+	[MAC_CONF_GROUP] = HCMD_ARR(iwl_mld_mac_conf_names),
+	[DATA_PATH_GROUP] = HCMD_ARR(iwl_mld_data_path_names),
+	[SCAN_GROUP] = HCMD_ARR(iwl_mld_scan_names),
+	[LOCATION_GROUP] = HCMD_ARR(iwl_mld_location_names),
+	[REGULATORY_AND_NVM_GROUP] = HCMD_ARR(iwl_mld_reg_and_nvm_names),
+	[DEBUG_GROUP] = HCMD_ARR(iwl_mld_debug_names),
+	[PHY_OPS_GROUP] = HCMD_ARR(iwl_mld_phy_names),
+	[STATISTICS_GROUP] = HCMD_ARR(iwl_mld_statistics_names),
+	[PROT_OFFLOAD_GROUP] = HCMD_ARR(iwl_mld_prot_offload_names),
+	[BT_COEX_GROUP] = HCMD_ARR(iwl_mld_coex_names),
+};
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_groups);
+
+#if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
+const unsigned int global_iwl_mld_goups_size = ARRAY_SIZE(iwl_mld_groups);
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(global_iwl_mld_goups_size);
+#endif
+
+static void
+iwl_mld_configure_trans(struct iwl_op_mode *op_mode)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+	static const u8 no_reclaim_cmds[] = {TX_CMD};
+	struct iwl_trans *trans = mld->trans;
+	u32 eckv_value;
+
+	iwl_bios_setup_step(trans, &mld->fwrt);
+	iwl_uefi_get_step_table(trans);
+
+	if (iwl_bios_get_eckv(&mld->fwrt, &eckv_value))
+		IWL_DEBUG_RADIO(mld, "ECKV table doesn't exist in BIOS\n");
+	else
+		trans->conf.ext_32khz_clock_valid = !!eckv_value;
+
+	trans->conf.rx_buf_size = iwl_amsdu_size_to_rxb_size();
+	trans->conf.command_groups = iwl_mld_groups;
+	trans->conf.command_groups_size = ARRAY_SIZE(iwl_mld_groups);
+	trans->conf.fw_reset_handshake = true;
+	trans->conf.queue_alloc_cmd_ver =
+		iwl_fw_lookup_cmd_ver(mld->fw, WIDE_ID(DATA_PATH_GROUP,
+						       SCD_QUEUE_CONFIG_CMD),
+				      0);
+	trans->conf.cb_data_offs = offsetof(struct ieee80211_tx_info,
+					    driver_data[2]);
+	BUILD_BUG_ON(sizeof(no_reclaim_cmds) >
+		     sizeof(trans->conf.no_reclaim_cmds));
+	memcpy(trans->conf.no_reclaim_cmds, no_reclaim_cmds,
+	       sizeof(no_reclaim_cmds));
+	trans->conf.n_no_reclaim_cmds = ARRAY_SIZE(no_reclaim_cmds);
+
+	trans->conf.rx_mpdu_cmd = REPLY_RX_MPDU_CMD;
+	trans->conf.rx_mpdu_cmd_hdr_size = sizeof(struct iwl_rx_mpdu_desc);
+	trans->conf.wide_cmd_header = true;
+
+	iwl_trans_op_mode_enter(trans, op_mode);
+}
+
+/*
+ *****************************************************
+ * op mode ops functions
+ *****************************************************
+ */
+
+#define NUM_FW_LOAD_RETRIES	3
+static struct iwl_op_mode *
+iwl_op_mode_mld_start(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
+		      const struct iwl_fw *fw, struct dentry *dbgfs_dir)
+{
+	struct ieee80211_hw *hw;
+	struct iwl_op_mode *op_mode;
+	struct iwl_mld *mld;
+	int ret;
+
+	/* Allocate and initialize a new hardware device */
+	hw = ieee80211_alloc_hw(sizeof(struct iwl_op_mode) +
+				sizeof(struct iwl_mld),
+				&iwl_mld_hw_ops);
+	if (!hw)
+		return ERR_PTR(-ENOMEM);
+
+	op_mode = hw->priv;
+
+	op_mode->ops = &iwl_mld_ops;
+
+	mld = IWL_OP_MODE_GET_MLD(op_mode);
+
+	iwl_construct_mld(mld, trans, cfg, fw, hw, dbgfs_dir);
+
+	/* we'll verify later it matches between commands */
+	mld->fw_rates_ver_3 = iwl_fw_lookup_cmd_ver(mld->fw, TX_CMD, 0) >= 11;
+
+	iwl_mld_construct_fw_runtime(mld, trans, fw, dbgfs_dir);
+
+	iwl_mld_get_bios_tables(mld);
+	iwl_uefi_get_sgom_table(trans, &mld->fwrt);
+	mld->bios_enable_puncturing = iwl_uefi_get_puncturing(&mld->fwrt);
+
+	iwl_mld_hw_set_regulatory(mld);
+
+	/* Configure transport layer with the opmode specific params */
+	iwl_mld_configure_trans(op_mode);
+
+	/* needed for regulatory init */
+	rtnl_lock();
+	/* Needed for sending commands */
+	wiphy_lock(mld->wiphy);
+
+	for (int i = 0; i < NUM_FW_LOAD_RETRIES; i++) {
+		ret = iwl_mld_load_fw(mld);
+		if (!ret)
+			break;
+	}
+
+	if (!ret) {
+		mld->nvm_data = iwl_get_nvm(mld->trans, mld->fw, 0, 0);
+		if (IS_ERR(mld->nvm_data)) {
+			IWL_ERR(mld, "Failed to read NVM: %d\n", ret);
+			ret = PTR_ERR(mld->nvm_data);
+		}
+	}
+
+	if (ret) {
+		wiphy_unlock(mld->wiphy);
+		rtnl_unlock();
+		goto err;
+	}
+
+	/* We are about to stop the FW. Notifications may require an
+	 * operational FW, so handle them all here before we stop.
+	 */
+	wiphy_work_flush(mld->wiphy, &mld->async_handlers_wk);
+
+	iwl_mld_stop_fw(mld);
+
+	wiphy_unlock(mld->wiphy);
+	rtnl_unlock();
+
+	ret = iwl_mld_leds_init(mld);
+	if (ret)
+		goto free_nvm;
+
+	ret = iwl_mld_alloc_scan_cmd(mld);
+	if (ret)
+		goto leds_exit;
+
+	ret = iwl_mld_low_latency_init(mld);
+	if (ret)
+		goto free_scan_cmd;
+
+	ret = iwl_mld_register_hw(mld);
+	if (ret)
+		goto low_latency_free;
+
+	iwl_mld_toggle_tx_ant(mld, &mld->mgmt_tx_ant);
+
+	iwl_mld_add_debugfs_files(mld, dbgfs_dir);
+	iwl_mld_thermal_initialize(mld);
+
+	iwl_mld_ptp_init(mld);
+
+	return op_mode;
+
+low_latency_free:
+	iwl_mld_low_latency_free(mld);
+free_scan_cmd:
+	kfree(mld->scan.cmd);
+leds_exit:
+	iwl_mld_leds_exit(mld);
+free_nvm:
+	kfree(mld->nvm_data);
+err:
+	iwl_trans_op_mode_leave(mld->trans);
+	ieee80211_free_hw(mld->hw);
+	return ERR_PTR(ret);
+}
+
+static void
+iwl_op_mode_mld_stop(struct iwl_op_mode *op_mode)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+
+	iwl_mld_ptp_remove(mld);
+	iwl_mld_leds_exit(mld);
+
+	iwl_mld_thermal_exit(mld);
+
+	wiphy_lock(mld->wiphy);
+	iwl_mld_low_latency_stop(mld);
+	iwl_mld_deinit_time_sync(mld);
+	wiphy_unlock(mld->wiphy);
+
+	ieee80211_unregister_hw(mld->hw);
+
+	iwl_fw_runtime_free(&mld->fwrt);
+	iwl_mld_low_latency_free(mld);
+
+	iwl_trans_op_mode_leave(mld->trans);
+
+	kfree(mld->nvm_data);
+	kfree(mld->scan.cmd);
+	kfree(mld->channel_survey);
+	kfree(mld->error_recovery_buf);
+	kfree(mld->mcast_filter_cmd);
+
+	ieee80211_free_hw(mld->hw);
+}
+
+static void iwl_mld_queue_state_change(struct iwl_op_mode *op_mode,
+				       int hw_queue, bool queue_full)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+	struct ieee80211_txq *txq;
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_txq *mld_txq;
+
+	rcu_read_lock();
+
+	txq = rcu_dereference(mld->fw_id_to_txq[hw_queue]);
+	if (!txq) {
+		rcu_read_unlock();
+
+		if (queue_full) {
+			/* An internal queue is not expected to become full */
+			IWL_WARN(mld,
+				 "Internal hw_queue %d is full! stopping all queues\n",
+				 hw_queue);
+			/* Stop all queues, as an internal queue is not
+			 * mapped to a mac80211 one
+			 */
+			ieee80211_stop_queues(mld->hw);
+		} else {
+			ieee80211_wake_queues(mld->hw);
+		}
+
+		return;
+	}
+
+	mld_txq = iwl_mld_txq_from_mac80211(txq);
+	mld_sta = txq->sta ? iwl_mld_sta_from_mac80211(txq->sta) : NULL;
+
+	mld_txq->status.stop_full = queue_full;
+
+	if (!queue_full && mld_sta &&
+	    mld_sta->sta_state != IEEE80211_STA_NOTEXIST) {
+		local_bh_disable();
+		iwl_mld_tx_from_txq(mld, txq);
+		local_bh_enable();
+	}
+
+	rcu_read_unlock();
+}
+
+static void
+iwl_mld_queue_full(struct iwl_op_mode *op_mode, int hw_queue)
+{
+	iwl_mld_queue_state_change(op_mode, hw_queue, true);
+}
+
+static void
+iwl_mld_queue_not_full(struct iwl_op_mode *op_mode, int hw_queue)
+{
+	iwl_mld_queue_state_change(op_mode, hw_queue, false);
+}
+
+static bool
+iwl_mld_set_hw_rfkill_state(struct iwl_op_mode *op_mode, bool state)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+
+	iwl_mld_set_hwkill(mld, state);
+
+	return false;
+}
+
+static void
+iwl_mld_free_skb(struct iwl_op_mode *op_mode, struct sk_buff *skb)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+	iwl_trans_free_tx_cmd(mld->trans, info->driver_data[1]);
+	ieee80211_free_txskb(mld->hw, skb);
+}
+
+static void iwl_mld_read_error_recovery_buffer(struct iwl_mld *mld)
+{
+	u32 src_size = mld->fw->ucode_capa.error_log_size;
+	u32 src_addr = mld->fw->ucode_capa.error_log_addr;
+	u8 *recovery_buf;
+	int ret;
+
+	/* no recovery buffer size defined in a TLV */
+	if (!src_size)
+		return;
+
+	recovery_buf = kzalloc(src_size, GFP_ATOMIC);
+	if (!recovery_buf)
+		return;
+
+	ret = iwl_trans_read_mem_bytes(mld->trans, src_addr,
+				       recovery_buf, src_size);
+	if (ret) {
+		IWL_ERR(mld, "Failed to read error recovery buffer (%d)\n",
+			ret);
+		kfree(recovery_buf);
+		return;
+	}
+
+	mld->error_recovery_buf = recovery_buf;
+}
+
+static void iwl_mld_restart_nic(struct iwl_mld *mld)
+{
+	iwl_mld_read_error_recovery_buffer(mld);
+
+	mld->fwrt.trans->dbg.restart_required = false;
+
+	ieee80211_restart_hw(mld->hw);
+}
+
+static void
+iwl_mld_nic_error(struct iwl_op_mode *op_mode,
+		  enum iwl_fw_error_type type)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+	bool trans_dead = iwl_trans_is_dead(mld->trans);
+
+	if (type == IWL_ERR_TYPE_CMD_QUEUE_FULL)
+		IWL_ERR(mld, "Command queue full!\n");
+	else if (!trans_dead && !mld->fw_status.do_not_dump_once)
+		iwl_fwrt_dump_error_logs(&mld->fwrt);
+
+	mld->fw_status.do_not_dump_once = false;
+
+	/* It is necessary to abort any os scan here because mac80211 requires
+	 * having the scan cleared before restarting.
+	 * We'll reset the scan_status to NONE in restart cleanup in
+	 * the next drv_start() call from mac80211. If ieee80211_hw_restart
+	 * isn't called scan status will stay busy.
+	 */
+	iwl_mld_report_scan_aborted(mld);
+
+	/*
+	 * This should be first thing before trying to collect any
+	 * data to avoid endless loops if any HW error happens while
+	 * collecting debug data.
+	 * It might not actually be true that we'll restart, but the
+	 * setting doesn't matter if we're going to be unbound either.
+	 */
+	if (type != IWL_ERR_TYPE_RESET_HS_TIMEOUT &&
+	    mld->fw_status.running)
+		mld->fw_status.in_hw_restart = true;
+}
+
+static void iwl_mld_dump_error(struct iwl_op_mode *op_mode,
+			       struct iwl_fw_error_dump_mode *mode)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+
+	/* if we come in from opmode we have the mutex held */
+	if (mode->context == IWL_ERR_CONTEXT_FROM_OPMODE) {
+		lockdep_assert_wiphy(mld->wiphy);
+		iwl_fw_error_collect(&mld->fwrt);
+	} else {
+		wiphy_lock(mld->wiphy);
+		if (mode->context != IWL_ERR_CONTEXT_ABORT)
+			iwl_fw_error_collect(&mld->fwrt);
+		wiphy_unlock(mld->wiphy);
+	}
+}
+
+static bool iwl_mld_sw_reset(struct iwl_op_mode *op_mode,
+			     enum iwl_fw_error_type type)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+
+	/* SW reset can happen for TOP error w/o NIC error, so
+	 * also abort scan here and set in_hw_restart, when we
+	 * had a NIC error both were already done.
+	 */
+	iwl_mld_report_scan_aborted(mld);
+	mld->fw_status.in_hw_restart = true;
+
+	/* Do restart only in the following conditions are met:
+	 * - we consider the FW as running
+	 * - The trigger that brought us here is defined as one that requires
+	 *   a restart (in the debug TLVs)
+	 */
+	if (!mld->fw_status.running || !mld->fwrt.trans->dbg.restart_required)
+		return false;
+
+	iwl_mld_restart_nic(mld);
+	return true;
+}
+
+static void
+iwl_mld_time_point(struct iwl_op_mode *op_mode,
+		   enum iwl_fw_ini_time_point tp_id,
+		   union iwl_dbg_tlv_tp_data *tp_data)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+
+	iwl_dbg_tlv_time_point(&mld->fwrt, tp_id, tp_data);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static void iwl_mld_device_powered_off(struct iwl_op_mode *op_mode)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+
+	wiphy_lock(mld->wiphy);
+	iwl_mld_stop_fw(mld);
+	mld->fw_status.in_d3 = false;
+	wiphy_unlock(mld->wiphy);
+}
+#else
+static void iwl_mld_device_powered_off(struct iwl_op_mode *op_mode)
+{}
+#endif
+
+static void iwl_mld_dump(struct iwl_op_mode *op_mode)
+{
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+	struct iwl_fw_runtime *fwrt = &mld->fwrt;
+
+	if (!iwl_trans_fw_running(fwrt->trans))
+		return;
+
+	iwl_dbg_tlv_time_point(fwrt, IWL_FW_INI_TIME_POINT_USER_TRIGGER, NULL);
+}
+
+static const struct iwl_op_mode_ops iwl_mld_ops = {
+	.start = iwl_op_mode_mld_start,
+	.stop = iwl_op_mode_mld_stop,
+	.rx = iwl_mld_rx,
+	.rx_rss = iwl_mld_rx_rss,
+	.queue_full = iwl_mld_queue_full,
+	.queue_not_full = iwl_mld_queue_not_full,
+	.hw_rf_kill = iwl_mld_set_hw_rfkill_state,
+	.free_skb = iwl_mld_free_skb,
+	.nic_error = iwl_mld_nic_error,
+	.dump_error = iwl_mld_dump_error,
+	.sw_reset = iwl_mld_sw_reset,
+	.time_point = iwl_mld_time_point,
+	.device_powered_off = pm_sleep_ptr(iwl_mld_device_powered_off),
+	.dump = iwl_mld_dump,
+};
+
+struct iwl_mld_mod_params iwlmld_mod_params = {
+	.power_scheme = IWL_POWER_SCHEME_BPS,
+};
+
+module_param_named(power_scheme, iwlmld_mod_params.power_scheme, int, 0444);
+MODULE_PARM_DESC(power_scheme,
+		 "power management scheme: 1-active, 2-balanced, default: 2");
diff --git a/sys/contrib/dev/iwlwifi/mld/mld.h b/sys/contrib/dev/iwlwifi/mld/mld.h
new file mode 100644
index 000000000000..94dc9da6360d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mld.h
@@ -0,0 +1,592 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_h__
+#define __iwl_mld_h__
+
+#include <linux/leds.h>
+#include <net/mac80211.h>
+
+#include "iwl-trans.h"
+#include "iwl-op-mode.h"
+#include "fw/runtime.h"
+#include "fw/notif-wait.h"
+#include "fw/api/commands.h"
+#include "fw/api/scan.h"
+#include "fw/api/mac-cfg.h"
+#include "fw/api/mac.h"
+#include "fw/api/phy-ctxt.h"
+#include "fw/api/datapath.h"
+#include "fw/api/rx.h"
+#include "fw/api/rs.h"
+#include "fw/api/context.h"
+#include "fw/api/coex.h"
+#include "fw/api/location.h"
+
+#include "fw/dbg.h"
+
+#include "notif.h"
+#include "scan.h"
+#include "rx.h"
+#include "thermal.h"
+#include "low_latency.h"
+#include "constants.h"
+#include "ptp.h"
+#include "time_sync.h"
+#include "ftm-initiator.h"
+
+/**
+ * DOC: Introduction
+ *
+ * iwlmld is an operation mode (a.k.a. op_mode) for Intel wireless devices.
+ * It is used for devices that ship after 2024 which typically support
+ * the WiFi-7 features. MLD stands for multi-link device. Note that there are
+ * devices that do not support WiFi-7 or even WiFi 6E and yet use iwlmld, but
+ * the firmware APIs used in this driver are WiFi-7 compatible.
+ *
+ * In the architecture of iwlwifi, an op_mode is a layer that translates
+ * mac80211's APIs into commands for the firmware and, of course, notifications
+ * from the firmware to mac80211's APIs. An op_mode must implement the
+ * interface defined in iwl-op-mode.h to interact with the transport layer
+ * which allows to send and receive data to the device, start the hardware,
+ * etc...
+ */
+
+/**
+ * DOC: Locking policy
+ *
+ * iwlmld has a very simple locking policy: it doesn't have any mutexes. It
+ * relies on cfg80211's wiphy->mtx and takes the lock when needed. All the
+ * control flows originating from mac80211 already acquired the lock, so that
+ * part is trivial, but also notifications that are received from the firmware
+ * and handled asynchronously are handled only after having taken the lock.
+ * This is described in notif.c.
+ * There are spin_locks needed to synchronize with the data path, around the
+ * allocation of the queues, for example.
+ */
+
+/**
+ * DOC: Debugfs
+ *
+ * iwlmld adds its share of debugfs hooks and its handlers are synchronized
+ * with the wiphy_lock using wiphy_locked_debugfs. This avoids races against
+ * resources deletion while the debugfs hook is being used.
+ */
+
+/**
+ * DOC: Main resources
+ *
+ * iwlmld is designed with the life cycle of the resource in mind. The
+ * resources are:
+ *
+ *  - struct iwl_mld (matches mac80211's struct ieee80211_hw)
+ *
+ *  - struct iwl_mld_vif (matches macu80211's struct ieee80211_vif)
+ *    iwl_mld_vif contains an array of pointers to struct iwl_mld_link
+ *    which describe the links for this vif.
+ *
+ *  - struct iwl_mld_sta (matches mac80211's struct ieee80211_sta)
+ *    iwl_mld_sta contains an array of points to struct iwl_mld_link_sta
+ *    which describes the link stations for this station
+ *
+ * Each object has properties that can survive a firmware reset or not.
+ * Asynchronous firmware notifications can declare themselves as dependent on a
+ * certain instance of those resources and that means that the notifications
+ * will be cancelled once the instance is destroyed.
+ */
+
+#define IWL_MLD_MAX_ADDRESSES		5
+
+/**
+ * struct iwl_mld - MLD op mode
+ *
+ * @fw_id_to_bss_conf: maps a fw id of a link to the corresponding
+ *	ieee80211_bss_conf.
+ * @fw_id_to_vif: maps a fw id of a MAC context to the corresponding
+ *	ieee80211_vif. Mapping is valid only when the MAC exists in the fw.
+ * @fw_id_to_txq: maps a fw id of a txq to the corresponding
+ *	ieee80211_txq.
+ * @used_phy_ids: a bitmap of the phy IDs used. If a bit is set, it means
+ *	that the index of this bit is already used as a PHY id.
+ * @num_igtks: the number if iGTKs that were sent to the FW.
+ * @monitor: monitor related data
+ * @monitor.on: does a monitor vif exist (singleton hence bool)
+ * @monitor.ampdu_ref: the id of the A-MPDU for sniffer
+ * @monitor.ampdu_toggle: the state of the previous packet to track A-MPDU
+ * @monitor.cur_aid: current association id tracked by the sniffer
+ * @monitor.cur_bssid: current bssid tracked by the sniffer
+ * @monitor.ptp_time: set the Rx mactime using the device's PTP clock time
+ * @monitor.p80: primary channel position relative to he whole bandwidth, in
+ * steps of 80 MHz
+ * @fw_id_to_link_sta: maps a fw id of a sta to the corresponding
+ *	ieee80211_link_sta. This is not cleaned up on restart since we want to
+ *	preserve the fw sta ids during a restart (for SN/PN restoring).
+ *	FW ids of internal stations will be mapped to ERR_PTR, and will be
+ *	re-allocated during a restart, so make sure to free it in restart
+ *	cleanup using iwl_mld_free_internal_sta
+ * @netdetect: indicates the FW is in suspend mode with netdetect configured
+ * @p2p_device_vif: points to the p2p device vif if exists
+ * @bt_is_active: indicates that BT is active
+ * @dev: pointer to device struct. For printing purposes
+ * @trans: pointer to the transport layer
+ * @cfg: pointer to the device configuration
+ * @fw: a pointer to the fw object
+ * @hw: pointer to the hw object.
+ * @wiphy: a pointer to the wiphy struct, for easier access to it.
+ * @nvm_data: pointer to the nvm_data that includes all our capabilities
+ * @fwrt: fw runtime data
+ * @debugfs_dir: debugfs directory
+ * @notif_wait: notification wait related data.
+ * @async_handlers_list: a list of all async RX handlers. When a notifciation
+ *	with an async handler is received, it is added to this list.
+ *	When &async_handlers_wk runs - it runs these handlers one by one.
+ * @async_handlers_lock: a lock for &async_handlers_list. Sync
+ *	&async_handlers_wk and RX notifcation path.
+ * @async_handlers_wk: A work to run all async RX handlers from
+ *	&async_handlers_list.
+ * @ct_kill_exit_wk: worker to exit thermal kill
+ * @fw_status: bitmap of fw status bits
+ * @running: true if the firmware is running
+ * @do_not_dump_once: true if firmware dump must be prevented once
+ * @in_d3: indicates FW is in suspend mode and should be resumed
+ * @resuming: indicates the driver is resuming from wowlan
+ * @in_hw_restart: indicates that we are currently in restart flow.
+ *	rather than restarted. Should be unset upon restart.
+ * @radio_kill: bitmap of radio kill status
+ * @radio_kill.hw: radio is killed by hw switch
+ * @radio_kill.ct: radio is killed because the device it too hot
+ * @power_budget_mw: maximum cTDP power budget as defined for this system and
+ *	device
+ * @addresses: device MAC addresses.
+ * @scan: instance of the scan object
+ * @channel_survey: channel survey information collected during scan
+ * @wowlan: WoWLAN support data.
+ * @debug_max_sleep: maximum sleep time in D3 (for debug purposes)
+ * @led: the led device
+ * @mcc_src: the source id of the MCC, comes from the firmware
+ * @bios_enable_puncturing: is puncturing enabled by bios
+ * @fw_id_to_ba: maps a fw (BA) id to a corresponding Block Ack session data.
+ * @num_rx_ba_sessions: tracks the number of active Rx Block Ack (BA) sessions.
+ *	the driver ensures that new BA sessions are blocked once the maximum
+ *	supported by the firmware is reached, preventing firmware asserts.
+ * @rxq_sync: manages RX queue sync state
+ * @txqs_to_add: a list of &ieee80211_txq's to allocate in &add_txqs_wk
+ * @add_txqs_wk: a worker to allocate txqs.
+ * @add_txqs_lock: to lock the &txqs_to_add list.
+ * @error_recovery_buf: pointer to the recovery buffer that will be read
+ *	from firmware upon fw/hw error and sent back to the firmware in
+ *	reconfig flow (after NIC reset).
+ * @mcast_filter_cmd: pointer to the multicast filter command.
+ * @mgmt_tx_ant: stores the last TX antenna index; used for setting
+ *	TX rate_n_flags for non-STA mgmt frames (toggles on every TX failure).
+ * @fw_rates_ver_3: FW rates are in version 3
+ * @low_latency: low-latency manager.
+ * @tzone: thermal zone device's data
+ * @cooling_dev: cooling device's related data
+ * @ibss_manager: in IBSS mode (only one vif can be active), indicates what
+ *	firmware indicated about having transmitted the last beacon, i.e.
+ *	being IBSS manager for that time and needing to respond to probe
+ *	requests
+ * @ptp_data: data of the PTP clock
+ * @time_sync: time sync data.
+ * @ftm_initiator: FTM initiator data
+ */
+struct iwl_mld {
+	/* Add here fields that need clean up on restart */
+	struct_group(zeroed_on_hw_restart,
+		struct ieee80211_bss_conf __rcu *fw_id_to_bss_conf[IWL_FW_MAX_LINK_ID + 1];
+		struct ieee80211_vif __rcu *fw_id_to_vif[NUM_MAC_INDEX_DRIVER];
+		struct ieee80211_txq __rcu *fw_id_to_txq[IWL_MAX_TVQM_QUEUES];
+		u8 used_phy_ids: NUM_PHY_CTX;
+		u8 num_igtks;
+		struct {
+			bool on;
+			u32 ampdu_ref;
+			bool ampdu_toggle;
+			u8 p80;
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+			__le16 cur_aid;
+			u8 cur_bssid[ETH_ALEN];
+			bool ptp_time;
+#endif
+		} monitor;
+#ifdef CONFIG_PM_SLEEP
+		bool netdetect;
+#endif /* CONFIG_PM_SLEEP */
+		struct ieee80211_vif *p2p_device_vif;
+		bool bt_is_active;
+	);
+	struct ieee80211_link_sta __rcu *fw_id_to_link_sta[IWL_STATION_COUNT_MAX];
+	/* And here fields that survive a fw restart */
+	struct device *dev;
+	struct iwl_trans *trans;
+	const struct iwl_rf_cfg *cfg;
+	const struct iwl_fw *fw;
+	struct ieee80211_hw *hw;
+	struct wiphy *wiphy;
+	struct iwl_nvm_data *nvm_data;
+	struct iwl_fw_runtime fwrt;
+	struct dentry *debugfs_dir;
+	struct iwl_notif_wait_data notif_wait;
+	struct list_head async_handlers_list;
+	spinlock_t async_handlers_lock;
+	struct wiphy_work async_handlers_wk;
+	struct wiphy_delayed_work ct_kill_exit_wk;
+
+	struct {
+		u32 running:1,
+		    do_not_dump_once:1,
+#ifdef CONFIG_PM_SLEEP
+		    in_d3:1,
+		    resuming:1,
+#endif
+		    in_hw_restart:1;
+
+	} fw_status;
+
+	struct {
+		u32 hw:1,
+		    ct:1;
+	} radio_kill;
+
+	u32 power_budget_mw;
+
+	struct mac_address addresses[IWL_MLD_MAX_ADDRESSES];
+	struct iwl_mld_scan scan;
+	struct iwl_mld_survey *channel_survey;
+#ifdef CONFIG_PM_SLEEP
+	struct wiphy_wowlan_support wowlan;
+	u32 debug_max_sleep;
+#endif /* CONFIG_PM_SLEEP */
+#ifdef CONFIG_IWLWIFI_LEDS
+	struct led_classdev led;
+#endif
+	enum iwl_mcc_source mcc_src;
+	bool bios_enable_puncturing;
+
+	struct iwl_mld_baid_data __rcu *fw_id_to_ba[IWL_MAX_BAID];
+	u8 num_rx_ba_sessions;
+
+	struct iwl_mld_rx_queues_sync rxq_sync;
+
+	struct list_head txqs_to_add;
+	struct wiphy_work add_txqs_wk;
+	spinlock_t add_txqs_lock;
+
+	u8 *error_recovery_buf;
+	struct iwl_mcast_filter_cmd *mcast_filter_cmd;
+
+	u8 mgmt_tx_ant;
+
+	bool fw_rates_ver_3;
+
+	struct iwl_mld_low_latency low_latency;
+
+	bool ibss_manager;
+#ifdef CONFIG_THERMAL
+	struct thermal_zone_device *tzone;
+	struct iwl_mld_cooling_device cooling_dev;
+#endif
+
+	struct ptp_data ptp_data;
+
+	struct iwl_mld_time_sync_data __rcu *time_sync;
+
+	struct ftm_initiator_data ftm_initiator;
+};
+
+/* memset the part of the struct that requires cleanup on restart */
+#define CLEANUP_STRUCT(_ptr)                             \
+	memset((void *)&(_ptr)->zeroed_on_hw_restart, 0, \
+	       sizeof((_ptr)->zeroed_on_hw_restart))
+
+/* Cleanup function for struct iwl_mld, will be called in restart */
+static inline void
+iwl_cleanup_mld(struct iwl_mld *mld)
+{
+	CLEANUP_STRUCT(mld);
+	CLEANUP_STRUCT(&mld->scan);
+
+#ifdef CONFIG_PM_SLEEP
+	mld->fw_status.in_d3 = false;
+#endif
+
+	iwl_mld_low_latency_restart_cleanup(mld);
+}
+
+enum iwl_power_scheme {
+	IWL_POWER_SCHEME_CAM = 1,
+	IWL_POWER_SCHEME_BPS,
+};
+
+/**
+ * struct iwl_mld_mod_params - module parameters for iwlmld
+ * @power_scheme: one of enum iwl_power_scheme
+ */
+struct iwl_mld_mod_params {
+	int power_scheme;
+};
+
+extern struct iwl_mld_mod_params iwlmld_mod_params;
+
+/* Extract MLD priv from op_mode */
+#define IWL_OP_MODE_GET_MLD(_iwl_op_mode)		\
+	((struct iwl_mld *)(_iwl_op_mode)->op_mode_specific)
+
+#define IWL_MAC80211_GET_MLD(_hw)			\
+	IWL_OP_MODE_GET_MLD((struct iwl_op_mode *)((_hw)->priv))
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+void
+iwl_mld_add_debugfs_files(struct iwl_mld *mld, struct dentry *debugfs_dir);
+#else
+static inline void
+iwl_mld_add_debugfs_files(struct iwl_mld *mld, struct dentry *debugfs_dir)
+{}
+#endif
+
+int iwl_mld_load_fw(struct iwl_mld *mld);
+void iwl_mld_stop_fw(struct iwl_mld *mld);
+int iwl_mld_start_fw(struct iwl_mld *mld);
+void iwl_mld_send_recovery_cmd(struct iwl_mld *mld, u32 flags);
+
+static inline void iwl_mld_set_ctkill(struct iwl_mld *mld, bool state)
+{
+	mld->radio_kill.ct = state;
+
+	wiphy_rfkill_set_hw_state(mld->wiphy,
+				  mld->radio_kill.hw || mld->radio_kill.ct);
+}
+
+static inline void iwl_mld_set_hwkill(struct iwl_mld *mld, bool state)
+{
+	mld->radio_kill.hw = state;
+
+	wiphy_rfkill_set_hw_state(mld->wiphy,
+				  mld->radio_kill.hw || mld->radio_kill.ct);
+}
+
+static inline u8 iwl_mld_get_valid_tx_ant(const struct iwl_mld *mld)
+{
+	u8 tx_ant = mld->fw->valid_tx_ant;
+
+	if (mld->nvm_data && mld->nvm_data->valid_tx_ant)
+		tx_ant &= mld->nvm_data->valid_tx_ant;
+
+	return tx_ant;
+}
+
+static inline u8 iwl_mld_get_valid_rx_ant(const struct iwl_mld *mld)
+{
+	u8 rx_ant = mld->fw->valid_rx_ant;
+
+	if (mld->nvm_data && mld->nvm_data->valid_rx_ant)
+		rx_ant &= mld->nvm_data->valid_rx_ant;
+
+	return rx_ant;
+}
+
+static inline u8 iwl_mld_nl80211_band_to_fw(enum nl80211_band band)
+{
+	switch (band) {
+	case NL80211_BAND_2GHZ:
+		return PHY_BAND_24;
+	case NL80211_BAND_5GHZ:
+		return PHY_BAND_5;
+	case NL80211_BAND_6GHZ:
+		return PHY_BAND_6;
+	default:
+		WARN_ONCE(1, "Unsupported band (%u)\n", band);
+		return PHY_BAND_5;
+	}
+}
+
+static inline u8 iwl_mld_phy_band_to_nl80211(u8 phy_band)
+{
+	switch (phy_band) {
+	case PHY_BAND_24:
+		return NL80211_BAND_2GHZ;
+	case PHY_BAND_5:
+		return NL80211_BAND_5GHZ;
+	case PHY_BAND_6:
+		return NL80211_BAND_6GHZ;
+	default:
+		WARN_ONCE(1, "Unsupported phy band (%u)\n", phy_band);
+		return NL80211_BAND_5GHZ;
+	}
+}
+
+static inline int
+iwl_mld_legacy_hw_idx_to_mac80211_idx(u32 rate_n_flags,
+				      enum nl80211_band band)
+{
+	int format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+	int rate = rate_n_flags & RATE_LEGACY_RATE_MSK;
+	bool is_lb = band == NL80211_BAND_2GHZ;
+
+	if (format == RATE_MCS_MOD_TYPE_LEGACY_OFDM)
+		return is_lb ? rate + IWL_FIRST_OFDM_RATE : rate;
+
+	/* CCK is not allowed in 5 GHz */
+	return is_lb ? rate : -1;
+}
+
+extern const struct ieee80211_ops iwl_mld_hw_ops;
+
+/**
+ * enum iwl_rx_handler_context: context for Rx handler
+ * @RX_HANDLER_SYNC: this means that it will be called in the Rx path
+ *	which can't acquire the wiphy->mutex.
+ * @RX_HANDLER_ASYNC: If the handler needs to hold wiphy->mutex
+ *	(and only in this case!), it should be set as ASYNC. In that case,
+ *	it will be called from a worker with wiphy->mutex held.
+ */
+enum iwl_rx_handler_context {
+	RX_HANDLER_SYNC,
+	RX_HANDLER_ASYNC,
+};
+
+/**
+ * struct iwl_rx_handler: handler for FW notification
+ * @val_fn: input validation function.
+ * @sizes: an array that mapps a version to the expected size.
+ * @fn: the function is called when notification is handled
+ * @cmd_id: command id
+ * @n_sizes: number of elements in &sizes.
+ * @context: see &iwl_rx_handler_context
+ * @obj_type: the type of the object that this handler is related to.
+ *	See &iwl_mld_object_type. Use IWL_MLD_OBJECT_TYPE_NONE if not related.
+ * @cancel: function to cancel the notification. valid only if obj_type is not
+ *	IWL_MLD_OBJECT_TYPE_NONE.
+ */
+struct iwl_rx_handler {
+	union {
+		bool (*val_fn)(struct iwl_mld *mld, struct iwl_rx_packet *pkt);
+		const struct iwl_notif_struct_size *sizes;
+	};
+	void (*fn)(struct iwl_mld *mld, struct iwl_rx_packet *pkt);
+	u16 cmd_id;
+	u8 n_sizes;
+	u8 context;
+	enum iwl_mld_object_type obj_type;
+	bool (*cancel)(struct iwl_mld *mld, struct iwl_rx_packet *pkt,
+		       u32 obj_id);
+};
+
+/**
+ * struct iwl_notif_struct_size: map a notif ver to the expected size
+ *
+ * @size: the size to expect
+ * @ver: the version of the notification
+ */
+struct iwl_notif_struct_size {
+	u32 size:24, ver:8;
+};
+
+#if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
+extern const struct iwl_hcmd_arr iwl_mld_groups[];
+extern const unsigned int global_iwl_mld_goups_size;
+extern const struct iwl_rx_handler iwl_mld_rx_handlers[];
+extern const unsigned int iwl_mld_rx_handlers_num;
+
+bool
+iwl_mld_is_dup(struct iwl_mld *mld, struct ieee80211_sta *sta,
+	       struct ieee80211_hdr *hdr,
+	       const struct iwl_rx_mpdu_desc *mpdu_desc,
+	       struct ieee80211_rx_status *rx_status, int queue);
+
+void iwl_construct_mld(struct iwl_mld *mld, struct iwl_trans *trans,
+		       const struct iwl_rf_cfg *cfg, const struct iwl_fw *fw,
+		       struct ieee80211_hw *hw, struct dentry *dbgfs_dir);
+#endif
+
+#define IWL_MLD_INVALID_FW_ID 0xff
+
+#define IWL_MLD_ALLOC_FN(_type, _mac80211_type)						\
+static int										\
+iwl_mld_allocate_##_type##_fw_id(struct iwl_mld *mld,					\
+				 u8 *fw_id,				\
+				 struct ieee80211_##_mac80211_type *mac80211_ptr)	\
+{											\
+	u8 rand = IWL_MLD_DIS_RANDOM_FW_ID ? 0 : get_random_u8();			\
+	u8 arr_sz = ARRAY_SIZE(mld->fw_id_to_##_mac80211_type);				\
+	if (__builtin_types_compatible_p(typeof(*mac80211_ptr),				\
+					 struct ieee80211_link_sta))			\
+		arr_sz = mld->fw->ucode_capa.num_stations;				\
+	if (__builtin_types_compatible_p(typeof(*mac80211_ptr),				\
+					 struct ieee80211_bss_conf))			\
+		arr_sz = mld->fw->ucode_capa.num_links;					\
+	for (int i = 0; i < arr_sz; i++) {						\
+		u8 idx = (i + rand) % arr_sz;						\
+		if (rcu_access_pointer(mld->fw_id_to_##_mac80211_type[idx]))		\
+			continue;							\
+		IWL_DEBUG_INFO(mld, "Allocated at index %d / %d\n", idx, arr_sz);	\
+		*fw_id = idx;								\
+		rcu_assign_pointer(mld->fw_id_to_##_mac80211_type[idx], mac80211_ptr);	\
+		return 0;								\
+	}										\
+	return -ENOSPC;									\
+}
+
+static inline struct ieee80211_bss_conf *
+iwl_mld_fw_id_to_link_conf(struct iwl_mld *mld, u8 fw_link_id)
+{
+	if (IWL_FW_CHECK(mld, fw_link_id >= mld->fw->ucode_capa.num_links,
+			 "Invalid fw_link_id: %d\n", fw_link_id))
+		return NULL;
+
+	return wiphy_dereference(mld->wiphy,
+				 mld->fw_id_to_bss_conf[fw_link_id]);
+}
+
+#define MSEC_TO_TU(_msec)	((_msec) * 1000 / 1024)
+
+void iwl_mld_add_vif_debugfs(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif);
+void iwl_mld_add_link_debugfs(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf,
+			      struct dentry *dir);
+void iwl_mld_add_link_sta_debugfs(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_link_sta *link_sta,
+				  struct dentry *dir);
+
+/* Utilities */
+
+static inline u8 iwl_mld_mac80211_ac_to_fw_tx_fifo(enum ieee80211_ac_numbers ac)
+{
+	static const u8 mac80211_ac_to_fw_tx_fifo[] = {
+		IWL_BZ_EDCA_TX_FIFO_VO,
+		IWL_BZ_EDCA_TX_FIFO_VI,
+		IWL_BZ_EDCA_TX_FIFO_BE,
+		IWL_BZ_EDCA_TX_FIFO_BK,
+		IWL_BZ_TRIG_TX_FIFO_VO,
+		IWL_BZ_TRIG_TX_FIFO_VI,
+		IWL_BZ_TRIG_TX_FIFO_BE,
+		IWL_BZ_TRIG_TX_FIFO_BK,
+	};
+	return mac80211_ac_to_fw_tx_fifo[ac];
+}
+
+static inline u32
+iwl_mld_get_lmac_id(struct iwl_mld *mld, enum nl80211_band band)
+{
+	if (!fw_has_capa(&mld->fw->ucode_capa,
+			 IWL_UCODE_TLV_CAPA_CDB_SUPPORT) ||
+	    band == NL80211_BAND_2GHZ)
+		return IWL_LMAC_24G_INDEX;
+	return IWL_LMAC_5G_INDEX;
+}
+
+/* Check if we had an error, but reconfig flow didn't start yet */
+static inline bool iwl_mld_error_before_recovery(struct iwl_mld *mld)
+{
+	return mld->fw_status.in_hw_restart &&
+		!iwl_trans_fw_running(mld->trans);
+}
+
+int iwl_mld_tdls_sta_count(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/mlo.c b/sys/contrib/dev/iwlwifi/mld/mlo.c
new file mode 100644
index 000000000000..e57f5388fe77
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mlo.c
@@ -0,0 +1,1225 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include "mlo.h"
+#include "phy.h"
+
+/* Block reasons helper */
+#define HANDLE_EMLSR_BLOCKED_REASONS(HOW)	\
+	HOW(PREVENTION)			\
+	HOW(WOWLAN)			\
+	HOW(ROC)			\
+	HOW(NON_BSS)			\
+	HOW(TMP_NON_BSS)		\
+	HOW(TPT)
+
+static const char *
+iwl_mld_get_emlsr_blocked_string(enum iwl_mld_emlsr_blocked blocked)
+{
+	/* Using switch without "default" will warn about missing entries  */
+	switch (blocked) {
+#define REASON_CASE(x) case IWL_MLD_EMLSR_BLOCKED_##x: return #x;
+	HANDLE_EMLSR_BLOCKED_REASONS(REASON_CASE)
+#undef REASON_CASE
+	}
+
+	return "ERROR";
+}
+
+static void iwl_mld_print_emlsr_blocked(struct iwl_mld *mld, u32 mask)
+{
+#define NAME_FMT(x) "%s"
+#define NAME_PR(x) (mask & IWL_MLD_EMLSR_BLOCKED_##x) ? "[" #x "]" : "",
+	IWL_DEBUG_INFO(mld,
+		       "EMLSR blocked = " HANDLE_EMLSR_BLOCKED_REASONS(NAME_FMT)
+		       " (0x%x)\n",
+		       HANDLE_EMLSR_BLOCKED_REASONS(NAME_PR)
+		       mask);
+#undef NAME_FMT
+#undef NAME_PR
+}
+
+/* Exit reasons helper */
+#define HANDLE_EMLSR_EXIT_REASONS(HOW)	\
+	HOW(BLOCK)			\
+	HOW(MISSED_BEACON)		\
+	HOW(FAIL_ENTRY)			\
+	HOW(CSA)			\
+	HOW(EQUAL_BAND)			\
+	HOW(LOW_RSSI)			\
+	HOW(LINK_USAGE)			\
+	HOW(BT_COEX)			\
+	HOW(CHAN_LOAD)			\
+	HOW(RFI)			\
+	HOW(FW_REQUEST)			\
+	HOW(INVALID)
+
+static const char *
+iwl_mld_get_emlsr_exit_string(enum iwl_mld_emlsr_exit exit)
+{
+	/* Using switch without "default" will warn about missing entries  */
+	switch (exit) {
+#define REASON_CASE(x) case IWL_MLD_EMLSR_EXIT_##x: return #x;
+	HANDLE_EMLSR_EXIT_REASONS(REASON_CASE)
+#undef REASON_CASE
+	}
+
+	return "ERROR";
+}
+
+static void iwl_mld_print_emlsr_exit(struct iwl_mld *mld, u32 mask)
+{
+#define NAME_FMT(x) "%s"
+#define NAME_PR(x) (mask & IWL_MLD_EMLSR_EXIT_##x) ? "[" #x "]" : "",
+	IWL_DEBUG_INFO(mld,
+		       "EMLSR exit = " HANDLE_EMLSR_EXIT_REASONS(NAME_FMT)
+		       " (0x%x)\n",
+		       HANDLE_EMLSR_EXIT_REASONS(NAME_PR)
+		       mask);
+#undef NAME_FMT
+#undef NAME_PR
+}
+
+void iwl_mld_emlsr_prevent_done_wk(struct wiphy *wiphy, struct wiphy_work *wk)
+{
+	struct iwl_mld_vif *mld_vif = container_of(wk, struct iwl_mld_vif,
+						   emlsr.prevent_done_wk.work);
+	struct ieee80211_vif *vif =
+		container_of((void *)mld_vif, struct ieee80211_vif, drv_priv);
+
+	if (WARN_ON(!(mld_vif->emlsr.blocked_reasons &
+		      IWL_MLD_EMLSR_BLOCKED_PREVENTION)))
+		return;
+
+	iwl_mld_unblock_emlsr(mld_vif->mld, vif,
+			      IWL_MLD_EMLSR_BLOCKED_PREVENTION);
+}
+
+void iwl_mld_emlsr_tmp_non_bss_done_wk(struct wiphy *wiphy,
+				       struct wiphy_work *wk)
+{
+	struct iwl_mld_vif *mld_vif = container_of(wk, struct iwl_mld_vif,
+						   emlsr.tmp_non_bss_done_wk.work);
+	struct ieee80211_vif *vif =
+		container_of((void *)mld_vif, struct ieee80211_vif, drv_priv);
+
+	if (WARN_ON(!(mld_vif->emlsr.blocked_reasons &
+		      IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS)))
+		return;
+
+	iwl_mld_unblock_emlsr(mld_vif->mld, vif,
+			      IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS);
+}
+
+#define IWL_MLD_TRIGGER_LINK_SEL_TIME	(HZ * IWL_MLD_TRIGGER_LINK_SEL_TIME_SEC)
+#define IWL_MLD_SCAN_EXPIRE_TIME	(HZ * IWL_MLD_SCAN_EXPIRE_TIME_SEC)
+
+/* Exit reasons that can cause longer EMLSR prevention */
+#define IWL_MLD_PREVENT_EMLSR_REASONS	(IWL_MLD_EMLSR_EXIT_MISSED_BEACON	| \
+					 IWL_MLD_EMLSR_EXIT_LINK_USAGE		| \
+					 IWL_MLD_EMLSR_EXIT_FW_REQUEST)
+#define IWL_MLD_PREVENT_EMLSR_TIMEOUT	(HZ * 400)
+
+#define IWL_MLD_EMLSR_PREVENT_SHORT	(HZ * 300)
+#define IWL_MLD_EMLSR_PREVENT_LONG	(HZ * 600)
+
+static void iwl_mld_check_emlsr_prevention(struct iwl_mld *mld,
+					   struct iwl_mld_vif *mld_vif,
+					   enum iwl_mld_emlsr_exit reason)
+{
+	unsigned long delay;
+
+	/*
+	 * Reset the counter if more than 400 seconds have passed between one
+	 * exit and the other, or if we exited due to a different reason.
+	 * Will also reset the counter after the long prevention is done.
+	 */
+	if (time_after(jiffies, mld_vif->emlsr.last_exit_ts +
+				IWL_MLD_PREVENT_EMLSR_TIMEOUT) ||
+	    mld_vif->emlsr.last_exit_reason != reason)
+		mld_vif->emlsr.exit_repeat_count = 0;
+
+	mld_vif->emlsr.last_exit_reason = reason;
+	mld_vif->emlsr.last_exit_ts = jiffies;
+	mld_vif->emlsr.exit_repeat_count++;
+
+	/*
+	 * Do not add a prevention when the reason was a block. For a block,
+	 * EMLSR will be enabled again on unblock.
+	 */
+	if (reason == IWL_MLD_EMLSR_EXIT_BLOCK)
+		return;
+
+	/* Set prevention for a minimum of 30 seconds */
+	mld_vif->emlsr.blocked_reasons |= IWL_MLD_EMLSR_BLOCKED_PREVENTION;
+	delay = IWL_MLD_TRIGGER_LINK_SEL_TIME;
+
+	/* Handle repeats for reasons that can cause long prevention */
+	if (mld_vif->emlsr.exit_repeat_count > 1 &&
+	    reason & IWL_MLD_PREVENT_EMLSR_REASONS) {
+		if (mld_vif->emlsr.exit_repeat_count == 2)
+			delay = IWL_MLD_EMLSR_PREVENT_SHORT;
+		else
+			delay = IWL_MLD_EMLSR_PREVENT_LONG;
+
+		/*
+		 * The timeouts are chosen so that this will not happen, i.e.
+		 * IWL_MLD_EMLSR_PREVENT_LONG > IWL_MLD_PREVENT_EMLSR_TIMEOUT
+		 */
+		WARN_ON(mld_vif->emlsr.exit_repeat_count > 3);
+	}
+
+	IWL_DEBUG_INFO(mld,
+		       "Preventing EMLSR for %ld seconds due to %u exits with the reason = %s (0x%x)\n",
+		       delay / HZ, mld_vif->emlsr.exit_repeat_count,
+		       iwl_mld_get_emlsr_exit_string(reason), reason);
+
+	wiphy_delayed_work_queue(mld->wiphy,
+				 &mld_vif->emlsr.prevent_done_wk, delay);
+}
+
+static void iwl_mld_clear_avg_chan_load_iter(struct ieee80211_hw *hw,
+					     struct ieee80211_chanctx_conf *ctx,
+					     void *dat)
+{
+	struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(ctx);
+
+	/* It is ok to do it for all chanctx (and not only for the ones that
+	 * belong to the EMLSR vif) since EMLSR is not allowed if there is
+	 * another vif.
+	 */
+	phy->avg_channel_load_not_by_us = 0;
+}
+
+static int _iwl_mld_exit_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			       enum iwl_mld_emlsr_exit exit, u8 link_to_keep,
+			       bool sync)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	u16 new_active_links;
+	int ret = 0;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* On entry failure need to exit anyway, even if entered from debugfs */
+	if (exit != IWL_MLD_EMLSR_EXIT_FAIL_ENTRY && !IWL_MLD_AUTO_EML_ENABLE)
+		return 0;
+
+	/* Ignore exit request if EMLSR is not active */
+	if (!iwl_mld_emlsr_active(vif))
+		return 0;
+
+	if (WARN_ON(!ieee80211_vif_is_mld(vif) || !mld_vif->authorized))
+		return 0;
+
+	if (WARN_ON(!(vif->active_links & BIT(link_to_keep))))
+		link_to_keep = __ffs(vif->active_links);
+
+	new_active_links = BIT(link_to_keep);
+	IWL_DEBUG_INFO(mld,
+		       "Exiting EMLSR. reason = %s (0x%x). Current active links=0x%x, new active links = 0x%x\n",
+		       iwl_mld_get_emlsr_exit_string(exit), exit,
+		       vif->active_links, new_active_links);
+
+	if (sync)
+		ret = ieee80211_set_active_links(vif, new_active_links);
+	else
+		ieee80211_set_active_links_async(vif, new_active_links);
+
+	/* Update latest exit reason and check EMLSR prevention */
+	iwl_mld_check_emlsr_prevention(mld, mld_vif, exit);
+
+	/* channel_load_not_by_us is invalid when in EMLSR.
+	 * Clear it so wrong values won't be used.
+	 */
+	ieee80211_iter_chan_contexts_atomic(mld->hw,
+					    iwl_mld_clear_avg_chan_load_iter,
+					    NULL);
+
+	return ret;
+}
+
+void iwl_mld_exit_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			enum iwl_mld_emlsr_exit exit, u8 link_to_keep)
+{
+	_iwl_mld_exit_emlsr(mld, vif, exit, link_to_keep, false);
+}
+
+static int _iwl_mld_emlsr_block(struct iwl_mld *mld, struct ieee80211_vif *vif,
+				enum iwl_mld_emlsr_blocked reason,
+				u8 link_to_keep, bool sync)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (!IWL_MLD_AUTO_EML_ENABLE || !iwl_mld_vif_has_emlsr_cap(vif))
+		return 0;
+
+	if (mld_vif->emlsr.blocked_reasons & reason)
+		return 0;
+
+	mld_vif->emlsr.blocked_reasons |= reason;
+
+	IWL_DEBUG_INFO(mld,
+		       "Blocking EMLSR mode. reason = %s (0x%x)\n",
+		       iwl_mld_get_emlsr_blocked_string(reason), reason);
+	iwl_mld_print_emlsr_blocked(mld, mld_vif->emlsr.blocked_reasons);
+
+	if (reason == IWL_MLD_EMLSR_BLOCKED_TPT)
+		wiphy_delayed_work_cancel(mld_vif->mld->wiphy,
+					  &mld_vif->emlsr.check_tpt_wk);
+
+	return _iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_BLOCK,
+				   link_to_keep, sync);
+}
+
+void iwl_mld_block_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			 enum iwl_mld_emlsr_blocked reason, u8 link_to_keep)
+{
+	_iwl_mld_emlsr_block(mld, vif, reason, link_to_keep, false);
+}
+
+int iwl_mld_block_emlsr_sync(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			     enum iwl_mld_emlsr_blocked reason, u8 link_to_keep)
+{
+	return _iwl_mld_emlsr_block(mld, vif, reason, link_to_keep, true);
+}
+
+#define IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS_TIMEOUT (10 * HZ)
+
+static void iwl_mld_vif_iter_emlsr_block_tmp_non_bss(void *_data, u8 *mac,
+						     struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	if (!iwl_mld_vif_has_emlsr_cap(vif))
+		return;
+
+	ret = iwl_mld_block_emlsr_sync(mld_vif->mld, vif,
+				       IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS,
+				       iwl_mld_get_primary_link(vif));
+	if (ret)
+		return;
+
+	wiphy_delayed_work_queue(mld_vif->mld->wiphy,
+				 &mld_vif->emlsr.tmp_non_bss_done_wk,
+				 IWL_MLD_EMLSR_BLOCKED_TMP_NON_BSS_TIMEOUT);
+}
+
+void iwl_mld_emlsr_block_tmp_non_bss(struct iwl_mld *mld)
+{
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_vif_iter_emlsr_block_tmp_non_bss,
+						NULL);
+}
+
+static void _iwl_mld_select_links(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif);
+
+void iwl_mld_unblock_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			   enum iwl_mld_emlsr_blocked reason)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (!IWL_MLD_AUTO_EML_ENABLE || !iwl_mld_vif_has_emlsr_cap(vif))
+		return;
+
+	if (!(mld_vif->emlsr.blocked_reasons & reason))
+		return;
+
+	mld_vif->emlsr.blocked_reasons &= ~reason;
+
+	IWL_DEBUG_INFO(mld,
+		       "Unblocking EMLSR mode. reason = %s (0x%x)\n",
+		       iwl_mld_get_emlsr_blocked_string(reason), reason);
+	iwl_mld_print_emlsr_blocked(mld, mld_vif->emlsr.blocked_reasons);
+
+	if (reason == IWL_MLD_EMLSR_BLOCKED_TPT)
+		wiphy_delayed_work_queue(mld_vif->mld->wiphy,
+					 &mld_vif->emlsr.check_tpt_wk,
+					 round_jiffies_relative(IWL_MLD_TPT_COUNT_WINDOW));
+
+	if (mld_vif->emlsr.blocked_reasons)
+		return;
+
+	IWL_DEBUG_INFO(mld, "EMLSR is unblocked\n");
+	iwl_mld_int_mlo_scan(mld, vif);
+}
+
+static void
+iwl_mld_vif_iter_emlsr_mode_notif(void *data, u8 *mac,
+				  struct ieee80211_vif *vif)
+{
+	const struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	enum iwl_mvm_fw_esr_recommendation action;
+	const struct iwl_esr_mode_notif *notif = NULL;
+
+	if (iwl_fw_lookup_notif_ver(mld_vif->mld->fw, DATA_PATH_GROUP,
+				    ESR_MODE_NOTIF, 0) > 1) {
+		notif = (void *)data;
+		action = le32_to_cpu(notif->action);
+	} else {
+		const struct iwl_esr_mode_notif_v1 *notif_v1 = (void *)data;
+
+		action = le32_to_cpu(notif_v1->action);
+	}
+
+	if (!iwl_mld_vif_has_emlsr_cap(vif))
+		return;
+
+	switch (action) {
+	case ESR_RECOMMEND_LEAVE:
+		if (notif)
+			IWL_DEBUG_INFO(mld_vif->mld,
+				       "FW recommend leave reason = 0x%x\n",
+				       le32_to_cpu(notif->leave_reason_mask));
+
+		iwl_mld_exit_emlsr(mld_vif->mld, vif,
+				   IWL_MLD_EMLSR_EXIT_FW_REQUEST,
+				   iwl_mld_get_primary_link(vif));
+		break;
+	case ESR_FORCE_LEAVE:
+		if (notif)
+			IWL_DEBUG_INFO(mld_vif->mld,
+				       "FW force leave reason = 0x%x\n",
+				       le32_to_cpu(notif->leave_reason_mask));
+		fallthrough;
+	case ESR_RECOMMEND_ENTER:
+	default:
+		IWL_WARN(mld_vif->mld, "Unexpected EMLSR notification: %d\n",
+			 action);
+	}
+}
+
+void iwl_mld_handle_emlsr_mode_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt)
+{
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_vif_iter_emlsr_mode_notif,
+						pkt->data);
+}
+
+static void
+iwl_mld_vif_iter_disconnect_emlsr(void *data, u8 *mac,
+				  struct ieee80211_vif *vif)
+{
+	if (!iwl_mld_vif_has_emlsr_cap(vif))
+		return;
+
+	ieee80211_connection_loss(vif);
+}
+
+void iwl_mld_handle_emlsr_trans_fail_notif(struct iwl_mld *mld,
+					   struct iwl_rx_packet *pkt)
+{
+	const struct iwl_esr_trans_fail_notif *notif = (const void *)pkt->data;
+	u32 fw_link_id = le32_to_cpu(notif->link_id);
+	struct ieee80211_bss_conf *bss_conf =
+		iwl_mld_fw_id_to_link_conf(mld, fw_link_id);
+
+	IWL_DEBUG_INFO(mld, "Failed to %s EMLSR on link %d (FW: %d), reason %d\n",
+		       le32_to_cpu(notif->activation) ? "enter" : "exit",
+		       bss_conf ? bss_conf->link_id : -1,
+		       le32_to_cpu(notif->link_id),
+		       le32_to_cpu(notif->err_code));
+
+	if (IWL_FW_CHECK(mld, !bss_conf,
+			 "FW reported failure to %sactivate EMLSR on a non-existing link: %d\n",
+			 le32_to_cpu(notif->activation) ? "" : "de",
+			 fw_link_id)) {
+		ieee80211_iterate_active_interfaces_mtx(
+			mld->hw, IEEE80211_IFACE_ITER_NORMAL,
+			iwl_mld_vif_iter_disconnect_emlsr, NULL);
+		return;
+	}
+
+	/* Disconnect if we failed to deactivate a link */
+	if (!le32_to_cpu(notif->activation)) {
+		ieee80211_connection_loss(bss_conf->vif);
+		return;
+	}
+
+	/*
+	 * We failed to activate the second link, go back to the link specified
+	 * by the firmware as that is the one that is still valid now.
+	 */
+	iwl_mld_exit_emlsr(mld, bss_conf->vif, IWL_MLD_EMLSR_EXIT_FAIL_ENTRY,
+			   bss_conf->link_id);
+}
+
+/* Active non-station link tracking */
+static void iwl_mld_count_non_bss_links(void *_data, u8 *mac,
+					struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int *count = _data;
+
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_STATION)
+		return;
+
+	*count += iwl_mld_count_active_links(mld_vif->mld, vif);
+}
+
+struct iwl_mld_update_emlsr_block_data {
+	bool block;
+	int result;
+};
+
+static void
+iwl_mld_vif_iter_update_emlsr_non_bss_block(void *_data, u8 *mac,
+					    struct ieee80211_vif *vif)
+{
+	struct iwl_mld_update_emlsr_block_data *data = _data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	if (data->block) {
+		ret = iwl_mld_block_emlsr_sync(mld_vif->mld, vif,
+					       IWL_MLD_EMLSR_BLOCKED_NON_BSS,
+					       iwl_mld_get_primary_link(vif));
+		if (ret)
+			data->result = ret;
+	} else {
+		iwl_mld_unblock_emlsr(mld_vif->mld, vif,
+				      IWL_MLD_EMLSR_BLOCKED_NON_BSS);
+	}
+}
+
+int iwl_mld_emlsr_check_non_bss_block(struct iwl_mld *mld,
+				      int pending_link_changes)
+{
+	/* An active link of a non-station vif blocks EMLSR. Upon activation
+	 * block EMLSR on the bss vif. Upon deactivation, check if this link
+	 * was the last non-station link active, and if so unblock the bss vif
+	 */
+	struct iwl_mld_update_emlsr_block_data block_data = {};
+	int count = pending_link_changes;
+
+	/* No need to count if we are activating a non-BSS link */
+	if (count <= 0)
+		ieee80211_iterate_active_interfaces_mtx(mld->hw,
+							IEEE80211_IFACE_ITER_NORMAL,
+							iwl_mld_count_non_bss_links,
+							&count);
+
+	/*
+	 * We could skip updating it if the block change did not change (and
+	 * pending_link_changes is non-zero).
+	 */
+	block_data.block = !!count;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_vif_iter_update_emlsr_non_bss_block,
+						&block_data);
+
+	return block_data.result;
+}
+
+#define EMLSR_SEC_LINK_MIN_PERC 10
+#define EMLSR_MIN_TX 3000
+#define EMLSR_MIN_RX 400
+
+void iwl_mld_emlsr_check_tpt(struct wiphy *wiphy, struct wiphy_work *wk)
+{
+	struct iwl_mld_vif *mld_vif = container_of(wk, struct iwl_mld_vif,
+						   emlsr.check_tpt_wk.work);
+	struct ieee80211_vif *vif =
+		container_of((void *)mld_vif, struct ieee80211_vif, drv_priv);
+	struct iwl_mld *mld = mld_vif->mld;
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_link *sec_link;
+	unsigned long total_tx = 0, total_rx = 0;
+	unsigned long sec_link_tx = 0, sec_link_rx = 0;
+	u8 sec_link_tx_perc, sec_link_rx_perc;
+	s8 sec_link_id;
+
+	if (!iwl_mld_vif_has_emlsr_cap(vif) || !mld_vif->ap_sta)
+		return;
+
+	mld_sta = iwl_mld_sta_from_mac80211(mld_vif->ap_sta);
+
+	/* We only count for the AP sta in a MLO connection */
+	if (!mld_sta->mpdu_counters)
+		return;
+
+	/* This wk should only run when the TPT blocker isn't set.
+	 * When the blocker is set, the decision to remove it, as well as
+	 * clearing the counters is done in DP (to avoid having a wk every
+	 * 5 seconds when idle. When the blocker is unset, we are not idle anyway)
+	 */
+	if (WARN_ON(mld_vif->emlsr.blocked_reasons & IWL_MLD_EMLSR_BLOCKED_TPT))
+		return;
+	/*
+	 * TPT is unblocked, need to check if the TPT criteria is still met.
+	 *
+	 * If EMLSR is active for at least 5 seconds, then we also
+	 * need to check the secondary link requirements.
+	 */
+	if (iwl_mld_emlsr_active(vif) &&
+	    time_is_before_jiffies(mld_vif->emlsr.last_entry_ts +
+				   IWL_MLD_TPT_COUNT_WINDOW)) {
+		sec_link_id = iwl_mld_get_other_link(vif, iwl_mld_get_primary_link(vif));
+		sec_link = iwl_mld_link_dereference_check(mld_vif, sec_link_id);
+		if (WARN_ON_ONCE(!sec_link))
+			return;
+		/* We need the FW ID here */
+		sec_link_id = sec_link->fw_id;
+	} else {
+		sec_link_id = -1;
+	}
+
+	/* Sum up RX and TX MPDUs from the different queues/links */
+	for (int q = 0; q < mld->trans->info.num_rxqs; q++) {
+		struct iwl_mld_per_q_mpdu_counter *queue_counter =
+			&mld_sta->mpdu_counters[q];
+
+		spin_lock_bh(&queue_counter->lock);
+
+		/* The link IDs that doesn't exist will contain 0 */
+		for (int link = 0;
+		     link < ARRAY_SIZE(queue_counter->per_link);
+		     link++) {
+			total_tx += queue_counter->per_link[link].tx;
+			total_rx += queue_counter->per_link[link].rx;
+		}
+
+		if (sec_link_id != -1) {
+			sec_link_tx += queue_counter->per_link[sec_link_id].tx;
+			sec_link_rx += queue_counter->per_link[sec_link_id].rx;
+		}
+
+		memset(queue_counter->per_link, 0,
+		       sizeof(queue_counter->per_link));
+
+		spin_unlock_bh(&queue_counter->lock);
+	}
+
+	IWL_DEBUG_INFO(mld, "total Tx MPDUs: %ld. total Rx MPDUs: %ld\n",
+		       total_tx, total_rx);
+
+	/* If we don't have enough MPDUs - exit EMLSR */
+	if (total_tx < IWL_MLD_ENTER_EMLSR_TPT_THRESH &&
+	    total_rx < IWL_MLD_ENTER_EMLSR_TPT_THRESH) {
+		iwl_mld_block_emlsr(mld, vif, IWL_MLD_EMLSR_BLOCKED_TPT,
+				    iwl_mld_get_primary_link(vif));
+		return;
+	}
+
+	/* EMLSR is not active */
+	if (sec_link_id == -1)
+		return;
+
+	IWL_DEBUG_INFO(mld, "Secondary Link %d: Tx MPDUs: %ld. Rx MPDUs: %ld\n",
+		       sec_link_id, sec_link_tx, sec_link_rx);
+
+	/* Calculate the percentage of the secondary link TX/RX */
+	sec_link_tx_perc = total_tx ? sec_link_tx * 100 / total_tx : 0;
+	sec_link_rx_perc = total_rx ? sec_link_rx * 100 / total_rx : 0;
+
+	/*
+	 * The TX/RX percentage is checked only if it exceeds the required
+	 * minimum. In addition, RX is checked only if the TX check failed.
+	 */
+	if ((total_tx > EMLSR_MIN_TX &&
+	     sec_link_tx_perc < EMLSR_SEC_LINK_MIN_PERC) ||
+	    (total_rx > EMLSR_MIN_RX &&
+	     sec_link_rx_perc < EMLSR_SEC_LINK_MIN_PERC)) {
+		iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_LINK_USAGE,
+				   iwl_mld_get_primary_link(vif));
+		return;
+	}
+
+	/* Check again when the next window ends  */
+	wiphy_delayed_work_queue(mld_vif->mld->wiphy,
+				 &mld_vif->emlsr.check_tpt_wk,
+				 round_jiffies_relative(IWL_MLD_TPT_COUNT_WINDOW));
+}
+
+void iwl_mld_emlsr_unblock_tpt_wk(struct wiphy *wiphy, struct wiphy_work *wk)
+{
+	struct iwl_mld_vif *mld_vif = container_of(wk, struct iwl_mld_vif,
+						   emlsr.unblock_tpt_wk);
+	struct ieee80211_vif *vif =
+		container_of((void *)mld_vif, struct ieee80211_vif, drv_priv);
+
+	iwl_mld_unblock_emlsr(mld_vif->mld, vif, IWL_MLD_EMLSR_BLOCKED_TPT);
+}
+
+/*
+ * Link selection
+ */
+
+s8 iwl_mld_get_emlsr_rssi_thresh(struct iwl_mld *mld,
+				 const struct cfg80211_chan_def *chandef,
+				 bool low)
+{
+	if (WARN_ON(chandef->chan->band != NL80211_BAND_2GHZ &&
+		    chandef->chan->band != NL80211_BAND_5GHZ &&
+		    chandef->chan->band != NL80211_BAND_6GHZ))
+		return S8_MAX;
+
+#define RSSI_THRESHOLD(_low, _bw)			\
+	(_low) ? IWL_MLD_LOW_RSSI_THRESH_##_bw##MHZ	\
+	       : IWL_MLD_HIGH_RSSI_THRESH_##_bw##MHZ
+
+	switch (chandef->width) {
+	case NL80211_CHAN_WIDTH_20_NOHT:
+	case NL80211_CHAN_WIDTH_20:
+	/* 320 MHz has the same thresholds as 20 MHz */
+	case NL80211_CHAN_WIDTH_320:
+		return RSSI_THRESHOLD(low, 20);
+	case NL80211_CHAN_WIDTH_40:
+		return RSSI_THRESHOLD(low, 40);
+	case NL80211_CHAN_WIDTH_80:
+		return RSSI_THRESHOLD(low, 80);
+	case NL80211_CHAN_WIDTH_160:
+		return RSSI_THRESHOLD(low, 160);
+	default:
+		WARN_ON(1);
+		return S8_MAX;
+	}
+#undef RSSI_THRESHOLD
+}
+
+static u32
+iwl_mld_emlsr_disallowed_with_link(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct iwl_mld_link_sel_data *link,
+				   bool primary)
+{
+	struct wiphy *wiphy = mld->wiphy;
+	struct ieee80211_bss_conf *conf;
+	u32 ret = 0;
+
+	conf = wiphy_dereference(wiphy, vif->link_conf[link->link_id]);
+	if (WARN_ON_ONCE(!conf))
+		return IWL_MLD_EMLSR_EXIT_INVALID;
+
+	if (link->chandef->chan->band == NL80211_BAND_2GHZ && mld->bt_is_active)
+		ret |= IWL_MLD_EMLSR_EXIT_BT_COEX;
+
+	if (link->signal <
+	    iwl_mld_get_emlsr_rssi_thresh(mld, link->chandef, false))
+		ret |= IWL_MLD_EMLSR_EXIT_LOW_RSSI;
+
+	if (conf->csa_active)
+		ret |= IWL_MLD_EMLSR_EXIT_CSA;
+
+	if (ret) {
+		IWL_DEBUG_INFO(mld,
+			       "Link %d is not allowed for EMLSR as %s\n",
+			       link->link_id,
+			       primary ? "primary" : "secondary");
+		iwl_mld_print_emlsr_exit(mld, ret);
+	}
+
+	return ret;
+}
+
+static u8
+iwl_mld_set_link_sel_data(struct iwl_mld *mld,
+			  struct ieee80211_vif *vif,
+			  struct iwl_mld_link_sel_data *data,
+			  unsigned long usable_links,
+			  u8 *best_link_idx)
+{
+	u8 n_data = 0;
+	u16 max_grade = 0;
+	unsigned long link_id;
+
+	/*
+	 * TODO: don't select links that weren't discovered in the last scan
+	 * This requires mac80211 (or cfg80211) changes to forward/track when
+	 * a BSS was last updated. cfg80211 already tracks this information but
+	 * it is not exposed within the kernel.
+	 */
+	for_each_set_bit(link_id, &usable_links, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, link_id);
+
+		if (WARN_ON_ONCE(!link_conf))
+			continue;
+
+		/* Ignore any BSS that was not seen in the last MLO scan */
+		if (ktime_before(link_conf->bss->ts_boottime,
+				 mld->scan.last_mlo_scan_time))
+			continue;
+
+		data[n_data].link_id = link_id;
+		data[n_data].chandef = &link_conf->chanreq.oper;
+		data[n_data].signal = MBM_TO_DBM(link_conf->bss->signal);
+		data[n_data].grade = iwl_mld_get_link_grade(mld, link_conf);
+
+		if (n_data == 0 || data[n_data].grade > max_grade) {
+			max_grade = data[n_data].grade;
+			*best_link_idx = n_data;
+		}
+		n_data++;
+	}
+
+	return n_data;
+}
+
+static u32
+iwl_mld_get_min_chan_load_thresh(struct ieee80211_chanctx_conf *chanctx)
+{
+	const struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(chanctx);
+
+	switch (phy->chandef.width) {
+	case NL80211_CHAN_WIDTH_320:
+	case NL80211_CHAN_WIDTH_160:
+		return 5;
+	case NL80211_CHAN_WIDTH_80:
+		return 7;
+	default:
+		break;
+	}
+	return 10;
+}
+
+static bool
+iwl_mld_channel_load_allows_emlsr(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif,
+				  const struct iwl_mld_link_sel_data *a,
+				  const struct iwl_mld_link_sel_data *b)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *link_a =
+		iwl_mld_link_dereference_check(mld_vif, a->link_id);
+	struct ieee80211_chanctx_conf *chanctx_a = NULL;
+	u32 bw_a, bw_b, ratio;
+	u32 primary_load_perc;
+
+	if (!link_a || !link_a->active) {
+		IWL_DEBUG_EHT(mld, "Primary link is not active. Can't enter EMLSR\n");
+		return false;
+	}
+
+	chanctx_a = wiphy_dereference(mld->wiphy, link_a->chan_ctx);
+
+	if (WARN_ON(!chanctx_a))
+		return false;
+
+	primary_load_perc =
+		iwl_mld_phy_from_mac80211(chanctx_a)->avg_channel_load_not_by_us;
+
+	IWL_DEBUG_EHT(mld, "Average channel load not by us: %u\n", primary_load_perc);
+
+	if (primary_load_perc < iwl_mld_get_min_chan_load_thresh(chanctx_a)) {
+		IWL_DEBUG_EHT(mld, "Channel load is below the minimum threshold\n");
+		return false;
+	}
+
+	if (iwl_mld_vif_low_latency(mld_vif)) {
+		IWL_DEBUG_EHT(mld, "Low latency vif, EMLSR is allowed\n");
+		return true;
+	}
+
+	if (a->chandef->width <= b->chandef->width)
+		return true;
+
+	bw_a = cfg80211_chandef_get_width(a->chandef);
+	bw_b = cfg80211_chandef_get_width(b->chandef);
+	ratio = bw_a / bw_b;
+
+	switch (ratio) {
+	case 2:
+		return primary_load_perc > 25;
+	case 4:
+		return primary_load_perc > 40;
+	case 8:
+	case 16:
+		return primary_load_perc > 50;
+	}
+
+	return false;
+}
+
+VISIBLE_IF_IWLWIFI_KUNIT u32
+iwl_mld_emlsr_pair_state(struct ieee80211_vif *vif,
+			 struct iwl_mld_link_sel_data *a,
+			 struct iwl_mld_link_sel_data *b)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld *mld = mld_vif->mld;
+	u32 reason_mask = 0;
+
+	/* Per-link considerations */
+	reason_mask = iwl_mld_emlsr_disallowed_with_link(mld, vif, a, true);
+	if (reason_mask)
+		return reason_mask;
+
+	reason_mask = iwl_mld_emlsr_disallowed_with_link(mld, vif, b, false);
+	if (reason_mask)
+		return reason_mask;
+
+	if (a->chandef->chan->band == b->chandef->chan->band) {
+		const struct cfg80211_chan_def *c_low = a->chandef;
+		const struct cfg80211_chan_def *c_high = b->chandef;
+		u32 c_low_upper_edge, c_high_lower_edge;
+
+		if (c_low->chan->center_freq > c_high->chan->center_freq)
+			swap(c_low, c_high);
+
+		c_low_upper_edge = c_low->chan->center_freq +
+				   cfg80211_chandef_get_width(c_low) / 2;
+		c_high_lower_edge = c_high->chan->center_freq -
+				    cfg80211_chandef_get_width(c_high) / 2;
+
+		if (a->chandef->chan->band == NL80211_BAND_5GHZ &&
+		    c_low_upper_edge <= 5330 && c_high_lower_edge >= 5490) {
+			/* This case is fine - HW/FW can deal with it, there's
+			 * enough separation between the two channels.
+			 */
+		} else {
+			reason_mask |= IWL_MLD_EMLSR_EXIT_EQUAL_BAND;
+		}
+	}
+	if (!iwl_mld_channel_load_allows_emlsr(mld, vif, a, b))
+		reason_mask |= IWL_MLD_EMLSR_EXIT_CHAN_LOAD;
+
+	if (reason_mask) {
+		IWL_DEBUG_INFO(mld,
+			       "Links %d and %d are not a valid pair for EMLSR\n",
+			       a->link_id, b->link_id);
+		IWL_DEBUG_INFO(mld,
+			       "Links bandwidth are: %d and %d\n",
+			       nl80211_chan_width_to_mhz(a->chandef->width),
+			       nl80211_chan_width_to_mhz(b->chandef->width));
+		iwl_mld_print_emlsr_exit(mld, reason_mask);
+	}
+
+	return reason_mask;
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_emlsr_pair_state);
+
+/* Calculation is done with fixed-point with a scaling factor of 1/256 */
+#define SCALE_FACTOR 256
+
+/*
+ * Returns the combined grade of two given links.
+ * Returns 0 if EMLSR is not allowed with these 2 links.
+ */
+static
+unsigned int iwl_mld_get_emlsr_grade(struct iwl_mld *mld,
+				     struct ieee80211_vif *vif,
+				     struct iwl_mld_link_sel_data *a,
+				     struct iwl_mld_link_sel_data *b,
+				     u8 *primary_id)
+{
+	struct ieee80211_bss_conf *primary_conf;
+	struct wiphy *wiphy = ieee80211_vif_to_wdev(vif)->wiphy;
+	unsigned int primary_load;
+
+	lockdep_assert_wiphy(wiphy);
+
+	/* a is always primary, b is always secondary */
+	if (b->grade > a->grade)
+		swap(a, b);
+
+	*primary_id = a->link_id;
+
+	if (iwl_mld_emlsr_pair_state(vif, a, b))
+		return 0;
+
+	primary_conf = wiphy_dereference(wiphy, vif->link_conf[*primary_id]);
+
+	if (WARN_ON_ONCE(!primary_conf))
+		return 0;
+
+	primary_load = iwl_mld_get_chan_load(mld, primary_conf);
+
+	/* The more the primary link is loaded, the more worthwhile EMLSR becomes */
+	return a->grade + ((b->grade * primary_load) / SCALE_FACTOR);
+}
+
+static void _iwl_mld_select_links(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif)
+{
+	struct iwl_mld_link_sel_data data[IEEE80211_MLD_MAX_NUM_LINKS];
+	struct iwl_mld_link_sel_data *best_link;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int max_active_links = iwl_mld_max_active_links(mld, vif);
+	u16 new_active, usable_links = ieee80211_vif_usable_links(vif);
+	u8 best_idx, new_primary, n_data;
+	u16 max_grade;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (!mld_vif->authorized || hweight16(usable_links) <= 1)
+		return;
+
+	if (WARN(ktime_before(mld->scan.last_mlo_scan_time,
+			      ktime_sub_ns(ktime_get_boottime_ns(),
+					   5ULL * NSEC_PER_SEC)),
+		"Last MLO scan was too long ago, can't select links\n"))
+		return;
+
+	/* The logic below is simple and not suited for more than 2 links */
+	WARN_ON_ONCE(max_active_links > 2);
+
+	n_data = iwl_mld_set_link_sel_data(mld, vif, data, usable_links,
+					   &best_idx);
+
+	if (!n_data) {
+		IWL_DEBUG_EHT(mld,
+			      "Couldn't find a valid grade for any link!\n");
+		return;
+	}
+
+	/* Default to selecting the single best link */
+	best_link = &data[best_idx];
+	new_primary = best_link->link_id;
+	new_active = BIT(best_link->link_id);
+	max_grade = best_link->grade;
+
+	/* If EMLSR is not possible, activate the best link */
+	if (max_active_links == 1 || n_data == 1 ||
+	    !iwl_mld_vif_has_emlsr_cap(vif) || !IWL_MLD_AUTO_EML_ENABLE ||
+	    mld_vif->emlsr.blocked_reasons)
+		goto set_active;
+
+	/* Try to find the best link combination */
+	for (u8 a = 0; a < n_data; a++) {
+		for (u8 b = a + 1; b < n_data; b++) {
+			u8 best_in_pair;
+			u16 emlsr_grade =
+				iwl_mld_get_emlsr_grade(mld, vif,
+							&data[a], &data[b],
+							&best_in_pair);
+
+			/*
+			 * Prefer (new) EMLSR combination to prefer EMLSR over
+			 * a single link.
+			 */
+			if (emlsr_grade < max_grade)
+				continue;
+
+			max_grade = emlsr_grade;
+			new_primary = best_in_pair;
+			new_active = BIT(data[a].link_id) |
+				     BIT(data[b].link_id);
+		}
+	}
+
+set_active:
+	IWL_DEBUG_INFO(mld, "Link selection result: 0x%x. Primary = %d\n",
+		       new_active, new_primary);
+
+	mld_vif->emlsr.selected_primary = new_primary;
+	mld_vif->emlsr.selected_links = new_active;
+
+	ieee80211_set_active_links_async(vif, new_active);
+}
+
+static void iwl_mld_vif_iter_select_links(void *_data, u8 *mac,
+					  struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld *mld = mld_vif->mld;
+
+	_iwl_mld_select_links(mld, vif);
+}
+
+void iwl_mld_select_links(struct iwl_mld *mld)
+{
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_vif_iter_select_links,
+						NULL);
+}
+
+static void iwl_mld_emlsr_check_bt_iter(void *_data, u8 *mac,
+					struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld *mld = mld_vif->mld;
+	struct ieee80211_bss_conf *link;
+	unsigned int link_id;
+
+	if (!iwl_mld_vif_has_emlsr_cap(vif))
+		return;
+
+	if (!mld->bt_is_active) {
+		iwl_mld_retry_emlsr(mld, vif);
+		return;
+	}
+
+	/* BT is turned ON but we are not in EMLSR, nothing to do */
+	if (!iwl_mld_emlsr_active(vif))
+		return;
+
+	/* In EMLSR and BT is turned ON */
+
+	for_each_vif_active_link(vif, link, link_id) {
+		if (WARN_ON(!link->chanreq.oper.chan))
+			continue;
+
+		if (link->chanreq.oper.chan->band == NL80211_BAND_2GHZ) {
+			iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_BT_COEX,
+					   iwl_mld_get_primary_link(vif));
+			return;
+		}
+	}
+}
+
+void iwl_mld_emlsr_check_bt(struct iwl_mld *mld)
+{
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_emlsr_check_bt_iter,
+						NULL);
+}
+
+struct iwl_mld_chan_load_data {
+	struct iwl_mld_phy *phy;
+	u32 prev_chan_load_not_by_us;
+};
+
+static void iwl_mld_chan_load_update_iter(void *_data, u8 *mac,
+					  struct ieee80211_vif *vif)
+{
+	struct iwl_mld_chan_load_data *data = _data;
+	const struct iwl_mld_phy *phy = data->phy;
+	struct ieee80211_chanctx_conf *chanctx =
+		container_of((const void *)phy, struct ieee80211_chanctx_conf,
+			     drv_priv);
+	struct iwl_mld *mld = iwl_mld_vif_from_mac80211(vif)->mld;
+	struct ieee80211_bss_conf *prim_link;
+	unsigned int prim_link_id;
+
+	prim_link_id = iwl_mld_get_primary_link(vif);
+	prim_link = link_conf_dereference_protected(vif, prim_link_id);
+
+	if (WARN_ON(!prim_link))
+		return;
+
+	if (chanctx != rcu_access_pointer(prim_link->chanctx_conf))
+		return;
+
+	if (iwl_mld_emlsr_active(vif)) {
+		int chan_load = iwl_mld_get_chan_load_by_others(mld, prim_link,
+								true);
+
+		if (chan_load < 0)
+			return;
+
+		/* chan_load is in range [0,255] */
+		if (chan_load < NORMALIZE_PERCENT_TO_255(IWL_MLD_EXIT_EMLSR_CHAN_LOAD))
+			iwl_mld_exit_emlsr(mld, vif,
+					   IWL_MLD_EMLSR_EXIT_CHAN_LOAD,
+					   prim_link_id);
+	} else {
+		u32 old_chan_load = data->prev_chan_load_not_by_us;
+		u32 new_chan_load = phy->avg_channel_load_not_by_us;
+		u32 min_thresh = iwl_mld_get_min_chan_load_thresh(chanctx);
+
+#define THRESHOLD_CROSSED(threshold) \
+	(old_chan_load <= (threshold) && new_chan_load > (threshold))
+
+		if (THRESHOLD_CROSSED(min_thresh) || THRESHOLD_CROSSED(25) ||
+		    THRESHOLD_CROSSED(40) || THRESHOLD_CROSSED(50))
+			iwl_mld_retry_emlsr(mld, vif);
+#undef THRESHOLD_CROSSED
+	}
+}
+
+void iwl_mld_emlsr_check_chan_load(struct ieee80211_hw *hw,
+				   struct iwl_mld_phy *phy,
+				   u32 prev_chan_load_not_by_us)
+{
+	struct iwl_mld_chan_load_data data = {
+		.phy = phy,
+		.prev_chan_load_not_by_us = prev_chan_load_not_by_us,
+	};
+
+	ieee80211_iterate_active_interfaces_mtx(hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_chan_load_update_iter,
+						&data);
+}
+
+void iwl_mld_retry_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (!IWL_MLD_AUTO_EML_ENABLE || !iwl_mld_vif_has_emlsr_cap(vif) ||
+	    iwl_mld_emlsr_active(vif) || mld_vif->emlsr.blocked_reasons)
+		return;
+
+	iwl_mld_int_mlo_scan(mld, vif);
+}
+
+static void iwl_mld_ignore_tpt_iter(void *data, u8 *mac,
+				    struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld *mld = mld_vif->mld;
+	struct iwl_mld_sta *mld_sta;
+	bool *start = (void *)data;
+
+	/* check_tpt_wk is only used when TPT block isn't set */
+	if (mld_vif->emlsr.blocked_reasons & IWL_MLD_EMLSR_BLOCKED_TPT ||
+	    !IWL_MLD_AUTO_EML_ENABLE || !mld_vif->ap_sta)
+		return;
+
+	mld_sta = iwl_mld_sta_from_mac80211(mld_vif->ap_sta);
+
+	/* We only count for the AP sta in a MLO connection */
+	if (!mld_sta->mpdu_counters)
+		return;
+
+	if (*start) {
+		wiphy_delayed_work_cancel(mld_vif->mld->wiphy,
+					  &mld_vif->emlsr.check_tpt_wk);
+		IWL_DEBUG_EHT(mld, "TPT check disabled\n");
+		return;
+	}
+
+	/* Clear the counters so we start from the beginning */
+	for (int q = 0; q < mld->trans->info.num_rxqs; q++) {
+		struct iwl_mld_per_q_mpdu_counter *queue_counter =
+			&mld_sta->mpdu_counters[q];
+
+		spin_lock_bh(&queue_counter->lock);
+
+		memset(queue_counter->per_link, 0,
+		       sizeof(queue_counter->per_link));
+
+		spin_unlock_bh(&queue_counter->lock);
+	}
+
+	/* Schedule the check in 5 seconds */
+	wiphy_delayed_work_queue(mld_vif->mld->wiphy,
+				 &mld_vif->emlsr.check_tpt_wk,
+				 round_jiffies_relative(IWL_MLD_TPT_COUNT_WINDOW));
+	IWL_DEBUG_EHT(mld, "TPT check enabled\n");
+}
+
+void iwl_mld_start_ignoring_tpt_updates(struct iwl_mld *mld)
+{
+	bool start = true;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_ignore_tpt_iter,
+						&start);
+}
+
+void iwl_mld_stop_ignoring_tpt_updates(struct iwl_mld *mld)
+{
+	bool start = false;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_ignore_tpt_iter,
+						&start);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/mlo.h b/sys/contrib/dev/iwlwifi/mld/mlo.h
new file mode 100644
index 000000000000..d936589fe39d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/mlo.h
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_mlo_h__
+#define __iwl_mld_mlo_h__
+
+#include <linux/ieee80211.h>
+#include <linux/types.h>
+#include <net/mac80211.h>
+#include "iwl-config.h"
+#include "iwl-trans.h"
+#include "iface.h"
+#include "phy.h"
+
+struct iwl_mld;
+
+void iwl_mld_emlsr_prevent_done_wk(struct wiphy *wiphy, struct wiphy_work *wk);
+void iwl_mld_emlsr_tmp_non_bss_done_wk(struct wiphy *wiphy,
+				       struct wiphy_work *wk);
+
+static inline bool iwl_mld_emlsr_active(struct ieee80211_vif *vif)
+{
+	/* Set on phy context activation, so should be a good proxy */
+	return !!(vif->driver_flags & IEEE80211_VIF_EML_ACTIVE);
+}
+
+static inline bool iwl_mld_vif_has_emlsr_cap(struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	/* We only track/permit EMLSR state once authorized */
+	if (!mld_vif->authorized)
+		return false;
+
+	/* No EMLSR on dual radio devices */
+	return ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_STATION &&
+	       ieee80211_vif_is_mld(vif) &&
+	       vif->cfg.eml_cap & IEEE80211_EML_CAP_EMLSR_SUPP &&
+	       !CSR_HW_RFID_IS_CDB(mld_vif->mld->trans->info.hw_rf_id);
+}
+
+static inline int
+iwl_mld_max_active_links(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	if (vif->type == NL80211_IFTYPE_AP)
+		return mld->fw->ucode_capa.num_beacons;
+
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_STATION)
+		return IWL_FW_MAX_ACTIVE_LINKS_NUM;
+
+	/* For now, do not accept more links on other interface types */
+	return 1;
+}
+
+static inline int
+iwl_mld_count_active_links(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *mld_link;
+	int n_active = 0;
+
+	for_each_mld_vif_valid_link(mld_vif, mld_link) {
+		if (rcu_access_pointer(mld_link->chan_ctx))
+			n_active++;
+	}
+
+	return n_active;
+}
+
+static inline u8 iwl_mld_get_primary_link(struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	lockdep_assert_wiphy(mld_vif->mld->wiphy);
+
+	if (!ieee80211_vif_is_mld(vif) || WARN_ON(!vif->active_links))
+		return 0;
+
+	/* In AP mode, there is no primary link */
+	if (vif->type == NL80211_IFTYPE_AP)
+		return __ffs(vif->active_links);
+
+	if (iwl_mld_emlsr_active(vif) &&
+	    !WARN_ON(!(BIT(mld_vif->emlsr.primary) & vif->active_links)))
+		return mld_vif->emlsr.primary;
+
+	return __ffs(vif->active_links);
+}
+
+/*
+ * For non-MLO/single link, this will return the deflink/single active link,
+ * respectively
+ */
+static inline u8 iwl_mld_get_other_link(struct ieee80211_vif *vif, u8 link_id)
+{
+	switch (hweight16(vif->active_links)) {
+	case 0:
+		return 0;
+	default:
+		WARN_ON(1);
+		fallthrough;
+	case 1:
+		return __ffs(vif->active_links);
+	case 2:
+		return __ffs(vif->active_links & ~BIT(link_id));
+	}
+}
+
+s8 iwl_mld_get_emlsr_rssi_thresh(struct iwl_mld *mld,
+				 const struct cfg80211_chan_def *chandef,
+				 bool low);
+
+/* EMLSR block/unblock and exit */
+void iwl_mld_block_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			 enum iwl_mld_emlsr_blocked reason, u8 link_to_keep);
+int iwl_mld_block_emlsr_sync(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			     enum iwl_mld_emlsr_blocked reason, u8 link_to_keep);
+void iwl_mld_unblock_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			   enum iwl_mld_emlsr_blocked reason);
+void iwl_mld_exit_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			enum iwl_mld_emlsr_exit exit, u8 link_to_keep);
+
+int iwl_mld_emlsr_check_non_bss_block(struct iwl_mld *mld,
+				      int pending_link_changes);
+
+void iwl_mld_handle_emlsr_mode_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt);
+void iwl_mld_handle_emlsr_trans_fail_notif(struct iwl_mld *mld,
+					   struct iwl_rx_packet *pkt);
+
+void iwl_mld_emlsr_check_tpt(struct wiphy *wiphy, struct wiphy_work *wk);
+void iwl_mld_emlsr_unblock_tpt_wk(struct wiphy *wiphy, struct wiphy_work *wk);
+
+void iwl_mld_select_links(struct iwl_mld *mld);
+
+void iwl_mld_emlsr_check_bt(struct iwl_mld *mld);
+
+void iwl_mld_emlsr_check_chan_load(struct ieee80211_hw *hw,
+				   struct iwl_mld_phy *phy,
+				   u32 prev_chan_load_not_by_us);
+
+/**
+ * iwl_mld_retry_emlsr - Retry entering EMLSR
+ * @mld: MLD context
+ * @vif: VIF to retry EMLSR on
+ *
+ * Retry entering EMLSR on the given VIF.
+ * Use this if one of the parameters that can prevent EMLSR has changed.
+ */
+void iwl_mld_retry_emlsr(struct iwl_mld *mld, struct ieee80211_vif *vif);
+
+struct iwl_mld_link_sel_data {
+	u8 link_id;
+	const struct cfg80211_chan_def *chandef;
+	s32 signal;
+	u16 grade;
+};
+
+void iwl_mld_emlsr_block_tmp_non_bss(struct iwl_mld *mld);
+
+#if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
+u32 iwl_mld_emlsr_pair_state(struct ieee80211_vif *vif,
+			     struct iwl_mld_link_sel_data *a,
+			     struct iwl_mld_link_sel_data *b);
+#endif
+
+void iwl_mld_start_ignoring_tpt_updates(struct iwl_mld *mld);
+void iwl_mld_stop_ignoring_tpt_updates(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_mlo_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/notif.c b/sys/contrib/dev/iwlwifi/mld/notif.c
new file mode 100644
index 000000000000..497967b6cc8e
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/notif.c
@@ -0,0 +1,725 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include "mld.h"
+#include "notif.h"
+#include "scan.h"
+#include "iface.h"
+#include "mlo.h"
+#include "iwl-trans.h"
+#include "fw/file.h"
+#include "fw/dbg.h"
+#include "fw/api/cmdhdr.h"
+#include "fw/api/mac-cfg.h"
+#include "session-protect.h"
+#include "fw/api/time-event.h"
+#include "fw/api/tx.h"
+#include "fw/api/rs.h"
+#include "fw/api/offload.h"
+#include "fw/api/stats.h"
+#include "fw/api/rfi.h"
+#include "fw/api/coex.h"
+
+#include "mcc.h"
+#include "link.h"
+#include "tx.h"
+#include "rx.h"
+#include "tlc.h"
+#include "agg.h"
+#include "mac80211.h"
+#include "thermal.h"
+#include "roc.h"
+#include "stats.h"
+#include "coex.h"
+#include "time_sync.h"
+#include "ftm-initiator.h"
+
+/* Please use this in an increasing order of the versions */
+#define CMD_VER_ENTRY(_ver, _struct)			\
+	{ .size = sizeof(struct _struct), .ver = _ver },
+#define CMD_VERSIONS(name, ...)				\
+	static const struct iwl_notif_struct_size	\
+	iwl_notif_struct_sizes_##name[] = { __VA_ARGS__ };
+
+#define RX_HANDLER_NO_OBJECT(_grp, _cmd, _name, _context)		\
+	{.cmd_id = WIDE_ID(_grp, _cmd),					\
+	 .context = _context,						\
+	 .fn = iwl_mld_handle_##_name,					\
+	 .sizes = iwl_notif_struct_sizes_##_name,			\
+	 .n_sizes = ARRAY_SIZE(iwl_notif_struct_sizes_##_name),		\
+	},
+
+/* Use this for Rx handlers that do not need notification validation */
+#define RX_HANDLER_NO_VAL(_grp, _cmd, _name, _context)			\
+	{.cmd_id = WIDE_ID(_grp, _cmd),					\
+	 .context = _context,						\
+	 .fn = iwl_mld_handle_##_name,					\
+	},
+
+#define RX_HANDLER_VAL_FN(_grp, _cmd, _name, _context)			\
+	{ .cmd_id = WIDE_ID(_grp, _cmd),				\
+	  .context = _context,						\
+	  .fn = iwl_mld_handle_##_name,					\
+	  .val_fn = iwl_mld_validate_##_name,				\
+	},
+
+#define DEFINE_SIMPLE_CANCELLATION(name, notif_struct, id_member)		\
+static bool iwl_mld_cancel_##name##_notif(struct iwl_mld *mld,			\
+					  struct iwl_rx_packet *pkt,		\
+					  u32 obj_id)				\
+{										\
+	const struct notif_struct *notif = (const void *)pkt->data;		\
+										\
+	return obj_id == _Generic((notif)->id_member,				\
+				  __le32: le32_to_cpu((notif)->id_member),	\
+				  __le16: le16_to_cpu((notif)->id_member),	\
+				  u8: (notif)->id_member);			\
+}
+
+/* Currently only defined for the RX_HANDLER_SIZES options. Use this for
+ * notifications that belong to a specific object, and that should be
+ * canceled when the object is removed
+ */
+#define RX_HANDLER_OF_OBJ(_grp, _cmd, _name, _obj_type)			\
+	{.cmd_id = WIDE_ID(_grp, _cmd),					\
+	/* Only async handlers can be canceled */			\
+	 .context = RX_HANDLER_ASYNC,					\
+	 .fn = iwl_mld_handle_##_name,					\
+	 .sizes = iwl_notif_struct_sizes_##_name,			\
+	 .n_sizes = ARRAY_SIZE(iwl_notif_struct_sizes_##_name),		\
+	 .obj_type = IWL_MLD_OBJECT_TYPE_##_obj_type,			\
+	 .cancel = iwl_mld_cancel_##_name,				\
+	 },
+
+#define RX_HANDLER_OF_LINK(_grp, _cmd, _name)				\
+	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, LINK)			\
+
+#define RX_HANDLER_OF_VIF(_grp, _cmd, _name)				\
+	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, VIF)			\
+
+#define RX_HANDLER_OF_STA(_grp, _cmd, _name)				\
+	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, STA)			\
+
+#define RX_HANDLER_OF_ROC(_grp, _cmd, _name)				\
+	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, ROC)
+
+#define RX_HANDLER_OF_SCAN(_grp, _cmd, _name)				\
+	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, SCAN)
+
+#define RX_HANDLER_OF_FTM_REQ(_grp, _cmd, _name)				\
+	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, FTM_REQ)
+
+static void iwl_mld_handle_mfuart_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt)
+{
+	struct iwl_mfuart_load_notif *mfuart_notif = (void *)pkt->data;
+
+	IWL_DEBUG_INFO(mld,
+		       "MFUART: installed ver: 0x%08x, external ver: 0x%08x\n",
+		       le32_to_cpu(mfuart_notif->installed_ver),
+		       le32_to_cpu(mfuart_notif->external_ver));
+	IWL_DEBUG_INFO(mld,
+		       "MFUART: status: 0x%08x, duration: 0x%08x image size: 0x%08x\n",
+		       le32_to_cpu(mfuart_notif->status),
+		       le32_to_cpu(mfuart_notif->duration),
+		       le32_to_cpu(mfuart_notif->image_size));
+}
+
+static void iwl_mld_mu_mimo_iface_iterator(void *_data, u8 *mac,
+					   struct ieee80211_vif *vif)
+{
+	struct ieee80211_bss_conf *bss_conf = &vif->bss_conf;
+	unsigned int link_id = 0;
+
+	if (WARN(hweight16(vif->active_links) > 1,
+		 "no support for this notif while in EMLSR 0x%x\n",
+		 vif->active_links))
+		return;
+
+	if (ieee80211_vif_is_mld(vif)) {
+		link_id = __ffs(vif->active_links);
+		bss_conf = link_conf_dereference_check(vif, link_id);
+	}
+
+	if (!WARN_ON(!bss_conf) && bss_conf->mu_mimo_owner) {
+		const struct iwl_mu_group_mgmt_notif *notif = _data;
+
+		BUILD_BUG_ON(sizeof(notif->membership_status) !=
+			     WLAN_MEMBERSHIP_LEN);
+		BUILD_BUG_ON(sizeof(notif->user_position) !=
+			     WLAN_USER_POSITION_LEN);
+
+		/* MU-MIMO Group Id action frame is little endian. We treat
+		 * the data received from firmware as if it came from the
+		 * action frame, so no conversion is needed.
+		 */
+		ieee80211_update_mu_groups(vif, link_id,
+#if defined(__linux__)
+					   (u8 *)&notif->membership_status,
+					   (u8 *)&notif->user_position);
+#elif defined(__FreeBSD__)
+					   (const u8 *)&notif->membership_status,
+					   (const u8 *)&notif->user_position);
+#endif
+	}
+}
+
+/* This handler is called in SYNC mode because it needs to be serialized with
+ * Rx as specified in ieee80211_update_mu_groups()'s documentation.
+ */
+static void iwl_mld_handle_mu_mimo_grp_notif(struct iwl_mld *mld,
+					     struct iwl_rx_packet *pkt)
+{
+	struct iwl_mu_group_mgmt_notif *notif = (void *)pkt->data;
+
+	ieee80211_iterate_active_interfaces_atomic(mld->hw,
+						   IEEE80211_IFACE_ITER_NORMAL,
+						   iwl_mld_mu_mimo_iface_iterator,
+						   notif);
+}
+
+static void
+iwl_mld_handle_channel_switch_start_notif(struct iwl_mld *mld,
+					  struct iwl_rx_packet *pkt)
+{
+	struct iwl_channel_switch_start_notif *notif = (void *)pkt->data;
+	u32 link_id = le32_to_cpu(notif->link_id);
+	struct ieee80211_bss_conf *link_conf =
+		iwl_mld_fw_id_to_link_conf(mld, link_id);
+	struct ieee80211_vif *vif;
+
+	if (WARN_ON(!link_conf))
+		return;
+
+	vif = link_conf->vif;
+
+	IWL_DEBUG_INFO(mld,
+		       "CSA Start Notification with vif type: %d, link_id: %d\n",
+		       vif->type,
+		       link_conf->link_id);
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_AP:
+		/* We don't support canceling a CSA as it was advertised
+		 * by the AP itself
+		 */
+		if (!link_conf->csa_active)
+			return;
+
+		ieee80211_csa_finish(vif, link_conf->link_id);
+		break;
+	case NL80211_IFTYPE_STATION:
+		if (!link_conf->csa_active) {
+			/* Either unexpected cs notif or mac80211 chose to
+			 * ignore, for example in channel switch to same channel
+			 */
+			struct iwl_cancel_channel_switch_cmd cmd = {
+				.id = cpu_to_le32(link_id),
+			};
+
+			if (iwl_mld_send_cmd_pdu(mld,
+						 WIDE_ID(MAC_CONF_GROUP,
+							 CANCEL_CHANNEL_SWITCH_CMD),
+						 &cmd))
+				IWL_ERR(mld,
+					"Failed to cancel the channel switch\n");
+			return;
+		}
+
+		ieee80211_chswitch_done(vif, true, link_conf->link_id);
+		break;
+
+	default:
+		WARN(1, "CSA on invalid vif type: %d", vif->type);
+	}
+}
+
+static void
+iwl_mld_handle_channel_switch_error_notif(struct iwl_mld *mld,
+					  struct iwl_rx_packet *pkt)
+{
+	struct iwl_channel_switch_error_notif *notif = (void *)pkt->data;
+	struct ieee80211_bss_conf *link_conf;
+	struct ieee80211_vif *vif;
+	u32 link_id = le32_to_cpu(notif->link_id);
+	u32 csa_err_mask = le32_to_cpu(notif->csa_err_mask);
+
+	link_conf = iwl_mld_fw_id_to_link_conf(mld, link_id);
+	if (WARN_ON(!link_conf))
+		return;
+
+	vif = link_conf->vif;
+
+	IWL_DEBUG_INFO(mld, "FW reports CSA error: id=%u, csa_err_mask=%u\n",
+		       link_id, csa_err_mask);
+
+	if (csa_err_mask & (CS_ERR_COUNT_ERROR |
+			    CS_ERR_LONG_DELAY_AFTER_CS |
+			    CS_ERR_TX_BLOCK_TIMER_EXPIRED))
+		ieee80211_channel_switch_disconnect(vif);
+}
+
+static void iwl_mld_handle_beacon_notification(struct iwl_mld *mld,
+					       struct iwl_rx_packet *pkt)
+{
+	struct iwl_extended_beacon_notif *beacon = (void *)pkt->data;
+
+	mld->ibss_manager = !!beacon->ibss_mgr_status;
+}
+
+/**
+ * DOC: Notification versioning
+ *
+ * The firmware's notifications change from time to time. In order to
+ * differentiate between different versions of the same notification, the
+ * firmware advertises the version of each notification.
+ * Here are listed all the notifications that are supported. Several versions
+ * of the same notification can be allowed at the same time:
+ *
+ * CMD_VERSION(my_multi_version_notif,
+ *	       CMD_VER_ENTRY(1, iwl_my_multi_version_notif_ver1)
+ *	       CMD_VER_ENTRY(2, iwl_my_multi_version_notif_ver2)
+ *
+ * etc...
+ *
+ * The driver will enforce that the notification coming from the firmware
+ * has its version listed here and it'll also enforce that the firmware sent
+ * at least enough bytes to cover the structure listed in the CMD_VER_ENTRY.
+ */
+
+CMD_VERSIONS(scan_complete_notif,
+	     CMD_VER_ENTRY(1, iwl_umac_scan_complete))
+CMD_VERSIONS(scan_iter_complete_notif,
+	     CMD_VER_ENTRY(2, iwl_umac_scan_iter_complete_notif))
+CMD_VERSIONS(channel_survey_notif,
+	     CMD_VER_ENTRY(1, iwl_umac_scan_channel_survey_notif))
+CMD_VERSIONS(mfuart_notif,
+	     CMD_VER_ENTRY(2, iwl_mfuart_load_notif))
+CMD_VERSIONS(update_mcc,
+	     CMD_VER_ENTRY(1, iwl_mcc_chub_notif))
+CMD_VERSIONS(session_prot_notif,
+	     CMD_VER_ENTRY(3, iwl_session_prot_notif))
+CMD_VERSIONS(missed_beacon_notif,
+	     CMD_VER_ENTRY(5, iwl_missed_beacons_notif))
+CMD_VERSIONS(tx_resp_notif,
+	     CMD_VER_ENTRY(8, iwl_tx_resp)
+	     CMD_VER_ENTRY(9, iwl_tx_resp))
+CMD_VERSIONS(compressed_ba_notif,
+	     CMD_VER_ENTRY(5, iwl_compressed_ba_notif)
+	     CMD_VER_ENTRY(6, iwl_compressed_ba_notif)
+	     CMD_VER_ENTRY(7, iwl_compressed_ba_notif))
+CMD_VERSIONS(tlc_notif,
+	     CMD_VER_ENTRY(3, iwl_tlc_update_notif)
+	     CMD_VER_ENTRY(4, iwl_tlc_update_notif))
+CMD_VERSIONS(mu_mimo_grp_notif,
+	     CMD_VER_ENTRY(1, iwl_mu_group_mgmt_notif))
+CMD_VERSIONS(channel_switch_start_notif,
+	     CMD_VER_ENTRY(3, iwl_channel_switch_start_notif))
+CMD_VERSIONS(channel_switch_error_notif,
+	     CMD_VER_ENTRY(2, iwl_channel_switch_error_notif))
+CMD_VERSIONS(ct_kill_notif,
+	     CMD_VER_ENTRY(2, ct_kill_notif))
+CMD_VERSIONS(temp_notif,
+	     CMD_VER_ENTRY(2, iwl_dts_measurement_notif))
+CMD_VERSIONS(roc_notif,
+	     CMD_VER_ENTRY(1, iwl_roc_notif))
+CMD_VERSIONS(probe_resp_data_notif,
+	     CMD_VER_ENTRY(1, iwl_probe_resp_data_notif))
+CMD_VERSIONS(datapath_monitor_notif,
+	     CMD_VER_ENTRY(1, iwl_datapath_monitor_notif))
+CMD_VERSIONS(stats_oper_notif,
+	     CMD_VER_ENTRY(3, iwl_system_statistics_notif_oper))
+CMD_VERSIONS(stats_oper_part1_notif,
+	     CMD_VER_ENTRY(4, iwl_system_statistics_part1_notif_oper))
+CMD_VERSIONS(bt_coex_notif,
+	     CMD_VER_ENTRY(1, iwl_bt_coex_profile_notif))
+CMD_VERSIONS(beacon_notification,
+	     CMD_VER_ENTRY(6, iwl_extended_beacon_notif))
+CMD_VERSIONS(emlsr_mode_notif,
+	     CMD_VER_ENTRY(1, iwl_esr_mode_notif_v1)
+	     CMD_VER_ENTRY(2, iwl_esr_mode_notif))
+CMD_VERSIONS(emlsr_trans_fail_notif,
+	     CMD_VER_ENTRY(1, iwl_esr_trans_fail_notif))
+CMD_VERSIONS(uapsd_misbehaving_ap_notif,
+	     CMD_VER_ENTRY(1, iwl_uapsd_misbehaving_ap_notif))
+CMD_VERSIONS(time_msmt_notif,
+	     CMD_VER_ENTRY(1, iwl_time_msmt_notify))
+CMD_VERSIONS(time_sync_confirm_notif,
+	     CMD_VER_ENTRY(1, iwl_time_msmt_cfm_notify))
+CMD_VERSIONS(ftm_resp_notif, CMD_VER_ENTRY(10, iwl_tof_range_rsp_ntfy))
+CMD_VERSIONS(beacon_filter_notif, CMD_VER_ENTRY(2, iwl_beacon_filter_notif))
+
+DEFINE_SIMPLE_CANCELLATION(session_prot, iwl_session_prot_notif, mac_link_id)
+DEFINE_SIMPLE_CANCELLATION(tlc, iwl_tlc_update_notif, sta_id)
+DEFINE_SIMPLE_CANCELLATION(channel_switch_start,
+			   iwl_channel_switch_start_notif, link_id)
+DEFINE_SIMPLE_CANCELLATION(channel_switch_error,
+			   iwl_channel_switch_error_notif, link_id)
+DEFINE_SIMPLE_CANCELLATION(datapath_monitor, iwl_datapath_monitor_notif,
+			   link_id)
+DEFINE_SIMPLE_CANCELLATION(roc, iwl_roc_notif, activity)
+DEFINE_SIMPLE_CANCELLATION(scan_complete, iwl_umac_scan_complete, uid)
+DEFINE_SIMPLE_CANCELLATION(probe_resp_data, iwl_probe_resp_data_notif,
+			   mac_id)
+DEFINE_SIMPLE_CANCELLATION(uapsd_misbehaving_ap, iwl_uapsd_misbehaving_ap_notif,
+			   mac_id)
+DEFINE_SIMPLE_CANCELLATION(ftm_resp, iwl_tof_range_rsp_ntfy, request_id)
+DEFINE_SIMPLE_CANCELLATION(beacon_filter, iwl_beacon_filter_notif, link_id)
+
+/**
+ * DOC: Handlers for fw notifications
+ *
+ * Here are listed the notifications IDs (including the group ID), the handler
+ * of the notification and how it should be called:
+ *
+ *  - RX_HANDLER_SYNC: will be called as part of the Rx path
+ *  - RX_HANDLER_ASYNC: will be handled in a working with the wiphy_lock held
+ *
+ * This means that if the firmware sends two notifications A and B in that
+ * order and notification A is RX_HANDLER_ASYNC and notification is
+ * RX_HANDLER_SYNC, the handler of B will likely be called before the handler
+ * of A.
+ *
+ * This list should be in order of frequency for performance purposes.
+ * The handler can be one from two contexts, see &iwl_rx_handler_context
+ *
+ * A handler can declare that it relies on a specific object in which case it
+ * can be cancelled in case the object is deleted. In order to use this
+ * mechanism, a cancellation function is needed. The cancellation function must
+ * receive an object id (the index of that object in the firmware) and a
+ * notification payload. It'll return true if that specific notification should
+ * be cancelled upon the obliteration of the specific instance of the object.
+ *
+ * DEFINE_SIMPLE_CANCELLATION allows to easily create a cancellation function
+ * that wills simply return true if a given object id matches the object id in
+ * the firmware notification.
+ */
+
+VISIBLE_IF_IWLWIFI_KUNIT
+const struct iwl_rx_handler iwl_mld_rx_handlers[] = {
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP, TX_CMD, tx_resp_notif,
+			     RX_HANDLER_SYNC)
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP, BA_NOTIF, compressed_ba_notif,
+			     RX_HANDLER_SYNC)
+	RX_HANDLER_OF_SCAN(LEGACY_GROUP, SCAN_COMPLETE_UMAC,
+			   scan_complete_notif)
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP, SCAN_ITERATION_COMPLETE_UMAC,
+			     scan_iter_complete_notif,
+			     RX_HANDLER_SYNC)
+	RX_HANDLER_NO_VAL(LEGACY_GROUP, MATCH_FOUND_NOTIFICATION,
+			  match_found_notif, RX_HANDLER_SYNC)
+
+	RX_HANDLER_NO_OBJECT(SCAN_GROUP, CHANNEL_SURVEY_NOTIF,
+			     channel_survey_notif,
+			     RX_HANDLER_ASYNC)
+
+	RX_HANDLER_NO_OBJECT(STATISTICS_GROUP, STATISTICS_OPER_NOTIF,
+			     stats_oper_notif, RX_HANDLER_ASYNC)
+	RX_HANDLER_NO_OBJECT(STATISTICS_GROUP, STATISTICS_OPER_PART1_NOTIF,
+			     stats_oper_part1_notif, RX_HANDLER_ASYNC)
+
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP, MFUART_LOAD_NOTIFICATION,
+			     mfuart_notif, RX_HANDLER_SYNC)
+
+	RX_HANDLER_NO_OBJECT(PHY_OPS_GROUP, DTS_MEASUREMENT_NOTIF_WIDE,
+			     temp_notif, RX_HANDLER_ASYNC)
+	RX_HANDLER_OF_LINK(MAC_CONF_GROUP, SESSION_PROTECTION_NOTIF,
+			   session_prot_notif)
+	RX_HANDLER_OF_LINK(MAC_CONF_GROUP, MISSED_BEACONS_NOTIF,
+			   missed_beacon_notif)
+	RX_HANDLER_OF_STA(DATA_PATH_GROUP, TLC_MNG_UPDATE_NOTIF, tlc_notif)
+	RX_HANDLER_OF_LINK(MAC_CONF_GROUP, CHANNEL_SWITCH_START_NOTIF,
+			   channel_switch_start_notif)
+	RX_HANDLER_OF_LINK(MAC_CONF_GROUP, CHANNEL_SWITCH_ERROR_NOTIF,
+			   channel_switch_error_notif)
+	RX_HANDLER_OF_ROC(MAC_CONF_GROUP, ROC_NOTIF, roc_notif)
+	RX_HANDLER_NO_OBJECT(DATA_PATH_GROUP, MU_GROUP_MGMT_NOTIF,
+			     mu_mimo_grp_notif, RX_HANDLER_SYNC)
+	RX_HANDLER_OF_VIF(MAC_CONF_GROUP, PROBE_RESPONSE_DATA_NOTIF,
+			  probe_resp_data_notif)
+	RX_HANDLER_NO_OBJECT(PHY_OPS_GROUP, CT_KILL_NOTIFICATION,
+			     ct_kill_notif, RX_HANDLER_ASYNC)
+	RX_HANDLER_OF_LINK(DATA_PATH_GROUP, MONITOR_NOTIF,
+			   datapath_monitor_notif)
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP, MCC_CHUB_UPDATE_CMD, update_mcc,
+			     RX_HANDLER_ASYNC)
+	RX_HANDLER_NO_OBJECT(BT_COEX_GROUP, PROFILE_NOTIF,
+			     bt_coex_notif, RX_HANDLER_ASYNC)
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP, BEACON_NOTIFICATION,
+			     beacon_notification, RX_HANDLER_ASYNC)
+	RX_HANDLER_NO_OBJECT(DATA_PATH_GROUP, ESR_MODE_NOTIF,
+			     emlsr_mode_notif, RX_HANDLER_ASYNC)
+	RX_HANDLER_NO_OBJECT(MAC_CONF_GROUP, EMLSR_TRANS_FAIL_NOTIF,
+			     emlsr_trans_fail_notif, RX_HANDLER_ASYNC)
+	RX_HANDLER_OF_VIF(LEGACY_GROUP, PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION,
+			  uapsd_misbehaving_ap_notif)
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP,
+			     WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION,
+			     time_msmt_notif, RX_HANDLER_SYNC)
+	RX_HANDLER_NO_OBJECT(LEGACY_GROUP,
+			     WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION,
+			     time_sync_confirm_notif, RX_HANDLER_ASYNC)
+	RX_HANDLER_OF_LINK(DATA_PATH_GROUP, BEACON_FILTER_IN_NOTIF,
+			   beacon_filter_notif)
+	RX_HANDLER_OF_FTM_REQ(LOCATION_GROUP, TOF_RANGE_RESPONSE_NOTIF,
+			      ftm_resp_notif)
+};
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_rx_handlers);
+
+#if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
+const unsigned int iwl_mld_rx_handlers_num = ARRAY_SIZE(iwl_mld_rx_handlers);
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_rx_handlers_num);
+#endif
+
+static bool
+iwl_mld_notif_is_valid(struct iwl_mld *mld, struct iwl_rx_packet *pkt,
+		       const struct iwl_rx_handler *handler)
+{
+	unsigned int size = iwl_rx_packet_payload_len(pkt);
+	size_t notif_ver;
+
+	/* If n_sizes == 0, it indicates that a validation function may be used
+	 * or that no validation is required.
+	 */
+	if (!handler->n_sizes) {
+		if (handler->val_fn)
+			return handler->val_fn(mld, pkt);
+		return true;
+	}
+
+	notif_ver = iwl_fw_lookup_notif_ver(mld->fw,
+					    iwl_cmd_groupid(handler->cmd_id),
+					    iwl_cmd_opcode(handler->cmd_id),
+					    IWL_FW_CMD_VER_UNKNOWN);
+
+	for (int i = 0; i < handler->n_sizes; i++) {
+		if (handler->sizes[i].ver != notif_ver)
+			continue;
+
+		if (IWL_FW_CHECK(mld, size < handler->sizes[i].size,
+				 "unexpected notification 0x%04x size %d, need %d\n",
+				 handler->cmd_id, size, handler->sizes[i].size))
+			return false;
+		return true;
+	}
+
+	IWL_FW_CHECK_FAILED(mld,
+			    "notif 0x%04x ver %zu missing expected size, use version %u size\n",
+			    handler->cmd_id, notif_ver,
+			    handler->sizes[handler->n_sizes - 1].ver);
+
+	return size < handler->sizes[handler->n_sizes - 1].size;
+}
+
+struct iwl_async_handler_entry {
+	struct list_head list;
+	struct iwl_rx_cmd_buffer rxb;
+	const struct iwl_rx_handler *rx_h;
+};
+
+static void
+iwl_mld_log_async_handler_op(struct iwl_mld *mld, const char *op,
+			     struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+
+	IWL_DEBUG_HC(mld,
+		     "%s async handler for notif %s (%.2x.%2x, seq 0x%x)\n",
+		     op, iwl_get_cmd_string(mld->trans,
+		     WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)),
+		     pkt->hdr.group_id, pkt->hdr.cmd,
+		     le16_to_cpu(pkt->hdr.sequence));
+}
+
+static void iwl_mld_rx_notif(struct iwl_mld *mld,
+			     struct iwl_rx_cmd_buffer *rxb,
+			     struct iwl_rx_packet *pkt)
+{
+	for (int i = 0; i < ARRAY_SIZE(iwl_mld_rx_handlers); i++) {
+		const struct iwl_rx_handler *rx_h = &iwl_mld_rx_handlers[i];
+		struct iwl_async_handler_entry *entry;
+
+		if (rx_h->cmd_id != WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd))
+			continue;
+
+		if (!iwl_mld_notif_is_valid(mld, pkt, rx_h))
+			return;
+
+		if (rx_h->context == RX_HANDLER_SYNC) {
+			rx_h->fn(mld, pkt);
+			break;
+		}
+
+		entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
+		/* we can't do much... */
+		if (!entry)
+			return;
+
+		/* Set the async handler entry */
+		entry->rxb._page = rxb_steal_page(rxb);
+		entry->rxb._offset = rxb->_offset;
+		entry->rxb._rx_page_order = rxb->_rx_page_order;
+
+		entry->rx_h = rx_h;
+
+		/* Add it to the list and queue the work */
+		spin_lock(&mld->async_handlers_lock);
+		list_add_tail(&entry->list, &mld->async_handlers_list);
+		spin_unlock(&mld->async_handlers_lock);
+
+		wiphy_work_queue(mld->hw->wiphy,
+				 &mld->async_handlers_wk);
+
+		iwl_mld_log_async_handler_op(mld, "Queued", rxb);
+		break;
+	}
+
+	iwl_notification_wait_notify(&mld->notif_wait, pkt);
+}
+
+void iwl_mld_rx(struct iwl_op_mode *op_mode, struct napi_struct *napi,
+		struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+	u16 cmd_id = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
+
+	if (likely(cmd_id == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
+		iwl_mld_rx_mpdu(mld, napi, rxb, 0);
+	else if (cmd_id == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE))
+		iwl_mld_handle_frame_release_notif(mld, napi, pkt, 0);
+	else if (cmd_id == WIDE_ID(LEGACY_GROUP, BAR_FRAME_RELEASE))
+		iwl_mld_handle_bar_frame_release_notif(mld, napi, pkt, 0);
+	else if (unlikely(cmd_id == WIDE_ID(DATA_PATH_GROUP,
+					    RX_QUEUES_NOTIFICATION)))
+		iwl_mld_handle_rx_queues_sync_notif(mld, napi, pkt, 0);
+	else if (cmd_id == WIDE_ID(DATA_PATH_GROUP, RX_NO_DATA_NOTIF))
+		iwl_mld_rx_monitor_no_data(mld, napi, pkt, 0);
+	else
+		iwl_mld_rx_notif(mld, rxb, pkt);
+}
+
+void iwl_mld_rx_rss(struct iwl_op_mode *op_mode, struct napi_struct *napi,
+		    struct iwl_rx_cmd_buffer *rxb, unsigned int queue)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	struct iwl_mld *mld = IWL_OP_MODE_GET_MLD(op_mode);
+	u16 cmd_id = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
+
+	if (unlikely(queue >= mld->trans->info.num_rxqs))
+		return;
+
+	if (likely(cmd_id == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
+		iwl_mld_rx_mpdu(mld, napi, rxb, queue);
+	else if (unlikely(cmd_id == WIDE_ID(DATA_PATH_GROUP,
+					    RX_QUEUES_NOTIFICATION)))
+		iwl_mld_handle_rx_queues_sync_notif(mld, napi, pkt, queue);
+	else if (unlikely(cmd_id == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE)))
+		iwl_mld_handle_frame_release_notif(mld, napi, pkt, queue);
+}
+
+void iwl_mld_delete_handlers(struct iwl_mld *mld, const u16 *cmds, int n_cmds)
+{
+	struct iwl_async_handler_entry *entry, *tmp;
+
+	spin_lock_bh(&mld->async_handlers_lock);
+	list_for_each_entry_safe(entry, tmp, &mld->async_handlers_list, list) {
+		bool match = false;
+
+		for (int i = 0; i < n_cmds; i++) {
+			if (entry->rx_h->cmd_id == cmds[i]) {
+				match = true;
+				break;
+			}
+		}
+
+		if (!match)
+			continue;
+
+		iwl_mld_log_async_handler_op(mld, "Delete", &entry->rxb);
+		iwl_free_rxb(&entry->rxb);
+		list_del(&entry->list);
+		kfree(entry);
+	}
+	spin_unlock_bh(&mld->async_handlers_lock);
+}
+
+void iwl_mld_async_handlers_wk(struct wiphy *wiphy, struct wiphy_work *wk)
+{
+	struct iwl_mld *mld =
+		container_of(wk, struct iwl_mld, async_handlers_wk);
+	struct iwl_async_handler_entry *entry, *tmp;
+	LIST_HEAD(local_list);
+
+	/* Sync with Rx path with a lock. Remove all the entries from this
+	 * list, add them to a local one (lock free), and then handle them.
+	 */
+	spin_lock_bh(&mld->async_handlers_lock);
+	list_splice_init(&mld->async_handlers_list, &local_list);
+	spin_unlock_bh(&mld->async_handlers_lock);
+
+	list_for_each_entry_safe(entry, tmp, &local_list, list) {
+		iwl_mld_log_async_handler_op(mld, "Handle", &entry->rxb);
+		entry->rx_h->fn(mld, rxb_addr(&entry->rxb));
+		iwl_free_rxb(&entry->rxb);
+		list_del(&entry->list);
+		kfree(entry);
+	}
+}
+
+void iwl_mld_cancel_async_notifications(struct iwl_mld *mld)
+{
+	struct iwl_async_handler_entry *entry, *tmp;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	wiphy_work_cancel(mld->wiphy, &mld->async_handlers_wk);
+
+	spin_lock_bh(&mld->async_handlers_lock);
+	list_for_each_entry_safe(entry, tmp, &mld->async_handlers_list, list) {
+		iwl_mld_log_async_handler_op(mld, "Purged", &entry->rxb);
+		iwl_free_rxb(&entry->rxb);
+		list_del(&entry->list);
+		kfree(entry);
+	}
+	spin_unlock_bh(&mld->async_handlers_lock);
+}
+
+void iwl_mld_cancel_notifications_of_object(struct iwl_mld *mld,
+					    enum iwl_mld_object_type obj_type,
+					    u32 obj_id)
+{
+	struct iwl_async_handler_entry *entry, *tmp;
+	LIST_HEAD(cancel_list);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(obj_type == IWL_MLD_OBJECT_TYPE_NONE))
+		return;
+
+	/* Sync with RX path and remove matching entries from the async list */
+	spin_lock_bh(&mld->async_handlers_lock);
+	list_for_each_entry_safe(entry, tmp, &mld->async_handlers_list, list) {
+		const struct iwl_rx_handler *rx_h = entry->rx_h;
+
+		if (rx_h->obj_type != obj_type || WARN_ON(!rx_h->cancel))
+			continue;
+
+		if (rx_h->cancel(mld, rxb_addr(&entry->rxb), obj_id)) {
+			iwl_mld_log_async_handler_op(mld, "Cancel", &entry->rxb);
+			list_del(&entry->list);
+			list_add_tail(&entry->list, &cancel_list);
+		}
+	}
+
+	spin_unlock_bh(&mld->async_handlers_lock);
+
+	/* Free the matching entries outside of the spinlock */
+	list_for_each_entry_safe(entry, tmp, &cancel_list, list) {
+		iwl_free_rxb(&entry->rxb);
+		list_del(&entry->list);
+		kfree(entry);
+	}
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/notif.h b/sys/contrib/dev/iwlwifi/mld/notif.h
new file mode 100644
index 000000000000..adcdd9dec192
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/notif.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_notif_h__
+#define __iwl_mld_notif_h__
+
+struct iwl_mld;
+
+void iwl_mld_rx(struct iwl_op_mode *op_mode, struct napi_struct *napi,
+		struct iwl_rx_cmd_buffer *rxb);
+
+void iwl_mld_rx_rss(struct iwl_op_mode *op_mode, struct napi_struct *napi,
+		    struct iwl_rx_cmd_buffer *rxb, unsigned int queue);
+
+void iwl_mld_async_handlers_wk(struct wiphy *wiphy, struct wiphy_work *wk);
+
+void iwl_mld_cancel_async_notifications(struct iwl_mld *mld);
+
+enum iwl_mld_object_type {
+	IWL_MLD_OBJECT_TYPE_NONE,
+	IWL_MLD_OBJECT_TYPE_LINK,
+	IWL_MLD_OBJECT_TYPE_STA,
+	IWL_MLD_OBJECT_TYPE_VIF,
+	IWL_MLD_OBJECT_TYPE_ROC,
+	IWL_MLD_OBJECT_TYPE_SCAN,
+	IWL_MLD_OBJECT_TYPE_FTM_REQ,
+};
+
+void iwl_mld_cancel_notifications_of_object(struct iwl_mld *mld,
+					    enum iwl_mld_object_type obj_type,
+					    u32 obj_id);
+void iwl_mld_delete_handlers(struct iwl_mld *mld, const u16 *cmds, int n_cmds);
+
+#endif /* __iwl_mld_notif_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/phy.c b/sys/contrib/dev/iwlwifi/mld/phy.c
new file mode 100644
index 000000000000..1d93fb9e4dbf
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/phy.c
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <net/mac80211.h>
+
+#include "phy.h"
+#include "hcmd.h"
+#include "fw/api/phy-ctxt.h"
+
+int iwl_mld_allocate_fw_phy_id(struct iwl_mld *mld)
+{
+	int id;
+	unsigned long used = mld->used_phy_ids;
+
+	for_each_clear_bit(id, &used, NUM_PHY_CTX) {
+		mld->used_phy_ids |= BIT(id);
+		return id;
+	}
+
+	return -ENOSPC;
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_allocate_fw_phy_id);
+
+struct iwl_mld_chanctx_usage_data {
+	struct iwl_mld *mld;
+	struct ieee80211_chanctx_conf *ctx;
+	bool use_def;
+};
+
+static bool iwl_mld_chanctx_fils_enabled(struct ieee80211_vif *vif,
+					 struct ieee80211_chanctx_conf *ctx)
+{
+	if (vif->type != NL80211_IFTYPE_AP)
+		return false;
+
+	return cfg80211_channel_is_psc(ctx->def.chan) ||
+		(ctx->def.chan->band == NL80211_BAND_6GHZ &&
+		 ctx->def.width >= NL80211_CHAN_WIDTH_80);
+}
+
+static void iwl_mld_chanctx_usage_iter(void *_data, u8 *mac,
+				       struct ieee80211_vif *vif)
+{
+	struct iwl_mld_chanctx_usage_data *data = _data;
+	struct ieee80211_bss_conf *link_conf;
+	int link_id;
+
+	for_each_vif_active_link(vif, link_conf, link_id) {
+		if (rcu_access_pointer(link_conf->chanctx_conf) != data->ctx)
+			continue;
+
+		if (vif->type == NL80211_IFTYPE_AP && link_conf->ftm_responder)
+			data->use_def = true;
+
+		if (iwl_mld_chanctx_fils_enabled(vif, data->ctx))
+			data->use_def = true;
+	}
+}
+
+struct cfg80211_chan_def *
+iwl_mld_get_chandef_from_chanctx(struct iwl_mld *mld,
+				 struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mld_chanctx_usage_data data = {
+		.mld = mld,
+		.ctx = ctx,
+	};
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_chanctx_usage_iter,
+						&data);
+
+	return data.use_def ? &ctx->def : &ctx->min_def;
+}
+
+static u8
+iwl_mld_nl80211_width_to_fw(enum nl80211_chan_width width)
+{
+	switch (width) {
+	case NL80211_CHAN_WIDTH_20_NOHT:
+	case NL80211_CHAN_WIDTH_20:
+		return IWL_PHY_CHANNEL_MODE20;
+	case NL80211_CHAN_WIDTH_40:
+		return IWL_PHY_CHANNEL_MODE40;
+	case NL80211_CHAN_WIDTH_80:
+		return IWL_PHY_CHANNEL_MODE80;
+	case NL80211_CHAN_WIDTH_160:
+		return IWL_PHY_CHANNEL_MODE160;
+	case NL80211_CHAN_WIDTH_320:
+		return IWL_PHY_CHANNEL_MODE320;
+	default:
+		WARN(1, "Invalid channel width=%u", width);
+		return IWL_PHY_CHANNEL_MODE20;
+	}
+}
+
+/* Maps the driver specific control channel position (relative to the center
+ * freq) definitions to the fw values
+ */
+u8 iwl_mld_get_fw_ctrl_pos(const struct cfg80211_chan_def *chandef)
+{
+	int offs = chandef->chan->center_freq - chandef->center_freq1;
+	int abs_offs = abs(offs);
+	u8 ret;
+
+	if (offs == 0) {
+		/* The FW is expected to check the control channel position only
+		 * when in HT/VHT and the channel width is not 20MHz. Return
+		 * this value as the default one.
+		 */
+		return 0;
+	}
+
+	/* this results in a value 0-7, i.e. fitting into 0b0111 */
+	ret = (abs_offs - 10) / 20;
+	/* But we need the value to be in 0b1011 because 0b0100 is
+	 * IWL_PHY_CTRL_POS_ABOVE, so shift bit 2 up to land in
+	 * IWL_PHY_CTRL_POS_OFFS_EXT (0b1000)
+	 */
+	ret = (ret & IWL_PHY_CTRL_POS_OFFS_MSK) |
+	      ((ret & BIT(2)) << 1);
+	/* and add the above bit */
+	ret |= (offs > 0) * IWL_PHY_CTRL_POS_ABOVE;
+
+	return ret;
+}
+
+int iwl_mld_phy_fw_action(struct iwl_mld *mld,
+			  struct ieee80211_chanctx_conf *ctx, u32 action)
+{
+	struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(ctx);
+	struct cfg80211_chan_def *chandef = &phy->chandef;
+	struct iwl_phy_context_cmd cmd = {
+		.id_and_color = cpu_to_le32(phy->fw_id),
+		.action = cpu_to_le32(action),
+		.puncture_mask = cpu_to_le16(chandef->punctured),
+		/* Channel info */
+		.ci.channel = cpu_to_le32(chandef->chan->hw_value),
+		.ci.band = iwl_mld_nl80211_band_to_fw(chandef->chan->band),
+		.ci.width = iwl_mld_nl80211_width_to_fw(chandef->width),
+		.ci.ctrl_pos = iwl_mld_get_fw_ctrl_pos(chandef),
+	};
+	int ret;
+
+	if (ctx->ap.chan) {
+		cmd.sbb_bandwidth =
+			iwl_mld_nl80211_width_to_fw(ctx->ap.width);
+		cmd.sbb_ctrl_channel_loc = iwl_mld_get_fw_ctrl_pos(&ctx->ap);
+	}
+
+	ret = iwl_mld_send_cmd_pdu(mld, PHY_CONTEXT_CMD, &cmd);
+	if (ret)
+		IWL_ERR(mld, "Failed to send PHY_CONTEXT_CMD ret = %d\n", ret);
+
+	return ret;
+}
+
+static u32 iwl_mld_get_phy_config(struct iwl_mld *mld)
+{
+	u32 phy_config = ~(FW_PHY_CFG_TX_CHAIN |
+			   FW_PHY_CFG_RX_CHAIN);
+	u32 valid_rx_ant = iwl_mld_get_valid_rx_ant(mld);
+	u32 valid_tx_ant = iwl_mld_get_valid_tx_ant(mld);
+
+	phy_config |= valid_tx_ant << FW_PHY_CFG_TX_CHAIN_POS |
+		      valid_rx_ant << FW_PHY_CFG_RX_CHAIN_POS;
+
+	return mld->fw->phy_config & phy_config;
+}
+
+int iwl_mld_send_phy_cfg_cmd(struct iwl_mld *mld)
+{
+	const struct iwl_tlv_calib_ctrl *default_calib =
+		&mld->fw->default_calib[IWL_UCODE_REGULAR];
+	struct iwl_phy_cfg_cmd_v3 cmd = {
+		.phy_cfg = cpu_to_le32(iwl_mld_get_phy_config(mld)),
+		.calib_control.event_trigger = default_calib->event_trigger,
+		.calib_control.flow_trigger = default_calib->flow_trigger,
+		.phy_specific_cfg = mld->fwrt.phy_filters,
+	};
+
+	IWL_DEBUG_INFO(mld, "Sending Phy CFG command: 0x%x\n", cmd.phy_cfg);
+
+	return iwl_mld_send_cmd_pdu(mld, PHY_CONFIGURATION_CMD, &cmd);
+}
+
+void iwl_mld_update_phy_chandef(struct iwl_mld *mld,
+				struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(ctx);
+	struct cfg80211_chan_def *chandef =
+		iwl_mld_get_chandef_from_chanctx(mld, ctx);
+
+	phy->chandef = *chandef;
+	iwl_mld_phy_fw_action(mld, ctx, FW_CTXT_ACTION_MODIFY);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/phy.h b/sys/contrib/dev/iwlwifi/mld/phy.h
new file mode 100644
index 000000000000..0deaf179f07c
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/phy.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_phy_h__
+#define __iwl_mld_phy_h__
+
+#include "mld.h"
+
+/**
+ * struct iwl_mld_phy - PHY configuration parameters
+ *
+ * @fw_id: fw id of the phy.
+ * @chandef: the last chandef that mac80211 configured the driver
+ *	with. Used to detect a no-op when the chanctx changes.
+ * @channel_load_by_us: channel load on this channel caused by
+ *	the NIC itself, as indicated by firmware
+ * @avg_channel_load_not_by_us: averaged channel load on this channel caused by
+ *	others. This value is invalid when in EMLSR (due to FW limitations)
+ * @mld: pointer to the MLD context
+ */
+struct iwl_mld_phy {
+	/* Add here fields that need clean up on hw restart */
+	struct_group(zeroed_on_hw_restart,
+		u8 fw_id;
+		struct cfg80211_chan_def chandef;
+	);
+	/* And here fields that survive a hw restart */
+	u32 channel_load_by_us;
+	u32 avg_channel_load_not_by_us;
+	struct iwl_mld *mld;
+};
+
+static inline struct iwl_mld_phy *
+iwl_mld_phy_from_mac80211(struct ieee80211_chanctx_conf *channel)
+{
+	return (void *)channel->drv_priv;
+}
+
+/* Cleanup function for struct iwl_mld_phy, will be called in restart */
+static inline void
+iwl_mld_cleanup_phy(struct iwl_mld *mld, struct iwl_mld_phy *phy)
+{
+	CLEANUP_STRUCT(phy);
+}
+
+int iwl_mld_allocate_fw_phy_id(struct iwl_mld *mld);
+int iwl_mld_phy_fw_action(struct iwl_mld *mld,
+			  struct ieee80211_chanctx_conf *ctx, u32 action);
+struct cfg80211_chan_def *
+iwl_mld_get_chandef_from_chanctx(struct iwl_mld *mld,
+				 struct ieee80211_chanctx_conf *ctx);
+u8 iwl_mld_get_fw_ctrl_pos(const struct cfg80211_chan_def *chandef);
+
+int iwl_mld_send_phy_cfg_cmd(struct iwl_mld *mld);
+
+void iwl_mld_update_phy_chandef(struct iwl_mld *mld,
+				struct ieee80211_chanctx_conf *ctx);
+
+#endif /* __iwl_mld_phy_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/power.c b/sys/contrib/dev/iwlwifi/mld/power.c
new file mode 100644
index 000000000000..f664b277adf7
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/power.c
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <net/mac80211.h>
+
+#include "mld.h"
+#include "hcmd.h"
+#include "power.h"
+#include "iface.h"
+#include "link.h"
+#include "constants.h"
+
+static void iwl_mld_vif_ps_iterator(void *data, u8 *mac,
+				    struct ieee80211_vif *vif)
+{
+	bool *ps_enable = (bool *)data;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	*ps_enable &= !mld_vif->ps_disabled;
+}
+
+int iwl_mld_update_device_power(struct iwl_mld *mld, bool d3)
+{
+	struct iwl_device_power_cmd cmd = {};
+	bool enable_ps = false;
+
+	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM) {
+		enable_ps = true;
+
+		/* Disable power save if any STA interface has
+		 * power save turned off
+		 */
+		ieee80211_iterate_active_interfaces_mtx(mld->hw,
+							IEEE80211_IFACE_ITER_NORMAL,
+							iwl_mld_vif_ps_iterator,
+							&enable_ps);
+	}
+
+	if (enable_ps)
+		cmd.flags |=
+			cpu_to_le16(DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK);
+
+	if (d3)
+		cmd.flags |=
+			cpu_to_le16(DEVICE_POWER_FLAGS_NO_SLEEP_TILL_D3_MSK);
+
+	IWL_DEBUG_POWER(mld,
+			"Sending device power command with flags = 0x%X\n",
+			cmd.flags);
+
+	return iwl_mld_send_cmd_pdu(mld, POWER_TABLE_CMD, &cmd);
+}
+
+int iwl_mld_enable_beacon_filter(struct iwl_mld *mld,
+				 const struct ieee80211_bss_conf *link_conf,
+				 bool d3)
+{
+	struct iwl_beacon_filter_cmd cmd = {
+		IWL_BF_CMD_CONFIG_DEFAULTS,
+		.bf_enable_beacon_filter = cpu_to_le32(1),
+		.ba_enable_beacon_abort = cpu_to_le32(1),
+	};
+
+	if (ieee80211_vif_type_p2p(link_conf->vif) != NL80211_IFTYPE_STATION)
+		return 0;
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	if (iwl_mld_vif_from_mac80211(link_conf->vif)->disable_bf)
+		return 0;
+#endif
+
+	if (link_conf->cqm_rssi_thold) {
+		cmd.bf_energy_delta =
+			cpu_to_le32(link_conf->cqm_rssi_hyst);
+		/* fw uses an absolute value for this */
+		cmd.bf_roaming_state =
+			cpu_to_le32(-link_conf->cqm_rssi_thold);
+	}
+
+	if (d3)
+		cmd.ba_escape_timer = cpu_to_le32(IWL_BA_ESCAPE_TIMER_D3);
+
+	return iwl_mld_send_cmd_pdu(mld, REPLY_BEACON_FILTERING_CMD,
+				    &cmd);
+}
+
+int iwl_mld_disable_beacon_filter(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif)
+{
+	struct iwl_beacon_filter_cmd cmd = {};
+
+	if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION)
+		return 0;
+
+	return iwl_mld_send_cmd_pdu(mld, REPLY_BEACON_FILTERING_CMD,
+				    &cmd);
+}
+
+static bool iwl_mld_power_is_radar(struct iwl_mld *mld,
+				   const struct ieee80211_bss_conf *link_conf)
+{
+	const struct ieee80211_chanctx_conf *chanctx_conf;
+
+	chanctx_conf = wiphy_dereference(mld->wiphy, link_conf->chanctx_conf);
+
+	if (WARN_ON(!chanctx_conf))
+		return false;
+
+	return chanctx_conf->def.chan->flags & IEEE80211_CHAN_RADAR;
+}
+
+static void iwl_mld_power_configure_uapsd(struct iwl_mld *mld,
+					  struct iwl_mld_link *link,
+					  struct iwl_mac_power_cmd *cmd,
+					  bool ps_poll)
+{
+	bool tid_found = false;
+
+	cmd->rx_data_timeout_uapsd =
+		cpu_to_le32(IWL_MLD_UAPSD_RX_DATA_TIMEOUT);
+	cmd->tx_data_timeout_uapsd =
+		cpu_to_le32(IWL_MLD_UAPSD_TX_DATA_TIMEOUT);
+
+	 /* set advanced pm flag with no uapsd ACs to enable ps-poll */
+	if (ps_poll) {
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_ADVANCE_PM_ENA_MSK);
+		return;
+	}
+
+	for (enum ieee80211_ac_numbers ac = IEEE80211_AC_VO;
+	     ac <= IEEE80211_AC_BK;
+	     ac++) {
+		if (!link->queue_params[ac].uapsd)
+			continue;
+
+		cmd->flags |=
+			cpu_to_le16(POWER_FLAGS_ADVANCE_PM_ENA_MSK |
+				    POWER_FLAGS_UAPSD_MISBEHAVING_ENA_MSK);
+
+		cmd->uapsd_ac_flags |= BIT(ac);
+
+		/* QNDP TID - the highest TID with no admission control */
+		if (!tid_found && !link->queue_params[ac].acm) {
+			tid_found = true;
+			switch (ac) {
+			case IEEE80211_AC_VO:
+				cmd->qndp_tid = 6;
+				break;
+			case IEEE80211_AC_VI:
+				cmd->qndp_tid = 5;
+				break;
+			case IEEE80211_AC_BE:
+				cmd->qndp_tid = 0;
+				break;
+			case IEEE80211_AC_BK:
+				cmd->qndp_tid = 1;
+				break;
+			}
+		}
+	}
+
+	if (cmd->uapsd_ac_flags == (BIT(IEEE80211_AC_VO) |
+				    BIT(IEEE80211_AC_VI) |
+				    BIT(IEEE80211_AC_BE) |
+				    BIT(IEEE80211_AC_BK))) {
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_SNOOZE_ENA_MSK);
+		cmd->snooze_interval = cpu_to_le16(IWL_MLD_PS_SNOOZE_INTERVAL);
+		cmd->snooze_window = cpu_to_le16(IWL_MLD_PS_SNOOZE_WINDOW);
+	}
+
+	cmd->uapsd_max_sp = mld->hw->uapsd_max_sp_len;
+}
+
+static void
+iwl_mld_power_config_skip_dtim(struct iwl_mld *mld,
+			       const struct ieee80211_bss_conf *link_conf,
+			       struct iwl_mac_power_cmd *cmd)
+{
+	unsigned int dtimper_tu;
+	unsigned int dtimper;
+	unsigned int skip;
+
+	dtimper = link_conf->dtim_period ?: 1;
+	dtimper_tu = dtimper * link_conf->beacon_int;
+
+	if (dtimper >= 10 || iwl_mld_power_is_radar(mld, link_conf))
+		return;
+
+	if (WARN_ON(!dtimper_tu))
+		return;
+
+	/* configure skip over dtim up to 900 TU DTIM interval */
+	skip = max_t(int, 1, 900 / dtimper_tu);
+
+	cmd->skip_dtim_periods = skip;
+	cmd->flags |= cpu_to_le16(POWER_FLAGS_SKIP_OVER_DTIM_MSK);
+}
+
+#define POWER_KEEP_ALIVE_PERIOD_SEC    25
+static void iwl_mld_power_build_cmd(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    struct iwl_mac_power_cmd *cmd,
+				    bool d3)
+{
+	int dtimper, bi;
+	int keep_alive;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_bss_conf *link_conf = &vif->bss_conf;
+	struct iwl_mld_link *link = &mld_vif->deflink;
+	bool ps_poll = false;
+
+	cmd->id_and_color = cpu_to_le32(mld_vif->fw_id);
+
+	if (ieee80211_vif_is_mld(vif)) {
+		int link_id;
+
+		if (WARN_ON(!vif->active_links))
+			return;
+
+		/* The firmware consumes one single configuration for the vif
+		 * and can't differentiate between links, just pick the lowest
+		 * link_id's configuration and use that.
+		 */
+		link_id = __ffs(vif->active_links);
+		link_conf = link_conf_dereference_check(vif, link_id);
+		link = iwl_mld_link_dereference_check(mld_vif, link_id);
+
+		if (WARN_ON(!link_conf || !link))
+			return;
+	}
+	dtimper = link_conf->dtim_period;
+	bi = link_conf->beacon_int;
+
+	/* Regardless of power management state the driver must set
+	 * keep alive period. FW will use it for sending keep alive NDPs
+	 * immediately after association. Check that keep alive period
+	 * is at least 3 * DTIM
+	 */
+	keep_alive = DIV_ROUND_UP(ieee80211_tu_to_usec(3 * dtimper * bi),
+				  USEC_PER_SEC);
+	keep_alive = max(keep_alive, POWER_KEEP_ALIVE_PERIOD_SEC);
+	cmd->keep_alive_seconds = cpu_to_le16(keep_alive);
+
+	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM)
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_POWER_SAVE_ENA_MSK);
+
+	if (!vif->cfg.ps || iwl_mld_tdls_sta_count(mld) > 0)
+		return;
+
+	cmd->flags |= cpu_to_le16(POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK);
+
+	if (iwl_fw_lookup_cmd_ver(mld->fw, MAC_PM_POWER_TABLE, 0) >= 2)
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_ENABLE_SMPS_MSK);
+
+	/* firmware supports LPRX for beacons at rate 1 Mbps or 6 Mbps only */
+	if (link_conf->beacon_rate &&
+	    (link_conf->beacon_rate->bitrate == 10 ||
+	     link_conf->beacon_rate->bitrate == 60)) {
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_LPRX_ENA_MSK);
+		cmd->lprx_rssi_threshold = POWER_LPRX_RSSI_THRESHOLD;
+	}
+
+	if (d3) {
+		iwl_mld_power_config_skip_dtim(mld, link_conf, cmd);
+		cmd->rx_data_timeout =
+			cpu_to_le32(IWL_MLD_WOWLAN_PS_RX_DATA_TIMEOUT);
+		cmd->tx_data_timeout =
+			cpu_to_le32(IWL_MLD_WOWLAN_PS_TX_DATA_TIMEOUT);
+	} else if (iwl_mld_vif_low_latency(mld_vif) && vif->p2p) {
+		cmd->tx_data_timeout =
+			cpu_to_le32(IWL_MLD_SHORT_PS_TX_DATA_TIMEOUT);
+		cmd->rx_data_timeout =
+			cpu_to_le32(IWL_MLD_SHORT_PS_RX_DATA_TIMEOUT);
+	} else {
+		cmd->rx_data_timeout =
+			cpu_to_le32(IWL_MLD_DEFAULT_PS_RX_DATA_TIMEOUT);
+		cmd->tx_data_timeout =
+			cpu_to_le32(IWL_MLD_DEFAULT_PS_TX_DATA_TIMEOUT);
+	}
+
+	/* uAPSD is only enabled for specific certifications. For those cases,
+	 * mac80211 will allow uAPSD. Always call iwl_mld_power_configure_uapsd
+	 * which will look at what mac80211 is saying.
+	 */
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	ps_poll = mld_vif->use_ps_poll;
+#endif
+	iwl_mld_power_configure_uapsd(mld, link, cmd, ps_poll);
+}
+
+int iwl_mld_update_mac_power(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			     bool d3)
+{
+	struct iwl_mac_power_cmd cmd = {};
+
+	iwl_mld_power_build_cmd(mld, vif, &cmd, d3);
+
+	return iwl_mld_send_cmd_pdu(mld, MAC_PM_POWER_TABLE, &cmd);
+}
+
+static void
+iwl_mld_tpe_sta_cmd_data(struct iwl_txpower_constraints_cmd *cmd,
+			 const struct ieee80211_bss_conf *link)
+{
+	u8 i;
+
+	/* NOTE: the 0 here is IEEE80211_TPE_CAT_6GHZ_DEFAULT,
+	 * we fully ignore IEEE80211_TPE_CAT_6GHZ_SUBORDINATE
+	 */
+
+	BUILD_BUG_ON(ARRAY_SIZE(cmd->psd_pwr) !=
+		     ARRAY_SIZE(link->tpe.psd_local[0].power));
+
+	/* if not valid, mac80211 puts default (max value) */
+	for (i = 0; i < ARRAY_SIZE(cmd->psd_pwr); i++)
+		cmd->psd_pwr[i] = min(link->tpe.psd_local[0].power[i],
+				      link->tpe.psd_reg_client[0].power[i]);
+
+	BUILD_BUG_ON(ARRAY_SIZE(cmd->eirp_pwr) !=
+		     ARRAY_SIZE(link->tpe.max_local[0].power));
+
+	for (i = 0; i < ARRAY_SIZE(cmd->eirp_pwr); i++)
+		cmd->eirp_pwr[i] = min(link->tpe.max_local[0].power[i],
+				       link->tpe.max_reg_client[0].power[i]);
+}
+
+void
+iwl_mld_send_ap_tx_power_constraint_cmd(struct iwl_mld *mld,
+					struct ieee80211_vif *vif,
+					struct ieee80211_bss_conf *link)
+{
+	struct iwl_txpower_constraints_cmd cmd = {};
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (!mld_link->active)
+		return;
+
+	if (link->chanreq.oper.chan->band != NL80211_BAND_6GHZ)
+		return;
+
+	cmd.link_id = cpu_to_le16(mld_link->fw_id);
+	memset(cmd.psd_pwr, DEFAULT_TPE_TX_POWER, sizeof(cmd.psd_pwr));
+	memset(cmd.eirp_pwr, DEFAULT_TPE_TX_POWER, sizeof(cmd.eirp_pwr));
+
+	if (vif->type == NL80211_IFTYPE_AP) {
+		cmd.ap_type = cpu_to_le16(IWL_6GHZ_AP_TYPE_VLP);
+	} else if (link->power_type == IEEE80211_REG_UNSET_AP) {
+		return;
+	} else {
+		cmd.ap_type = cpu_to_le16(link->power_type - 1);
+		iwl_mld_tpe_sta_cmd_data(&cmd, link);
+	}
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(PHY_OPS_GROUP,
+					   AP_TX_POWER_CONSTRAINTS_CMD),
+				   &cmd);
+	if (ret)
+		IWL_ERR(mld,
+			"failed to send AP_TX_POWER_CONSTRAINTS_CMD (%d)\n",
+			ret);
+}
+
+int iwl_mld_set_tx_power(struct iwl_mld *mld,
+			 struct ieee80211_bss_conf *link_conf,
+			 s16 tx_power)
+{
+	u32 cmd_id = REDUCE_TX_POWER_CMD;
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link_conf);
+	u16 u_tx_power = tx_power == IWL_DEFAULT_MAX_TX_POWER ?
+		IWL_DEV_MAX_TX_POWER : 8 * tx_power;
+	struct iwl_dev_tx_power_cmd cmd = {
+		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_LINK),
+		.common.pwr_restriction = cpu_to_le16(u_tx_power),
+	};
+	int len = sizeof(cmd.common) + sizeof(cmd.v10);
+
+	if (WARN_ON(!mld_link))
+		return -ENODEV;
+
+	cmd.common.link_id = cpu_to_le32(mld_link->fw_id);
+
+	return iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd, len);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/power.h b/sys/contrib/dev/iwlwifi/mld/power.h
new file mode 100644
index 000000000000..05ce27bef106
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/power.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_power_h__
+#define __iwl_mld_power_h__
+
+#include <net/mac80211.h>
+
+#include "mld.h"
+
+int iwl_mld_update_device_power(struct iwl_mld *mld, bool d3);
+
+int iwl_mld_enable_beacon_filter(struct iwl_mld *mld,
+				 const struct ieee80211_bss_conf *link_conf,
+				 bool d3);
+
+int iwl_mld_disable_beacon_filter(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif);
+
+int iwl_mld_update_mac_power(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			     bool d3);
+
+void
+iwl_mld_send_ap_tx_power_constraint_cmd(struct iwl_mld *mld,
+					struct ieee80211_vif *vif,
+					struct ieee80211_bss_conf *link);
+
+int iwl_mld_set_tx_power(struct iwl_mld *mld,
+			 struct ieee80211_bss_conf *link_conf,
+			 s16 tx_power);
+
+#endif /* __iwl_mld_power_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/ptp.c b/sys/contrib/dev/iwlwifi/mld/ptp.c
new file mode 100644
index 000000000000..ffeb37a7f830
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/ptp.c
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+
+#include "mld.h"
+#include "iwl-debug.h"
+#include "hcmd.h"
+#include "ptp.h"
+#include <linux/timekeeping.h>
+
+/* The scaled_ppm parameter is ppm (parts per million) with a 16-bit fractional
+ * part, which means that a value of 1 in one of those fields actually means
+ * 2^-16 ppm, and 2^16=65536 is 1 ppm.
+ */
+#define PTP_SCALE_FACTOR	65536000000ULL
+
+#define IWL_PTP_GP2_WRAP	0x100000000ULL
+#define IWL_PTP_WRAP_TIME	(3600 * HZ)
+#define IWL_PTP_WRAP_THRESHOLD_USEC	(5000)
+
+static int iwl_mld_get_systime(struct iwl_mld *mld, u32 *gp2)
+{
+	*gp2 = iwl_read_prph(mld->trans, mld->trans->mac_cfg->base->gp2_reg_addr);
+
+	if (*gp2 == 0x5a5a5a5a)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void iwl_mld_ptp_update_new_read(struct iwl_mld *mld, u32 gp2)
+{
+	IWL_DEBUG_PTP(mld, "PTP: last_gp2=%u, new gp2 read=%u\n",
+		      mld->ptp_data.last_gp2, gp2);
+
+	/* If the difference is above the threshold, assume it's a wraparound.
+	 * Otherwise assume it's an old read and ignore it.
+	 */
+	if (gp2 < mld->ptp_data.last_gp2) {
+		if (mld->ptp_data.last_gp2 - gp2 <
+		    IWL_PTP_WRAP_THRESHOLD_USEC) {
+			IWL_DEBUG_PTP(mld,
+				      "PTP: ignore old read (gp2=%u, last_gp2=%u)\n",
+				      gp2, mld->ptp_data.last_gp2);
+			return;
+		}
+
+		mld->ptp_data.wrap_counter++;
+		IWL_DEBUG_PTP(mld,
+			      "PTP: wraparound detected (new counter=%u)\n",
+			      mld->ptp_data.wrap_counter);
+	}
+
+	mld->ptp_data.last_gp2 = gp2;
+	schedule_delayed_work(&mld->ptp_data.dwork, IWL_PTP_WRAP_TIME);
+}
+
+u64 iwl_mld_ptp_get_adj_time(struct iwl_mld *mld, u64 base_time_ns)
+{
+	struct ptp_data *data = &mld->ptp_data;
+	u64 scale_time_gp2_ns = mld->ptp_data.scale_update_gp2 * NSEC_PER_USEC;
+	u64 res;
+	u64 diff;
+	s64 scaled_diff;
+
+	lockdep_assert_held(&data->lock);
+
+	iwl_mld_ptp_update_new_read(mld,
+				    div64_u64(base_time_ns, NSEC_PER_USEC));
+
+	base_time_ns = base_time_ns +
+		(data->wrap_counter * IWL_PTP_GP2_WRAP * NSEC_PER_USEC);
+
+	/* It is possible that a GP2 timestamp was received from fw before the
+	 * last scale update.
+	 */
+	if (base_time_ns < scale_time_gp2_ns) {
+		diff = scale_time_gp2_ns - base_time_ns;
+		scaled_diff = -mul_u64_u64_div_u64(diff,
+						   data->scaled_freq,
+						   PTP_SCALE_FACTOR);
+	} else {
+		diff = base_time_ns - scale_time_gp2_ns;
+		scaled_diff = mul_u64_u64_div_u64(diff,
+						  data->scaled_freq,
+						  PTP_SCALE_FACTOR);
+	}
+
+	IWL_DEBUG_PTP(mld, "base_time=%llu diff ns=%llu scaled_diff_ns=%lld\n",
+		      (unsigned long long)base_time_ns,
+		      (unsigned long long)diff, (long long)scaled_diff);
+
+	res = data->scale_update_adj_time_ns + data->delta + scaled_diff;
+
+	IWL_DEBUG_PTP(mld, "scale_update_ns=%llu delta=%lld adj=%llu\n",
+		      (unsigned long long)data->scale_update_adj_time_ns,
+		      (long long)data->delta, (unsigned long long)res);
+	return res;
+}
+
+static int iwl_mld_ptp_gettime(struct ptp_clock_info *ptp,
+			       struct timespec64 *ts)
+{
+	struct iwl_mld *mld = container_of(ptp, struct iwl_mld,
+					   ptp_data.ptp_clock_info);
+	struct ptp_data *data = &mld->ptp_data;
+	u32 gp2;
+	u64 ns;
+
+	if (iwl_mld_get_systime(mld, &gp2)) {
+		IWL_DEBUG_PTP(mld, "PTP: gettime: failed to read systime\n");
+		return -EIO;
+	}
+
+	spin_lock_bh(&data->lock);
+	ns = iwl_mld_ptp_get_adj_time(mld, (u64)gp2 * NSEC_PER_USEC);
+	spin_unlock_bh(&data->lock);
+
+	*ts = ns_to_timespec64(ns);
+	return 0;
+}
+
+static int iwl_mld_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	struct iwl_mld *mld = container_of(ptp, struct iwl_mld,
+					   ptp_data.ptp_clock_info);
+	struct ptp_data *data = &mld->ptp_data;
+
+	spin_lock_bh(&data->lock);
+	data->delta += delta;
+	IWL_DEBUG_PTP(mld, "delta=%lld, new delta=%lld\n", (long long)delta,
+		      (long long)data->delta);
+	spin_unlock_bh(&data->lock);
+	return 0;
+}
+
+static int iwl_mld_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
+{
+	struct iwl_mld *mld = container_of(ptp, struct iwl_mld,
+					   ptp_data.ptp_clock_info);
+	struct ptp_data *data = &mld->ptp_data;
+	u32 gp2;
+
+	/* Must call iwl_mld_ptp_get_adj_time() before updating
+	 * data->scale_update_gp2 or data->scaled_freq since
+	 * scale_update_adj_time_ns should reflect the previous scaled_freq.
+	 */
+	if (iwl_mld_get_systime(mld, &gp2)) {
+		IWL_DEBUG_PTP(mld, "adjfine: failed to read systime\n");
+		return -EBUSY;
+	}
+
+	spin_lock_bh(&data->lock);
+	data->scale_update_adj_time_ns =
+		iwl_mld_ptp_get_adj_time(mld, gp2 * NSEC_PER_USEC);
+	data->scale_update_gp2 = gp2;
+
+	/* scale_update_adj_time_ns now relects the configured delta, the
+	 * wrap_counter and the previous scaled frequency. Thus delta and
+	 * wrap_counter should be reset, and the scale frequency is updated
+	 * to the new frequency.
+	 */
+	data->delta = 0;
+	data->wrap_counter = 0;
+	data->scaled_freq = PTP_SCALE_FACTOR + scaled_ppm;
+	IWL_DEBUG_PTP(mld, "adjfine: scaled_ppm=%ld new=%llu\n",
+		      scaled_ppm, (unsigned long long)data->scaled_freq);
+	spin_unlock_bh(&data->lock);
+	return 0;
+}
+
+static void iwl_mld_ptp_work(struct work_struct *wk)
+{
+	struct iwl_mld *mld = container_of(wk, struct iwl_mld,
+					   ptp_data.dwork.work);
+	struct ptp_data *data = &mld->ptp_data;
+	u32 gp2;
+
+	spin_lock_bh(&data->lock);
+	if (!iwl_mld_get_systime(mld, &gp2))
+		iwl_mld_ptp_update_new_read(mld, gp2);
+	else
+		IWL_DEBUG_PTP(mld, "PTP work: failed to read GP2\n");
+	spin_unlock_bh(&data->lock);
+}
+
+static int
+iwl_mld_get_crosstimestamp_fw(struct iwl_mld *mld, u32 *gp2, u64 *sys_time)
+{
+	struct iwl_synced_time_cmd synced_time_cmd = {
+		.operation = cpu_to_le32(IWL_SYNCED_TIME_OPERATION_READ_BOTH)
+	};
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(DATA_PATH_GROUP, WNM_PLATFORM_PTM_REQUEST_CMD),
+		.flags = CMD_WANT_SKB,
+		.data[0] = &synced_time_cmd,
+		.len[0] = sizeof(synced_time_cmd),
+	};
+	struct iwl_synced_time_rsp *resp;
+	struct iwl_rx_packet *pkt;
+	int ret;
+	u64 gp2_10ns;
+
+	wiphy_lock(mld->wiphy);
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	wiphy_unlock(mld->wiphy);
+	if (ret)
+		return ret;
+
+	pkt = cmd.resp_pkt;
+
+	if (iwl_rx_packet_payload_len(pkt) != sizeof(*resp)) {
+		IWL_DEBUG_PTP(mld, "PTP: Invalid PTM command response\n");
+		iwl_free_resp(&cmd);
+		return -EIO;
+	}
+
+	resp = (void *)pkt->data;
+
+	gp2_10ns = (u64)le32_to_cpu(resp->gp2_timestamp_hi) << 32 |
+		le32_to_cpu(resp->gp2_timestamp_lo);
+	*gp2 = div_u64(gp2_10ns, 100);
+
+	*sys_time = (u64)le32_to_cpu(resp->platform_timestamp_hi) << 32 |
+		le32_to_cpu(resp->platform_timestamp_lo);
+
+	iwl_free_resp(&cmd);
+	return ret;
+}
+
+static int
+iwl_mld_phc_get_crosstimestamp(struct ptp_clock_info *ptp,
+			       struct system_device_crosststamp *xtstamp)
+{
+	struct iwl_mld *mld = container_of(ptp, struct iwl_mld,
+					   ptp_data.ptp_clock_info);
+	struct ptp_data *data = &mld->ptp_data;
+	int ret = 0;
+	/* Raw value read from GP2 register in usec */
+	u32 gp2;
+	/* GP2 value in ns*/
+	s64 gp2_ns;
+	/* System (wall) time */
+	ktime_t sys_time;
+
+	memset(xtstamp, 0, sizeof(struct system_device_crosststamp));
+
+	ret = iwl_mld_get_crosstimestamp_fw(mld, &gp2, &sys_time);
+	if (ret) {
+		IWL_DEBUG_PTP(mld,
+			      "PTP: fw get_crosstimestamp failed (ret=%d)\n",
+			      ret);
+		return ret;
+	}
+
+	spin_lock_bh(&data->lock);
+	gp2_ns = iwl_mld_ptp_get_adj_time(mld, (u64)gp2 * NSEC_PER_USEC);
+	spin_unlock_bh(&data->lock);
+
+	IWL_DEBUG_PTP(mld,
+		      "Got Sync Time: GP2:%u, last_GP2: %u, GP2_ns: %lld, sys_time: %lld\n",
+		      gp2, mld->ptp_data.last_gp2, gp2_ns, (s64)sys_time);
+
+	/* System monotonic raw time is not used */
+	xtstamp->device = ns_to_ktime(gp2_ns);
+	xtstamp->sys_realtime = sys_time;
+
+	return ret;
+}
+
+void iwl_mld_ptp_init(struct iwl_mld *mld)
+{
+	if (WARN_ON(mld->ptp_data.ptp_clock))
+		return;
+
+	spin_lock_init(&mld->ptp_data.lock);
+	INIT_DELAYED_WORK(&mld->ptp_data.dwork, iwl_mld_ptp_work);
+
+	mld->ptp_data.ptp_clock_info.owner = THIS_MODULE;
+	mld->ptp_data.ptp_clock_info.gettime64 = iwl_mld_ptp_gettime;
+	mld->ptp_data.ptp_clock_info.max_adj = 0x7fffffff;
+	mld->ptp_data.ptp_clock_info.adjtime = iwl_mld_ptp_adjtime;
+	mld->ptp_data.ptp_clock_info.adjfine = iwl_mld_ptp_adjfine;
+	mld->ptp_data.scaled_freq = PTP_SCALE_FACTOR;
+	mld->ptp_data.ptp_clock_info.getcrosststamp =
+					iwl_mld_phc_get_crosstimestamp;
+
+	/* Give a short 'friendly name' to identify the PHC clock */
+	snprintf(mld->ptp_data.ptp_clock_info.name,
+		 sizeof(mld->ptp_data.ptp_clock_info.name),
+		 "%s", "iwlwifi-PTP");
+
+	mld->ptp_data.ptp_clock =
+		ptp_clock_register(&mld->ptp_data.ptp_clock_info, mld->dev);
+
+	if (IS_ERR_OR_NULL(mld->ptp_data.ptp_clock)) {
+		IWL_ERR(mld, "Failed to register PHC clock (%ld)\n",
+			PTR_ERR(mld->ptp_data.ptp_clock));
+		mld->ptp_data.ptp_clock = NULL;
+	} else {
+		IWL_DEBUG_INFO(mld, "Registered PHC clock: %s, with index: %d\n",
+			       mld->ptp_data.ptp_clock_info.name,
+			       ptp_clock_index(mld->ptp_data.ptp_clock));
+	}
+}
+
+void iwl_mld_ptp_remove(struct iwl_mld *mld)
+{
+	if (mld->ptp_data.ptp_clock) {
+		IWL_DEBUG_INFO(mld, "Unregistering PHC clock: %s, with index: %d\n",
+			       mld->ptp_data.ptp_clock_info.name,
+			       ptp_clock_index(mld->ptp_data.ptp_clock));
+
+		ptp_clock_unregister(mld->ptp_data.ptp_clock);
+		mld->ptp_data.ptp_clock = NULL;
+		mld->ptp_data.last_gp2 = 0;
+		mld->ptp_data.wrap_counter = 0;
+		cancel_delayed_work_sync(&mld->ptp_data.dwork);
+	}
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/ptp.h b/sys/contrib/dev/iwlwifi/mld/ptp.h
new file mode 100644
index 000000000000..f3d18dd304e5
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/ptp.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+#ifndef __iwl_mld_ptp_h__
+#define __iwl_mld_ptp_h__
+
+#include <linux/ptp_clock_kernel.h>
+
+/**
+ * struct ptp_data - PTP hardware clock data
+ *
+ * @ptp_clock: struct ptp_clock pointer returned by the ptp_clock_register()
+ *	function.
+ * @ptp_clock_info: struct ptp_clock_info that describes a PTP hardware clock
+ * @lock: protects the time adjustments data
+ * @delta: delta between hardware clock and ptp clock in nanoseconds
+ * @scale_update_gp2: GP2 time when the scale was last updated
+ * @scale_update_adj_time_ns: adjusted time when the scale was last updated,
+ *	in nanoseconds
+ * @scaled_freq: clock frequency offset, scaled to 65536000000
+ * @last_gp2: the last GP2 reading from the hardware, used for tracking GP2
+ *	wraparounds
+ * @wrap_counter: number of wraparounds since scale_update_adj_time_ns
+ * @dwork: worker scheduled every 1 hour to detect workarounds
+ */
+struct ptp_data {
+	struct ptp_clock *ptp_clock;
+	struct ptp_clock_info ptp_clock_info;
+
+	spinlock_t lock;
+	s64 delta;
+	u32 scale_update_gp2;
+	u64 scale_update_adj_time_ns;
+	u64 scaled_freq;
+	u32 last_gp2;
+	u32 wrap_counter;
+	struct delayed_work dwork;
+};
+
+void iwl_mld_ptp_init(struct iwl_mld *mld);
+void iwl_mld_ptp_remove(struct iwl_mld *mld);
+u64 iwl_mld_ptp_get_adj_time(struct iwl_mld *mld, u64 base_time_ns);
+
+#endif /* __iwl_mld_ptp_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/regulatory.c b/sys/contrib/dev/iwlwifi/mld/regulatory.c
new file mode 100644
index 000000000000..75d2f5cb23a7
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/regulatory.c
@@ -0,0 +1,369 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include <linux/dmi.h>
+
+#include "fw/regulatory.h"
+#include "fw/acpi.h"
+#include "fw/uefi.h"
+
+#include "regulatory.h"
+#include "mld.h"
+#include "hcmd.h"
+
+void iwl_mld_get_bios_tables(struct iwl_mld *mld)
+{
+	int ret;
+
+	iwl_acpi_get_guid_lock_status(&mld->fwrt);
+
+	ret = iwl_bios_get_ppag_table(&mld->fwrt);
+	if (ret < 0) {
+		IWL_DEBUG_RADIO(mld,
+				"PPAG BIOS table invalid or unavailable. (%d)\n",
+				ret);
+	}
+
+	ret = iwl_bios_get_wrds_table(&mld->fwrt);
+	if (ret < 0) {
+		IWL_DEBUG_RADIO(mld,
+				"WRDS SAR BIOS table invalid or unavailable. (%d)\n",
+				ret);
+
+		/* If not available, don't fail and don't bother with EWRD and
+		 * WGDS
+		 */
+
+		if (!iwl_bios_get_wgds_table(&mld->fwrt)) {
+			/* If basic SAR is not available, we check for WGDS,
+			 * which should *not* be available either. If it is
+			 * available, issue an error, because we can't use SAR
+			 * Geo without basic SAR.
+			 */
+			IWL_ERR(mld, "BIOS contains WGDS but no WRDS\n");
+		}
+
+	} else {
+		ret = iwl_bios_get_ewrd_table(&mld->fwrt);
+		/* If EWRD is not available, we can still use
+		 * WRDS, so don't fail.
+		 */
+		if (ret < 0)
+			IWL_DEBUG_RADIO(mld,
+					"EWRD SAR BIOS table invalid or unavailable. (%d)\n",
+					ret);
+
+		ret = iwl_bios_get_wgds_table(&mld->fwrt);
+		if (ret < 0)
+			IWL_DEBUG_RADIO(mld,
+					"Geo SAR BIOS table invalid or unavailable. (%d)\n",
+					ret);
+		/* we don't fail if the table is not available */
+	}
+
+	iwl_uefi_get_uats_table(mld->trans, &mld->fwrt);
+
+	iwl_bios_get_phy_filters(&mld->fwrt);
+}
+
+static int iwl_mld_geo_sar_init(struct iwl_mld *mld)
+{
+	u32 cmd_id = WIDE_ID(PHY_OPS_GROUP, PER_CHAIN_LIMIT_OFFSET_CMD);
+	/* Only set to South Korea if the table revision is 1 */
+	__le32 sk = cpu_to_le32(mld->fwrt.geo_rev == 1 ? 1 : 0);
+	union iwl_geo_tx_power_profiles_cmd cmd = {
+		.v5.ops = cpu_to_le32(IWL_PER_CHAIN_OFFSET_SET_TABLES),
+		.v5.table_revision = sk,
+	};
+	int ret;
+
+	ret = iwl_sar_geo_fill_table(&mld->fwrt, &cmd.v5.table[0][0],
+				     ARRAY_SIZE(cmd.v5.table[0]),
+				     BIOS_GEO_MAX_PROFILE_NUM);
+
+	/* It is a valid scenario to not support SAR, or miss wgds table,
+	 * but in that case there is no need to send the command.
+	 */
+	if (ret)
+		return 0;
+
+	return iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd, sizeof(cmd.v5));
+}
+
+int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b)
+{
+	u32 cmd_id = REDUCE_TX_POWER_CMD;
+	struct iwl_dev_tx_power_cmd cmd = {
+		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_CHAINS),
+		.v10.flags = cpu_to_le32(mld->fwrt.reduced_power_flags),
+	};
+	int ret;
+
+	/* TODO: CDB - support IWL_NUM_CHAIN_TABLES_V2 */
+	ret = iwl_sar_fill_profile(&mld->fwrt, &cmd.v10.per_chain[0][0][0],
+				   IWL_NUM_CHAIN_TABLES, IWL_NUM_SUB_BANDS_V2,
+				   prof_a, prof_b);
+	/* return on error or if the profile is disabled (positive number) */
+	if (ret)
+		return ret;
+
+	return iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd,
+				    sizeof(cmd.common) + sizeof(cmd.v10));
+}
+
+int iwl_mld_init_sar(struct iwl_mld *mld)
+{
+	int chain_a_prof = 1;
+	int chain_b_prof = 1;
+	int ret;
+
+	/* If no profile was chosen by the user yet, choose profile 1 (WRDS) as
+	 * default for both chains
+	 */
+	if (mld->fwrt.sar_chain_a_profile && mld->fwrt.sar_chain_b_profile) {
+		chain_a_prof = mld->fwrt.sar_chain_a_profile;
+		chain_b_prof = mld->fwrt.sar_chain_b_profile;
+	}
+
+	ret = iwl_mld_config_sar_profile(mld, chain_a_prof, chain_b_prof);
+	if (ret < 0)
+		return ret;
+
+	if (ret)
+		return 0;
+
+	return iwl_mld_geo_sar_init(mld);
+}
+
+int iwl_mld_init_sgom(struct iwl_mld *mld)
+{
+	int ret;
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP,
+			      SAR_OFFSET_MAPPING_TABLE_CMD),
+		.data[0] = &mld->fwrt.sgom_table,
+		.len[0] =  sizeof(mld->fwrt.sgom_table),
+		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
+	};
+
+	if (!mld->fwrt.sgom_enabled) {
+		IWL_DEBUG_RADIO(mld, "SGOM table is disabled\n");
+		return 0;
+	}
+
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	if (ret)
+		IWL_ERR(mld,
+			"failed to send SAR_OFFSET_MAPPING_CMD (%d)\n", ret);
+
+	return ret;
+}
+
+static int iwl_mld_ppag_send_cmd(struct iwl_mld *mld)
+{
+	union iwl_ppag_table_cmd cmd = {};
+	int ret, len;
+
+	ret = iwl_fill_ppag_table(&mld->fwrt, &cmd, &len);
+	/* Not supporting PPAG table is a valid scenario */
+	if (ret < 0)
+		return 0;
+
+	IWL_DEBUG_RADIO(mld, "Sending PER_PLATFORM_ANT_GAIN_CMD\n");
+	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(PHY_OPS_GROUP,
+						PER_PLATFORM_ANT_GAIN_CMD),
+				   &cmd, len);
+	if (ret < 0)
+		IWL_ERR(mld, "failed to send PER_PLATFORM_ANT_GAIN_CMD (%d)\n",
+			ret);
+
+	return ret;
+}
+
+int iwl_mld_init_ppag(struct iwl_mld *mld)
+{
+	/* no need to read the table, done in INIT stage */
+
+	if (!(iwl_is_ppag_approved(&mld->fwrt)))
+		return 0;
+
+	return iwl_mld_ppag_send_cmd(mld);
+}
+
+void iwl_mld_configure_lari(struct iwl_mld *mld)
+{
+	struct iwl_fw_runtime *fwrt = &mld->fwrt;
+	struct iwl_lari_config_change_cmd cmd = {
+		.config_bitmap = iwl_get_lari_config_bitmap(fwrt),
+	};
+	bool has_raw_dsm_capa = fw_has_capa(&fwrt->fw->ucode_capa,
+					    IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE);
+	int ret;
+	u32 value;
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_11AX_ENABLEMENT, &value);
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= DSM_11AX_ALLOW_BITMAP;
+		cmd.oem_11ax_allow_bitmap = cpu_to_le32(value);
+	}
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ENABLE_UNII4_CHAN, &value);
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= DSM_UNII4_ALLOW_BITMAP;
+		cmd.oem_unii4_allow_bitmap = cpu_to_le32(value);
+	}
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ACTIVATE_CHANNEL, &value);
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= CHAN_STATE_ACTIVE_BITMAP_CMD_V12;
+		cmd.chan_state_active_bitmap = cpu_to_le32(value);
+	}
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ENABLE_6E, &value);
+	if (!ret)
+		cmd.oem_uhb_allow_bitmap = cpu_to_le32(value);
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_FORCE_DISABLE_CHANNELS, &value);
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= DSM_FORCE_DISABLE_CHANNELS_ALLOWED_BITMAP;
+		cmd.force_disable_channels_bitmap = cpu_to_le32(value);
+	}
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ENERGY_DETECTION_THRESHOLD,
+			       &value);
+	if (!ret) {
+		if (!has_raw_dsm_capa)
+			value &= DSM_EDT_ALLOWED_BITMAP;
+		cmd.edt_bitmap = cpu_to_le32(value);
+	}
+
+	ret = iwl_bios_get_wbem(fwrt, &value);
+	if (!ret)
+		cmd.oem_320mhz_allow_bitmap = cpu_to_le32(value);
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ENABLE_11BE, &value);
+	if (!ret)
+		cmd.oem_11be_allow_bitmap = cpu_to_le32(value);
+
+	if (!cmd.config_bitmap &&
+	    !cmd.oem_uhb_allow_bitmap &&
+	    !cmd.oem_11ax_allow_bitmap &&
+	    !cmd.oem_unii4_allow_bitmap &&
+	    !cmd.chan_state_active_bitmap &&
+	    !cmd.force_disable_channels_bitmap &&
+	    !cmd.edt_bitmap &&
+	    !cmd.oem_320mhz_allow_bitmap &&
+	    !cmd.oem_11be_allow_bitmap)
+		return;
+
+	IWL_DEBUG_RADIO(mld,
+			"sending LARI_CONFIG_CHANGE, config_bitmap=0x%x, oem_11ax_allow_bitmap=0x%x\n",
+			le32_to_cpu(cmd.config_bitmap),
+			le32_to_cpu(cmd.oem_11ax_allow_bitmap));
+	IWL_DEBUG_RADIO(mld,
+			"sending LARI_CONFIG_CHANGE, oem_unii4_allow_bitmap=0x%x, chan_state_active_bitmap=0x%x\n",
+			le32_to_cpu(cmd.oem_unii4_allow_bitmap),
+			le32_to_cpu(cmd.chan_state_active_bitmap));
+	IWL_DEBUG_RADIO(mld,
+			"sending LARI_CONFIG_CHANGE, oem_uhb_allow_bitmap=0x%x, force_disable_channels_bitmap=0x%x\n",
+			le32_to_cpu(cmd.oem_uhb_allow_bitmap),
+			le32_to_cpu(cmd.force_disable_channels_bitmap));
+	IWL_DEBUG_RADIO(mld,
+			"sending LARI_CONFIG_CHANGE, edt_bitmap=0x%x, oem_320mhz_allow_bitmap=0x%x\n",
+			le32_to_cpu(cmd.edt_bitmap),
+			le32_to_cpu(cmd.oem_320mhz_allow_bitmap));
+	IWL_DEBUG_RADIO(mld,
+			"sending LARI_CONFIG_CHANGE, oem_11be_allow_bitmap=0x%x\n",
+			le32_to_cpu(cmd.oem_11be_allow_bitmap));
+
+	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(REGULATORY_AND_NVM_GROUP,
+						LARI_CONFIG_CHANGE), &cmd);
+	if (ret)
+		IWL_DEBUG_RADIO(mld,
+				"Failed to send LARI_CONFIG_CHANGE (%d)\n",
+				ret);
+}
+
+void iwl_mld_init_uats(struct iwl_mld *mld)
+{
+	int ret;
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP,
+			      MCC_ALLOWED_AP_TYPE_CMD),
+		.data[0] = &mld->fwrt.uats_table,
+		.len[0] =  sizeof(mld->fwrt.uats_table),
+		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
+	};
+
+	if (!mld->fwrt.uats_valid)
+		return;
+
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	if (ret)
+		IWL_ERR(mld, "failed to send MCC_ALLOWED_AP_TYPE_CMD (%d)\n",
+			ret);
+}
+
+void iwl_mld_init_tas(struct iwl_mld *mld)
+{
+	int ret;
+	struct iwl_tas_data data = {};
+	struct iwl_tas_config_cmd cmd = {};
+	u32 cmd_id = WIDE_ID(REGULATORY_AND_NVM_GROUP, TAS_CONFIG);
+
+	BUILD_BUG_ON(ARRAY_SIZE(data.block_list_array) !=
+		     IWL_WTAS_BLACK_LIST_MAX);
+	BUILD_BUG_ON(ARRAY_SIZE(cmd.block_list_array) !=
+		     IWL_WTAS_BLACK_LIST_MAX);
+
+	if (!fw_has_capa(&mld->fw->ucode_capa, IWL_UCODE_TLV_CAPA_TAS_CFG)) {
+		IWL_DEBUG_RADIO(mld, "TAS not enabled in FW\n");
+		return;
+	}
+
+	ret = iwl_bios_get_tas_table(&mld->fwrt, &data);
+	if (ret < 0) {
+		IWL_DEBUG_RADIO(mld,
+				"TAS table invalid or unavailable. (%d)\n",
+				ret);
+		return;
+	}
+
+	if (!iwl_is_tas_approved()) {
+		IWL_DEBUG_RADIO(mld,
+				"System vendor '%s' is not in the approved list, disabling TAS in US and Canada.\n",
+				dmi_get_system_info(DMI_SYS_VENDOR) ?: "<unknown>");
+		if ((!iwl_add_mcc_to_tas_block_list(data.block_list_array,
+						    &data.block_list_size,
+						    IWL_MCC_US)) ||
+		    (!iwl_add_mcc_to_tas_block_list(data.block_list_array,
+						    &data.block_list_size,
+						    IWL_MCC_CANADA))) {
+			IWL_DEBUG_RADIO(mld,
+					"Unable to add US/Canada to TAS block list, disabling TAS\n");
+			return;
+		}
+	} else {
+		IWL_DEBUG_RADIO(mld,
+				"System vendor '%s' is in the approved list.\n",
+				dmi_get_system_info(DMI_SYS_VENDOR) ?: "<unknown>");
+	}
+
+	cmd.block_list_size = cpu_to_le16(data.block_list_size);
+	for (u8 i = 0; i < data.block_list_size; i++)
+		cmd.block_list_array[i] =
+			cpu_to_le16(data.block_list_array[i]);
+	cmd.tas_config_info.table_source = data.table_source;
+	cmd.tas_config_info.table_revision = data.table_revision;
+	cmd.tas_config_info.value = cpu_to_le32(data.tas_selection);
+
+	ret = iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd);
+	if (ret)
+		IWL_DEBUG_RADIO(mld, "failed to send TAS_CONFIG (%d)\n", ret);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/regulatory.h b/sys/contrib/dev/iwlwifi/mld/regulatory.h
new file mode 100644
index 000000000000..3b01c645adda
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/regulatory.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_regulatory_h__
+#define __iwl_mld_regulatory_h__
+
+#include "mld.h"
+
+void iwl_mld_get_bios_tables(struct iwl_mld *mld);
+void iwl_mld_configure_lari(struct iwl_mld *mld);
+void iwl_mld_init_uats(struct iwl_mld *mld);
+void iwl_mld_init_tas(struct iwl_mld *mld);
+
+int iwl_mld_init_ppag(struct iwl_mld *mld);
+
+int iwl_mld_init_sgom(struct iwl_mld *mld);
+
+int iwl_mld_init_sar(struct iwl_mld *mld);
+
+int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b);
+
+#endif /* __iwl_mld_regulatory_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/roc.c b/sys/contrib/dev/iwlwifi/mld/roc.c
new file mode 100644
index 000000000000..e85f45bce79a
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/roc.c
@@ -0,0 +1,265 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024 - 2025 Intel Corporation
+ */
+#include <net/cfg80211.h>
+#include <net/mac80211.h>
+
+#include "mld.h"
+#include "roc.h"
+#include "hcmd.h"
+#include "iface.h"
+#include "sta.h"
+#include "mlo.h"
+
+#include "fw/api/context.h"
+#include "fw/api/time-event.h"
+
+#define AUX_ROC_MAX_DELAY MSEC_TO_TU(200)
+
+static void
+iwl_mld_vif_iter_emlsr_block_roc(void *data, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int *result = data;
+	int ret;
+
+	ret = iwl_mld_block_emlsr_sync(mld_vif->mld, vif,
+				       IWL_MLD_EMLSR_BLOCKED_ROC,
+				       iwl_mld_get_primary_link(vif));
+	if (ret)
+		*result = ret;
+}
+
+struct iwl_mld_roc_iter_data {
+	enum iwl_roc_activity activity;
+	struct ieee80211_vif *vif;
+	bool found;
+};
+
+static void iwl_mld_find_roc_vif_iter(void *data, u8 *mac,
+				      struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_roc_iter_data *roc_data = data;
+
+	if (mld_vif->roc_activity != roc_data->activity)
+		return;
+
+	/* The FW supports one ROC of each type simultaneously */
+	if (WARN_ON(roc_data->found)) {
+		roc_data->vif = NULL;
+		return;
+	}
+
+	roc_data->found = true;
+	roc_data->vif = vif;
+}
+
+static struct ieee80211_vif *
+iwl_mld_find_roc_vif(struct iwl_mld *mld, enum iwl_roc_activity activity)
+{
+	struct iwl_mld_roc_iter_data roc_data = {
+		.activity = activity,
+		.found = false,
+	};
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_find_roc_vif_iter,
+						&roc_data);
+
+	return roc_data.vif;
+}
+
+int iwl_mld_start_roc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		      struct ieee80211_channel *channel, int duration,
+		      enum ieee80211_roc_type type)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_int_sta *aux_sta = &mld_vif->aux_sta;
+	struct iwl_roc_req cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_ADD),
+	};
+	u8 ver = iwl_fw_lookup_cmd_ver(mld->fw,
+				       WIDE_ID(MAC_CONF_GROUP, ROC_CMD), 0);
+	u16 cmd_len = ver < 6 ? sizeof(struct iwl_roc_req_v5) : sizeof(cmd);
+	enum iwl_roc_activity activity;
+	int ret = 0;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+	    vif->type != NL80211_IFTYPE_STATION) {
+		IWL_ERR(mld, "NOT SUPPORTED: ROC on vif->type %d\n",
+			vif->type);
+
+		return -EOPNOTSUPP;
+	}
+
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
+		switch (type) {
+		case IEEE80211_ROC_TYPE_NORMAL:
+			activity = ROC_ACTIVITY_P2P_DISC;
+			break;
+		case IEEE80211_ROC_TYPE_MGMT_TX:
+			activity = ROC_ACTIVITY_P2P_NEG;
+			break;
+		default:
+			WARN_ONCE(1, "Got an invalid P2P ROC type\n");
+			return -EINVAL;
+		}
+	} else {
+		activity = ROC_ACTIVITY_HOTSPOT;
+	}
+
+	/* The FW supports one ROC of each type simultaneously */
+	if (WARN_ON(iwl_mld_find_roc_vif(mld, activity)))
+		return -EBUSY;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_vif_iter_emlsr_block_roc,
+						&ret);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_add_aux_sta(mld, aux_sta);
+	if (ret)
+		return ret;
+
+	cmd.activity = cpu_to_le32(activity);
+	cmd.sta_id = cpu_to_le32(aux_sta->sta_id);
+	cmd.channel_info.channel = cpu_to_le32(channel->hw_value);
+	cmd.channel_info.band = iwl_mld_nl80211_band_to_fw(channel->band);
+	cmd.channel_info.width = IWL_PHY_CHANNEL_MODE20;
+	cmd.max_delay = cpu_to_le32(AUX_ROC_MAX_DELAY);
+	cmd.duration = cpu_to_le32(MSEC_TO_TU(duration));
+
+	memcpy(cmd.node_addr, vif->addr, ETH_ALEN);
+
+	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(MAC_CONF_GROUP, ROC_CMD),
+				   &cmd, cmd_len);
+	if (ret) {
+		IWL_ERR(mld, "Couldn't send the ROC_CMD\n");
+		return ret;
+	}
+
+	mld_vif->roc_activity = activity;
+
+	return 0;
+}
+
+static void
+iwl_mld_vif_iter_emlsr_unblock_roc(void *data, u8 *mac,
+				   struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	iwl_mld_unblock_emlsr(mld_vif->mld, vif, IWL_MLD_EMLSR_BLOCKED_ROC);
+}
+
+static void iwl_mld_destroy_roc(struct iwl_mld *mld,
+				struct ieee80211_vif *vif,
+				struct iwl_mld_vif *mld_vif)
+{
+	mld_vif->roc_activity = ROC_NUM_ACTIVITIES;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_vif_iter_emlsr_unblock_roc,
+						NULL);
+
+	/* wait until every tx has seen that roc_activity has been reset */
+	synchronize_net();
+	/* from here, no new tx will be added
+	 * we can flush the Tx on the queues
+	 */
+
+	iwl_mld_flush_link_sta_txqs(mld, mld_vif->aux_sta.sta_id);
+
+	iwl_mld_remove_aux_sta(mld, vif);
+}
+
+int iwl_mld_cancel_roc(struct ieee80211_hw *hw,
+		       struct ieee80211_vif *vif)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_roc_req cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
+	};
+	u8 ver = iwl_fw_lookup_cmd_ver(mld->fw,
+				       WIDE_ID(MAC_CONF_GROUP, ROC_CMD), 0);
+	u16 cmd_len = ver < 6 ? sizeof(struct iwl_roc_req_v5) : sizeof(cmd);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+		    vif->type != NL80211_IFTYPE_STATION))
+		return -EOPNOTSUPP;
+
+	/* No roc activity running it's probably already done */
+	if (mld_vif->roc_activity == ROC_NUM_ACTIVITIES)
+		return 0;
+
+	cmd.activity = cpu_to_le32(mld_vif->roc_activity);
+
+	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(MAC_CONF_GROUP, ROC_CMD),
+				   &cmd, cmd_len);
+	if (ret)
+		IWL_ERR(mld, "Couldn't send the command to cancel the ROC\n");
+
+	/* We may have raced with the firmware expiring the ROC instance at
+	 * this very moment. In that case, we can have a notification in the
+	 * async processing queue. However, none can arrive _after_ this as
+	 * ROC_CMD was sent synchronously, i.e. we waited for a response and
+	 * the firmware cannot refer to this ROC after the response. Thus,
+	 * if we just cancel the notification (if there's one) we'll be at a
+	 * clean state for any possible next ROC.
+	 */
+	iwl_mld_cancel_notifications_of_object(mld, IWL_MLD_OBJECT_TYPE_ROC,
+					       mld_vif->roc_activity);
+
+	iwl_mld_destroy_roc(mld, vif, mld_vif);
+
+	return 0;
+}
+
+void iwl_mld_handle_roc_notif(struct iwl_mld *mld,
+			      struct iwl_rx_packet *pkt)
+{
+	const struct iwl_roc_notif *notif = (void *)pkt->data;
+	u32 activity = le32_to_cpu(notif->activity);
+	struct iwl_mld_vif *mld_vif;
+	struct ieee80211_vif *vif;
+
+	vif = iwl_mld_find_roc_vif(mld, activity);
+	if (WARN_ON(!vif))
+		return;
+
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+	/* It is possible that the ROC was canceled
+	 * but the notification was already fired.
+	 */
+	if (mld_vif->roc_activity != activity)
+		return;
+
+	if (le32_to_cpu(notif->success) &&
+	    le32_to_cpu(notif->started)) {
+		/* We had a successful start */
+		ieee80211_ready_on_channel(mld->hw);
+	} else {
+		/* ROC was not successful, tell the firmware to remove it */
+		if (le32_to_cpu(notif->started))
+			iwl_mld_cancel_roc(mld->hw, vif);
+		else
+			iwl_mld_destroy_roc(mld, vif, mld_vif);
+		/* we need to let know mac80211 about end OR
+		 * an unsuccessful start
+		 */
+		ieee80211_remain_on_channel_expired(mld->hw);
+	}
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/roc.h b/sys/contrib/dev/iwlwifi/mld/roc.h
new file mode 100644
index 000000000000..985d7f20a3d7
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/roc.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_roc_h__
+#define __iwl_mld_roc_h__
+
+#include <net/mac80211.h>
+
+int iwl_mld_start_roc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		      struct ieee80211_channel *channel, int duration,
+		      enum ieee80211_roc_type type);
+
+int iwl_mld_cancel_roc(struct ieee80211_hw *hw,
+		       struct ieee80211_vif *vif);
+
+void iwl_mld_handle_roc_notif(struct iwl_mld *mld,
+			      struct iwl_rx_packet *pkt);
+
+#endif /* __iwl_mld_roc_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/rx.c b/sys/contrib/dev/iwlwifi/mld/rx.c
new file mode 100644
index 000000000000..b6dedd1ecd4d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/rx.c
@@ -0,0 +1,2137 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include <net/mac80211.h>
+#include <kunit/static_stub.h>
+
+#include "mld.h"
+#include "sta.h"
+#include "agg.h"
+#include "rx.h"
+#include "hcmd.h"
+#include "iface.h"
+#include "time_sync.h"
+#include "fw/dbg.h"
+#include "fw/api/rx.h"
+
+/* stores relevant PHY data fields extracted from iwl_rx_mpdu_desc */
+struct iwl_mld_rx_phy_data {
+	enum iwl_rx_phy_info_type info_type;
+	__le32 data0;
+	__le32 data1;
+	__le32 data2;
+	__le32 data3;
+	__le32 eht_data4;
+	__le32 data5;
+	__le16 data4;
+	bool first_subframe;
+	bool with_data;
+	__le32 rx_vec[4];
+	u32 rate_n_flags;
+	u32 gp2_on_air_rise;
+	u16 phy_info;
+	u8 energy_a, energy_b;
+};
+
+static void
+iwl_mld_fill_phy_data(struct iwl_mld *mld,
+		      struct iwl_rx_mpdu_desc *desc,
+		      struct iwl_mld_rx_phy_data *phy_data)
+{
+	phy_data->phy_info = le16_to_cpu(desc->phy_info);
+	phy_data->rate_n_flags = iwl_v3_rate_from_v2_v3(desc->v3.rate_n_flags,
+							mld->fw_rates_ver_3);
+	phy_data->gp2_on_air_rise = le32_to_cpu(desc->v3.gp2_on_air_rise);
+	phy_data->energy_a = desc->v3.energy_a;
+	phy_data->energy_b = desc->v3.energy_b;
+	phy_data->data0 = desc->v3.phy_data0;
+	phy_data->data1 = desc->v3.phy_data1;
+	phy_data->data2 = desc->v3.phy_data2;
+	phy_data->data3 = desc->v3.phy_data3;
+	phy_data->data4 = desc->phy_data4;
+	phy_data->eht_data4 = desc->phy_eht_data4;
+	phy_data->data5 = desc->v3.phy_data5;
+	phy_data->with_data = true;
+}
+
+static inline int iwl_mld_check_pn(struct iwl_mld *mld, struct sk_buff *skb,
+				   int queue, struct ieee80211_sta *sta)
+{
+	struct ieee80211_hdr *hdr = (void *)skb_mac_header(skb);
+	struct ieee80211_rx_status *stats = IEEE80211_SKB_RXCB(skb);
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_ptk_pn *ptk_pn;
+	int res;
+	u8 tid, keyidx;
+	u8 pn[IEEE80211_CCMP_PN_LEN];
+	u8 *extiv;
+
+	/* multicast and non-data only arrives on default queue; avoid checking
+	 * for default queue - we don't want to replicate all the logic that's
+	 * necessary for checking the PN on fragmented frames, leave that
+	 * to mac80211
+	 */
+	if (queue == 0 || !ieee80211_is_data(hdr->frame_control) ||
+	    is_multicast_ether_addr(hdr->addr1))
+		return 0;
+
+	if (!(stats->flag & RX_FLAG_DECRYPTED))
+		return 0;
+
+	/* if we are here - this for sure is either CCMP or GCMP */
+	if (!sta) {
+		IWL_DEBUG_DROP(mld,
+			       "expected hw-decrypted unicast frame for station\n");
+		return -1;
+	}
+
+	mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+	extiv = (u8 *)hdr + ieee80211_hdrlen(hdr->frame_control);
+	keyidx = extiv[3] >> 6;
+
+	ptk_pn = rcu_dereference(mld_sta->ptk_pn[keyidx]);
+	if (!ptk_pn)
+		return -1;
+
+	if (ieee80211_is_data_qos(hdr->frame_control))
+		tid = ieee80211_get_tid(hdr);
+	else
+		tid = 0;
+
+	/* we don't use HCCA/802.11 QoS TSPECs, so drop such frames */
+	if (tid >= IWL_MAX_TID_COUNT)
+		return -1;
+
+	/* load pn */
+	pn[0] = extiv[7];
+	pn[1] = extiv[6];
+	pn[2] = extiv[5];
+	pn[3] = extiv[4];
+	pn[4] = extiv[1];
+	pn[5] = extiv[0];
+
+	res = memcmp(pn, ptk_pn->q[queue].pn[tid], IEEE80211_CCMP_PN_LEN);
+	if (res < 0)
+		return -1;
+	if (!res && !(stats->flag & RX_FLAG_ALLOW_SAME_PN))
+		return -1;
+
+	memcpy(ptk_pn->q[queue].pn[tid], pn, IEEE80211_CCMP_PN_LEN);
+	stats->flag |= RX_FLAG_PN_VALIDATED;
+
+	return 0;
+}
+
+/* iwl_mld_pass_packet_to_mac80211 - passes the packet for mac80211 */
+void iwl_mld_pass_packet_to_mac80211(struct iwl_mld *mld,
+				     struct napi_struct *napi,
+				     struct sk_buff *skb, int queue,
+				     struct ieee80211_sta *sta)
+{
+	KUNIT_STATIC_STUB_REDIRECT(iwl_mld_pass_packet_to_mac80211,
+				   mld, napi, skb, queue, sta);
+
+	if (unlikely(iwl_mld_check_pn(mld, skb, queue, sta))) {
+		kfree_skb(skb);
+		return;
+	}
+
+	ieee80211_rx_napi(mld->hw, sta, skb, napi);
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_pass_packet_to_mac80211);
+
+static bool iwl_mld_used_average_energy(struct iwl_mld *mld, int link_id,
+					struct ieee80211_hdr *hdr,
+					struct ieee80211_rx_status *rx_status)
+{
+	struct ieee80211_bss_conf *link_conf;
+	struct iwl_mld_link *mld_link;
+
+	if (unlikely(!hdr || link_id < 0))
+		return false;
+
+	if (likely(!ieee80211_is_beacon(hdr->frame_control)))
+		return false;
+
+	/*
+	 * if link ID is >= valid ones then that means the RX
+	 * was on the AUX link and no correction is needed
+	 */
+	if (link_id >= mld->fw->ucode_capa.num_links)
+		return false;
+
+	/* for the link conf lookup */
+	guard(rcu)();
+
+	link_conf = rcu_dereference(mld->fw_id_to_bss_conf[link_id]);
+	if (!link_conf)
+		return false;
+
+	mld_link = iwl_mld_link_from_mac80211(link_conf);
+	if (!mld_link)
+		return false;
+
+	/*
+	 * If we know the link by link ID then the frame was
+	 * received for the link, so by filtering it means it
+	 * was from the AP the link is connected to.
+	 */
+
+	/* skip also in case we don't have it (yet) */
+	if (!mld_link->average_beacon_energy)
+		return false;
+
+	IWL_DEBUG_STATS(mld, "energy override by average %d\n",
+			mld_link->average_beacon_energy);
+	rx_status->signal = -mld_link->average_beacon_energy;
+	return true;
+}
+
+static void iwl_mld_fill_signal(struct iwl_mld *mld, int link_id,
+				struct ieee80211_hdr *hdr,
+				struct ieee80211_rx_status *rx_status,
+				struct iwl_mld_rx_phy_data *phy_data)
+{
+	u32 rate_n_flags = phy_data->rate_n_flags;
+	int energy_a = phy_data->energy_a;
+	int energy_b = phy_data->energy_b;
+	int max_energy;
+
+	energy_a = energy_a ? -energy_a : S8_MIN;
+	energy_b = energy_b ? -energy_b : S8_MIN;
+	max_energy = max(energy_a, energy_b);
+
+	IWL_DEBUG_STATS(mld, "energy in A %d B %d, and max %d\n",
+			energy_a, energy_b, max_energy);
+
+	if (iwl_mld_used_average_energy(mld, link_id, hdr, rx_status))
+		return;
+
+	rx_status->signal = max_energy;
+	rx_status->chains = u32_get_bits(rate_n_flags, RATE_MCS_ANT_AB_MSK);
+	rx_status->chain_signal[0] = energy_a;
+	rx_status->chain_signal[1] = energy_b;
+}
+
+static void
+iwl_mld_decode_he_phy_ru_alloc(struct iwl_mld_rx_phy_data *phy_data,
+			       struct ieee80211_radiotap_he *he,
+			       struct ieee80211_radiotap_he_mu *he_mu,
+			       struct ieee80211_rx_status *rx_status)
+{
+	/* Unfortunately, we have to leave the mac80211 data
+	 * incorrect for the case that we receive an HE-MU
+	 * transmission and *don't* have the HE phy data (due
+	 * to the bits being used for TSF). This shouldn't
+	 * happen though as management frames where we need
+	 * the TSF/timers are not be transmitted in HE-MU.
+	 */
+	u8 ru = le32_get_bits(phy_data->data1, IWL_RX_PHY_DATA1_HE_RU_ALLOC_MASK);
+	u32 rate_n_flags = phy_data->rate_n_flags;
+	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK;
+	u8 offs = 0;
+
+	rx_status->bw = RATE_INFO_BW_HE_RU;
+
+	he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN);
+
+	switch (ru) {
+	case 0 ... 36:
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_26;
+		offs = ru;
+		break;
+	case 37 ... 52:
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_52;
+		offs = ru - 37;
+		break;
+	case 53 ... 60:
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_106;
+		offs = ru - 53;
+		break;
+	case 61 ... 64:
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_242;
+		offs = ru - 61;
+		break;
+	case 65 ... 66:
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_484;
+		offs = ru - 65;
+		break;
+	case 67:
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_996;
+		break;
+	case 68:
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_2x996;
+		break;
+	}
+	he->data2 |= le16_encode_bits(offs,
+				      IEEE80211_RADIOTAP_HE_DATA2_RU_OFFSET);
+	he->data2 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA2_PRISEC_80_KNOWN |
+				 IEEE80211_RADIOTAP_HE_DATA2_RU_OFFSET_KNOWN);
+	if (phy_data->data1 & cpu_to_le32(IWL_RX_PHY_DATA1_HE_RU_ALLOC_SEC80))
+		he->data2 |=
+			cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA2_PRISEC_80_SEC);
+
+#define CHECK_BW(bw) \
+	BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_MU_FLAGS2_BW_FROM_SIG_A_BW_ ## bw ## MHZ != \
+		     RATE_MCS_CHAN_WIDTH_##bw >> RATE_MCS_CHAN_WIDTH_POS); \
+	BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA6_TB_PPDU_BW_ ## bw ## MHZ != \
+		     RATE_MCS_CHAN_WIDTH_##bw >> RATE_MCS_CHAN_WIDTH_POS)
+	CHECK_BW(20);
+	CHECK_BW(40);
+	CHECK_BW(80);
+	CHECK_BW(160);
+
+	if (he_mu)
+		he_mu->flags2 |=
+			le16_encode_bits(u32_get_bits(rate_n_flags,
+						      RATE_MCS_CHAN_WIDTH_MSK),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS2_BW_FROM_SIG_A_BW);
+	else if (he_type == RATE_MCS_HE_TYPE_TRIG)
+		he->data6 |=
+			cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA6_TB_PPDU_BW_KNOWN) |
+			le16_encode_bits(u32_get_bits(rate_n_flags,
+						      RATE_MCS_CHAN_WIDTH_MSK),
+					 IEEE80211_RADIOTAP_HE_DATA6_TB_PPDU_BW);
+}
+
+static void
+iwl_mld_decode_he_mu_ext(struct iwl_mld_rx_phy_data *phy_data,
+			 struct ieee80211_radiotap_he_mu *he_mu)
+{
+	u32 phy_data2 = le32_to_cpu(phy_data->data2);
+	u32 phy_data3 = le32_to_cpu(phy_data->data3);
+	u16 phy_data4 = le16_to_cpu(phy_data->data4);
+	u32 rate_n_flags = phy_data->rate_n_flags;
+
+	if (u32_get_bits(phy_data4, IWL_RX_PHY_DATA4_HE_MU_EXT_CH1_CRC_OK)) {
+		he_mu->flags1 |=
+			cpu_to_le16(IEEE80211_RADIOTAP_HE_MU_FLAGS1_CH1_RU_KNOWN |
+				    IEEE80211_RADIOTAP_HE_MU_FLAGS1_CH1_CTR_26T_RU_KNOWN);
+
+		he_mu->flags1 |=
+			le16_encode_bits(u32_get_bits(phy_data4,
+						      IWL_RX_PHY_DATA4_HE_MU_EXT_CH1_CTR_RU),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS1_CH1_CTR_26T_RU);
+
+		he_mu->ru_ch1[0] = u32_get_bits(phy_data2,
+						IWL_RX_PHY_DATA2_HE_MU_EXT_CH1_RU0);
+		he_mu->ru_ch1[1] = u32_get_bits(phy_data3,
+						IWL_RX_PHY_DATA3_HE_MU_EXT_CH1_RU1);
+		he_mu->ru_ch1[2] = u32_get_bits(phy_data2,
+						IWL_RX_PHY_DATA2_HE_MU_EXT_CH1_RU2);
+		he_mu->ru_ch1[3] = u32_get_bits(phy_data3,
+						IWL_RX_PHY_DATA3_HE_MU_EXT_CH1_RU3);
+	}
+
+	if (u32_get_bits(phy_data4, IWL_RX_PHY_DATA4_HE_MU_EXT_CH2_CRC_OK) &&
+	    (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) != RATE_MCS_CHAN_WIDTH_20) {
+		he_mu->flags1 |=
+			cpu_to_le16(IEEE80211_RADIOTAP_HE_MU_FLAGS1_CH2_RU_KNOWN |
+				    IEEE80211_RADIOTAP_HE_MU_FLAGS1_CH2_CTR_26T_RU_KNOWN);
+
+		he_mu->flags2 |=
+			le16_encode_bits(u32_get_bits(phy_data4,
+						      IWL_RX_PHY_DATA4_HE_MU_EXT_CH2_CTR_RU),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS2_CH2_CTR_26T_RU);
+
+		he_mu->ru_ch2[0] = u32_get_bits(phy_data2,
+						IWL_RX_PHY_DATA2_HE_MU_EXT_CH2_RU0);
+		he_mu->ru_ch2[1] = u32_get_bits(phy_data3,
+						IWL_RX_PHY_DATA3_HE_MU_EXT_CH2_RU1);
+		he_mu->ru_ch2[2] = u32_get_bits(phy_data2,
+						IWL_RX_PHY_DATA2_HE_MU_EXT_CH2_RU2);
+		he_mu->ru_ch2[3] = u32_get_bits(phy_data3,
+						IWL_RX_PHY_DATA3_HE_MU_EXT_CH2_RU3);
+	}
+}
+
+static void
+iwl_mld_decode_he_phy_data(struct iwl_mld_rx_phy_data *phy_data,
+			   struct ieee80211_radiotap_he *he,
+			   struct ieee80211_radiotap_he_mu *he_mu,
+			   struct ieee80211_rx_status *rx_status,
+			   int queue)
+{
+	switch (phy_data->info_type) {
+	case IWL_RX_PHY_INFO_TYPE_NONE:
+	case IWL_RX_PHY_INFO_TYPE_CCK:
+	case IWL_RX_PHY_INFO_TYPE_OFDM_LGCY:
+	case IWL_RX_PHY_INFO_TYPE_HT:
+	case IWL_RX_PHY_INFO_TYPE_VHT_SU:
+	case IWL_RX_PHY_INFO_TYPE_VHT_MU:
+	case IWL_RX_PHY_INFO_TYPE_EHT_MU:
+	case IWL_RX_PHY_INFO_TYPE_EHT_TB:
+	case IWL_RX_PHY_INFO_TYPE_EHT_MU_EXT:
+	case IWL_RX_PHY_INFO_TYPE_EHT_TB_EXT:
+		return;
+	case IWL_RX_PHY_INFO_TYPE_HE_TB_EXT:
+		he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE2_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE3_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE4_KNOWN);
+		he->data4 |= le16_encode_bits(le32_get_bits(phy_data->data2,
+							    IWL_RX_PHY_DATA2_HE_TB_EXT_SPTL_REUSE1),
+					      IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE1);
+		he->data4 |= le16_encode_bits(le32_get_bits(phy_data->data2,
+							    IWL_RX_PHY_DATA2_HE_TB_EXT_SPTL_REUSE2),
+					      IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE2);
+		he->data4 |= le16_encode_bits(le32_get_bits(phy_data->data2,
+							    IWL_RX_PHY_DATA2_HE_TB_EXT_SPTL_REUSE3),
+					      IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE3);
+		he->data4 |= le16_encode_bits(le32_get_bits(phy_data->data2,
+							    IWL_RX_PHY_DATA2_HE_TB_EXT_SPTL_REUSE4),
+					      IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE4);
+		fallthrough;
+	case IWL_RX_PHY_INFO_TYPE_HE_SU:
+	case IWL_RX_PHY_INFO_TYPE_HE_MU:
+	case IWL_RX_PHY_INFO_TYPE_HE_MU_EXT:
+	case IWL_RX_PHY_INFO_TYPE_HE_TB:
+		/* HE common */
+		he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_LDPC_XSYMSEG_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA1_DOPPLER_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA1_BSS_COLOR_KNOWN);
+		he->data2 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA2_PRE_FEC_PAD_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA2_PE_DISAMBIG_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA2_TXOP_KNOWN |
+					 IEEE80211_RADIOTAP_HE_DATA2_NUM_LTF_SYMS_KNOWN);
+		he->data3 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_BSS_COLOR_MASK),
+					      IEEE80211_RADIOTAP_HE_DATA3_BSS_COLOR);
+		if (phy_data->info_type != IWL_RX_PHY_INFO_TYPE_HE_TB &&
+		    phy_data->info_type != IWL_RX_PHY_INFO_TYPE_HE_TB_EXT) {
+			he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_UL_DL_KNOWN);
+			he->data3 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_UPLINK),
+						      IEEE80211_RADIOTAP_HE_DATA3_UL_DL);
+		}
+		he->data3 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_LDPC_EXT_SYM),
+					      IEEE80211_RADIOTAP_HE_DATA3_LDPC_XSYMSEG);
+		he->data5 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_PRE_FEC_PAD_MASK),
+					      IEEE80211_RADIOTAP_HE_DATA5_PRE_FEC_PAD);
+		he->data5 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_PE_DISAMBIG),
+					      IEEE80211_RADIOTAP_HE_DATA5_PE_DISAMBIG);
+		he->data5 |= le16_encode_bits(le32_get_bits(phy_data->data1,
+							    IWL_RX_PHY_DATA1_HE_LTF_NUM_MASK),
+					      IEEE80211_RADIOTAP_HE_DATA5_NUM_LTF_SYMS);
+		he->data6 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_TXOP_DUR_MASK),
+					      IEEE80211_RADIOTAP_HE_DATA6_TXOP);
+		he->data6 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_DOPPLER),
+					      IEEE80211_RADIOTAP_HE_DATA6_DOPPLER);
+		break;
+	}
+
+	switch (phy_data->info_type) {
+	case IWL_RX_PHY_INFO_TYPE_HE_MU_EXT:
+	case IWL_RX_PHY_INFO_TYPE_HE_MU:
+	case IWL_RX_PHY_INFO_TYPE_HE_SU:
+		he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE_KNOWN);
+		he->data4 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_SPATIAL_REUSE_MASK),
+					      IEEE80211_RADIOTAP_HE_DATA4_SU_MU_SPTL_REUSE);
+		break;
+	default:
+		/* nothing here */
+		break;
+	}
+
+	switch (phy_data->info_type) {
+	case IWL_RX_PHY_INFO_TYPE_HE_MU_EXT:
+		he_mu->flags1 |=
+			le16_encode_bits(le16_get_bits(phy_data->data4,
+						       IWL_RX_PHY_DATA4_HE_MU_EXT_SIGB_DCM),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_DCM);
+		he_mu->flags1 |=
+			le16_encode_bits(le16_get_bits(phy_data->data4,
+						       IWL_RX_PHY_DATA4_HE_MU_EXT_SIGB_MCS_MASK),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_MCS);
+		he_mu->flags2 |=
+			le16_encode_bits(le16_get_bits(phy_data->data4,
+						       IWL_RX_PHY_DATA4_HE_MU_EXT_PREAMBLE_PUNC_TYPE_MASK),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS2_PUNC_FROM_SIG_A_BW);
+		iwl_mld_decode_he_mu_ext(phy_data, he_mu);
+		fallthrough;
+	case IWL_RX_PHY_INFO_TYPE_HE_MU:
+		he_mu->flags2 |=
+			le16_encode_bits(le32_get_bits(phy_data->data1,
+						       IWL_RX_PHY_DATA1_HE_MU_SIBG_SYM_OR_USER_NUM_MASK),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS2_SIG_B_SYMS_USERS);
+		he_mu->flags2 |=
+			le16_encode_bits(le32_get_bits(phy_data->data1,
+						       IWL_RX_PHY_DATA1_HE_MU_SIGB_COMPRESSION),
+					 IEEE80211_RADIOTAP_HE_MU_FLAGS2_SIG_B_COMP);
+		fallthrough;
+	case IWL_RX_PHY_INFO_TYPE_HE_TB:
+	case IWL_RX_PHY_INFO_TYPE_HE_TB_EXT:
+		iwl_mld_decode_he_phy_ru_alloc(phy_data, he, he_mu, rx_status);
+		break;
+	case IWL_RX_PHY_INFO_TYPE_HE_SU:
+		he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BEAM_CHANGE_KNOWN);
+		he->data3 |= le16_encode_bits(le32_get_bits(phy_data->data0,
+							    IWL_RX_PHY_DATA0_HE_BEAM_CHNG),
+					      IEEE80211_RADIOTAP_HE_DATA3_BEAM_CHANGE);
+		break;
+	default:
+		/* nothing */
+		break;
+	}
+}
+
+static void iwl_mld_rx_he(struct iwl_mld *mld, struct sk_buff *skb,
+			  struct iwl_mld_rx_phy_data *phy_data,
+			  int queue)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	struct ieee80211_radiotap_he *he = NULL;
+	struct ieee80211_radiotap_he_mu *he_mu = NULL;
+	u32 rate_n_flags = phy_data->rate_n_flags;
+	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK;
+	u8 ltf;
+	static const struct ieee80211_radiotap_he known = {
+		.data1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN |
+				     IEEE80211_RADIOTAP_HE_DATA1_DATA_DCM_KNOWN |
+				     IEEE80211_RADIOTAP_HE_DATA1_STBC_KNOWN	|
+				     IEEE80211_RADIOTAP_HE_DATA1_CODING_KNOWN),
+		.data2 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA2_GI_KNOWN |
+				     IEEE80211_RADIOTAP_HE_DATA2_TXBF_KNOWN),
+	};
+	static const struct ieee80211_radiotap_he_mu mu_known = {
+		.flags1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_MCS_KNOWN |
+				      IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_DCM_KNOWN |
+				      IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_SYMS_USERS_KNOWN |
+				      IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_COMP_KNOWN),
+		.flags2 = cpu_to_le16(IEEE80211_RADIOTAP_HE_MU_FLAGS2_PUNC_FROM_SIG_A_BW_KNOWN |
+				      IEEE80211_RADIOTAP_HE_MU_FLAGS2_BW_FROM_SIG_A_BW_KNOWN),
+	};
+	u16 phy_info = phy_data->phy_info;
+
+	he = skb_put_data(skb, &known, sizeof(known));
+	rx_status->flag |= RX_FLAG_RADIOTAP_HE;
+
+	if (phy_data->info_type == IWL_RX_PHY_INFO_TYPE_HE_MU ||
+	    phy_data->info_type == IWL_RX_PHY_INFO_TYPE_HE_MU_EXT) {
+		he_mu = skb_put_data(skb, &mu_known, sizeof(mu_known));
+		rx_status->flag |= RX_FLAG_RADIOTAP_HE_MU;
+	}
+
+	/* report the AMPDU-EOF bit on single frames */
+	if (!queue && !(phy_info & IWL_RX_MPDU_PHY_AMPDU)) {
+		rx_status->flag |= RX_FLAG_AMPDU_DETAILS;
+		rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
+		if (phy_data->data0 & cpu_to_le32(IWL_RX_PHY_DATA0_HE_DELIM_EOF))
+			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
+	}
+
+	if (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD)
+		iwl_mld_decode_he_phy_data(phy_data, he, he_mu, rx_status,
+					   queue);
+
+	/* update aggregation data for monitor sake on default queue */
+	if (!queue && (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) &&
+	    (phy_info & IWL_RX_MPDU_PHY_AMPDU) && phy_data->first_subframe) {
+		rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
+		if (phy_data->data0 & cpu_to_le32(IWL_RX_PHY_DATA0_EHT_DELIM_EOF))
+			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
+	}
+
+	if (he_type == RATE_MCS_HE_TYPE_EXT_SU &&
+	    rate_n_flags & RATE_MCS_HE_106T_MSK) {
+		rx_status->bw = RATE_INFO_BW_HE_RU;
+		rx_status->he_ru = NL80211_RATE_INFO_HE_RU_ALLOC_106;
+	}
+
+	/* actually data is filled in mac80211 */
+	if (he_type == RATE_MCS_HE_TYPE_SU ||
+	    he_type == RATE_MCS_HE_TYPE_EXT_SU)
+		he->data1 |=
+			cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN);
+
+#define CHECK_TYPE(F)							\
+	BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA1_FORMAT_ ## F !=	\
+		     (RATE_MCS_HE_TYPE_ ## F >> RATE_MCS_HE_TYPE_POS))
+
+	CHECK_TYPE(SU);
+	CHECK_TYPE(EXT_SU);
+	CHECK_TYPE(MU);
+	CHECK_TYPE(TRIG);
+
+	he->data1 |= cpu_to_le16(he_type >> RATE_MCS_HE_TYPE_POS);
+
+	if (rate_n_flags & RATE_MCS_BF_MSK)
+		he->data5 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA5_TXBF);
+
+	switch ((rate_n_flags & RATE_MCS_HE_GI_LTF_MSK) >>
+		RATE_MCS_HE_GI_LTF_POS) {
+	case 0:
+		if (he_type == RATE_MCS_HE_TYPE_TRIG)
+			rx_status->he_gi = NL80211_RATE_INFO_HE_GI_1_6;
+		else
+			rx_status->he_gi = NL80211_RATE_INFO_HE_GI_0_8;
+		if (he_type == RATE_MCS_HE_TYPE_MU)
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+		else
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_1X;
+		break;
+	case 1:
+		if (he_type == RATE_MCS_HE_TYPE_TRIG)
+			rx_status->he_gi = NL80211_RATE_INFO_HE_GI_1_6;
+		else
+			rx_status->he_gi = NL80211_RATE_INFO_HE_GI_0_8;
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+		break;
+	case 2:
+		if (he_type == RATE_MCS_HE_TYPE_TRIG) {
+			rx_status->he_gi = NL80211_RATE_INFO_HE_GI_3_2;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+		} else {
+			rx_status->he_gi = NL80211_RATE_INFO_HE_GI_1_6;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+		}
+		break;
+	case 3:
+		rx_status->he_gi = NL80211_RATE_INFO_HE_GI_3_2;
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+		break;
+	case 4:
+		rx_status->he_gi = NL80211_RATE_INFO_HE_GI_0_8;
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+		break;
+	default:
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_UNKNOWN;
+	}
+
+	he->data5 |= le16_encode_bits(ltf,
+				      IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE);
+}
+
+static void iwl_mld_decode_lsig(struct sk_buff *skb,
+				struct iwl_mld_rx_phy_data *phy_data)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	struct ieee80211_radiotap_lsig *lsig;
+
+	switch (phy_data->info_type) {
+	case IWL_RX_PHY_INFO_TYPE_HT:
+	case IWL_RX_PHY_INFO_TYPE_VHT_SU:
+	case IWL_RX_PHY_INFO_TYPE_VHT_MU:
+	case IWL_RX_PHY_INFO_TYPE_HE_TB_EXT:
+	case IWL_RX_PHY_INFO_TYPE_HE_SU:
+	case IWL_RX_PHY_INFO_TYPE_HE_MU:
+	case IWL_RX_PHY_INFO_TYPE_HE_MU_EXT:
+	case IWL_RX_PHY_INFO_TYPE_HE_TB:
+	case IWL_RX_PHY_INFO_TYPE_EHT_MU:
+	case IWL_RX_PHY_INFO_TYPE_EHT_TB:
+	case IWL_RX_PHY_INFO_TYPE_EHT_MU_EXT:
+	case IWL_RX_PHY_INFO_TYPE_EHT_TB_EXT:
+		lsig = skb_put(skb, sizeof(*lsig));
+		lsig->data1 = cpu_to_le16(IEEE80211_RADIOTAP_LSIG_DATA1_LENGTH_KNOWN);
+		lsig->data2 = le16_encode_bits(le32_get_bits(phy_data->data1,
+							     IWL_RX_PHY_DATA1_LSIG_LEN_MASK),
+					       IEEE80211_RADIOTAP_LSIG_DATA2_LENGTH);
+		rx_status->flag |= RX_FLAG_RADIOTAP_LSIG;
+		break;
+	default:
+		break;
+	}
+}
+
+/* Put a TLV on the skb and return data pointer
+ *
+ * Also pad the len to 4 and zero out all data part
+ */
+static void *
+iwl_mld_radiotap_put_tlv(struct sk_buff *skb, u16 type, u16 len)
+{
+	struct ieee80211_radiotap_tlv *tlv;
+
+	tlv = skb_put(skb, sizeof(*tlv));
+	tlv->type = cpu_to_le16(type);
+	tlv->len = cpu_to_le16(len);
+	return skb_put_zero(skb, ALIGN(len, 4));
+}
+
+#define LE32_DEC_ENC(value, dec_bits, enc_bits) \
+	le32_encode_bits(le32_get_bits(value, dec_bits), enc_bits)
+
+#define IWL_MLD_ENC_USIG_VALUE_MASK(usig, in_value, dec_bits, enc_bits) do { \
+	typeof(enc_bits) _enc_bits = enc_bits; \
+	typeof(usig) _usig = usig; \
+	(_usig)->mask |= cpu_to_le32(_enc_bits); \
+	(_usig)->value |= LE32_DEC_ENC(in_value, dec_bits, _enc_bits); \
+} while (0)
+
+#define __IWL_MLD_ENC_EHT_RU(rt_data, rt_ru, fw_data, fw_ru) \
+	eht->data[(rt_data)] |= \
+		(cpu_to_le32 \
+		 (IEEE80211_RADIOTAP_EHT_DATA ## rt_data ## _RU_ALLOC_CC_ ## rt_ru ## _KNOWN) | \
+		 LE32_DEC_ENC(data ## fw_data, \
+			      IWL_RX_PHY_DATA ## fw_data ## _EHT_MU_EXT_RU_ALLOC_ ## fw_ru, \
+			      IEEE80211_RADIOTAP_EHT_DATA ## rt_data ## _RU_ALLOC_CC_ ## rt_ru))
+
+#define _IWL_MLD_ENC_EHT_RU(rt_data, rt_ru, fw_data, fw_ru)	\
+	__IWL_MLD_ENC_EHT_RU(rt_data, rt_ru, fw_data, fw_ru)
+
+#define IEEE80211_RADIOTAP_RU_DATA_1_1_1	1
+#define IEEE80211_RADIOTAP_RU_DATA_2_1_1	2
+#define IEEE80211_RADIOTAP_RU_DATA_1_1_2	2
+#define IEEE80211_RADIOTAP_RU_DATA_2_1_2	2
+#define IEEE80211_RADIOTAP_RU_DATA_1_2_1	3
+#define IEEE80211_RADIOTAP_RU_DATA_2_2_1	3
+#define IEEE80211_RADIOTAP_RU_DATA_1_2_2	3
+#define IEEE80211_RADIOTAP_RU_DATA_2_2_2	4
+
+#define IWL_RX_RU_DATA_A1			2
+#define IWL_RX_RU_DATA_A2			2
+#define IWL_RX_RU_DATA_B1			2
+#define IWL_RX_RU_DATA_B2			4
+#define IWL_RX_RU_DATA_C1			3
+#define IWL_RX_RU_DATA_C2			3
+#define IWL_RX_RU_DATA_D1			4
+#define IWL_RX_RU_DATA_D2			4
+
+#define IWL_MLD_ENC_EHT_RU(rt_ru, fw_ru)				\
+	_IWL_MLD_ENC_EHT_RU(IEEE80211_RADIOTAP_RU_DATA_ ## rt_ru,	\
+			    rt_ru,					\
+			    IWL_RX_RU_DATA_ ## fw_ru,			\
+			    fw_ru)
+
+static void iwl_mld_decode_eht_ext_mu(struct iwl_mld *mld,
+				      struct iwl_mld_rx_phy_data *phy_data,
+				      struct ieee80211_rx_status *rx_status,
+				      struct ieee80211_radiotap_eht *eht,
+				      struct ieee80211_radiotap_eht_usig *usig)
+{
+	if (phy_data->with_data) {
+		__le32 data1 = phy_data->data1;
+		__le32 data2 = phy_data->data2;
+		__le32 data3 = phy_data->data3;
+		__le32 data4 = phy_data->eht_data4;
+		__le32 data5 = phy_data->data5;
+		u32 phy_bw = phy_data->rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK;
+
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TYPE_AND_COMP,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_MU_PUNC_CH_CODE,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, data4,
+					    IWL_RX_PHY_DATA4_EHT_MU_EXT_SIGB_MCS,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS);
+		IWL_MLD_ENC_USIG_VALUE_MASK
+			(usig, data1, IWL_RX_PHY_DATA1_EHT_MU_NUM_SIG_SYM_USIGA2,
+			 IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS);
+
+		eht->user_info[0] |=
+			cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN) |
+			LE32_DEC_ENC(data5, IWL_RX_PHY_DATA5_EHT_MU_STA_ID_USR,
+				     IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID);
+
+		eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_NR_NON_OFDMA_USERS_M);
+		eht->data[7] |= LE32_DEC_ENC
+			(data5, IWL_RX_PHY_DATA5_EHT_MU_NUM_USR_NON_OFDMA,
+			 IEEE80211_RADIOTAP_EHT_DATA7_NUM_OF_NON_OFDMA_USERS);
+
+		/*
+		 * Hardware labels the content channels/RU allocation values
+		 * as follows:
+		 *           Content Channel 1		Content Channel 2
+		 *   20 MHz: A1
+		 *   40 MHz: A1				B1
+		 *   80 MHz: A1 C1			B1 D1
+		 *  160 MHz: A1 C1 A2 C2		B1 D1 B2 D2
+		 *  320 MHz: A1 C1 A2 C2 A3 C3 A4 C4	B1 D1 B2 D2 B3 D3 B4 D4
+		 *
+		 * However firmware can only give us A1-D2, so the higher
+		 * frequencies are missing.
+		 */
+
+		switch (phy_bw) {
+		case RATE_MCS_CHAN_WIDTH_320:
+			/* additional values are missing in RX metadata */
+			fallthrough;
+		case RATE_MCS_CHAN_WIDTH_160:
+			/* content channel 1 */
+			IWL_MLD_ENC_EHT_RU(1_2_1, A2);
+			IWL_MLD_ENC_EHT_RU(1_2_2, C2);
+			/* content channel 2 */
+			IWL_MLD_ENC_EHT_RU(2_2_1, B2);
+			IWL_MLD_ENC_EHT_RU(2_2_2, D2);
+			fallthrough;
+		case RATE_MCS_CHAN_WIDTH_80:
+			/* content channel 1 */
+			IWL_MLD_ENC_EHT_RU(1_1_2, C1);
+			/* content channel 2 */
+			IWL_MLD_ENC_EHT_RU(2_1_2, D1);
+			fallthrough;
+		case RATE_MCS_CHAN_WIDTH_40:
+			/* content channel 2 */
+			IWL_MLD_ENC_EHT_RU(2_1_1, B1);
+			fallthrough;
+		case RATE_MCS_CHAN_WIDTH_20:
+			IWL_MLD_ENC_EHT_RU(1_1_1, A1);
+			break;
+		}
+	} else {
+		__le32 usig_a1 = phy_data->rx_vec[0];
+		__le32 usig_a2 = phy_data->rx_vec[1];
+
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a1,
+					    IWL_RX_USIG_A1_DISREGARD,
+					    IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a1,
+					    IWL_RX_USIG_A1_VALIDATE,
+					    IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_PPDU_TYPE,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_PUNC_CHANNEL,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B8,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_SIG_MCS,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS);
+		IWL_MLD_ENC_USIG_VALUE_MASK
+			(usig, usig_a2, IWL_RX_USIG_A2_EHT_SIG_SYM_NUM,
+			 IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_CRC_OK,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC);
+	}
+}
+
+static void iwl_mld_decode_eht_ext_tb(struct iwl_mld *mld,
+				      struct iwl_mld_rx_phy_data *phy_data,
+				      struct ieee80211_rx_status *rx_status,
+				      struct ieee80211_radiotap_eht *eht,
+				      struct ieee80211_radiotap_eht_usig *usig)
+{
+	if (phy_data->with_data) {
+		__le32 data5 = phy_data->data5;
+
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TYPE_AND_COMP,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TB_SPATIAL_REUSE1,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1);
+
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TB_SPATIAL_REUSE2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2);
+	} else {
+		__le32 usig_a1 = phy_data->rx_vec[0];
+		__le32 usig_a2 = phy_data->rx_vec[1];
+
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a1,
+					    IWL_RX_USIG_A1_DISREGARD,
+					    IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_PPDU_TYPE,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_TRIG_SPATIAL_REUSE_1,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_TRIG_SPATIAL_REUSE_2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_TRIG_USIG2_DISREGARD,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD);
+		IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_CRC_OK,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC);
+	}
+}
+
+static void iwl_mld_decode_eht_ru(struct iwl_mld *mld,
+				  struct ieee80211_rx_status *rx_status,
+				  struct ieee80211_radiotap_eht *eht)
+{
+	u32 ru = le32_get_bits(eht->data[8],
+			       IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_B7_B1);
+	enum nl80211_eht_ru_alloc nl_ru;
+
+	/* Using D1.5 Table 9-53a - Encoding of PS160 and RU Allocation subfields
+	 * in an EHT variant User Info field
+	 */
+
+	switch (ru) {
+	case 0 ... 36:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_26;
+		break;
+	case 37 ... 52:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_52;
+		break;
+	case 53 ... 60:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_106;
+		break;
+	case 61 ... 64:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_242;
+		break;
+	case 65 ... 66:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_484;
+		break;
+	case 67:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_996;
+		break;
+	case 68:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_2x996;
+		break;
+	case 69:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_4x996;
+		break;
+	case 70 ... 81:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_52P26;
+		break;
+	case 82 ... 89:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_106P26;
+		break;
+	case 90 ... 93:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_484P242;
+		break;
+	case 94 ... 95:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_996P484;
+		break;
+	case 96 ... 99:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_996P484P242;
+		break;
+	case 100 ... 103:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_2x996P484;
+		break;
+	case 104:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_3x996;
+		break;
+	case 105 ... 106:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_3x996P484;
+		break;
+	default:
+		return;
+	}
+
+	rx_status->bw = RATE_INFO_BW_EHT_RU;
+	rx_status->eht.ru = nl_ru;
+}
+
+static void iwl_mld_decode_eht_phy_data(struct iwl_mld *mld,
+					struct iwl_mld_rx_phy_data *phy_data,
+					struct ieee80211_rx_status *rx_status,
+					struct ieee80211_radiotap_eht *eht,
+					struct ieee80211_radiotap_eht_usig *usig)
+
+{
+	__le32 data0 = phy_data->data0;
+	__le32 data1 = phy_data->data1;
+	__le32 usig_a1 = phy_data->rx_vec[0];
+	u8 info_type = phy_data->info_type;
+
+	/* Not in EHT range */
+	if (info_type < IWL_RX_PHY_INFO_TYPE_EHT_MU ||
+	    info_type > IWL_RX_PHY_INFO_TYPE_EHT_TB_EXT)
+		return;
+
+	usig->common |= cpu_to_le32
+		(IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL_KNOWN |
+		 IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR_KNOWN);
+	if (phy_data->with_data) {
+		usig->common |= LE32_DEC_ENC(data0,
+					     IWL_RX_PHY_DATA0_EHT_UPLINK,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL);
+		usig->common |= LE32_DEC_ENC(data0,
+					     IWL_RX_PHY_DATA0_EHT_BSS_COLOR_MASK,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR);
+	} else {
+		usig->common |= LE32_DEC_ENC(usig_a1,
+					     IWL_RX_USIG_A1_UL_FLAG,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL);
+		usig->common |= LE32_DEC_ENC(usig_a1,
+					     IWL_RX_USIG_A1_BSS_COLOR,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR);
+	}
+
+	usig->common |=
+		cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_VALIDATE_BITS_CHECKED);
+	usig->common |=
+		LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_VALIDATE,
+			     IEEE80211_RADIOTAP_EHT_USIG_COMMON_VALIDATE_BITS_OK);
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE);
+	eht->data[0] |= LE32_DEC_ENC(data0,
+				     IWL_RX_PHY_DATA0_ETH_SPATIAL_REUSE_MASK,
+				     IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE);
+
+	/* All RU allocating size/index is in TB format */
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_RU_ALLOC_TB_FMT);
+	eht->data[8] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PS160,
+				     IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_PS_160);
+	eht->data[8] |= LE32_DEC_ENC(data1, IWL_RX_PHY_DATA1_EHT_RU_ALLOC_B0,
+				     IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_B0);
+	eht->data[8] |= LE32_DEC_ENC(data1, IWL_RX_PHY_DATA1_EHT_RU_ALLOC_B1_B7,
+				     IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_B7_B1);
+
+	iwl_mld_decode_eht_ru(mld, rx_status, eht);
+
+	/* We only get here in case of IWL_RX_MPDU_PHY_TSF_OVERLOAD is set
+	 * which is on only in case of monitor mode so no need to check monitor
+	 * mode
+	 */
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_PRIMARY_80);
+	eht->data[1] |=
+		le32_encode_bits(mld->monitor.p80,
+				 IEEE80211_RADIOTAP_EHT_DATA1_PRIMARY_80);
+
+	usig->common |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP_KNOWN);
+	if (phy_data->with_data)
+		usig->common |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_TXOP_DUR_MASK,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP);
+	else
+		usig->common |= LE32_DEC_ENC(usig_a1, IWL_RX_USIG_A1_TXOP_DURATION,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP);
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_LDPC_EXTRA_SYM_OM);
+	eht->data[0] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_LDPC_EXT_SYM,
+				     IEEE80211_RADIOTAP_EHT_DATA0_LDPC_EXTRA_SYM_OM);
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_PRE_PADD_FACOR_OM);
+	eht->data[0] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PRE_FEC_PAD_MASK,
+				    IEEE80211_RADIOTAP_EHT_DATA0_PRE_PADD_FACOR_OM);
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_PE_DISAMBIGUITY_OM);
+	eht->data[0] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PE_DISAMBIG,
+				     IEEE80211_RADIOTAP_EHT_DATA0_PE_DISAMBIGUITY_OM);
+
+	/* TODO: what about IWL_RX_PHY_DATA0_EHT_BW320_SLOT */
+
+	if (!le32_get_bits(data0, IWL_RX_PHY_DATA0_EHT_SIGA_CRC_OK))
+		usig->common |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC);
+
+	usig->common |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER_KNOWN);
+	usig->common |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PHY_VER,
+				     IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER);
+
+	/*
+	 * TODO: what about TB - IWL_RX_PHY_DATA1_EHT_TB_PILOT_TYPE,
+	 *			 IWL_RX_PHY_DATA1_EHT_TB_LOW_SS
+	 */
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF);
+	eht->data[0] |= LE32_DEC_ENC(data1, IWL_RX_PHY_DATA1_EHT_SIG_LTF_NUM,
+				     IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF);
+
+	if (info_type == IWL_RX_PHY_INFO_TYPE_EHT_TB_EXT ||
+	    info_type == IWL_RX_PHY_INFO_TYPE_EHT_TB)
+		iwl_mld_decode_eht_ext_tb(mld, phy_data, rx_status, eht, usig);
+
+	if (info_type == IWL_RX_PHY_INFO_TYPE_EHT_MU_EXT ||
+	    info_type == IWL_RX_PHY_INFO_TYPE_EHT_MU)
+		iwl_mld_decode_eht_ext_mu(mld, phy_data, rx_status, eht, usig);
+}
+
+static void iwl_mld_rx_eht(struct iwl_mld *mld, struct sk_buff *skb,
+			   struct iwl_mld_rx_phy_data *phy_data,
+			   int queue)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	struct ieee80211_radiotap_eht *eht;
+	struct ieee80211_radiotap_eht_usig *usig;
+	size_t eht_len = sizeof(*eht);
+
+	u32 rate_n_flags = phy_data->rate_n_flags;
+	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK;
+	/* EHT and HE have the same values for LTF */
+	u8 ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_UNKNOWN;
+	u16 phy_info = phy_data->phy_info;
+	u32 bw;
+
+	/* u32 for 1 user_info */
+	if (phy_data->with_data)
+		eht_len += sizeof(u32);
+
+	eht = iwl_mld_radiotap_put_tlv(skb, IEEE80211_RADIOTAP_EHT, eht_len);
+
+	usig = iwl_mld_radiotap_put_tlv(skb, IEEE80211_RADIOTAP_EHT_USIG,
+					sizeof(*usig));
+	rx_status->flag |= RX_FLAG_RADIOTAP_TLV_AT_END;
+	usig->common |=
+		cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW_KNOWN);
+
+	/* specific handling for 320MHz */
+	bw = u32_get_bits(rate_n_flags, RATE_MCS_CHAN_WIDTH_MSK);
+	if (bw == RATE_MCS_CHAN_WIDTH_320_VAL)
+		bw += le32_get_bits(phy_data->data0,
+				    IWL_RX_PHY_DATA0_EHT_BW320_SLOT);
+
+	usig->common |= cpu_to_le32
+		(FIELD_PREP(IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW, bw));
+
+	/* report the AMPDU-EOF bit on single frames */
+	if (!queue && !(phy_info & IWL_RX_MPDU_PHY_AMPDU)) {
+		rx_status->flag |= RX_FLAG_AMPDU_DETAILS;
+		rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
+		if (phy_data->data0 &
+		    cpu_to_le32(IWL_RX_PHY_DATA0_EHT_DELIM_EOF))
+			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
+	}
+
+	/* update aggregation data for monitor sake on default queue */
+	if (!queue && (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) &&
+	    (phy_info & IWL_RX_MPDU_PHY_AMPDU) && phy_data->first_subframe) {
+		rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
+		if (phy_data->data0 &
+		    cpu_to_le32(IWL_RX_PHY_DATA0_EHT_DELIM_EOF))
+			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
+	}
+
+	if (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD)
+		iwl_mld_decode_eht_phy_data(mld, phy_data, rx_status, eht, usig);
+
+#define CHECK_TYPE(F)							\
+	BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA1_FORMAT_ ## F !=	\
+		     (RATE_MCS_HE_TYPE_ ## F >> RATE_MCS_HE_TYPE_POS))
+
+	CHECK_TYPE(SU);
+	CHECK_TYPE(EXT_SU);
+	CHECK_TYPE(MU);
+	CHECK_TYPE(TRIG);
+
+	switch (u32_get_bits(rate_n_flags, RATE_MCS_HE_GI_LTF_MSK)) {
+	case 0:
+		if (he_type == RATE_MCS_HE_TYPE_TRIG) {
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_1_6;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_1X;
+		} else {
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_0_8;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+		}
+		break;
+	case 1:
+		rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_1_6;
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+		break;
+	case 2:
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+		if (he_type == RATE_MCS_HE_TYPE_TRIG)
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_3_2;
+		else
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_0_8;
+		break;
+	case 3:
+		if (he_type != RATE_MCS_HE_TYPE_TRIG) {
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_3_2;
+		}
+		break;
+	default:
+		/* nothing here */
+		break;
+	}
+
+	if (ltf != IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_UNKNOWN) {
+		eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_GI);
+		eht->data[0] |= cpu_to_le32
+			(FIELD_PREP(IEEE80211_RADIOTAP_EHT_DATA0_LTF,
+				    ltf) |
+			 FIELD_PREP(IEEE80211_RADIOTAP_EHT_DATA0_GI,
+				    rx_status->eht.gi));
+	}
+
+	if (!phy_data->with_data) {
+		eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_NSS_S |
+					  IEEE80211_RADIOTAP_EHT_KNOWN_BEAMFORMED_S);
+		eht->data[7] |=
+			le32_encode_bits(le32_get_bits(phy_data->rx_vec[2],
+						       RX_NO_DATA_RX_VEC2_EHT_NSTS_MSK),
+					 IEEE80211_RADIOTAP_EHT_DATA7_NSS_S);
+		if (rate_n_flags & RATE_MCS_BF_MSK)
+			eht->data[7] |=
+				cpu_to_le32(IEEE80211_RADIOTAP_EHT_DATA7_BEAMFORMED_S);
+	} else {
+		eht->user_info[0] |=
+			cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_KNOWN_O |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_KNOWN_O |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_DATA_FOR_USER);
+
+		if (rate_n_flags & RATE_MCS_BF_MSK)
+			eht->user_info[0] |=
+				cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_O);
+
+		if (rate_n_flags & RATE_MCS_LDPC_MSK)
+			eht->user_info[0] |=
+				cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_CODING);
+
+		eht->user_info[0] |= cpu_to_le32
+			(FIELD_PREP(IEEE80211_RADIOTAP_EHT_USER_INFO_MCS,
+				    u32_get_bits(rate_n_flags,
+						 RATE_VHT_MCS_RATE_CODE_MSK)) |
+			 FIELD_PREP(IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_O,
+				    u32_get_bits(rate_n_flags,
+						 RATE_MCS_NSS_MSK)));
+	}
+}
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+static void iwl_mld_add_rtap_sniffer_config(struct iwl_mld *mld,
+					    struct sk_buff *skb)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	struct ieee80211_radiotap_vendor_content *radiotap;
+	const u16 vendor_data_len = sizeof(mld->monitor.cur_aid);
+
+	if (!mld->monitor.cur_aid)
+		return;
+
+	radiotap =
+		iwl_mld_radiotap_put_tlv(skb,
+					 IEEE80211_RADIOTAP_VENDOR_NAMESPACE,
+					 sizeof(*radiotap) + vendor_data_len);
+
+	/* Intel OUI */
+	radiotap->oui[0] = 0xf6;
+	radiotap->oui[1] = 0x54;
+	radiotap->oui[2] = 0x25;
+	/* radiotap sniffer config sub-namespace */
+	radiotap->oui_subtype = 1;
+	radiotap->vendor_type = 0;
+
+	/* fill the data now */
+	memcpy(radiotap->data, &mld->monitor.cur_aid,
+	       sizeof(mld->monitor.cur_aid));
+
+	rx_status->flag |= RX_FLAG_RADIOTAP_TLV_AT_END;
+}
+#endif
+
+/* Note: hdr can be NULL */
+static void iwl_mld_rx_fill_status(struct iwl_mld *mld, int link_id,
+				   struct ieee80211_hdr *hdr,
+				   struct sk_buff *skb,
+				   struct iwl_mld_rx_phy_data *phy_data,
+				   int queue)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	u32 format = phy_data->rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+	u32 rate_n_flags = phy_data->rate_n_flags;
+	u8 stbc = u32_get_bits(rate_n_flags, RATE_MCS_STBC_MSK);
+	bool is_sgi = rate_n_flags & RATE_MCS_SGI_MSK;
+
+	phy_data->info_type = IWL_RX_PHY_INFO_TYPE_NONE;
+
+	if (phy_data->phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD)
+		phy_data->info_type =
+			le32_get_bits(phy_data->data1,
+				      IWL_RX_PHY_DATA1_INFO_TYPE_MASK);
+
+	/* set the preamble flag if appropriate */
+	if (format == RATE_MCS_MOD_TYPE_CCK &&
+	    phy_data->phy_info & IWL_RX_MPDU_PHY_SHORT_PREAMBLE)
+		rx_status->enc_flags |= RX_ENC_FLAG_SHORTPRE;
+
+	iwl_mld_fill_signal(mld, link_id, hdr, rx_status, phy_data);
+
+	/* This may be overridden by iwl_mld_rx_he() to HE_RU */
+	switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) {
+	case RATE_MCS_CHAN_WIDTH_20:
+		break;
+	case RATE_MCS_CHAN_WIDTH_40:
+		rx_status->bw = RATE_INFO_BW_40;
+		break;
+	case RATE_MCS_CHAN_WIDTH_80:
+		rx_status->bw = RATE_INFO_BW_80;
+		break;
+	case RATE_MCS_CHAN_WIDTH_160:
+		rx_status->bw = RATE_INFO_BW_160;
+		break;
+	case RATE_MCS_CHAN_WIDTH_320:
+		rx_status->bw = RATE_INFO_BW_320;
+		break;
+	}
+
+	/* must be before L-SIG data */
+	if (format == RATE_MCS_MOD_TYPE_HE)
+		iwl_mld_rx_he(mld, skb, phy_data, queue);
+
+	iwl_mld_decode_lsig(skb, phy_data);
+
+	rx_status->device_timestamp = phy_data->gp2_on_air_rise;
+
+	/* using TLV format and must be after all fixed len fields */
+	if (format == RATE_MCS_MOD_TYPE_EHT)
+		iwl_mld_rx_eht(mld, skb, phy_data, queue);
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	if (unlikely(mld->monitor.on)) {
+		iwl_mld_add_rtap_sniffer_config(mld, skb);
+
+		if (mld->monitor.ptp_time) {
+			u64 adj_time =
+			    iwl_mld_ptp_get_adj_time(mld,
+						     phy_data->gp2_on_air_rise *
+						     NSEC_PER_USEC);
+
+			rx_status->mactime = div64_u64(adj_time, NSEC_PER_USEC);
+			rx_status->flag |= RX_FLAG_MACTIME_IS_RTAP_TS64;
+			rx_status->flag &= ~RX_FLAG_MACTIME;
+		}
+	}
+#endif
+
+	if (format != RATE_MCS_MOD_TYPE_CCK && is_sgi)
+		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
+
+	if (rate_n_flags & RATE_MCS_LDPC_MSK)
+		rx_status->enc_flags |= RX_ENC_FLAG_LDPC;
+
+	switch (format) {
+	case RATE_MCS_MOD_TYPE_HT:
+		rx_status->encoding = RX_ENC_HT;
+		rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags);
+		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
+		break;
+	case RATE_MCS_MOD_TYPE_VHT:
+	case RATE_MCS_MOD_TYPE_HE:
+	case RATE_MCS_MOD_TYPE_EHT:
+		if (format == RATE_MCS_MOD_TYPE_VHT) {
+			rx_status->encoding = RX_ENC_VHT;
+		} else if (format == RATE_MCS_MOD_TYPE_HE) {
+			rx_status->encoding = RX_ENC_HE;
+			rx_status->he_dcm =
+				!!(rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK);
+		} else if (format == RATE_MCS_MOD_TYPE_EHT) {
+			rx_status->encoding = RX_ENC_EHT;
+		}
+
+		rx_status->nss = u32_get_bits(rate_n_flags,
+					      RATE_MCS_NSS_MSK) + 1;
+		rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK;
+		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
+		break;
+	default: {
+		int rate =
+		    iwl_mld_legacy_hw_idx_to_mac80211_idx(rate_n_flags,
+							  rx_status->band);
+
+		/* valid rate */
+		if (rate >= 0 && rate <= 0xFF) {
+			rx_status->rate_idx = rate;
+			break;
+		}
+
+		/* invalid rate */
+		rx_status->rate_idx = 0;
+
+		if (net_ratelimit())
+			IWL_ERR(mld, "invalid rate_n_flags=0x%x, band=%d\n",
+				rate_n_flags, rx_status->band);
+		break;
+		}
+	}
+}
+
+/* iwl_mld_create_skb adds the rxb to a new skb */
+static int iwl_mld_build_rx_skb(struct iwl_mld *mld, struct sk_buff *skb,
+				struct ieee80211_hdr *hdr, u16 len,
+				u8 crypt_len, struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	struct iwl_rx_mpdu_desc *desc = (void *)pkt->data;
+	unsigned int headlen, fraglen, pad_len = 0;
+	unsigned int hdrlen = ieee80211_hdrlen(hdr->frame_control);
+	u8 mic_crc_len = u8_get_bits(desc->mac_flags1,
+				     IWL_RX_MPDU_MFLG1_MIC_CRC_LEN_MASK) << 1;
+
+	if (desc->mac_flags2 & IWL_RX_MPDU_MFLG2_PAD) {
+		len -= 2;
+		pad_len = 2;
+	}
+
+	/* For non monitor interface strip the bytes the RADA might not have
+	 * removed (it might be disabled, e.g. for mgmt frames). As a monitor
+	 * interface cannot exist with other interfaces, this removal is safe
+	 * and sufficient, in monitor mode there's no decryption being done.
+	 */
+	if (len > mic_crc_len && !ieee80211_hw_check(mld->hw, RX_INCLUDES_FCS))
+		len -= mic_crc_len;
+
+	/* If frame is small enough to fit in skb->head, pull it completely.
+	 * If not, only pull ieee80211_hdr (including crypto if present, and
+	 * an additional 8 bytes for SNAP/ethertype, see below) so that
+	 * splice() or TCP coalesce are more efficient.
+	 *
+	 * Since, in addition, ieee80211_data_to_8023() always pull in at
+	 * least 8 bytes (possibly more for mesh) we can do the same here
+	 * to save the cost of doing it later. That still doesn't pull in
+	 * the actual IP header since the typical case has a SNAP header.
+	 * If the latter changes (there are efforts in the standards group
+	 * to do so) we should revisit this and ieee80211_data_to_8023().
+	 */
+	headlen = (len <= skb_tailroom(skb)) ? len : hdrlen + crypt_len + 8;
+
+	/* The firmware may align the packet to DWORD.
+	 * The padding is inserted after the IV.
+	 * After copying the header + IV skip the padding if
+	 * present before copying packet data.
+	 */
+	hdrlen += crypt_len;
+
+	if (unlikely(headlen < hdrlen))
+		return -EINVAL;
+
+	/* Since data doesn't move data while putting data on skb and that is
+	 * the only way we use, data + len is the next place that hdr would
+	 * be put
+	 */
+	skb_set_mac_header(skb, skb->len);
+	skb_put_data(skb, hdr, hdrlen);
+	skb_put_data(skb, (u8 *)hdr + hdrlen + pad_len, headlen - hdrlen);
+
+	if (skb->ip_summed == CHECKSUM_COMPLETE) {
+		struct {
+			u8 hdr[6];
+			__be16 type;
+		} __packed *shdr = (void *)((u8 *)hdr + hdrlen + pad_len);
+
+		if (unlikely(headlen - hdrlen < sizeof(*shdr) ||
+			     !ether_addr_equal(shdr->hdr, rfc1042_header) ||
+			     (shdr->type != htons(ETH_P_IP) &&
+			      shdr->type != htons(ETH_P_ARP) &&
+			      shdr->type != htons(ETH_P_IPV6) &&
+			      shdr->type != htons(ETH_P_8021Q) &&
+			      shdr->type != htons(ETH_P_PAE) &&
+			      shdr->type != htons(ETH_P_TDLS))))
+			skb->ip_summed = CHECKSUM_NONE;
+	}
+
+	fraglen = len - headlen;
+
+	if (fraglen) {
+		int offset = (u8 *)hdr + headlen + pad_len -
+			     (u8 *)rxb_addr(rxb) + rxb_offset(rxb);
+
+		skb_add_rx_frag(skb, 0, rxb_steal_page(rxb), offset,
+				fraglen, rxb->truesize);
+	}
+
+	return 0;
+}
+
+/* returns true if a packet is a duplicate or invalid tid and
+ * should be dropped. Updates AMSDU PN tracking info
+ */
+VISIBLE_IF_IWLWIFI_KUNIT
+bool
+iwl_mld_is_dup(struct iwl_mld *mld, struct ieee80211_sta *sta,
+	       struct ieee80211_hdr *hdr,
+	       const struct iwl_rx_mpdu_desc *mpdu_desc,
+	       struct ieee80211_rx_status *rx_status, int queue)
+{
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_rxq_dup_data *dup_data;
+	u8 tid, sub_frame_idx;
+
+	if (WARN_ON(!sta))
+		return false;
+
+	mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+	if (WARN_ON_ONCE(!mld_sta->dup_data))
+		return false;
+
+	dup_data = &mld_sta->dup_data[queue];
+
+	/* Drop duplicate 802.11 retransmissions
+	 * (IEEE 802.11-2020: 10.3.2.14 "Duplicate detection and recovery")
+	 */
+	if (ieee80211_is_ctl(hdr->frame_control) ||
+	    ieee80211_is_any_nullfunc(hdr->frame_control) ||
+	    is_multicast_ether_addr(hdr->addr1))
+		return false;
+
+	if (ieee80211_is_data_qos(hdr->frame_control)) {
+		/* frame has qos control */
+		tid = ieee80211_get_tid(hdr);
+		if (tid >= IWL_MAX_TID_COUNT)
+			return true;
+	} else {
+		tid = IWL_MAX_TID_COUNT;
+	}
+
+	/* If this wasn't a part of an A-MSDU the sub-frame index will be 0 */
+	sub_frame_idx = mpdu_desc->amsdu_info &
+		IWL_RX_MPDU_AMSDU_SUBFRAME_IDX_MASK;
+
+	if (IWL_FW_CHECK(mld,
+			 sub_frame_idx > 0 &&
+			 !(mpdu_desc->mac_flags2 & IWL_RX_MPDU_MFLG2_AMSDU),
+			 "got sub_frame_idx=%d but A-MSDU flag is not set\n",
+			 sub_frame_idx))
+		return true;
+
+	if (unlikely(ieee80211_has_retry(hdr->frame_control) &&
+		     dup_data->last_seq[tid] == hdr->seq_ctrl &&
+		     dup_data->last_sub_frame_idx[tid] >= sub_frame_idx))
+		return true;
+
+	/* Allow same PN as the first subframe for following sub frames */
+	if (dup_data->last_seq[tid] == hdr->seq_ctrl &&
+	    sub_frame_idx > dup_data->last_sub_frame_idx[tid])
+		rx_status->flag |= RX_FLAG_ALLOW_SAME_PN;
+
+	dup_data->last_seq[tid] = hdr->seq_ctrl;
+	dup_data->last_sub_frame_idx[tid] = sub_frame_idx;
+
+	rx_status->flag |= RX_FLAG_DUP_VALIDATED;
+
+	return false;
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_is_dup);
+
+static void iwl_mld_update_last_rx_timestamp(struct iwl_mld *mld, u8 baid)
+{
+	unsigned long now = jiffies;
+	unsigned long timeout;
+	struct iwl_mld_baid_data *ba_data;
+
+	ba_data = rcu_dereference(mld->fw_id_to_ba[baid]);
+	if (!ba_data) {
+		IWL_DEBUG_HT(mld, "BAID %d not found in map\n", baid);
+		return;
+	}
+
+	if (!ba_data->timeout)
+		return;
+
+	/* To minimize cache bouncing between RX queues, avoid frequent updates
+	 * to last_rx_timestamp. update it only when the timeout period has
+	 * passed. The worst-case scenario is the session expiring after
+	 * approximately 2 * timeout, which is negligible (the update is
+	 * atomic).
+	 */
+	timeout = TU_TO_JIFFIES(ba_data->timeout);
+	if (time_is_before_jiffies(ba_data->last_rx_timestamp + timeout))
+		ba_data->last_rx_timestamp = now;
+}
+
+/* Processes received packets for a station.
+ * Sets *drop to true if the packet should be dropped.
+ * Returns the station if found, or NULL otherwise.
+ */
+static struct ieee80211_sta *
+iwl_mld_rx_with_sta(struct iwl_mld *mld, struct ieee80211_hdr *hdr,
+		    struct sk_buff *skb,
+		    const struct iwl_rx_mpdu_desc *mpdu_desc,
+		    const struct iwl_rx_packet *pkt, int queue, bool *drop)
+{
+	struct ieee80211_sta *sta = NULL;
+	struct ieee80211_link_sta *link_sta = NULL;
+	struct ieee80211_rx_status *rx_status;
+	u8 baid;
+
+	if (mpdu_desc->status & cpu_to_le32(IWL_RX_MPDU_STATUS_SRC_STA_FOUND)) {
+		u8 sta_id = le32_get_bits(mpdu_desc->status,
+					  IWL_RX_MPDU_STATUS_STA_ID);
+
+		if (IWL_FW_CHECK(mld,
+				 sta_id >= mld->fw->ucode_capa.num_stations,
+				 "rx_mpdu: invalid sta_id %d\n", sta_id))
+			return NULL;
+
+		link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]);
+		if (!IS_ERR_OR_NULL(link_sta))
+			sta = link_sta->sta;
+	} else if (!is_multicast_ether_addr(hdr->addr2)) {
+		/* Passing NULL is fine since we prevent two stations with the
+		 * same address from being added.
+		 */
+		sta = ieee80211_find_sta_by_ifaddr(mld->hw, hdr->addr2, NULL);
+	}
+
+	/* we may not have any station yet */
+	if (!sta)
+		return NULL;
+
+	rx_status = IEEE80211_SKB_RXCB(skb);
+
+	if (link_sta && sta->valid_links) {
+		rx_status->link_valid = true;
+		rx_status->link_id = link_sta->link_id;
+	}
+
+	/* fill checksum */
+	if (ieee80211_is_data(hdr->frame_control) &&
+	    pkt->len_n_flags & cpu_to_le32(FH_RSCSR_RPA_EN)) {
+		u16 hwsum = be16_to_cpu(mpdu_desc->v3.raw_xsum);
+
+		skb->ip_summed = CHECKSUM_COMPLETE;
+		skb->csum = csum_unfold(~(__force __sum16)hwsum);
+	}
+
+	if (iwl_mld_is_dup(mld, sta, hdr, mpdu_desc, rx_status, queue)) {
+		IWL_DEBUG_DROP(mld, "Dropping duplicate packet 0x%x\n",
+			       le16_to_cpu(hdr->seq_ctrl));
+		*drop = true;
+		return NULL;
+	}
+
+	baid = le32_get_bits(mpdu_desc->reorder_data,
+			     IWL_RX_MPDU_REORDER_BAID_MASK);
+	if (baid != IWL_RX_REORDER_DATA_INVALID_BAID)
+		iwl_mld_update_last_rx_timestamp(mld, baid);
+
+	if (link_sta && ieee80211_is_data(hdr->frame_control)) {
+		u8 sub_frame_idx = mpdu_desc->amsdu_info &
+			IWL_RX_MPDU_AMSDU_SUBFRAME_IDX_MASK;
+
+		/* 0 means not an A-MSDU, and 1 means a new A-MSDU */
+		if (!sub_frame_idx || sub_frame_idx == 1)
+			iwl_mld_count_mpdu_rx(link_sta, queue, 1);
+
+		if (!is_multicast_ether_addr(hdr->addr1))
+			iwl_mld_low_latency_update_counters(mld, hdr, sta,
+							    queue);
+	}
+
+	return sta;
+}
+
+#define KEY_IDX_LEN 2
+
+static int iwl_mld_rx_mgmt_prot(struct ieee80211_sta *sta,
+				struct ieee80211_hdr *hdr,
+				struct ieee80211_rx_status *rx_status,
+				u32 mpdu_status,
+				u32 mpdu_len)
+{
+	struct wireless_dev *wdev;
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_vif *mld_vif;
+	u8 keyidx;
+	struct ieee80211_key_conf *key;
+	const u8 *frame = (void *)hdr;
+
+	if ((mpdu_status & IWL_RX_MPDU_STATUS_SEC_MASK) ==
+	     IWL_RX_MPDU_STATUS_SEC_NONE)
+		return 0;
+
+	/* For non-beacon, we don't really care. But beacons may
+	 * be filtered out, and we thus need the firmware's replay
+	 * detection, otherwise beacons the firmware previously
+	 * filtered could be replayed, or something like that, and
+	 * it can filter a lot - though usually only if nothing has
+	 * changed.
+	 */
+	if (!ieee80211_is_beacon(hdr->frame_control))
+		return 0;
+
+	if (!sta)
+		return -1;
+
+	mld_sta = iwl_mld_sta_from_mac80211(sta);
+	mld_vif = iwl_mld_vif_from_mac80211(mld_sta->vif);
+
+	/* key mismatch - will also report !MIC_OK but we shouldn't count it */
+	if (!(mpdu_status & IWL_RX_MPDU_STATUS_KEY_VALID))
+		goto report;
+
+	/* good cases */
+	if (likely(mpdu_status & IWL_RX_MPDU_STATUS_MIC_OK &&
+		   !(mpdu_status & IWL_RX_MPDU_STATUS_REPLAY_ERROR))) {
+		rx_status->flag |= RX_FLAG_DECRYPTED;
+		return 0;
+	}
+
+	/* both keys will have the same cipher and MIC length, use
+	 * whichever one is available
+	 */
+	key = rcu_dereference(mld_vif->bigtks[0]);
+	if (!key) {
+		key = rcu_dereference(mld_vif->bigtks[1]);
+		if (!key)
+			goto report;
+	}
+
+	if (mpdu_len < key->icv_len + IEEE80211_GMAC_PN_LEN + KEY_IDX_LEN)
+		goto report;
+
+	/* get the real key ID */
+	keyidx = frame[mpdu_len - key->icv_len - IEEE80211_GMAC_PN_LEN - KEY_IDX_LEN];
+	/* and if that's the other key, look it up */
+	if (keyidx != key->keyidx) {
+		/* shouldn't happen since firmware checked, but be safe
+		 * in case the MIC length is wrong too, for example
+		 */
+		if (keyidx != 6 && keyidx != 7)
+			return -1;
+
+		key = rcu_dereference(mld_vif->bigtks[keyidx - 6]);
+		if (!key)
+			goto report;
+	}
+
+	/* Report status to mac80211 */
+	if (!(mpdu_status & IWL_RX_MPDU_STATUS_MIC_OK))
+		ieee80211_key_mic_failure(key);
+	else if (mpdu_status & IWL_RX_MPDU_STATUS_REPLAY_ERROR)
+		ieee80211_key_replay(key);
+report:
+	wdev = ieee80211_vif_to_wdev(mld_sta->vif);
+	if (wdev->netdev)
+		cfg80211_rx_unprot_mlme_mgmt(wdev->netdev, (void *)hdr,
+					     mpdu_len);
+
+	return -1;
+}
+
+static int iwl_mld_rx_crypto(struct iwl_mld *mld,
+			     struct ieee80211_sta *sta,
+			     struct ieee80211_hdr *hdr,
+			     struct ieee80211_rx_status *rx_status,
+			     struct iwl_rx_mpdu_desc *desc, int queue,
+			     u32 pkt_flags, u8 *crypto_len)
+{
+	u32 status = le32_to_cpu(desc->status);
+
+	if (unlikely(ieee80211_is_mgmt(hdr->frame_control) &&
+		     !ieee80211_has_protected(hdr->frame_control)))
+		return iwl_mld_rx_mgmt_prot(sta, hdr, rx_status, status,
+					    le16_to_cpu(desc->mpdu_len));
+
+	if (!ieee80211_has_protected(hdr->frame_control) ||
+	    (status & IWL_RX_MPDU_STATUS_SEC_MASK) ==
+	    IWL_RX_MPDU_STATUS_SEC_NONE)
+		return 0;
+
+	switch (status & IWL_RX_MPDU_STATUS_SEC_MASK) {
+	case IWL_RX_MPDU_STATUS_SEC_CCM:
+	case IWL_RX_MPDU_STATUS_SEC_GCM:
+		BUILD_BUG_ON(IEEE80211_CCMP_PN_LEN != IEEE80211_GCMP_PN_LEN);
+		if (!(status & IWL_RX_MPDU_STATUS_MIC_OK)) {
+			IWL_DEBUG_DROP(mld,
+				       "Dropping packet, bad MIC (CCM/GCM)\n");
+			return -1;
+		}
+
+		rx_status->flag |= RX_FLAG_DECRYPTED | RX_FLAG_MIC_STRIPPED;
+		*crypto_len = IEEE80211_CCMP_HDR_LEN;
+		return 0;
+	case IWL_RX_MPDU_STATUS_SEC_TKIP:
+		if (!(status & IWL_RX_MPDU_STATUS_ICV_OK))
+			return -1;
+
+		if (!(status & RX_MPDU_RES_STATUS_MIC_OK))
+			rx_status->flag |= RX_FLAG_MMIC_ERROR;
+
+		if (pkt_flags & FH_RSCSR_RADA_EN) {
+			rx_status->flag |= RX_FLAG_ICV_STRIPPED;
+			rx_status->flag |= RX_FLAG_MMIC_STRIPPED;
+		}
+
+		*crypto_len = IEEE80211_TKIP_IV_LEN;
+		rx_status->flag |= RX_FLAG_DECRYPTED;
+		return 0;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static void iwl_mld_rx_update_ampdu_ref(struct iwl_mld *mld,
+					struct iwl_mld_rx_phy_data *phy_data,
+					struct ieee80211_rx_status *rx_status)
+{
+	bool toggle_bit =
+		phy_data->phy_info & IWL_RX_MPDU_PHY_AMPDU_TOGGLE;
+
+	rx_status->flag |= RX_FLAG_AMPDU_DETAILS;
+	/* Toggle is switched whenever new aggregation starts. Make
+	 * sure ampdu_reference is never 0 so we can later use it to
+	 * see if the frame was really part of an A-MPDU or not.
+	 */
+	if (toggle_bit != mld->monitor.ampdu_toggle) {
+		mld->monitor.ampdu_ref++;
+		if (mld->monitor.ampdu_ref == 0)
+			mld->monitor.ampdu_ref++;
+		mld->monitor.ampdu_toggle = toggle_bit;
+		phy_data->first_subframe = true;
+	}
+	rx_status->ampdu_reference = mld->monitor.ampdu_ref;
+}
+
+static void
+iwl_mld_fill_rx_status_band_freq(struct ieee80211_rx_status *rx_status,
+				 u8 band, u8 channel)
+{
+	rx_status->band = iwl_mld_phy_band_to_nl80211(band);
+	rx_status->freq = ieee80211_channel_to_frequency(channel,
+							 rx_status->band);
+}
+
+void iwl_mld_rx_mpdu(struct iwl_mld *mld, struct napi_struct *napi,
+		     struct iwl_rx_cmd_buffer *rxb, int queue)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	struct iwl_mld_rx_phy_data phy_data = {};
+	struct iwl_rx_mpdu_desc *mpdu_desc = (void *)pkt->data;
+	struct ieee80211_sta *sta;
+	struct ieee80211_hdr *hdr;
+	struct sk_buff *skb;
+	size_t mpdu_desc_size = sizeof(*mpdu_desc);
+	bool drop = false;
+	u8 crypto_len = 0, band, link_id;
+	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
+	u32 mpdu_len;
+	enum iwl_mld_reorder_result reorder_res;
+	struct ieee80211_rx_status *rx_status;
+
+	if (unlikely(mld->fw_status.in_hw_restart))
+		return;
+
+	if (IWL_FW_CHECK(mld, pkt_len < mpdu_desc_size,
+			 "Bad REPLY_RX_MPDU_CMD size (%d)\n", pkt_len))
+		return;
+
+	mpdu_len = le16_to_cpu(mpdu_desc->mpdu_len);
+
+	if (IWL_FW_CHECK(mld, mpdu_len + mpdu_desc_size > pkt_len,
+			 "FW lied about packet len (%d)\n", pkt_len))
+		return;
+
+	/* Don't use dev_alloc_skb(), we'll have enough headroom once
+	 * ieee80211_hdr pulled.
+	 */
+	skb = alloc_skb(128, GFP_ATOMIC);
+	if (!skb) {
+		IWL_ERR(mld, "alloc_skb failed\n");
+		return;
+	}
+
+	hdr = (void *)(pkt->data + mpdu_desc_size);
+
+	iwl_mld_fill_phy_data(mld, mpdu_desc, &phy_data);
+
+	if (mpdu_desc->mac_flags2 & IWL_RX_MPDU_MFLG2_PAD) {
+		/* If the device inserted padding it means that (it thought)
+		 * the 802.11 header wasn't a multiple of 4 bytes long. In
+		 * this case, reserve two bytes at the start of the SKB to
+		 * align the payload properly in case we end up copying it.
+		 */
+		skb_reserve(skb, 2);
+	}
+
+	rx_status = IEEE80211_SKB_RXCB(skb);
+
+	/* this is needed early */
+	band = u8_get_bits(mpdu_desc->mac_phy_band,
+			   IWL_RX_MPDU_MAC_PHY_BAND_BAND_MASK);
+	iwl_mld_fill_rx_status_band_freq(rx_status, band,
+					 mpdu_desc->v3.channel);
+
+
+	rcu_read_lock();
+
+	sta = iwl_mld_rx_with_sta(mld, hdr, skb, mpdu_desc, pkt, queue, &drop);
+	if (drop)
+		goto drop;
+
+	/* update aggregation data for monitor sake on default queue */
+	if (!queue && (phy_data.phy_info & IWL_RX_MPDU_PHY_AMPDU))
+		iwl_mld_rx_update_ampdu_ref(mld, &phy_data, rx_status);
+
+	/* Keep packets with CRC errors (and with overrun) for monitor mode
+	 * (otherwise the firmware discards them) but mark them as bad.
+	 */
+	if (!(mpdu_desc->status & cpu_to_le32(IWL_RX_MPDU_STATUS_CRC_OK)) ||
+	    !(mpdu_desc->status & cpu_to_le32(IWL_RX_MPDU_STATUS_OVERRUN_OK))) {
+		IWL_DEBUG_RX(mld, "Bad CRC or FIFO: 0x%08X.\n",
+			     le32_to_cpu(mpdu_desc->status));
+		rx_status->flag |= RX_FLAG_FAILED_FCS_CRC;
+	}
+
+	if (likely(!(phy_data.phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD))) {
+		rx_status->mactime =
+			le64_to_cpu(mpdu_desc->v3.tsf_on_air_rise);
+
+		/* TSF as indicated by the firmware is at INA time */
+		rx_status->flag |= RX_FLAG_MACTIME_PLCP_START;
+	}
+
+	/* management stuff on default queue */
+	if (!queue && unlikely(ieee80211_is_beacon(hdr->frame_control) ||
+			       ieee80211_is_probe_resp(hdr->frame_control))) {
+		rx_status->boottime_ns = ktime_get_boottime_ns();
+
+		if (mld->scan.pass_all_sched_res ==
+				SCHED_SCAN_PASS_ALL_STATE_ENABLED)
+			mld->scan.pass_all_sched_res =
+				SCHED_SCAN_PASS_ALL_STATE_FOUND;
+	}
+
+	link_id = u8_get_bits(mpdu_desc->mac_phy_band,
+			      IWL_RX_MPDU_MAC_PHY_BAND_LINK_MASK);
+
+	iwl_mld_rx_fill_status(mld, link_id, hdr, skb, &phy_data, queue);
+
+	if (iwl_mld_rx_crypto(mld, sta, hdr, rx_status, mpdu_desc, queue,
+			      le32_to_cpu(pkt->len_n_flags), &crypto_len))
+		goto drop;
+
+	if (iwl_mld_build_rx_skb(mld, skb, hdr, mpdu_len, crypto_len, rxb))
+		goto drop;
+
+	/* time sync frame is saved and will be released later when the
+	 * notification with the timestamps arrives.
+	 */
+	if (iwl_mld_time_sync_frame(mld, skb, hdr->addr2))
+		goto out;
+
+	reorder_res = iwl_mld_reorder(mld, napi, queue, sta, skb, mpdu_desc);
+	switch (reorder_res) {
+	case IWL_MLD_PASS_SKB:
+		break;
+	case IWL_MLD_DROP_SKB:
+		goto drop;
+	case IWL_MLD_BUFFERED_SKB:
+		goto out;
+	default:
+		WARN_ON(1);
+		goto drop;
+	}
+
+	iwl_mld_pass_packet_to_mac80211(mld, napi, skb, queue, sta);
+
+	goto out;
+
+drop:
+	kfree_skb(skb);
+out:
+	rcu_read_unlock();
+}
+
+#define SYNC_RX_QUEUE_TIMEOUT (HZ)
+void iwl_mld_sync_rx_queues(struct iwl_mld *mld,
+			    enum iwl_mld_internal_rxq_notif_type type,
+			    const void *notif_payload, u32 notif_payload_size)
+{
+	u8 num_rx_queues = mld->trans->info.num_rxqs;
+	struct {
+		struct iwl_rxq_sync_cmd sync_cmd;
+		struct iwl_mld_internal_rxq_notif notif;
+	} __packed cmd = {
+		.sync_cmd.rxq_mask = cpu_to_le32(BIT(num_rx_queues) - 1),
+		.sync_cmd.count =
+			cpu_to_le32(sizeof(struct iwl_mld_internal_rxq_notif) +
+				    notif_payload_size),
+		.notif.type = type,
+		.notif.cookie = mld->rxq_sync.cookie,
+	};
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(DATA_PATH_GROUP, TRIGGER_RX_QUEUES_NOTIF_CMD),
+		.data[0] = &cmd,
+		.len[0] = sizeof(cmd),
+		.data[1] = notif_payload,
+		.len[1] = notif_payload_size,
+	};
+	int ret;
+
+	/* size must be a multiple of DWORD */
+	if (WARN_ON(cmd.sync_cmd.count & cpu_to_le32(3)))
+		return;
+
+	mld->rxq_sync.state = (1 << num_rx_queues) - 1;
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+	if (ret) {
+		IWL_ERR(mld, "Failed to trigger RX queues sync (%d)\n", ret);
+		goto out;
+	}
+
+	ret = wait_event_timeout(mld->rxq_sync.waitq,
+				 READ_ONCE(mld->rxq_sync.state) == 0,
+				 SYNC_RX_QUEUE_TIMEOUT);
+	WARN_ONCE(!ret, "RXQ sync failed: state=0x%lx, cookie=%d\n",
+		  mld->rxq_sync.state, mld->rxq_sync.cookie);
+
+out:
+	mld->rxq_sync.state = 0;
+	mld->rxq_sync.cookie++;
+}
+
+void iwl_mld_handle_rx_queues_sync_notif(struct iwl_mld *mld,
+					 struct napi_struct *napi,
+					 struct iwl_rx_packet *pkt, int queue)
+{
+	struct iwl_rxq_sync_notification *notif;
+	struct iwl_mld_internal_rxq_notif *internal_notif;
+	u32 len = iwl_rx_packet_payload_len(pkt);
+	size_t combined_notif_len = sizeof(*notif) + sizeof(*internal_notif);
+
+	notif = (void *)pkt->data;
+	internal_notif = (void *)notif->payload;
+
+	if (IWL_FW_CHECK(mld, len < combined_notif_len,
+			 "invalid notification size %u (%zu)\n",
+			 len, combined_notif_len))
+		return;
+
+	len -= combined_notif_len;
+
+	if (IWL_FW_CHECK(mld, mld->rxq_sync.cookie != internal_notif->cookie,
+			 "received expired RX queue sync message (cookie=%d expected=%d q[%d])\n",
+			 internal_notif->cookie, mld->rxq_sync.cookie, queue))
+		return;
+
+	switch (internal_notif->type) {
+	case IWL_MLD_RXQ_EMPTY:
+		IWL_FW_CHECK(mld, len,
+			     "invalid empty notification size %d\n", len);
+		break;
+	case IWL_MLD_RXQ_NOTIF_DEL_BA:
+		if (IWL_FW_CHECK(mld, len != sizeof(struct iwl_mld_delba_data),
+				 "invalid delba notification size %u (%zu)\n",
+				 len, sizeof(struct iwl_mld_delba_data)))
+			break;
+		iwl_mld_del_ba(mld, queue, (void *)internal_notif->payload);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+	}
+
+	IWL_FW_CHECK(mld, !test_and_clear_bit(queue, &mld->rxq_sync.state),
+		     "RXQ sync: queue %d responded a second time!\n", queue);
+
+	if (READ_ONCE(mld->rxq_sync.state) == 0)
+		wake_up(&mld->rxq_sync.waitq);
+}
+
+void iwl_mld_rx_monitor_no_data(struct iwl_mld *mld, struct napi_struct *napi,
+				struct iwl_rx_packet *pkt, int queue)
+{
+	struct iwl_rx_no_data_ver_3 *desc;
+	struct iwl_mld_rx_phy_data phy_data;
+	struct ieee80211_rx_status *rx_status;
+	struct sk_buff *skb;
+	u32 format, rssi;
+	u8 channel;
+
+	if (unlikely(mld->fw_status.in_hw_restart))
+		return;
+
+	if (IWL_FW_CHECK(mld, iwl_rx_packet_payload_len(pkt) < sizeof(*desc),
+			 "Bad RX_NO_DATA_NOTIF size (%d)\n",
+			 iwl_rx_packet_payload_len(pkt)))
+		return;
+
+	desc = (void *)pkt->data;
+
+	rssi = le32_to_cpu(desc->rssi);
+	channel = u32_get_bits(rssi, RX_NO_DATA_CHANNEL_MSK);
+
+	phy_data.energy_a = u32_get_bits(rssi, RX_NO_DATA_CHAIN_A_MSK);
+	phy_data.energy_b = u32_get_bits(rssi, RX_NO_DATA_CHAIN_B_MSK);
+	phy_data.data0 = desc->phy_info[0];
+	phy_data.data1 = desc->phy_info[1];
+	phy_data.phy_info = IWL_RX_MPDU_PHY_TSF_OVERLOAD;
+	phy_data.gp2_on_air_rise = le32_to_cpu(desc->on_air_rise_time);
+	phy_data.rate_n_flags = iwl_v3_rate_from_v2_v3(desc->rate,
+						       mld->fw_rates_ver_3);
+	phy_data.with_data = false;
+
+	BUILD_BUG_ON(sizeof(phy_data.rx_vec) != sizeof(desc->rx_vec));
+	memcpy(phy_data.rx_vec, desc->rx_vec, sizeof(phy_data.rx_vec));
+
+	format = phy_data.rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+
+	/* Don't use dev_alloc_skb(), we'll have enough headroom once
+	 * ieee80211_hdr pulled.
+	 */
+	skb = alloc_skb(128, GFP_ATOMIC);
+	if (!skb) {
+		IWL_ERR(mld, "alloc_skb failed\n");
+		return;
+	}
+
+	rx_status = IEEE80211_SKB_RXCB(skb);
+
+	/* 0-length PSDU */
+	rx_status->flag |= RX_FLAG_NO_PSDU;
+
+	/* mark as failed PLCP on any errors to skip checks in mac80211 */
+	if (le32_get_bits(desc->info, RX_NO_DATA_INFO_ERR_MSK) !=
+	    RX_NO_DATA_INFO_ERR_NONE)
+		rx_status->flag |= RX_FLAG_FAILED_PLCP_CRC;
+
+	switch (le32_get_bits(desc->info, RX_NO_DATA_INFO_TYPE_MSK)) {
+	case RX_NO_DATA_INFO_TYPE_NDP:
+		rx_status->zero_length_psdu_type =
+			IEEE80211_RADIOTAP_ZERO_LEN_PSDU_SOUNDING;
+		break;
+	case RX_NO_DATA_INFO_TYPE_MU_UNMATCHED:
+	case RX_NO_DATA_INFO_TYPE_TB_UNMATCHED:
+		rx_status->zero_length_psdu_type =
+			IEEE80211_RADIOTAP_ZERO_LEN_PSDU_NOT_CAPTURED;
+		break;
+	default:
+		rx_status->zero_length_psdu_type =
+			IEEE80211_RADIOTAP_ZERO_LEN_PSDU_VENDOR;
+		break;
+	}
+
+	rx_status->band = channel > 14 ? NL80211_BAND_5GHZ :
+		NL80211_BAND_2GHZ;
+
+	rx_status->freq = ieee80211_channel_to_frequency(channel,
+							 rx_status->band);
+
+	/* link ID is ignored for NULL header */
+	iwl_mld_rx_fill_status(mld, -1, NULL, skb, &phy_data, queue);
+
+	/* No more radiotap info should be added after this point.
+	 * Mark it as mac header for upper layers to know where
+	 * the radiotap header ends.
+	 */
+	skb_set_mac_header(skb, skb->len);
+
+	/* Override the nss from the rx_vec since the rate_n_flags has
+	 * only 1 bit for the nss which gives a max of 2 ss but there
+	 * may be up to 8 spatial streams.
+	 */
+	switch (format) {
+	case RATE_MCS_MOD_TYPE_VHT:
+		rx_status->nss =
+			le32_get_bits(desc->rx_vec[0],
+				      RX_NO_DATA_RX_VEC0_VHT_NSTS_MSK) + 1;
+		break;
+	case RATE_MCS_MOD_TYPE_HE:
+		rx_status->nss =
+			le32_get_bits(desc->rx_vec[0],
+				      RX_NO_DATA_RX_VEC0_HE_NSTS_MSK) + 1;
+		break;
+	case RATE_MCS_MOD_TYPE_EHT:
+		rx_status->nss =
+			le32_get_bits(desc->rx_vec[2],
+				      RX_NO_DATA_RX_VEC2_EHT_NSTS_MSK) + 1;
+	}
+
+	/* pass the packet to mac80211 */
+	rcu_read_lock();
+	ieee80211_rx_napi(mld->hw, NULL, skb, napi);
+	rcu_read_unlock();
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/rx.h b/sys/contrib/dev/iwlwifi/mld/rx.h
new file mode 100644
index 000000000000..2beabd7e70b1
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/rx.h
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_rx_h__
+#define __iwl_mld_rx_h__
+
+#include "mld.h"
+
+/**
+ * enum iwl_mld_internal_rxq_notif_type - RX queue sync notif types
+ *
+ * @IWL_MLD_RXQ_EMPTY: empty sync notification
+ * @IWL_MLD_RXQ_NOTIF_DEL_BA: notify RSS queues of delBA
+ */
+enum iwl_mld_internal_rxq_notif_type {
+	IWL_MLD_RXQ_EMPTY,
+	IWL_MLD_RXQ_NOTIF_DEL_BA,
+};
+
+/**
+ * struct iwl_mld_internal_rxq_notif - @iwl_rxq_sync_cmd internal data.
+ * This data is echoed by the firmware to all RSS queues and should be DWORD
+ * aligned. FW is agnostic to the data, so there are no endianness requirements
+ *
+ * @type: one of &iwl_mld_internal_rxq_notif_type
+ * @cookie: unique internal cookie to identify old notifications
+ * @reserved: reserved for alignment
+ * @payload: data to send to RX queues based on the type (may be empty)
+ */
+struct iwl_mld_internal_rxq_notif {
+	u8 type;
+	u8 reserved[3];
+	u32 cookie;
+	u8 payload[];
+} __packed;
+
+/**
+ * struct iwl_mld_rx_queues_sync - RX queues sync data
+ *
+ * @waitq: wait queue for RX queues sync completion
+ * @cookie: unique id to correlate sync requests with responses
+ * @state: bitmask representing the sync state of RX queues
+ *	all RX queues bits are set before sending the command, and the
+ *	corresponding queue bit cleared upon handling the notification
+ */
+struct iwl_mld_rx_queues_sync {
+	wait_queue_head_t waitq;
+	u32 cookie;
+	unsigned long state;
+};
+
+void iwl_mld_rx_mpdu(struct iwl_mld *mld, struct napi_struct *napi,
+		     struct iwl_rx_cmd_buffer *rxb, int queue);
+
+void iwl_mld_sync_rx_queues(struct iwl_mld *mld,
+			    enum iwl_mld_internal_rxq_notif_type type,
+			    const void *notif_payload, u32 notif_payload_size);
+
+void iwl_mld_handle_rx_queues_sync_notif(struct iwl_mld *mld,
+					 struct napi_struct *napi,
+					 struct iwl_rx_packet *pkt, int queue);
+
+void iwl_mld_pass_packet_to_mac80211(struct iwl_mld *mld,
+				     struct napi_struct *napi,
+				     struct sk_buff *skb, int queue,
+				     struct ieee80211_sta *sta);
+
+void iwl_mld_rx_monitor_no_data(struct iwl_mld *mld, struct napi_struct *napi,
+				struct iwl_rx_packet *pkt, int queue);
+
+#endif /* __iwl_mld_agg_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/scan.c b/sys/contrib/dev/iwlwifi/mld/scan.c
new file mode 100644
index 000000000000..62f97a18a16c
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/scan.c
@@ -0,0 +1,2181 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <linux/crc32.h>
+
+#include "iwl-utils.h"
+
+#include "mld.h"
+#include "scan.h"
+#include "hcmd.h"
+#include "iface.h"
+#include "phy.h"
+#include "mlo.h"
+
+#include "fw/api/scan.h"
+#include "fw/dbg.h"
+
+#define IWL_SCAN_DWELL_ACTIVE 10
+#define IWL_SCAN_DWELL_PASSIVE 110
+#define IWL_SCAN_NUM_OF_FRAGS 3
+
+/* adaptive dwell max budget time [TU] for full scan */
+#define IWL_SCAN_ADWELL_MAX_BUDGET_FULL_SCAN 300
+
+/* adaptive dwell max budget time [TU] for directed scan */
+#define IWL_SCAN_ADWELL_MAX_BUDGET_DIRECTED_SCAN 100
+
+/* adaptive dwell default high band APs number */
+#define IWL_SCAN_ADWELL_DEFAULT_HB_N_APS 8
+
+/* adaptive dwell default low band APs number */
+#define IWL_SCAN_ADWELL_DEFAULT_LB_N_APS 2
+
+/* adaptive dwell default APs number for P2P social channels (1, 6, 11) */
+#define IWL_SCAN_ADWELL_DEFAULT_N_APS_SOCIAL 10
+
+/* adaptive dwell number of APs override for P2P friendly GO channels */
+#define IWL_SCAN_ADWELL_N_APS_GO_FRIENDLY 10
+
+/* adaptive dwell number of APs override for P2P social channels */
+#define IWL_SCAN_ADWELL_N_APS_SOCIAL_CHS 2
+
+/* adaptive dwell number of APs override mask for p2p friendly GO */
+#define IWL_SCAN_ADWELL_N_APS_GO_FRIENDLY_BIT BIT(20)
+
+/* adaptive dwell number of APs override mask for social channels */
+#define IWL_SCAN_ADWELL_N_APS_SOCIAL_CHS_BIT BIT(21)
+
+#define SCAN_TIMEOUT_MSEC (30000 * HZ)
+
+/* minimal number of 2GHz and 5GHz channels in the regular scan request */
+#define IWL_MLD_6GHZ_PASSIVE_SCAN_MIN_CHANS 4
+
+enum iwl_mld_scan_type {
+	IWL_SCAN_TYPE_NOT_SET,
+	IWL_SCAN_TYPE_UNASSOC,
+	IWL_SCAN_TYPE_WILD,
+	IWL_SCAN_TYPE_MILD,
+	IWL_SCAN_TYPE_FRAGMENTED,
+	IWL_SCAN_TYPE_FAST_BALANCE,
+};
+
+struct iwl_mld_scan_timing_params {
+	u32 suspend_time;
+	u32 max_out_time;
+};
+
+static const struct iwl_mld_scan_timing_params scan_timing[] = {
+	[IWL_SCAN_TYPE_UNASSOC] = {
+		.suspend_time = 0,
+		.max_out_time = 0,
+	},
+	[IWL_SCAN_TYPE_WILD] = {
+		.suspend_time = 30,
+		.max_out_time = 120,
+	},
+	[IWL_SCAN_TYPE_MILD] = {
+		.suspend_time = 120,
+		.max_out_time = 120,
+	},
+	[IWL_SCAN_TYPE_FRAGMENTED] = {
+		.suspend_time = 95,
+		.max_out_time = 44,
+	},
+	[IWL_SCAN_TYPE_FAST_BALANCE] = {
+		.suspend_time = 30,
+		.max_out_time = 37,
+	},
+};
+
+struct iwl_mld_scan_params {
+	enum iwl_mld_scan_type type;
+	u32 n_channels;
+	u16 delay;
+	int n_ssids;
+	struct cfg80211_ssid *ssids;
+	struct ieee80211_channel **channels;
+	u32 flags;
+	u8 *mac_addr;
+	u8 *mac_addr_mask;
+	bool no_cck;
+	bool pass_all;
+	int n_match_sets;
+	struct iwl_scan_probe_req preq;
+	struct cfg80211_match_set *match_sets;
+	int n_scan_plans;
+	struct cfg80211_sched_scan_plan *scan_plans;
+	bool iter_notif;
+	bool respect_p2p_go;
+	u8 fw_link_id;
+	struct cfg80211_scan_6ghz_params *scan_6ghz_params;
+	u32 n_6ghz_params;
+	bool scan_6ghz;
+	bool enable_6ghz_passive;
+	u8 bssid[ETH_ALEN] __aligned(2);
+};
+
+struct iwl_mld_scan_respect_p2p_go_iter_data {
+	struct ieee80211_vif *current_vif;
+	bool p2p_go;
+};
+
+static void iwl_mld_scan_respect_p2p_go_iter(void *_data, u8 *mac,
+					     struct ieee80211_vif *vif)
+{
+	struct iwl_mld_scan_respect_p2p_go_iter_data *data = _data;
+
+	/* exclude the given vif */
+	if (vif == data->current_vif)
+		return;
+
+	/* TODO: CDB check the band of the GO */
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_P2P_GO &&
+	    iwl_mld_vif_from_mac80211(vif)->ap_ibss_active)
+		data->p2p_go = true;
+}
+
+static bool iwl_mld_get_respect_p2p_go(struct iwl_mld *mld,
+				       struct ieee80211_vif *vif,
+				       bool low_latency)
+{
+	struct iwl_mld_scan_respect_p2p_go_iter_data data = {
+		.current_vif = vif,
+		.p2p_go = false,
+	};
+
+	if (!low_latency)
+		return false;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_scan_respect_p2p_go_iter,
+						&data);
+
+	return data.p2p_go;
+}
+
+struct iwl_mld_scan_iter_data {
+	struct ieee80211_vif *current_vif;
+	bool active_vif;
+	bool is_dcm_with_p2p_go;
+	bool global_low_latency;
+};
+
+static void iwl_mld_scan_iterator(void *_data, u8 *mac,
+				  struct ieee80211_vif *vif)
+{
+	struct iwl_mld_scan_iter_data *data = _data;
+	struct ieee80211_vif *curr_vif = data->current_vif;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_vif *curr_mld_vif;
+	unsigned long curr_vif_active_links;
+	u16 link_id;
+
+	data->global_low_latency |= iwl_mld_vif_low_latency(mld_vif);
+
+	if ((ieee80211_vif_is_mld(vif) && vif->active_links) ||
+	    (vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+	     mld_vif->deflink.active))
+		data->active_vif = true;
+
+	if (vif == curr_vif)
+		return;
+
+	if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_P2P_GO)
+		return;
+
+	/* Currently P2P GO can't be AP MLD so the logic below assumes that */
+	WARN_ON_ONCE(ieee80211_vif_is_mld(vif));
+
+	curr_vif_active_links =
+		ieee80211_vif_is_mld(curr_vif) ? curr_vif->active_links : 1;
+
+	curr_mld_vif = iwl_mld_vif_from_mac80211(curr_vif);
+
+	for_each_set_bit(link_id, &curr_vif_active_links,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct iwl_mld_link *curr_mld_link =
+			iwl_mld_link_dereference_check(curr_mld_vif, link_id);
+
+		if (WARN_ON(!curr_mld_link))
+			return;
+
+		if (rcu_access_pointer(curr_mld_link->chan_ctx) &&
+		    rcu_access_pointer(mld_vif->deflink.chan_ctx) !=
+		    rcu_access_pointer(curr_mld_link->chan_ctx)) {
+			data->is_dcm_with_p2p_go = true;
+			return;
+		}
+	}
+}
+
+static enum
+iwl_mld_scan_type iwl_mld_get_scan_type(struct iwl_mld *mld,
+					struct ieee80211_vif *vif,
+					struct iwl_mld_scan_iter_data *data)
+{
+	enum iwl_mld_traffic_load load = mld->scan.traffic_load.status;
+
+	/* A scanning AP interface probably wants to generate a survey to do
+	 * ACS (automatic channel selection).
+	 * Force a non-fragmented scan in that case.
+	 */
+	if (ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_AP)
+		return IWL_SCAN_TYPE_WILD;
+
+	if (!data->active_vif)
+		return IWL_SCAN_TYPE_UNASSOC;
+
+	if ((load == IWL_MLD_TRAFFIC_HIGH || data->global_low_latency) &&
+	    vif->type != NL80211_IFTYPE_P2P_DEVICE)
+		return IWL_SCAN_TYPE_FRAGMENTED;
+
+	/* In case of DCM with P2P GO set all scan requests as
+	 * fast-balance scan
+	 */
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    data->is_dcm_with_p2p_go)
+		return IWL_SCAN_TYPE_FAST_BALANCE;
+
+	if (load >= IWL_MLD_TRAFFIC_MEDIUM || data->global_low_latency)
+		return IWL_SCAN_TYPE_MILD;
+
+	return IWL_SCAN_TYPE_WILD;
+}
+
+static u8 *
+iwl_mld_scan_add_2ghz_elems(struct iwl_mld *mld, const u8 *ies,
+			    size_t len, u8 *const pos)
+{
+	static const u8 before_ds_params[] = {
+	    WLAN_EID_SSID,
+	    WLAN_EID_SUPP_RATES,
+	    WLAN_EID_REQUEST,
+	    WLAN_EID_EXT_SUPP_RATES,
+	};
+	size_t offs;
+	u8 *newpos = pos;
+
+	offs = ieee80211_ie_split(ies, len,
+				  before_ds_params,
+				  ARRAY_SIZE(before_ds_params),
+				  0);
+
+	memcpy(newpos, ies, offs);
+	newpos += offs;
+
+	/* Add a placeholder for DS Parameter Set element */
+	*newpos++ = WLAN_EID_DS_PARAMS;
+	*newpos++ = 1;
+	*newpos++ = 0;
+
+	memcpy(newpos, ies + offs, len - offs);
+	newpos += len - offs;
+
+	return newpos;
+}
+
+static void
+iwl_mld_scan_add_tpc_report_elem(u8 *pos)
+{
+	pos[0] = WLAN_EID_VENDOR_SPECIFIC;
+	pos[1] = WFA_TPC_IE_LEN - 2;
+	pos[2] = (WLAN_OUI_MICROSOFT >> 16) & 0xff;
+	pos[3] = (WLAN_OUI_MICROSOFT >> 8) & 0xff;
+	pos[4] = WLAN_OUI_MICROSOFT & 0xff;
+	pos[5] = WLAN_OUI_TYPE_MICROSOFT_TPC;
+	pos[6] = 0;
+	/* pos[7] - tx power will be inserted by the FW */
+	pos[7] = 0;
+	pos[8] = 0;
+}
+
+static u32
+iwl_mld_scan_ooc_priority(enum iwl_mld_scan_status scan_status)
+{
+	if (scan_status == IWL_MLD_SCAN_REGULAR)
+		return IWL_SCAN_PRIORITY_EXT_6;
+	if (scan_status == IWL_MLD_SCAN_INT_MLO)
+		return IWL_SCAN_PRIORITY_EXT_4;
+
+	return IWL_SCAN_PRIORITY_EXT_2;
+}
+
+static bool
+iwl_mld_scan_is_regular(struct iwl_mld_scan_params *params)
+{
+	return params->n_scan_plans == 1 &&
+		params->scan_plans[0].iterations == 1;
+}
+
+static bool
+iwl_mld_scan_is_fragmented(enum iwl_mld_scan_type type)
+{
+	return (type == IWL_SCAN_TYPE_FRAGMENTED ||
+		type == IWL_SCAN_TYPE_FAST_BALANCE);
+}
+
+static int
+iwl_mld_scan_uid_by_status(struct iwl_mld *mld, int status)
+{
+	for (int i = 0; i < ARRAY_SIZE(mld->scan.uid_status); i++)
+		if (mld->scan.uid_status[i] == status)
+			return i;
+
+	return -ENOENT;
+}
+
+static const char *
+iwl_mld_scan_ebs_status_str(enum iwl_scan_ebs_status status)
+{
+	switch (status) {
+	case IWL_SCAN_EBS_SUCCESS:
+		return "successful";
+	case IWL_SCAN_EBS_INACTIVE:
+		return "inactive";
+	case IWL_SCAN_EBS_FAILED:
+	case IWL_SCAN_EBS_CHAN_NOT_FOUND:
+	default:
+		return "failed";
+	}
+}
+
+static int
+iwl_mld_scan_ssid_exist(u8 *ssid, u8 ssid_len, struct iwl_ssid_ie *ssid_list)
+{
+	for (int i = 0; i < PROBE_OPTION_MAX; i++) {
+		if (!ssid_list[i].len)
+			return -1;
+		if (ssid_list[i].len == ssid_len &&
+		    !memcmp(ssid_list[i].ssid, ssid, ssid_len))
+			return i;
+	}
+
+	return -1;
+}
+
+static bool
+iwl_mld_scan_fits(struct iwl_mld *mld, int n_ssids,
+		  struct ieee80211_scan_ies *ies, int n_channels)
+{
+	return ((n_ssids <= PROBE_OPTION_MAX) &&
+		(n_channels <= mld->fw->ucode_capa.n_scan_channels) &&
+		(ies->common_ie_len + ies->len[NL80211_BAND_2GHZ] +
+		 ies->len[NL80211_BAND_5GHZ] + ies->len[NL80211_BAND_6GHZ] <=
+		 iwl_mld_scan_max_template_size()));
+}
+
+static void
+iwl_mld_scan_build_probe_req(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			     struct ieee80211_scan_ies *ies,
+			     struct iwl_mld_scan_params *params)
+{
+	struct ieee80211_mgmt *frame = (void *)params->preq.buf;
+	u8 *pos, *newpos;
+	const u8 *mac_addr = params->flags & NL80211_SCAN_FLAG_RANDOM_ADDR ?
+		params->mac_addr : NULL;
+
+	if (mac_addr)
+		get_random_mask_addr(frame->sa, mac_addr,
+				     params->mac_addr_mask);
+	else
+		memcpy(frame->sa, vif->addr, ETH_ALEN);
+
+	frame->frame_control = cpu_to_le16(IEEE80211_STYPE_PROBE_REQ);
+	eth_broadcast_addr(frame->da);
+	ether_addr_copy(frame->bssid, params->bssid);
+	frame->seq_ctrl = 0;
+
+	pos = frame->u.probe_req.variable;
+	*pos++ = WLAN_EID_SSID;
+	*pos++ = 0;
+
+	params->preq.mac_header.offset = 0;
+	params->preq.mac_header.len = cpu_to_le16(24 + 2);
+
+	/* Insert DS parameter set element on 2.4 GHz band */
+	newpos = iwl_mld_scan_add_2ghz_elems(mld,
+					     ies->ies[NL80211_BAND_2GHZ],
+					     ies->len[NL80211_BAND_2GHZ],
+					     pos);
+	params->preq.band_data[0].offset = cpu_to_le16(pos - params->preq.buf);
+	params->preq.band_data[0].len = cpu_to_le16(newpos - pos);
+	pos = newpos;
+
+	memcpy(pos, ies->ies[NL80211_BAND_5GHZ],
+	       ies->len[NL80211_BAND_5GHZ]);
+	params->preq.band_data[1].offset = cpu_to_le16(pos - params->preq.buf);
+	params->preq.band_data[1].len =
+	    cpu_to_le16(ies->len[NL80211_BAND_5GHZ]);
+	pos += ies->len[NL80211_BAND_5GHZ];
+
+	memcpy(pos, ies->ies[NL80211_BAND_6GHZ],
+	       ies->len[NL80211_BAND_6GHZ]);
+	params->preq.band_data[2].offset = cpu_to_le16(pos - params->preq.buf);
+	params->preq.band_data[2].len =
+		cpu_to_le16(ies->len[NL80211_BAND_6GHZ]);
+	pos += ies->len[NL80211_BAND_6GHZ];
+
+	memcpy(pos, ies->common_ies, ies->common_ie_len);
+	params->preq.common_data.offset = cpu_to_le16(pos - params->preq.buf);
+
+	iwl_mld_scan_add_tpc_report_elem(pos + ies->common_ie_len);
+	params->preq.common_data.len = cpu_to_le16(ies->common_ie_len +
+						   WFA_TPC_IE_LEN);
+}
+
+static u16
+iwl_mld_scan_get_cmd_gen_flags(struct iwl_mld *mld,
+			       struct iwl_mld_scan_params *params,
+			       struct ieee80211_vif *vif,
+			       enum iwl_mld_scan_status scan_status)
+{
+	u16 flags = 0;
+
+	/* If no direct SSIDs are provided perform a passive scan. Otherwise,
+	 * if there is a single SSID which is not the broadcast SSID, assume
+	 * that the scan is intended for roaming purposes and thus enable Rx on
+	 * all chains to improve chances of hearing the beacons/probe responses.
+	 */
+	if (params->n_ssids == 0)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_FORCE_PASSIVE;
+	else if (params->n_ssids == 1 && params->ssids[0].ssid_len)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_USE_ALL_RX_CHAINS;
+
+	if (params->pass_all)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_PASS_ALL;
+	else
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_MATCH;
+
+	if (iwl_mld_scan_is_fragmented(params->type))
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_FRAGMENTED_LMAC1;
+
+	if (!iwl_mld_scan_is_regular(params))
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_PERIODIC;
+
+	if (params->iter_notif ||
+	    mld->scan.pass_all_sched_res == SCHED_SCAN_PASS_ALL_STATE_ENABLED)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_NTFY_ITER_COMPLETE;
+
+	if (scan_status == IWL_MLD_SCAN_SCHED ||
+	    scan_status == IWL_MLD_SCAN_NETDETECT)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_PREEMPTIVE;
+
+	if (params->flags & (NL80211_SCAN_FLAG_ACCEPT_BCAST_PROBE_RESP |
+			     NL80211_SCAN_FLAG_OCE_PROBE_REQ_HIGH_TX_RATE |
+			     NL80211_SCAN_FLAG_FILS_MAX_CHANNEL_TIME))
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_OCE;
+
+	if ((scan_status == IWL_MLD_SCAN_SCHED ||
+	     scan_status == IWL_MLD_SCAN_NETDETECT) &&
+	    params->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_TRIGGER_UHB_SCAN;
+
+	if (params->enable_6ghz_passive)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_6GHZ_PASSIVE_SCAN;
+
+	flags |= IWL_UMAC_SCAN_GEN_FLAGS_V2_ADAPTIVE_DWELL;
+
+	return flags;
+}
+
+static u8
+iwl_mld_scan_get_cmd_gen_flags2(struct iwl_mld *mld,
+				struct iwl_mld_scan_params *params,
+				struct ieee80211_vif *vif,
+				enum iwl_mld_scan_status scan_status,
+				u16 gen_flags)
+{
+	u8 flags = 0;
+
+	/* TODO: CDB */
+	if (params->respect_p2p_go)
+		flags |= IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_RESPECT_P2P_GO_LB |
+			IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_RESPECT_P2P_GO_HB;
+
+	if (params->scan_6ghz)
+		flags |= IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_DONT_TOGGLE_ANT;
+
+	/* For AP interfaces, request survey data for regular scans and if
+	 * it is supported. For non-AP interfaces, EBS will be enabled and
+	 * the results may be missing information for some channels.
+	 */
+	if (scan_status == IWL_MLD_SCAN_REGULAR &&
+	    ieee80211_vif_type_p2p(vif) == NL80211_IFTYPE_AP &&
+	    gen_flags & IWL_UMAC_SCAN_GEN_FLAGS_V2_FORCE_PASSIVE &&
+	    iwl_fw_lookup_notif_ver(mld->fw, SCAN_GROUP,
+				    CHANNEL_SURVEY_NOTIF, 0) >= 1)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS2_COLLECT_CHANNEL_STATS;
+
+	return flags;
+}
+
+static void
+iwl_mld_scan_cmd_set_dwell(struct iwl_mld *mld,
+			   struct iwl_scan_general_params_v11 *gp,
+			   struct iwl_mld_scan_params *params)
+{
+	const struct iwl_mld_scan_timing_params *timing =
+		&scan_timing[params->type];
+
+	gp->adwell_default_social_chn =
+	    IWL_SCAN_ADWELL_DEFAULT_N_APS_SOCIAL;
+	gp->adwell_default_2g = IWL_SCAN_ADWELL_DEFAULT_LB_N_APS;
+	gp->adwell_default_5g = IWL_SCAN_ADWELL_DEFAULT_HB_N_APS;
+
+	if (params->n_ssids && params->ssids[0].ssid_len)
+		gp->adwell_max_budget =
+			cpu_to_le16(IWL_SCAN_ADWELL_MAX_BUDGET_DIRECTED_SCAN);
+	else
+		gp->adwell_max_budget =
+			cpu_to_le16(IWL_SCAN_ADWELL_MAX_BUDGET_FULL_SCAN);
+
+	gp->scan_priority = cpu_to_le32(IWL_SCAN_PRIORITY_EXT_6);
+
+	gp->max_out_of_time[SCAN_LB_LMAC_IDX] = cpu_to_le32(timing->max_out_time);
+	gp->suspend_time[SCAN_LB_LMAC_IDX] = cpu_to_le32(timing->suspend_time);
+
+	gp->active_dwell[SCAN_LB_LMAC_IDX] = IWL_SCAN_DWELL_ACTIVE;
+	gp->passive_dwell[SCAN_LB_LMAC_IDX] = IWL_SCAN_DWELL_PASSIVE;
+	gp->active_dwell[SCAN_HB_LMAC_IDX] = IWL_SCAN_DWELL_ACTIVE;
+	gp->passive_dwell[SCAN_HB_LMAC_IDX] = IWL_SCAN_DWELL_PASSIVE;
+
+	IWL_DEBUG_SCAN(mld,
+		       "Scan: adwell_max_budget=%d max_out_of_time=%d suspend_time=%d\n",
+		       gp->adwell_max_budget,
+		       gp->max_out_of_time[SCAN_LB_LMAC_IDX],
+		       gp->suspend_time[SCAN_LB_LMAC_IDX]);
+}
+
+static void
+iwl_mld_scan_cmd_set_gen_params(struct iwl_mld *mld,
+				struct iwl_mld_scan_params *params,
+				struct ieee80211_vif *vif,
+				struct iwl_scan_general_params_v11 *gp,
+				enum iwl_mld_scan_status scan_status)
+{
+	u16 gen_flags = iwl_mld_scan_get_cmd_gen_flags(mld, params, vif,
+						       scan_status);
+	u8 gen_flags2 = iwl_mld_scan_get_cmd_gen_flags2(mld, params, vif,
+							scan_status,
+							gen_flags);
+
+	IWL_DEBUG_SCAN(mld, "General: flags=0x%x, flags2=0x%x\n",
+		       gen_flags, gen_flags2);
+
+	gp->flags = cpu_to_le16(gen_flags);
+	gp->flags2 = gen_flags2;
+
+	iwl_mld_scan_cmd_set_dwell(mld, gp, params);
+
+	if (gen_flags & IWL_UMAC_SCAN_GEN_FLAGS_V2_FRAGMENTED_LMAC1)
+		gp->num_of_fragments[SCAN_LB_LMAC_IDX] = IWL_SCAN_NUM_OF_FRAGS;
+
+	if (params->fw_link_id != IWL_MLD_INVALID_FW_ID)
+		gp->scan_start_mac_or_link_id = params->fw_link_id;
+}
+
+static int
+iwl_mld_scan_cmd_set_sched_params(struct iwl_mld_scan_params *params,
+				  struct iwl_scan_umac_schedule *schedule,
+				  __le16 *delay)
+{
+	if (WARN_ON(!params->n_scan_plans ||
+		    params->n_scan_plans > IWL_MAX_SCHED_SCAN_PLANS))
+		return -EINVAL;
+
+	for (int i = 0; i < params->n_scan_plans; i++) {
+		struct cfg80211_sched_scan_plan *scan_plan =
+		    &params->scan_plans[i];
+
+		schedule[i].iter_count = scan_plan->iterations;
+		schedule[i].interval =
+		    cpu_to_le16(scan_plan->interval);
+	}
+
+	/* If the number of iterations of the last scan plan is set to zero,
+	 * it should run infinitely. However, this is not always the case.
+	 * For example, when regular scan is requested the driver sets one scan
+	 * plan with one iteration.
+	 */
+	if (!schedule[params->n_scan_plans - 1].iter_count)
+		schedule[params->n_scan_plans - 1].iter_count = 0xff;
+
+	*delay = cpu_to_le16(params->delay);
+
+	return 0;
+}
+
+/* We insert the SSIDs in an inverted order, because the FW will
+ * invert it back.
+ */
+static void
+iwl_mld_scan_cmd_build_ssids(struct iwl_mld_scan_params *params,
+			     struct iwl_ssid_ie *ssids, u32 *ssid_bitmap)
+{
+	int i, j;
+	int index;
+	u32 tmp_bitmap = 0;
+
+	/* copy SSIDs from match list. iwl_config_sched_scan_profiles()
+	 * uses the order of these ssids to config match list.
+	 */
+	for (i = 0, j = params->n_match_sets - 1;
+	     j >= 0 && i < PROBE_OPTION_MAX;
+	     i++, j--) {
+		/* skip empty SSID match_sets */
+		if (!params->match_sets[j].ssid.ssid_len)
+			continue;
+
+		ssids[i].id = WLAN_EID_SSID;
+		ssids[i].len = params->match_sets[j].ssid.ssid_len;
+		memcpy(ssids[i].ssid, params->match_sets[j].ssid.ssid,
+		       ssids[i].len);
+	}
+
+	/* add SSIDs from scan SSID list */
+	for (j = params->n_ssids - 1;
+	     j >= 0 && i < PROBE_OPTION_MAX;
+	     i++, j--) {
+		index = iwl_mld_scan_ssid_exist(params->ssids[j].ssid,
+						params->ssids[j].ssid_len,
+						ssids);
+		if (index < 0) {
+			ssids[i].id = WLAN_EID_SSID;
+			ssids[i].len = params->ssids[j].ssid_len;
+			memcpy(ssids[i].ssid, params->ssids[j].ssid,
+			       ssids[i].len);
+			tmp_bitmap |= BIT(i);
+		} else {
+			tmp_bitmap |= BIT(index);
+		}
+	}
+
+	if (ssid_bitmap)
+		*ssid_bitmap = tmp_bitmap;
+}
+
+static void
+iwl_mld_scan_fill_6g_chan_list(struct iwl_mld_scan_params *params,
+			       struct iwl_scan_probe_params_v4 *pp)
+{
+	int j, idex_s = 0, idex_b = 0;
+	struct cfg80211_scan_6ghz_params *scan_6ghz_params =
+		params->scan_6ghz_params;
+
+	for (j = 0;
+	     j < params->n_ssids && idex_s < SCAN_SHORT_SSID_MAX_SIZE;
+	     j++) {
+		if (!params->ssids[j].ssid_len)
+			continue;
+
+		pp->short_ssid[idex_s] =
+			cpu_to_le32(~crc32_le(~0, params->ssids[j].ssid,
+					      params->ssids[j].ssid_len));
+
+		/* hidden 6ghz scan */
+		pp->direct_scan[idex_s].id = WLAN_EID_SSID;
+		pp->direct_scan[idex_s].len = params->ssids[j].ssid_len;
+		memcpy(pp->direct_scan[idex_s].ssid, params->ssids[j].ssid,
+		       params->ssids[j].ssid_len);
+		idex_s++;
+	}
+
+	/* Populate the arrays of the short SSIDs and the BSSIDs using the 6GHz
+	 * collocated parameters. This might not be optimal, as this processing
+	 * does not (yet) correspond to the actual channels, so it is possible
+	 * that some entries would be left out.
+	 */
+	for (j = 0; j < params->n_6ghz_params; j++) {
+		int k;
+
+		/* First, try to place the short SSID */
+		if (scan_6ghz_params[j].short_ssid_valid) {
+			for (k = 0; k < idex_s; k++) {
+				if (pp->short_ssid[k] ==
+				    cpu_to_le32(scan_6ghz_params[j].short_ssid))
+					break;
+			}
+
+			if (k == idex_s && idex_s < SCAN_SHORT_SSID_MAX_SIZE) {
+				pp->short_ssid[idex_s++] =
+					cpu_to_le32(scan_6ghz_params[j].short_ssid);
+			}
+		}
+
+		/* try to place BSSID for the same entry */
+		for (k = 0; k < idex_b; k++) {
+			if (!memcmp(&pp->bssid_array[k],
+				    scan_6ghz_params[j].bssid, ETH_ALEN))
+				break;
+		}
+
+		if (k == idex_b && idex_b < SCAN_BSSID_MAX_SIZE &&
+		    !WARN_ONCE(!is_valid_ether_addr(scan_6ghz_params[j].bssid),
+			       "scan: invalid BSSID at index %u, index_b=%u\n",
+			       j, idex_b)) {
+			memcpy(&pp->bssid_array[idex_b++],
+			       scan_6ghz_params[j].bssid, ETH_ALEN);
+		}
+	}
+
+	pp->short_ssid_num = idex_s;
+	pp->bssid_num = idex_b;
+}
+
+static void
+iwl_mld_scan_cmd_set_probe_params(struct iwl_mld_scan_params *params,
+				  struct iwl_scan_probe_params_v4 *pp,
+				  u32 *bitmap_ssid)
+{
+	pp->preq = params->preq;
+
+	if (params->scan_6ghz) {
+		iwl_mld_scan_fill_6g_chan_list(params, pp);
+		return;
+	}
+
+	/* relevant only for 2.4 GHz /5 GHz scan */
+	iwl_mld_scan_cmd_build_ssids(params, pp->direct_scan, bitmap_ssid);
+}
+
+static bool
+iwl_mld_scan_use_ebs(struct iwl_mld *mld, struct ieee80211_vif *vif,
+		     bool low_latency)
+{
+	const struct iwl_ucode_capabilities *capa = &mld->fw->ucode_capa;
+
+	/* We can only use EBS if:
+	 *	1. the feature is supported.
+	 *	2. the last EBS was successful.
+	 *	3. it's not a p2p find operation.
+	 *	4. we are not in low latency mode,
+	 *	   or if fragmented ebs is supported by the FW
+	 *	5. the VIF is not an AP interface (scan wants survey results)
+	 */
+	return ((capa->flags & IWL_UCODE_TLV_FLAGS_EBS_SUPPORT) &&
+		!mld->scan.last_ebs_failed &&
+		vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+		(!low_latency || fw_has_api(capa, IWL_UCODE_TLV_API_FRAG_EBS)) &&
+		ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_AP);
+}
+
+static u8
+iwl_mld_scan_cmd_set_chan_flags(struct iwl_mld *mld,
+				struct iwl_mld_scan_params *params,
+				struct ieee80211_vif *vif,
+				bool low_latency)
+{
+	u8 flags = 0;
+
+	flags |= IWL_SCAN_CHANNEL_FLAG_ENABLE_CHAN_ORDER;
+
+	if (iwl_mld_scan_use_ebs(mld, vif, low_latency))
+		flags |= IWL_SCAN_CHANNEL_FLAG_EBS |
+			 IWL_SCAN_CHANNEL_FLAG_EBS_ACCURATE |
+			 IWL_SCAN_CHANNEL_FLAG_CACHE_ADD;
+
+	/* set fragmented ebs for fragmented scan */
+	if (iwl_mld_scan_is_fragmented(params->type))
+		flags |= IWL_SCAN_CHANNEL_FLAG_EBS_FRAG;
+
+	/* Force EBS in case the scan is a fragmented and there is a need
+	 * to take P2P GO operation into consideration during scan operation.
+	 */
+	/* TODO: CDB */
+	if (iwl_mld_scan_is_fragmented(params->type) &&
+	    params->respect_p2p_go) {
+		IWL_DEBUG_SCAN(mld, "Respect P2P GO. Force EBS\n");
+		flags |= IWL_SCAN_CHANNEL_FLAG_FORCE_EBS;
+	}
+
+	return flags;
+}
+
+static const u8 p2p_go_friendly_chs[] = {
+	36, 40, 44, 48, 149, 153, 157, 161, 165,
+};
+
+static const u8 social_chs[] = {
+	1, 6, 11
+};
+
+static u32 iwl_mld_scan_ch_n_aps_flag(enum nl80211_iftype vif_type, u8 ch_id)
+{
+	if (vif_type != NL80211_IFTYPE_P2P_DEVICE)
+		return 0;
+
+	for (int i = 0; i < ARRAY_SIZE(p2p_go_friendly_chs); i++) {
+		if (ch_id == p2p_go_friendly_chs[i])
+			return IWL_SCAN_ADWELL_N_APS_GO_FRIENDLY_BIT;
+	}
+
+	for (int i = 0; i < ARRAY_SIZE(social_chs); i++) {
+		if (ch_id == social_chs[i])
+			return IWL_SCAN_ADWELL_N_APS_SOCIAL_CHS_BIT;
+	}
+
+	return 0;
+}
+
+static void
+iwl_mld_scan_cmd_set_channels(struct iwl_mld *mld,
+			      struct ieee80211_channel **channels,
+			      struct iwl_scan_channel_params_v7 *cp,
+			      int n_channels, u32 flags,
+			      enum nl80211_iftype vif_type)
+{
+	for (int i = 0; i < n_channels; i++) {
+		enum nl80211_band band = channels[i]->band;
+		struct iwl_scan_channel_cfg_umac *cfg = &cp->channel_config[i];
+		u8 iwl_band = iwl_mld_nl80211_band_to_fw(band);
+		u32 n_aps_flag =
+			iwl_mld_scan_ch_n_aps_flag(vif_type,
+						   channels[i]->hw_value);
+
+		if (IWL_MLD_ADAPTIVE_DWELL_NUM_APS_OVERRIDE)
+			n_aps_flag = IWL_SCAN_ADWELL_N_APS_GO_FRIENDLY_BIT;
+
+		cfg->flags = cpu_to_le32(flags | n_aps_flag);
+		cfg->channel_num = channels[i]->hw_value;
+		if (cfg80211_channel_is_psc(channels[i]))
+			cfg->flags = 0;
+
+		if (band == NL80211_BAND_6GHZ) {
+			/* 6 GHz channels should only appear in a scan request
+			 * that has scan_6ghz set. The only exception is MLO
+			 * scan, which has to be passive.
+			 */
+			WARN_ON_ONCE(cfg->flags != 0);
+			cfg->flags =
+				cpu_to_le32(IWL_UHB_CHAN_CFG_FLAG_FORCE_PASSIVE);
+		}
+
+		cfg->v2.iter_count = 1;
+		cfg->v2.iter_interval = 0;
+		cfg->flags |= cpu_to_le32(iwl_band <<
+					  IWL_CHAN_CFG_FLAGS_BAND_POS);
+	}
+}
+
+static u8
+iwl_mld_scan_cfg_channels_6g(struct iwl_mld *mld,
+			     struct iwl_mld_scan_params *params,
+			     u32 n_channels,
+			     struct iwl_scan_probe_params_v4 *pp,
+			     struct iwl_scan_channel_params_v7 *cp,
+			     enum nl80211_iftype vif_type)
+{
+	struct cfg80211_scan_6ghz_params *scan_6ghz_params =
+		params->scan_6ghz_params;
+	u32 i;
+	u8 ch_cnt;
+
+	for (i = 0, ch_cnt = 0; i < params->n_channels; i++) {
+		struct iwl_scan_channel_cfg_umac *cfg =
+			&cp->channel_config[ch_cnt];
+
+		u32 s_ssid_bitmap = 0, bssid_bitmap = 0, flags = 0;
+		u8 k, n_s_ssids = 0, n_bssids = 0;
+		u8 max_s_ssids, max_bssids;
+		bool force_passive = false, found = false, allow_passive = true,
+		     unsolicited_probe_on_chan = false, psc_no_listen = false;
+		s8 psd_20 = IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED;
+
+		/* Avoid performing passive scan on non PSC channels unless the
+		 * scan is specifically a passive scan, i.e., no SSIDs
+		 * configured in the scan command.
+		 */
+		if (!cfg80211_channel_is_psc(params->channels[i]) &&
+		    !params->n_6ghz_params && params->n_ssids)
+			continue;
+
+		cfg->channel_num = params->channels[i]->hw_value;
+		cfg->flags |=
+			cpu_to_le32(PHY_BAND_6 << IWL_CHAN_CFG_FLAGS_BAND_POS);
+
+		cfg->v5.iter_count = 1;
+		cfg->v5.iter_interval = 0;
+
+		for (u32 j = 0; j < params->n_6ghz_params; j++) {
+			s8 tmp_psd_20;
+
+			if (!(scan_6ghz_params[j].channel_idx == i))
+				continue;
+
+			unsolicited_probe_on_chan |=
+				scan_6ghz_params[j].unsolicited_probe;
+
+			/* Use the highest PSD value allowed as advertised by
+			 * APs for this channel
+			 */
+			tmp_psd_20 = scan_6ghz_params[j].psd_20;
+			if (tmp_psd_20 !=
+			    IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED &&
+			    (psd_20 ==
+			     IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED ||
+			     psd_20 < tmp_psd_20))
+				psd_20 = tmp_psd_20;
+
+			psc_no_listen |= scan_6ghz_params[j].psc_no_listen;
+		}
+
+		/* In the following cases apply passive scan:
+		 * 1. Non fragmented scan:
+		 *	- PSC channel with NO_LISTEN_FLAG on should be treated
+		 *	  like non PSC channel
+		 *	- Non PSC channel with more than 3 short SSIDs or more
+		 *	  than 9 BSSIDs.
+		 *	- Non PSC Channel with unsolicited probe response and
+		 *	  more than 2 short SSIDs or more than 6 BSSIDs.
+		 *	- PSC channel with more than 2 short SSIDs or more than
+		 *	  6 BSSIDs.
+		 * 2. Fragmented scan:
+		 *	- PSC channel with more than 1 SSID or 3 BSSIDs.
+		 *	- Non PSC channel with more than 2 SSIDs or 6 BSSIDs.
+		 *	- Non PSC channel with unsolicited probe response and
+		 *	  more than 1 SSID or more than 3 BSSIDs.
+		 */
+		if (!iwl_mld_scan_is_fragmented(params->type)) {
+			if (!cfg80211_channel_is_psc(params->channels[i]) ||
+			    psc_no_listen) {
+				if (unsolicited_probe_on_chan) {
+					max_s_ssids = 2;
+					max_bssids = 6;
+				} else {
+					max_s_ssids = 3;
+					max_bssids = 9;
+				}
+			} else {
+				max_s_ssids = 2;
+				max_bssids = 6;
+			}
+		} else if (cfg80211_channel_is_psc(params->channels[i])) {
+			max_s_ssids = 1;
+			max_bssids = 3;
+		} else {
+			if (unsolicited_probe_on_chan) {
+				max_s_ssids = 1;
+				max_bssids = 3;
+			} else {
+				max_s_ssids = 2;
+				max_bssids = 6;
+			}
+		}
+
+		/* To optimize the scan time, i.e., reduce the scan dwell time
+		 * on each channel, the below logic tries to set 3 direct BSSID
+		 * probe requests for each broadcast probe request with a short
+		 * SSID.
+		 */
+		for (u32 j = 0; j < params->n_6ghz_params; j++) {
+			if (!(scan_6ghz_params[j].channel_idx == i))
+				continue;
+
+			found = false;
+
+			for (k = 0;
+			     k < pp->short_ssid_num && n_s_ssids < max_s_ssids;
+			     k++) {
+				if (!scan_6ghz_params[j].unsolicited_probe &&
+				    le32_to_cpu(pp->short_ssid[k]) ==
+				    scan_6ghz_params[j].short_ssid) {
+					/* Relevant short SSID bit set */
+					if (s_ssid_bitmap & BIT(k)) {
+						found = true;
+						break;
+					}
+
+					/* Prefer creating BSSID entries unless
+					 * the short SSID probe can be done in
+					 * the same channel dwell iteration.
+					 *
+					 * We also need to create a short SSID
+					 * entry for any hidden AP.
+					 */
+					if (3 * n_s_ssids > n_bssids &&
+					    !pp->direct_scan[k].len)
+						break;
+
+					/* Hidden AP, cannot do passive scan */
+					if (pp->direct_scan[k].len)
+						allow_passive = false;
+
+					s_ssid_bitmap |= BIT(k);
+					n_s_ssids++;
+					found = true;
+					break;
+				}
+			}
+
+			if (found)
+				continue;
+
+			for (k = 0; k < pp->bssid_num; k++) {
+				if (memcmp(&pp->bssid_array[k],
+					   scan_6ghz_params[j].bssid,
+					   ETH_ALEN))
+					continue;
+
+				if (bssid_bitmap & BIT(k))
+					break;
+
+				if (n_bssids < max_bssids) {
+					bssid_bitmap |= BIT(k);
+					n_bssids++;
+				} else {
+					force_passive = TRUE;
+				}
+
+				break;
+			}
+		}
+
+		if (cfg80211_channel_is_psc(params->channels[i]) &&
+		    psc_no_listen)
+			flags |= IWL_UHB_CHAN_CFG_FLAG_PSC_CHAN_NO_LISTEN;
+
+		if (unsolicited_probe_on_chan)
+			flags |= IWL_UHB_CHAN_CFG_FLAG_UNSOLICITED_PROBE_RES;
+
+		if ((allow_passive && force_passive) ||
+		    (!(bssid_bitmap | s_ssid_bitmap) &&
+		     !cfg80211_channel_is_psc(params->channels[i])))
+			flags |= IWL_UHB_CHAN_CFG_FLAG_FORCE_PASSIVE;
+		else
+			flags |= bssid_bitmap | (s_ssid_bitmap << 16);
+
+		cfg->flags |= cpu_to_le32(flags);
+		cfg->v5.psd_20 = psd_20;
+
+		ch_cnt++;
+	}
+
+	if (params->n_channels > ch_cnt)
+		IWL_DEBUG_SCAN(mld,
+			       "6GHz: reducing number channels: (%u->%u)\n",
+			       params->n_channels, ch_cnt);
+
+	return ch_cnt;
+}
+
+static int
+iwl_mld_scan_cmd_set_6ghz_chan_params(struct iwl_mld *mld,
+				      struct iwl_mld_scan_params *params,
+				      struct ieee80211_vif *vif,
+				      struct iwl_scan_req_params_v17 *scan_p,
+				      enum iwl_mld_scan_status scan_status)
+{
+	struct iwl_scan_channel_params_v7 *chan_p = &scan_p->channel_params;
+	struct iwl_scan_probe_params_v4 *probe_p = &scan_p->probe_params;
+
+	chan_p->flags = iwl_mld_scan_get_cmd_gen_flags(mld, params, vif,
+						       scan_status);
+	chan_p->count = iwl_mld_scan_cfg_channels_6g(mld, params,
+						     params->n_channels,
+						     probe_p, chan_p,
+						     vif->type);
+	if (!chan_p->count)
+		return -EINVAL;
+
+	if (!params->n_ssids ||
+	    (params->n_ssids == 1 && !params->ssids[0].ssid_len))
+		chan_p->flags |= IWL_SCAN_CHANNEL_FLAG_6G_PSC_NO_FILTER;
+
+	return 0;
+}
+
+static int
+iwl_mld_scan_cmd_set_chan_params(struct iwl_mld *mld,
+				 struct iwl_mld_scan_params *params,
+				 struct ieee80211_vif *vif,
+				 struct iwl_scan_req_params_v17 *scan_p,
+				 bool low_latency,
+				 enum iwl_mld_scan_status scan_status,
+				 u32 channel_cfg_flags)
+{
+	struct iwl_scan_channel_params_v7 *cp = &scan_p->channel_params;
+	struct ieee80211_supported_band *sband =
+		&mld->nvm_data->bands[NL80211_BAND_6GHZ];
+
+	cp->n_aps_override[0] = IWL_SCAN_ADWELL_N_APS_GO_FRIENDLY;
+	cp->n_aps_override[1] = IWL_SCAN_ADWELL_N_APS_SOCIAL_CHS;
+
+	if (IWL_MLD_ADAPTIVE_DWELL_NUM_APS_OVERRIDE)
+		cp->n_aps_override[0] = IWL_MLD_ADAPTIVE_DWELL_NUM_APS_OVERRIDE;
+
+	if (params->scan_6ghz)
+		return iwl_mld_scan_cmd_set_6ghz_chan_params(mld, params,
+							     vif, scan_p,
+							     scan_status);
+
+	/* relevant only for 2.4 GHz/5 GHz scan */
+	cp->flags = iwl_mld_scan_cmd_set_chan_flags(mld, params, vif,
+						    low_latency);
+	cp->count = params->n_channels;
+
+	iwl_mld_scan_cmd_set_channels(mld, params->channels, cp,
+				      params->n_channels, channel_cfg_flags,
+				      vif->type);
+
+	if (!params->enable_6ghz_passive)
+		return 0;
+
+	/* fill 6 GHz passive scan cfg */
+	for (int i = 0; i < sband->n_channels; i++) {
+		struct ieee80211_channel *channel =
+			&sband->channels[i];
+		struct iwl_scan_channel_cfg_umac *cfg =
+			&cp->channel_config[cp->count];
+
+		if (!cfg80211_channel_is_psc(channel))
+			continue;
+
+		cfg->channel_num = channel->hw_value;
+		cfg->v5.iter_count = 1;
+		cfg->v5.iter_interval = 0;
+		cfg->v5.psd_20 =
+			IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED;
+		cfg->flags = cpu_to_le32(PHY_BAND_6 <<
+					 IWL_CHAN_CFG_FLAGS_BAND_POS);
+		cp->count++;
+	}
+
+	return 0;
+}
+
+static int
+iwl_mld_scan_build_cmd(struct iwl_mld *mld, struct ieee80211_vif *vif,
+		       struct iwl_mld_scan_params *params,
+		       enum iwl_mld_scan_status scan_status,
+		       bool low_latency)
+{
+	struct iwl_scan_req_umac_v17 *cmd = mld->scan.cmd;
+	struct iwl_scan_req_params_v17 *scan_p = &cmd->scan_params;
+	u32 bitmap_ssid = 0;
+	int uid, ret;
+
+	memset(mld->scan.cmd, 0, mld->scan.cmd_size);
+
+	/* find a free UID entry */
+	uid = iwl_mld_scan_uid_by_status(mld, IWL_MLD_SCAN_NONE);
+	if (uid < 0)
+		return uid;
+
+	cmd->uid = cpu_to_le32(uid);
+	cmd->ooc_priority =
+		cpu_to_le32(iwl_mld_scan_ooc_priority(scan_status));
+
+	iwl_mld_scan_cmd_set_gen_params(mld, params, vif,
+					&scan_p->general_params, scan_status);
+
+	ret = iwl_mld_scan_cmd_set_sched_params(params,
+						scan_p->periodic_params.schedule,
+						&scan_p->periodic_params.delay);
+	if (ret)
+		return ret;
+
+	iwl_mld_scan_cmd_set_probe_params(params, &scan_p->probe_params,
+					  &bitmap_ssid);
+
+	ret = iwl_mld_scan_cmd_set_chan_params(mld, params, vif, scan_p,
+					       low_latency, scan_status,
+					       bitmap_ssid);
+	if (ret)
+		return ret;
+
+	return uid;
+}
+
+static bool
+iwl_mld_scan_pass_all(struct iwl_mld *mld,
+		      struct cfg80211_sched_scan_request *req)
+{
+	if (req->n_match_sets && req->match_sets[0].ssid.ssid_len) {
+		IWL_DEBUG_SCAN(mld,
+			       "Sending scheduled scan with filtering, n_match_sets %d\n",
+			       req->n_match_sets);
+		mld->scan.pass_all_sched_res = SCHED_SCAN_PASS_ALL_STATE_DISABLED;
+		return false;
+	}
+
+	IWL_DEBUG_SCAN(mld, "Sending Scheduled scan without filtering\n");
+	mld->scan.pass_all_sched_res = SCHED_SCAN_PASS_ALL_STATE_ENABLED;
+
+	return true;
+}
+
+static int
+iwl_mld_config_sched_scan_profiles(struct iwl_mld *mld,
+				   struct cfg80211_sched_scan_request *req)
+{
+	struct iwl_host_cmd hcmd = {
+		.id = SCAN_OFFLOAD_UPDATE_PROFILES_CMD,
+		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
+	};
+	struct iwl_scan_offload_profile *profile;
+	struct iwl_scan_offload_profile_cfg_data *cfg_data;
+	struct iwl_scan_offload_profile_cfg *profile_cfg;
+	struct iwl_scan_offload_blocklist *blocklist;
+	u32 blocklist_size = IWL_SCAN_MAX_BLACKLIST_LEN * sizeof(*blocklist);
+	u32 cmd_size = blocklist_size + sizeof(*profile_cfg);
+	u8 *cmd;
+	int ret;
+
+	if (WARN_ON(req->n_match_sets > IWL_SCAN_MAX_PROFILES_V2))
+		return -EIO;
+
+	cmd = kzalloc(cmd_size, GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	hcmd.data[0] = cmd;
+	hcmd.len[0] = cmd_size;
+
+	blocklist = (struct iwl_scan_offload_blocklist *)cmd;
+	profile_cfg = (struct iwl_scan_offload_profile_cfg *)(cmd + blocklist_size);
+
+	/* No blocklist configuration */
+	cfg_data = &profile_cfg->data;
+	cfg_data->num_profiles = req->n_match_sets;
+	cfg_data->active_clients = SCAN_CLIENT_SCHED_SCAN;
+	cfg_data->pass_match = SCAN_CLIENT_SCHED_SCAN;
+	cfg_data->match_notify = SCAN_CLIENT_SCHED_SCAN;
+
+	if (!req->n_match_sets || !req->match_sets[0].ssid.ssid_len)
+		cfg_data->any_beacon_notify = SCAN_CLIENT_SCHED_SCAN;
+
+	for (int i = 0; i < req->n_match_sets; i++) {
+		profile = &profile_cfg->profiles[i];
+
+		/* Support any cipher and auth algorithm */
+		profile->unicast_cipher = 0xff;
+		profile->auth_alg = IWL_AUTH_ALGO_UNSUPPORTED |
+			IWL_AUTH_ALGO_NONE | IWL_AUTH_ALGO_PSK |
+			IWL_AUTH_ALGO_8021X | IWL_AUTH_ALGO_SAE |
+			IWL_AUTH_ALGO_8021X_SHA384 | IWL_AUTH_ALGO_OWE;
+		profile->network_type = IWL_NETWORK_TYPE_ANY;
+		profile->band_selection = IWL_SCAN_OFFLOAD_SELECT_ANY;
+		profile->client_bitmap = SCAN_CLIENT_SCHED_SCAN;
+		profile->ssid_index = i;
+	}
+
+	IWL_DEBUG_SCAN(mld,
+		       "Sending scheduled scan profile config (n_match_sets=%u)\n",
+		       req->n_match_sets);
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+
+	kfree(cmd);
+
+	return ret;
+}
+
+static int
+iwl_mld_sched_scan_handle_non_psc_channels(struct iwl_mld_scan_params *params,
+					   bool *non_psc_included)
+{
+	int i, j;
+
+	*non_psc_included = false;
+	/* for 6 GHZ band only PSC channels need to be added */
+	for (i = 0; i < params->n_channels; i++) {
+		struct ieee80211_channel *channel = params->channels[i];
+
+		if (channel->band == NL80211_BAND_6GHZ &&
+		    !cfg80211_channel_is_psc(channel)) {
+			*non_psc_included = true;
+			break;
+		}
+	}
+
+	if (!*non_psc_included)
+		return 0;
+
+	params->channels =
+		kmemdup(params->channels,
+			sizeof(params->channels[0]) * params->n_channels,
+			GFP_KERNEL);
+	if (!params->channels)
+		return -ENOMEM;
+
+	for (i = j = 0; i < params->n_channels; i++) {
+		if (params->channels[i]->band == NL80211_BAND_6GHZ &&
+		    !cfg80211_channel_is_psc(params->channels[i]))
+			continue;
+		params->channels[j++] = params->channels[i];
+	}
+
+	params->n_channels = j;
+
+	return 0;
+}
+
+static void
+iwl_mld_scan_6ghz_passive_scan(struct iwl_mld *mld,
+			       struct iwl_mld_scan_params *params,
+			       struct ieee80211_vif *vif)
+{
+	struct ieee80211_supported_band *sband =
+		&mld->nvm_data->bands[NL80211_BAND_6GHZ];
+	u32 n_disabled, i;
+
+	params->enable_6ghz_passive = false;
+
+	/* 6 GHz passive scan may be enabled in the first 2.4 GHz/5 GHz scan
+	 * phase to discover geo location if no AP's are found. Skip it when
+	 * we're in the 6 GHz scan phase.
+	 */
+	if (params->scan_6ghz)
+		return;
+
+	/* 6 GHz passive scan allowed only on station interface  */
+	if (vif->type != NL80211_IFTYPE_STATION) {
+		IWL_DEBUG_SCAN(mld,
+			       "6GHz passive scan: not station interface\n");
+		return;
+	}
+
+	/* 6 GHz passive scan is allowed in a defined time interval following
+	 * HW reset or resume flow, or while not associated and a large
+	 * interval has passed since the last 6 GHz passive scan.
+	 */
+	if ((vif->cfg.assoc ||
+	     time_after(mld->scan.last_6ghz_passive_jiffies +
+			(IWL_MLD_6GHZ_PASSIVE_SCAN_TIMEOUT * HZ), jiffies)) &&
+	    (time_before(mld->scan.last_start_time_jiffies +
+			 (IWL_MLD_6GHZ_PASSIVE_SCAN_ASSOC_TIMEOUT * HZ),
+			 jiffies))) {
+		IWL_DEBUG_SCAN(mld, "6GHz passive scan: %s\n",
+			       vif->cfg.assoc ? "associated" :
+			       "timeout did not expire");
+		return;
+	}
+
+	/* not enough channels in the regular scan request */
+	if (params->n_channels < IWL_MLD_6GHZ_PASSIVE_SCAN_MIN_CHANS) {
+		IWL_DEBUG_SCAN(mld,
+			       "6GHz passive scan: not enough channels %d\n",
+			       params->n_channels);
+		return;
+	}
+
+	for (i = 0; i < params->n_ssids; i++) {
+		if (!params->ssids[i].ssid_len)
+			break;
+	}
+
+	/* not a wildcard scan, so cannot enable passive 6 GHz scan */
+	if (i == params->n_ssids) {
+		IWL_DEBUG_SCAN(mld,
+			       "6GHz passive scan: no wildcard SSID\n");
+		return;
+	}
+
+	if (!sband || !sband->n_channels) {
+		IWL_DEBUG_SCAN(mld,
+			       "6GHz passive scan: no 6GHz channels\n");
+		return;
+	}
+
+	for (i = 0, n_disabled = 0; i < sband->n_channels; i++) {
+		if (sband->channels[i].flags & (IEEE80211_CHAN_DISABLED))
+			n_disabled++;
+	}
+
+	/* Not all the 6 GHz channels are disabled, so no need for 6 GHz
+	 * passive scan
+	 */
+	if (n_disabled != sband->n_channels) {
+		IWL_DEBUG_SCAN(mld,
+			       "6GHz passive scan: 6GHz channels enabled\n");
+		return;
+	}
+
+	/* all conditions to enable 6 GHz passive scan are satisfied */
+	IWL_DEBUG_SCAN(mld, "6GHz passive scan: can be enabled\n");
+	params->enable_6ghz_passive = true;
+}
+
+static void
+iwl_mld_scan_set_link_id(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			 struct iwl_mld_scan_params *params,
+			 s8 tsf_report_link_id,
+			 enum iwl_mld_scan_status scan_status)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *link;
+
+	if (tsf_report_link_id < 0) {
+		if (vif->active_links)
+			tsf_report_link_id = __ffs(vif->active_links);
+		else
+			tsf_report_link_id = 0;
+	}
+
+	link = iwl_mld_link_dereference_check(mld_vif, tsf_report_link_id);
+	if (!WARN_ON(!link)) {
+		params->fw_link_id = link->fw_id;
+		/* we to store fw_link_id only for regular scan,
+		 * and use it in scan complete notif
+		 */
+		if (scan_status == IWL_MLD_SCAN_REGULAR)
+			mld->scan.fw_link_id = link->fw_id;
+	} else {
+		mld->scan.fw_link_id = IWL_MLD_INVALID_FW_ID;
+		params->fw_link_id = IWL_MLD_INVALID_FW_ID;
+	}
+}
+
+static int
+_iwl_mld_single_scan_start(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			   struct cfg80211_scan_request *req,
+			   struct ieee80211_scan_ies *ies,
+			   enum iwl_mld_scan_status scan_status)
+{
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(LONG_GROUP, SCAN_REQ_UMAC),
+		.len = { mld->scan.cmd_size, },
+		.data = { mld->scan.cmd, },
+		.dataflags = { IWL_HCMD_DFL_NOCOPY, },
+	};
+	struct iwl_mld_scan_iter_data scan_iter_data = {
+		.current_vif = vif,
+	};
+	struct cfg80211_sched_scan_plan scan_plan = {.iterations = 1};
+	struct iwl_mld_scan_params params = {};
+	int ret, uid;
+
+	/* we should have failed registration if scan_cmd was NULL */
+	if (WARN_ON(!mld->scan.cmd))
+		return -ENOMEM;
+
+	if (!iwl_mld_scan_fits(mld, req->n_ssids, ies, req->n_channels))
+		return -ENOBUFS;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_scan_iterator,
+						&scan_iter_data);
+
+	params.type = iwl_mld_get_scan_type(mld, vif, &scan_iter_data);
+	params.n_ssids = req->n_ssids;
+	params.flags = req->flags;
+	params.n_channels = req->n_channels;
+	params.delay = 0;
+	params.ssids = req->ssids;
+	params.channels = req->channels;
+	params.mac_addr = req->mac_addr;
+	params.mac_addr_mask = req->mac_addr_mask;
+	params.no_cck = req->no_cck;
+	params.pass_all = true;
+	params.n_match_sets = 0;
+	params.match_sets = NULL;
+	params.scan_plans = &scan_plan;
+	params.n_scan_plans = 1;
+
+	params.n_6ghz_params = req->n_6ghz_params;
+	params.scan_6ghz_params = req->scan_6ghz_params;
+	params.scan_6ghz = req->scan_6ghz;
+
+	ether_addr_copy(params.bssid, req->bssid);
+	/* TODO: CDB - per-band flag */
+	params.respect_p2p_go =
+		iwl_mld_get_respect_p2p_go(mld, vif,
+					   scan_iter_data.global_low_latency);
+
+	if (req->duration)
+		params.iter_notif = true;
+
+	iwl_mld_scan_set_link_id(mld, vif, &params, req->tsf_report_link_id,
+				 scan_status);
+
+	iwl_mld_scan_build_probe_req(mld, vif, ies, &params);
+
+	iwl_mld_scan_6ghz_passive_scan(mld, &params, vif);
+
+	uid = iwl_mld_scan_build_cmd(mld, vif, &params, scan_status,
+				     scan_iter_data.global_low_latency);
+	if (uid < 0)
+		return uid;
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+	if (ret) {
+		IWL_ERR(mld, "Scan failed! ret %d\n", ret);
+		return ret;
+	}
+
+	IWL_DEBUG_SCAN(mld, "Scan request send success: status=%u, uid=%u\n",
+		       scan_status, uid);
+
+	mld->scan.uid_status[uid] = scan_status;
+	mld->scan.status |= scan_status;
+
+	if (params.enable_6ghz_passive)
+		mld->scan.last_6ghz_passive_jiffies = jiffies;
+
+	return 0;
+}
+
+static int
+iwl_mld_scan_send_abort_cmd_status(struct iwl_mld *mld, int uid, u32 *status)
+{
+	struct iwl_umac_scan_abort abort_cmd = {
+		.uid = cpu_to_le32(uid),
+	};
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(LONG_GROUP, SCAN_ABORT_UMAC),
+		.flags = CMD_WANT_SKB,
+		.data = { &abort_cmd },
+		.len[0] = sizeof(abort_cmd),
+	};
+	struct iwl_rx_packet *pkt;
+	struct iwl_cmd_response *resp;
+	u32 resp_len;
+	int ret;
+
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	if (ret)
+		return ret;
+
+	pkt = cmd.resp_pkt;
+
+	resp_len = iwl_rx_packet_payload_len(pkt);
+	if (IWL_FW_CHECK(mld, resp_len != sizeof(*resp),
+			 "Scan Abort: unexpected response length %d\n",
+			 resp_len)) {
+		ret = -EIO;
+		goto out;
+	}
+
+	resp = (void *)pkt->data;
+	*status = le32_to_cpu(resp->status);
+
+out:
+	iwl_free_resp(&cmd);
+	return ret;
+}
+
+static int
+iwl_mld_scan_abort(struct iwl_mld *mld, int type, int uid, bool *wait)
+{
+	enum iwl_umac_scan_abort_status status;
+	int ret;
+
+	*wait = true;
+
+	IWL_DEBUG_SCAN(mld, "Sending scan abort, uid %u\n", uid);
+
+	ret = iwl_mld_scan_send_abort_cmd_status(mld, uid, &status);
+
+	IWL_DEBUG_SCAN(mld, "Scan abort: ret=%d status=%u\n", ret, status);
+
+	/* We don't need to wait to scan complete in the following cases:
+	 * 1. Driver failed to send the scan abort cmd.
+	 * 2. The FW is no longer familiar with the scan that needs to be
+	 *    stopped. It is expected that the scan complete notification was
+	 *    already received but not yet processed.
+	 *
+	 * In both cases the flow should continue similar to the case that the
+	 * scan was really aborted.
+	 */
+	if (ret || status == IWL_UMAC_SCAN_ABORT_STATUS_NOT_FOUND)
+		*wait = false;
+
+	return ret;
+}
+
+static int
+iwl_mld_scan_stop_wait(struct iwl_mld *mld, int type, int uid)
+{
+	struct iwl_notification_wait wait_scan_done;
+	static const u16 scan_comp_notif[] = { SCAN_COMPLETE_UMAC };
+	bool wait = true;
+	int ret;
+
+	iwl_init_notification_wait(&mld->notif_wait, &wait_scan_done,
+				   scan_comp_notif,
+				   ARRAY_SIZE(scan_comp_notif),
+				   NULL, NULL);
+
+	IWL_DEBUG_SCAN(mld, "Preparing to stop scan, type=%x\n", type);
+
+	ret = iwl_mld_scan_abort(mld, type, uid, &wait);
+	if (ret) {
+		IWL_DEBUG_SCAN(mld, "couldn't stop scan type=%d\n", type);
+		goto return_no_wait;
+	}
+
+	if (!wait) {
+		IWL_DEBUG_SCAN(mld, "no need to wait for scan type=%d\n", type);
+		goto return_no_wait;
+	}
+
+	return iwl_wait_notification(&mld->notif_wait, &wait_scan_done, HZ);
+
+return_no_wait:
+	iwl_remove_notification(&mld->notif_wait, &wait_scan_done);
+	return ret;
+}
+
+int iwl_mld_sched_scan_start(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct cfg80211_sched_scan_request *req,
+			     struct ieee80211_scan_ies *ies,
+			     int type)
+{
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(LONG_GROUP, SCAN_REQ_UMAC),
+		.len = { mld->scan.cmd_size, },
+		.data = { mld->scan.cmd, },
+		.dataflags = { IWL_HCMD_DFL_NOCOPY, },
+	};
+	struct iwl_mld_scan_params params = {};
+	struct iwl_mld_scan_iter_data scan_iter_data = {
+		.current_vif = vif,
+	};
+	bool non_psc_included = false;
+	int ret, uid;
+
+	/* we should have failed registration if scan_cmd was NULL */
+	if (WARN_ON(!mld->scan.cmd))
+		return -ENOMEM;
+
+	/* FW supports only a single periodic scan */
+	if (mld->scan.status & (IWL_MLD_SCAN_SCHED | IWL_MLD_SCAN_NETDETECT))
+		return -EBUSY;
+
+	ieee80211_iterate_active_interfaces_mtx(mld->hw,
+						IEEE80211_IFACE_ITER_NORMAL,
+						iwl_mld_scan_iterator,
+						&scan_iter_data);
+
+	params.type = iwl_mld_get_scan_type(mld, vif, &scan_iter_data);
+	params.flags = req->flags;
+	params.n_ssids = req->n_ssids;
+	params.ssids = req->ssids;
+	params.n_channels = req->n_channels;
+	params.channels = req->channels;
+	params.mac_addr = req->mac_addr;
+	params.mac_addr_mask = req->mac_addr_mask;
+	params.no_cck = false;
+	params.pass_all =  iwl_mld_scan_pass_all(mld, req);
+	params.n_match_sets = req->n_match_sets;
+	params.match_sets = req->match_sets;
+	params.n_scan_plans = req->n_scan_plans;
+	params.scan_plans = req->scan_plans;
+	/* TODO: CDB - per-band flag */
+	params.respect_p2p_go =
+		iwl_mld_get_respect_p2p_go(mld, vif,
+					   scan_iter_data.global_low_latency);
+
+	/* UMAC scan supports up to 16-bit delays, trim it down to 16-bits */
+	params.delay = req->delay > U16_MAX ? U16_MAX : req->delay;
+
+	eth_broadcast_addr(params.bssid);
+
+	ret = iwl_mld_config_sched_scan_profiles(mld, req);
+	if (ret)
+		return ret;
+
+	iwl_mld_scan_build_probe_req(mld, vif, ies, &params);
+
+	ret = iwl_mld_sched_scan_handle_non_psc_channels(&params,
+							 &non_psc_included);
+	if (ret)
+		goto out;
+
+	if (!iwl_mld_scan_fits(mld, req->n_ssids, ies, params.n_channels)) {
+		ret = -ENOBUFS;
+		goto out;
+	}
+
+	uid = iwl_mld_scan_build_cmd(mld, vif, &params, type,
+				     scan_iter_data.global_low_latency);
+	if (uid < 0) {
+		ret = uid;
+		goto out;
+	}
+
+	ret = iwl_mld_send_cmd(mld, &hcmd);
+	if (!ret) {
+		IWL_DEBUG_SCAN(mld,
+			       "Sched scan request send success: type=%u, uid=%u\n",
+			       type, uid);
+		mld->scan.uid_status[uid] = type;
+		mld->scan.status |= type;
+	} else {
+		IWL_ERR(mld, "Sched scan failed! ret %d\n", ret);
+		mld->scan.pass_all_sched_res = SCHED_SCAN_PASS_ALL_STATE_DISABLED;
+	}
+
+out:
+	if (non_psc_included)
+		kfree(params.channels);
+	return ret;
+}
+
+int iwl_mld_scan_stop(struct iwl_mld *mld, int type, bool notify)
+{
+	int uid, ret;
+
+	IWL_DEBUG_SCAN(mld,
+		       "Request to stop scan: type=0x%x, status=0x%x\n",
+		       type, mld->scan.status);
+
+	if (!(mld->scan.status & type))
+		return 0;
+
+	uid = iwl_mld_scan_uid_by_status(mld, type);
+	/* must be valid, we just checked it's running */
+	if (WARN_ON_ONCE(uid < 0))
+		return uid;
+
+	ret = iwl_mld_scan_stop_wait(mld, type, uid);
+	if (ret)
+		IWL_DEBUG_SCAN(mld, "Failed to stop scan\n");
+
+	/* Clear the scan status so the next scan requests will
+	 * succeed and mark the scan as stopping, so that the Rx
+	 * handler doesn't do anything, as the scan was stopped from
+	 * above. Also remove the handler to not notify mac80211
+	 * erroneously after a new scan starts, for example.
+	 */
+	mld->scan.status &= ~type;
+	mld->scan.uid_status[uid] = IWL_MLD_SCAN_NONE;
+	iwl_mld_cancel_notifications_of_object(mld, IWL_MLD_OBJECT_TYPE_SCAN,
+					       uid);
+
+	if (type == IWL_MLD_SCAN_REGULAR) {
+		if (notify) {
+			struct cfg80211_scan_info info = {
+				.aborted = true,
+			};
+
+			ieee80211_scan_completed(mld->hw, &info);
+		}
+	} else if (notify) {
+		ieee80211_sched_scan_stopped(mld->hw);
+		mld->scan.pass_all_sched_res = SCHED_SCAN_PASS_ALL_STATE_DISABLED;
+	}
+
+	return ret;
+}
+
+int iwl_mld_regular_scan_start(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			       struct cfg80211_scan_request *req,
+			       struct ieee80211_scan_ies *ies)
+{
+	/* Clear survey data when starting the first part of a regular scan */
+	if (req->first_part && mld->channel_survey)
+		memset(mld->channel_survey->channels, 0,
+		       sizeof(mld->channel_survey->channels[0]) *
+		       mld->channel_survey->n_channels);
+
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
+		iwl_mld_emlsr_block_tmp_non_bss(mld);
+
+	return _iwl_mld_single_scan_start(mld, vif, req, ies,
+					  IWL_MLD_SCAN_REGULAR);
+}
+
+static void iwl_mld_int_mlo_scan_start(struct iwl_mld *mld,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_channel **channels,
+				       size_t n_channels)
+{
+	struct cfg80211_scan_request *req __free(kfree) = NULL;
+	struct ieee80211_scan_ies ies = {};
+	size_t size;
+	int ret;
+
+	IWL_DEBUG_SCAN(mld, "Starting Internal MLO scan: n_channels=%zu\n",
+		       n_channels);
+
+	size = struct_size(req, channels, n_channels);
+	req = kzalloc(size, GFP_KERNEL);
+	if (!req)
+		return;
+
+	/* set the requested channels */
+	for (int i = 0; i < n_channels; i++)
+		req->channels[i] = channels[i];
+
+	req->n_channels = n_channels;
+
+	/* set the rates */
+	for (int i = 0; i < NUM_NL80211_BANDS; i++)
+		if (mld->wiphy->bands[i])
+			req->rates[i] =
+				(1 << mld->wiphy->bands[i]->n_bitrates) - 1;
+
+	req->wdev = ieee80211_vif_to_wdev(vif);
+	req->wiphy = mld->wiphy;
+	req->scan_start = jiffies;
+	req->tsf_report_link_id = -1;
+
+	ret = _iwl_mld_single_scan_start(mld, vif, req, &ies,
+					 IWL_MLD_SCAN_INT_MLO);
+
+	if (!ret)
+		mld->scan.last_mlo_scan_time = ktime_get_boottime_ns();
+
+	IWL_DEBUG_SCAN(mld, "Internal MLO scan: ret=%d\n", ret);
+}
+
+#define IWL_MLD_MLO_SCAN_BLOCKOUT_TIME		5 /* seconds */
+
+void iwl_mld_int_mlo_scan(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct ieee80211_channel *channels[IEEE80211_MLD_MAX_NUM_LINKS];
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	unsigned long usable_links = ieee80211_vif_usable_links(vif);
+	size_t n_channels = 0;
+	u8 link_id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (!IWL_MLD_AUTO_EML_ENABLE || !vif->cfg.assoc ||
+	    !ieee80211_vif_is_mld(vif) || hweight16(vif->valid_links) == 1)
+		return;
+
+	if (mld->scan.status & IWL_MLD_SCAN_INT_MLO) {
+		IWL_DEBUG_SCAN(mld, "Internal MLO scan is already running\n");
+		return;
+	}
+
+	if (mld_vif->last_link_activation_time > ktime_get_boottime_seconds() -
+						 IWL_MLD_MLO_SCAN_BLOCKOUT_TIME) {
+		/* timing doesn't matter much, so use the blockout time */
+		wiphy_delayed_work_queue(mld->wiphy,
+					 &mld_vif->mlo_scan_start_wk,
+					 IWL_MLD_MLO_SCAN_BLOCKOUT_TIME);
+		return;
+	}
+
+	for_each_set_bit(link_id, &usable_links, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_check(vif, link_id);
+
+		if (WARN_ON_ONCE(!link_conf))
+			continue;
+
+		channels[n_channels++] = link_conf->chanreq.oper.chan;
+	}
+
+	if (!n_channels)
+		return;
+
+	iwl_mld_int_mlo_scan_start(mld, vif, channels, n_channels);
+}
+
+void iwl_mld_handle_scan_iter_complete_notif(struct iwl_mld *mld,
+					     struct iwl_rx_packet *pkt)
+{
+	struct iwl_umac_scan_iter_complete_notif *notif = (void *)pkt->data;
+	u32 uid = __le32_to_cpu(notif->uid);
+
+	if (IWL_FW_CHECK(mld, uid >= ARRAY_SIZE(mld->scan.uid_status),
+			 "FW reports out-of-range scan UID %d\n", uid))
+		return;
+
+	if (mld->scan.uid_status[uid] == IWL_MLD_SCAN_REGULAR)
+		mld->scan.start_tsf = le64_to_cpu(notif->start_tsf);
+
+	IWL_DEBUG_SCAN(mld,
+		       "UMAC Scan iteration complete: status=0x%x scanned_channels=%d\n",
+		       notif->status, notif->scanned_channels);
+
+	if (mld->scan.pass_all_sched_res == SCHED_SCAN_PASS_ALL_STATE_FOUND) {
+		IWL_DEBUG_SCAN(mld, "Pass all scheduled scan results found\n");
+		ieee80211_sched_scan_results(mld->hw);
+		mld->scan.pass_all_sched_res = SCHED_SCAN_PASS_ALL_STATE_ENABLED;
+	}
+
+	IWL_DEBUG_SCAN(mld,
+		       "UMAC Scan iteration complete: scan started at %llu (TSF)\n",
+		       le64_to_cpu(notif->start_tsf));
+}
+
+void iwl_mld_handle_match_found_notif(struct iwl_mld *mld,
+				      struct iwl_rx_packet *pkt)
+{
+	IWL_DEBUG_SCAN(mld, "Scheduled scan results\n");
+	ieee80211_sched_scan_results(mld->hw);
+}
+
+void iwl_mld_handle_scan_complete_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt)
+{
+	struct iwl_umac_scan_complete *notif = (void *)pkt->data;
+	bool aborted = (notif->status == IWL_SCAN_OFFLOAD_ABORTED);
+	u32 uid = __le32_to_cpu(notif->uid);
+
+	if (IWL_FW_CHECK(mld, uid >= ARRAY_SIZE(mld->scan.uid_status),
+			 "FW reports out-of-range scan UID %d\n", uid))
+		return;
+
+	IWL_DEBUG_SCAN(mld,
+		       "Scan completed: uid=%u type=%u, status=%s, EBS=%s\n",
+		       uid, mld->scan.uid_status[uid],
+		       notif->status == IWL_SCAN_OFFLOAD_COMPLETED ?
+				"completed" : "aborted",
+		       iwl_mld_scan_ebs_status_str(notif->ebs_status));
+	IWL_DEBUG_SCAN(mld, "Scan completed: scan_status=0x%x\n",
+		       mld->scan.status);
+	IWL_DEBUG_SCAN(mld,
+		       "Scan completed: line=%u, iter=%u, elapsed time=%u\n",
+		       notif->last_schedule, notif->last_iter,
+		       __le32_to_cpu(notif->time_from_last_iter));
+
+	if (IWL_FW_CHECK(mld, !(mld->scan.uid_status[uid] & mld->scan.status),
+			 "FW reports scan UID %d we didn't trigger\n", uid))
+		return;
+
+	/* if the scan is already stopping, we don't need to notify mac80211 */
+	if (mld->scan.uid_status[uid] == IWL_MLD_SCAN_REGULAR) {
+		struct cfg80211_scan_info info = {
+			.aborted = aborted,
+			.scan_start_tsf = mld->scan.start_tsf,
+		};
+		int fw_link_id = mld->scan.fw_link_id;
+		struct ieee80211_bss_conf *link_conf = NULL;
+
+		if (fw_link_id != IWL_MLD_INVALID_FW_ID)
+			link_conf =
+				wiphy_dereference(mld->wiphy,
+						  mld->fw_id_to_bss_conf[fw_link_id]);
+
+		/* It is possible that by the time the scan is complete the
+		 * link was already removed and is not valid.
+		 */
+		if (link_conf)
+			ether_addr_copy(info.tsf_bssid, link_conf->bssid);
+		else
+			IWL_DEBUG_SCAN(mld, "Scan link is no longer valid\n");
+
+		ieee80211_scan_completed(mld->hw, &info);
+
+		/* Scan is over, we can check again the tpt counters */
+		iwl_mld_stop_ignoring_tpt_updates(mld);
+	} else if (mld->scan.uid_status[uid] == IWL_MLD_SCAN_SCHED) {
+		ieee80211_sched_scan_stopped(mld->hw);
+		mld->scan.pass_all_sched_res = SCHED_SCAN_PASS_ALL_STATE_DISABLED;
+	} else if (mld->scan.uid_status[uid] == IWL_MLD_SCAN_INT_MLO) {
+		IWL_DEBUG_SCAN(mld, "Internal MLO scan completed\n");
+
+		/*
+		 * We limit link selection to internal MLO scans as otherwise
+		 * we do not know whether all channels were covered.
+		 */
+		iwl_mld_select_links(mld);
+	}
+
+	mld->scan.status &= ~mld->scan.uid_status[uid];
+
+	IWL_DEBUG_SCAN(mld, "Scan completed: after update: scan_status=0x%x\n",
+		       mld->scan.status);
+
+	mld->scan.uid_status[uid] = IWL_MLD_SCAN_NONE;
+
+	if (notif->ebs_status != IWL_SCAN_EBS_SUCCESS &&
+	    notif->ebs_status != IWL_SCAN_EBS_INACTIVE)
+		mld->scan.last_ebs_failed = true;
+}
+
+/* This function is used in nic restart flow, to inform mac80211 about scans
+ * that were aborted by restart flow or by an assert.
+ */
+void iwl_mld_report_scan_aborted(struct iwl_mld *mld)
+{
+	int uid;
+
+	uid = iwl_mld_scan_uid_by_status(mld, IWL_MLD_SCAN_REGULAR);
+	if (uid >= 0) {
+		struct cfg80211_scan_info info = {
+			.aborted = true,
+		};
+
+		ieee80211_scan_completed(mld->hw, &info);
+		mld->scan.uid_status[uid] = IWL_MLD_SCAN_NONE;
+	}
+
+	uid = iwl_mld_scan_uid_by_status(mld, IWL_MLD_SCAN_SCHED);
+	if (uid >= 0) {
+		mld->scan.pass_all_sched_res = SCHED_SCAN_PASS_ALL_STATE_DISABLED;
+		mld->scan.uid_status[uid] = IWL_MLD_SCAN_NONE;
+
+		/* sched scan will be restarted by mac80211 in reconfig.
+		 * report to mac80211 that sched scan stopped only if we won't
+		 * restart the firmware.
+		 */
+		if (!iwlwifi_mod_params.fw_restart)
+			ieee80211_sched_scan_stopped(mld->hw);
+	}
+
+	uid = iwl_mld_scan_uid_by_status(mld, IWL_MLD_SCAN_INT_MLO);
+	if (uid >= 0) {
+		IWL_DEBUG_SCAN(mld, "Internal MLO scan aborted\n");
+		mld->scan.uid_status[uid] = IWL_MLD_SCAN_NONE;
+	}
+
+	BUILD_BUG_ON(IWL_MLD_SCAN_NONE != 0);
+	memset(mld->scan.uid_status, 0, sizeof(mld->scan.uid_status));
+}
+
+int iwl_mld_alloc_scan_cmd(struct iwl_mld *mld)
+{
+	u8 scan_cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, SCAN_REQ_UMAC,
+						IWL_FW_CMD_VER_UNKNOWN);
+	size_t scan_cmd_size;
+
+	if (scan_cmd_ver == 17) {
+		scan_cmd_size = sizeof(struct iwl_scan_req_umac_v17);
+	} else {
+		IWL_ERR(mld, "Unexpected scan cmd version %d\n", scan_cmd_ver);
+		return -EINVAL;
+	}
+
+	mld->scan.cmd = kmalloc(scan_cmd_size, GFP_KERNEL);
+	if (!mld->scan.cmd)
+		return -ENOMEM;
+
+	mld->scan.cmd_size = scan_cmd_size;
+
+	return 0;
+}
+
+static int iwl_mld_chanidx_from_phy(struct iwl_mld *mld,
+				    enum nl80211_band band,
+				    u16 phy_chan_num)
+{
+	struct ieee80211_supported_band *sband = mld->wiphy->bands[band];
+
+	if (WARN_ON_ONCE(!sband))
+		return -EINVAL;
+
+	for (int chan_idx = 0; chan_idx < sband->n_channels; chan_idx++) {
+		struct ieee80211_channel *channel = &sband->channels[chan_idx];
+
+		if (channel->hw_value == phy_chan_num)
+			return chan_idx;
+	}
+
+	return -EINVAL;
+}
+
+void iwl_mld_handle_channel_survey_notif(struct iwl_mld *mld,
+					 struct iwl_rx_packet *pkt)
+{
+	const struct iwl_umac_scan_channel_survey_notif *notif =
+		(void *)pkt->data;
+	struct iwl_mld_survey_channel *info;
+	enum nl80211_band band;
+	int chan_idx;
+
+	if (!mld->channel_survey) {
+		size_t n_channels = 0;
+
+		for (band = 0; band < NUM_NL80211_BANDS; band++) {
+			if (!mld->wiphy->bands[band])
+				continue;
+
+			n_channels += mld->wiphy->bands[band]->n_channels;
+		}
+
+		mld->channel_survey = kzalloc(struct_size(mld->channel_survey,
+							  channels, n_channels),
+							  GFP_KERNEL);
+
+		if (!mld->channel_survey)
+			return;
+
+		mld->channel_survey->n_channels = n_channels;
+		n_channels = 0;
+		for (band = 0; band < NUM_NL80211_BANDS; band++) {
+			if (!mld->wiphy->bands[band])
+				continue;
+
+			mld->channel_survey->bands[band] =
+				&mld->channel_survey->channels[n_channels];
+			n_channels += mld->wiphy->bands[band]->n_channels;
+		}
+	}
+
+	band = iwl_mld_phy_band_to_nl80211(le32_to_cpu(notif->band));
+	chan_idx = iwl_mld_chanidx_from_phy(mld, band,
+					    le32_to_cpu(notif->channel));
+	if (WARN_ON_ONCE(chan_idx < 0))
+		return;
+
+	IWL_DEBUG_SCAN(mld, "channel survey received for freq %d\n",
+		       mld->wiphy->bands[band]->channels[chan_idx].center_freq);
+
+	info = &mld->channel_survey->bands[band][chan_idx];
+
+	/* Times are all in ms */
+	info->time = le32_to_cpu(notif->active_time);
+	info->time_busy = le32_to_cpu(notif->busy_time);
+	info->noise =
+		iwl_average_neg_dbm(notif->noise, ARRAY_SIZE(notif->noise));
+}
+
+int iwl_mld_mac80211_get_survey(struct ieee80211_hw *hw, int idx,
+				struct survey_info *survey)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	int curr_idx = 0;
+
+	if (!mld->channel_survey)
+		return -ENOENT;
+
+	/* Iterate bands/channels to find the requested index.
+	 * Logically this returns the entry with index "idx" from a flattened
+	 * survey result array that only contains channels with information.
+	 * The current index into this flattened array is tracked in curr_idx.
+	 */
+	for (enum nl80211_band band = 0; band < NUM_NL80211_BANDS; band++) {
+		struct ieee80211_supported_band *sband =
+			mld->wiphy->bands[band];
+
+		if (!sband)
+			continue;
+
+		for (int per_band_idx = 0;
+		     per_band_idx < sband->n_channels;
+		     per_band_idx++) {
+			struct iwl_mld_survey_channel *info =
+				&mld->channel_survey->bands[band][per_band_idx];
+
+			/* Skip entry entirely, it was not reported/scanned,
+			 * do not increase curr_idx for this entry.
+			 */
+			if (!info->time)
+				continue;
+
+			/* Search did not reach the requested entry yet,
+			 * increment curr_idx and continue.
+			 */
+			if (idx != curr_idx) {
+				curr_idx++;
+				continue;
+			}
+
+			/* Found (the next) channel to report */
+			survey->channel = &sband->channels[per_band_idx];
+			survey->filled = SURVEY_INFO_TIME |
+					 SURVEY_INFO_TIME_BUSY;
+			survey->time = info->time;
+			survey->time_busy = info->time_busy;
+			survey->noise = info->noise;
+			if (survey->noise < 0)
+				survey->filled |= SURVEY_INFO_NOISE_DBM;
+
+			return 0;
+		}
+	}
+
+	return -ENOENT;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/scan.h b/sys/contrib/dev/iwlwifi/mld/scan.h
new file mode 100644
index 000000000000..69110f0cfc8e
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/scan.h
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifndef __iwl_mld_scan_h__
+#define __iwl_mld_scan_h__
+
+int iwl_mld_alloc_scan_cmd(struct iwl_mld *mld);
+
+int iwl_mld_regular_scan_start(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			       struct cfg80211_scan_request *req,
+			       struct ieee80211_scan_ies *ies);
+
+void iwl_mld_int_mlo_scan(struct iwl_mld *mld, struct ieee80211_vif *vif);
+
+void iwl_mld_handle_scan_iter_complete_notif(struct iwl_mld *mld,
+					     struct iwl_rx_packet *pkt);
+
+int iwl_mld_scan_stop(struct iwl_mld *mld, int type, bool notify);
+
+int iwl_mld_sched_scan_start(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct cfg80211_sched_scan_request *req,
+			     struct ieee80211_scan_ies *ies,
+			     int type);
+
+void iwl_mld_handle_match_found_notif(struct iwl_mld *mld,
+				      struct iwl_rx_packet *pkt);
+
+void iwl_mld_handle_scan_complete_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt);
+
+int iwl_mld_mac80211_get_survey(struct ieee80211_hw *hw, int idx,
+				struct survey_info *survey);
+
+void iwl_mld_handle_channel_survey_notif(struct iwl_mld *mld,
+					 struct iwl_rx_packet *pkt);
+
+#define WFA_TPC_IE_LEN 9
+
+static inline int iwl_mld_scan_max_template_size(void)
+{
+#define MAC_HDR_LEN 24
+#define DS_IE_LEN 3
+#define SSID_IE_LEN 2
+
+/* driver create the 802.11 header, WFA TPC IE, DS parameter and SSID IE */
+#define DRIVER_TOTAL_IES_LEN \
+	(MAC_HDR_LEN + WFA_TPC_IE_LEN + DS_IE_LEN + SSID_IE_LEN)
+
+	BUILD_BUG_ON(SCAN_OFFLOAD_PROBE_REQ_SIZE < DRIVER_TOTAL_IES_LEN);
+
+	return SCAN_OFFLOAD_PROBE_REQ_SIZE - DRIVER_TOTAL_IES_LEN;
+}
+
+void iwl_mld_report_scan_aborted(struct iwl_mld *mld);
+
+enum iwl_mld_scan_status {
+	IWL_MLD_SCAN_NONE		= 0,
+	IWL_MLD_SCAN_REGULAR		= BIT(0),
+	IWL_MLD_SCAN_SCHED		= BIT(1),
+	IWL_MLD_SCAN_NETDETECT		= BIT(2),
+	IWL_MLD_SCAN_INT_MLO		= BIT(3),
+};
+
+/* enum iwl_mld_pass_all_sched_results_states - Defines the states for
+ * handling/passing scheduled scan results to mac80211
+ * @SCHED_SCAN_PASS_ALL_STATE_DISABLED: Don't pass all scan results, only when
+ *	a match found.
+ * @SCHED_SCAN_PASS_ALL_STATE_ENABLED: Pass all scan results is enabled
+ *	(no filtering).
+ * @SCHED_SCAN_PASS_ALL_STATE_FOUND: A scan result is found, pass it on the
+ *	next scan iteration complete notification.
+ */
+enum iwl_mld_pass_all_sched_results_states {
+	SCHED_SCAN_PASS_ALL_STATE_DISABLED,
+	SCHED_SCAN_PASS_ALL_STATE_ENABLED,
+	SCHED_SCAN_PASS_ALL_STATE_FOUND,
+};
+
+/**
+ * enum iwl_mld_traffic_load - Levels of traffic load
+ *
+ * @IWL_MLD_TRAFFIC_LOW: low traffic load
+ * @IWL_MLD_TRAFFIC_MEDIUM: medium traffic load
+ * @IWL_MLD_TRAFFIC_HIGH: high traffic load
+ */
+enum iwl_mld_traffic_load {
+	IWL_MLD_TRAFFIC_LOW,
+	IWL_MLD_TRAFFIC_MEDIUM,
+	IWL_MLD_TRAFFIC_HIGH,
+};
+
+/**
+ * struct iwl_mld_scan - Scan data
+ * @status: scan status, a combination of %enum iwl_mld_scan_status,
+ *	reflects the %scan.uid_status array.
+ * @uid_status: array to track the scan status per uid.
+ * @start_tsf: start time of last scan in TSF of the link that requested
+ *	the scan.
+ * @last_ebs_failed: true if the last EBS (Energy Based Scan) failed.
+ * @pass_all_sched_res: see %enum iwl_mld_pass_all_sched_results_states.
+ * @fw_link_id: the current (regular) scan fw link id, used by scan
+ *	complete notif.
+ * @traffic_load: traffic load related data
+ * @traffic_load.last_stats_ts_usec: The timestamp of the last statistics
+ *	notification, used to calculate the elapsed time between two
+ *	notifications and determine the traffic load
+ * @traffic_load.status: The current traffic load status, see
+ *	&enum iwl_mld_traffic_load
+ * @cmd_size: size of %cmd.
+ * @cmd: pointer to scan cmd buffer (allocated once in op mode start).
+ * @last_6ghz_passive_jiffies: stores the last 6GHz passive scan time
+ *	in jiffies.
+ * @last_start_time_jiffies: stores the last start time in jiffies
+ *	(interface up/reset/resume).
+ * @last_mlo_scan_time: start time of the last MLO scan in nanoseconds since
+ *	boot.
+ */
+struct iwl_mld_scan {
+	/* Add here fields that need clean up on restart */
+	struct_group(zeroed_on_hw_restart,
+		unsigned int status;
+		u32 uid_status[IWL_MAX_UMAC_SCANS];
+		u64 start_tsf;
+		bool last_ebs_failed;
+		enum iwl_mld_pass_all_sched_results_states pass_all_sched_res;
+		u8 fw_link_id;
+		struct {
+			u32 last_stats_ts_usec;
+			enum iwl_mld_traffic_load status;
+		} traffic_load;
+	);
+	/* And here fields that survive a fw restart */
+	size_t cmd_size;
+	void *cmd;
+	unsigned long last_6ghz_passive_jiffies;
+	unsigned long last_start_time_jiffies;
+	u64 last_mlo_scan_time;
+};
+
+/**
+ * struct iwl_mld_survey_channel - per-channel survey information
+ *
+ * Driver version of &struct survey_info with just the data we want to report.
+ *
+ * @time: time in ms the radio was on the channel
+ * @time_busy: time in ms the channel was sensed busy
+ * @noise: channel noise in dBm
+ */
+struct iwl_mld_survey_channel {
+	u32 time;
+	u32 time_busy;
+	s8 noise;
+};
+
+/**
+ * struct iwl_mld_survey - survey information
+ *
+ * Survey information for all available channels.
+ *
+ * @bands: per-band array for per-channel survey data, points into @channels
+ * @n_channels: Number of @channels entries that are allocated
+ * @channels: per-channel information
+ */
+struct iwl_mld_survey {
+	struct iwl_mld_survey_channel *bands[NUM_NL80211_BANDS];
+
+	int n_channels;
+	struct iwl_mld_survey_channel channels[] __counted_by(n_channels);
+};
+
+#endif /* __iwl_mld_scan_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/session-protect.c b/sys/contrib/dev/iwlwifi/mld/session-protect.c
new file mode 100644
index 000000000000..dbb5615dc3f6
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/session-protect.c
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#include "session-protect.h"
+#include "fw/api/time-event.h"
+#include "fw/api/context.h"
+#include "iface.h"
+#include <net/mac80211.h>
+
+void iwl_mld_handle_session_prot_notif(struct iwl_mld *mld,
+				       struct iwl_rx_packet *pkt)
+{
+	struct iwl_session_prot_notif *notif = (void *)pkt->data;
+	int fw_link_id = le32_to_cpu(notif->mac_link_id);
+	struct ieee80211_bss_conf *link_conf =
+		iwl_mld_fw_id_to_link_conf(mld, fw_link_id);
+	struct ieee80211_vif *vif;
+	struct iwl_mld_vif *mld_vif;
+	struct iwl_mld_session_protect *session_protect;
+
+	if (WARN_ON(!link_conf))
+		return;
+
+	vif = link_conf->vif;
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+	session_protect = &mld_vif->session_protect;
+
+	if (!le32_to_cpu(notif->status)) {
+		memset(session_protect, 0, sizeof(*session_protect));
+	} else if (le32_to_cpu(notif->start)) {
+		/* End_jiffies indicates an active session */
+		session_protect->session_requested = false;
+		session_protect->end_jiffies =
+			TU_TO_EXP_TIME(session_protect->duration);
+		/* !session_protect->end_jiffies means inactive session */
+		if (!session_protect->end_jiffies)
+			session_protect->end_jiffies = 1;
+	} else {
+		memset(session_protect, 0, sizeof(*session_protect));
+	}
+}
+
+static int _iwl_mld_schedule_session_protection(struct iwl_mld *mld,
+						struct ieee80211_vif *vif,
+						u32 duration, u32 min_duration,
+						int link_id)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *link =
+		iwl_mld_link_dereference_check(mld_vif, link_id);
+	struct iwl_mld_session_protect *session_protect =
+		&mld_vif->session_protect;
+	struct iwl_session_prot_cmd cmd = {
+		.id_and_color = cpu_to_le32(link->fw_id),
+		.action = cpu_to_le32(FW_CTXT_ACTION_ADD),
+		.conf_id = cpu_to_le32(SESSION_PROTECT_CONF_ASSOC),
+		.duration_tu = cpu_to_le32(MSEC_TO_TU(duration)),
+	};
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	WARN(hweight16(vif->active_links) > 1,
+	     "Session protection isn't allowed with more than one active link");
+
+	if (session_protect->end_jiffies &&
+	    time_after(session_protect->end_jiffies,
+		       TU_TO_EXP_TIME(min_duration))) {
+		IWL_DEBUG_TE(mld, "We have ample in the current session: %u\n",
+			     jiffies_to_msecs(session_protect->end_jiffies -
+					      jiffies));
+		return -EALREADY;
+	}
+
+	IWL_DEBUG_TE(mld, "Add a new session protection, duration %d TU\n",
+		     le32_to_cpu(cmd.duration_tu));
+
+	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(MAC_CONF_GROUP,
+						SESSION_PROTECTION_CMD), &cmd);
+
+	if (ret)
+		return ret;
+
+	/* end_jiffies will be updated when handling session_prot_notif */
+	session_protect->end_jiffies = 0;
+	session_protect->duration = duration;
+	session_protect->session_requested = true;
+
+	return 0;
+}
+
+void iwl_mld_schedule_session_protection(struct iwl_mld *mld,
+					 struct ieee80211_vif *vif,
+					 u32 duration, u32 min_duration,
+					 int link_id)
+{
+	int ret;
+
+	ret = _iwl_mld_schedule_session_protection(mld, vif, duration,
+						   min_duration, link_id);
+	if (ret && ret != -EALREADY)
+		IWL_ERR(mld,
+			"Couldn't send the SESSION_PROTECTION_CMD (%d)\n",
+			ret);
+}
+
+struct iwl_mld_session_start_data {
+	struct iwl_mld *mld;
+	struct ieee80211_bss_conf *link_conf;
+	bool success;
+};
+
+static bool iwl_mld_session_start_fn(struct iwl_notif_wait_data *notif_wait,
+				     struct iwl_rx_packet *pkt, void *_data)
+{
+	struct iwl_session_prot_notif *notif = (void *)pkt->data;
+	unsigned int pkt_len = iwl_rx_packet_payload_len(pkt);
+	struct iwl_mld_session_start_data *data = _data;
+	struct ieee80211_bss_conf *link_conf;
+	struct iwl_mld *mld = data->mld;
+	int fw_link_id;
+
+	if (IWL_FW_CHECK(mld, pkt_len < sizeof(*notif),
+			 "short session prot notif (%d)\n",
+			 pkt_len))
+		return false;
+
+	fw_link_id = le32_to_cpu(notif->mac_link_id);
+	link_conf = iwl_mld_fw_id_to_link_conf(mld, fw_link_id);
+
+	if (link_conf != data->link_conf)
+		return false;
+
+	if (!le32_to_cpu(notif->status))
+		return true;
+
+	if (notif->start) {
+		data->success = true;
+		return true;
+	}
+
+	return false;
+}
+
+int iwl_mld_start_session_protection(struct iwl_mld *mld,
+				     struct ieee80211_vif *vif,
+				     u32 duration, u32 min_duration,
+				     int link_id, unsigned long timeout)
+{
+	static const u16 start_notif[] = { SESSION_PROTECTION_NOTIF };
+	struct iwl_notification_wait start_wait;
+	struct iwl_mld_session_start_data data = {
+		.mld = mld,
+		.link_conf = wiphy_dereference(mld->wiphy,
+					       vif->link_conf[link_id]),
+	};
+	int ret;
+
+	if (WARN_ON(!data.link_conf))
+		return -EINVAL;
+
+	iwl_init_notification_wait(&mld->notif_wait, &start_wait,
+				   start_notif, ARRAY_SIZE(start_notif),
+				   iwl_mld_session_start_fn, &data);
+
+	ret = _iwl_mld_schedule_session_protection(mld, vif, duration,
+						   min_duration, link_id);
+
+	if (ret) {
+		iwl_remove_notification(&mld->notif_wait, &start_wait);
+		return ret == -EALREADY ? 0 : ret;
+	}
+
+	ret = iwl_wait_notification(&mld->notif_wait, &start_wait, timeout);
+	if (ret)
+		return ret;
+	return data.success ? 0 : -EIO;
+}
+
+int iwl_mld_cancel_session_protection(struct iwl_mld *mld,
+				      struct ieee80211_vif *vif,
+				      int link_id)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_link *link =
+		iwl_mld_link_dereference_check(mld_vif, link_id);
+	struct iwl_mld_session_protect *session_protect =
+		&mld_vif->session_protect;
+	struct iwl_session_prot_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
+		.conf_id = cpu_to_le32(SESSION_PROTECT_CONF_ASSOC),
+	};
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* If there isn't an active session or a requested one for this
+	 * link do nothing
+	 */
+	if (!session_protect->session_requested &&
+	    !session_protect->end_jiffies)
+		return 0;
+
+	if (WARN_ON(!link))
+		return -EINVAL;
+
+	cmd.id_and_color = cpu_to_le32(link->fw_id);
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(MAC_CONF_GROUP,
+					   SESSION_PROTECTION_CMD), &cmd);
+	if (ret) {
+		IWL_ERR(mld,
+			"Couldn't send the SESSION_PROTECTION_CMD\n");
+		return ret;
+	}
+
+	memset(session_protect, 0, sizeof(*session_protect));
+
+	return 0;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/session-protect.h b/sys/contrib/dev/iwlwifi/mld/session-protect.h
new file mode 100644
index 000000000000..642bec8451a1
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/session-protect.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#ifndef __session_protect_h__
+#define __session_protect_h__
+
+#include "mld.h"
+#include "hcmd.h"
+#include <net/mac80211.h>
+#include "fw/api/mac-cfg.h"
+
+/**
+ * DOC: session protection
+ *
+ * Session protection is an API from the firmware that allows the driver to
+ * request time on medium. This is needed before the association when we need
+ * to be on medium for the association frame exchange. Once we configure the
+ * firmware as 'associated', the firmware will allocate time on medium without
+ * needed a session protection.
+ *
+ * TDLS discover uses this API as well even after association to ensure that
+ * other activities internal to the firmware will not interrupt our presence
+ * on medium.
+ */
+
+/**
+ * struct iwl_mld_session_protect - session protection parameters
+ * @end_jiffies: expected end_jiffies of current session protection.
+ *	0 if not active
+ * @duration: the duration in tu of current session
+ * @session_requested: A session protection command was sent and wasn't yet
+ *	answered
+ */
+struct iwl_mld_session_protect {
+	unsigned long end_jiffies;
+	u32 duration;
+	bool session_requested;
+};
+
+#define IWL_MLD_SESSION_PROTECTION_ASSOC_TIME_MS 900
+#define IWL_MLD_SESSION_PROTECTION_MIN_TIME_MS 400
+
+/**
+ * iwl_mld_handle_session_prot_notif - handles %SESSION_PROTECTION_NOTIF
+ * @mld: the mld component
+ * @pkt: the RX packet containing the notification
+ */
+void iwl_mld_handle_session_prot_notif(struct iwl_mld *mld,
+				       struct iwl_rx_packet *pkt);
+
+/**
+ * iwl_mld_schedule_session_protection - schedule a session protection
+ * @mld: the mld component
+ * @vif: the virtual interface for which the protection issued
+ * @duration: the requested duration of the protection
+ * @min_duration: the minimum duration of the protection
+ * @link_id: The link to schedule a session protection for
+ */
+void iwl_mld_schedule_session_protection(struct iwl_mld *mld,
+					 struct ieee80211_vif *vif,
+					 u32 duration, u32 min_duration,
+					 int link_id);
+
+/**
+ * iwl_mld_start_session_protection - start a session protection
+ * @mld: the mld component
+ * @vif: the virtual interface for which the protection issued
+ * @duration: the requested duration of the protection
+ * @min_duration: the minimum duration of the protection
+ * @link_id: The link to schedule a session protection for
+ * @timeout: timeout for waiting
+ *
+ * This schedules the session protection, and waits for it to start
+ * (with timeout)
+ *
+ * Returns: 0 if successful, error code otherwise
+ */
+int iwl_mld_start_session_protection(struct iwl_mld *mld,
+				     struct ieee80211_vif *vif,
+				     u32 duration, u32 min_duration,
+				     int link_id, unsigned long timeout);
+
+/**
+ * iwl_mld_cancel_session_protection - cancel the session protection.
+ * @mld: the mld component
+ * @vif: the virtual interface for which the session is issued
+ * @link_id: cancel the session protection for given link
+ *
+ * This functions cancels the session protection which is an act of good
+ * citizenship. If it is not needed any more it should be canceled because
+ * the other mac contexts wait for the medium during that time.
+ *
+ * Returns: 0 if successful, error code otherwise
+ *
+ */
+int iwl_mld_cancel_session_protection(struct iwl_mld *mld,
+				      struct ieee80211_vif *vif,
+				      int link_id);
+
+#endif /* __session_protect_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/sta.c b/sys/contrib/dev/iwlwifi/mld/sta.c
new file mode 100644
index 000000000000..8fb51209b4a6
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/sta.c
@@ -0,0 +1,1321 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include <linux/ieee80211.h>
+#include <kunit/static_stub.h>
+
+#include "sta.h"
+#include "hcmd.h"
+#include "iface.h"
+#include "mlo.h"
+#include "key.h"
+#include "agg.h"
+#include "tlc.h"
+#include "fw/api/sta.h"
+#include "fw/api/mac.h"
+#include "fw/api/rx.h"
+
+int iwl_mld_fw_sta_id_from_link_sta(struct iwl_mld *mld,
+				    struct ieee80211_link_sta *link_sta)
+{
+	struct iwl_mld_link_sta *mld_link_sta;
+
+	/* This function should only be used with the wiphy lock held,
+	 * In other cases, it is not guaranteed that the link_sta will exist
+	 * in the driver too, and it is checked here.
+	 */
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* This is not meant to be called with a NULL pointer */
+	if (WARN_ON(!link_sta))
+		return -ENOENT;
+
+	mld_link_sta = iwl_mld_link_sta_from_mac80211(link_sta);
+	if (!mld_link_sta) {
+		WARN_ON(!iwl_mld_error_before_recovery(mld));
+		return -ENOENT;
+	}
+
+	return mld_link_sta->fw_id;
+}
+
+static void
+iwl_mld_fill_ampdu_size_and_dens(struct ieee80211_link_sta *link_sta,
+				 struct ieee80211_bss_conf *link,
+				 __le32 *tx_ampdu_max_size,
+				 __le32 *tx_ampdu_spacing)
+{
+	u32 agg_size = 0, mpdu_dens = 0;
+
+	if (WARN_ON(!link_sta || !link))
+		return;
+
+	/* Note that we always use only legacy & highest supported PPDUs, so
+	 * of Draft P802.11be D.30 Table 10-12a--Fields used for calculating
+	 * the maximum A-MPDU size of various PPDU types in different bands,
+	 * we only need to worry about the highest supported PPDU type here.
+	 */
+
+	if (link_sta->ht_cap.ht_supported) {
+		agg_size = link_sta->ht_cap.ampdu_factor;
+		mpdu_dens = link_sta->ht_cap.ampdu_density;
+	}
+
+	if (link->chanreq.oper.chan->band == NL80211_BAND_6GHZ) {
+		/* overwrite HT values on 6 GHz */
+		mpdu_dens =
+			le16_get_bits(link_sta->he_6ghz_capa.capa,
+				      IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START);
+		agg_size =
+			le16_get_bits(link_sta->he_6ghz_capa.capa,
+				      IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP);
+	} else if (link_sta->vht_cap.vht_supported) {
+		/* if VHT supported overwrite HT value */
+		agg_size =
+			u32_get_bits(link_sta->vht_cap.cap,
+				     IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK);
+	}
+
+	/* D6.0 10.12.2 A-MPDU length limit rules
+	 * A STA indicates the maximum length of the A-MPDU preEOF padding
+	 * that it can receive in an HE PPDU in the Maximum A-MPDU Length
+	 * Exponent field in its HT Capabilities, VHT Capabilities,
+	 * and HE 6 GHz Band Capabilities elements (if present) and the
+	 * Maximum AMPDU Length Exponent Extension field in its HE
+	 * Capabilities element
+	 */
+	if (link_sta->he_cap.has_he)
+		agg_size +=
+			u8_get_bits(link_sta->he_cap.he_cap_elem.mac_cap_info[3],
+				    IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_MASK);
+
+	if (link_sta->eht_cap.has_eht)
+		agg_size +=
+			u8_get_bits(link_sta->eht_cap.eht_cap_elem.mac_cap_info[1],
+				    IEEE80211_EHT_MAC_CAP1_MAX_AMPDU_LEN_MASK);
+
+	/* Limit to max A-MPDU supported by FW */
+	agg_size = min_t(u32, agg_size,
+			 STA_FLG_MAX_AGG_SIZE_4M >> STA_FLG_MAX_AGG_SIZE_SHIFT);
+
+	*tx_ampdu_max_size = cpu_to_le32(agg_size);
+	*tx_ampdu_spacing = cpu_to_le32(mpdu_dens);
+}
+
+static u8 iwl_mld_get_uapsd_acs(struct ieee80211_sta *sta)
+{
+	u8 uapsd_acs = 0;
+
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BK)
+		uapsd_acs |= BIT(AC_BK);
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BE)
+		uapsd_acs |= BIT(AC_BE);
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VI)
+		uapsd_acs |= BIT(AC_VI);
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VO)
+		uapsd_acs |= BIT(AC_VO);
+
+	return uapsd_acs | uapsd_acs << 4;
+}
+
+static u8 iwl_mld_he_get_ppe_val(u8 *ppe, u8 ppe_pos_bit)
+{
+	u8 byte_num = ppe_pos_bit / 8;
+	u8 bit_num = ppe_pos_bit % 8;
+	u8 residue_bits;
+	u8 res;
+
+	if (bit_num <= 5)
+		return (ppe[byte_num] >> bit_num) &
+		       (BIT(IEEE80211_PPE_THRES_INFO_PPET_SIZE) - 1);
+
+	/* If bit_num > 5, we have to combine bits with next byte.
+	 * Calculate how many bits we need to take from current byte (called
+	 * here "residue_bits"), and add them to bits from next byte.
+	 */
+
+	residue_bits = 8 - bit_num;
+
+	res = (ppe[byte_num + 1] &
+	       (BIT(IEEE80211_PPE_THRES_INFO_PPET_SIZE - residue_bits) - 1)) <<
+	      residue_bits;
+	res += (ppe[byte_num] >> bit_num) & (BIT(residue_bits) - 1);
+
+	return res;
+}
+
+static void iwl_mld_parse_ppe(struct iwl_mld *mld,
+			      struct iwl_he_pkt_ext_v2 *pkt_ext, u8 nss,
+			      u8 ru_index_bitmap, u8 *ppe, u8 ppe_pos_bit,
+			      bool inheritance)
+{
+	/* FW currently supports only nss == MAX_HE_SUPP_NSS
+	 *
+	 * If nss > MAX: we can ignore values we don't support
+	 * If nss < MAX: we can set zeros in other streams
+	 */
+	if (nss > MAX_HE_SUPP_NSS) {
+		IWL_DEBUG_INFO(mld, "Got NSS = %d - trimming to %d\n", nss,
+			       MAX_HE_SUPP_NSS);
+		nss = MAX_HE_SUPP_NSS;
+	}
+
+	for (int i = 0; i < nss; i++) {
+		u8 ru_index_tmp = ru_index_bitmap << 1;
+		u8 low_th = IWL_HE_PKT_EXT_NONE, high_th = IWL_HE_PKT_EXT_NONE;
+
+		for (u8 bw = 0;
+		     bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]);
+		     bw++) {
+			ru_index_tmp >>= 1;
+
+			/* According to the 11be spec, if for a specific BW the PPE Thresholds
+			 * isn't present - it should inherit the thresholds from the last
+			 * BW for which we had PPE Thresholds. In 11ax though, we don't have
+			 * this inheritance - continue in this case
+			 */
+			if (!(ru_index_tmp & 1)) {
+				if (inheritance)
+					goto set_thresholds;
+				else
+					continue;
+			}
+
+			high_th = iwl_mld_he_get_ppe_val(ppe, ppe_pos_bit);
+			ppe_pos_bit += IEEE80211_PPE_THRES_INFO_PPET_SIZE;
+			low_th = iwl_mld_he_get_ppe_val(ppe, ppe_pos_bit);
+			ppe_pos_bit += IEEE80211_PPE_THRES_INFO_PPET_SIZE;
+
+set_thresholds:
+			pkt_ext->pkt_ext_qam_th[i][bw][0] = low_th;
+			pkt_ext->pkt_ext_qam_th[i][bw][1] = high_th;
+		}
+	}
+}
+
+static void iwl_mld_set_pkt_ext_from_he_ppe(struct iwl_mld *mld,
+					    struct ieee80211_link_sta *link_sta,
+					    struct iwl_he_pkt_ext_v2 *pkt_ext,
+					    bool inheritance)
+{
+	u8 nss = (link_sta->he_cap.ppe_thres[0] &
+		  IEEE80211_PPE_THRES_NSS_MASK) + 1;
+	u8 *ppe = &link_sta->he_cap.ppe_thres[0];
+	u8 ru_index_bitmap =
+		u8_get_bits(*ppe,
+			    IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK);
+	/* Starting after PPE header */
+	u8 ppe_pos_bit = IEEE80211_HE_PPE_THRES_INFO_HEADER_SIZE;
+
+	iwl_mld_parse_ppe(mld, pkt_ext, nss, ru_index_bitmap, ppe, ppe_pos_bit,
+			  inheritance);
+}
+
+static int
+iwl_mld_set_pkt_ext_from_nominal_padding(struct iwl_he_pkt_ext_v2 *pkt_ext,
+					 u8 nominal_padding)
+{
+	int low_th = -1;
+	int high_th = -1;
+
+	/* all the macros are the same for EHT and HE */
+	switch (nominal_padding) {
+	case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_0US:
+		low_th = IWL_HE_PKT_EXT_NONE;
+		high_th = IWL_HE_PKT_EXT_NONE;
+		break;
+	case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US:
+		low_th = IWL_HE_PKT_EXT_BPSK;
+		high_th = IWL_HE_PKT_EXT_NONE;
+		break;
+	case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_16US:
+	case IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_20US:
+		low_th = IWL_HE_PKT_EXT_NONE;
+		high_th = IWL_HE_PKT_EXT_BPSK;
+		break;
+	}
+
+	if (low_th < 0 || high_th < 0)
+		return -EINVAL;
+
+	/* Set the PPE thresholds accordingly */
+	for (int i = 0; i < MAX_HE_SUPP_NSS; i++) {
+		for (u8 bw = 0;
+			bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]);
+			bw++) {
+			pkt_ext->pkt_ext_qam_th[i][bw][0] = low_th;
+			pkt_ext->pkt_ext_qam_th[i][bw][1] = high_th;
+		}
+	}
+
+	return 0;
+}
+
+static void iwl_mld_get_optimal_ppe_info(struct iwl_he_pkt_ext_v2 *pkt_ext,
+					 u8 nominal_padding)
+{
+	for (int i = 0; i < MAX_HE_SUPP_NSS; i++) {
+		for (u8 bw = 0; bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]);
+		     bw++) {
+			u8 *qam_th = &pkt_ext->pkt_ext_qam_th[i][bw][0];
+
+			if (nominal_padding >
+			    IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US &&
+			    qam_th[1] == IWL_HE_PKT_EXT_NONE)
+				qam_th[1] = IWL_HE_PKT_EXT_4096QAM;
+			else if (nominal_padding ==
+				 IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_8US &&
+				 qam_th[0] == IWL_HE_PKT_EXT_NONE &&
+				 qam_th[1] == IWL_HE_PKT_EXT_NONE)
+				qam_th[0] = IWL_HE_PKT_EXT_4096QAM;
+		}
+	}
+}
+
+static void iwl_mld_fill_pkt_ext(struct iwl_mld *mld,
+				 struct ieee80211_link_sta *link_sta,
+				 struct iwl_he_pkt_ext_v2 *pkt_ext)
+{
+	if (WARN_ON(!link_sta))
+		return;
+
+	/* Initialize the PPE thresholds to "None" (7), as described in Table
+	 * 9-262ac of 80211.ax/D3.0.
+	 */
+	memset(pkt_ext, IWL_HE_PKT_EXT_NONE, sizeof(*pkt_ext));
+
+	if (link_sta->eht_cap.has_eht) {
+		u8 nominal_padding =
+			u8_get_bits(link_sta->eht_cap.eht_cap_elem.phy_cap_info[5],
+				    IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK);
+
+		/* If PPE Thresholds exists, parse them into a FW-familiar
+		 * format.
+		 */
+		if (link_sta->eht_cap.eht_cap_elem.phy_cap_info[5] &
+		    IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT) {
+			u8 nss = (link_sta->eht_cap.eht_ppe_thres[0] &
+				IEEE80211_EHT_PPE_THRES_NSS_MASK) + 1;
+			u8 *ppe = &link_sta->eht_cap.eht_ppe_thres[0];
+			u8 ru_index_bitmap =
+				u16_get_bits(*ppe,
+					     IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK);
+			 /* Starting after PPE header */
+			u8 ppe_pos_bit = IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE;
+
+			iwl_mld_parse_ppe(mld, pkt_ext, nss, ru_index_bitmap,
+					  ppe, ppe_pos_bit, true);
+		/* EHT PPE Thresholds doesn't exist - set the API according to
+		 * HE PPE Tresholds
+		 */
+		} else if (link_sta->he_cap.he_cap_elem.phy_cap_info[6] &
+			   IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
+			/* Even though HE Capabilities IE doesn't contain PPE
+			 * Thresholds for BW 320Mhz, thresholds for this BW will
+			 * be filled in with the same values as 160Mhz, due to
+			 * the inheritance, as required.
+			 */
+			iwl_mld_set_pkt_ext_from_he_ppe(mld, link_sta, pkt_ext,
+							true);
+
+			/* According to the requirements, for MCSs 12-13 the
+			 * maximum value between HE PPE Threshold and Common
+			 * Nominal Packet Padding needs to be taken
+			 */
+			iwl_mld_get_optimal_ppe_info(pkt_ext, nominal_padding);
+
+		/* if PPE Thresholds doesn't present in both EHT IE and HE IE -
+		 * take the Thresholds from Common Nominal Packet Padding field
+		 */
+		} else {
+			iwl_mld_set_pkt_ext_from_nominal_padding(pkt_ext,
+								 nominal_padding);
+		}
+	} else if (link_sta->he_cap.has_he) {
+		/* If PPE Thresholds exist, parse them into a FW-familiar format. */
+		if (link_sta->he_cap.he_cap_elem.phy_cap_info[6] &
+			IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
+			iwl_mld_set_pkt_ext_from_he_ppe(mld, link_sta, pkt_ext,
+							false);
+		/* PPE Thresholds doesn't exist - set the API PPE values
+		 * according to Common Nominal Packet Padding field.
+		 */
+		} else {
+			u8 nominal_padding =
+				u8_get_bits(link_sta->he_cap.he_cap_elem.phy_cap_info[9],
+					    IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK);
+			if (nominal_padding != IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED)
+				iwl_mld_set_pkt_ext_from_nominal_padding(pkt_ext,
+									 nominal_padding);
+		}
+	}
+
+	for (int i = 0; i < MAX_HE_SUPP_NSS; i++) {
+		for (int bw = 0;
+		     bw < ARRAY_SIZE(*pkt_ext->pkt_ext_qam_th[i]);
+		     bw++) {
+			u8 *qam_th =
+				&pkt_ext->pkt_ext_qam_th[i][bw][0];
+
+			IWL_DEBUG_HT(mld,
+				     "PPE table: nss[%d] bw[%d] PPET8 = %d, PPET16 = %d\n",
+				     i, bw, qam_th[0], qam_th[1]);
+		}
+	}
+}
+
+static u32 iwl_mld_get_htc_flags(struct ieee80211_link_sta *link_sta)
+{
+	u8 *mac_cap_info =
+		&link_sta->he_cap.he_cap_elem.mac_cap_info[0];
+	u32 htc_flags = 0;
+
+	if (mac_cap_info[0] & IEEE80211_HE_MAC_CAP0_HTC_HE)
+		htc_flags |= IWL_HE_HTC_SUPPORT;
+	if ((mac_cap_info[1] & IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION) ||
+	    (mac_cap_info[2] & IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION)) {
+		u8 link_adap =
+			((mac_cap_info[2] &
+			  IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION) << 1) +
+			 (mac_cap_info[1] &
+			  IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION);
+
+		if (link_adap == 2)
+			htc_flags |=
+				IWL_HE_HTC_LINK_ADAP_UNSOLICITED;
+		else if (link_adap == 3)
+			htc_flags |= IWL_HE_HTC_LINK_ADAP_BOTH;
+	}
+	if (mac_cap_info[2] & IEEE80211_HE_MAC_CAP2_BSR)
+		htc_flags |= IWL_HE_HTC_BSR_SUPP;
+	if (mac_cap_info[3] & IEEE80211_HE_MAC_CAP3_OMI_CONTROL)
+		htc_flags |= IWL_HE_HTC_OMI_SUPP;
+	if (mac_cap_info[4] & IEEE80211_HE_MAC_CAP4_BQR)
+		htc_flags |= IWL_HE_HTC_BQR_SUPP;
+
+	return htc_flags;
+}
+
+static int iwl_mld_send_sta_cmd(struct iwl_mld *mld,
+				const struct iwl_sta_cfg_cmd *cmd)
+{
+	u32 cmd_id = WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD);
+	int cmd_len = iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 0) > 1 ?
+		      sizeof(*cmd) :
+		      sizeof(struct iwl_sta_cfg_cmd_v1);
+	int ret = iwl_mld_send_cmd_pdu(mld, cmd_id, cmd, cmd_len);
+	if (ret)
+		IWL_ERR(mld, "STA_CONFIG_CMD send failed, ret=0x%x\n", ret);
+	return ret;
+}
+
+static int
+iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
+			   struct ieee80211_link_sta *link_sta)
+{
+	struct ieee80211_sta *sta = link_sta->sta;
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct ieee80211_bss_conf *link;
+	struct iwl_mld_link *mld_link;
+	struct iwl_sta_cfg_cmd cmd = {};
+	int fw_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	link = link_conf_dereference_protected(mld_sta->vif,
+					       link_sta->link_id);
+
+	mld_link = iwl_mld_link_from_mac80211(link);
+
+	if (WARN_ON(!link || !mld_link) || fw_id < 0)
+		return -EINVAL;
+
+	cmd.sta_id = cpu_to_le32(fw_id);
+	cmd.station_type = cpu_to_le32(mld_sta->sta_type);
+	cmd.link_id = cpu_to_le32(mld_link->fw_id);
+
+	memcpy(&cmd.peer_mld_address, sta->addr, ETH_ALEN);
+	memcpy(&cmd.peer_link_address, link_sta->addr, ETH_ALEN);
+
+	if (mld_sta->sta_state >= IEEE80211_STA_ASSOC)
+		cmd.assoc_id = cpu_to_le32(sta->aid);
+
+	if (sta->mfp || mld_sta->sta_state < IEEE80211_STA_AUTHORIZED)
+		cmd.mfp = cpu_to_le32(1);
+
+	switch (link_sta->rx_nss) {
+	case 1:
+		cmd.mimo = cpu_to_le32(0);
+		break;
+	case 2 ... 8:
+		cmd.mimo = cpu_to_le32(1);
+		break;
+	}
+
+	switch (link_sta->smps_mode) {
+	case IEEE80211_SMPS_AUTOMATIC:
+	case IEEE80211_SMPS_NUM_MODES:
+		WARN_ON(1);
+		break;
+	case IEEE80211_SMPS_STATIC:
+		/* override NSS */
+		cmd.mimo = cpu_to_le32(0);
+		break;
+	case IEEE80211_SMPS_DYNAMIC:
+		cmd.mimo_protection = cpu_to_le32(1);
+		break;
+	case IEEE80211_SMPS_OFF:
+		/* nothing */
+		break;
+	}
+
+	iwl_mld_fill_ampdu_size_and_dens(link_sta, link,
+					 &cmd.tx_ampdu_max_size,
+					 &cmd.tx_ampdu_spacing);
+
+	if (sta->wme) {
+		cmd.sp_length =
+			cpu_to_le32(sta->max_sp ? sta->max_sp * 2 : 128);
+		cmd.uapsd_acs = cpu_to_le32(iwl_mld_get_uapsd_acs(sta));
+	}
+
+	if (link_sta->he_cap.has_he) {
+		cmd.trig_rnd_alloc =
+			cpu_to_le32(link->uora_exists ? 1 : 0);
+
+		/* PPE Thresholds */
+		iwl_mld_fill_pkt_ext(mld, link_sta, &cmd.pkt_ext);
+
+		/* HTC flags */
+		cmd.htc_flags =
+			cpu_to_le32(iwl_mld_get_htc_flags(link_sta));
+
+		if (link_sta->he_cap.he_cap_elem.mac_cap_info[2] &
+		    IEEE80211_HE_MAC_CAP2_ACK_EN)
+			cmd.ack_enabled = cpu_to_le32(1);
+	}
+
+	return iwl_mld_send_sta_cmd(mld, &cmd);
+}
+
+IWL_MLD_ALLOC_FN(link_sta, link_sta)
+
+static int
+iwl_mld_add_link_sta(struct iwl_mld *mld, struct ieee80211_link_sta *link_sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	struct iwl_mld_link_sta *mld_link_sta;
+	int ret;
+	u8 fw_id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* We will fail to add it to the FW anyway */
+	if (iwl_mld_error_before_recovery(mld))
+		return -ENODEV;
+
+	mld_link_sta = iwl_mld_link_sta_from_mac80211(link_sta);
+
+	/* We need to preserve the fw sta ids during a restart, since the fw
+	 * will recover SN/PN for them, this is why the mld_link_sta exists.
+	 */
+	if (mld_link_sta) {
+		/* But if we are not restarting, this is not OK */
+		WARN_ON(!mld->fw_status.in_hw_restart);
+
+		/* Avoid adding a STA that is already in FW to avoid an assert */
+		if (WARN_ON(mld_link_sta->in_fw))
+			return -EINVAL;
+
+		fw_id = mld_link_sta->fw_id;
+		goto add_to_fw;
+	}
+
+	/* Allocate a fw id and map it to the link_sta */
+	ret = iwl_mld_allocate_link_sta_fw_id(mld, &fw_id, link_sta);
+	if (ret)
+		return ret;
+
+	if (link_sta == &link_sta->sta->deflink) {
+		mld_link_sta = &mld_sta->deflink;
+	} else {
+		mld_link_sta = kzalloc(sizeof(*mld_link_sta), GFP_KERNEL);
+		if (!mld_link_sta)
+			return -ENOMEM;
+	}
+
+	mld_link_sta->fw_id = fw_id;
+	rcu_assign_pointer(mld_sta->link[link_sta->link_id], mld_link_sta);
+
+add_to_fw:
+	ret = iwl_mld_add_modify_sta_cmd(mld, link_sta);
+	if (ret) {
+		RCU_INIT_POINTER(mld->fw_id_to_link_sta[fw_id], NULL);
+		RCU_INIT_POINTER(mld_sta->link[link_sta->link_id], NULL);
+		if (link_sta != &link_sta->sta->deflink)
+			kfree(mld_link_sta);
+		return ret;
+	}
+	mld_link_sta->in_fw = true;
+
+	return 0;
+}
+
+static int iwl_mld_rm_sta_from_fw(struct iwl_mld *mld, u8 fw_sta_id)
+{
+	struct iwl_remove_sta_cmd cmd = {
+		.sta_id = cpu_to_le32(fw_sta_id),
+	};
+	int ret;
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(MAC_CONF_GROUP, STA_REMOVE_CMD),
+				   &cmd);
+	if (ret)
+		IWL_ERR(mld, "Failed to remove station. Id=%d\n", fw_sta_id);
+
+	return ret;
+}
+
+static void
+iwl_mld_remove_link_sta(struct iwl_mld *mld,
+			struct ieee80211_link_sta *link_sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	struct iwl_mld_link_sta *mld_link_sta =
+		iwl_mld_link_sta_from_mac80211(link_sta);
+
+	if (WARN_ON(!mld_link_sta))
+		return;
+
+	iwl_mld_rm_sta_from_fw(mld, mld_link_sta->fw_id);
+	mld_link_sta->in_fw = false;
+
+	/* Now that the STA doesn't exist in FW, we don't expect any new
+	 * notifications for it. Cancel the ones that are already pending
+	 */
+	iwl_mld_cancel_notifications_of_object(mld, IWL_MLD_OBJECT_TYPE_STA,
+					       mld_link_sta->fw_id);
+
+	/* This will not be done upon reconfig, so do it also when
+	 * failed to remove from fw
+	 */
+	RCU_INIT_POINTER(mld->fw_id_to_link_sta[mld_link_sta->fw_id], NULL);
+	RCU_INIT_POINTER(mld_sta->link[link_sta->link_id], NULL);
+	if (mld_link_sta != &mld_sta->deflink)
+		kfree_rcu(mld_link_sta, rcu_head);
+}
+
+static void iwl_mld_set_max_amsdu_len(struct iwl_mld *mld,
+				      struct ieee80211_link_sta *link_sta)
+{
+	const struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+
+	/* For EHT, HE and VHT we can use the value as it was calculated by
+	 * mac80211. For HT, mac80211 doesn't enforce to 4095, so force it
+	 * here
+	 */
+	if (link_sta->eht_cap.has_eht || link_sta->he_cap.has_he ||
+	    link_sta->vht_cap.vht_supported ||
+	    !ht_cap->ht_supported ||
+	    !(ht_cap->cap & IEEE80211_HT_CAP_MAX_AMSDU))
+		return;
+
+	link_sta->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_BA;
+	ieee80211_sta_recalc_aggregates(link_sta->sta);
+}
+
+int iwl_mld_update_all_link_stations(struct iwl_mld *mld,
+				     struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct ieee80211_link_sta *link_sta;
+	int link_id;
+
+	for_each_sta_active_link(mld_sta->vif, sta, link_sta, link_id) {
+		int ret = iwl_mld_add_modify_sta_cmd(mld, link_sta);
+
+		if (ret)
+			return ret;
+
+		if (mld_sta->sta_state == IEEE80211_STA_ASSOC)
+			iwl_mld_set_max_amsdu_len(mld, link_sta);
+	}
+	return 0;
+}
+
+static void iwl_mld_destroy_sta(struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+	kfree(mld_sta->dup_data);
+	kfree(mld_sta->mpdu_counters);
+}
+
+static int
+iwl_mld_alloc_dup_data(struct iwl_mld *mld, struct iwl_mld_sta *mld_sta)
+{
+	struct iwl_mld_rxq_dup_data *dup_data;
+
+	if (mld->fw_status.in_hw_restart)
+		return 0;
+
+	dup_data = kcalloc(mld->trans->info.num_rxqs, sizeof(*dup_data),
+			   GFP_KERNEL);
+	if (!dup_data)
+		return -ENOMEM;
+
+	/* Initialize all the last_seq values to 0xffff which can never
+	 * compare equal to the frame's seq_ctrl in the check in
+	 * iwl_mld_is_dup() since the lower 4 bits are the fragment
+	 * number and fragmented packets don't reach that function.
+	 *
+	 * This thus allows receiving a packet with seqno 0 and the
+	 * retry bit set as the very first packet on a new TID.
+	 */
+	for (int q = 0; q < mld->trans->info.num_rxqs; q++)
+		memset(dup_data[q].last_seq, 0xff,
+		       sizeof(dup_data[q].last_seq));
+	mld_sta->dup_data = dup_data;
+
+	return 0;
+}
+
+static void iwl_mld_alloc_mpdu_counters(struct iwl_mld *mld,
+					struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct ieee80211_vif *vif = mld_sta->vif;
+
+	if (mld->fw_status.in_hw_restart)
+		return;
+
+	/* MPDUs are counted only when EMLSR is possible */
+	if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION ||
+	    sta->tdls || !ieee80211_vif_is_mld(vif))
+		return;
+
+	mld_sta->mpdu_counters = kcalloc(mld->trans->info.num_rxqs,
+					 sizeof(*mld_sta->mpdu_counters),
+					 GFP_KERNEL);
+	if (!mld_sta->mpdu_counters)
+		return;
+
+	for (int q = 0; q < mld->trans->info.num_rxqs; q++)
+		spin_lock_init(&mld_sta->mpdu_counters[q].lock);
+}
+
+static int
+iwl_mld_init_sta(struct iwl_mld *mld, struct ieee80211_sta *sta,
+		 struct ieee80211_vif *vif, enum iwl_fw_sta_type type)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+	mld_sta->vif = vif;
+	mld_sta->sta_type = type;
+	mld_sta->mld = mld;
+
+	if (!mld->fw_status.in_hw_restart)
+		for (int i = 0; i < ARRAY_SIZE(sta->txq); i++)
+			iwl_mld_init_txq(iwl_mld_txq_from_mac80211(sta->txq[i]));
+
+	iwl_mld_alloc_mpdu_counters(mld, sta);
+
+	iwl_mld_toggle_tx_ant(mld, &mld_sta->data_tx_ant);
+
+	return iwl_mld_alloc_dup_data(mld, mld_sta);
+}
+
+int iwl_mld_add_sta(struct iwl_mld *mld, struct ieee80211_sta *sta,
+		    struct ieee80211_vif *vif, enum iwl_fw_sta_type type)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct ieee80211_link_sta *link_sta;
+	int link_id;
+	int ret;
+
+	ret = iwl_mld_init_sta(mld, sta, vif, type);
+	if (ret)
+		return ret;
+
+	/* We could have add only the deflink link_sta, but it will not work
+	 * in the restart case if the single link that is active during
+	 * reconfig is not the deflink one.
+	 */
+	for_each_sta_active_link(mld_sta->vif, sta, link_sta, link_id) {
+		ret = iwl_mld_add_link_sta(mld, link_sta);
+		if (ret)
+			goto destroy_sta;
+	}
+
+	return 0;
+
+destroy_sta:
+	iwl_mld_destroy_sta(sta);
+
+	return ret;
+}
+
+void iwl_mld_flush_sta_txqs(struct iwl_mld *mld, struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct ieee80211_link_sta *link_sta;
+	int link_id;
+
+	for_each_sta_active_link(mld_sta->vif, sta, link_sta, link_id) {
+		int fw_sta_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
+
+		if (fw_sta_id < 0)
+			continue;
+
+		iwl_mld_flush_link_sta_txqs(mld, fw_sta_id);
+	}
+}
+
+void iwl_mld_wait_sta_txqs_empty(struct iwl_mld *mld, struct ieee80211_sta *sta)
+{
+	/* Avoid a warning in iwl_trans_wait_txq_empty if are anyway on the way
+	 * to a restart.
+	 */
+	if (iwl_mld_error_before_recovery(mld))
+		return;
+
+	for (int i = 0; i < ARRAY_SIZE(sta->txq); i++) {
+		struct iwl_mld_txq *mld_txq =
+			iwl_mld_txq_from_mac80211(sta->txq[i]);
+
+		if (!mld_txq->status.allocated)
+			continue;
+
+		iwl_trans_wait_txq_empty(mld->trans, mld_txq->fw_id);
+	}
+}
+
+void iwl_mld_remove_sta(struct iwl_mld *mld, struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct ieee80211_vif *vif = mld_sta->vif;
+	struct ieee80211_link_sta *link_sta;
+	u8 link_id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* Tell the HW to flush the queues */
+	iwl_mld_flush_sta_txqs(mld, sta);
+
+	/* Wait for trans to empty its queues */
+	iwl_mld_wait_sta_txqs_empty(mld, sta);
+
+	/* Now we can remove the queues */
+	for (int i = 0; i < ARRAY_SIZE(sta->txq); i++)
+		iwl_mld_remove_txq(mld, sta->txq[i]);
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		/* Mac8011 will remove the groupwise keys after the sta is
+		 * removed, but FW expects all the keys to be removed before
+		 * the STA is, so remove them all here.
+		 */
+		if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls)
+			iwl_mld_remove_ap_keys(mld, vif, sta, link_id);
+
+		/* Remove the link_sta */
+		iwl_mld_remove_link_sta(mld, link_sta);
+	}
+
+	iwl_mld_destroy_sta(sta);
+}
+
+u32 iwl_mld_fw_sta_id_mask(struct iwl_mld *mld, struct ieee80211_sta *sta)
+{
+	struct ieee80211_vif *vif = iwl_mld_sta_from_mac80211(sta)->vif;
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+	u32 result = 0;
+
+	KUNIT_STATIC_STUB_REDIRECT(iwl_mld_fw_sta_id_mask, mld, sta);
+
+	/* This function should only be used with the wiphy lock held,
+	 * In other cases, it is not guaranteed that the link_sta will exist
+	 * in the driver too, and it is checked in
+	 * iwl_mld_fw_sta_id_from_link_sta.
+	 */
+	lockdep_assert_wiphy(mld->wiphy);
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		int fw_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
+
+		if (!(fw_id < 0))
+			result |= BIT(fw_id);
+	}
+
+	return result;
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_fw_sta_id_mask);
+
+static void iwl_mld_count_mpdu(struct ieee80211_link_sta *link_sta, int queue,
+			       u32 count, bool tx)
+{
+	struct iwl_mld_per_q_mpdu_counter *queue_counter;
+	struct iwl_mld_per_link_mpdu_counter *link_counter;
+	struct iwl_mld_vif *mld_vif;
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_link *mld_link;
+	struct iwl_mld *mld;
+	int total_mpdus = 0;
+
+	if (WARN_ON(!link_sta))
+		return;
+
+	mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	if (!mld_sta->mpdu_counters)
+		return;
+
+	mld_vif = iwl_mld_vif_from_mac80211(mld_sta->vif);
+	mld_link = iwl_mld_link_dereference_check(mld_vif, link_sta->link_id);
+
+	if (WARN_ON_ONCE(!mld_link))
+		return;
+
+	queue_counter = &mld_sta->mpdu_counters[queue];
+
+	mld = mld_vif->mld;
+
+	/* If it the window is over, first clear the counters.
+	 * When we are not blocked by TPT, the window is managed by check_tpt_wk
+	 */
+	if ((mld_vif->emlsr.blocked_reasons & IWL_MLD_EMLSR_BLOCKED_TPT) &&
+	    time_is_before_jiffies(queue_counter->window_start_time +
+					IWL_MLD_TPT_COUNT_WINDOW)) {
+		memset(queue_counter->per_link, 0,
+		       sizeof(queue_counter->per_link));
+		queue_counter->window_start_time = jiffies;
+
+		IWL_DEBUG_INFO(mld, "MPDU counters are cleared\n");
+	}
+
+	link_counter = &queue_counter->per_link[mld_link->fw_id];
+
+	spin_lock_bh(&queue_counter->lock);
+
+	/* Update the statistics for this TPT measurement window */
+	if (tx)
+		link_counter->tx += count;
+	else
+		link_counter->rx += count;
+
+	/*
+	 * Next, evaluate whether we should queue an unblock,
+	 * skip this if we are not blocked due to low throughput.
+	 */
+	if (!(mld_vif->emlsr.blocked_reasons & IWL_MLD_EMLSR_BLOCKED_TPT))
+		goto unlock;
+
+	for (int i = 0; i <= IWL_FW_MAX_LINK_ID; i++)
+		total_mpdus += tx ? queue_counter->per_link[i].tx :
+				    queue_counter->per_link[i].rx;
+
+	/* Unblock is already queued if the threshold was reached before */
+	if (total_mpdus - count >= IWL_MLD_ENTER_EMLSR_TPT_THRESH)
+		goto unlock;
+
+	if (total_mpdus >= IWL_MLD_ENTER_EMLSR_TPT_THRESH)
+		wiphy_work_queue(mld->wiphy, &mld_vif->emlsr.unblock_tpt_wk);
+
+unlock:
+	spin_unlock_bh(&queue_counter->lock);
+}
+
+/* must be called under rcu_read_lock() */
+void iwl_mld_count_mpdu_rx(struct ieee80211_link_sta *link_sta, int queue,
+			   u32 count)
+{
+	iwl_mld_count_mpdu(link_sta, queue, count, false);
+}
+
+/* must be called under rcu_read_lock() */
+void iwl_mld_count_mpdu_tx(struct ieee80211_link_sta *link_sta, u32 count)
+{
+	/* use queue 0 for all TX */
+	iwl_mld_count_mpdu(link_sta, 0, count, true);
+}
+
+static int iwl_mld_allocate_internal_txq(struct iwl_mld *mld,
+					 struct iwl_mld_int_sta *internal_sta,
+					 u8 tid)
+{
+	u32 sta_mask = BIT(internal_sta->sta_id);
+	int queue, size;
+
+	size = max_t(u32, IWL_MGMT_QUEUE_SIZE,
+		     mld->trans->mac_cfg->base->min_txq_size);
+
+	queue = iwl_trans_txq_alloc(mld->trans, 0, sta_mask, tid, size,
+				    IWL_WATCHDOG_DISABLED);
+
+	if (queue >= 0)
+		IWL_DEBUG_TX_QUEUES(mld,
+				    "Enabling TXQ #%d for sta mask 0x%x tid %d\n",
+				    queue, sta_mask, tid);
+	return queue;
+}
+
+static int iwl_mld_send_aux_sta_cmd(struct iwl_mld *mld,
+				    const struct iwl_mld_int_sta *internal_sta)
+{
+	struct iwl_aux_sta_cmd cmd = {
+		.sta_id = cpu_to_le32(internal_sta->sta_id),
+		/* TODO: CDB - properly set the lmac_id */
+		.lmac_id = cpu_to_le32(IWL_LMAC_24G_INDEX),
+	};
+
+	return iwl_mld_send_cmd_pdu(mld, WIDE_ID(MAC_CONF_GROUP, AUX_STA_CMD),
+				    &cmd);
+}
+
+static int
+iwl_mld_add_internal_sta_to_fw(struct iwl_mld *mld,
+			       const struct iwl_mld_int_sta *internal_sta,
+			       u8 fw_link_id,
+			       const u8 *addr)
+{
+	struct iwl_sta_cfg_cmd cmd = {};
+
+	if (internal_sta->sta_type == STATION_TYPE_AUX)
+		return iwl_mld_send_aux_sta_cmd(mld, internal_sta);
+
+	cmd.sta_id = cpu_to_le32((u8)internal_sta->sta_id);
+	cmd.link_id = cpu_to_le32(fw_link_id);
+	cmd.station_type = cpu_to_le32(internal_sta->sta_type);
+
+	/* FW doesn't allow to add a IGTK/BIGTK if the sta isn't marked as MFP.
+	 * On the other hand, FW will never check this flag during RX since
+	 * an AP/GO doesn't receive protected broadcast management frames.
+	 * So, we can set it unconditionally.
+	 */
+	if (internal_sta->sta_type == STATION_TYPE_BCAST_MGMT)
+		cmd.mfp = cpu_to_le32(1);
+
+	if (addr) {
+		memcpy(cmd.peer_mld_address, addr, ETH_ALEN);
+		memcpy(cmd.peer_link_address, addr, ETH_ALEN);
+	}
+
+	return iwl_mld_send_sta_cmd(mld, &cmd);
+}
+
+static int iwl_mld_add_internal_sta(struct iwl_mld *mld,
+				    struct iwl_mld_int_sta *internal_sta,
+				    enum iwl_fw_sta_type sta_type,
+				    u8 fw_link_id, const u8 *addr, u8 tid)
+{
+	int ret, queue_id;
+
+	ret = iwl_mld_allocate_link_sta_fw_id(mld,
+					      &internal_sta->sta_id,
+					      ERR_PTR(-EINVAL));
+	if (ret)
+		return ret;
+
+	internal_sta->sta_type = sta_type;
+
+	ret = iwl_mld_add_internal_sta_to_fw(mld, internal_sta, fw_link_id,
+					     addr);
+	if (ret)
+		goto err;
+
+	queue_id = iwl_mld_allocate_internal_txq(mld, internal_sta, tid);
+	if (queue_id < 0) {
+		iwl_mld_rm_sta_from_fw(mld, internal_sta->sta_id);
+		ret = queue_id;
+		goto err;
+	}
+
+	internal_sta->queue_id = queue_id;
+
+	return 0;
+err:
+	iwl_mld_free_internal_sta(mld, internal_sta);
+	return ret;
+}
+
+int iwl_mld_add_bcast_sta(struct iwl_mld *mld,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	const u8 bcast_addr[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
+	const u8 *addr;
+
+	if (WARN_ON(!mld_link))
+		return -EINVAL;
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_AP &&
+		    vif->type != NL80211_IFTYPE_ADHOC))
+		return -EINVAL;
+
+	addr = vif->type == NL80211_IFTYPE_ADHOC ? link->bssid : bcast_addr;
+
+	return iwl_mld_add_internal_sta(mld, &mld_link->bcast_sta,
+					STATION_TYPE_BCAST_MGMT,
+					mld_link->fw_id, addr,
+					IWL_MGMT_TID);
+}
+
+int iwl_mld_add_mcast_sta(struct iwl_mld *mld,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	const u8 mcast_addr[] = {0x03, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+	if (WARN_ON(!mld_link))
+		return -EINVAL;
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_AP &&
+		    vif->type != NL80211_IFTYPE_ADHOC))
+		return -EINVAL;
+
+	return iwl_mld_add_internal_sta(mld, &mld_link->mcast_sta,
+					STATION_TYPE_MCAST,
+					mld_link->fw_id, mcast_addr, 0);
+}
+
+int iwl_mld_add_aux_sta(struct iwl_mld *mld,
+			struct iwl_mld_int_sta *internal_sta)
+{
+	return iwl_mld_add_internal_sta(mld, internal_sta, STATION_TYPE_AUX,
+					0, NULL, IWL_MAX_TID_COUNT);
+}
+
+int iwl_mld_add_mon_sta(struct iwl_mld *mld,
+			struct ieee80211_vif *vif,
+			struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+
+	if (WARN_ON(!mld_link))
+		return -EINVAL;
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_MONITOR))
+		return -EINVAL;
+
+	return iwl_mld_add_internal_sta(mld, &mld_link->mon_sta,
+					STATION_TYPE_BCAST_MGMT,
+					mld_link->fw_id, NULL,
+					IWL_MAX_TID_COUNT);
+}
+
+static void iwl_mld_remove_internal_sta(struct iwl_mld *mld,
+					struct iwl_mld_int_sta *internal_sta,
+					bool flush, u8 tid)
+{
+	if (WARN_ON_ONCE(internal_sta->sta_id == IWL_INVALID_STA ||
+			 internal_sta->queue_id == IWL_MLD_INVALID_QUEUE))
+		return;
+
+	if (flush)
+		iwl_mld_flush_link_sta_txqs(mld, internal_sta->sta_id);
+
+	iwl_mld_free_txq(mld, BIT(internal_sta->sta_id),
+			 tid, internal_sta->queue_id);
+
+	iwl_mld_rm_sta_from_fw(mld, internal_sta->sta_id);
+
+	iwl_mld_free_internal_sta(mld, internal_sta);
+}
+
+void iwl_mld_remove_bcast_sta(struct iwl_mld *mld,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+
+	if (WARN_ON(!mld_link))
+		return;
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_AP &&
+		    vif->type != NL80211_IFTYPE_ADHOC))
+		return;
+
+	iwl_mld_remove_internal_sta(mld, &mld_link->bcast_sta, true,
+				    IWL_MGMT_TID);
+}
+
+void iwl_mld_remove_mcast_sta(struct iwl_mld *mld,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+
+	if (WARN_ON(!mld_link))
+		return;
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_AP &&
+		    vif->type != NL80211_IFTYPE_ADHOC))
+		return;
+
+	iwl_mld_remove_internal_sta(mld, &mld_link->mcast_sta, true, 0);
+}
+
+void iwl_mld_remove_aux_sta(struct iwl_mld *mld,
+			    struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+		    vif->type != NL80211_IFTYPE_STATION))
+		return;
+
+	iwl_mld_remove_internal_sta(mld, &mld_vif->aux_sta, false,
+				    IWL_MAX_TID_COUNT);
+}
+
+void iwl_mld_remove_mon_sta(struct iwl_mld *mld,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+
+	if (WARN_ON(!mld_link))
+		return;
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_MONITOR))
+		return;
+
+	iwl_mld_remove_internal_sta(mld, &mld_link->mon_sta, false,
+				    IWL_MAX_TID_COUNT);
+}
+
+static int iwl_mld_update_sta_resources(struct iwl_mld *mld,
+					struct ieee80211_vif *vif,
+					struct ieee80211_sta *sta,
+					u32 old_sta_mask,
+					u32 new_sta_mask)
+{
+	int ret;
+
+	ret = iwl_mld_update_sta_txqs(mld, sta, old_sta_mask, new_sta_mask);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_update_sta_keys(mld, vif, sta, old_sta_mask, new_sta_mask);
+	if (ret)
+		return ret;
+
+	return iwl_mld_update_sta_baids(mld, old_sta_mask, new_sta_mask);
+}
+
+int iwl_mld_update_link_stas(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta,
+			     u16 old_links, u16 new_links)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_link_sta *mld_link_sta;
+	unsigned long links_to_add = ~old_links & new_links;
+	unsigned long links_to_rem = old_links & ~new_links;
+	unsigned long old_links_long = old_links;
+	unsigned long sta_mask_added = 0;
+	u32 current_sta_mask = 0, sta_mask_to_rem = 0;
+	unsigned int link_id, sta_id;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	for_each_set_bit(link_id, &old_links_long,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		mld_link_sta =
+			iwl_mld_link_sta_dereference_check(mld_sta, link_id);
+
+		if (WARN_ON(!mld_link_sta))
+			return -EINVAL;
+
+		current_sta_mask |= BIT(mld_link_sta->fw_id);
+		if (links_to_rem & BIT(link_id))
+			sta_mask_to_rem |= BIT(mld_link_sta->fw_id);
+	}
+
+	if (sta_mask_to_rem) {
+		ret = iwl_mld_update_sta_resources(mld, vif, sta,
+						   current_sta_mask,
+						   current_sta_mask &
+							~sta_mask_to_rem);
+		if (ret)
+			return ret;
+
+		current_sta_mask &= ~sta_mask_to_rem;
+	}
+
+	for_each_set_bit(link_id, &links_to_rem, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_protected(sta, link_id);
+
+		if (WARN_ON(!link_sta))
+			return -EINVAL;
+
+		iwl_mld_remove_link_sta(mld, link_sta);
+	}
+
+	for_each_set_bit(link_id, &links_to_add, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_protected(sta, link_id);
+		struct ieee80211_bss_conf *link;
+
+		if (WARN_ON(!link_sta))
+			return -EINVAL;
+
+		ret = iwl_mld_add_link_sta(mld, link_sta);
+		if (ret)
+			goto remove_added_link_stas;
+
+		mld_link_sta =
+			iwl_mld_link_sta_dereference_check(mld_sta,
+							   link_id);
+
+		link = link_conf_dereference_protected(mld_sta->vif,
+						       link_sta->link_id);
+
+		iwl_mld_set_max_amsdu_len(mld, link_sta);
+		iwl_mld_config_tlc_link(mld, vif, link, link_sta);
+
+		sta_mask_added |= BIT(mld_link_sta->fw_id);
+	}
+
+	if (sta_mask_added) {
+		ret = iwl_mld_update_sta_resources(mld, vif, sta,
+						   current_sta_mask,
+						   current_sta_mask |
+							sta_mask_added);
+		if (ret)
+			goto remove_added_link_stas;
+	}
+
+	/* We couldn't activate the links before it has a STA. Now we can */
+	for_each_set_bit(link_id, &links_to_add, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link =
+			link_conf_dereference_protected(mld_sta->vif, link_id);
+
+		if (WARN_ON(!link))
+			continue;
+
+		iwl_mld_activate_link(mld, link);
+	}
+
+	return 0;
+
+remove_added_link_stas:
+	for_each_set_bit(sta_id, &sta_mask_added, mld->fw->ucode_capa.num_stations) {
+		struct ieee80211_link_sta *link_sta =
+			wiphy_dereference(mld->wiphy,
+					  mld->fw_id_to_link_sta[sta_id]);
+
+		if (WARN_ON(!link_sta))
+			continue;
+
+		iwl_mld_remove_link_sta(mld, link_sta);
+	}
+
+	return ret;
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/sta.h b/sys/contrib/dev/iwlwifi/mld/sta.h
new file mode 100644
index 000000000000..1897b121aae2
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/sta.h
@@ -0,0 +1,273 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#ifndef __iwl_mld_sta_h__
+#define __iwl_mld_sta_h__
+
+#include <net/mac80211.h>
+
+#include "mld.h"
+#include "tx.h"
+
+/**
+ * struct iwl_mld_rxq_dup_data - Duplication detection data, per STA & Rx queue
+ * @last_seq: last sequence per tid.
+ * @last_sub_frame_idx: the index of the last subframe in an A-MSDU. This value
+ *	will be zero if the packet is not part of an A-MSDU.
+ */
+struct iwl_mld_rxq_dup_data {
+	__le16 last_seq[IWL_MAX_TID_COUNT + 1];
+	u8 last_sub_frame_idx[IWL_MAX_TID_COUNT + 1];
+} ____cacheline_aligned_in_smp;
+
+/**
+ * struct iwl_mld_link_sta - link-level station
+ *
+ * This represents the link-level sta - the driver level equivalent to the
+ * ieee80211_link_sta
+ *
+ * @last_rate_n_flags: rate_n_flags from the last &iwl_tlc_update_notif
+ * @signal_avg: the signal average coming from the firmware
+ * @in_fw: whether the link STA is uploaded to the FW (false during restart)
+ * @rcu_head: RCU head for freeing this object
+ * @fw_id: the FW id of this link sta.
+ */
+struct iwl_mld_link_sta {
+	/* Add here fields that need clean up on restart */
+	struct_group(zeroed_on_hw_restart,
+		u32 last_rate_n_flags;
+		bool in_fw;
+		s8 signal_avg;
+	);
+	/* And here fields that survive a fw restart */
+	struct rcu_head rcu_head;
+	u32 fw_id;
+};
+
+#define iwl_mld_link_sta_dereference_check(mld_sta, link_id)		\
+	rcu_dereference_check((mld_sta)->link[link_id],			\
+			      lockdep_is_held(&mld_sta->mld->wiphy->mtx))
+
+#define for_each_mld_link_sta(mld_sta, link_sta, link_id)		\
+	for (link_id = 0; link_id < ARRAY_SIZE((mld_sta)->link);	\
+	     link_id++)							\
+		if ((link_sta =						\
+			iwl_mld_link_sta_dereference_check(mld_sta, link_id)))
+
+#define IWL_NUM_DEFAULT_KEYS 4
+
+/* struct iwl_mld_ptk_pn - Holds Packet Number (PN) per TID.
+ * @rcu_head: RCU head for freeing this data.
+ * @pn: Array storing PN for each TID.
+ */
+struct iwl_mld_ptk_pn {
+	struct rcu_head rcu_head;
+	struct {
+		u8 pn[IWL_MAX_TID_COUNT][IEEE80211_CCMP_PN_LEN];
+	} ____cacheline_aligned_in_smp q[];
+};
+
+/**
+ * struct iwl_mld_per_link_mpdu_counter - per-link TX/RX MPDU counters
+ *
+ * @tx: Number of TX MPDUs.
+ * @rx: Number of RX MPDUs.
+ */
+struct iwl_mld_per_link_mpdu_counter {
+	u32 tx;
+	u32 rx;
+};
+
+/**
+ * struct iwl_mld_per_q_mpdu_counter - per-queue MPDU counter
+ *
+ * @lock: Needed to protect the counters when modified from statistics.
+ * @per_link: per-link counters.
+ * @window_start_time: timestamp of the counting-window start
+ */
+struct iwl_mld_per_q_mpdu_counter {
+	spinlock_t lock;
+	struct iwl_mld_per_link_mpdu_counter per_link[IWL_FW_MAX_LINK_ID + 1];
+	unsigned long window_start_time;
+} ____cacheline_aligned_in_smp;
+
+/**
+ * struct iwl_mld_sta - representation of a station in the driver.
+ *
+ * This represent the MLD-level sta, and will not be added to the FW.
+ * Embedded in ieee80211_sta.
+ *
+ * @vif: pointer the vif object.
+ * @sta_state: station state according to enum %ieee80211_sta_state
+ * @sta_type: type of this station. See &enum iwl_fw_sta_type
+ * @mld: a pointer to the iwl_mld object
+ * @dup_data: per queue duplicate packet detection data
+ * @data_tx_ant: stores the last TX antenna index; used for setting
+ *	TX rate_n_flags for injected data frames (toggles on every TX failure).
+ * @tid_to_baid: a simple map of TID to Block-Ack fw id
+ * @deflink: This holds the default link STA information, for non MLO STA all
+ *	link specific STA information is accessed through @deflink or through
+ *	link[0] which points to address of @deflink. For MLO Link STA
+ *	the first added link STA will point to deflink.
+ * @link: reference to Link Sta entries. For Non MLO STA, except 1st link,
+ *	i.e link[0] all links would be assigned to NULL by default and
+ *	would access link information via @deflink or link[0]. For MLO
+ *	STA, first link STA being added will point its link pointer to
+ *	@deflink address and remaining would be allocated and the address
+ *	would be assigned to link[link_id] where link_id is the id assigned
+ *	by the AP.
+ * @ptk_pn: Array of pointers to PTK PN data, used to track the Packet Number
+ *	per key index and per queue (TID).
+ * @mpdu_counters: RX/TX MPDUs counters for each queue.
+ */
+struct iwl_mld_sta {
+	/* Add here fields that need clean up on restart */
+	struct_group(zeroed_on_hw_restart,
+		enum ieee80211_sta_state sta_state;
+		enum iwl_fw_sta_type sta_type;
+	);
+	/* And here fields that survive a fw restart */
+	struct iwl_mld *mld;
+	struct ieee80211_vif *vif;
+	struct iwl_mld_rxq_dup_data *dup_data;
+	u8 tid_to_baid[IWL_MAX_TID_COUNT];
+	u8 data_tx_ant;
+
+	struct iwl_mld_link_sta deflink;
+	struct iwl_mld_link_sta __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
+	struct iwl_mld_ptk_pn __rcu *ptk_pn[IWL_NUM_DEFAULT_KEYS];
+	struct iwl_mld_per_q_mpdu_counter *mpdu_counters;
+};
+
+static inline struct iwl_mld_sta *
+iwl_mld_sta_from_mac80211(struct ieee80211_sta *sta)
+{
+	return (void *)sta->drv_priv;
+}
+
+static inline void
+iwl_mld_cleanup_sta(void *data, struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_link_sta *mld_link_sta;
+	u8 link_id;
+
+	for (int i = 0; i < ARRAY_SIZE(sta->txq); i++)
+		CLEANUP_STRUCT(iwl_mld_txq_from_mac80211(sta->txq[i]));
+
+	for_each_mld_link_sta(mld_sta, mld_link_sta, link_id) {
+		CLEANUP_STRUCT(mld_link_sta);
+
+		if (!ieee80211_vif_is_mld(mld_sta->vif)) {
+			/* not an MLD STA; only has the deflink with ID zero */
+			WARN_ON(link_id);
+			continue;
+		}
+
+		if (mld_sta->vif->active_links & BIT(link_id))
+			continue;
+
+		/* Should not happen as link removal should always succeed */
+		WARN_ON(1);
+		RCU_INIT_POINTER(mld_sta->link[link_id], NULL);
+		RCU_INIT_POINTER(mld_sta->mld->fw_id_to_link_sta[mld_link_sta->fw_id],
+				 NULL);
+		if (mld_link_sta != &mld_sta->deflink)
+			kfree_rcu(mld_link_sta, rcu_head);
+	}
+
+	CLEANUP_STRUCT(mld_sta);
+}
+
+static inline struct iwl_mld_link_sta *
+iwl_mld_link_sta_from_mac80211(struct ieee80211_link_sta *link_sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+
+	return iwl_mld_link_sta_dereference_check(mld_sta, link_sta->link_id);
+}
+
+int iwl_mld_add_sta(struct iwl_mld *mld, struct ieee80211_sta *sta,
+		    struct ieee80211_vif *vif, enum iwl_fw_sta_type type);
+void iwl_mld_remove_sta(struct iwl_mld *mld, struct ieee80211_sta *sta);
+int iwl_mld_fw_sta_id_from_link_sta(struct iwl_mld *mld,
+				    struct ieee80211_link_sta *link_sta);
+u32 iwl_mld_fw_sta_id_mask(struct iwl_mld *mld, struct ieee80211_sta *sta);
+int iwl_mld_update_all_link_stations(struct iwl_mld *mld,
+				     struct ieee80211_sta *sta);
+void iwl_mld_flush_sta_txqs(struct iwl_mld *mld, struct ieee80211_sta *sta);
+void iwl_mld_wait_sta_txqs_empty(struct iwl_mld *mld,
+				 struct ieee80211_sta *sta);
+void iwl_mld_count_mpdu_rx(struct ieee80211_link_sta *link_sta, int queue,
+			   u32 count);
+void iwl_mld_count_mpdu_tx(struct ieee80211_link_sta *link_sta, u32 count);
+
+/**
+ * struct iwl_mld_int_sta - representation of an internal station
+ * (a station that exist in FW and in driver, but not in mac80211)
+ *
+ * @sta_id: the index of the station in the fw
+ * @queue_id: the if of the queue used by the station
+ * @sta_type: station type. One of &iwl_fw_sta_type
+ */
+struct iwl_mld_int_sta {
+	u8 sta_id;
+	u32 queue_id;
+	enum iwl_fw_sta_type sta_type;
+};
+
+static inline void
+iwl_mld_init_internal_sta(struct iwl_mld_int_sta *internal_sta)
+{
+	internal_sta->sta_id = IWL_INVALID_STA;
+	internal_sta->queue_id = IWL_MLD_INVALID_QUEUE;
+}
+
+static inline void
+iwl_mld_free_internal_sta(struct iwl_mld *mld,
+			  struct iwl_mld_int_sta *internal_sta)
+{
+	if (WARN_ON(internal_sta->sta_id == IWL_INVALID_STA))
+		return;
+
+	RCU_INIT_POINTER(mld->fw_id_to_link_sta[internal_sta->sta_id], NULL);
+	iwl_mld_init_internal_sta(internal_sta);
+}
+
+int iwl_mld_add_bcast_sta(struct iwl_mld *mld,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link);
+
+int iwl_mld_add_mcast_sta(struct iwl_mld *mld,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link);
+
+int iwl_mld_add_aux_sta(struct iwl_mld *mld,
+			struct iwl_mld_int_sta *internal_sta);
+
+int iwl_mld_add_mon_sta(struct iwl_mld *mld,
+			struct ieee80211_vif *vif,
+			struct ieee80211_bss_conf *link);
+
+void iwl_mld_remove_bcast_sta(struct iwl_mld *mld,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link);
+
+void iwl_mld_remove_mcast_sta(struct iwl_mld *mld,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link);
+
+void iwl_mld_remove_aux_sta(struct iwl_mld *mld,
+			    struct ieee80211_vif *vif);
+
+void iwl_mld_remove_mon_sta(struct iwl_mld *mld,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_bss_conf *link);
+
+int iwl_mld_update_link_stas(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta,
+			     u16 old_links, u16 new_links);
+#endif /* __iwl_mld_sta_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/stats.c b/sys/contrib/dev/iwlwifi/mld/stats.c
new file mode 100644
index 000000000000..cbc64db5eab6
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/stats.c
@@ -0,0 +1,518 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include "mld.h"
+#include "stats.h"
+#include "sta.h"
+#include "mlo.h"
+#include "hcmd.h"
+#include "iface.h"
+#include "scan.h"
+#include "phy.h"
+#include "fw/api/stats.h"
+
+static int iwl_mld_send_fw_stats_cmd(struct iwl_mld *mld, u32 cfg_mask,
+				     u32 cfg_time, u32 type_mask)
+{
+	u32 cmd_id = WIDE_ID(SYSTEM_GROUP, SYSTEM_STATISTICS_CMD);
+	struct iwl_system_statistics_cmd stats_cmd = {
+		.cfg_mask = cpu_to_le32(cfg_mask),
+		.config_time_sec = cpu_to_le32(cfg_time),
+		.type_id_mask = cpu_to_le32(type_mask),
+	};
+
+	return iwl_mld_send_cmd_pdu(mld, cmd_id, &stats_cmd);
+}
+
+int iwl_mld_clear_stats_in_fw(struct iwl_mld *mld)
+{
+	u32 cfg_mask = IWL_STATS_CFG_FLG_ON_DEMAND_NTFY_MSK;
+	u32 type_mask = IWL_STATS_NTFY_TYPE_ID_OPER |
+			IWL_STATS_NTFY_TYPE_ID_OPER_PART1;
+
+	return iwl_mld_send_fw_stats_cmd(mld, cfg_mask, 0, type_mask);
+}
+
+static void
+iwl_mld_fill_stats_from_oper_notif(struct iwl_mld *mld,
+				   struct iwl_rx_packet *pkt,
+				   u8 fw_sta_id, struct station_info *sinfo)
+{
+	const struct iwl_system_statistics_notif_oper *notif =
+		(void *)&pkt->data;
+	const struct iwl_stats_ntfy_per_sta *per_sta =
+		&notif->per_sta[fw_sta_id];
+	struct ieee80211_link_sta *link_sta;
+	struct iwl_mld_link_sta *mld_link_sta;
+
+	/* 0 isn't a valid value, but FW might send 0.
+	 * In that case, set the latest non-zero value we stored
+	 */
+	rcu_read_lock();
+
+	link_sta = rcu_dereference(mld->fw_id_to_link_sta[fw_sta_id]);
+	if (IS_ERR_OR_NULL(link_sta))
+		goto unlock;
+
+	mld_link_sta = iwl_mld_link_sta_from_mac80211(link_sta);
+	if (WARN_ON(!mld_link_sta))
+		goto unlock;
+
+	if (per_sta->average_energy)
+		mld_link_sta->signal_avg =
+			-(s8)le32_to_cpu(per_sta->average_energy);
+
+	sinfo->signal_avg = mld_link_sta->signal_avg;
+	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
+
+unlock:
+	rcu_read_unlock();
+}
+
+struct iwl_mld_stats_data {
+	u8 fw_sta_id;
+	struct station_info *sinfo;
+	struct iwl_mld *mld;
+};
+
+static bool iwl_mld_wait_stats_handler(struct iwl_notif_wait_data *notif_data,
+				       struct iwl_rx_packet *pkt, void *data)
+{
+	struct iwl_mld_stats_data *stats_data = data;
+	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
+
+	switch (cmd) {
+	case WIDE_ID(STATISTICS_GROUP, STATISTICS_OPER_NOTIF):
+		iwl_mld_fill_stats_from_oper_notif(stats_data->mld, pkt,
+						   stats_data->fw_sta_id,
+						   stats_data->sinfo);
+		break;
+	case WIDE_ID(STATISTICS_GROUP, STATISTICS_OPER_PART1_NOTIF):
+		break;
+	case WIDE_ID(SYSTEM_GROUP, SYSTEM_STATISTICS_END_NOTIF):
+		return true;
+	}
+
+	return false;
+}
+
+static int
+iwl_mld_fw_stats_to_mac80211(struct iwl_mld *mld, struct iwl_mld_sta *mld_sta,
+			     struct station_info *sinfo)
+{
+	u32 cfg_mask = IWL_STATS_CFG_FLG_ON_DEMAND_NTFY_MSK |
+		       IWL_STATS_CFG_FLG_RESET_MSK;
+	u32 type_mask = IWL_STATS_NTFY_TYPE_ID_OPER |
+			IWL_STATS_NTFY_TYPE_ID_OPER_PART1;
+	static const u16 notifications[] = {
+		WIDE_ID(STATISTICS_GROUP, STATISTICS_OPER_NOTIF),
+		WIDE_ID(STATISTICS_GROUP, STATISTICS_OPER_PART1_NOTIF),
+		WIDE_ID(SYSTEM_GROUP, SYSTEM_STATISTICS_END_NOTIF),
+	};
+	struct iwl_mld_stats_data wait_stats_data = {
+		/* We don't support drv_sta_statistics in EMLSR */
+		.fw_sta_id = mld_sta->deflink.fw_id,
+		.sinfo = sinfo,
+		.mld = mld,
+	};
+	struct iwl_notification_wait stats_wait;
+	int ret;
+
+	iwl_init_notification_wait(&mld->notif_wait, &stats_wait,
+				   notifications, ARRAY_SIZE(notifications),
+				   iwl_mld_wait_stats_handler,
+				   &wait_stats_data);
+
+	ret = iwl_mld_send_fw_stats_cmd(mld, cfg_mask, 0, type_mask);
+	if (ret) {
+		iwl_remove_notification(&mld->notif_wait, &stats_wait);
+		return ret;
+	}
+
+	/* Wait 500ms for OPERATIONAL, PART1, and END notifications,
+	 * which should be sufficient for the firmware to gather data
+	 * from all LMACs and send notifications to the host.
+	 */
+	ret = iwl_wait_notification(&mld->notif_wait, &stats_wait, HZ / 2);
+	if (ret)
+		return ret;
+
+	/* When periodic statistics are sent, FW will clear its statistics DB.
+	 * If the statistics request here happens shortly afterwards,
+	 * the response will contain data collected over a short time
+	 * interval. The response we got here shouldn't be processed by
+	 * the general statistics processing because it's incomplete.
+	 * So, we delete it from the list so it won't be processed.
+	 */
+	iwl_mld_delete_handlers(mld, notifications, ARRAY_SIZE(notifications));
+
+	return 0;
+}
+
+#define PERIODIC_STATS_SECONDS 5
+
+int iwl_mld_request_periodic_fw_stats(struct iwl_mld *mld, bool enable)
+{
+	u32 cfg_mask = enable ? 0 : IWL_STATS_CFG_FLG_DISABLE_NTFY_MSK;
+	u32 type_mask = enable ? (IWL_STATS_NTFY_TYPE_ID_OPER |
+				  IWL_STATS_NTFY_TYPE_ID_OPER_PART1) : 0;
+	u32 cfg_time = enable ? PERIODIC_STATS_SECONDS : 0;
+
+	return iwl_mld_send_fw_stats_cmd(mld, cfg_mask, cfg_time, type_mask);
+}
+
+static void iwl_mld_sta_stats_fill_txrate(struct iwl_mld_sta *mld_sta,
+					  struct station_info *sinfo)
+{
+	struct rate_info *rinfo = &sinfo->txrate;
+	u32 rate_n_flags = mld_sta->deflink.last_rate_n_flags;
+	u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+	u32 gi_ltf;
+
+	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
+
+	switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) {
+	case RATE_MCS_CHAN_WIDTH_20:
+		rinfo->bw = RATE_INFO_BW_20;
+		break;
+	case RATE_MCS_CHAN_WIDTH_40:
+		rinfo->bw = RATE_INFO_BW_40;
+		break;
+	case RATE_MCS_CHAN_WIDTH_80:
+		rinfo->bw = RATE_INFO_BW_80;
+		break;
+	case RATE_MCS_CHAN_WIDTH_160:
+		rinfo->bw = RATE_INFO_BW_160;
+		break;
+	case RATE_MCS_CHAN_WIDTH_320:
+		rinfo->bw = RATE_INFO_BW_320;
+		break;
+	}
+
+	if (format == RATE_MCS_MOD_TYPE_CCK ||
+	    format == RATE_MCS_MOD_TYPE_LEGACY_OFDM) {
+		int rate = u32_get_bits(rate_n_flags, RATE_LEGACY_RATE_MSK);
+
+		/* add the offset needed to get to the legacy ofdm indices */
+		if (format == RATE_MCS_MOD_TYPE_LEGACY_OFDM)
+			rate += IWL_FIRST_OFDM_RATE;
+
+		switch (rate) {
+		case IWL_RATE_1M_INDEX:
+			rinfo->legacy = 10;
+			break;
+		case IWL_RATE_2M_INDEX:
+			rinfo->legacy = 20;
+			break;
+		case IWL_RATE_5M_INDEX:
+			rinfo->legacy = 55;
+			break;
+		case IWL_RATE_11M_INDEX:
+			rinfo->legacy = 110;
+			break;
+		case IWL_RATE_6M_INDEX:
+			rinfo->legacy = 60;
+			break;
+		case IWL_RATE_9M_INDEX:
+			rinfo->legacy = 90;
+			break;
+		case IWL_RATE_12M_INDEX:
+			rinfo->legacy = 120;
+			break;
+		case IWL_RATE_18M_INDEX:
+			rinfo->legacy = 180;
+			break;
+		case IWL_RATE_24M_INDEX:
+			rinfo->legacy = 240;
+			break;
+		case IWL_RATE_36M_INDEX:
+			rinfo->legacy = 360;
+			break;
+		case IWL_RATE_48M_INDEX:
+			rinfo->legacy = 480;
+			break;
+		case IWL_RATE_54M_INDEX:
+			rinfo->legacy = 540;
+		}
+		return;
+	}
+
+	rinfo->nss = u32_get_bits(rate_n_flags, RATE_MCS_NSS_MSK) + 1;
+
+	if (format == RATE_MCS_MOD_TYPE_HT)
+		rinfo->mcs = RATE_HT_MCS_INDEX(rate_n_flags);
+	else
+		rinfo->mcs = u32_get_bits(rate_n_flags, RATE_MCS_CODE_MSK);
+
+	if (rate_n_flags & RATE_MCS_SGI_MSK)
+		rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
+
+	switch (format) {
+	case RATE_MCS_MOD_TYPE_EHT:
+		rinfo->flags |= RATE_INFO_FLAGS_EHT_MCS;
+		break;
+	case RATE_MCS_MOD_TYPE_HE:
+		gi_ltf = u32_get_bits(rate_n_flags, RATE_MCS_HE_GI_LTF_MSK);
+
+		rinfo->flags |= RATE_INFO_FLAGS_HE_MCS;
+
+		if (rate_n_flags & RATE_MCS_HE_106T_MSK) {
+			rinfo->bw = RATE_INFO_BW_HE_RU;
+			rinfo->he_ru_alloc = NL80211_RATE_INFO_HE_RU_ALLOC_106;
+		}
+
+		switch (rate_n_flags & RATE_MCS_HE_TYPE_MSK) {
+		case RATE_MCS_HE_TYPE_SU:
+		case RATE_MCS_HE_TYPE_EXT_SU:
+			if (gi_ltf == 0 || gi_ltf == 1)
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_0_8;
+			else if (gi_ltf == 2)
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_1_6;
+			else if (gi_ltf == 3)
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_3_2;
+			else
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_0_8;
+			break;
+		case RATE_MCS_HE_TYPE_MU:
+			if (gi_ltf == 0 || gi_ltf == 1)
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_0_8;
+			else if (gi_ltf == 2)
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_1_6;
+			else
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_3_2;
+			break;
+		case RATE_MCS_HE_TYPE_TRIG:
+			if (gi_ltf == 0 || gi_ltf == 1)
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_1_6;
+			else
+				rinfo->he_gi = NL80211_RATE_INFO_HE_GI_3_2;
+			break;
+		}
+
+		if (rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK)
+			rinfo->he_dcm = 1;
+		break;
+	case RATE_MCS_MOD_TYPE_HT:
+		rinfo->flags |= RATE_INFO_FLAGS_MCS;
+		break;
+	case RATE_MCS_MOD_TYPE_VHT:
+		rinfo->flags |= RATE_INFO_FLAGS_VHT_MCS;
+		break;
+	}
+}
+
+void iwl_mld_mac80211_sta_statistics(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_sta *sta,
+				     struct station_info *sinfo)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+	/* This API is not EMLSR ready, so we cannot provide complete
+	 * information if EMLSR is active
+	 */
+	if (hweight16(vif->active_links) > 1)
+		return;
+
+	if (iwl_mld_fw_stats_to_mac80211(mld_sta->mld, mld_sta, sinfo))
+		return;
+
+	iwl_mld_sta_stats_fill_txrate(mld_sta, sinfo);
+
+	/* TODO: NL80211_STA_INFO_BEACON_RX */
+
+	/* TODO: NL80211_STA_INFO_BEACON_SIGNAL_AVG */
+}
+
+#define IWL_MLD_TRAFFIC_LOAD_MEDIUM_THRESH	10 /* percentage */
+#define IWL_MLD_TRAFFIC_LOAD_HIGH_THRESH	50 /* percentage */
+#define IWL_MLD_TRAFFIC_LOAD_MIN_WINDOW_USEC	(500 * 1000)
+
+static u8 iwl_mld_stats_load_percentage(u32 last_ts_usec, u32 curr_ts_usec,
+					u32 total_airtime_usec)
+{
+	u32 elapsed_usec = curr_ts_usec - last_ts_usec;
+
+	if (elapsed_usec < IWL_MLD_TRAFFIC_LOAD_MIN_WINDOW_USEC)
+		return 0;
+
+	return (100 * total_airtime_usec / elapsed_usec);
+}
+
+static void iwl_mld_stats_recalc_traffic_load(struct iwl_mld *mld,
+					      u32 total_airtime_usec,
+					      u32 curr_ts_usec)
+{
+	u32 last_ts_usec = mld->scan.traffic_load.last_stats_ts_usec;
+	u8 load_prec;
+
+	/* Skip the calculation as this is the first notification received */
+	if (!last_ts_usec)
+		goto out;
+
+	load_prec = iwl_mld_stats_load_percentage(last_ts_usec, curr_ts_usec,
+						  total_airtime_usec);
+
+	if (load_prec > IWL_MLD_TRAFFIC_LOAD_HIGH_THRESH)
+		mld->scan.traffic_load.status = IWL_MLD_TRAFFIC_HIGH;
+	else if (load_prec > IWL_MLD_TRAFFIC_LOAD_MEDIUM_THRESH)
+		mld->scan.traffic_load.status = IWL_MLD_TRAFFIC_MEDIUM;
+	else
+		mld->scan.traffic_load.status = IWL_MLD_TRAFFIC_LOW;
+
+out:
+	mld->scan.traffic_load.last_stats_ts_usec = curr_ts_usec;
+}
+
+static void iwl_mld_update_link_sig(struct ieee80211_vif *vif, int sig,
+				    struct ieee80211_bss_conf *bss_conf)
+{
+	struct iwl_mld *mld = iwl_mld_vif_from_mac80211(vif)->mld;
+	int exit_emlsr_thresh;
+
+	if (sig == 0) {
+		IWL_DEBUG_RX(mld, "RSSI is 0 - skip signal based decision\n");
+		return;
+	}
+
+	/* TODO: task=statistics handle CQM notifications */
+
+	if (sig < IWL_MLD_LOW_RSSI_MLO_SCAN_THRESH)
+		iwl_mld_int_mlo_scan(mld, vif);
+
+	if (!iwl_mld_emlsr_active(vif))
+		return;
+
+	/* We are in EMLSR, check if we need to exit */
+	exit_emlsr_thresh =
+		iwl_mld_get_emlsr_rssi_thresh(mld, &bss_conf->chanreq.oper,
+					      true);
+
+	if (sig < exit_emlsr_thresh)
+		iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_LOW_RSSI,
+				   iwl_mld_get_other_link(vif,
+							  bss_conf->link_id));
+}
+
+static void
+iwl_mld_process_per_link_stats(struct iwl_mld *mld,
+			       const struct iwl_stats_ntfy_per_link *per_link,
+			       u32 curr_ts_usec)
+{
+	u32 total_airtime_usec = 0;
+
+	for (u32 fw_id = 0;
+	     fw_id < ARRAY_SIZE(mld->fw_id_to_bss_conf);
+	     fw_id++) {
+		const struct iwl_stats_ntfy_per_link *link_stats;
+		struct ieee80211_bss_conf *bss_conf;
+		int sig;
+
+		bss_conf = wiphy_dereference(mld->wiphy,
+					     mld->fw_id_to_bss_conf[fw_id]);
+		if (!bss_conf || bss_conf->vif->type != NL80211_IFTYPE_STATION)
+			continue;
+
+		link_stats = &per_link[fw_id];
+
+		total_airtime_usec += le32_to_cpu(link_stats->air_time);
+
+		sig = -le32_to_cpu(link_stats->beacon_filter_average_energy);
+		iwl_mld_update_link_sig(bss_conf->vif, sig, bss_conf);
+
+		/* TODO: parse more fields here (task=statistics)*/
+	}
+
+	iwl_mld_stats_recalc_traffic_load(mld, total_airtime_usec,
+					  curr_ts_usec);
+}
+
+static void
+iwl_mld_process_per_sta_stats(struct iwl_mld *mld,
+			      const struct iwl_stats_ntfy_per_sta *per_sta)
+{
+	for (int i = 0; i < mld->fw->ucode_capa.num_stations; i++) {
+		struct ieee80211_link_sta *link_sta =
+			wiphy_dereference(mld->wiphy,
+					  mld->fw_id_to_link_sta[i]);
+		struct iwl_mld_link_sta *mld_link_sta;
+		s8 avg_energy =
+			-(s8)le32_to_cpu(per_sta[i].average_energy);
+
+		if (IS_ERR_OR_NULL(link_sta) || !avg_energy)
+			continue;
+
+		mld_link_sta = iwl_mld_link_sta_from_mac80211(link_sta);
+		if (WARN_ON(!mld_link_sta))
+			continue;
+
+		mld_link_sta->signal_avg = avg_energy;
+	}
+}
+
+static void iwl_mld_fill_chanctx_stats(struct ieee80211_hw *hw,
+				       struct ieee80211_chanctx_conf *ctx,
+				       void *data)
+{
+	struct iwl_mld_phy *phy = iwl_mld_phy_from_mac80211(ctx);
+	const struct iwl_stats_ntfy_per_phy *per_phy = data;
+	u32 new_load, old_load;
+
+	if (WARN_ON(phy->fw_id >= IWL_STATS_MAX_PHY_OPERATIONAL))
+		return;
+
+	phy->channel_load_by_us =
+		le32_to_cpu(per_phy[phy->fw_id].channel_load_by_us);
+
+	old_load = phy->avg_channel_load_not_by_us;
+	new_load = le32_to_cpu(per_phy[phy->fw_id].channel_load_not_by_us);
+
+	if (IWL_FW_CHECK(phy->mld,
+			 new_load != IWL_STATS_UNKNOWN_CHANNEL_LOAD &&
+				new_load > 100,
+			 "Invalid channel load %u\n", new_load))
+		return;
+
+	if (new_load != IWL_STATS_UNKNOWN_CHANNEL_LOAD) {
+		/* update giving a weight of 0.5 for the old value */
+		phy->avg_channel_load_not_by_us = (new_load >> 1) +
+						  (old_load >> 1);
+	}
+
+	iwl_mld_emlsr_check_chan_load(hw, phy, old_load);
+}
+
+static void
+iwl_mld_process_per_phy_stats(struct iwl_mld *mld,
+			      const struct iwl_stats_ntfy_per_phy *per_phy)
+{
+	ieee80211_iter_chan_contexts_mtx(mld->hw,
+					 iwl_mld_fill_chanctx_stats,
+					 (void *)(uintptr_t)per_phy);
+
+}
+
+void iwl_mld_handle_stats_oper_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt)
+{
+	const struct iwl_system_statistics_notif_oper *stats =
+		(void *)&pkt->data;
+	u32 curr_ts_usec = le32_to_cpu(stats->time_stamp);
+
+	BUILD_BUG_ON(ARRAY_SIZE(stats->per_sta) != IWL_STATION_COUNT_MAX);
+	BUILD_BUG_ON(ARRAY_SIZE(stats->per_link) <
+		     ARRAY_SIZE(mld->fw_id_to_bss_conf));
+
+	iwl_mld_process_per_link_stats(mld, stats->per_link, curr_ts_usec);
+	iwl_mld_process_per_sta_stats(mld, stats->per_sta);
+	iwl_mld_process_per_phy_stats(mld, stats->per_phy);
+}
+
+void iwl_mld_handle_stats_oper_part1_notif(struct iwl_mld *mld,
+					   struct iwl_rx_packet *pkt)
+{
+	/* TODO */
+}
+
diff --git a/sys/contrib/dev/iwlwifi/mld/stats.h b/sys/contrib/dev/iwlwifi/mld/stats.h
new file mode 100644
index 000000000000..de434e66d555
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/stats.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_stats_h__
+#define __iwl_mld_stats_h__
+
+int iwl_mld_request_periodic_fw_stats(struct iwl_mld *mld, bool enable);
+
+void iwl_mld_mac80211_sta_statistics(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_sta *sta,
+				     struct station_info *sinfo);
+
+void iwl_mld_handle_stats_oper_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt);
+void iwl_mld_handle_stats_oper_part1_notif(struct iwl_mld *mld,
+					   struct iwl_rx_packet *pkt);
+
+int iwl_mld_clear_stats_in_fw(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_stats_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/Makefile b/sys/contrib/dev/iwlwifi/mld/tests/Makefile
new file mode 100644
index 000000000000..36317feb923b
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+iwlmld-tests-y += module.o hcmd.o utils.o link.o rx.o agg.o link-selection.o
+
+ccflags-y += -I$(src)/../
+obj-$(CONFIG_IWLWIFI_KUNIT_TESTS) += iwlmld-tests.o
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/agg.c b/sys/contrib/dev/iwlwifi/mld/tests/agg.c
new file mode 100644
index 000000000000..29b0248cec3d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/agg.c
@@ -0,0 +1,663 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for channel helper functions
+ *
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <kunit/test.h>
+#include <kunit/static_stub.h>
+#include <kunit/skbuff.h>
+
+#include "utils.h"
+#include "mld.h"
+#include "sta.h"
+#include "agg.h"
+#include "rx.h"
+
+#define FC_QOS_DATA (IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_DATA)
+#define BA_WINDOW_SIZE 64
+#define QUEUE 0
+
+static const struct reorder_buffer_case {
+	const char *desc;
+	struct {
+		/* ieee80211_hdr fields */
+		u16 fc;
+		u8 tid;
+		bool multicast;
+		/* iwl_rx_mpdu_desc fields */
+		u16 nssn;
+		/* used also for setting hdr::seq_ctrl */
+		u16 sn;
+		u8 baid;
+		bool amsdu;
+		bool last_subframe;
+		bool old_sn;
+		bool dup;
+	} rx_pkt;
+	struct {
+		bool valid;
+		u16 head_sn;
+		u8 baid;
+		u16 num_entries;
+		/* The test prepares the reorder buffer with fake skbs based
+		 * on the sequence numbers provided in @entries array.
+		 */
+		struct {
+			u16 sn;
+			/* Set add_subframes > 0 to simulate an A-MSDU by
+			 * queueing additional @add_subframes skbs in the
+			 * appropriate reorder buffer entry (based on the @sn)
+			 */
+			u8 add_subframes;
+		} entries[BA_WINDOW_SIZE];
+	} reorder_buf_state;
+	struct {
+		enum iwl_mld_reorder_result reorder_res;
+		u16 head_sn;
+		u16 num_stored;
+		u16 skb_release_order[BA_WINDOW_SIZE];
+		u16 skb_release_order_count;
+	} expected;
+} reorder_buffer_cases[] = {
+	{
+		.desc = "RX packet with invalid BAID",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.baid = IWL_RX_REORDER_DATA_INVALID_BAID,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+		},
+		.expected = {
+			/* Invalid BAID should not be buffered. The frame is
+			 * passed to the network stack immediately.
+			 */
+			.reorder_res = IWL_MLD_PASS_SKB,
+			.num_stored = 0,
+		},
+	},
+	{
+		.desc = "RX Multicast packet",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.multicast = true,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+		},
+		.expected = {
+			/* Multicast packets are not buffered. The packet is
+			 * passed to the network stack immediately.
+			 */
+			.reorder_res = IWL_MLD_PASS_SKB,
+			.num_stored = 0,
+		},
+	},
+	{
+		.desc = "RX non-QoS data",
+		.rx_pkt = {
+			.fc = IEEE80211_FTYPE_DATA,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+		},
+		.expected = {
+			/* non-QoS data frames do not require reordering.
+			 * The packet is passed to the network stack
+			 * immediately.
+			 */
+		.reorder_res = IWL_MLD_PASS_SKB,
+		},
+	},
+	{
+		.desc = "RX old SN packet, reorder buffer is not yet valid",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.old_sn = true,
+		},
+		.reorder_buf_state = {
+			.valid = false,
+		},
+		.expected = {
+			/* The buffer is invalid and the RX packet has an old
+			 * SN. The packet is passed to the network stack
+			 * immediately.
+			 */
+			.reorder_res = IWL_MLD_PASS_SKB,
+		},
+	},
+	{
+		.desc = "RX old SN packet, reorder buffer valid",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.old_sn = true,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+		},
+		.expected = {
+			/* The buffer is valid and the RX packet has an old SN.
+			 * The packet should be dropped.
+			 */
+			.reorder_res = IWL_MLD_DROP_SKB,
+			.num_stored = 0,
+			.head_sn = 100,
+		},
+	},
+	{
+		.desc = "RX duplicate packet",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.dup = true,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+		},
+		.expected = {
+			/* Duplicate packets should be dropped */
+			.reorder_res = IWL_MLD_DROP_SKB,
+			.num_stored = 0,
+			.head_sn = 100,
+		},
+	},
+	{
+		.desc = "RX In-order packet, sn < nssn",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 100,
+			.nssn = 101,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+		},
+		.expected = {
+			/* 1. Reorder buffer is empty.
+			 * 2. RX packet SN is in order and less than NSSN.
+			 * Packet is released to the network stack immediately
+			 * and buffer->head_sn is updated to NSSN.
+			 */
+			.reorder_res = IWL_MLD_PASS_SKB,
+			.num_stored = 0,
+			.head_sn = 101,
+		},
+	},
+	{
+		.desc = "RX In-order packet, sn == head_sn",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 101,
+			.nssn = 100,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 101,
+		},
+		.expected = {
+			/* 1. Reorder buffer is empty.
+			 * 2. RX packet SN is equal to buffer->head_sn.
+			 * Packet is released to the network stack immediately
+			 * and buffer->head_sn is incremented.
+			 */
+			.reorder_res = IWL_MLD_PASS_SKB,
+			.num_stored = 0,
+			.head_sn = 102,
+		},
+	},
+	{
+		.desc = "RX In-order packet, IEEE80211_MAX_SN wrap around",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = IEEE80211_MAX_SN,
+			.nssn = IEEE80211_MAX_SN - 1,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = IEEE80211_MAX_SN,
+		},
+		.expected = {
+			/* 1. Reorder buffer is empty.
+			 * 2. RX SN == buffer->head_sn == IEEE80211_MAX_SN
+			 * Packet is released to the network stack immediately
+			 * and buffer->head_sn is incremented correctly (wraps
+			 * around to 0).
+			 */
+			.reorder_res = IWL_MLD_PASS_SKB,
+			.num_stored = 0,
+			.head_sn = 0,
+		},
+	},
+	{
+		.desc = "RX Out-of-order packet, pending packet in buffer",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 100,
+			.nssn = 102,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+			.num_entries = 1,
+			.entries[0].sn = 101,
+		},
+		.expected = {
+			/* 1. Reorder buffer contains one packet with SN=101.
+			 * 2. RX packet SN = buffer->head_sn.
+			 * Both packets are released (in order) to the network
+			 * stack as there are no gaps.
+			 */
+			.reorder_res = IWL_MLD_BUFFERED_SKB,
+			.num_stored = 0,
+			.head_sn = 102,
+			.skb_release_order = {100, 101},
+			.skb_release_order_count = 2,
+		},
+	},
+	{
+		.desc = "RX Out-of-order packet, pending packet in buffer (wrap around)",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 0,
+			.nssn = 1,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = IEEE80211_MAX_SN - 1,
+			.num_entries = 1,
+			.entries[0].sn = IEEE80211_MAX_SN,
+		},
+		.expected = {
+			/* 1. Reorder buffer contains one packet with
+			 *    SN=IEEE80211_MAX_SN.
+			 * 2. RX Packet SN = 0 (after wrap around)
+			 * Both packets are released (in order) to the network
+			 * stack as there are no gaps.
+			 */
+			.reorder_res = IWL_MLD_BUFFERED_SKB,
+			.num_stored = 0,
+			.head_sn = 1,
+			.skb_release_order = { 4095, 0 },
+			.skb_release_order_count = 2,
+		},
+	},
+	{
+		.desc = "RX Out-of-order packet, filling 1/2 holes in buffer, release RX packet",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 100,
+			.nssn = 101,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+			.num_entries = 1,
+			.entries[0].sn = 102,
+		},
+		.expected = {
+			/* 1. Reorder buffer contains one packet with SN=102.
+			 * 2. There are 2 holes at SN={100, 101}.
+			 * Only the RX packet (SN=100) is released, there is
+			 * still a hole at 101.
+			 */
+			.reorder_res = IWL_MLD_BUFFERED_SKB,
+			.num_stored = 1,
+			.head_sn = 101,
+			.skb_release_order = {100},
+			.skb_release_order_count = 1,
+		},
+	},
+	{
+		.desc = "RX Out-of-order packet, filling 1/2 holes, release 2 packets",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 102,
+			.nssn = 103,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+			.num_entries = 3,
+			.entries[0].sn = 101,
+			.entries[1].sn = 104,
+			.entries[2].sn = 105,
+		},
+		.expected = {
+			/* 1. Reorder buffer contains three packets.
+			 * 2. RX packet fills one of two holes (at SN=102).
+			 * Two packets are released (until the next hole at
+			 * SN=103).
+			 */
+			.reorder_res = IWL_MLD_BUFFERED_SKB,
+			.num_stored = 2,
+			.head_sn = 103,
+			.skb_release_order = {101, 102},
+			.skb_release_order_count = 2,
+		},
+	},
+		{
+		.desc = "RX Out-of-order packet, filling 1/1 holes, no packets released",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 102,
+			.nssn = 100,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+			.num_entries = 3,
+			.entries[0].sn = 101,
+			.entries[1].sn = 103,
+			.entries[2].sn = 104,
+		},
+		.expected = {
+			/* 1. Reorder buffer contains three packets:
+			 *    SN={101, 103, 104}.
+			 * 2. RX packet fills a hole (SN=102), but NSSN is
+			 *    smaller than buffered frames.
+			 * No packets can be released yet and buffer->head_sn
+			 * is not updated.
+			 */
+			.reorder_res = IWL_MLD_BUFFERED_SKB,
+			.num_stored = 4,
+			.head_sn = 100,
+		},
+	},
+	{
+		.desc = "RX In-order A-MSDU, last subframe",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 100,
+			.nssn = 101,
+			.amsdu = true,
+			.last_subframe = true,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+			.num_entries = 1,
+			.entries[0] = {
+				.sn = 100,
+				.add_subframes = 1,
+			},
+		},
+		.expected = {
+			/* 1. Reorder buffer contains a 2-sub frames A-MSDU
+			 *    at SN=100.
+			 * 2. RX packet is the last SN=100 A-MSDU subframe
+			 * All packets are released in order (3 x SN=100).
+			 */
+			.reorder_res = IWL_MLD_BUFFERED_SKB,
+			.num_stored = 0,
+			.head_sn = 101,
+			.skb_release_order = {100, 100, 100},
+			.skb_release_order_count = 3,
+		},
+	},
+	{
+		.desc = "RX In-order A-MSDU, not the last subframe",
+		.rx_pkt = {
+			.fc = FC_QOS_DATA,
+			.sn = 100,
+			.nssn = 101,
+			.amsdu = true,
+			.last_subframe = false,
+		},
+		.reorder_buf_state = {
+			.valid = true,
+			.head_sn = 100,
+			.num_entries = 1,
+			.entries[0] = {
+				.sn = 100,
+				.add_subframes = 1,
+			},
+		},
+		.expected = {
+			/* 1. Reorder buffer contains a 2-sub frames A-MSDU
+			 *    at SN=100.
+			 * 2. RX packet additional SN=100 A-MSDU subframe,
+			 *    but not the last one
+			 * No packets are released and head_sn is not updated.
+			 */
+			.reorder_res = IWL_MLD_BUFFERED_SKB,
+			.num_stored = 3,
+			.head_sn = 100,
+		},
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(test_reorder_buffer, reorder_buffer_cases, desc);
+
+static struct sk_buff_head g_released_skbs;
+static u16 g_num_released_skbs;
+
+/* Add released skbs from reorder buffer to a global list; This allows us
+ * to verify the correct release order of packets after they pass through the
+ * simulated reorder logic.
+ */
+static void
+fake_iwl_mld_pass_packet_to_mac80211(struct iwl_mld *mld,
+				     struct napi_struct *napi,
+				     struct sk_buff *skb, int queue,
+				     struct ieee80211_sta *sta)
+{
+	__skb_queue_tail(&g_released_skbs, skb);
+	g_num_released_skbs++;
+}
+
+static u32
+fake_iwl_mld_fw_sta_id_mask(struct iwl_mld *mld, struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_link_sta *mld_link_sta;
+	u8 link_id;
+	u32 sta_mask = 0;
+
+	/* This is the expectation in the real function */
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* We can't use for_each_sta_active_link */
+	for_each_mld_link_sta(mld_sta, mld_link_sta, link_id)
+		sta_mask |= BIT(mld_link_sta->fw_id);
+	return sta_mask;
+}
+
+static struct iwl_rx_mpdu_desc *setup_mpdu_desc(void)
+{
+	struct kunit *test = kunit_get_current_test();
+	const struct reorder_buffer_case *param =
+		(const void *)(test->param_value);
+	struct iwl_rx_mpdu_desc *mpdu_desc;
+
+	KUNIT_ALLOC_AND_ASSERT(test, mpdu_desc);
+
+	mpdu_desc->reorder_data |=
+		le32_encode_bits(param->rx_pkt.baid,
+				 IWL_RX_MPDU_REORDER_BAID_MASK);
+	mpdu_desc->reorder_data |=
+		le32_encode_bits(param->rx_pkt.sn,
+				 IWL_RX_MPDU_REORDER_SN_MASK);
+	mpdu_desc->reorder_data |=
+		le32_encode_bits(param->rx_pkt.nssn,
+				 IWL_RX_MPDU_REORDER_NSSN_MASK);
+	if (param->rx_pkt.old_sn)
+		mpdu_desc->reorder_data |=
+			cpu_to_le32(IWL_RX_MPDU_REORDER_BA_OLD_SN);
+
+	if (param->rx_pkt.dup)
+		mpdu_desc->status |= cpu_to_le32(IWL_RX_MPDU_STATUS_DUPLICATE);
+
+	if (param->rx_pkt.amsdu) {
+		mpdu_desc->mac_flags2 |= IWL_RX_MPDU_MFLG2_AMSDU;
+		if (param->rx_pkt.last_subframe)
+			mpdu_desc->amsdu_info |=
+				IWL_RX_MPDU_AMSDU_LAST_SUBFRAME;
+	}
+
+	return mpdu_desc;
+}
+
+static struct sk_buff *alloc_and_setup_skb(u16 fc, u16 seq_ctrl, u8 tid,
+					   bool mcast)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct ieee80211_hdr hdr = {
+		.frame_control = cpu_to_le16(fc),
+		.seq_ctrl = cpu_to_le16(seq_ctrl),
+	};
+	struct sk_buff *skb;
+
+	skb = kunit_zalloc_skb(test, 128, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, skb);
+
+	if (ieee80211_is_data_qos(hdr.frame_control)) {
+		u8 *qc = ieee80211_get_qos_ctl(&hdr);
+
+		qc[0] = tid & IEEE80211_QOS_CTL_TID_MASK;
+	}
+
+	if (mcast)
+		hdr.addr1[0] = 0x1;
+
+	skb_set_mac_header(skb, skb->len);
+	skb_put_data(skb, &hdr, ieee80211_hdrlen(hdr.frame_control));
+
+	return skb;
+}
+
+static struct iwl_mld_reorder_buffer *
+setup_reorder_buffer(struct iwl_mld_baid_data *baid_data)
+{
+	struct kunit *test = kunit_get_current_test();
+	const struct reorder_buffer_case *param =
+		(const void *)(test->param_value);
+	struct iwl_mld_reorder_buffer *buffer = baid_data->reorder_buf;
+	struct iwl_mld_reorder_buf_entry *entries = baid_data->entries;
+	struct sk_buff *fake_skb;
+
+	buffer->valid = param->reorder_buf_state.valid;
+	buffer->head_sn = param->reorder_buf_state.head_sn;
+	buffer->queue = QUEUE;
+
+	for (int i = 0; i < baid_data->buf_size; i++)
+		__skb_queue_head_init(&entries[i].frames);
+
+	for (int i = 0; i < param->reorder_buf_state.num_entries; i++) {
+		u16 sn = param->reorder_buf_state.entries[i].sn;
+		int index = sn % baid_data->buf_size;
+		u8 add_subframes =
+			param->reorder_buf_state.entries[i].add_subframes;
+		/* create 1 skb per entry + additional skbs per num of
+		 * requested subframes
+		 */
+		u8 num_skbs = 1 + add_subframes;
+
+		for (int j = 0; j < num_skbs; j++) {
+			fake_skb = alloc_and_setup_skb(FC_QOS_DATA, sn, 0,
+						       false);
+			__skb_queue_tail(&entries[index].frames, fake_skb);
+			buffer->num_stored++;
+		}
+	}
+
+	return buffer;
+}
+
+static struct iwl_mld_reorder_buffer *setup_ba_data(struct ieee80211_sta *sta)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	const struct reorder_buffer_case *param =
+		(const void *)(test->param_value);
+	struct iwl_mld_baid_data *baid_data = NULL;
+	struct iwl_mld_reorder_buffer *buffer;
+	u32 reorder_buf_size = BA_WINDOW_SIZE * sizeof(baid_data->entries[0]);
+	u8 baid = param->reorder_buf_state.baid;
+
+	/* Assuming only 1 RXQ */
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, baid_data,
+				    sizeof(*baid_data) + reorder_buf_size);
+
+	baid_data->baid = baid;
+	baid_data->tid = param->rx_pkt.tid;
+	baid_data->buf_size = BA_WINDOW_SIZE;
+
+	wiphy_lock(mld->wiphy);
+	baid_data->sta_mask = iwl_mld_fw_sta_id_mask(mld, sta);
+	wiphy_unlock(mld->wiphy);
+
+	baid_data->entries_per_queue = BA_WINDOW_SIZE;
+
+	buffer = setup_reorder_buffer(baid_data);
+
+	KUNIT_EXPECT_NULL(test, rcu_access_pointer(mld->fw_id_to_ba[baid]));
+	rcu_assign_pointer(mld->fw_id_to_ba[baid], baid_data);
+
+	return buffer;
+}
+
+static void test_reorder_buffer(struct kunit *test)
+{
+	struct iwl_mld *mld = test->priv;
+	const struct reorder_buffer_case *param =
+		(const void *)(test->param_value);
+	struct iwl_rx_mpdu_desc *mpdu_desc;
+	struct ieee80211_vif *vif;
+	struct ieee80211_sta *sta;
+	struct sk_buff *skb;
+	struct iwl_mld_reorder_buffer *buffer;
+	enum iwl_mld_reorder_result reorder_res;
+	u16 skb_release_order_count = param->expected.skb_release_order_count;
+	u16 skb_idx = 0;
+
+	/* Init globals and activate stubs */
+	__skb_queue_head_init(&g_released_skbs);
+	g_num_released_skbs = 0;
+	kunit_activate_static_stub(test, iwl_mld_fw_sta_id_mask,
+				   fake_iwl_mld_fw_sta_id_mask);
+	kunit_activate_static_stub(test, iwl_mld_pass_packet_to_mac80211,
+				   fake_iwl_mld_pass_packet_to_mac80211);
+
+	vif = iwlmld_kunit_add_vif(false, NL80211_IFTYPE_STATION);
+	sta = iwlmld_kunit_setup_sta(vif, IEEE80211_STA_AUTHORIZED, -1);
+
+	/* Prepare skb, mpdu_desc, BA data and the reorder buffer */
+	skb = alloc_and_setup_skb(param->rx_pkt.fc, param->rx_pkt.sn,
+				  param->rx_pkt.tid, param->rx_pkt.multicast);
+	buffer = setup_ba_data(sta);
+	mpdu_desc = setup_mpdu_desc();
+
+	rcu_read_lock();
+	reorder_res = iwl_mld_reorder(mld, NULL, QUEUE, sta, skb, mpdu_desc);
+	rcu_read_unlock();
+
+	KUNIT_ASSERT_EQ(test, reorder_res, param->expected.reorder_res);
+	KUNIT_ASSERT_EQ(test, buffer->num_stored, param->expected.num_stored);
+	KUNIT_ASSERT_EQ(test, buffer->head_sn, param->expected.head_sn);
+
+	/* Verify skbs release order */
+	KUNIT_ASSERT_EQ(test, skb_release_order_count, g_num_released_skbs);
+	while ((skb = __skb_dequeue(&g_released_skbs))) {
+		struct ieee80211_hdr *hdr = (void *)skb_mac_header(skb);
+
+		KUNIT_ASSERT_EQ(test, le16_to_cpu(hdr->seq_ctrl),
+				param->expected.skb_release_order[skb_idx]);
+		skb_idx++;
+	}
+	KUNIT_ASSERT_EQ(test, skb_idx, skb_release_order_count);
+}
+
+static struct kunit_case reorder_buffer_test_cases[] = {
+	KUNIT_CASE_PARAM(test_reorder_buffer, test_reorder_buffer_gen_params),
+	{},
+};
+
+static struct kunit_suite reorder_buffer = {
+	.name = "iwlmld-reorder-buffer",
+	.test_cases = reorder_buffer_test_cases,
+	.init = iwlmld_kunit_test_init,
+};
+
+kunit_test_suite(reorder_buffer);
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/hcmd.c b/sys/contrib/dev/iwlwifi/mld/tests/hcmd.c
new file mode 100644
index 000000000000..0e3b9417dd63
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/hcmd.c
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for channel helper functions
+ *
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <kunit/test.h>
+
+#include <iwl-trans.h>
+#include "mld.h"
+
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+static void test_hcmd_names_sorted(struct kunit *test)
+{
+	int i;
+
+	for (i = 0; i < global_iwl_mld_goups_size; i++) {
+		const struct iwl_hcmd_arr *arr = &iwl_mld_groups[i];
+		int j;
+
+		if (!arr->arr)
+			continue;
+		for (j = 0; j < arr->size - 1; j++)
+			KUNIT_EXPECT_LE(test, arr->arr[j].cmd_id,
+					arr->arr[j + 1].cmd_id);
+	}
+}
+
+static void test_hcmd_names_for_rx(struct kunit *test)
+{
+	static struct iwl_trans t = {
+		.conf.command_groups = iwl_mld_groups,
+	};
+
+	t.conf.command_groups_size = global_iwl_mld_goups_size;
+
+	for (unsigned int i = 0; i < iwl_mld_rx_handlers_num; i++) {
+		const struct iwl_rx_handler *rxh;
+		const char *name;
+
+		rxh = &iwl_mld_rx_handlers[i];
+
+		name = iwl_get_cmd_string(&t, rxh->cmd_id);
+		KUNIT_EXPECT_NOT_NULL(test, name);
+		KUNIT_EXPECT_NE_MSG(test, strcmp(name, "UNKNOWN"), 0,
+				    "ID 0x%04x is UNKNOWN", rxh->cmd_id);
+	}
+}
+
+static struct kunit_case hcmd_names_cases[] = {
+	KUNIT_CASE(test_hcmd_names_sorted),
+	KUNIT_CASE(test_hcmd_names_for_rx),
+	{},
+};
+
+static struct kunit_suite hcmd_names = {
+	.name = "iwlmld-hcmd-names",
+	.test_cases = hcmd_names_cases,
+};
+
+kunit_test_suite(hcmd_names);
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/link-selection.c b/sys/contrib/dev/iwlwifi/mld/tests/link-selection.c
new file mode 100644
index 000000000000..766c24db3613
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/link-selection.c
@@ -0,0 +1,339 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for link selection functions
+ *
+ * Copyright (C) 2025 Intel Corporation
+ */
+#include <kunit/static_stub.h>
+
+#include "utils.h"
+#include "mld.h"
+#include "link.h"
+#include "iface.h"
+#include "phy.h"
+#include "mlo.h"
+
+static const struct link_grading_test_case {
+	const char *desc;
+	struct {
+		struct {
+			u8 link_id;
+			const struct cfg80211_chan_def *chandef;
+			bool active;
+			s32 signal;
+			bool has_chan_util_elem;
+			u8 chan_util; /* 0-255 , used only if has_chan_util_elem is true */
+			u8 chan_load_by_us; /* 0-100, used only if active is true */;
+		} link;
+	} input;
+	unsigned int expected_grade;
+} link_grading_cases[] = {
+	{
+		.desc = "channel util of 128 (50%)",
+		.input.link = {
+			.link_id = 0,
+			.chandef = &chandef_2ghz_20mhz,
+			.active = false,
+			.has_chan_util_elem = true,
+			.chan_util = 128,
+		},
+		.expected_grade = 86,
+	},
+	{
+		.desc = "channel util of 180 (70%)",
+		.input.link = {
+			.link_id = 0,
+			.chandef = &chandef_2ghz_20mhz,
+			.active = false,
+			.has_chan_util_elem = true,
+			.chan_util = 180,
+		},
+		.expected_grade = 51,
+	},
+	{
+		.desc = "channel util of 180 (70%), channel load by us of 10%",
+		.input.link = {
+			.link_id = 0,
+			.chandef = &chandef_2ghz_20mhz,
+			.has_chan_util_elem = true,
+			.chan_util = 180,
+			.active = true,
+			.chan_load_by_us = 10,
+		},
+		.expected_grade = 67,
+	},
+		{
+		.desc = "no channel util element",
+		.input.link = {
+			.link_id = 0,
+			.chandef = &chandef_2ghz_20mhz,
+			.active = true,
+		},
+		.expected_grade = 120,
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(link_grading, link_grading_cases, desc);
+
+static void setup_link(struct ieee80211_bss_conf *link)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	const struct link_grading_test_case *test_param =
+		(const void *)(test->param_value);
+
+	KUNIT_ALLOC_AND_ASSERT(test, link->bss);
+
+	link->bss->signal = DBM_TO_MBM(test_param->input.link.signal);
+
+	link->chanreq.oper = *test_param->input.link.chandef;
+
+	if (test_param->input.link.has_chan_util_elem) {
+		struct cfg80211_bss_ies *ies;
+		struct ieee80211_bss_load_elem bss_load = {
+			.channel_util = test_param->input.link.chan_util,
+		};
+		struct element *elem =
+			iwlmld_kunit_gen_element(WLAN_EID_QBSS_LOAD,
+						 &bss_load,
+						 sizeof(bss_load));
+		unsigned int elem_len = sizeof(*elem) + sizeof(bss_load);
+
+		KUNIT_ALLOC_AND_ASSERT_SIZE(test, ies, sizeof(*ies) + elem_len);
+		memcpy(ies->data, elem, elem_len);
+		ies->len = elem_len;
+		rcu_assign_pointer(link->bss->beacon_ies, ies);
+		rcu_assign_pointer(link->bss->ies, ies);
+	}
+
+	if (test_param->input.link.active) {
+		struct ieee80211_chanctx_conf *chan_ctx =
+			wiphy_dereference(mld->wiphy, link->chanctx_conf);
+		struct iwl_mld_phy *phy;
+
+		KUNIT_ASSERT_NOT_NULL(test, chan_ctx);
+
+		phy = iwl_mld_phy_from_mac80211(chan_ctx);
+
+		phy->channel_load_by_us = test_param->input.link.chan_load_by_us;
+	}
+}
+
+static void test_link_grading(struct kunit *test)
+{
+	struct iwl_mld *mld = test->priv;
+	const struct link_grading_test_case *test_param =
+		(const void *)(test->param_value);
+	struct ieee80211_vif *vif;
+	struct ieee80211_bss_conf *link;
+	unsigned int actual_grade;
+	/* Extract test case parameters */
+	u8 link_id = test_param->input.link.link_id;
+	bool active = test_param->input.link.active;
+	u16 valid_links;
+	struct iwl_mld_kunit_link assoc_link = {
+		.chandef = test_param->input.link.chandef,
+	};
+
+	/* If the link is not active, use a different link as the assoc link */
+	if (active) {
+		assoc_link.id = link_id;
+		valid_links = BIT(link_id);
+	} else {
+		assoc_link.id = BIT(ffz(BIT(link_id)));
+		valid_links = BIT(assoc_link.id) | BIT(link_id);
+	}
+
+	vif = iwlmld_kunit_setup_mlo_assoc(valid_links, &assoc_link);
+
+	wiphy_lock(mld->wiphy);
+	link = wiphy_dereference(mld->wiphy, vif->link_conf[link_id]);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	setup_link(link);
+
+	actual_grade = iwl_mld_get_link_grade(mld, link);
+	wiphy_unlock(mld->wiphy);
+
+	/* Assert that the returned grade matches the expected grade */
+	KUNIT_EXPECT_EQ(test, actual_grade, test_param->expected_grade);
+}
+
+static struct kunit_case link_selection_cases[] = {
+	KUNIT_CASE_PARAM(test_link_grading, link_grading_gen_params),
+	{},
+};
+
+static struct kunit_suite link_selection = {
+	.name = "iwlmld-link-selection-tests",
+	.test_cases = link_selection_cases,
+	.init = iwlmld_kunit_test_init,
+};
+
+kunit_test_suite(link_selection);
+
+static const struct link_pair_case {
+	const char *desc;
+	const struct cfg80211_chan_def *chandef_a, *chandef_b;
+	bool low_latency_vif;
+	u32 chan_load_not_by_us;
+	bool primary_link_active;
+	u32 expected_result;
+} link_pair_cases[] = {
+	{
+		.desc = "Unequal bandwidth, primary link inactive, EMLSR not allowed",
+		.low_latency_vif = false,
+		.primary_link_active = false,
+		.chandef_a = &chandef_5ghz_40mhz,
+		.chandef_b = &chandef_6ghz_20mhz,
+		.expected_result = IWL_MLD_EMLSR_EXIT_CHAN_LOAD,
+	},
+	{
+		.desc = "Equal bandwidths, sufficient channel load, EMLSR allowed",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 11,
+		.chandef_a = &chandef_5ghz_40mhz,
+		.chandef_b = &chandef_6ghz_40mhz,
+		.expected_result = 0,
+	},
+	{
+		.desc = "Equal bandwidths, insufficient channel load, EMLSR not allowed",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 6,
+		.chandef_a = &chandef_5ghz_80mhz,
+		.chandef_b = &chandef_6ghz_80mhz,
+		.expected_result = IWL_MLD_EMLSR_EXIT_CHAN_LOAD,
+	},
+	{
+		.desc = "Low latency VIF, sufficient channel load, EMLSR allowed",
+		.low_latency_vif = true,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 6,
+		.chandef_a = &chandef_5ghz_160mhz,
+		.chandef_b = &chandef_6ghz_160mhz,
+		.expected_result = 0,
+	},
+	{
+		.desc = "Different bandwidths (2x ratio), primary link load permits EMLSR",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 30,
+		.chandef_a = &chandef_5ghz_40mhz,
+		.chandef_b = &chandef_6ghz_20mhz,
+		.expected_result = 0,
+	},
+	{
+		.desc = "Different bandwidths (4x ratio), primary link load permits EMLSR",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 45,
+		.chandef_a = &chandef_5ghz_80mhz,
+		.chandef_b = &chandef_6ghz_20mhz,
+		.expected_result = 0,
+	},
+	{
+		.desc = "Different bandwidths (16x ratio), primary link load insufficient",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 45,
+		.chandef_a = &chandef_6ghz_320mhz,
+		.chandef_b = &chandef_5ghz_20mhz,
+		.expected_result = IWL_MLD_EMLSR_EXIT_CHAN_LOAD,
+	},
+	{
+		.desc = "Same band not allowed (2.4 GHz)",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 30,
+		.chandef_a = &chandef_2ghz_20mhz,
+		.chandef_b = &chandef_2ghz_11_20mhz,
+		.expected_result = IWL_MLD_EMLSR_EXIT_EQUAL_BAND,
+	},
+	{
+		.desc = "Same band not allowed (5 GHz)",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 30,
+		.chandef_a = &chandef_5ghz_40mhz,
+		.chandef_b = &chandef_5ghz_40mhz,
+		.expected_result = IWL_MLD_EMLSR_EXIT_EQUAL_BAND,
+	},
+	{
+		.desc = "Same band allowed (5 GHz separated)",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 30,
+		.chandef_a = &chandef_5ghz_40mhz,
+		.chandef_b = &chandef_5ghz_120_40mhz,
+		.expected_result = 0,
+	},
+	{
+		.desc = "Same band not allowed (6 GHz)",
+		.low_latency_vif = false,
+		.primary_link_active = true,
+		.chan_load_not_by_us = 30,
+		.chandef_a = &chandef_6ghz_160mhz,
+		.chandef_b = &chandef_6ghz_221_160mhz,
+		.expected_result = IWL_MLD_EMLSR_EXIT_EQUAL_BAND,
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(link_pair, link_pair_cases, desc);
+
+static void test_iwl_mld_link_pair_allows_emlsr(struct kunit *test)
+{
+	const struct link_pair_case *params = test->param_value;
+	struct iwl_mld *mld = test->priv;
+	struct ieee80211_vif *vif;
+	/* link A is the primary and link B is the secondary */
+	struct iwl_mld_link_sel_data a = {
+		.chandef = params->chandef_a,
+		.link_id = 4,
+	};
+	struct iwl_mld_link_sel_data b = {
+		.chandef = params->chandef_b,
+		.link_id = 5,
+	};
+	struct iwl_mld_kunit_link assoc_link = {
+		.chandef = params->primary_link_active ? a.chandef : b.chandef,
+		.id = params->primary_link_active ? a.link_id : b.link_id,
+	};
+	u32 result;
+
+	vif = iwlmld_kunit_setup_mlo_assoc(BIT(a.link_id) | BIT(b.link_id),
+					   &assoc_link);
+
+	if (params->low_latency_vif)
+		iwl_mld_vif_from_mac80211(vif)->low_latency_causes = 1;
+
+	wiphy_lock(mld->wiphy);
+
+	/* Simulate channel load */
+	if (params->primary_link_active) {
+		struct iwl_mld_phy *phy =
+			iwlmld_kunit_get_phy_of_link(vif, a.link_id);
+
+		phy->avg_channel_load_not_by_us = params->chan_load_not_by_us;
+	}
+
+	result = iwl_mld_emlsr_pair_state(vif, &a, &b);
+
+	wiphy_unlock(mld->wiphy);
+
+	KUNIT_EXPECT_EQ(test, result, params->expected_result);
+}
+
+static struct kunit_case link_pair_criteria_test_cases[] = {
+	KUNIT_CASE_PARAM(test_iwl_mld_link_pair_allows_emlsr, link_pair_gen_params),
+	{}
+};
+
+static struct kunit_suite link_pair_criteria_tests = {
+	.name = "iwlmld_link_pair_allows_emlsr",
+	.test_cases = link_pair_criteria_test_cases,
+	.init = iwlmld_kunit_test_init,
+};
+
+kunit_test_suite(link_pair_criteria_tests);
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/link.c b/sys/contrib/dev/iwlwifi/mld/tests/link.c
new file mode 100644
index 000000000000..69a0d67858bf
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/link.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for channel helper functions
+ *
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <kunit/static_stub.h>
+
+#include "utils.h"
+#include "mld.h"
+#include "link.h"
+#include "iface.h"
+#include "fw/api/mac-cfg.h"
+
+static const struct missed_beacon_test_case {
+	const char *desc;
+	struct {
+		struct iwl_missed_beacons_notif notif;
+		bool emlsr;
+	} input;
+	struct {
+		bool disconnected;
+		bool emlsr;
+	} output;
+} missed_beacon_cases[] = {
+	{
+		.desc = "no EMLSR, no disconnect",
+		.input.notif = {
+			.consec_missed_beacons = cpu_to_le32(4),
+		},
+	},
+	{
+		.desc = "no EMLSR, no beacon loss since Rx, no disconnect",
+		.input.notif = {
+			.consec_missed_beacons = cpu_to_le32(20),
+		},
+	},
+	{
+		.desc = "no EMLSR, beacon loss since Rx, disconnect",
+		.input.notif = {
+			.consec_missed_beacons = cpu_to_le32(20),
+			.consec_missed_beacons_since_last_rx =
+				cpu_to_le32(10),
+		},
+		.output.disconnected = true,
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(test_missed_beacon, missed_beacon_cases, desc);
+
+static void fake_ieee80211_connection_loss(struct ieee80211_vif *vif)
+{
+	vif->cfg.assoc = false;
+}
+
+static void test_missed_beacon(struct kunit *test)
+{
+	struct iwl_mld *mld = test->priv;
+	struct iwl_missed_beacons_notif *notif;
+	const struct missed_beacon_test_case *test_param =
+		(const void *)(test->param_value);
+	struct ieee80211_vif *vif;
+	struct iwl_rx_packet *pkt;
+	struct iwl_mld_kunit_link link1 = {
+		.id = 0,
+		.chandef = &chandef_6ghz_160mhz,
+	};
+	struct iwl_mld_kunit_link link2 = {
+		.id = 1,
+		.chandef = &chandef_5ghz_80mhz,
+	};
+
+	kunit_activate_static_stub(test, ieee80211_connection_loss,
+				   fake_ieee80211_connection_loss);
+	pkt = iwl_mld_kunit_create_pkt(test_param->input.notif);
+	notif = (void *)pkt->data;
+
+	if (test_param->input.emlsr) {
+		vif = iwlmld_kunit_assoc_emlsr(&link1, &link2);
+	} else {
+		struct iwl_mld_vif *mld_vif;
+
+		vif = iwlmld_kunit_setup_non_mlo_assoc(&link1);
+		mld_vif = iwl_mld_vif_from_mac80211(vif);
+		notif->link_id = cpu_to_le32(mld_vif->deflink.fw_id);
+	}
+
+	wiphy_lock(mld->wiphy);
+
+	iwl_mld_handle_missed_beacon_notif(mld, pkt);
+
+	wiphy_unlock(mld->wiphy);
+
+	KUNIT_ASSERT_NE(test, vif->cfg.assoc, test_param->output.disconnected);
+
+	/* TODO: add test cases for esr and check */
+}
+
+static struct kunit_case link_cases[] = {
+	KUNIT_CASE_PARAM(test_missed_beacon, test_missed_beacon_gen_params),
+	{},
+};
+
+static struct kunit_suite link = {
+	.name = "iwlmld-link",
+	.test_cases = link_cases,
+	.init = iwlmld_kunit_test_init,
+};
+
+kunit_test_suite(link);
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/module.c b/sys/contrib/dev/iwlwifi/mld/tests/module.c
new file mode 100644
index 000000000000..5d9818587b23
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/module.c
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * This is just module boilerplate for the iwlmld kunit module.
+ *
+ * Copyright (C) 2024 Intel Corporation
+ */
+#include <linux/module.h>
+
+MODULE_IMPORT_NS("IWLWIFI");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("kunit tests for iwlmld");
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/rx.c b/sys/contrib/dev/iwlwifi/mld/tests/rx.c
new file mode 100644
index 000000000000..20cb4e03ab41
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/rx.c
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for channel helper functions
+ *
+ * Copyright (C) 2024 Intel Corporation
+ */
+#include <kunit/test.h>
+#include "utils.h"
+#include "iwl-trans.h"
+#include "mld.h"
+#include "sta.h"
+
+static const struct is_dup_case {
+	const char *desc;
+	struct {
+		/* ieee80211_hdr fields */
+		__le16 fc;
+		__le16 seq;
+		u8 tid;
+		bool multicast;
+		/* iwl_rx_mpdu_desc fields */
+		bool is_amsdu;
+		u8 sub_frame_idx;
+	} rx_pkt;
+	struct {
+		__le16 last_seq;
+		u8 last_sub_frame_idx;
+		u8 tid;
+	} dup_data_state;
+	struct {
+		bool is_dup;
+		u32 rx_status_flag;
+	} result;
+} is_dup_cases[] = {
+	{
+		.desc = "Control frame",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_CTL),
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = 0,
+		}
+	},
+	{
+		.desc = "Null func frame",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_NULLFUNC),
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = 0,
+		}
+	},
+	{
+		.desc = "Multicast data",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA),
+			.multicast = true,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = 0,
+		}
+	},
+	{
+		.desc = "QoS null func frame",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_NULLFUNC),
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = 0,
+		}
+	},
+	{
+		.desc = "QoS data new sequence",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA),
+			.seq = __cpu_to_le16(0x101),
+		},
+		.dup_data_state = {
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = RX_FLAG_DUP_VALIDATED,
+		},
+	},
+	{
+		.desc = "QoS data same sequence, no retry",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA),
+			.seq = __cpu_to_le16(0x100),
+		},
+		.dup_data_state = {
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = RX_FLAG_DUP_VALIDATED,
+		},
+	},
+	{
+		.desc = "QoS data same sequence, has retry",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA |
+					    IEEE80211_FCTL_RETRY),
+			.seq = __cpu_to_le16(0x100),
+		},
+		.dup_data_state = {
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = true,
+			.rx_status_flag = 0,
+		},
+	},
+	{
+		.desc = "QoS data invalid tid",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA),
+			.seq = __cpu_to_le16(0x100),
+			.tid = IWL_MAX_TID_COUNT + 1,
+		},
+		.result = {
+			.is_dup = true,
+			.rx_status_flag = 0,
+		},
+	},
+	{
+		.desc = "non-QoS data, same sequence, same tid, no retry",
+		.rx_pkt = {
+			/* Driver will use tid = IWL_MAX_TID_COUNT */
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA),
+			.seq = __cpu_to_le16(0x100),
+		},
+		.dup_data_state = {
+			.tid = IWL_MAX_TID_COUNT,
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = RX_FLAG_DUP_VALIDATED,
+		},
+	},
+	{
+		.desc = "non-QoS data, same sequence, same tid, has retry",
+		.rx_pkt = {
+			/* Driver will use tid = IWL_MAX_TID_COUNT */
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_FCTL_RETRY),
+			.seq = __cpu_to_le16(0x100),
+		},
+		.dup_data_state = {
+			.tid = IWL_MAX_TID_COUNT,
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = true,
+			.rx_status_flag = 0,
+		},
+	},
+	{
+		.desc = "non-QoS data, same sequence on different tid's",
+		.rx_pkt = {
+			/* Driver will use tid = IWL_MAX_TID_COUNT */
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA),
+			.seq = __cpu_to_le16(0x100),
+		},
+		.dup_data_state = {
+			.tid = 7,
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = RX_FLAG_DUP_VALIDATED,
+		},
+	},
+	{
+		.desc = "A-MSDU new subframe, allow same PN",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA),
+			.seq = __cpu_to_le16(0x100),
+			.is_amsdu = true,
+			.sub_frame_idx = 1,
+		},
+		.dup_data_state = {
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = RX_FLAG_ALLOW_SAME_PN |
+					  RX_FLAG_DUP_VALIDATED,
+		},
+	},
+	{
+		.desc = "A-MSDU subframe with smaller idx, disallow same PN",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA),
+			.seq = __cpu_to_le16(0x100),
+			.is_amsdu = true,
+			.sub_frame_idx = 1,
+		},
+		.dup_data_state = {
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 2,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = RX_FLAG_DUP_VALIDATED,
+		},
+	},
+	{
+		.desc = "A-MSDU same subframe, no retry, disallow same PN",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA),
+			.seq = __cpu_to_le16(0x100),
+			.is_amsdu = true,
+			.sub_frame_idx = 0,
+		},
+		.dup_data_state = {
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = false,
+			.rx_status_flag = RX_FLAG_DUP_VALIDATED,
+		},
+	},
+	{
+		.desc = "A-MSDU same subframe, has retry",
+		.rx_pkt = {
+			.fc = __cpu_to_le16(IEEE80211_FTYPE_DATA |
+					    IEEE80211_STYPE_QOS_DATA |
+					    IEEE80211_FCTL_RETRY),
+			.seq = __cpu_to_le16(0x100),
+			.is_amsdu = true,
+			.sub_frame_idx = 0,
+		},
+		.dup_data_state = {
+			.last_seq = __cpu_to_le16(0x100),
+			.last_sub_frame_idx = 0,
+		},
+		.result = {
+			.is_dup = true,
+			.rx_status_flag = 0,
+		},
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(test_is_dup, is_dup_cases, desc);
+
+static void
+setup_dup_data_state(struct ieee80211_sta *sta)
+{
+	struct kunit *test = kunit_get_current_test();
+	const struct is_dup_case *param = (const void *)(test->param_value);
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	u8 tid = param->dup_data_state.tid;
+	struct iwl_mld_rxq_dup_data *dup_data;
+
+	/* Allocate dup_data only for 1 queue */
+	KUNIT_ALLOC_AND_ASSERT(test, dup_data);
+
+	/* Initialize dup data, see iwl_mld_alloc_dup_data */
+	memset(dup_data->last_seq, 0xff, sizeof(dup_data->last_seq));
+
+	dup_data->last_seq[tid] = param->dup_data_state.last_seq;
+	dup_data->last_sub_frame_idx[tid] =
+		param->dup_data_state.last_sub_frame_idx;
+
+	mld_sta->dup_data = dup_data;
+}
+
+static void setup_rx_pkt(const struct is_dup_case *param,
+			 struct ieee80211_hdr *hdr,
+			 struct iwl_rx_mpdu_desc *mpdu_desc)
+{
+	u8 tid = param->rx_pkt.tid;
+
+	/* Set "new rx packet" header */
+	hdr->frame_control = param->rx_pkt.fc;
+	hdr->seq_ctrl = param->rx_pkt.seq;
+
+	if (ieee80211_is_data_qos(hdr->frame_control)) {
+		u8 *qc = ieee80211_get_qos_ctl(hdr);
+
+		qc[0] = tid & IEEE80211_QOS_CTL_TID_MASK;
+	}
+
+	if (param->rx_pkt.multicast)
+		hdr->addr1[0] = 0x1;
+
+	/* Set mpdu_desc */
+	mpdu_desc->amsdu_info = param->rx_pkt.sub_frame_idx &
+				IWL_RX_MPDU_AMSDU_SUBFRAME_IDX_MASK;
+	if (param->rx_pkt.is_amsdu)
+		mpdu_desc->mac_flags2 |= IWL_RX_MPDU_MFLG2_AMSDU;
+}
+
+static void test_is_dup(struct kunit *test)
+{
+	const struct is_dup_case *param = (const void *)(test->param_value);
+	struct iwl_mld *mld = test->priv;
+	struct iwl_rx_mpdu_desc mpdu_desc = { };
+	struct ieee80211_rx_status rx_status = { };
+	struct ieee80211_vif *vif;
+	struct ieee80211_sta *sta;
+	struct ieee80211_hdr hdr;
+
+	vif = iwlmld_kunit_add_vif(false, NL80211_IFTYPE_STATION);
+	sta = iwlmld_kunit_setup_sta(vif, IEEE80211_STA_AUTHORIZED, -1);
+
+	/* Prepare test case state */
+	setup_dup_data_state(sta);
+	setup_rx_pkt(param, &hdr, &mpdu_desc);
+
+	KUNIT_EXPECT_EQ(test,
+			iwl_mld_is_dup(mld, sta, &hdr, &mpdu_desc, &rx_status,
+				       0), /* assuming only 1 queue */
+			param->result.is_dup);
+	KUNIT_EXPECT_EQ(test, rx_status.flag, param->result.rx_status_flag);
+}
+
+static struct kunit_case is_dup_test_cases[] = {
+	KUNIT_CASE_PARAM(test_is_dup, test_is_dup_gen_params),
+	{},
+};
+
+static struct kunit_suite is_dup = {
+	.name = "iwlmld-rx-is-dup",
+	.test_cases = is_dup_test_cases,
+	.init = iwlmld_kunit_test_init,
+};
+
+kunit_test_suite(is_dup);
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/utils.c b/sys/contrib/dev/iwlwifi/mld/tests/utils.c
new file mode 100644
index 000000000000..26cf27be762d
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/utils.c
@@ -0,0 +1,503 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for channel helper functions
+ *
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#include <kunit/test.h>
+#include <kunit/test-bug.h>
+
+#include "utils.h"
+
+#include <linux/device.h>
+
+#include "fw/api/scan.h"
+#include "fw/api/mac-cfg.h"
+#include "iwl-trans.h"
+#include "mld.h"
+#include "iface.h"
+#include "link.h"
+#include "phy.h"
+#include "sta.h"
+
+int iwlmld_kunit_test_init(struct kunit *test)
+{
+	struct iwl_mld *mld;
+	struct iwl_trans *trans;
+	const struct iwl_rf_cfg *cfg;
+	struct iwl_fw *fw;
+	struct ieee80211_hw *hw;
+
+	KUNIT_ALLOC_AND_ASSERT(test, trans);
+	KUNIT_ALLOC_AND_ASSERT(test, trans->dev);
+	KUNIT_ALLOC_AND_ASSERT(test, cfg);
+	KUNIT_ALLOC_AND_ASSERT(test, fw);
+	KUNIT_ALLOC_AND_ASSERT(test, hw);
+	KUNIT_ALLOC_AND_ASSERT(test, hw->wiphy);
+
+	mutex_init(&hw->wiphy->mtx);
+
+	/* Allocate and initialize the mld structure */
+	KUNIT_ALLOC_AND_ASSERT(test, mld);
+	iwl_construct_mld(mld, trans, cfg, fw, hw, NULL);
+
+	fw->ucode_capa.num_stations = IWL_STATION_COUNT_MAX;
+	fw->ucode_capa.num_links = IWL_FW_MAX_LINK_ID + 1;
+
+	mld->fwrt.trans = trans;
+	mld->fwrt.fw = fw;
+	mld->fwrt.dev = trans->dev;
+
+	/* TODO: add priv_size to hw allocation and setup hw->priv to enable
+	 * testing mac80211 callbacks
+	 */
+
+	KUNIT_ALLOC_AND_ASSERT(test, mld->nvm_data);
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, mld->scan.cmd,
+				    sizeof(struct iwl_scan_req_umac_v17));
+	mld->scan.cmd_size = sizeof(struct iwl_scan_req_umac_v17);
+
+	/* This is not the state at the end of the regular opmode_start,
+	 * but it is more common to need it. Explicitly undo this if needed.
+	 */
+	mld->trans->state = IWL_TRANS_FW_ALIVE;
+	mld->fw_status.running = true;
+
+	/* Avoid passing mld struct around */
+	test->priv = mld;
+	return 0;
+}
+
+IWL_MLD_ALLOC_FN(link, bss_conf)
+
+static void iwlmld_kunit_init_link(struct ieee80211_vif *vif,
+				   struct ieee80211_bss_conf *link,
+				   struct iwl_mld_link *mld_link, int link_id)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	/* setup mac80211 link */
+	rcu_assign_pointer(vif->link_conf[link_id], link);
+	link->link_id = link_id;
+	link->vif = vif;
+	link->beacon_int = 100;
+	link->dtim_period = 3;
+	link->qos = true;
+
+	/* and mld_link */
+	ret = iwl_mld_allocate_link_fw_id(mld, &mld_link->fw_id, link);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	rcu_assign_pointer(mld_vif->link[link_id], mld_link);
+	rcu_assign_pointer(vif->link_conf[link_id], link);
+}
+
+IWL_MLD_ALLOC_FN(vif, vif)
+
+/* Helper function to add and initialize a VIF for KUnit tests */
+struct ieee80211_vif *iwlmld_kunit_add_vif(bool mlo, enum nl80211_iftype type)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	struct ieee80211_vif *vif;
+	struct iwl_mld_vif *mld_vif;
+	int ret;
+
+	/* TODO: support more types */
+	KUNIT_ASSERT_EQ(test, type, NL80211_IFTYPE_STATION);
+
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, vif,
+				    sizeof(*vif) + sizeof(*mld_vif));
+
+	vif->type = type;
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+	mld_vif->mld = mld;
+
+	ret = iwl_mld_allocate_vif_fw_id(mld, &mld_vif->fw_id, vif);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* TODO: revisit (task=EHT) */
+	if (mlo)
+		return vif;
+
+	/* Initialize the default link */
+	iwlmld_kunit_init_link(vif, &vif->bss_conf, &mld_vif->deflink, 0);
+
+	return vif;
+}
+
+/* Use only for MLO vif */
+struct ieee80211_bss_conf *
+iwlmld_kunit_add_link(struct ieee80211_vif *vif, int link_id)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct ieee80211_bss_conf *link;
+	struct iwl_mld_link *mld_link;
+
+	KUNIT_ALLOC_AND_ASSERT(test, link);
+	KUNIT_ALLOC_AND_ASSERT(test, mld_link);
+
+	iwlmld_kunit_init_link(vif, link, mld_link, link_id);
+	vif->valid_links |= BIT(link_id);
+
+	return link;
+}
+
+struct ieee80211_chanctx_conf *
+iwlmld_kunit_add_chanctx(const struct cfg80211_chan_def *def)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	struct ieee80211_chanctx_conf *ctx;
+	struct iwl_mld_phy *phy;
+	int fw_id;
+
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, ctx, sizeof(*ctx) + sizeof(*phy));
+
+	/* Setup the chanctx conf */
+	ctx->def = *def;
+	ctx->min_def = *def;
+	ctx->ap = *def;
+
+	/* and the iwl_mld_phy */
+	phy = iwl_mld_phy_from_mac80211(ctx);
+
+	fw_id = iwl_mld_allocate_fw_phy_id(mld);
+	KUNIT_ASSERT_GE(test, fw_id, 0);
+
+	phy->fw_id = fw_id;
+	phy->mld = mld;
+	phy->chandef = *def;
+
+	return ctx;
+}
+
+void iwlmld_kunit_assign_chanctx_to_link(struct ieee80211_vif *vif,
+					 struct ieee80211_bss_conf *link,
+					 struct ieee80211_chanctx_conf *ctx)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	struct iwl_mld_link *mld_link;
+
+	KUNIT_EXPECT_NULL(test, rcu_access_pointer(link->chanctx_conf));
+	rcu_assign_pointer(link->chanctx_conf, ctx);
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	mld_link = iwl_mld_link_from_mac80211(link);
+
+	KUNIT_EXPECT_NULL(test, rcu_access_pointer(mld_link->chan_ctx));
+	KUNIT_EXPECT_FALSE(test, mld_link->active);
+
+	rcu_assign_pointer(mld_link->chan_ctx, ctx);
+	mld_link->active = true;
+
+	if (ieee80211_vif_is_mld(vif))
+		vif->active_links |= BIT(link->link_id);
+}
+
+IWL_MLD_ALLOC_FN(link_sta, link_sta)
+
+static void iwlmld_kunit_add_link_sta(struct ieee80211_sta *sta,
+				      struct ieee80211_link_sta *link_sta,
+				      struct iwl_mld_link_sta *mld_link_sta,
+				      u8 link_id)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld *mld = test->priv;
+	u8 fw_id;
+	int ret;
+
+	/* initialize mac80211's link_sta */
+	link_sta->link_id = link_id;
+	rcu_assign_pointer(sta->link[link_id], link_sta);
+
+	link_sta->sta = sta;
+
+	/* and the iwl_mld_link_sta */
+	ret = iwl_mld_allocate_link_sta_fw_id(mld, &fw_id, link_sta);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	mld_link_sta->fw_id = fw_id;
+
+	rcu_assign_pointer(mld_sta->link[link_id], mld_link_sta);
+}
+
+static struct ieee80211_link_sta *
+iwlmld_kunit_alloc_link_sta(struct ieee80211_sta *sta, int link_id)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct ieee80211_link_sta *link_sta;
+	struct iwl_mld_link_sta *mld_link_sta;
+
+	/* Only valid for MLO */
+	KUNIT_ASSERT_TRUE(test, sta->valid_links);
+
+	KUNIT_ALLOC_AND_ASSERT(test, link_sta);
+	KUNIT_ALLOC_AND_ASSERT(test, mld_link_sta);
+
+	iwlmld_kunit_add_link_sta(sta, link_sta, mld_link_sta, link_id);
+
+	sta->valid_links |= BIT(link_id);
+
+	return link_sta;
+}
+
+/* Allocate and initialize a STA with the first link_sta */
+static struct ieee80211_sta *
+iwlmld_kunit_add_sta(struct ieee80211_vif *vif, int link_id)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct ieee80211_sta *sta;
+	struct iwl_mld_sta *mld_sta;
+
+	/* Allocate memory for ieee80211_sta with embedded iwl_mld_sta */
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, sta, sizeof(*sta) + sizeof(*mld_sta));
+
+	/* TODO: allocate and initialize the TXQs ? */
+
+	mld_sta = iwl_mld_sta_from_mac80211(sta);
+	mld_sta->vif = vif;
+	mld_sta->mld = test->priv;
+
+	/* TODO: adjust for internal stations */
+	mld_sta->sta_type = STATION_TYPE_PEER;
+
+	if (link_id >= 0) {
+		iwlmld_kunit_add_link_sta(sta, &sta->deflink,
+					  &mld_sta->deflink, link_id);
+		sta->valid_links = BIT(link_id);
+	} else {
+		iwlmld_kunit_add_link_sta(sta, &sta->deflink,
+					  &mld_sta->deflink, 0);
+	}
+	return sta;
+}
+
+/* Move s STA to a state */
+static void iwlmld_kunit_move_sta_state(struct ieee80211_vif *vif,
+					struct ieee80211_sta *sta,
+					enum ieee80211_sta_state state)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld_sta *mld_sta;
+	struct iwl_mld_vif *mld_vif;
+
+	/* The sta will be removed automatically at the end of the test */
+	KUNIT_ASSERT_NE(test, state, IEEE80211_STA_NOTEXIST);
+
+	mld_sta = iwl_mld_sta_from_mac80211(sta);
+	mld_sta->sta_state = state;
+
+	mld_vif = iwl_mld_vif_from_mac80211(mld_sta->vif);
+	mld_vif->authorized = state == IEEE80211_STA_AUTHORIZED;
+
+	if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls)
+		mld_vif->ap_sta = sta;
+}
+
+struct ieee80211_sta *iwlmld_kunit_setup_sta(struct ieee80211_vif *vif,
+					     enum ieee80211_sta_state state,
+					     int link_id)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct ieee80211_sta *sta;
+
+	/* The sta will be removed automatically at the end of the test */
+	KUNIT_ASSERT_NE(test, state, IEEE80211_STA_NOTEXIST);
+
+	/* First - allocate and init the STA */
+	sta = iwlmld_kunit_add_sta(vif, link_id);
+
+	/* Now move it all the way to the wanted state */
+	for (enum ieee80211_sta_state _state = IEEE80211_STA_NONE;
+	     _state <= state; _state++)
+		iwlmld_kunit_move_sta_state(vif, sta, state);
+
+	return sta;
+}
+
+static void iwlmld_kunit_set_vif_associated(struct ieee80211_vif *vif)
+{
+	/* TODO: setup chanreq */
+	/* TODO setup capabilities */
+
+	vif->cfg.assoc = 1;
+}
+
+static struct ieee80211_vif *
+iwlmld_kunit_setup_assoc(bool mlo, struct iwl_mld_kunit_link *assoc_link)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	struct ieee80211_vif *vif;
+	struct ieee80211_bss_conf *link;
+	struct ieee80211_chanctx_conf *chan_ctx;
+
+	KUNIT_ASSERT_TRUE(test, mlo || assoc_link->id == 0);
+
+	vif = iwlmld_kunit_add_vif(mlo, NL80211_IFTYPE_STATION);
+
+	if (mlo)
+		link = iwlmld_kunit_add_link(vif, assoc_link->id);
+	else
+		link = &vif->bss_conf;
+
+	chan_ctx = iwlmld_kunit_add_chanctx(assoc_link->chandef);
+
+	wiphy_lock(mld->wiphy);
+	iwlmld_kunit_assign_chanctx_to_link(vif, link, chan_ctx);
+	wiphy_unlock(mld->wiphy);
+
+	/* The AP sta will now be pointer to by mld_vif->ap_sta */
+	iwlmld_kunit_setup_sta(vif, IEEE80211_STA_AUTHORIZED, assoc_link->id);
+
+	iwlmld_kunit_set_vif_associated(vif);
+
+	return vif;
+}
+
+struct ieee80211_vif *
+iwlmld_kunit_setup_mlo_assoc(u16 valid_links,
+			     struct iwl_mld_kunit_link *assoc_link)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct ieee80211_vif *vif;
+
+	KUNIT_ASSERT_TRUE(test,
+			  hweight16(valid_links) == 1 ||
+			  hweight16(valid_links) == 2);
+	KUNIT_ASSERT_TRUE(test, valid_links & BIT(assoc_link->id));
+
+	vif = iwlmld_kunit_setup_assoc(true, assoc_link);
+
+	/* Add the other link, if applicable */
+	if (hweight16(valid_links) > 1) {
+		u8 other_link_id = ffs(valid_links & ~BIT(assoc_link->id)) - 1;
+
+		iwlmld_kunit_add_link(vif, other_link_id);
+	}
+
+	return vif;
+}
+
+struct ieee80211_vif *
+iwlmld_kunit_setup_non_mlo_assoc(struct iwl_mld_kunit_link *assoc_link)
+{
+	return iwlmld_kunit_setup_assoc(false, assoc_link);
+}
+
+struct iwl_rx_packet *
+_iwl_mld_kunit_create_pkt(const void *notif, size_t notif_sz)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_rx_packet *pkt;
+
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, pkt, sizeof(pkt) + notif_sz);
+
+	memcpy(pkt->data, notif, notif_sz);
+	pkt->len_n_flags = cpu_to_le32(sizeof(pkt->hdr) + notif_sz);
+
+	return pkt;
+}
+
+struct ieee80211_vif *iwlmld_kunit_assoc_emlsr(struct iwl_mld_kunit_link *link1,
+					       struct iwl_mld_kunit_link *link2)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	struct ieee80211_vif *vif;
+	struct ieee80211_bss_conf *link;
+	struct ieee80211_chanctx_conf *chan_ctx;
+	struct ieee80211_sta *sta;
+	struct iwl_mld_vif *mld_vif;
+	u16 valid_links = BIT(link1->id) | BIT(link2->id);
+
+	KUNIT_ASSERT_TRUE(test, hweight16(valid_links) == 2);
+
+	vif = iwlmld_kunit_setup_mlo_assoc(valid_links, link1);
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	/* Activate second link */
+	wiphy_lock(mld->wiphy);
+
+	link = wiphy_dereference(mld->wiphy, vif->link_conf[link2->id]);
+	KUNIT_EXPECT_NOT_NULL(test, link);
+
+	chan_ctx = iwlmld_kunit_add_chanctx(link2->chandef);
+	iwlmld_kunit_assign_chanctx_to_link(vif, link, chan_ctx);
+
+	wiphy_unlock(mld->wiphy);
+
+	/* And other link sta */
+	sta = mld_vif->ap_sta;
+	KUNIT_EXPECT_NOT_NULL(test, sta);
+
+	iwlmld_kunit_alloc_link_sta(sta, link2->id);
+
+	return vif;
+}
+
+struct element *iwlmld_kunit_gen_element(u8 id, const void *data, size_t len)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct element *elem;
+
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, elem, sizeof(*elem) + len);
+
+	elem->id = id;
+	elem->datalen = len;
+	memcpy(elem->data, data, len);
+
+	return elem;
+}
+
+struct iwl_mld_phy *iwlmld_kunit_get_phy_of_link(struct ieee80211_vif *vif,
+						 u8 link_id)
+{
+	struct kunit *test = kunit_get_current_test();
+	struct iwl_mld *mld = test->priv;
+	struct ieee80211_chanctx_conf *chanctx;
+	struct ieee80211_bss_conf *link =
+		wiphy_dereference(mld->wiphy, vif->link_conf[link_id]);
+
+	KUNIT_EXPECT_NOT_NULL(test, link);
+
+	chanctx = wiphy_dereference(mld->wiphy, link->chanctx_conf);
+	KUNIT_EXPECT_NOT_NULL(test, chanctx);
+
+	return iwl_mld_phy_from_mac80211(chanctx);
+}
+
+static const struct chandef_case {
+	const char *desc;
+	const struct cfg80211_chan_def *chandef;
+} chandef_cases[] = {
+#define CHANDEF(c, ...) { .desc = "chandef " #c " valid", .chandef = &c, },
+	CHANDEF_LIST
+#undef CHANDEF
+};
+
+KUNIT_ARRAY_PARAM_DESC(chandef, chandef_cases, desc);
+
+static void test_iwl_mld_chandef_valid(struct kunit *test)
+{
+	const struct chandef_case *params = test->param_value;
+
+	KUNIT_EXPECT_EQ(test, true, cfg80211_chandef_valid(params->chandef));
+}
+
+static struct kunit_case chandef_test_cases[] = {
+	KUNIT_CASE_PARAM(test_iwl_mld_chandef_valid, chandef_gen_params),
+	{}
+};
+
+static struct kunit_suite chandef_tests = {
+	.name = "iwlmld_valid_test_chandefs",
+	.test_cases = chandef_test_cases,
+};
+
+kunit_test_suite(chandef_tests);
diff --git a/sys/contrib/dev/iwlwifi/mld/tests/utils.h b/sys/contrib/dev/iwlwifi/mld/tests/utils.h
new file mode 100644
index 000000000000..edf8eef4e81a
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tests/utils.h
@@ -0,0 +1,140 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#ifndef __iwl_mld_kunit_utils_h__
+#define __iwl_mld_kunit_utils_h__
+
+#include <net/mac80211.h>
+#include <kunit/test-bug.h>
+
+struct iwl_mld;
+
+int iwlmld_kunit_test_init(struct kunit *test);
+
+struct iwl_mld_kunit_link {
+	const struct cfg80211_chan_def *chandef;
+	u8 id;
+};
+
+enum nl80211_iftype;
+
+struct ieee80211_vif *iwlmld_kunit_add_vif(bool mlo, enum nl80211_iftype type);
+
+struct ieee80211_bss_conf *
+iwlmld_kunit_add_link(struct ieee80211_vif *vif, int link_id);
+
+#define KUNIT_ALLOC_AND_ASSERT_SIZE(test, ptr, size)			\
+do {									\
+	(ptr) = kunit_kzalloc((test), (size), GFP_KERNEL);		\
+	KUNIT_ASSERT_NOT_NULL((test), (ptr));				\
+} while (0)
+
+#define KUNIT_ALLOC_AND_ASSERT(test, ptr)				\
+	KUNIT_ALLOC_AND_ASSERT_SIZE(test, ptr, sizeof(*(ptr)))
+
+#define CHANNEL(_name, _band, _freq)				\
+static struct ieee80211_channel _name = {			\
+	.band = (_band),					\
+	.center_freq = (_freq),					\
+	.hw_value = (_freq),					\
+}
+
+CHANNEL(chan_2ghz, NL80211_BAND_2GHZ, 2412);
+CHANNEL(chan_2ghz_11, NL80211_BAND_2GHZ, 2462);
+CHANNEL(chan_5ghz, NL80211_BAND_5GHZ, 5200);
+CHANNEL(chan_5ghz_120, NL80211_BAND_5GHZ, 5600);
+CHANNEL(chan_6ghz, NL80211_BAND_6GHZ, 6115);
+CHANNEL(chan_6ghz_221, NL80211_BAND_6GHZ, 7055);
+/* Feel free to add more */
+#undef CHANNEL
+
+#define CHANDEF_LIST \
+	CHANDEF(chandef_2ghz_20mhz, chan_2ghz, 2412,		\
+		NL80211_CHAN_WIDTH_20)				\
+	CHANDEF(chandef_2ghz_40mhz, chan_2ghz, 2422,		\
+		NL80211_CHAN_WIDTH_40)				\
+	CHANDEF(chandef_2ghz_11_20mhz, chan_2ghz_11, 2462,	\
+		NL80211_CHAN_WIDTH_20)				\
+	CHANDEF(chandef_5ghz_20mhz, chan_5ghz, 5200,		\
+		NL80211_CHAN_WIDTH_20)				\
+	CHANDEF(chandef_5ghz_40mhz, chan_5ghz, 5210,		\
+		NL80211_CHAN_WIDTH_40)				\
+	CHANDEF(chandef_5ghz_80mhz, chan_5ghz, 5210,		\
+		NL80211_CHAN_WIDTH_80)				\
+	CHANDEF(chandef_5ghz_160mhz, chan_5ghz, 5250,		\
+		NL80211_CHAN_WIDTH_160)				\
+	CHANDEF(chandef_5ghz_120_40mhz, chan_5ghz_120, 5610,	\
+		NL80211_CHAN_WIDTH_40)				\
+	CHANDEF(chandef_6ghz_20mhz, chan_6ghz, 6115,		\
+		NL80211_CHAN_WIDTH_20)				\
+	CHANDEF(chandef_6ghz_40mhz, chan_6ghz, 6125,		\
+		NL80211_CHAN_WIDTH_40)				\
+	CHANDEF(chandef_6ghz_80mhz, chan_6ghz, 6145,		\
+		NL80211_CHAN_WIDTH_80)				\
+	CHANDEF(chandef_6ghz_160mhz, chan_6ghz, 6185,		\
+		NL80211_CHAN_WIDTH_160)				\
+	CHANDEF(chandef_6ghz_320mhz, chan_6ghz, 6105,		\
+		NL80211_CHAN_WIDTH_320)				\
+	CHANDEF(chandef_6ghz_221_160mhz, chan_6ghz_221, 6985,	\
+		NL80211_CHAN_WIDTH_160)				\
+	/* Feel free to add more */
+
+#define CHANDEF(_name, _channel, _freq1, _width)		\
+__maybe_unused static const struct cfg80211_chan_def _name = {	\
+	.chan = &(_channel),					\
+	.center_freq1 = (_freq1),				\
+	.width = (_width),					\
+};
+CHANDEF_LIST
+#undef CHANDEF
+
+struct ieee80211_chanctx_conf *
+iwlmld_kunit_add_chanctx(const struct cfg80211_chan_def *def);
+
+void iwlmld_kunit_assign_chanctx_to_link(struct ieee80211_vif *vif,
+					 struct ieee80211_bss_conf *link,
+					 struct ieee80211_chanctx_conf *ctx);
+
+/* Allocate a sta, initialize it and move it to the wanted state */
+struct ieee80211_sta *iwlmld_kunit_setup_sta(struct ieee80211_vif *vif,
+					     enum ieee80211_sta_state state,
+					     int link_id);
+
+struct ieee80211_vif *
+iwlmld_kunit_setup_mlo_assoc(u16 valid_links,
+			     struct iwl_mld_kunit_link *assoc_link);
+
+struct ieee80211_vif *
+iwlmld_kunit_setup_non_mlo_assoc(struct iwl_mld_kunit_link *assoc_link);
+
+struct iwl_rx_packet *
+_iwl_mld_kunit_create_pkt(const void *notif, size_t notif_sz);
+
+#define iwl_mld_kunit_create_pkt(_notif)	\
+	_iwl_mld_kunit_create_pkt(&(_notif), sizeof(_notif))
+
+struct ieee80211_vif *
+iwlmld_kunit_assoc_emlsr(struct iwl_mld_kunit_link *link1,
+			 struct iwl_mld_kunit_link *link2);
+
+struct element *iwlmld_kunit_gen_element(u8 id, const void *data, size_t len);
+
+/**
+ * iwlmld_kunit_get_phy_of_link - Get the phy of a link
+ *
+ * @vif: The vif to get the phy from.
+ * @link_id: The id of the link to get the phy for.
+ *
+ * given a vif and link id, return the phy pointer of that link.
+ * This assumes that the link exists, and that it had a chanctx
+ * assigned.
+ * If this is not the case, the test will fail.
+ *
+ * Return: phy pointer.
+ */
+struct iwl_mld_phy *iwlmld_kunit_get_phy_of_link(struct ieee80211_vif *vif,
+						 u8 link_id);
+
+#endif /* __iwl_mld_kunit_utils_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/thermal.c b/sys/contrib/dev/iwlwifi/mld/thermal.c
new file mode 100644
index 000000000000..f8a8c35066be
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/thermal.c
@@ -0,0 +1,467 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+#ifdef CONFIG_THERMAL
+#include <linux/sort.h>
+#include <linux/thermal.h>
+#endif
+
+#include "fw/api/phy.h"
+
+#include "thermal.h"
+#include "mld.h"
+#include "hcmd.h"
+
+#define IWL_MLD_NUM_CTDP_STEPS		20
+#define IWL_MLD_MIN_CTDP_BUDGET_MW	150
+
+#define IWL_MLD_CT_KILL_DURATION (5 * HZ)
+
+void iwl_mld_handle_ct_kill_notif(struct iwl_mld *mld,
+				  struct iwl_rx_packet *pkt)
+{
+	const struct ct_kill_notif *notif = (const void *)pkt->data;
+
+	IWL_ERR(mld,
+		"CT Kill notification: temp = %d, DTS = 0x%x, Scheme 0x%x - Enter CT Kill\n",
+		le16_to_cpu(notif->temperature), notif->dts,
+		notif->scheme);
+
+	iwl_mld_set_ctkill(mld, true);
+
+	wiphy_delayed_work_queue(mld->wiphy, &mld->ct_kill_exit_wk,
+				 round_jiffies_relative(IWL_MLD_CT_KILL_DURATION));
+}
+
+static void iwl_mld_exit_ctkill(struct wiphy *wiphy, struct wiphy_work *wk)
+{
+	struct iwl_mld *mld;
+
+	mld = container_of(wk, struct iwl_mld, ct_kill_exit_wk.work);
+
+	IWL_ERR(mld, "Exit CT Kill\n");
+	iwl_mld_set_ctkill(mld, false);
+}
+
+void iwl_mld_handle_temp_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt)
+{
+	const struct iwl_dts_measurement_notif *notif =
+		(const void *)pkt->data;
+	int temp;
+	u32 ths_crossed;
+
+	temp = le32_to_cpu(notif->temp);
+
+	/* shouldn't be negative, but since it's s32, make sure it isn't */
+	if (IWL_FW_CHECK(mld, temp < 0, "negative temperature %d\n", temp))
+		return;
+
+	ths_crossed = le32_to_cpu(notif->threshold_idx);
+
+	/* 0xFF in ths_crossed means the notification is not related
+	 * to a trip, so we can ignore it here.
+	 */
+	if (ths_crossed == 0xFF)
+		return;
+
+	IWL_DEBUG_TEMP(mld, "Temp = %d Threshold crossed = %d\n",
+		       temp, ths_crossed);
+
+	if (IWL_FW_CHECK(mld, ths_crossed >= IWL_MAX_DTS_TRIPS,
+			 "bad threshold: %d\n", ths_crossed))
+		return;
+
+#ifdef CONFIG_THERMAL
+	if (mld->tzone)
+		thermal_zone_device_update(mld->tzone, THERMAL_TRIP_VIOLATED);
+#endif /* CONFIG_THERMAL */
+}
+
+#ifdef CONFIG_THERMAL
+static int iwl_mld_get_temp(struct iwl_mld *mld, s32 *temp)
+{
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(PHY_OPS_GROUP, CMD_DTS_MEASUREMENT_TRIGGER_WIDE),
+		.flags = CMD_WANT_SKB,
+	};
+	const struct iwl_dts_measurement_resp *resp;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	if (ret) {
+		IWL_ERR(mld,
+			"Failed to send the temperature measurement command (err=%d)\n",
+			ret);
+		return ret;
+	}
+
+	if (iwl_rx_packet_payload_len(cmd.resp_pkt) < sizeof(*resp)) {
+		IWL_ERR(mld,
+			"Failed to get a valid response to DTS measurement\n");
+		ret = -EIO;
+		goto free_resp;
+	}
+
+	resp = (const void *)cmd.resp_pkt->data;
+	*temp = le32_to_cpu(resp->temp);
+
+	IWL_DEBUG_TEMP(mld,
+		       "Got temperature measurement response: temp=%d\n",
+		       *temp);
+
+free_resp:
+	iwl_free_resp(&cmd);
+	return ret;
+}
+
+static int compare_temps(const void *a, const void *b)
+{
+	return ((s16)le16_to_cpu(*(const __le16 *)a) -
+		(s16)le16_to_cpu(*(const __le16 *)b));
+}
+
+struct iwl_trip_walk_data {
+	__le16 *thresholds;
+	int count;
+};
+
+static int iwl_trip_temp_iter(struct thermal_trip *trip, void *arg)
+{
+	struct iwl_trip_walk_data *twd = arg;
+
+	if (trip->temperature == THERMAL_TEMP_INVALID)
+		return 0;
+
+	twd->thresholds[twd->count++] =
+		cpu_to_le16((s16)(trip->temperature / 1000));
+	return 0;
+}
+#endif
+
+int iwl_mld_config_temp_report_ths(struct iwl_mld *mld)
+{
+	struct temp_report_ths_cmd cmd = {0};
+	int ret;
+#ifdef CONFIG_THERMAL
+	struct iwl_trip_walk_data twd = {
+		.thresholds = cmd.thresholds,
+		.count = 0
+	};
+
+	if (!mld->tzone)
+		goto send;
+
+	/* The thermal core holds an array of temperature trips that are
+	 * unsorted and uncompressed, the FW should get it compressed and
+	 * sorted.
+	 */
+
+	/* compress trips to cmd array, remove uninitialized values*/
+	for_each_thermal_trip(mld->tzone, iwl_trip_temp_iter, &twd);
+
+	cmd.num_temps = cpu_to_le32(twd.count);
+	if (twd.count)
+		sort(cmd.thresholds, twd.count, sizeof(s16),
+		     compare_temps, NULL);
+
+send:
+#endif
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(PHY_OPS_GROUP,
+						TEMP_REPORTING_THRESHOLDS_CMD),
+				   &cmd);
+	if (ret)
+		IWL_ERR(mld, "TEMP_REPORT_THS_CMD command failed (err=%d)\n",
+			ret);
+
+	return ret;
+}
+
+#ifdef CONFIG_THERMAL
+static int iwl_mld_tzone_get_temp(struct thermal_zone_device *device,
+				  int *temperature)
+{
+	struct iwl_mld *mld = thermal_zone_device_priv(device);
+	int temp;
+	int ret = 0;
+
+	wiphy_lock(mld->wiphy);
+
+	if (!mld->fw_status.running) {
+		/* Tell the core that there is no valid temperature value to
+		 * return, but it need not worry about this.
+		 */
+		*temperature = THERMAL_TEMP_INVALID;
+		goto unlock;
+	}
+
+	ret = iwl_mld_get_temp(mld, &temp);
+	if (ret)
+		goto unlock;
+
+	*temperature = temp * 1000;
+unlock:
+	wiphy_unlock(mld->wiphy);
+	return ret;
+}
+
+static int iwl_mld_tzone_set_trip_temp(struct thermal_zone_device *device,
+				       const struct thermal_trip *trip,
+				       int temp)
+{
+	struct iwl_mld *mld = thermal_zone_device_priv(device);
+	int ret;
+
+	wiphy_lock(mld->wiphy);
+
+	if (!mld->fw_status.running) {
+		ret = -EIO;
+		goto unlock;
+	}
+
+	if ((temp / 1000) > S16_MAX) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	ret = iwl_mld_config_temp_report_ths(mld);
+unlock:
+	wiphy_unlock(mld->wiphy);
+	return ret;
+}
+
+static  struct thermal_zone_device_ops tzone_ops = {
+	.get_temp = iwl_mld_tzone_get_temp,
+	.set_trip_temp = iwl_mld_tzone_set_trip_temp,
+};
+
+static void iwl_mld_thermal_zone_register(struct iwl_mld *mld)
+{
+	int ret;
+	char name[16];
+	static atomic_t counter = ATOMIC_INIT(0);
+	struct thermal_trip trips[IWL_MAX_DTS_TRIPS] = {
+		[0 ... IWL_MAX_DTS_TRIPS - 1] = {
+			.temperature = THERMAL_TEMP_INVALID,
+			.type = THERMAL_TRIP_PASSIVE,
+			.flags = THERMAL_TRIP_FLAG_RW_TEMP,
+		},
+	};
+
+	BUILD_BUG_ON(ARRAY_SIZE(name) >= THERMAL_NAME_LENGTH);
+
+	sprintf(name, "iwlwifi_%u", atomic_inc_return(&counter) & 0xFF);
+	mld->tzone =
+		thermal_zone_device_register_with_trips(name, trips,
+							IWL_MAX_DTS_TRIPS,
+							mld, &tzone_ops,
+							NULL, 0, 0);
+	if (IS_ERR(mld->tzone)) {
+		IWL_DEBUG_TEMP(mld,
+			       "Failed to register to thermal zone (err = %ld)\n",
+			       PTR_ERR(mld->tzone));
+		mld->tzone = NULL;
+		return;
+	}
+
+	ret = thermal_zone_device_enable(mld->tzone);
+	if (ret) {
+		IWL_DEBUG_TEMP(mld, "Failed to enable thermal zone\n");
+		thermal_zone_device_unregister(mld->tzone);
+	}
+}
+
+int iwl_mld_config_ctdp(struct iwl_mld *mld, u32 state,
+			enum iwl_ctdp_cmd_operation op)
+{
+	struct iwl_ctdp_cmd cmd = {
+		.operation = cpu_to_le32(op),
+		.window_size = 0,
+	};
+	u32 budget;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* Do a linear scale from IWL_MLD_MIN_CTDP_BUDGET_MW to the configured
+	 * maximum in the predefined number of steps.
+	 */
+	budget = ((mld->power_budget_mw - IWL_MLD_MIN_CTDP_BUDGET_MW) *
+		  (IWL_MLD_NUM_CTDP_STEPS - 1 - state)) /
+		 (IWL_MLD_NUM_CTDP_STEPS - 1) +
+		 IWL_MLD_MIN_CTDP_BUDGET_MW;
+	cmd.budget = cpu_to_le32(budget);
+
+	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(PHY_OPS_GROUP, CTDP_CONFIG_CMD),
+				   &cmd);
+
+	if (ret) {
+		IWL_ERR(mld, "cTDP command failed (err=%d)\n", ret);
+		return ret;
+	}
+
+	if (op == CTDP_CMD_OPERATION_START)
+		mld->cooling_dev.cur_state = state;
+
+	return 0;
+}
+
+static int iwl_mld_tcool_get_max_state(struct thermal_cooling_device *cdev,
+				       unsigned long *state)
+{
+	*state = IWL_MLD_NUM_CTDP_STEPS - 1;
+
+	return 0;
+}
+
+static int iwl_mld_tcool_get_cur_state(struct thermal_cooling_device *cdev,
+				       unsigned long *state)
+{
+	struct iwl_mld *mld = (struct iwl_mld *)(cdev->devdata);
+
+	*state = mld->cooling_dev.cur_state;
+
+	return 0;
+}
+
+static int iwl_mld_tcool_set_cur_state(struct thermal_cooling_device *cdev,
+				       unsigned long new_state)
+{
+	struct iwl_mld *mld = (struct iwl_mld *)(cdev->devdata);
+	int ret;
+
+	wiphy_lock(mld->wiphy);
+
+	if (!mld->fw_status.running) {
+		ret = -EIO;
+		goto unlock;
+	}
+
+	if (new_state >= IWL_MLD_NUM_CTDP_STEPS) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	ret = iwl_mld_config_ctdp(mld, new_state, CTDP_CMD_OPERATION_START);
+
+unlock:
+	wiphy_unlock(mld->wiphy);
+	return ret;
+}
+
+static const struct thermal_cooling_device_ops tcooling_ops = {
+	.get_max_state = iwl_mld_tcool_get_max_state,
+	.get_cur_state = iwl_mld_tcool_get_cur_state,
+	.set_cur_state = iwl_mld_tcool_set_cur_state,
+};
+
+static void iwl_mld_cooling_device_register(struct iwl_mld *mld)
+{
+	char name[] = "iwlwifi";
+
+	BUILD_BUG_ON(ARRAY_SIZE(name) >= THERMAL_NAME_LENGTH);
+
+	mld->cooling_dev.cdev =
+		thermal_cooling_device_register(name,
+						mld,
+						&tcooling_ops);
+
+	if (IS_ERR(mld->cooling_dev.cdev)) {
+		IWL_DEBUG_TEMP(mld,
+			       "Failed to register to cooling device (err = %ld)\n",
+			       PTR_ERR(mld->cooling_dev.cdev));
+		mld->cooling_dev.cdev = NULL;
+		return;
+	}
+}
+
+static void iwl_mld_thermal_zone_unregister(struct iwl_mld *mld)
+{
+	if (!mld->tzone)
+		return;
+
+	IWL_DEBUG_TEMP(mld, "Thermal zone device unregister\n");
+	if (mld->tzone) {
+		thermal_zone_device_unregister(mld->tzone);
+		mld->tzone = NULL;
+	}
+}
+
+static void iwl_mld_cooling_device_unregister(struct iwl_mld *mld)
+{
+	if (!mld->cooling_dev.cdev)
+		return;
+
+	IWL_DEBUG_TEMP(mld, "Cooling device unregister\n");
+	if (mld->cooling_dev.cdev) {
+		thermal_cooling_device_unregister(mld->cooling_dev.cdev);
+		mld->cooling_dev.cdev = NULL;
+	}
+}
+#endif /* CONFIG_THERMAL */
+
+static u32 iwl_mld_ctdp_get_max_budget(struct iwl_mld *mld)
+{
+	u64 bios_power_budget = 0;
+	u32 default_power_budget;
+
+	switch (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id)) {
+	case IWL_CFG_RF_TYPE_GF:
+		/* dual-radio devices have a higher budget */
+		if (CSR_HW_RFID_IS_CDB(mld->trans->info.hw_rf_id))
+			default_power_budget = 5200;
+		else
+			default_power_budget = 2880;
+		break;
+	case IWL_CFG_RF_TYPE_FM:
+		default_power_budget = 3450;
+		break;
+	case IWL_CFG_RF_TYPE_WH:
+	case IWL_CFG_RF_TYPE_PE:
+	default:
+		default_power_budget = 5550;
+		break;
+	}
+
+	iwl_bios_get_pwr_limit(&mld->fwrt, &bios_power_budget);
+
+	/* 32bit in UEFI, 16bit in ACPI; use BIOS value if it is in range */
+	if (bios_power_budget &&
+	    bios_power_budget != 0xffff && bios_power_budget != 0xffffffff &&
+	    bios_power_budget >= IWL_MLD_MIN_CTDP_BUDGET_MW &&
+	    bios_power_budget <= default_power_budget)
+		return (u32)bios_power_budget;
+
+	return default_power_budget;
+}
+
+void iwl_mld_thermal_initialize(struct iwl_mld *mld)
+{
+	lockdep_assert_not_held(&mld->wiphy->mtx);
+
+	wiphy_delayed_work_init(&mld->ct_kill_exit_wk, iwl_mld_exit_ctkill);
+
+	mld->power_budget_mw = iwl_mld_ctdp_get_max_budget(mld);
+	IWL_DEBUG_TEMP(mld, "cTDP power budget: %d mW\n", mld->power_budget_mw);
+
+#ifdef CONFIG_THERMAL
+	iwl_mld_cooling_device_register(mld);
+	iwl_mld_thermal_zone_register(mld);
+#endif
+}
+
+void iwl_mld_thermal_exit(struct iwl_mld *mld)
+{
+	wiphy_lock(mld->wiphy);
+	wiphy_delayed_work_cancel(mld->wiphy, &mld->ct_kill_exit_wk);
+	wiphy_unlock(mld->wiphy);
+
+#ifdef CONFIG_THERMAL
+	iwl_mld_cooling_device_unregister(mld);
+	iwl_mld_thermal_zone_unregister(mld);
+#endif
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/thermal.h b/sys/contrib/dev/iwlwifi/mld/thermal.h
new file mode 100644
index 000000000000..8c8893331b05
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/thermal.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_thermal_h__
+#define __iwl_mld_thermal_h__
+
+#include "iwl-trans.h"
+
+struct iwl_mld;
+
+#ifdef CONFIG_THERMAL
+#include <linux/thermal.h>
+
+/*
+ * struct iwl_mld_cooling_device
+ * @cur_state: current state
+ * @cdev: struct thermal cooling device
+ */
+struct iwl_mld_cooling_device {
+	u32 cur_state;
+	struct thermal_cooling_device *cdev;
+};
+
+int iwl_mld_config_ctdp(struct iwl_mld *mld, u32 state,
+			enum iwl_ctdp_cmd_operation op);
+#endif
+
+void iwl_mld_handle_temp_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt);
+void iwl_mld_handle_ct_kill_notif(struct iwl_mld *mld,
+				  struct iwl_rx_packet *pkt);
+int iwl_mld_config_temp_report_ths(struct iwl_mld *mld);
+void iwl_mld_thermal_initialize(struct iwl_mld *mld);
+void iwl_mld_thermal_exit(struct iwl_mld *mld);
+
+#endif /* __iwl_mld_thermal_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/time_sync.c b/sys/contrib/dev/iwlwifi/mld/time_sync.c
new file mode 100644
index 000000000000..50799f9bfccb
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/time_sync.c
@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+
+#include "mld.h"
+#include "hcmd.h"
+#include "ptp.h"
+#include "time_sync.h"
+#include <linux/ieee80211.h>
+
+static int iwl_mld_init_time_sync(struct iwl_mld *mld, u32 protocols,
+				  const u8 *addr)
+{
+	struct iwl_mld_time_sync_data *time_sync = kzalloc(sizeof(*time_sync),
+							   GFP_KERNEL);
+
+	if (!time_sync)
+		return -ENOMEM;
+
+	time_sync->active_protocols = protocols;
+	ether_addr_copy(time_sync->peer_addr, addr);
+	skb_queue_head_init(&time_sync->frame_list);
+	rcu_assign_pointer(mld->time_sync, time_sync);
+
+	return 0;
+}
+
+int iwl_mld_time_sync_fw_config(struct iwl_mld *mld)
+{
+	struct iwl_time_sync_cfg_cmd cmd = {};
+	struct iwl_mld_time_sync_data *time_sync;
+	int err;
+
+	time_sync = wiphy_dereference(mld->wiphy, mld->time_sync);
+	if (!time_sync)
+		return -EINVAL;
+
+	cmd.protocols = cpu_to_le32(time_sync->active_protocols);
+	ether_addr_copy(cmd.peer_addr, time_sync->peer_addr);
+
+	err = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(DATA_PATH_GROUP,
+					   WNM_80211V_TIMING_MEASUREMENT_CONFIG_CMD),
+				   &cmd);
+	if (err)
+		IWL_ERR(mld, "Failed to send time sync cfg cmd: %d\n", err);
+
+	return err;
+}
+
+int iwl_mld_time_sync_config(struct iwl_mld *mld, const u8 *addr, u32 protocols)
+{
+	struct iwl_mld_time_sync_data *time_sync;
+	int err;
+
+	time_sync = wiphy_dereference(mld->wiphy, mld->time_sync);
+
+	/* The fw only supports one peer. We do allow reconfiguration of the
+	 * same peer for cases of fw reset etc.
+	 */
+	if (time_sync && time_sync->active_protocols &&
+	    !ether_addr_equal(addr, time_sync->peer_addr)) {
+		IWL_DEBUG_INFO(mld, "Time sync: reject config for peer: %pM\n",
+			       addr);
+		return -ENOBUFS;
+	}
+
+	if (protocols & ~(IWL_TIME_SYNC_PROTOCOL_TM |
+			  IWL_TIME_SYNC_PROTOCOL_FTM))
+		return -EINVAL;
+
+	IWL_DEBUG_INFO(mld, "Time sync: set peer addr=%pM\n", addr);
+
+	iwl_mld_deinit_time_sync(mld);
+	err = iwl_mld_init_time_sync(mld, protocols, addr);
+	if (err)
+		return err;
+
+	err = iwl_mld_time_sync_fw_config(mld);
+	return err;
+}
+
+void iwl_mld_deinit_time_sync(struct iwl_mld *mld)
+{
+	struct iwl_mld_time_sync_data *time_sync =
+		wiphy_dereference(mld->wiphy, mld->time_sync);
+
+	if (!time_sync)
+		return;
+
+	RCU_INIT_POINTER(mld->time_sync, NULL);
+	skb_queue_purge(&time_sync->frame_list);
+	kfree_rcu(time_sync, rcu_head);
+}
+
+bool iwl_mld_time_sync_frame(struct iwl_mld *mld, struct sk_buff *skb, u8 *addr)
+{
+	struct iwl_mld_time_sync_data *time_sync;
+
+	rcu_read_lock();
+	time_sync = rcu_dereference(mld->time_sync);
+	if (time_sync && ether_addr_equal(time_sync->peer_addr, addr) &&
+	    (ieee80211_is_timing_measurement(skb) || ieee80211_is_ftm(skb))) {
+		skb_queue_tail(&time_sync->frame_list, skb);
+		rcu_read_unlock();
+		return true;
+	}
+	rcu_read_unlock();
+
+	return false;
+}
+
+static bool iwl_mld_is_skb_match(struct sk_buff *skb, u8 *addr, u8 dialog_token)
+{
+	struct ieee80211_mgmt *mgmt = (void *)skb->data;
+	u8 skb_dialog_token;
+
+	if (ieee80211_is_timing_measurement(skb))
+		skb_dialog_token = mgmt->u.action.u.wnm_timing_msr.dialog_token;
+	else
+		skb_dialog_token = mgmt->u.action.u.ftm.dialog_token;
+
+	if ((ether_addr_equal(mgmt->sa, addr) ||
+	     ether_addr_equal(mgmt->da, addr)) &&
+	    skb_dialog_token == dialog_token)
+		return true;
+
+	return false;
+}
+
+static struct sk_buff *iwl_mld_time_sync_find_skb(struct iwl_mld *mld, u8 *addr,
+						  u8 dialog_token)
+{
+	struct iwl_mld_time_sync_data *time_sync;
+	struct sk_buff *skb;
+
+	rcu_read_lock();
+
+	time_sync = rcu_dereference(mld->time_sync);
+	if (IWL_FW_CHECK(mld, !time_sync,
+			 "Time sync notification but time sync is not initialized\n")) {
+		rcu_read_unlock();
+		return NULL;
+	}
+
+	/* The notifications are expected to arrive in the same order of the
+	 * frames. If the incoming notification doesn't match the first SKB
+	 * in the queue, it means there was no time sync notification for this
+	 * SKB and it can be dropped.
+	 */
+	while ((skb = skb_dequeue(&time_sync->frame_list))) {
+		if (iwl_mld_is_skb_match(skb, addr, dialog_token))
+			break;
+
+		kfree_skb(skb);
+		skb = NULL;
+		IWL_DEBUG_DROP(mld,
+			       "Time sync: drop SKB without matching notification\n");
+	}
+	rcu_read_unlock();
+
+	return skb;
+}
+
+static u64 iwl_mld_get_64_bit(__le32 high, __le32 low)
+{
+	return ((u64)le32_to_cpu(high) << 32) | le32_to_cpu(low);
+}
+
+void iwl_mld_handle_time_msmt_notif(struct iwl_mld *mld,
+				    struct iwl_rx_packet *pkt)
+{
+	struct ptp_data *data = &mld->ptp_data;
+	struct iwl_time_msmt_notify *notif = (void *)pkt->data;
+	struct ieee80211_rx_status *rx_status;
+	struct skb_shared_hwtstamps *shwt;
+	u64 ts_10ns;
+	struct sk_buff *skb =
+		iwl_mld_time_sync_find_skb(mld, notif->peer_addr,
+					   le32_to_cpu(notif->dialog_token));
+	u64 adj_time;
+
+	if (IWL_FW_CHECK(mld, !skb, "Time sync event but no pending skb\n"))
+		return;
+
+	spin_lock_bh(&data->lock);
+	ts_10ns = iwl_mld_get_64_bit(notif->t2_hi, notif->t2_lo);
+	adj_time = iwl_mld_ptp_get_adj_time(mld, ts_10ns * 10);
+	shwt = skb_hwtstamps(skb);
+	shwt->hwtstamp = ktime_set(0, adj_time);
+
+	ts_10ns = iwl_mld_get_64_bit(notif->t3_hi, notif->t3_lo);
+	adj_time = iwl_mld_ptp_get_adj_time(mld, ts_10ns * 10);
+	rx_status = IEEE80211_SKB_RXCB(skb);
+	rx_status->ack_tx_hwtstamp = ktime_set(0, adj_time);
+	spin_unlock_bh(&data->lock);
+
+	IWL_DEBUG_INFO(mld,
+		       "Time sync: RX event - report frame t2=%llu t3=%llu\n",
+		       ktime_to_ns(shwt->hwtstamp),
+		       ktime_to_ns(rx_status->ack_tx_hwtstamp));
+	ieee80211_rx_napi(mld->hw, NULL, skb, NULL);
+}
+
+void iwl_mld_handle_time_sync_confirm_notif(struct iwl_mld *mld,
+					    struct iwl_rx_packet *pkt)
+{
+	struct ptp_data *data = &mld->ptp_data;
+	struct iwl_time_msmt_cfm_notify *notif = (void *)pkt->data;
+	struct ieee80211_tx_status status = {};
+	struct skb_shared_hwtstamps *shwt;
+	u64 ts_10ns, adj_time;
+
+	status.skb =
+		iwl_mld_time_sync_find_skb(mld, notif->peer_addr,
+					   le32_to_cpu(notif->dialog_token));
+
+	if (IWL_FW_CHECK(mld, !status.skb,
+			 "Time sync confirm but no pending skb\n"))
+		return;
+
+	spin_lock_bh(&data->lock);
+	ts_10ns = iwl_mld_get_64_bit(notif->t1_hi, notif->t1_lo);
+	adj_time = iwl_mld_ptp_get_adj_time(mld, ts_10ns * 10);
+	shwt = skb_hwtstamps(status.skb);
+	shwt->hwtstamp = ktime_set(0, adj_time);
+
+	ts_10ns = iwl_mld_get_64_bit(notif->t4_hi, notif->t4_lo);
+	adj_time = iwl_mld_ptp_get_adj_time(mld, ts_10ns * 10);
+	status.info = IEEE80211_SKB_CB(status.skb);
+	status.ack_hwtstamp = ktime_set(0, adj_time);
+	spin_unlock_bh(&data->lock);
+
+	IWL_DEBUG_INFO(mld,
+		       "Time sync: TX event - report frame t1=%llu t4=%llu\n",
+		       ktime_to_ns(shwt->hwtstamp),
+		       ktime_to_ns(status.ack_hwtstamp));
+	ieee80211_tx_status_ext(mld->hw, &status);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/time_sync.h b/sys/contrib/dev/iwlwifi/mld/time_sync.h
new file mode 100644
index 000000000000..2d4c5512e961
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/time_sync.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+#ifndef __iwl_mld_time_sync_h__
+#define __iwl_mld_time_sync_h__
+
+struct iwl_mld_time_sync_data {
+	struct rcu_head rcu_head;
+	u8 peer_addr[ETH_ALEN];
+	u32 active_protocols;
+	struct sk_buff_head frame_list;
+};
+
+int iwl_mld_time_sync_config(struct iwl_mld *mld, const u8 *addr,
+			     u32 protocols);
+int iwl_mld_time_sync_fw_config(struct iwl_mld *mld);
+void iwl_mld_deinit_time_sync(struct iwl_mld *mld);
+void iwl_mld_handle_time_msmt_notif(struct iwl_mld *mld,
+				    struct iwl_rx_packet *pkt);
+bool iwl_mld_time_sync_frame(struct iwl_mld *mld, struct sk_buff *skb,
+			     u8 *addr);
+void iwl_mld_handle_time_sync_confirm_notif(struct iwl_mld *mld,
+					    struct iwl_rx_packet *pkt);
+
+#endif /* __iwl_mld_time_sync_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/tlc.c b/sys/contrib/dev/iwlwifi/mld/tlc.c
new file mode 100644
index 000000000000..a9ca92c0455e
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tlc.c
@@ -0,0 +1,702 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024-2025 Intel Corporation
+ */
+
+#include <net/mac80211.h>
+
+#include "tlc.h"
+#include "hcmd.h"
+#include "sta.h"
+
+#include "fw/api/rs.h"
+#include "fw/api/context.h"
+#include "fw/api/dhc.h"
+
+static u8 iwl_mld_fw_bw_from_sta_bw(const struct ieee80211_link_sta *link_sta)
+{
+	switch (link_sta->bandwidth) {
+	case IEEE80211_STA_RX_BW_320:
+		return IWL_TLC_MNG_CH_WIDTH_320MHZ;
+	case IEEE80211_STA_RX_BW_160:
+		return IWL_TLC_MNG_CH_WIDTH_160MHZ;
+	case IEEE80211_STA_RX_BW_80:
+		return IWL_TLC_MNG_CH_WIDTH_80MHZ;
+	case IEEE80211_STA_RX_BW_40:
+		return IWL_TLC_MNG_CH_WIDTH_40MHZ;
+	case IEEE80211_STA_RX_BW_20:
+	default:
+		return IWL_TLC_MNG_CH_WIDTH_20MHZ;
+	}
+}
+
+static __le16
+iwl_mld_get_tlc_cmd_flags(struct iwl_mld *mld,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_link_sta *link_sta,
+			  const struct ieee80211_sta_he_cap *own_he_cap,
+			  const struct ieee80211_sta_eht_cap *own_eht_cap)
+{
+	struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
+	struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
+	bool has_vht = vht_cap->vht_supported;
+	u16 flags = 0;
+
+	/* STBC flags */
+	if (mld->cfg->ht_params.stbc &&
+	    (hweight8(iwl_mld_get_valid_tx_ant(mld)) > 1)) {
+		if (he_cap->has_he && he_cap->he_cap_elem.phy_cap_info[2] &
+				      IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
+		else if (vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
+		else if (ht_cap->cap & IEEE80211_HT_CAP_RX_STBC)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
+	}
+
+	/* LDPC */
+	if (mld->cfg->ht_params.ldpc &&
+	    ((ht_cap->cap & IEEE80211_HT_CAP_LDPC_CODING) ||
+	     (has_vht && (vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC))))
+		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+
+	if (he_cap->has_he && (he_cap->he_cap_elem.phy_cap_info[1] &
+	    IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
+		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+
+	if (own_he_cap &&
+	    !(own_he_cap->he_cap_elem.phy_cap_info[1] &
+			IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
+		flags &= ~IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+
+	/* DCM */
+	if (he_cap->has_he &&
+	    (he_cap->he_cap_elem.phy_cap_info[3] &
+	     IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_MASK &&
+	     own_he_cap &&
+	     own_he_cap->he_cap_elem.phy_cap_info[3] &
+			IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_MASK))
+		flags |= IWL_TLC_MNG_CFG_FLAGS_HE_DCM_NSS_1_MSK;
+
+	/* Extra EHT LTF */
+	if (own_eht_cap &&
+	    own_eht_cap->eht_cap_elem.phy_cap_info[5] &
+		IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF &&
+	    link_sta->eht_cap.has_eht &&
+	    link_sta->eht_cap.eht_cap_elem.phy_cap_info[5] &
+	    IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF) {
+		flags |= IWL_TLC_MNG_CFG_FLAGS_EHT_EXTRA_LTF_MSK;
+	}
+
+	return cpu_to_le16(flags);
+}
+
+static u8 iwl_mld_get_fw_chains(struct iwl_mld *mld)
+{
+	u8 chains = iwl_mld_get_valid_tx_ant(mld);
+	u8 fw_chains = 0;
+
+	if (chains & ANT_A)
+		fw_chains |= IWL_TLC_MNG_CHAIN_A_MSK;
+	if (chains & ANT_B)
+		fw_chains |= IWL_TLC_MNG_CHAIN_B_MSK;
+
+	return fw_chains;
+}
+
+static u8 iwl_mld_get_fw_sgi(struct ieee80211_link_sta *link_sta)
+{
+	struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
+	struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
+	u8 sgi_chwidths = 0;
+
+	/* If the association supports HE, HT/VHT rates will never be used for
+	 * Tx and therefor there's no need to set the
+	 * sgi-per-channel-width-support bits
+	 */
+	if (he_cap->has_he)
+		return 0;
+
+	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_20)
+		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_20MHZ);
+	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_40)
+		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_40MHZ);
+	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_80)
+		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_80MHZ);
+	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_160)
+		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_160MHZ);
+
+	return sgi_chwidths;
+}
+
+static int
+iwl_mld_get_highest_fw_mcs(const struct ieee80211_sta_vht_cap *vht_cap,
+			   int nss)
+{
+	u16 rx_mcs = le16_to_cpu(vht_cap->vht_mcs.rx_mcs_map) &
+		(0x3 << (2 * (nss - 1)));
+	rx_mcs >>= (2 * (nss - 1));
+
+	switch (rx_mcs) {
+	case IEEE80211_VHT_MCS_SUPPORT_0_7:
+		return IWL_TLC_MNG_HT_RATE_MCS7;
+	case IEEE80211_VHT_MCS_SUPPORT_0_8:
+		return IWL_TLC_MNG_HT_RATE_MCS8;
+	case IEEE80211_VHT_MCS_SUPPORT_0_9:
+		return IWL_TLC_MNG_HT_RATE_MCS9;
+	default:
+		WARN_ON_ONCE(1);
+		break;
+	}
+
+	return 0;
+}
+
+static void
+iwl_mld_fill_vht_rates(const struct ieee80211_link_sta *link_sta,
+		       const struct ieee80211_sta_vht_cap *vht_cap,
+		       struct iwl_tlc_config_cmd_v4 *cmd)
+{
+	u16 supp;
+	int i, highest_mcs;
+	u8 max_nss = link_sta->rx_nss;
+	struct ieee80211_vht_cap ieee_vht_cap = {
+		.vht_cap_info = cpu_to_le32(vht_cap->cap),
+		.supp_mcs = vht_cap->vht_mcs,
+	};
+
+	/* the station support only a single receive chain */
+	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
+		max_nss = 1;
+
+	for (i = 0; i < max_nss && i < IWL_TLC_NSS_MAX; i++) {
+		int nss = i + 1;
+
+		highest_mcs = iwl_mld_get_highest_fw_mcs(vht_cap, nss);
+		if (!highest_mcs)
+			continue;
+
+		supp = BIT(highest_mcs + 1) - 1;
+		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_20)
+			supp &= ~BIT(IWL_TLC_MNG_HT_RATE_MCS9);
+
+		cmd->ht_rates[i][IWL_TLC_MCS_PER_BW_80] = cpu_to_le16(supp);
+		/* Check if VHT extended NSS indicates that the bandwidth/NSS
+		 * configuration is supported - only for MCS 0 since we already
+		 * decoded the MCS bits anyway ourselves.
+		 */
+		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_160 &&
+		    ieee80211_get_vht_max_nss(&ieee_vht_cap,
+					      IEEE80211_VHT_CHANWIDTH_160MHZ,
+					      0, true, nss) >= nss)
+			cmd->ht_rates[i][IWL_TLC_MCS_PER_BW_160] =
+				cmd->ht_rates[i][IWL_TLC_MCS_PER_BW_80];
+	}
+}
+
+static u16 iwl_mld_he_mac80211_mcs_to_fw_mcs(u16 mcs)
+{
+	switch (mcs) {
+	case IEEE80211_HE_MCS_SUPPORT_0_7:
+		return BIT(IWL_TLC_MNG_HT_RATE_MCS7 + 1) - 1;
+	case IEEE80211_HE_MCS_SUPPORT_0_9:
+		return BIT(IWL_TLC_MNG_HT_RATE_MCS9 + 1) - 1;
+	case IEEE80211_HE_MCS_SUPPORT_0_11:
+		return BIT(IWL_TLC_MNG_HT_RATE_MCS11 + 1) - 1;
+	case IEEE80211_HE_MCS_NOT_SUPPORTED:
+		return 0;
+	}
+
+	WARN(1, "invalid HE MCS %d\n", mcs);
+	return 0;
+}
+
+static void
+iwl_mld_fill_he_rates(const struct ieee80211_link_sta *link_sta,
+		      const struct ieee80211_sta_he_cap *own_he_cap,
+		      struct iwl_tlc_config_cmd_v4 *cmd)
+{
+	const struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
+	u16 mcs_160 = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160);
+	u16 mcs_80 = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80);
+	u16 tx_mcs_80 = le16_to_cpu(own_he_cap->he_mcs_nss_supp.tx_mcs_80);
+	u16 tx_mcs_160 = le16_to_cpu(own_he_cap->he_mcs_nss_supp.tx_mcs_160);
+	int i;
+	u8 nss = link_sta->rx_nss;
+
+	/* the station support only a single receive chain */
+	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
+		nss = 1;
+
+	for (i = 0; i < nss && i < IWL_TLC_NSS_MAX; i++) {
+		u16 _mcs_160 = (mcs_160 >> (2 * i)) & 0x3;
+		u16 _mcs_80 = (mcs_80 >> (2 * i)) & 0x3;
+		u16 _tx_mcs_160 = (tx_mcs_160 >> (2 * i)) & 0x3;
+		u16 _tx_mcs_80 = (tx_mcs_80 >> (2 * i)) & 0x3;
+
+		/* If one side doesn't support - mark both as not supporting */
+		if (_mcs_80 == IEEE80211_HE_MCS_NOT_SUPPORTED ||
+		    _tx_mcs_80 == IEEE80211_HE_MCS_NOT_SUPPORTED) {
+			_mcs_80 = IEEE80211_HE_MCS_NOT_SUPPORTED;
+			_tx_mcs_80 = IEEE80211_HE_MCS_NOT_SUPPORTED;
+		}
+		if (_mcs_80 > _tx_mcs_80)
+			_mcs_80 = _tx_mcs_80;
+		cmd->ht_rates[i][IWL_TLC_MCS_PER_BW_80] =
+			cpu_to_le16(iwl_mld_he_mac80211_mcs_to_fw_mcs(_mcs_80));
+
+		/* If one side doesn't support - mark both as not supporting */
+		if (_mcs_160 == IEEE80211_HE_MCS_NOT_SUPPORTED ||
+		    _tx_mcs_160 == IEEE80211_HE_MCS_NOT_SUPPORTED) {
+			_mcs_160 = IEEE80211_HE_MCS_NOT_SUPPORTED;
+			_tx_mcs_160 = IEEE80211_HE_MCS_NOT_SUPPORTED;
+		}
+		if (_mcs_160 > _tx_mcs_160)
+			_mcs_160 = _tx_mcs_160;
+		cmd->ht_rates[i][IWL_TLC_MCS_PER_BW_160] =
+			cpu_to_le16(iwl_mld_he_mac80211_mcs_to_fw_mcs(_mcs_160));
+	}
+}
+
+static void iwl_mld_set_eht_mcs(__le16 ht_rates[][3],
+				enum IWL_TLC_MCS_PER_BW bw,
+				u8 max_nss, u16 mcs_msk)
+{
+	if (max_nss >= 2)
+		ht_rates[IWL_TLC_NSS_2][bw] |= cpu_to_le16(mcs_msk);
+
+	if (max_nss >= 1)
+		ht_rates[IWL_TLC_NSS_1][bw] |= cpu_to_le16(mcs_msk);
+}
+
+static const
+struct ieee80211_eht_mcs_nss_supp_bw *
+iwl_mld_get_eht_mcs_of_bw(enum IWL_TLC_MCS_PER_BW bw,
+			  const struct ieee80211_eht_mcs_nss_supp *eht_mcs)
+{
+	switch (bw) {
+	case IWL_TLC_MCS_PER_BW_80:
+		return &eht_mcs->bw._80;
+	case IWL_TLC_MCS_PER_BW_160:
+		return &eht_mcs->bw._160;
+	case IWL_TLC_MCS_PER_BW_320:
+		return &eht_mcs->bw._320;
+	default:
+		return NULL;
+	}
+}
+
+static u8 iwl_mld_get_eht_max_nss(u8 rx_nss, u8 tx_nss)
+{
+	u8 tx = u8_get_bits(tx_nss, IEEE80211_EHT_MCS_NSS_TX);
+	u8 rx = u8_get_bits(rx_nss, IEEE80211_EHT_MCS_NSS_RX);
+	/* the max nss that can be used,
+	 * is the min with our tx capa and the peer rx capa.
+	 */
+	return min(tx, rx);
+}
+
+#define MAX_NSS_MCS(mcs_num, rx, tx) \
+	iwl_mld_get_eht_max_nss((rx)->rx_tx_mcs ##mcs_num## _max_nss, \
+				(tx)->rx_tx_mcs ##mcs_num## _max_nss)
+
+static void
+iwl_mld_fill_eht_rates(struct ieee80211_vif *vif,
+		       const struct ieee80211_link_sta *link_sta,
+		       const struct ieee80211_sta_he_cap *own_he_cap,
+		       const struct ieee80211_sta_eht_cap *own_eht_cap,
+		       struct iwl_tlc_config_cmd_v4 *cmd)
+{
+	/* peer RX mcs capa */
+	const struct ieee80211_eht_mcs_nss_supp *eht_rx_mcs =
+		&link_sta->eht_cap.eht_mcs_nss_supp;
+	/* our TX mcs capa */
+	const struct ieee80211_eht_mcs_nss_supp *eht_tx_mcs =
+		&own_eht_cap->eht_mcs_nss_supp;
+
+	enum IWL_TLC_MCS_PER_BW bw;
+	struct ieee80211_eht_mcs_nss_supp_20mhz_only mcs_rx_20;
+	struct ieee80211_eht_mcs_nss_supp_20mhz_only mcs_tx_20;
+
+	/* peer is 20 MHz only */
+	if (vif->type == NL80211_IFTYPE_AP &&
+	    !(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
+	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
+		mcs_rx_20 = eht_rx_mcs->only_20mhz;
+	} else {
+		mcs_rx_20.rx_tx_mcs7_max_nss =
+			eht_rx_mcs->bw._80.rx_tx_mcs9_max_nss;
+		mcs_rx_20.rx_tx_mcs9_max_nss =
+			eht_rx_mcs->bw._80.rx_tx_mcs9_max_nss;
+		mcs_rx_20.rx_tx_mcs11_max_nss =
+			eht_rx_mcs->bw._80.rx_tx_mcs11_max_nss;
+		mcs_rx_20.rx_tx_mcs13_max_nss =
+			eht_rx_mcs->bw._80.rx_tx_mcs13_max_nss;
+	}
+
+	/* NIC is capable of 20 MHz only */
+	if (!(own_he_cap->he_cap_elem.phy_cap_info[0] &
+	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
+		mcs_tx_20 = eht_tx_mcs->only_20mhz;
+	} else {
+		mcs_tx_20.rx_tx_mcs7_max_nss =
+			eht_tx_mcs->bw._80.rx_tx_mcs9_max_nss;
+		mcs_tx_20.rx_tx_mcs9_max_nss =
+			eht_tx_mcs->bw._80.rx_tx_mcs9_max_nss;
+		mcs_tx_20.rx_tx_mcs11_max_nss =
+			eht_tx_mcs->bw._80.rx_tx_mcs11_max_nss;
+		mcs_tx_20.rx_tx_mcs13_max_nss =
+			eht_tx_mcs->bw._80.rx_tx_mcs13_max_nss;
+	}
+
+	/* rates for 20/40/80 MHz */
+	bw = IWL_TLC_MCS_PER_BW_80;
+	iwl_mld_set_eht_mcs(cmd->ht_rates, bw,
+			    MAX_NSS_MCS(7, &mcs_rx_20, &mcs_tx_20),
+			    GENMASK(7, 0));
+	iwl_mld_set_eht_mcs(cmd->ht_rates, bw,
+			    MAX_NSS_MCS(9, &mcs_rx_20, &mcs_tx_20),
+			    GENMASK(9, 8));
+	iwl_mld_set_eht_mcs(cmd->ht_rates, bw,
+			    MAX_NSS_MCS(11, &mcs_rx_20, &mcs_tx_20),
+			    GENMASK(11, 10));
+	iwl_mld_set_eht_mcs(cmd->ht_rates, bw,
+			    MAX_NSS_MCS(13, &mcs_rx_20, &mcs_tx_20),
+			    GENMASK(13, 12));
+
+	/* rates for 160/320 MHz */
+	for (bw = IWL_TLC_MCS_PER_BW_160; bw <= IWL_TLC_MCS_PER_BW_320; bw++) {
+		const struct ieee80211_eht_mcs_nss_supp_bw *mcs_rx =
+			iwl_mld_get_eht_mcs_of_bw(bw, eht_rx_mcs);
+		const struct ieee80211_eht_mcs_nss_supp_bw *mcs_tx =
+			iwl_mld_get_eht_mcs_of_bw(bw, eht_tx_mcs);
+
+		/* got unsupported index for bw */
+		if (!mcs_rx || !mcs_tx)
+			continue;
+
+		/* break out if we don't support the bandwidth */
+		if (cmd->max_ch_width < (bw + IWL_TLC_MNG_CH_WIDTH_80MHZ))
+			break;
+
+		iwl_mld_set_eht_mcs(cmd->ht_rates, bw,
+				    MAX_NSS_MCS(9, mcs_rx, mcs_tx),
+				    GENMASK(9, 0));
+		iwl_mld_set_eht_mcs(cmd->ht_rates, bw,
+				    MAX_NSS_MCS(11, mcs_rx, mcs_tx),
+				    GENMASK(11, 10));
+		iwl_mld_set_eht_mcs(cmd->ht_rates, bw,
+				    MAX_NSS_MCS(13, mcs_rx, mcs_tx),
+				    GENMASK(13, 12));
+	}
+
+	/* the station support only a single receive chain */
+	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC ||
+	    link_sta->rx_nss < 2)
+		memset(cmd->ht_rates[IWL_TLC_NSS_2], 0,
+		       sizeof(cmd->ht_rates[IWL_TLC_NSS_2]));
+}
+
+static void
+iwl_mld_fill_supp_rates(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			struct ieee80211_link_sta *link_sta,
+			struct ieee80211_supported_band *sband,
+			const struct ieee80211_sta_he_cap *own_he_cap,
+			const struct ieee80211_sta_eht_cap *own_eht_cap,
+			struct iwl_tlc_config_cmd_v4 *cmd)
+{
+	int i;
+	u16 non_ht_rates = 0;
+	unsigned long rates_bitmap;
+	const struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+	const struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
+	const struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
+
+	/* non HT rates */
+	rates_bitmap = link_sta->supp_rates[sband->band];
+	for_each_set_bit(i, &rates_bitmap, BITS_PER_LONG)
+		non_ht_rates |= BIT(sband->bitrates[i].hw_value);
+
+	cmd->non_ht_rates = cpu_to_le16(non_ht_rates);
+	cmd->mode = IWL_TLC_MNG_MODE_NON_HT;
+
+	if (link_sta->eht_cap.has_eht && own_he_cap && own_eht_cap) {
+		cmd->mode = IWL_TLC_MNG_MODE_EHT;
+		iwl_mld_fill_eht_rates(vif, link_sta, own_he_cap,
+				       own_eht_cap, cmd);
+	} else if (he_cap->has_he && own_he_cap) {
+		cmd->mode = IWL_TLC_MNG_MODE_HE;
+		iwl_mld_fill_he_rates(link_sta, own_he_cap, cmd);
+	} else if (vht_cap->vht_supported) {
+		cmd->mode = IWL_TLC_MNG_MODE_VHT;
+		iwl_mld_fill_vht_rates(link_sta, vht_cap, cmd);
+	} else if (ht_cap->ht_supported) {
+		cmd->mode = IWL_TLC_MNG_MODE_HT;
+		cmd->ht_rates[IWL_TLC_NSS_1][IWL_TLC_MCS_PER_BW_80] =
+			cpu_to_le16(ht_cap->mcs.rx_mask[0]);
+
+		/* the station support only a single receive chain */
+		if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
+			cmd->ht_rates[IWL_TLC_NSS_2][IWL_TLC_MCS_PER_BW_80] =
+				0;
+		else
+			cmd->ht_rates[IWL_TLC_NSS_2][IWL_TLC_MCS_PER_BW_80] =
+				cpu_to_le16(ht_cap->mcs.rx_mask[1]);
+	}
+}
+
+static void iwl_mld_send_tlc_cmd(struct iwl_mld *mld,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_link_sta *link_sta,
+				 enum nl80211_band band)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	struct ieee80211_supported_band *sband = mld->hw->wiphy->bands[band];
+	const struct ieee80211_sta_he_cap *own_he_cap =
+		ieee80211_get_he_iftype_cap_vif(sband, vif);
+	const struct ieee80211_sta_eht_cap *own_eht_cap =
+		ieee80211_get_eht_iftype_cap_vif(sband, vif);
+	struct iwl_tlc_config_cmd_v4 cmd = {
+		/* For AP mode, use 20 MHz until the STA is authorized */
+		.max_ch_width = mld_sta->sta_state > IEEE80211_STA_ASSOC ?
+			iwl_mld_fw_bw_from_sta_bw(link_sta) :
+			IWL_TLC_MNG_CH_WIDTH_20MHZ,
+		.flags = iwl_mld_get_tlc_cmd_flags(mld, vif, link_sta,
+						   own_he_cap, own_eht_cap),
+		.chains = iwl_mld_get_fw_chains(mld),
+		.sgi_ch_width_supp = iwl_mld_get_fw_sgi(link_sta),
+		.max_mpdu_len = cpu_to_le16(link_sta->agg.max_amsdu_len),
+	};
+	int fw_sta_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
+	int ret;
+
+	if (fw_sta_id < 0)
+		return;
+
+	cmd.sta_id = fw_sta_id;
+
+	iwl_mld_fill_supp_rates(mld, vif, link_sta, sband,
+				own_he_cap, own_eht_cap,
+				&cmd);
+
+	IWL_DEBUG_RATE(mld,
+		       "TLC CONFIG CMD, sta_id=%d, max_ch_width=%d, mode=%d\n",
+		       cmd.sta_id, cmd.max_ch_width, cmd.mode);
+
+	/* Send async since this can be called within a RCU-read section */
+	ret = iwl_mld_send_cmd_with_flags_pdu(mld, WIDE_ID(DATA_PATH_GROUP,
+							   TLC_MNG_CONFIG_CMD),
+					      CMD_ASYNC, &cmd);
+	if (ret)
+		IWL_ERR(mld, "Failed to send TLC cmd (%d)\n", ret);
+}
+
+int iwl_mld_send_tlc_dhc(struct iwl_mld *mld, u8 sta_id, u32 type, u32 data)
+{
+	struct {
+		struct iwl_dhc_cmd dhc;
+		struct iwl_dhc_tlc_cmd tlc;
+	} __packed cmd = {
+		.tlc.sta_id = sta_id,
+		.tlc.type = cpu_to_le32(type),
+		.tlc.data[0] = cpu_to_le32(data),
+		.dhc.length = cpu_to_le32(sizeof(cmd.tlc) >> 2),
+		.dhc.index_and_mask =
+			cpu_to_le32(DHC_TABLE_INTEGRATION | DHC_TARGET_UMAC |
+				    DHC_INTEGRATION_TLC_DEBUG_CONFIG),
+	};
+	int ret;
+
+	ret = iwl_mld_send_cmd_with_flags_pdu(mld,
+					      WIDE_ID(IWL_ALWAYS_LONG_GROUP,
+						      DEBUG_HOST_COMMAND),
+					      CMD_ASYNC, &cmd);
+	IWL_DEBUG_RATE(mld, "sta_id %d, type: 0x%X, value: 0x%X, ret%d\n",
+		       sta_id, type, data, ret);
+	return ret;
+}
+
+void iwl_mld_config_tlc_link(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf,
+			     struct ieee80211_link_sta *link_sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	enum nl80211_band band;
+
+	if (WARN_ON_ONCE(!link_conf->chanreq.oper.chan))
+		return;
+
+	/* Before we have information about a station, configure the A-MSDU RC
+	 * limit such that iwlmd and mac80211 would not be allowed to build
+	 * A-MSDUs.
+	 */
+	if (mld_sta->sta_state < IEEE80211_STA_ASSOC) {
+		link_sta->agg.max_rc_amsdu_len = 1;
+		ieee80211_sta_recalc_aggregates(link_sta->sta);
+	}
+
+	band = link_conf->chanreq.oper.chan->band;
+	iwl_mld_send_tlc_cmd(mld, vif, link_sta, band);
+}
+
+void iwl_mld_config_tlc(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			struct ieee80211_sta *sta)
+{
+	struct ieee80211_bss_conf *link;
+	int link_id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	for_each_vif_active_link(vif, link, link_id) {
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_check(sta, link_id);
+
+		if (!link || !link_sta)
+			continue;
+
+		iwl_mld_config_tlc_link(mld, vif, link, link_sta);
+	}
+}
+
+static u16
+iwl_mld_get_amsdu_size_of_tid(struct iwl_mld *mld,
+			      struct ieee80211_link_sta *link_sta,
+			      unsigned int tid)
+{
+	struct ieee80211_sta *sta = link_sta->sta;
+	struct ieee80211_vif *vif = iwl_mld_sta_from_mac80211(sta)->vif;
+	const u8 tid_to_mac80211_ac[] = {
+		IEEE80211_AC_BE,
+		IEEE80211_AC_BK,
+		IEEE80211_AC_BK,
+		IEEE80211_AC_BE,
+		IEEE80211_AC_VI,
+		IEEE80211_AC_VI,
+		IEEE80211_AC_VO,
+		IEEE80211_AC_VO,
+	};
+	unsigned int result = link_sta->agg.max_rc_amsdu_len;
+	u8 ac, txf, lmac;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* Don't send an AMSDU that will be longer than the TXF.
+	 * Add a security margin of 256 for the TX command + headers.
+	 * We also want to have the start of the next packet inside the
+	 * fifo to be able to send bursts.
+	 */
+
+	if (WARN_ON(tid >= ARRAY_SIZE(tid_to_mac80211_ac)))
+		return 0;
+
+	ac = tid_to_mac80211_ac[tid];
+
+	/* For HE redirect to trigger based fifos */
+	if (link_sta->he_cap.has_he)
+		ac += 4;
+
+	txf = iwl_mld_mac80211_ac_to_fw_tx_fifo(ac);
+
+	/* Only one link: take the lmac according to the band */
+	if (hweight16(sta->valid_links) <= 1) {
+		enum nl80211_band band;
+		struct ieee80211_bss_conf *link =
+			wiphy_dereference(mld->wiphy,
+					  vif->link_conf[link_sta->link_id]);
+
+		if (WARN_ON(!link || !link->chanreq.oper.chan))
+			band = NL80211_BAND_2GHZ;
+		else
+			band = link->chanreq.oper.chan->band;
+		lmac = iwl_mld_get_lmac_id(mld, band);
+
+	/* More than one link but with 2 lmacs: take the minimum */
+	} else if (fw_has_capa(&mld->fw->ucode_capa,
+			       IWL_UCODE_TLV_CAPA_CDB_SUPPORT)) {
+		lmac = IWL_LMAC_5G_INDEX;
+		result = min_t(unsigned int, result,
+			       mld->fwrt.smem_cfg.lmac[lmac].txfifo_size[txf] - 256);
+		lmac = IWL_LMAC_24G_INDEX;
+	/* More than one link but only one lmac */
+	} else {
+		lmac = IWL_LMAC_24G_INDEX;
+	}
+
+	return min_t(unsigned int, result,
+		     mld->fwrt.smem_cfg.lmac[lmac].txfifo_size[txf] - 256);
+}
+
+void iwl_mld_handle_tlc_notif(struct iwl_mld *mld,
+			      struct iwl_rx_packet *pkt)
+{
+	struct iwl_tlc_update_notif *notif = (void *)pkt->data;
+	struct ieee80211_link_sta *link_sta;
+	u32 flags = le32_to_cpu(notif->flags);
+	u32 enabled;
+	u16 size;
+
+	if (IWL_FW_CHECK(mld, notif->sta_id >= mld->fw->ucode_capa.num_stations,
+			 "Invalid sta id (%d) in TLC notification\n",
+			 notif->sta_id))
+		return;
+
+	link_sta = wiphy_dereference(mld->wiphy,
+				     mld->fw_id_to_link_sta[notif->sta_id]);
+
+	if (WARN(IS_ERR_OR_NULL(link_sta),
+		 "link_sta of sta id (%d) doesn't exist\n", notif->sta_id))
+		return;
+
+	if (flags & IWL_TLC_NOTIF_FLAG_RATE) {
+		struct iwl_mld_link_sta *mld_link_sta =
+			iwl_mld_link_sta_from_mac80211(link_sta);
+		char pretty_rate[100];
+
+		if (WARN_ON(!mld_link_sta))
+			return;
+
+		mld_link_sta->last_rate_n_flags =
+			iwl_v3_rate_from_v2_v3(notif->rate,
+					       mld->fw_rates_ver_3);
+
+		rs_pretty_print_rate(pretty_rate, sizeof(pretty_rate),
+				     mld_link_sta->last_rate_n_flags);
+		IWL_DEBUG_RATE(mld, "TLC notif: new rate = %s\n", pretty_rate);
+	}
+
+	/* We are done processing the notif */
+	if (!(flags & IWL_TLC_NOTIF_FLAG_AMSDU))
+		return;
+
+	enabled = le32_to_cpu(notif->amsdu_enabled);
+	size = le32_to_cpu(notif->amsdu_size);
+
+	if (size < 2000) {
+		size = 0;
+		enabled = 0;
+	}
+
+	if (IWL_FW_CHECK(mld, size > link_sta->agg.max_amsdu_len,
+			 "Invalid AMSDU len in TLC notif: %d (Max AMSDU len: %d)\n",
+			 size, link_sta->agg.max_amsdu_len))
+		return;
+
+	link_sta->agg.max_rc_amsdu_len = size;
+
+	for (int i = 0; i < IWL_MAX_TID_COUNT; i++) {
+		if (enabled & BIT(i))
+			link_sta->agg.max_tid_amsdu_len[i] =
+				iwl_mld_get_amsdu_size_of_tid(mld, link_sta, i);
+		else
+			link_sta->agg.max_tid_amsdu_len[i] = 1;
+	}
+
+	ieee80211_sta_recalc_aggregates(link_sta->sta);
+
+	IWL_DEBUG_RATE(mld,
+		       "AMSDU update. AMSDU size: %d, AMSDU selected size: %d, AMSDU TID bitmap 0x%X\n",
+		       le32_to_cpu(notif->amsdu_size), size, enabled);
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/tlc.h b/sys/contrib/dev/iwlwifi/mld/tlc.h
new file mode 100644
index 000000000000..c32f42e8840b
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tlc.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_tlc_h__
+#define __iwl_mld_tlc_h__
+
+#include "mld.h"
+
+void iwl_mld_config_tlc_link(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf,
+			     struct ieee80211_link_sta *link_sta);
+
+void iwl_mld_config_tlc(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			struct ieee80211_sta *sta);
+
+void iwl_mld_handle_tlc_notif(struct iwl_mld *mld,
+			      struct iwl_rx_packet *pkt);
+
+int iwl_mld_send_tlc_dhc(struct iwl_mld *mld, u8 sta_id, u32 type, u32 data);
+
+#endif /* __iwl_mld_tlc_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mld/tx.c b/sys/contrib/dev/iwlwifi/mld/tx.c
new file mode 100644
index 000000000000..3b4b575aadaa
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tx.c
@@ -0,0 +1,1394 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2024 - 2025 Intel Corporation
+ */
+#include <net/ip.h>
+
+#include "tx.h"
+#include "sta.h"
+#include "hcmd.h"
+#include "iwl-utils.h"
+#include "iface.h"
+
+#include "fw/dbg.h"
+
+#include "fw/api/tx.h"
+#include "fw/api/rs.h"
+#include "fw/api/txq.h"
+#include "fw/api/datapath.h"
+#include "fw/api/time-event.h"
+
+#define MAX_ANT_NUM 2
+
+/* Toggles between TX antennas. Receives the bitmask of valid TX antennas and
+ * the *index* used for the last TX, and returns the next valid *index* to use.
+ * In order to set it in the tx_cmd, must do BIT(idx).
+ */
+static u8 iwl_mld_next_ant(u8 valid, u8 last_idx)
+{
+	u8 index = last_idx;
+
+	for (int i = 0; i < MAX_ANT_NUM; i++) {
+		index = (index + 1) % MAX_ANT_NUM;
+		if (valid & BIT(index))
+			return index;
+	}
+
+	WARN_ONCE(1, "Failed to toggle between antennas 0x%x", valid);
+
+	return last_idx;
+}
+
+void iwl_mld_toggle_tx_ant(struct iwl_mld *mld, u8 *ant)
+{
+	*ant = iwl_mld_next_ant(iwl_mld_get_valid_tx_ant(mld), *ant);
+}
+
+static int
+iwl_mld_get_queue_size(struct iwl_mld *mld, struct ieee80211_txq *txq)
+{
+	struct ieee80211_sta *sta = txq->sta;
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+	int max_size = IWL_DEFAULT_QUEUE_SIZE;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	for_each_sta_active_link(txq->vif, sta, link_sta, link_id) {
+		if (link_sta->eht_cap.has_eht) {
+			max_size = IWL_DEFAULT_QUEUE_SIZE_EHT;
+			break;
+		}
+
+		if (link_sta->he_cap.has_he)
+			max_size = IWL_DEFAULT_QUEUE_SIZE_HE;
+	}
+
+	return max_size;
+}
+
+static int iwl_mld_allocate_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
+{
+	u8 tid = txq->tid == IEEE80211_NUM_TIDS ? IWL_MGMT_TID : txq->tid;
+	u32 fw_sta_mask = iwl_mld_fw_sta_id_mask(mld, txq->sta);
+	/* We can't know when the station is asleep or awake, so we
+	 * must disable the queue hang detection.
+	 */
+	unsigned int watchdog_timeout = txq->vif->type == NL80211_IFTYPE_AP ?
+				IWL_WATCHDOG_DISABLED :
+				mld->trans->mac_cfg->base->wd_timeout;
+	int queue, size;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (tid == IWL_MGMT_TID)
+		size = max_t(u32, IWL_MGMT_QUEUE_SIZE,
+			     mld->trans->mac_cfg->base->min_txq_size);
+	else
+		size = iwl_mld_get_queue_size(mld, txq);
+
+	queue = iwl_trans_txq_alloc(mld->trans, 0, fw_sta_mask, tid, size,
+				    watchdog_timeout);
+
+	if (queue >= 0)
+		IWL_DEBUG_TX_QUEUES(mld,
+				    "Enabling TXQ #%d for sta mask 0x%x tid %d\n",
+				    queue, fw_sta_mask, tid);
+	return queue;
+}
+
+static int iwl_mld_add_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
+{
+	struct iwl_mld_txq *mld_txq = iwl_mld_txq_from_mac80211(txq);
+	int id;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	/* This will alse send the SCD_QUEUE_CONFIG_CMD */
+	id = iwl_mld_allocate_txq(mld, txq);
+	if (id < 0)
+		return id;
+
+	mld_txq->fw_id = id;
+	mld_txq->status.allocated = true;
+
+	rcu_assign_pointer(mld->fw_id_to_txq[id], txq);
+
+	return 0;
+}
+
+void iwl_mld_add_txq_list(struct iwl_mld *mld)
+{
+	lockdep_assert_wiphy(mld->wiphy);
+
+	while (!list_empty(&mld->txqs_to_add)) {
+		struct ieee80211_txq *txq;
+		struct iwl_mld_txq *mld_txq =
+			list_first_entry(&mld->txqs_to_add, struct iwl_mld_txq,
+					 list);
+		int failed;
+
+		txq = container_of((void *)mld_txq, struct ieee80211_txq,
+				   drv_priv);
+
+		failed = iwl_mld_add_txq(mld, txq);
+
+		local_bh_disable();
+		spin_lock(&mld->add_txqs_lock);
+		list_del_init(&mld_txq->list);
+		spin_unlock(&mld->add_txqs_lock);
+		/* If the queue allocation failed, we can't transmit. Leave the
+		 * frames on the txq, maybe the attempt to allocate the queue
+		 * will succeed.
+		 */
+		if (!failed)
+			iwl_mld_tx_from_txq(mld, txq);
+		local_bh_enable();
+	}
+}
+
+void iwl_mld_add_txqs_wk(struct wiphy *wiphy, struct wiphy_work *wk)
+{
+	struct iwl_mld *mld = container_of(wk, struct iwl_mld,
+					   add_txqs_wk);
+
+	/* will reschedule to run after restart */
+	if (mld->fw_status.in_hw_restart)
+		return;
+
+	iwl_mld_add_txq_list(mld);
+}
+
+void
+iwl_mld_free_txq(struct iwl_mld *mld, u32 fw_sta_mask, u32 tid, u32 queue_id)
+{
+	struct iwl_scd_queue_cfg_cmd remove_cmd = {
+		.operation = cpu_to_le32(IWL_SCD_QUEUE_REMOVE),
+		.u.remove.tid = cpu_to_le32(tid),
+		.u.remove.sta_mask = cpu_to_le32(fw_sta_mask),
+	};
+
+	iwl_mld_send_cmd_pdu(mld,
+			     WIDE_ID(DATA_PATH_GROUP, SCD_QUEUE_CONFIG_CMD),
+			     &remove_cmd);
+
+	iwl_trans_txq_free(mld->trans, queue_id);
+}
+
+void iwl_mld_remove_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
+{
+	struct iwl_mld_txq *mld_txq = iwl_mld_txq_from_mac80211(txq);
+	u32 sta_msk, tid;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	spin_lock_bh(&mld->add_txqs_lock);
+	if (!list_empty(&mld_txq->list))
+		list_del_init(&mld_txq->list);
+	spin_unlock_bh(&mld->add_txqs_lock);
+
+	if (!mld_txq->status.allocated ||
+	    WARN_ON(mld_txq->fw_id >= ARRAY_SIZE(mld->fw_id_to_txq)))
+		return;
+
+	sta_msk = iwl_mld_fw_sta_id_mask(mld, txq->sta);
+
+	tid = txq->tid == IEEE80211_NUM_TIDS ? IWL_MGMT_TID :
+					       txq->tid;
+
+	iwl_mld_free_txq(mld, sta_msk, tid, mld_txq->fw_id);
+
+	RCU_INIT_POINTER(mld->fw_id_to_txq[mld_txq->fw_id], NULL);
+	mld_txq->status.allocated = false;
+}
+
+#define OPT_HDR(type, skb, off) \
+	(type *)(skb_network_header(skb) + (off))
+
+static __le32
+iwl_mld_get_offload_assist(struct sk_buff *skb, bool amsdu)
+{
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	u16 mh_len = ieee80211_hdrlen(hdr->frame_control);
+	u16 offload_assist = 0;
+#if IS_ENABLED(CONFIG_INET)
+	u8 protocol = 0;
+
+	/* Do not compute checksum if already computed */
+	if (skb->ip_summed != CHECKSUM_PARTIAL)
+		goto out;
+
+	/* We do not expect to be requested to csum stuff we do not support */
+
+	/* TBD: do we also need to check
+	 * !(mvm->hw->netdev_features & IWL_TX_CSUM_NETIF_FLAGS) now that all
+	 * the devices we support has this flags?
+	 */
+	if (WARN_ONCE(skb->protocol != htons(ETH_P_IP) &&
+		      skb->protocol != htons(ETH_P_IPV6),
+		      "No support for requested checksum\n")) {
+		skb_checksum_help(skb);
+		goto out;
+	}
+
+	if (skb->protocol == htons(ETH_P_IP)) {
+		protocol = ip_hdr(skb)->protocol;
+	} else {
+#if IS_ENABLED(CONFIG_IPV6)
+		struct ipv6hdr *ipv6h =
+			(struct ipv6hdr *)skb_network_header(skb);
+		unsigned int off = sizeof(*ipv6h);
+
+		protocol = ipv6h->nexthdr;
+		while (protocol != NEXTHDR_NONE && ipv6_ext_hdr(protocol)) {
+			struct ipv6_opt_hdr *hp;
+
+			/* only supported extension headers */
+			if (protocol != NEXTHDR_ROUTING &&
+			    protocol != NEXTHDR_HOP &&
+			    protocol != NEXTHDR_DEST) {
+				skb_checksum_help(skb);
+				goto out;
+			}
+
+			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
+			protocol = hp->nexthdr;
+			off += ipv6_optlen(hp);
+		}
+		/* if we get here - protocol now should be TCP/UDP */
+#endif
+	}
+
+	if (protocol != IPPROTO_TCP && protocol != IPPROTO_UDP) {
+		WARN_ON_ONCE(1);
+		skb_checksum_help(skb);
+		goto out;
+	}
+
+	/* enable L4 csum */
+	offload_assist |= BIT(TX_CMD_OFFLD_L4_EN);
+
+	/* Set offset to IP header (snap).
+	 * We don't support tunneling so no need to take care of inner header.
+	 * Size is in words.
+	 */
+	offload_assist |= (4 << TX_CMD_OFFLD_IP_HDR);
+
+	/* Do IPv4 csum for AMSDU only (no IP csum for Ipv6) */
+	if (skb->protocol == htons(ETH_P_IP) && amsdu) {
+		ip_hdr(skb)->check = 0;
+		offload_assist |= BIT(TX_CMD_OFFLD_L3_EN);
+	}
+
+	/* reset UDP/TCP header csum */
+	if (protocol == IPPROTO_TCP)
+		tcp_hdr(skb)->check = 0;
+	else
+		udp_hdr(skb)->check = 0;
+
+out:
+#endif
+	mh_len /= 2;
+	offload_assist |= mh_len << TX_CMD_OFFLD_MH_SIZE;
+
+	if (amsdu)
+		offload_assist |= BIT(TX_CMD_OFFLD_AMSDU);
+	else if (ieee80211_hdrlen(hdr->frame_control) % 4)
+		/* padding is inserted later in transport */
+		offload_assist |= BIT(TX_CMD_OFFLD_PAD);
+
+	return cpu_to_le32(offload_assist);
+}
+
+static void iwl_mld_get_basic_rates_and_band(struct iwl_mld *mld,
+					     struct ieee80211_vif *vif,
+					     struct ieee80211_tx_info *info,
+					     unsigned long *basic_rates,
+					     u8 *band)
+{
+	u32 link_id = u32_get_bits(info->control.flags,
+				   IEEE80211_TX_CTRL_MLO_LINK);
+
+	*basic_rates = vif->bss_conf.basic_rates;
+	*band = info->band;
+
+	if (link_id == IEEE80211_LINK_UNSPECIFIED &&
+	    ieee80211_vif_is_mld(vif)) {
+		/* shouldn't do this when >1 link is active */
+		WARN_ON(hweight16(vif->active_links) != 1);
+		link_id = __ffs(vif->active_links);
+	}
+
+	if (link_id < IEEE80211_LINK_UNSPECIFIED) {
+		struct ieee80211_bss_conf *link_conf;
+
+		rcu_read_lock();
+		link_conf = rcu_dereference(vif->link_conf[link_id]);
+		if (link_conf) {
+			*basic_rates = link_conf->basic_rates;
+			if (link_conf->chanreq.oper.chan)
+				*band = link_conf->chanreq.oper.chan->band;
+		}
+		rcu_read_unlock();
+	}
+}
+
+u8 iwl_mld_get_lowest_rate(struct iwl_mld *mld,
+			   struct ieee80211_tx_info *info,
+			   struct ieee80211_vif *vif)
+{
+	struct ieee80211_supported_band *sband;
+	u16 lowest_cck = IWL_RATE_COUNT, lowest_ofdm = IWL_RATE_COUNT;
+	unsigned long basic_rates;
+	u8 band, rate;
+	u32 i;
+
+	iwl_mld_get_basic_rates_and_band(mld, vif, info, &basic_rates, &band);
+
+	sband = mld->hw->wiphy->bands[band];
+	for_each_set_bit(i, &basic_rates, BITS_PER_LONG) {
+		u16 hw = sband->bitrates[i].hw_value;
+
+		if (hw >= IWL_FIRST_OFDM_RATE) {
+			if (lowest_ofdm > hw)
+				lowest_ofdm = hw;
+		} else if (lowest_cck > hw) {
+			lowest_cck = hw;
+		}
+	}
+
+	if (band == NL80211_BAND_2GHZ && !vif->p2p &&
+	    vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+	    !(info->flags & IEEE80211_TX_CTL_NO_CCK_RATE)) {
+		if (lowest_cck != IWL_RATE_COUNT)
+			rate = lowest_cck;
+		else if (lowest_ofdm != IWL_RATE_COUNT)
+			rate = lowest_ofdm;
+		else
+			rate = IWL_FIRST_CCK_RATE;
+	} else if (lowest_ofdm != IWL_RATE_COUNT) {
+		rate = lowest_ofdm;
+	} else {
+		rate = IWL_FIRST_OFDM_RATE;
+	}
+
+	return rate;
+}
+
+static u32 iwl_mld_mac80211_rate_idx_to_fw(struct iwl_mld *mld,
+					   struct ieee80211_tx_info *info,
+					   int rate_idx)
+{
+	u32 rate_flags = 0;
+	u8 rate_plcp;
+
+	/* if the rate isn't a well known legacy rate, take the lowest one */
+	if (rate_idx < 0 || rate_idx >= IWL_RATE_COUNT_LEGACY)
+		rate_idx = iwl_mld_get_lowest_rate(mld, info,
+						   info->control.vif);
+
+	WARN_ON_ONCE(rate_idx < 0);
+
+	/* Set CCK or OFDM flag */
+	if (rate_idx <= IWL_LAST_CCK_RATE)
+		rate_flags |= RATE_MCS_MOD_TYPE_CCK;
+	else
+		rate_flags |= RATE_MCS_MOD_TYPE_LEGACY_OFDM;
+
+	/* Legacy rates are indexed:
+	 * 0 - 3 for CCK and 0 - 7 for OFDM
+	 */
+	rate_plcp = (rate_idx >= IWL_FIRST_OFDM_RATE ?
+		     rate_idx - IWL_FIRST_OFDM_RATE : rate_idx);
+
+	return (u32)rate_plcp | rate_flags;
+}
+
+static u32 iwl_mld_get_tx_ant(struct iwl_mld *mld,
+			      struct ieee80211_tx_info *info,
+			      struct ieee80211_sta *sta, __le16 fc)
+{
+	if (sta && ieee80211_is_data(fc)) {
+		struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+		return BIT(mld_sta->data_tx_ant) << RATE_MCS_ANT_POS;
+	}
+
+	return BIT(mld->mgmt_tx_ant) << RATE_MCS_ANT_POS;
+}
+
+static u32 iwl_mld_get_inject_tx_rate(struct iwl_mld *mld,
+				      struct ieee80211_tx_info *info,
+				      struct ieee80211_sta *sta,
+				      __le16 fc)
+{
+	struct ieee80211_tx_rate *rate = &info->control.rates[0];
+	u32 result;
+
+	if (rate->flags & IEEE80211_TX_RC_VHT_MCS) {
+		u8 mcs = ieee80211_rate_get_vht_mcs(rate);
+		u8 nss = ieee80211_rate_get_vht_nss(rate);
+
+		result = RATE_MCS_MOD_TYPE_VHT;
+		result |= u32_encode_bits(mcs, RATE_MCS_CODE_MSK);
+		result |= u32_encode_bits(nss, RATE_MCS_NSS_MSK);
+
+		if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
+			result |= RATE_MCS_SGI_MSK;
+
+		if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+			result |= RATE_MCS_CHAN_WIDTH_40;
+		else if (rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
+			result |= RATE_MCS_CHAN_WIDTH_80;
+		else if (rate->flags & IEEE80211_TX_RC_160_MHZ_WIDTH)
+			result |= RATE_MCS_CHAN_WIDTH_160;
+	} else if (rate->flags & IEEE80211_TX_RC_MCS) {
+		/* only MCS 0-15 are supported */
+		u8 mcs = rate->idx & 7;
+		u8 nss = rate->idx > 7;
+
+		result = RATE_MCS_MOD_TYPE_HT;
+		result |= u32_encode_bits(mcs, RATE_MCS_CODE_MSK);
+		result |= u32_encode_bits(nss, RATE_MCS_NSS_MSK);
+
+		if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
+			result |= RATE_MCS_SGI_MSK;
+		if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+			result |= RATE_MCS_CHAN_WIDTH_40;
+		if (info->flags & IEEE80211_TX_CTL_LDPC)
+			result |= RATE_MCS_LDPC_MSK;
+		if (u32_get_bits(info->flags, IEEE80211_TX_CTL_STBC))
+			result |= RATE_MCS_STBC_MSK;
+	} else {
+		result = iwl_mld_mac80211_rate_idx_to_fw(mld, info, rate->idx);
+	}
+
+	if (info->control.antennas)
+		result |= u32_encode_bits(info->control.antennas,
+					  RATE_MCS_ANT_AB_MSK);
+	else
+		result |= iwl_mld_get_tx_ant(mld, info, sta, fc);
+
+	return result;
+}
+
+static __le32 iwl_mld_get_tx_rate_n_flags(struct iwl_mld *mld,
+					  struct ieee80211_tx_info *info,
+					  struct ieee80211_sta *sta, __le16 fc)
+{
+	u32 rate;
+
+	if (unlikely(info->control.flags & IEEE80211_TX_CTRL_RATE_INJECT))
+		rate = iwl_mld_get_inject_tx_rate(mld, info, sta, fc);
+	else
+		rate = iwl_mld_mac80211_rate_idx_to_fw(mld, info, -1) |
+		       iwl_mld_get_tx_ant(mld, info, sta, fc);
+
+	return iwl_v3_rate_to_v2_v3(rate, mld->fw_rates_ver_3);
+}
+
+static void
+iwl_mld_fill_tx_cmd_hdr(struct iwl_tx_cmd *tx_cmd,
+			struct sk_buff *skb, bool amsdu)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	struct ieee80211_vif *vif;
+
+	/* Copy MAC header from skb into command buffer */
+	memcpy(tx_cmd->hdr, hdr, ieee80211_hdrlen(hdr->frame_control));
+
+	if (!amsdu || !skb_is_gso(skb))
+		return;
+
+	/* As described in IEEE sta 802.11-2020, table 9-30 (Address
+	 * field contents), A-MSDU address 3 should contain the BSSID
+	 * address.
+	 *
+	 * In TSO, the skb header address 3 contains the original address 3 to
+	 * correctly create all the A-MSDU subframes headers from it.
+	 * Override now the address 3 in the command header with the BSSID.
+	 *
+	 * Note: we fill in the MLD address, but the firmware will do the
+	 * necessary translation to link address after encryption.
+	 */
+	vif = info->control.vif;
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		ether_addr_copy(tx_cmd->hdr->addr3, vif->cfg.ap_addr);
+		break;
+	case NL80211_IFTYPE_AP:
+		ether_addr_copy(tx_cmd->hdr->addr3, vif->addr);
+		break;
+	default:
+		break;
+	}
+}
+
+static void
+iwl_mld_fill_tx_cmd(struct iwl_mld *mld, struct sk_buff *skb,
+		    struct iwl_device_tx_cmd *dev_tx_cmd,
+		    struct ieee80211_sta *sta)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	struct iwl_mld_sta *mld_sta = sta ? iwl_mld_sta_from_mac80211(sta) :
+					    NULL;
+	struct iwl_tx_cmd *tx_cmd;
+	bool amsdu = ieee80211_is_data_qos(hdr->frame_control) &&
+		     (*ieee80211_get_qos_ctl(hdr) &
+		      IEEE80211_QOS_CTL_A_MSDU_PRESENT);
+	__le32 rate_n_flags = 0;
+	u16 flags = 0;
+
+	dev_tx_cmd->hdr.cmd = TX_CMD;
+
+	if (!info->control.hw_key)
+		flags |= IWL_TX_FLAGS_ENCRYPT_DIS;
+
+	/* For data and mgmt packets rate info comes from the fw.
+	 * Only set rate/antenna for injected frames with fixed rate, or
+	 * when no sta is given.
+	 */
+	if (unlikely(!sta ||
+		     info->control.flags & IEEE80211_TX_CTRL_RATE_INJECT)) {
+		flags |= IWL_TX_FLAGS_CMD_RATE;
+		rate_n_flags = iwl_mld_get_tx_rate_n_flags(mld, info, sta,
+							   hdr->frame_control);
+	} else if (!ieee80211_is_data(hdr->frame_control) ||
+		   (mld_sta &&
+		    mld_sta->sta_state < IEEE80211_STA_AUTHORIZED)) {
+		/* These are important frames */
+		flags |= IWL_TX_FLAGS_HIGH_PRI;
+	}
+
+	tx_cmd = (void *)dev_tx_cmd->payload;
+
+	iwl_mld_fill_tx_cmd_hdr(tx_cmd, skb, amsdu);
+
+	tx_cmd->offload_assist = iwl_mld_get_offload_assist(skb, amsdu);
+
+	/* Total # bytes to be transmitted */
+	tx_cmd->len = cpu_to_le16((u16)skb->len);
+
+	tx_cmd->flags = cpu_to_le16(flags);
+
+	tx_cmd->rate_n_flags = rate_n_flags;
+}
+
+/* Caller of this need to check that info->control.vif is not NULL */
+static struct iwl_mld_link *
+iwl_mld_get_link_from_tx_info(struct ieee80211_tx_info *info)
+{
+	struct iwl_mld_vif *mld_vif =
+		iwl_mld_vif_from_mac80211(info->control.vif);
+	u32 link_id = u32_get_bits(info->control.flags,
+				   IEEE80211_TX_CTRL_MLO_LINK);
+
+	if (link_id == IEEE80211_LINK_UNSPECIFIED) {
+		if (info->control.vif->active_links)
+			link_id = ffs(info->control.vif->active_links) - 1;
+		else
+			link_id = 0;
+	}
+
+	return rcu_dereference(mld_vif->link[link_id]);
+}
+
+static int
+iwl_mld_get_tx_queue_id(struct iwl_mld *mld, struct ieee80211_txq *txq,
+			struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	__le16 fc = hdr->frame_control;
+	struct iwl_mld_vif *mld_vif;
+	struct iwl_mld_link *link;
+
+	if (txq && txq->sta)
+		return iwl_mld_txq_from_mac80211(txq)->fw_id;
+
+	if (!info->control.vif)
+		return IWL_MLD_INVALID_QUEUE;
+
+	switch (info->control.vif->type) {
+	case NL80211_IFTYPE_AP:
+	case NL80211_IFTYPE_ADHOC:
+		link = iwl_mld_get_link_from_tx_info(info);
+
+		if (WARN_ON(!link))
+			break;
+
+		/* ucast disassociate/deauth frames without a station might
+		 * happen, especially with reason 7 ("Class 3 frame received
+		 * from nonassociated STA").
+		 */
+		if (ieee80211_is_mgmt(fc) &&
+		    (!ieee80211_is_bufferable_mmpdu(skb) ||
+		     ieee80211_is_deauth(fc) || ieee80211_is_disassoc(fc)))
+			return link->bcast_sta.queue_id;
+
+		if (is_multicast_ether_addr(hdr->addr1) &&
+		    !ieee80211_has_order(fc))
+			return link->mcast_sta.queue_id;
+
+		WARN_ONCE(info->control.vif->type != NL80211_IFTYPE_ADHOC,
+			  "Couldn't find a TXQ. fc=0x%02x", le16_to_cpu(fc));
+		return link->bcast_sta.queue_id;
+	case NL80211_IFTYPE_P2P_DEVICE:
+		mld_vif = iwl_mld_vif_from_mac80211(info->control.vif);
+
+		if (mld_vif->roc_activity != ROC_ACTIVITY_P2P_DISC &&
+		    mld_vif->roc_activity != ROC_ACTIVITY_P2P_NEG) {
+			IWL_DEBUG_DROP(mld,
+				       "Drop tx outside ROC with activity %d\n",
+				       mld_vif->roc_activity);
+			return IWL_MLD_INVALID_DROP_TX;
+		}
+
+		WARN_ON(!ieee80211_is_mgmt(fc));
+
+		return mld_vif->aux_sta.queue_id;
+	case NL80211_IFTYPE_MONITOR:
+		mld_vif = iwl_mld_vif_from_mac80211(info->control.vif);
+		return mld_vif->deflink.mon_sta.queue_id;
+	case NL80211_IFTYPE_STATION:
+		mld_vif = iwl_mld_vif_from_mac80211(info->control.vif);
+
+		if (!(info->flags & IEEE80211_TX_CTL_TX_OFFCHAN)) {
+			IWL_DEBUG_DROP(mld, "Drop tx not off-channel\n");
+			return IWL_MLD_INVALID_DROP_TX;
+		}
+
+		if (mld_vif->roc_activity != ROC_ACTIVITY_HOTSPOT) {
+			IWL_DEBUG_DROP(mld, "Drop tx outside ROC\n");
+			return IWL_MLD_INVALID_DROP_TX;
+		}
+
+		WARN_ON(!ieee80211_is_mgmt(fc));
+		return mld_vif->aux_sta.queue_id;
+	default:
+		WARN_ONCE(1, "Unsupported vif type\n");
+		break;
+	}
+
+	return IWL_MLD_INVALID_QUEUE;
+}
+
+static void iwl_mld_probe_resp_set_noa(struct iwl_mld *mld,
+				       struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct iwl_mld_link *mld_link =
+		&iwl_mld_vif_from_mac80211(info->control.vif)->deflink;
+	struct iwl_probe_resp_data *resp_data;
+	u8 *pos;
+
+	if (!info->control.vif->p2p)
+		return;
+
+	rcu_read_lock();
+
+	resp_data = rcu_dereference(mld_link->probe_resp_data);
+	if (!resp_data)
+		goto out;
+
+	if (!resp_data->notif.noa_active)
+		goto out;
+
+	if (skb_tailroom(skb) < resp_data->noa_len) {
+		if (pskb_expand_head(skb, 0, resp_data->noa_len, GFP_ATOMIC)) {
+			IWL_ERR(mld,
+				"Failed to reallocate probe resp\n");
+			goto out;
+		}
+	}
+
+	pos = skb_put(skb, resp_data->noa_len);
+
+	*pos++ = WLAN_EID_VENDOR_SPECIFIC;
+	/* Set length of IE body (not including ID and length itself) */
+	*pos++ = resp_data->noa_len - 2;
+	*pos++ = (WLAN_OUI_WFA >> 16) & 0xff;
+	*pos++ = (WLAN_OUI_WFA >> 8) & 0xff;
+	*pos++ = WLAN_OUI_WFA & 0xff;
+	*pos++ = WLAN_OUI_TYPE_WFA_P2P;
+
+	memcpy(pos, &resp_data->notif.noa_attr,
+	       resp_data->noa_len - sizeof(struct ieee80211_vendor_ie));
+
+out:
+	rcu_read_unlock();
+}
+
+/* This function must be called with BHs disabled */
+static int iwl_mld_tx_mpdu(struct iwl_mld *mld, struct sk_buff *skb,
+			   struct ieee80211_txq *txq)
+{
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_sta *sta = txq ? txq->sta : NULL;
+	struct iwl_device_tx_cmd *dev_tx_cmd;
+	int queue = iwl_mld_get_tx_queue_id(mld, txq, skb);
+	u8 tid = IWL_MAX_TID_COUNT;
+
+	if (WARN_ONCE(queue == IWL_MLD_INVALID_QUEUE, "Invalid TX Queue id") ||
+	    queue == IWL_MLD_INVALID_DROP_TX)
+		return -1;
+
+	if (unlikely(ieee80211_is_any_nullfunc(hdr->frame_control)))
+		return -1;
+
+	dev_tx_cmd = iwl_trans_alloc_tx_cmd(mld->trans);
+	if (unlikely(!dev_tx_cmd))
+		return -1;
+
+	if (unlikely(ieee80211_is_probe_resp(hdr->frame_control))) {
+		if (IWL_MLD_NON_TRANSMITTING_AP)
+			return -1;
+
+		iwl_mld_probe_resp_set_noa(mld, skb);
+	}
+
+	iwl_mld_fill_tx_cmd(mld, skb, dev_tx_cmd, sta);
+
+	if (ieee80211_is_data(hdr->frame_control)) {
+		if (ieee80211_is_data_qos(hdr->frame_control))
+			tid = ieee80211_get_tid(hdr);
+		else
+			tid = IWL_TID_NON_QOS;
+	}
+
+	IWL_DEBUG_TX(mld, "TX TID:%d from Q:%d len %d\n",
+		     tid, queue, skb->len);
+
+	/* From now on, we cannot access info->control */
+	memset(&info->status, 0, sizeof(info->status));
+	memset(info->driver_data, 0, sizeof(info->driver_data));
+
+	info->driver_data[1] = dev_tx_cmd;
+
+	if (iwl_trans_tx(mld->trans, skb, dev_tx_cmd, queue))
+		goto err;
+
+	/* Update low-latency counter when a packet is queued instead
+	 * of after TX, it makes sense for early low-latency detection
+	 */
+	if (sta)
+		iwl_mld_low_latency_update_counters(mld, hdr, sta, 0);
+
+	return 0;
+
+err:
+	iwl_trans_free_tx_cmd(mld->trans, dev_tx_cmd);
+	IWL_DEBUG_TX(mld, "TX from Q:%d dropped\n", queue);
+	return -1;
+}
+
+#ifdef CONFIG_INET
+
+/* This function handles the segmentation of a large TSO packet into multiple
+ * MPDUs, ensuring that the resulting segments conform to AMSDU limits and
+ * constraints.
+ */
+static int iwl_mld_tx_tso_segment(struct iwl_mld *mld, struct sk_buff *skb,
+				  struct ieee80211_sta *sta,
+				  struct sk_buff_head *mpdus_skbs)
+{
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	netdev_features_t netdev_flags = NETIF_F_CSUM_MASK | NETIF_F_SG;
+	unsigned int mss = skb_shinfo(skb)->gso_size;
+	unsigned int num_subframes, tcp_payload_len, subf_len;
+	u16 snap_ip_tcp, pad, max_tid_amsdu_len;
+	u8 tid;
+
+	snap_ip_tcp = 8 + skb_network_header_len(skb) + tcp_hdrlen(skb);
+
+	if (!ieee80211_is_data_qos(hdr->frame_control) ||
+	    !sta->cur->max_rc_amsdu_len)
+		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skbs);
+
+	/* Do not build AMSDU for IPv6 with extension headers.
+	 * Ask stack to segment and checksum the generated MPDUs for us.
+	 */
+	if (skb->protocol == htons(ETH_P_IPV6) &&
+	    ((struct ipv6hdr *)skb_network_header(skb))->nexthdr !=
+	    IPPROTO_TCP) {
+		netdev_flags &= ~NETIF_F_CSUM_MASK;
+		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skbs);
+	}
+
+	tid = ieee80211_get_tid(hdr);
+	if (WARN_ON_ONCE(tid >= IWL_MAX_TID_COUNT))
+		return -EINVAL;
+
+	max_tid_amsdu_len = sta->cur->max_tid_amsdu_len[tid];
+	if (!max_tid_amsdu_len)
+		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skbs);
+
+	/* Sub frame header + SNAP + IP header + TCP header + MSS */
+	subf_len = sizeof(struct ethhdr) + snap_ip_tcp + mss;
+	pad = (4 - subf_len) & 0x3;
+
+	/* If we have N subframes in the A-MSDU, then the A-MSDU's size is
+	 * N * subf_len + (N - 1) * pad.
+	 */
+	num_subframes = (max_tid_amsdu_len + pad) / (subf_len + pad);
+
+	if (sta->max_amsdu_subframes &&
+	    num_subframes > sta->max_amsdu_subframes)
+		num_subframes = sta->max_amsdu_subframes;
+
+	tcp_payload_len = skb_tail_pointer(skb) - skb_transport_header(skb) -
+		tcp_hdrlen(skb) + skb->data_len;
+
+	/* Make sure we have enough TBs for the A-MSDU:
+	 *	2 for each subframe
+	 *	1 more for each fragment
+	 *	1 more for the potential data in the header
+	 */
+	if ((num_subframes * 2 + skb_shinfo(skb)->nr_frags + 1) >
+	    mld->trans->info.max_skb_frags)
+		num_subframes = 1;
+
+	if (num_subframes > 1)
+		*ieee80211_get_qos_ctl(hdr) |= IEEE80211_QOS_CTL_A_MSDU_PRESENT;
+
+	/* This skb fits in one single A-MSDU */
+	if (tcp_payload_len <= num_subframes * mss) {
+		__skb_queue_tail(mpdus_skbs, skb);
+		return 0;
+	}
+
+	/* Trick the segmentation function to make it create SKBs that can fit
+	 * into one A-MSDU.
+	 */
+	return iwl_tx_tso_segment(skb, num_subframes, netdev_flags, mpdus_skbs);
+}
+
+/* Manages TSO (TCP Segmentation Offload) packet transmission by segmenting
+ * large packets when necessary and transmitting each segment as MPDU.
+ */
+static int iwl_mld_tx_tso(struct iwl_mld *mld, struct sk_buff *skb,
+			  struct ieee80211_txq *txq)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct sk_buff *orig_skb = skb;
+	struct sk_buff_head mpdus_skbs;
+	unsigned int payload_len;
+	int ret;
+
+	if (WARN_ON(!txq || !txq->sta))
+		return -1;
+
+	payload_len = skb_tail_pointer(skb) - skb_transport_header(skb) -
+		tcp_hdrlen(skb) + skb->data_len;
+
+	if (payload_len <= skb_shinfo(skb)->gso_size)
+		return iwl_mld_tx_mpdu(mld, skb, txq);
+
+	if (!info->control.vif)
+		return -1;
+
+	__skb_queue_head_init(&mpdus_skbs);
+
+	ret = iwl_mld_tx_tso_segment(mld, skb, txq->sta, &mpdus_skbs);
+	if (ret)
+		return ret;
+
+	WARN_ON(skb_queue_empty(&mpdus_skbs));
+
+	while (!skb_queue_empty(&mpdus_skbs)) {
+		skb = __skb_dequeue(&mpdus_skbs);
+
+		ret = iwl_mld_tx_mpdu(mld, skb, txq);
+		if (!ret)
+			continue;
+
+		/* Free skbs created as part of TSO logic that have not yet
+		 * been dequeued
+		 */
+		__skb_queue_purge(&mpdus_skbs);
+
+		/* skb here is not necessarily same as skb that entered
+		 * this method, so free it explicitly.
+		 */
+		if (skb == orig_skb)
+			ieee80211_free_txskb(mld->hw, skb);
+		else
+			kfree_skb(skb);
+
+		/* there was error, but we consumed skb one way or
+		 * another, so return 0
+		 */
+		return 0;
+	}
+
+	return 0;
+}
+#else
+static int iwl_mld_tx_tso(struct iwl_mld *mld, struct sk_buff *skb,
+			  struct ieee80211_txq *txq)
+{
+	/* Impossible to get TSO without CONFIG_INET */
+	WARN_ON(1);
+
+	return -1;
+}
+#endif /* CONFIG_INET */
+
+void iwl_mld_tx_skb(struct iwl_mld *mld, struct sk_buff *skb,
+		    struct ieee80211_txq *txq)
+{
+	if (skb_is_gso(skb)) {
+		if (!iwl_mld_tx_tso(mld, skb, txq))
+			return;
+		goto err;
+	}
+
+	if (likely(!iwl_mld_tx_mpdu(mld, skb, txq)))
+		return;
+
+err:
+	ieee80211_free_txskb(mld->hw, skb);
+}
+
+void iwl_mld_tx_from_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
+{
+	struct iwl_mld_txq *mld_txq = iwl_mld_txq_from_mac80211(txq);
+	struct sk_buff *skb = NULL;
+	u8 zero_addr[ETH_ALEN] = {};
+
+	/*
+	 * No need for threads to be pending here, they can leave the first
+	 * taker all the work.
+	 *
+	 * mld_txq->tx_request logic:
+	 *
+	 * If 0, no one is currently TXing, set to 1 to indicate current thread
+	 * will now start TX and other threads should quit.
+	 *
+	 * If 1, another thread is currently TXing, set to 2 to indicate to
+	 * that thread that there was another request. Since that request may
+	 * have raced with the check whether the queue is empty, the TXing
+	 * thread should check the queue's status one more time before leaving.
+	 * This check is done in order to not leave any TX hanging in the queue
+	 * until the next TX invocation (which may not even happen).
+	 *
+	 * If 2, another thread is currently TXing, and it will already double
+	 * check the queue, so do nothing.
+	 */
+	if (atomic_fetch_add_unless(&mld_txq->tx_request, 1, 2))
+		return;
+
+	rcu_read_lock();
+	do {
+		while (likely(!mld_txq->status.stop_full) &&
+		       (skb = ieee80211_tx_dequeue(mld->hw, txq)))
+			iwl_mld_tx_skb(mld, skb, txq);
+	} while (atomic_dec_return(&mld_txq->tx_request));
+
+	IWL_DEBUG_TX(mld, "TXQ of sta %pM tid %d is now empty\n",
+		     txq->sta ? txq->sta->addr : zero_addr, txq->tid);
+
+	rcu_read_unlock();
+}
+
+static void iwl_mld_hwrate_to_tx_rate(struct iwl_mld *mld,
+				      __le32 rate_n_flags_fw,
+				      struct ieee80211_tx_info *info)
+{
+	enum nl80211_band band = info->band;
+	struct ieee80211_tx_rate *tx_rate = &info->status.rates[0];
+	u32 rate_n_flags = iwl_v3_rate_from_v2_v3(rate_n_flags_fw,
+						  mld->fw_rates_ver_3);
+	u32 sgi = rate_n_flags & RATE_MCS_SGI_MSK;
+	u32 chan_width = rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK;
+	u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+
+	if (sgi)
+		tx_rate->flags |= IEEE80211_TX_RC_SHORT_GI;
+
+	switch (chan_width) {
+	case RATE_MCS_CHAN_WIDTH_20:
+		break;
+	case RATE_MCS_CHAN_WIDTH_40:
+		tx_rate->flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;
+		break;
+	case RATE_MCS_CHAN_WIDTH_80:
+		tx_rate->flags |= IEEE80211_TX_RC_80_MHZ_WIDTH;
+		break;
+	case RATE_MCS_CHAN_WIDTH_160:
+		tx_rate->flags |= IEEE80211_TX_RC_160_MHZ_WIDTH;
+		break;
+	default:
+		break;
+	}
+
+	switch (format) {
+	case RATE_MCS_MOD_TYPE_HT:
+		tx_rate->flags |= IEEE80211_TX_RC_MCS;
+		tx_rate->idx = RATE_HT_MCS_INDEX(rate_n_flags);
+		break;
+	case RATE_MCS_MOD_TYPE_VHT:
+		ieee80211_rate_set_vht(tx_rate,
+				       rate_n_flags & RATE_MCS_CODE_MSK,
+				       u32_get_bits(rate_n_flags,
+						    RATE_MCS_NSS_MSK) + 1);
+		tx_rate->flags |= IEEE80211_TX_RC_VHT_MCS;
+		break;
+	case RATE_MCS_MOD_TYPE_HE:
+	case RATE_MCS_MOD_TYPE_EHT:
+		/* mac80211 cannot do this without ieee80211_tx_status_ext()
+		 * but it only matters for radiotap
+		 */
+		tx_rate->idx = 0;
+		break;
+	default:
+		tx_rate->idx =
+			iwl_mld_legacy_hw_idx_to_mac80211_idx(rate_n_flags,
+							      band);
+		break;
+	}
+}
+
+void iwl_mld_handle_tx_resp_notif(struct iwl_mld *mld,
+				  struct iwl_rx_packet *pkt)
+{
+	struct iwl_tx_resp *tx_resp = (void *)pkt->data;
+	int txq_id = le16_to_cpu(tx_resp->tx_queue);
+	struct agg_tx_status *agg_status = &tx_resp->status;
+	u32 status = le16_to_cpu(agg_status->status);
+	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
+	size_t notif_size = sizeof(*tx_resp) + sizeof(u32);
+	int sta_id = IWL_TX_RES_GET_RA(tx_resp->ra_tid);
+	int tid = IWL_TX_RES_GET_TID(tx_resp->ra_tid);
+	struct ieee80211_link_sta *link_sta;
+	struct iwl_mld_sta *mld_sta;
+	u16 ssn;
+	struct sk_buff_head skbs;
+	u8 skb_freed = 0;
+	bool mgmt = false;
+	bool tx_failure = (status & TX_STATUS_MSK) != TX_STATUS_SUCCESS;
+
+	if (IWL_FW_CHECK(mld, tx_resp->frame_count != 1,
+			 "Invalid tx_resp notif frame_count (%d)\n",
+			 tx_resp->frame_count))
+		return;
+
+	/* validate the size of the variable part of the notif */
+	if (IWL_FW_CHECK(mld, notif_size != pkt_len,
+			 "Invalid tx_resp notif size (expected=%zu got=%u)\n",
+			 notif_size, pkt_len))
+		return;
+
+	ssn = le32_to_cpup((__le32 *)agg_status +
+			   tx_resp->frame_count) & 0xFFFF;
+
+	__skb_queue_head_init(&skbs);
+
+	/* we can free until ssn % q.n_bd not inclusive */
+	iwl_trans_reclaim(mld->trans, txq_id, ssn, &skbs, false);
+
+	while (!skb_queue_empty(&skbs)) {
+		struct sk_buff *skb = __skb_dequeue(&skbs);
+		struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+		struct ieee80211_hdr *hdr = (void *)skb->data;
+
+		skb_freed++;
+
+		iwl_trans_free_tx_cmd(mld->trans, info->driver_data[1]);
+
+		memset(&info->status, 0, sizeof(info->status));
+
+		info->flags &= ~(IEEE80211_TX_STAT_ACK | IEEE80211_TX_STAT_TX_FILTERED);
+
+		/* inform mac80211 about what happened with the frame */
+		switch (status & TX_STATUS_MSK) {
+		case TX_STATUS_SUCCESS:
+		case TX_STATUS_DIRECT_DONE:
+			info->flags |= IEEE80211_TX_STAT_ACK;
+			break;
+		default:
+			break;
+		}
+
+		/* If we are freeing multiple frames, mark all the frames
+		 * but the first one as acked, since they were acknowledged
+		 * before
+		 */
+		if (skb_freed > 1)
+			info->flags |= IEEE80211_TX_STAT_ACK;
+
+		if (tx_failure) {
+			enum iwl_fw_ini_time_point tp =
+				IWL_FW_INI_TIME_POINT_TX_FAILED;
+
+			if (ieee80211_is_action(hdr->frame_control))
+				tp = IWL_FW_INI_TIME_POINT_TX_WFD_ACTION_FRAME_FAILED;
+			else if (ieee80211_is_mgmt(hdr->frame_control))
+				mgmt = true;
+
+			iwl_dbg_tlv_time_point(&mld->fwrt, tp, NULL);
+		}
+
+		iwl_mld_hwrate_to_tx_rate(mld, tx_resp->initial_rate, info);
+
+		if (likely(!iwl_mld_time_sync_frame(mld, skb, hdr->addr1)))
+			ieee80211_tx_status_skb(mld->hw, skb);
+	}
+
+	IWL_DEBUG_TX_REPLY(mld,
+			   "TXQ %d status 0x%08x ssn=%d initial_rate 0x%x retries %d\n",
+			   txq_id, status, ssn, le32_to_cpu(tx_resp->initial_rate),
+			   tx_resp->failure_frame);
+
+	if (tx_failure && mgmt)
+		iwl_mld_toggle_tx_ant(mld, &mld->mgmt_tx_ant);
+
+	if (IWL_FW_CHECK(mld, sta_id >= mld->fw->ucode_capa.num_stations,
+			 "Got invalid sta_id (%d)\n", sta_id))
+		return;
+
+	rcu_read_lock();
+
+	link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]);
+	if (!link_sta) {
+		/* This can happen if the TX cmd was sent before pre_rcu_remove
+		 * but the TX response was received after
+		 */
+		IWL_DEBUG_TX_REPLY(mld,
+				   "Got valid sta_id (%d) but sta is NULL\n",
+				   sta_id);
+		goto out;
+	}
+
+	if (IS_ERR(link_sta))
+		goto out;
+
+	mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+
+	if (tx_failure && mld_sta->sta_state < IEEE80211_STA_AUTHORIZED)
+		iwl_mld_toggle_tx_ant(mld, &mld_sta->data_tx_ant);
+
+	if (tid < IWL_MAX_TID_COUNT)
+		iwl_mld_count_mpdu_tx(link_sta, 1);
+
+out:
+	rcu_read_unlock();
+}
+
+static void iwl_mld_tx_reclaim_txq(struct iwl_mld *mld, int txq, int index,
+				   bool in_flush)
+{
+	struct sk_buff_head reclaimed_skbs;
+
+	__skb_queue_head_init(&reclaimed_skbs);
+
+	iwl_trans_reclaim(mld->trans, txq, index, &reclaimed_skbs, in_flush);
+
+	while (!skb_queue_empty(&reclaimed_skbs)) {
+		struct sk_buff *skb = __skb_dequeue(&reclaimed_skbs);
+		struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+		iwl_trans_free_tx_cmd(mld->trans, info->driver_data[1]);
+
+		memset(&info->status, 0, sizeof(info->status));
+
+		/* Packet was transmitted successfully, failures come as single
+		 * frames because before failing a frame the firmware transmits
+		 * it without aggregation at least once.
+		 */
+		if (!in_flush)
+			info->flags |= IEEE80211_TX_STAT_ACK;
+		else
+			info->flags &= ~IEEE80211_TX_STAT_ACK;
+
+		ieee80211_tx_status_skb(mld->hw, skb);
+	}
+}
+
+int iwl_mld_flush_link_sta_txqs(struct iwl_mld *mld, u32 fw_sta_id)
+{
+	struct iwl_tx_path_flush_cmd_rsp *rsp;
+	struct iwl_tx_path_flush_cmd flush_cmd = {
+		.sta_id = cpu_to_le32(fw_sta_id),
+		.tid_mask = cpu_to_le16(0xffff),
+	};
+	struct iwl_host_cmd cmd = {
+		.id = TXPATH_FLUSH,
+		.len = { sizeof(flush_cmd), },
+		.data = { &flush_cmd, },
+		.flags = CMD_WANT_SKB,
+	};
+	int ret, num_flushed_queues;
+	u32 resp_len;
+
+	IWL_DEBUG_TX_QUEUES(mld, "flush for sta id %d tid mask 0x%x\n",
+			    fw_sta_id, 0xffff);
+
+	ret = iwl_mld_send_cmd(mld, &cmd);
+	if (ret) {
+		IWL_ERR(mld, "Failed to send flush command (%d)\n", ret);
+		return ret;
+	}
+
+	resp_len = iwl_rx_packet_payload_len(cmd.resp_pkt);
+	if (IWL_FW_CHECK(mld, resp_len != sizeof(*rsp),
+			 "Invalid TXPATH_FLUSH response len: %d\n",
+			 resp_len)) {
+		ret = -EIO;
+		goto free_rsp;
+	}
+
+	rsp = (void *)cmd.resp_pkt->data;
+
+	if (IWL_FW_CHECK(mld, le16_to_cpu(rsp->sta_id) != fw_sta_id,
+			 "sta_id %d != rsp_sta_id %d\n", fw_sta_id,
+			 le16_to_cpu(rsp->sta_id))) {
+		ret = -EIO;
+		goto free_rsp;
+	}
+
+	num_flushed_queues = le16_to_cpu(rsp->num_flushed_queues);
+	if (IWL_FW_CHECK(mld, num_flushed_queues > IWL_TX_FLUSH_QUEUE_RSP,
+			 "num_flushed_queues %d\n", num_flushed_queues)) {
+		ret = -EIO;
+		goto free_rsp;
+	}
+
+	for (int i = 0; i < num_flushed_queues; i++) {
+		struct iwl_flush_queue_info *queue_info = &rsp->queues[i];
+		int read_after = le16_to_cpu(queue_info->read_after_flush);
+		int txq_id = le16_to_cpu(queue_info->queue_num);
+
+		if (IWL_FW_CHECK(mld,
+				 txq_id >= ARRAY_SIZE(mld->fw_id_to_txq),
+				 "Invalid txq id %d\n", txq_id))
+			continue;
+
+		IWL_DEBUG_TX_QUEUES(mld,
+				    "tid %d txq_id %d read-before %d read-after %d\n",
+				    le16_to_cpu(queue_info->tid), txq_id,
+				    le16_to_cpu(queue_info->read_before_flush),
+				    read_after);
+
+		iwl_mld_tx_reclaim_txq(mld, txq_id, read_after, true);
+	}
+
+free_rsp:
+	iwl_free_resp(&cmd);
+	return ret;
+}
+
+int iwl_mld_ensure_queue(struct iwl_mld *mld, struct ieee80211_txq *txq)
+{
+	struct iwl_mld_txq *mld_txq = iwl_mld_txq_from_mac80211(txq);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (likely(mld_txq->status.allocated))
+		return 0;
+
+	ret = iwl_mld_add_txq(mld, txq);
+
+	spin_lock_bh(&mld->add_txqs_lock);
+	if (!list_empty(&mld_txq->list))
+		list_del_init(&mld_txq->list);
+	spin_unlock_bh(&mld->add_txqs_lock);
+
+	return ret;
+}
+
+int iwl_mld_update_sta_txqs(struct iwl_mld *mld,
+			    struct ieee80211_sta *sta,
+			    u32 old_sta_mask, u32 new_sta_mask)
+{
+	struct iwl_scd_queue_cfg_cmd cmd = {
+		.operation = cpu_to_le32(IWL_SCD_QUEUE_MODIFY),
+		.u.modify.old_sta_mask = cpu_to_le32(old_sta_mask),
+		.u.modify.new_sta_mask = cpu_to_le32(new_sta_mask),
+	};
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	for (int tid = 0; tid <= IWL_MAX_TID_COUNT; tid++) {
+		struct ieee80211_txq *txq =
+			sta->txq[tid != IWL_MAX_TID_COUNT ?
+					tid : IEEE80211_NUM_TIDS];
+		struct iwl_mld_txq *mld_txq =
+			iwl_mld_txq_from_mac80211(txq);
+		int ret;
+
+		if (!mld_txq->status.allocated)
+			continue;
+
+		if (tid == IWL_MAX_TID_COUNT)
+			cmd.u.modify.tid = cpu_to_le32(IWL_MGMT_TID);
+		else
+			cmd.u.modify.tid = cpu_to_le32(tid);
+
+		ret = iwl_mld_send_cmd_pdu(mld,
+					   WIDE_ID(DATA_PATH_GROUP,
+						   SCD_QUEUE_CONFIG_CMD),
+					   &cmd);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+void iwl_mld_handle_compressed_ba_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt)
+{
+	struct iwl_compressed_ba_notif *ba_res = (void *)pkt->data;
+	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
+	u16 tfd_cnt = le16_to_cpu(ba_res->tfd_cnt);
+	u8 sta_id = ba_res->sta_id;
+	struct ieee80211_link_sta *link_sta;
+
+	if (!tfd_cnt)
+		return;
+
+	if (IWL_FW_CHECK(mld, struct_size(ba_res, tfd, tfd_cnt) > pkt_len,
+			 "Short BA notif (tfd_cnt=%d, size:0x%x)\n",
+			 tfd_cnt, pkt_len))
+		return;
+
+	IWL_DEBUG_TX_REPLY(mld,
+			   "BA notif received from sta_id=%d, flags=0x%x, sent:%d, acked:%d\n",
+			   sta_id, le32_to_cpu(ba_res->flags),
+			   le16_to_cpu(ba_res->txed),
+			   le16_to_cpu(ba_res->done));
+
+	for (int i = 0; i < tfd_cnt; i++) {
+		struct iwl_compressed_ba_tfd *ba_tfd = &ba_res->tfd[i];
+		int txq_id = le16_to_cpu(ba_tfd->q_num);
+		int index = le16_to_cpu(ba_tfd->tfd_index);
+
+		if (IWL_FW_CHECK(mld,
+				 txq_id >= ARRAY_SIZE(mld->fw_id_to_txq),
+				 "Invalid txq id %d\n", txq_id))
+			continue;
+
+		iwl_mld_tx_reclaim_txq(mld, txq_id, index, false);
+	}
+
+	if (IWL_FW_CHECK(mld, sta_id >= mld->fw->ucode_capa.num_stations,
+			 "Got invalid sta_id (%d)\n", sta_id))
+		return;
+
+	rcu_read_lock();
+
+	link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]);
+	if (IWL_FW_CHECK(mld, IS_ERR_OR_NULL(link_sta),
+			 "Got valid sta_id (%d) but link_sta is NULL\n",
+			 sta_id))
+		goto out;
+
+	iwl_mld_count_mpdu_tx(link_sta, le16_to_cpu(ba_res->txed));
+out:
+	rcu_read_unlock();
+}
diff --git a/sys/contrib/dev/iwlwifi/mld/tx.h b/sys/contrib/dev/iwlwifi/mld/tx.h
new file mode 100644
index 000000000000..520f15f9d33c
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mld/tx.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+#ifndef __iwl_mld_tx_h__
+#define __iwl_mld_tx_h__
+
+#include "mld.h"
+
+#define IWL_MLD_INVALID_QUEUE		0xFFFF
+#define IWL_MLD_INVALID_DROP_TX		0xFFFE
+
+/**
+ * struct iwl_mld_txq - TX Queue data
+ *
+ * @fw_id: the fw id of this txq. Only valid when &status.allocated is true.
+ * @status: bitmap of the txq status
+ * @status.allocated: Indicates that the queue was allocated.
+ * @status.stop_full: Indicates that the queue is full and should stop TXing.
+ * @list: list pointer, for &mld::txqs_to_add
+ * @tx_request: makes sure that if there are multiple threads that want to tx
+ *	from this txq, only one of them will do all the TXing.
+ *	This is needed to avoid spinning the trans txq lock, which is expensive
+ */
+struct iwl_mld_txq {
+	/* Add here fields that need clean up on restart */
+	struct_group(zeroed_on_hw_restart,
+		u16 fw_id;
+		struct {
+			u8 allocated:1;
+			u8 stop_full:1;
+		} status;
+	);
+	struct list_head list;
+	atomic_t tx_request;
+	/* And here fields that survive a fw restart */
+};
+
+static inline void iwl_mld_init_txq(struct iwl_mld_txq *mld_txq)
+{
+	INIT_LIST_HEAD(&mld_txq->list);
+	atomic_set(&mld_txq->tx_request, 0);
+}
+
+static inline struct iwl_mld_txq *
+iwl_mld_txq_from_mac80211(struct ieee80211_txq *txq)
+{
+	return (void *)txq->drv_priv;
+}
+
+void iwl_mld_add_txqs_wk(struct wiphy *wiphy, struct wiphy_work *wk);
+void iwl_mld_remove_txq(struct iwl_mld *mld, struct ieee80211_txq *txq);
+void iwl_mld_add_txq_list(struct iwl_mld *mld);
+void
+iwl_mld_free_txq(struct iwl_mld *mld, u32 fw_sta_mask, u32 tid, u32 queue_id);
+void iwl_mld_tx_from_txq(struct iwl_mld *mld, struct ieee80211_txq *txq);
+void iwl_mld_handle_tx_resp_notif(struct iwl_mld *mld,
+				  struct iwl_rx_packet *pkt);
+int iwl_mld_flush_link_sta_txqs(struct iwl_mld *mld, u32 fw_sta_id);
+int iwl_mld_ensure_queue(struct iwl_mld *mld, struct ieee80211_txq *txq);
+
+int iwl_mld_update_sta_txqs(struct iwl_mld *mld,
+			    struct ieee80211_sta *sta,
+			    u32 old_sta_mask, u32 new_sta_mask);
+
+void iwl_mld_handle_compressed_ba_notif(struct iwl_mld *mld,
+					struct iwl_rx_packet *pkt);
+void iwl_mld_toggle_tx_ant(struct iwl_mld *mld, u8 *ant);
+
+u8 iwl_mld_get_lowest_rate(struct iwl_mld *mld,
+			   struct ieee80211_tx_info *info,
+			   struct ieee80211_vif *vif);
+
+void iwl_mld_tx_skb(struct iwl_mld *mld, struct sk_buff *skb,
+		    struct ieee80211_txq *txq);
+
+#endif /* __iwl_mld_tx_h__ */
diff --git a/sys/contrib/dev/iwlwifi/mvm/binding.c b/sys/contrib/dev/iwlwifi/mvm/binding.c
index 458b97930059..58e9a940024d 100644
--- a/sys/contrib/dev/iwlwifi/mvm/binding.c
+++ b/sys/contrib/dev/iwlwifi/mvm/binding.c
@@ -2,7 +2,7 @@
 /*
  * Copyright (C) 2012-2014, 2020 Intel Corporation
  * Copyright (C) 2016 Intel Deutschland GmbH
- * Copyright (C) 2022 Intel Corporation
+ * Copyright (C) 2022, 2024 Intel Corporation
  */
 #include <net/mac80211.h>
 #include "fw-api.h"
@@ -158,9 +158,8 @@ int iwl_mvm_binding_remove_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	ret = iwl_mvm_binding_update(mvm, vif, mvmvif->deflink.phy_ctxt,
 				     false);
 
-	if (!ret)
-		if (iwl_mvm_sf_update(mvm, vif, true))
-			IWL_ERR(mvm, "Failed to update SF state\n");
+	if (!ret && iwl_mvm_sf_update(mvm, vif, true))
+		IWL_ERR(mvm, "Failed to update SF state\n");
 
 	return ret;
 }
diff --git a/sys/contrib/dev/iwlwifi/mvm/coex.c b/sys/contrib/dev/iwlwifi/mvm/coex.c
index ad3e14a0d043..13cdc077d8d3 100644
--- a/sys/contrib/dev/iwlwifi/mvm/coex.c
+++ b/sys/contrib/dev/iwlwifi/mvm/coex.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2013-2014, 2018-2020, 2022-2024 Intel Corporation
+ * Copyright (C) 2013-2014, 2018-2020, 2022-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  */
 #include <linux/ieee80211.h>
@@ -181,7 +181,7 @@ static int iwl_mvm_bt_coex_reduced_txp(struct iwl_mvm *mvm, u8 sta_id,
 	struct iwl_mvm_sta *mvmsta;
 	u32 value;
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		return 0;
 
 	mvmsta = iwl_mvm_sta_from_staid_protected(mvm, sta_id);
@@ -208,7 +208,7 @@ static int iwl_mvm_bt_coex_reduced_txp(struct iwl_mvm *mvm, u8 sta_id,
 }
 
 struct iwl_bt_iterator_data {
-	struct iwl_bt_coex_profile_notif *notif;
+	struct iwl_bt_coex_prof_old_notif *notif;
 	struct iwl_mvm *mvm;
 	struct ieee80211_chanctx_conf *primary;
 	struct ieee80211_chanctx_conf *secondary;
@@ -266,10 +266,26 @@ iwl_mvm_bt_coex_calculate_esr_mode(struct iwl_mvm *mvm,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	bool have_wifi_loss_rate =
 		iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
-					BT_PROFILE_NOTIFICATION, 0) > 4;
+					BT_PROFILE_NOTIFICATION, 0) > 4 ||
+		iwl_fw_lookup_notif_ver(mvm->fw, BT_COEX_GROUP,
+					PROFILE_NOTIF, 0) >= 1;
+	u8 wifi_loss_mid_high_rssi;
+	u8 wifi_loss_low_rssi;
 	u8 wifi_loss_rate;
 
-	if (mvm->last_bt_notif.wifi_loss_low_rssi == BT_OFF)
+	if (iwl_fw_lookup_notif_ver(mvm->fw, BT_COEX_GROUP,
+				    PROFILE_NOTIF, 0) >= 1) {
+		/* For now, we consider 2.4 GHz band / ANT_A only */
+		wifi_loss_mid_high_rssi =
+			mvm->last_bt_wifi_loss.wifi_loss_mid_high_rssi[PHY_BAND_24][0];
+		wifi_loss_low_rssi =
+			mvm->last_bt_wifi_loss.wifi_loss_low_rssi[PHY_BAND_24][0];
+	} else {
+		wifi_loss_mid_high_rssi = mvm->last_bt_notif.wifi_loss_mid_high_rssi;
+		wifi_loss_low_rssi = mvm->last_bt_notif.wifi_loss_low_rssi;
+	}
+
+	if (wifi_loss_low_rssi == BT_OFF)
 		return true;
 
 	if (primary)
@@ -286,20 +302,20 @@ iwl_mvm_bt_coex_calculate_esr_mode(struct iwl_mvm *mvm,
 	 * we will get an update on this and exit eSR.
 	 */
 	if (!link_rssi)
-		wifi_loss_rate = mvm->last_bt_notif.wifi_loss_mid_high_rssi;
+		wifi_loss_rate = wifi_loss_mid_high_rssi;
 
 	else if (mvmvif->esr_active)
 		 /* RSSI needs to get really low to disable eSR... */
 		wifi_loss_rate =
 			link_rssi <= -IWL_MVM_BT_COEX_DISABLE_ESR_THRESH ?
-				mvm->last_bt_notif.wifi_loss_low_rssi :
-				mvm->last_bt_notif.wifi_loss_mid_high_rssi;
+				wifi_loss_low_rssi :
+				wifi_loss_mid_high_rssi;
 	else
 		/* ...And really high before we enable it back */
 		wifi_loss_rate =
 			link_rssi <= -IWL_MVM_BT_COEX_ENABLE_ESR_THRESH ?
-				mvm->last_bt_notif.wifi_loss_low_rssi :
-				mvm->last_bt_notif.wifi_loss_mid_high_rssi;
+				wifi_loss_low_rssi :
+				wifi_loss_mid_high_rssi;
 
 	return wifi_loss_rate <= IWL_MVM_BT_COEX_WIFI_LOSS_THRESH;
 }
@@ -509,6 +525,32 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 		iwl_mvm_bt_notif_per_link(mvm, vif, data, link_id);
 }
 
+/* must be called under rcu_read_lock */
+static void iwl_mvm_bt_coex_notif_iterator(void *_data, u8 *mac,
+					   struct ieee80211_vif *vif)
+{
+	struct iwl_mvm *mvm = _data;
+	struct ieee80211_bss_conf *link_conf;
+	unsigned int link_id;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	for_each_vif_active_link(vif, link_conf, link_id) {
+		struct ieee80211_chanctx_conf *chanctx_conf =
+			rcu_dereference_check(link_conf->chanctx_conf,
+					      lockdep_is_held(&mvm->mutex));
+
+		if ((!chanctx_conf ||
+		     chanctx_conf->def.chan->band != NL80211_BAND_2GHZ))
+			continue;
+
+		iwl_mvm_bt_coex_update_link_esr(mvm, vif, link_id);
+	}
+}
+
 static void iwl_mvm_bt_coex_notif_handle(struct iwl_mvm *mvm)
 {
 	struct iwl_bt_iterator_data data = {
@@ -527,7 +569,7 @@ static void iwl_mvm_bt_coex_notif_handle(struct iwl_mvm *mvm)
 					mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
 					iwl_mvm_bt_notif_iterator, &data);
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		rcu_read_unlock();
 		return;
 	}
@@ -591,11 +633,11 @@ static void iwl_mvm_bt_coex_notif_handle(struct iwl_mvm *mvm)
 	}
 }
 
-void iwl_mvm_rx_bt_coex_notif(struct iwl_mvm *mvm,
-			      struct iwl_rx_cmd_buffer *rxb)
+void iwl_mvm_rx_bt_coex_old_notif(struct iwl_mvm *mvm,
+				  struct iwl_rx_cmd_buffer *rxb)
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_bt_coex_profile_notif *notif = (void *)pkt->data;
+	struct iwl_bt_coex_prof_old_notif *notif = (void *)pkt->data;
 
 	IWL_DEBUG_COEX(mvm, "BT Coex Notification received\n");
 	IWL_DEBUG_COEX(mvm, "\tBT ci compliance %d\n", notif->bt_ci_compliance);
@@ -612,6 +654,22 @@ void iwl_mvm_rx_bt_coex_notif(struct iwl_mvm *mvm,
 	iwl_mvm_bt_coex_notif_handle(mvm);
 }
 
+void iwl_mvm_rx_bt_coex_notif(struct iwl_mvm *mvm,
+			      struct iwl_rx_cmd_buffer *rxb)
+{
+	const struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	const struct iwl_bt_coex_profile_notif *notif = (const void *)pkt->data;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	mvm->last_bt_wifi_loss = *notif;
+
+	ieee80211_iterate_active_interfaces(mvm->hw,
+					    IEEE80211_IFACE_ITER_NORMAL,
+					    iwl_mvm_bt_coex_notif_iterator,
+					    mvm);
+}
+
 void iwl_mvm_bt_rssi_event(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			   enum ieee80211_rssi_event_data rssi_event)
 {
@@ -628,7 +686,7 @@ void iwl_mvm_bt_rssi_event(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 * Rssi update while not associated - can happen since the statistics
 	 * are handled asynchronously
 	 */
-	if (mvmvif->deflink.ap_sta_id == IWL_MVM_INVALID_STA)
+	if (mvmvif->deflink.ap_sta_id == IWL_INVALID_STA)
 		return;
 
 	/* No BT - reports should be disabled */
diff --git a/sys/contrib/dev/iwlwifi/mvm/constants.h b/sys/contrib/dev/iwlwifi/mvm/constants.h
index c4c1e67b9ac7..776600ddaea6 100644
--- a/sys/contrib/dev/iwlwifi/mvm/constants.h
+++ b/sys/contrib/dev/iwlwifi/mvm/constants.h
@@ -16,6 +16,10 @@
 #define IWL_MVM_BT_COEX_WIFI_LOSS_THRESH	0
 #define IWL_MVM_TRIGGER_LINK_SEL_TIME_SEC	30
 #define IWL_MVM_TPT_COUNT_WINDOW_SEC		5
+#define IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS	5
+#define IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH	15
+#define IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED	11
+#define IWL_MVM_LOW_RSSI_MLO_SCAN_THRESH	-72
 
 #define IWL_MVM_DEFAULT_PS_TX_DATA_TIMEOUT	(100 * USEC_PER_MSEC)
 #define IWL_MVM_DEFAULT_PS_RX_DATA_TIMEOUT	(100 * USEC_PER_MSEC)
@@ -55,7 +59,6 @@
 #define IWL_MVM_RS_RSSI_BASED_INIT_RATE         0
 #define IWL_MVM_RS_80_20_FAR_RANGE_TWEAK	1
 #define IWL_MVM_TOF_IS_RESPONDER		0
-#define IWL_MVM_HW_CSUM_DISABLE			0
 #define IWL_MVM_ADWELL_ENABLE			1
 #define IWL_MVM_ADWELL_MAX_BUDGET		0
 #define IWL_MVM_TCM_LOAD_MEDIUM_THRESH		10 /* percentage */
@@ -109,7 +112,7 @@
 #define IWL_MVM_FTM_INITIATOR_SECURE_LTF	false
 #define IWL_MVM_FTM_RESP_NDP_SUPPORT		true
 #define IWL_MVM_FTM_RESP_LMR_FEEDBACK_SUPPORT	true
-#define IWL_MVM_FTM_NON_TB_MIN_TIME_BETWEEN_MSR	5
+#define IWL_MVM_FTM_NON_TB_MIN_TIME_BETWEEN_MSR	7
 #define IWL_MVM_FTM_NON_TB_MAX_TIME_BETWEEN_MSR	1000
 #define IWL_MVM_D3_DEBUG			false
 #define IWL_MVM_USE_TWT				true
@@ -125,7 +128,6 @@
 #define IWL_MVM_6GHZ_PASSIVE_SCAN_ASSOC_TIMEOUT 60   /* in seconds */
 #define IWL_MVM_MIN_BEACON_INTERVAL_TU		16
 #define IWL_MVM_AUTO_EML_ENABLE                 true
-#define IWL_MVM_MISSED_BEACONS_EXIT_ESR_THRESH	7
 
 #define IWL_MVM_HIGH_RSSI_THRESH_20MHZ		-67
 #define IWL_MVM_LOW_RSSI_THRESH_20MHZ		-71
diff --git a/sys/contrib/dev/iwlwifi/mvm/d3.c b/sys/contrib/dev/iwlwifi/mvm/d3.c
index 11d9911eabb1..c7d298294ec1 100644
--- a/sys/contrib/dev/iwlwifi/mvm/d3.c
+++ b/sys/contrib/dev/iwlwifi/mvm/d3.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -124,19 +124,17 @@ static void iwl_mvm_wowlan_program_keys(struct ieee80211_hw *hw,
 	switch (key->cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
 	case WLAN_CIPHER_SUITE_WEP104: { /* hack it for now */
-		struct {
-			struct iwl_mvm_wep_key_cmd wep_key_cmd;
-			struct iwl_mvm_wep_key wep_key;
-		} __packed wkc = {
-			.wep_key_cmd.mac_id_n_color =
-				cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->id,
-								mvmvif->color)),
-			.wep_key_cmd.num_keys = 1,
-			/* firmware sets STA_KEY_FLG_WEP_13BYTES */
-			.wep_key_cmd.decryption_type = STA_KEY_FLG_WEP,
-			.wep_key.key_index = key->keyidx,
-			.wep_key.key_size = key->keylen,
-		};
+		DEFINE_RAW_FLEX(struct iwl_mvm_wep_key_cmd, wkc, wep_key, 1);
+		struct iwl_mvm_wep_key *wep_key = wkc->wep_key;
+
+		wkc->mac_id_n_color =
+			cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->id,
+							mvmvif->color));
+		wkc->num_keys = 1;
+		/* firmware sets STA_KEY_FLG_WEP_13BYTES */
+		wkc->decryption_type = STA_KEY_FLG_WEP;
+		wep_key->key_index = key->keyidx;
+		wep_key->key_size = key->keylen;
 
 		/*
 		 * This will fail -- the key functions don't set support
@@ -146,18 +144,19 @@ static void iwl_mvm_wowlan_program_keys(struct ieee80211_hw *hw,
 		if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE)
 			break;
 
-		memcpy(&wkc.wep_key.key[3], key->key, key->keylen);
+		memcpy(&wep_key->key[3], key->key, key->keylen);
 		if (key->keyidx == mvmvif->tx_key_idx) {
 			/* TX key must be at offset 0 */
-			wkc.wep_key.key_offset = 0;
+			wep_key->key_offset = 0;
 		} else {
 			/* others start at 1 */
 			data->wep_key_idx++;
-			wkc.wep_key.key_offset = data->wep_key_idx;
+			wep_key->key_offset = data->wep_key_idx;
 		}
 
 		mutex_lock(&mvm->mutex);
-		ret = iwl_mvm_send_cmd_pdu(mvm, WEP_KEY, 0, sizeof(wkc), &wkc);
+		ret = iwl_mvm_send_cmd_pdu(mvm, WEP_KEY, 0,
+					   __struct_size(wkc), wkc);
 		data->error = ret != 0;
 
 		mvm->ptk_ivlen = key->iv_len;
@@ -216,7 +215,7 @@ static void iwl_mvm_wowlan_program_keys(struct ieee80211_hw *hw,
 }
 
 struct wowlan_key_rsc_tsc_data {
-	struct iwl_wowlan_rsc_tsc_params_cmd_v4 *rsc_tsc;
+	struct iwl_wowlan_rsc_tsc_params_cmd_ver_2 *rsc_tsc;
 	bool have_rsc_tsc;
 };
 
@@ -241,21 +240,21 @@ static void iwl_mvm_wowlan_get_rsc_tsc_data(struct ieee80211_hw *hw,
 			u64 pn64;
 
 			tkip_sc =
-			   data->rsc_tsc->params.all_tsc_rsc.tkip.unicast_rsc;
+			   data->rsc_tsc->all_tsc_rsc.tkip.unicast_rsc;
 			tkip_tx_sc =
-				&data->rsc_tsc->params.all_tsc_rsc.tkip.tsc;
+				&data->rsc_tsc->all_tsc_rsc.tkip.tsc;
 
 			pn64 = atomic64_read(&key->tx_pn);
 			tkip_tx_sc->iv16 = cpu_to_le16(TKIP_PN_TO_IV16(pn64));
 			tkip_tx_sc->iv32 = cpu_to_le32(TKIP_PN_TO_IV32(pn64));
 		} else {
 			tkip_sc =
-			  data->rsc_tsc->params.all_tsc_rsc.tkip.multicast_rsc;
+			  data->rsc_tsc->all_tsc_rsc.tkip.multicast_rsc;
 		}
 
 		/*
 		 * For non-QoS this relies on the fact that both the uCode and
-		 * mac80211 use TID 0 (as they need to to avoid replay attacks)
+		 * mac80211 use TID 0 (as they need to avoid replay attacks)
 		 * for checking the IV in the frames.
 		 */
 		for (i = 0; i < IWL_NUM_RSC; i++) {
@@ -274,15 +273,15 @@ static void iwl_mvm_wowlan_get_rsc_tsc_data(struct ieee80211_hw *hw,
 			u64 pn64;
 
 			aes_sc =
-			   data->rsc_tsc->params.all_tsc_rsc.aes.unicast_rsc;
+			   data->rsc_tsc->all_tsc_rsc.aes.unicast_rsc;
 			aes_tx_sc =
-				&data->rsc_tsc->params.all_tsc_rsc.aes.tsc;
+				&data->rsc_tsc->all_tsc_rsc.aes.tsc;
 
 			pn64 = atomic64_read(&key->tx_pn);
 			aes_tx_sc->pn = cpu_to_le64(pn64);
 		} else {
 			aes_sc =
-			   data->rsc_tsc->params.all_tsc_rsc.aes.multicast_rsc;
+			   data->rsc_tsc->all_tsc_rsc.aes.multicast_rsc;
 		}
 
 		/*
@@ -304,7 +303,7 @@ static void iwl_mvm_wowlan_get_rsc_tsc_data(struct ieee80211_hw *hw,
 
 			for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
 				pn = iwl_mvm_find_max_pn(key, ptk_pn, &seq, i,
-						mvm->trans->num_rx_queues);
+						mvm->trans->info.num_rxqs);
 				aes_sc[i].pn = cpu_to_le64((u64)pn[5] |
 							   ((u64)pn[4] << 8) |
 							   ((u64)pn[3] << 16) |
@@ -391,7 +390,7 @@ static void iwl_mvm_wowlan_get_rsc_v5_data(struct ieee80211_hw *hw,
 
 		/*
 		 * For non-QoS this relies on the fact that both the uCode and
-		 * mac80211 use TID 0 (as they need to to avoid replay attacks)
+		 * mac80211 use TID 0 (as they need to avoid replay attacks)
 		 * for checking the IV in the frames.
 		 */
 		for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
@@ -425,7 +424,7 @@ static void iwl_mvm_wowlan_get_rsc_v5_data(struct ieee80211_hw *hw,
 
 			for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
 				pn = iwl_mvm_find_max_pn(key, ptk_pn, &seq, i,
-						mvm->trans->num_rx_queues);
+						mvm->trans->info.num_rxqs);
 				rsc[i] = cpu_to_le64((u64)pn[5] |
 						     ((u64)pn[4] << 8) |
 						     ((u64)pn[3] << 16) |
@@ -485,30 +484,21 @@ static int iwl_mvm_wowlan_config_rsc_tsc(struct iwl_mvm *mvm,
 		else
 			ret = 0;
 		kfree(data.rsc);
-	} else if (ver == 4 || ver == 2 || ver == IWL_FW_CMD_VER_UNKNOWN) {
+	} else if (ver == 2 || ver == IWL_FW_CMD_VER_UNKNOWN) {
 		struct wowlan_key_rsc_tsc_data data = {};
-		int size;
 
 		data.rsc_tsc = kzalloc(sizeof(*data.rsc_tsc), GFP_KERNEL);
 		if (!data.rsc_tsc)
 			return -ENOMEM;
 
-		if (ver == 4) {
-			size = sizeof(*data.rsc_tsc);
-			data.rsc_tsc->sta_id =
-				cpu_to_le32(mvm_link->ap_sta_id);
-		} else {
-			/* ver == 2 || ver == IWL_FW_CMD_VER_UNKNOWN */
-			size = sizeof(data.rsc_tsc->params);
-		}
-
 		ieee80211_iter_keys(mvm->hw, vif,
 				    iwl_mvm_wowlan_get_rsc_tsc_data,
 				    &data);
 
 		if (data.have_rsc_tsc)
 			ret = iwl_mvm_send_cmd_pdu(mvm, WOWLAN_TSC_RSC_PARAM,
-						   CMD_ASYNC, size,
+						   CMD_ASYNC,
+						   sizeof(*data.rsc_tsc),
 						   data.rsc_tsc);
 		else
 			ret = 0;
@@ -926,7 +916,7 @@ static int iwl_mvm_switch_to_d3(struct iwl_mvm *mvm)
 static int
 iwl_mvm_get_wowlan_config(struct iwl_mvm *mvm,
 			  struct cfg80211_wowlan *wowlan,
-			  struct iwl_wowlan_config_cmd *wowlan_config_cmd,
+			  struct iwl_wowlan_config_cmd_v6 *wowlan_config_cmd,
 			  struct ieee80211_vif *vif, struct iwl_mvm_vif *mvmvif,
 			  struct ieee80211_sta *ap_sta)
 {
@@ -952,7 +942,8 @@ iwl_mvm_get_wowlan_config(struct iwl_mvm *mvm,
 		wowlan_config_cmd->non_qos_seq = cpu_to_le16(ret);
 	}
 
-	iwl_mvm_set_wowlan_qos_seq(mvm_ap_sta, wowlan_config_cmd);
+	if (iwl_fw_lookup_cmd_ver(mvm->fw, WOWLAN_CONFIGURATION, 0) < 7)
+		iwl_mvm_set_wowlan_qos_seq(mvm_ap_sta, wowlan_config_cmd);
 
 	if (wowlan->disconnect)
 		wowlan_config_cmd->wakeup_filter |=
@@ -1126,7 +1117,7 @@ static int iwl_mvm_wowlan_config_key_params(struct iwl_mvm *mvm,
 static int
 iwl_mvm_wowlan_config(struct iwl_mvm *mvm,
 		      struct cfg80211_wowlan *wowlan,
-		      struct iwl_wowlan_config_cmd *wowlan_config_cmd,
+		      struct iwl_wowlan_config_cmd_v6 *wowlan_config_cmd_v6,
 		      struct ieee80211_vif *vif, struct iwl_mvm_vif *mvmvif,
 		      struct iwl_mvm_vif_link_info *mvm_link,
 		      struct ieee80211_sta *ap_sta)
@@ -1135,7 +1126,7 @@ iwl_mvm_wowlan_config(struct iwl_mvm *mvm,
 	bool unified_image = fw_has_capa(&mvm->fw->ucode_capa,
 					 IWL_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG);
 
-	mvm->offload_tid = wowlan_config_cmd->offloading_tid;
+	mvm->offload_tid = wowlan_config_cmd_v6->offloading_tid;
 
 	if (!unified_image) {
 		ret = iwl_mvm_switch_to_d3(mvm);
@@ -1151,9 +1142,26 @@ iwl_mvm_wowlan_config(struct iwl_mvm *mvm,
 	if (ret)
 		return ret;
 
-	ret = iwl_mvm_send_cmd_pdu(mvm, WOWLAN_CONFIGURATION, 0,
-				   sizeof(*wowlan_config_cmd),
-				   wowlan_config_cmd);
+	if (iwl_fw_lookup_cmd_ver(mvm->fw, WOWLAN_CONFIGURATION, 0) > 6) {
+		struct iwl_wowlan_config_cmd wowlan_config_cmd = {
+			.wakeup_filter = wowlan_config_cmd_v6->wakeup_filter,
+			.wowlan_ba_teardown_tids =
+				wowlan_config_cmd_v6->wowlan_ba_teardown_tids,
+			.is_11n_connection =
+				wowlan_config_cmd_v6->is_11n_connection,
+			.offloading_tid = wowlan_config_cmd_v6->offloading_tid,
+			.flags = wowlan_config_cmd_v6->flags,
+			.sta_id = wowlan_config_cmd_v6->sta_id,
+		};
+
+		ret = iwl_mvm_send_cmd_pdu(mvm, WOWLAN_CONFIGURATION, 0,
+					   sizeof(wowlan_config_cmd),
+					   &wowlan_config_cmd);
+	} else {
+		ret = iwl_mvm_send_cmd_pdu(mvm, WOWLAN_CONFIGURATION, 0,
+					   sizeof(*wowlan_config_cmd_v6),
+					   wowlan_config_cmd_v6);
+	}
 	if (ret)
 		return ret;
 
@@ -1252,7 +1260,7 @@ static int __iwl_mvm_suspend(struct ieee80211_hw *hw,
 	};
 	struct iwl_host_cmd d3_cfg_cmd = {
 		.id = D3_CONFIG_CMD,
-		.flags = CMD_WANT_SKB | CMD_SEND_IN_D3,
+		.flags = CMD_WANT_SKB,
 		.data[0] = &d3_cfg_cmd_data,
 		.len[0] = sizeof(d3_cfg_cmd_data),
 	};
@@ -1292,7 +1300,7 @@ static int __iwl_mvm_suspend(struct ieee80211_hw *hw,
 		goto out_noreset;
 	}
 
-	if (mvm_link->ap_sta_id == IWL_MVM_INVALID_STA) {
+	if (mvm_link->ap_sta_id == IWL_INVALID_STA) {
 		/* if we're not associated, this must be netdetect */
 		if (!wowlan->nd_config) {
 			ret = 1;
@@ -1306,7 +1314,7 @@ static int __iwl_mvm_suspend(struct ieee80211_hw *hw,
 
 		mvm->net_detect = true;
 	} else {
-		struct iwl_wowlan_config_cmd wowlan_config_cmd = {
+		struct iwl_wowlan_config_cmd_v6 wowlan_config_cmd = {
 			.offloading_tid = 0,
 		};
 
@@ -1356,11 +1364,9 @@ static int __iwl_mvm_suspend(struct ieee80211_hw *hw,
 	 * recording before entering D3. In later devices the FW stops the
 	 * recording automatically.
 	 */
-	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000)
+	if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
 		iwl_fw_dbg_stop_restart_recording(&mvm->fwrt, NULL, true);
 
-	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_D3;
-
 	/* must be last -- this switches firmware state */
 	ret = iwl_mvm_send_cmd(mvm, &d3_cfg_cmd);
 	if (ret)
@@ -1381,13 +1387,6 @@ static int __iwl_mvm_suspend(struct ieee80211_hw *hw,
 	if (ret < 0) {
 		iwl_mvm_free_nd(mvm);
 
-		if (!unified_image) {
-			if (mvm->fw_restart > 0) {
-				mvm->fw_restart--;
-				ieee80211_restart_hw(mvm->hw);
-			}
-		}
-
 		clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status);
 	}
  out_noreset:
@@ -1402,7 +1401,9 @@ int iwl_mvm_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan)
 
 	iwl_mvm_pause_tcm(mvm, true);
 
+	mutex_lock(&mvm->mutex);
 	iwl_fw_runtime_suspend(&mvm->fwrt);
+	mutex_unlock(&mvm->mutex);
 
 	return __iwl_mvm_suspend(hw, wowlan, false);
 }
@@ -1427,6 +1428,7 @@ struct iwl_wowlan_status_data {
 	u16 non_qos_seq_ctr;
 	u16 qos_seq_ctr[8];
 	u8 tid_tear_down;
+	u8 tid_offloaded_tx;
 
 	struct {
 		/* including RX MIC key for TKIP */
@@ -1467,9 +1469,6 @@ struct iwl_wowlan_status_data {
 	struct iwl_multicast_key_data igtk;
 	struct iwl_multicast_key_data bigtk[WOWLAN_BIGTK_KEYS_NUM];
 
-	int num_mlo_keys;
-	struct iwl_wowlan_mlo_gtk mlo_keys[WOWLAN_MAX_MLO_KEYS];
-
 	u8 *wake_packet;
 };
 
@@ -1676,7 +1675,7 @@ static void iwl_mvm_set_aes_ptk_rx_seq(struct iwl_mvm *mvm,
 	for (tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
 		int i;
 
-		for (i = 1; i < mvm->trans->num_rx_queues; i++)
+		for (i = 1; i < mvm->trans->info.num_rxqs; i++)
 			memcpy(ptk_pn->q[i].pn[tid],
 			       status->ptk.aes.seq[tid].ccmp.pn,
 			       IEEE80211_CCMP_PN_LEN);
@@ -1685,7 +1684,7 @@ static void iwl_mvm_set_aes_ptk_rx_seq(struct iwl_mvm *mvm,
 }
 
 static void iwl_mvm_convert_key_counters(struct iwl_wowlan_status_data *status,
-					 union iwl_all_tsc_rsc *sc)
+					 union iwl_all_tsc_rsc *sc, u8 key_idx)
 {
 	int i;
 
@@ -1700,7 +1699,7 @@ static void iwl_mvm_convert_key_counters(struct iwl_wowlan_status_data *status,
 				      &status->gtk_seq[0].aes.seq[i]);
 	}
 	status->gtk_seq[0].valid = true;
-	status->gtk_seq[0].key_id = -1;
+	status->gtk_seq[0].key_id = key_idx;
 
 	/* PTK TX counter */
 	status->ptk.tkip.tx_pn = (u64)le16_to_cpu(sc->tkip.tsc.iv16) |
@@ -1783,8 +1782,7 @@ static void iwl_mvm_set_key_rx_seq_idx(struct ieee80211_key_conf *key,
 }
 
 static void iwl_mvm_set_key_rx_seq(struct ieee80211_key_conf *key,
-				   struct iwl_wowlan_status_data *status,
-				   bool installed)
+				   struct iwl_wowlan_status_data *status)
 {
 	int i;
 
@@ -1792,23 +1790,7 @@ static void iwl_mvm_set_key_rx_seq(struct ieee80211_key_conf *key,
 		if (!status->gtk_seq[i].valid)
 			continue;
 
-		/* Handle the case where we know the key ID */
-		if (status->gtk_seq[i].key_id == key->keyidx) {
-			s8 new_key_id = -1;
-
-			if (status->num_of_gtk_rekeys)
-				new_key_id = status->gtk[0].flags &
-						IWL_WOWLAN_GTK_IDX_MASK;
-
-			/* Don't install a new key's value to an old key */
-			if (new_key_id != key->keyidx)
-				iwl_mvm_set_key_rx_seq_idx(key, status, i);
-			continue;
-		}
-
-		/* handle the case where we didn't, last key only */
-		if (status->gtk_seq[i].key_id == -1 &&
-		    (!status->num_of_gtk_rekeys || installed))
+		if (status->gtk_seq[i].key_id == key->keyidx)
 			iwl_mvm_set_key_rx_seq_idx(key, status, i);
 	}
 }
@@ -1898,17 +1880,10 @@ iwl_mvm_d3_update_igtk_bigtk(struct iwl_wowlan_status_data *status,
 			     struct ieee80211_key_conf *key,
 			     struct iwl_multicast_key_data *key_data)
 {
-	if (status->num_of_gtk_rekeys && key_data->len) {
-		/* remove rekeyed key */
-		ieee80211_remove_key(key);
-	} else {
-		struct ieee80211_key_seq seq;
+	struct ieee80211_key_seq seq;
 
-		iwl_mvm_d3_set_igtk_bigtk_ipn(key_data,
-					      &seq,
-					      key->cipher);
-		ieee80211_set_key_rx_seq(key, 0, &seq);
-	}
+	iwl_mvm_d3_set_igtk_bigtk_ipn(key_data, &seq, key->cipher);
+	ieee80211_set_key_rx_seq(key, 0, &seq);
 }
 
 static void iwl_mvm_d3_update_keys(struct ieee80211_hw *hw,
@@ -1949,18 +1924,13 @@ static void iwl_mvm_d3_update_keys(struct ieee80211_hw *hw,
 			return;
 		}
 		keyidx = key->keyidx;
-		/* The current key is always sent by the FW, even if it wasn't
-		 * rekeyed during D3.
-		 * We remove an existing key if it has the same index as
-		 * a new key
+		/*
+		 * Update the seq even if there was a rekey. If there was a
+		 * rekey, we will update again after replacing the key
 		 */
-		if (status->num_of_gtk_rekeys &&
-		    ((status->gtk[0].len && keyidx == status->gtk[0].id) ||
-		     (status->gtk[1].len && keyidx == status->gtk[1].id))) {
-			ieee80211_remove_key(key);
-		} else {
-			iwl_mvm_set_key_rx_seq(key, data->status, false);
-		}
+		if ((status->gtk[0].len && keyidx == status->gtk[0].id) ||
+		    (status->gtk[1].len && keyidx == status->gtk[1].id))
+			iwl_mvm_set_key_rx_seq(key, status);
 		break;
 	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
 	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
@@ -1979,199 +1949,35 @@ static void iwl_mvm_d3_update_keys(struct ieee80211_hw *hw,
 	}
 }
 
-struct iwl_mvm_d3_mlo_old_keys {
-	u32 cipher[IEEE80211_MLD_MAX_NUM_LINKS][WOWLAN_MLO_GTK_KEY_NUM_TYPES];
-	struct ieee80211_key_conf *key[IEEE80211_MLD_MAX_NUM_LINKS][8];
-};
-
-static void iwl_mvm_mlo_key_ciphers(struct ieee80211_hw *hw,
-				    struct ieee80211_vif *vif,
-				    struct ieee80211_sta *sta,
-				    struct ieee80211_key_conf *key,
-				    void *data)
-{
-	struct iwl_mvm_d3_mlo_old_keys *old_keys = data;
-	enum iwl_wowlan_mlo_gtk_type key_type;
-
-	if (key->link_id < 0)
-		return;
-
-	if (WARN_ON(key->link_id >= IEEE80211_MLD_MAX_NUM_LINKS ||
-		    key->keyidx >= 8))
-		return;
-
-	if (WARN_ON(old_keys->key[key->link_id][key->keyidx]))
-		return;
-
-	switch (key->cipher) {
-	case WLAN_CIPHER_SUITE_CCMP:
-	case WLAN_CIPHER_SUITE_GCMP:
-	case WLAN_CIPHER_SUITE_GCMP_256:
-		key_type = WOWLAN_MLO_GTK_KEY_TYPE_GTK;
-		break;
-	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
-	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
-	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
-	case WLAN_CIPHER_SUITE_AES_CMAC:
-		if (key->keyidx == 4 || key->keyidx == 5) {
-			key_type = WOWLAN_MLO_GTK_KEY_TYPE_IGTK;
-			break;
-		} else if (key->keyidx == 6 || key->keyidx == 7) {
-			key_type = WOWLAN_MLO_GTK_KEY_TYPE_BIGTK;
-			break;
-		}
-		return;
-	default:
-		/* ignore WEP/TKIP or unknown ciphers */
-		return;
-	}
-
-	old_keys->cipher[key->link_id][key_type] = key->cipher;
-	old_keys->key[key->link_id][key->keyidx] = key;
-}
-
-static bool iwl_mvm_mlo_gtk_rekey(struct iwl_wowlan_status_data *status,
-				  struct ieee80211_vif *vif,
-				  struct iwl_mvm *mvm)
-{
-	int i;
-	struct iwl_mvm_d3_mlo_old_keys *old_keys;
-	bool ret = true;
-
-	IWL_DEBUG_WOWLAN(mvm, "Num of MLO Keys: %d\n", status->num_mlo_keys);
-	if (!status->num_mlo_keys)
-		return true;
-
-	old_keys = kzalloc(sizeof(*old_keys), GFP_KERNEL);
-	if (!old_keys)
-		return false;
-
-	/* find the cipher for each mlo key */
-	ieee80211_iter_keys(mvm->hw, vif, iwl_mvm_mlo_key_ciphers, old_keys);
-
-	for (i = 0; i < status->num_mlo_keys; i++) {
-		struct iwl_wowlan_mlo_gtk *mlo_key = &status->mlo_keys[i];
-		struct ieee80211_key_conf *key, *old_key;
-		struct ieee80211_key_seq seq;
-		struct {
-			struct ieee80211_key_conf conf;
-			u8 key[32];
-		} conf = {};
-		u16 flags = le16_to_cpu(mlo_key->flags);
-		int j, link_id, key_id, key_type;
-
-		link_id = u16_get_bits(flags, WOWLAN_MLO_GTK_FLAG_LINK_ID_MSK);
-		key_id = u16_get_bits(flags, WOWLAN_MLO_GTK_FLAG_KEY_ID_MSK);
-		key_type = u16_get_bits(flags,
-					WOWLAN_MLO_GTK_FLAG_KEY_TYPE_MSK);
-
-		if (!(vif->valid_links & BIT(link_id)))
-			continue;
-
-		if (WARN_ON(link_id >= IEEE80211_MLD_MAX_NUM_LINKS ||
-			    key_id >= 8 ||
-			    key_type >= WOWLAN_MLO_GTK_KEY_NUM_TYPES))
-			continue;
-
-		conf.conf.cipher = old_keys->cipher[link_id][key_type];
-		/* WARN_ON? */
-		if (!conf.conf.cipher)
-			continue;
-
-		conf.conf.keylen = 0;
-		switch (conf.conf.cipher) {
-		case WLAN_CIPHER_SUITE_CCMP:
-		case WLAN_CIPHER_SUITE_GCMP:
-			conf.conf.keylen = WLAN_KEY_LEN_CCMP;
-			break;
-		case WLAN_CIPHER_SUITE_GCMP_256:
-			conf.conf.keylen = WLAN_KEY_LEN_GCMP_256;
-			break;
-		case WLAN_CIPHER_SUITE_BIP_GMAC_128:
-			conf.conf.keylen = WLAN_KEY_LEN_BIP_GMAC_128;
-			break;
-		case WLAN_CIPHER_SUITE_BIP_GMAC_256:
-			conf.conf.keylen = WLAN_KEY_LEN_BIP_GMAC_256;
-			break;
-		case WLAN_CIPHER_SUITE_AES_CMAC:
-			conf.conf.keylen = WLAN_KEY_LEN_AES_CMAC;
-			break;
-		case WLAN_CIPHER_SUITE_BIP_CMAC_256:
-			conf.conf.keylen = WLAN_KEY_LEN_BIP_CMAC_256;
-			break;
-		}
-
-		if (WARN_ON(!conf.conf.keylen ||
-			    conf.conf.keylen > sizeof(conf.key)))
-			continue;
-
-		memcpy(conf.conf.key, mlo_key->key, conf.conf.keylen);
-		conf.conf.keyidx = key_id;
-
-		old_key = old_keys->key[link_id][key_id];
-		if (old_key) {
-			IWL_DEBUG_WOWLAN(mvm,
-					 "Remove MLO key id %d, link id %d\n",
-					 key_id, link_id);
-			ieee80211_remove_key(old_key);
-		}
-
-		IWL_DEBUG_WOWLAN(mvm, "Add MLO key id %d, link id %d\n",
-				 key_id, link_id);
-		key = ieee80211_gtk_rekey_add(vif, &conf.conf, link_id);
-		if (WARN_ON(IS_ERR(key))) {
-			ret = false;
-			goto out;
-		}
-
-		/*
-		 * mac80211 expects the pn in big-endian
-		 * also note that seq is a union of all cipher types
-		 * (ccmp, gcmp, cmac, gmac), and they all have the same
-		 * pn field (of length 6) so just copy it to ccmp.pn.
-		 */
-		for (j = 5; j >= 0; j--)
-			seq.ccmp.pn[5 - j] = mlo_key->pn[j];
-
-		/* group keys are non-QoS and use TID 0 */
-		ieee80211_set_key_rx_seq(key, 0, &seq);
-	}
-
-out:
-	kfree(old_keys);
-	return ret;
-}
-
 static bool iwl_mvm_gtk_rekey(struct iwl_wowlan_status_data *status,
 			      struct ieee80211_vif *vif,
 			      struct iwl_mvm *mvm, u32 gtk_cipher)
 {
 	int i, j;
 	struct ieee80211_key_conf *key;
-	struct {
-		struct ieee80211_key_conf conf;
-		u8 key[32];
-	} conf = {
-		.conf.cipher = gtk_cipher,
-	};
+	DEFINE_RAW_FLEX(struct ieee80211_key_conf, conf, key,
+			WOWLAN_KEY_MAX_SIZE);
 	int link_id = vif->active_links ? __ffs(vif->active_links) : -1;
+	u8 key_data[WOWLAN_KEY_MAX_SIZE];
+
+	conf->cipher = gtk_cipher;
 
 	BUILD_BUG_ON(WLAN_KEY_LEN_CCMP != WLAN_KEY_LEN_GCMP);
-	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_CCMP);
-	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_GCMP_256);
-	BUILD_BUG_ON(sizeof(conf.key) < WLAN_KEY_LEN_TKIP);
-	BUILD_BUG_ON(sizeof(conf.key) < sizeof(status->gtk[0].key));
+	BUILD_BUG_ON(WOWLAN_KEY_MAX_SIZE < WLAN_KEY_LEN_CCMP);
+	BUILD_BUG_ON(WOWLAN_KEY_MAX_SIZE < WLAN_KEY_LEN_GCMP_256);
+	BUILD_BUG_ON(WOWLAN_KEY_MAX_SIZE < WLAN_KEY_LEN_TKIP);
+	BUILD_BUG_ON(WOWLAN_KEY_MAX_SIZE < sizeof(status->gtk[0].key));
 
 	switch (gtk_cipher) {
 	case WLAN_CIPHER_SUITE_CCMP:
 	case WLAN_CIPHER_SUITE_GCMP:
-		conf.conf.keylen = WLAN_KEY_LEN_CCMP;
+		conf->keylen = WLAN_KEY_LEN_CCMP;
 		break;
 	case WLAN_CIPHER_SUITE_GCMP_256:
-		conf.conf.keylen = WLAN_KEY_LEN_GCMP_256;
+		conf->keylen = WLAN_KEY_LEN_GCMP_256;
 		break;
 	case WLAN_CIPHER_SUITE_TKIP:
-		conf.conf.keylen = WLAN_KEY_LEN_TKIP;
+		conf->keylen = WLAN_KEY_LEN_TKIP;
 		break;
 	default:
 		WARN_ON(1);
@@ -2181,16 +1987,22 @@ static bool iwl_mvm_gtk_rekey(struct iwl_wowlan_status_data *status,
 		if (!status->gtk[i].len)
 			continue;
 
-		conf.conf.keyidx = status->gtk[i].id;
+		conf->keyidx = status->gtk[i].id;
 		IWL_DEBUG_WOWLAN(mvm,
 				 "Received from FW GTK cipher %d, key index %d\n",
-				 conf.conf.cipher, conf.conf.keyidx);
-		memcpy(conf.conf.key, status->gtk[i].key,
+				 conf->cipher, conf->keyidx);
+		memcpy(conf->key, status->gtk[i].key,
 		       sizeof(status->gtk[i].key));
+		memcpy(key_data, status->gtk[i].key, sizeof(status->gtk[i].key));
 
-		key = ieee80211_gtk_rekey_add(vif, &conf.conf, link_id);
-		if (IS_ERR(key))
+		key = ieee80211_gtk_rekey_add(vif, status->gtk[i].id, key_data,
+					      sizeof(key_data), link_id);
+		if (IS_ERR(key)) {
+			/* FW may send also the old keys */
+			if (PTR_ERR(key) == -EALREADY)
+				continue;
 			return false;
+		}
 
 		for (j = 0; j < ARRAY_SIZE(status->gtk_seq); j++) {
 			if (!status->gtk_seq[j].valid ||
@@ -2210,55 +2022,66 @@ iwl_mvm_d3_igtk_bigtk_rekey_add(struct iwl_wowlan_status_data *status,
 				struct ieee80211_vif *vif, u32 cipher,
 				struct iwl_multicast_key_data *key_data)
 {
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	DEFINE_RAW_FLEX(struct ieee80211_key_conf, conf, key,
+			WOWLAN_KEY_MAX_SIZE);
 	struct ieee80211_key_conf *key_config;
-	struct {
-		struct ieee80211_key_conf conf;
-		u8 key[WOWLAN_KEY_MAX_SIZE];
-	} conf = {
-		.conf.cipher = cipher,
-		.conf.keyidx = key_data->id,
-	};
 	struct ieee80211_key_seq seq;
 	int link_id = vif->active_links ? __ffs(vif->active_links) : -1;
+	u8 key[WOWLAN_KEY_MAX_SIZE];
+	s8 keyidx = key_data->id;
+
+	conf->cipher = cipher;
+	conf->keyidx = keyidx;
 
 	if (!key_data->len)
 		return true;
 
-	iwl_mvm_d3_set_igtk_bigtk_ipn(key_data, &seq, conf.conf.cipher);
+	iwl_mvm_d3_set_igtk_bigtk_ipn(key_data, &seq, conf->cipher);
 
 	switch (cipher) {
 	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
-		conf.conf.keylen = WLAN_KEY_LEN_BIP_GMAC_128;
+		conf->keylen = WLAN_KEY_LEN_BIP_GMAC_128;
 		break;
 	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
-		conf.conf.keylen = WLAN_KEY_LEN_BIP_GMAC_256;
+		conf->keylen = WLAN_KEY_LEN_BIP_GMAC_256;
 		break;
 	case WLAN_CIPHER_SUITE_AES_CMAC:
-		conf.conf.keylen = WLAN_KEY_LEN_AES_CMAC;
+		conf->keylen = WLAN_KEY_LEN_AES_CMAC;
 		break;
 	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
-		conf.conf.keylen = WLAN_KEY_LEN_BIP_CMAC_256;
+		conf->keylen = WLAN_KEY_LEN_BIP_CMAC_256;
 		break;
 	default:
 		WARN_ON(1);
 	}
-	BUILD_BUG_ON(sizeof(conf.key) < sizeof(key_data->key));
-	memcpy(conf.conf.key, key_data->key, conf.conf.keylen);
+	BUILD_BUG_ON(WOWLAN_KEY_MAX_SIZE < sizeof(key_data->key));
+	memcpy(conf->key, key_data->key, conf->keylen);
 
-	key_config = ieee80211_gtk_rekey_add(vif, &conf.conf, link_id);
-	if (IS_ERR(key_config))
-		return false;
+	memcpy(key, key_data->key, sizeof(key_data->key));
+
+	key_config = ieee80211_gtk_rekey_add(vif, keyidx, key, sizeof(key),
+					     link_id);
+	if (IS_ERR(key_config)) {
+		/* FW may send also the old keys */
+		return PTR_ERR(key_config) == -EALREADY;
+	}
 	ieee80211_set_key_rx_seq(key_config, 0, &seq);
 
-	if (key_config->keyidx == 4 || key_config->keyidx == 5) {
-		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	if (keyidx == 4 || keyidx == 5) {
 		struct iwl_mvm_vif_link_info *mvm_link;
 
 		link_id = link_id < 0 ? 0 : link_id;
 		mvm_link = mvmvif->link[link_id];
+		if (mvm_link->igtk)
+			mvm_link->igtk->hw_key_idx = STA_KEY_IDX_INVALID;
 		mvm_link->igtk = key_config;
 	}
 
+	if (vif->type == NL80211_IFTYPE_STATION && (keyidx == 6 || keyidx == 7))
+		rcu_assign_pointer(mvmvif->bcn_prot.keys[keyidx - 6],
+				   key_config);
+
 	return true;
 }
 
@@ -2345,9 +2168,6 @@ static bool iwl_mvm_setup_connection_keep(struct iwl_mvm *mvm,
 				return false;
 		}
 
-		if (!iwl_mvm_mlo_gtk_rekey(status, vif, mvm))
-			return false;
-
 		ieee80211_gtk_rekey_notify(vif, vif->bss_conf.bssid,
 					   (void *)&replay_ctr, GFP_KERNEL);
 	}
@@ -2377,6 +2197,7 @@ static void iwl_mvm_convert_gtk_v2(struct iwl_wowlan_status_data *status,
 
 	status->gtk[0].len = data->key_len;
 	status->gtk[0].flags = data->key_flags;
+	status->gtk[0].id = status->gtk[0].flags & IWL_WOWLAN_GTK_IDX_MASK;
 
 	memcpy(status->gtk[0].key, data->key, sizeof(data->key));
 
@@ -2476,22 +2297,52 @@ static void iwl_mvm_convert_bigtk(struct iwl_wowlan_status_data *status,
 static void iwl_mvm_parse_wowlan_info_notif(struct iwl_mvm *mvm,
 					    struct iwl_wowlan_info_notif *data,
 					    struct iwl_wowlan_status_data *status,
-					    u32 len, bool has_mlo_keys)
+					    u32 len)
 {
-	u32 i;
-	u32 expected_len = sizeof(*data);
+	if (IWL_FW_CHECK(mvm, data->num_mlo_link_keys,
+			 "MLO is not supported, shouldn't receive MLO keys\n"))
+		return;
 
-	if (!data) {
-		IWL_ERR(mvm, "iwl_wowlan_info_notif data is NULL\n");
+	if (len < sizeof(*data)) {
+		IWL_ERR(mvm, "Invalid WoWLAN info notification!\n");
 		status = NULL;
 		return;
 	}
 
-	if (has_mlo_keys)
-		expected_len += (data->num_mlo_link_keys *
-				 sizeof(status->mlo_keys[0]));
+	if (mvm->fast_resume)
+		return;
 
-	if (len < expected_len) {
+	iwl_mvm_convert_key_counters_v5(status, &data->gtk[0].sc);
+	iwl_mvm_convert_gtk_v3(status, data->gtk);
+	iwl_mvm_convert_igtk(status, &data->igtk[0]);
+	iwl_mvm_convert_bigtk(status, data->bigtk);
+	status->replay_ctr = le64_to_cpu(data->replay_ctr);
+	status->pattern_number = le16_to_cpu(data->pattern_number);
+	status->tid_offloaded_tx = data->tid_offloaded_tx;
+	if (IWL_FW_CHECK(mvm,
+			 data->tid_offloaded_tx >=
+			 ARRAY_SIZE(status->qos_seq_ctr),
+			 "tid_offloaded_tx is out of bound %d\n",
+			 data->tid_offloaded_tx))
+		data->tid_offloaded_tx = 0;
+	status->qos_seq_ctr[data->tid_offloaded_tx] =
+		le16_to_cpu(data->qos_seq_ctr);
+	status->wakeup_reasons = le32_to_cpu(data->wakeup_reasons);
+	status->num_of_gtk_rekeys =
+		le32_to_cpu(data->num_of_gtk_rekeys);
+	status->received_beacons = le32_to_cpu(data->received_beacons);
+	status->tid_tear_down = data->tid_tear_down;
+}
+
+static void
+iwl_mvm_parse_wowlan_info_notif_v3(struct iwl_mvm *mvm,
+				   struct iwl_wowlan_info_notif_v3 *data,
+				   struct iwl_wowlan_status_data *status,
+				   u32 len)
+{
+	u32 i;
+
+	if (len < sizeof(*data)) {
 		IWL_ERR(mvm, "Invalid WoWLAN info notification!\n");
 		status = NULL;
 		return;
@@ -2514,33 +2365,16 @@ static void iwl_mvm_parse_wowlan_info_notif(struct iwl_mvm *mvm,
 		le32_to_cpu(data->num_of_gtk_rekeys);
 	status->received_beacons = le32_to_cpu(data->received_beacons);
 	status->tid_tear_down = data->tid_tear_down;
-
-	if (has_mlo_keys && data->num_mlo_link_keys) {
-		status->num_mlo_keys = data->num_mlo_link_keys;
-		if (IWL_FW_CHECK(mvm,
-				 status->num_mlo_keys > WOWLAN_MAX_MLO_KEYS,
-				 "Too many mlo keys: %d, max %d\n",
-				 status->num_mlo_keys, WOWLAN_MAX_MLO_KEYS))
-			status->num_mlo_keys = WOWLAN_MAX_MLO_KEYS;
-		memcpy(status->mlo_keys, data->mlo_gtks,
-		       status->num_mlo_keys * sizeof(status->mlo_keys[0]));
-	}
 }
 
 static void
-iwl_mvm_parse_wowlan_info_notif_v2(struct iwl_mvm *mvm,
-				   struct iwl_wowlan_info_notif_v2 *data,
+iwl_mvm_parse_wowlan_info_notif_v1(struct iwl_mvm *mvm,
+				   struct iwl_wowlan_info_notif_v1 *data,
 				   struct iwl_wowlan_status_data *status,
 				   u32 len)
 {
 	u32 i;
 
-	if (!data) {
-		IWL_ERR(mvm, "iwl_wowlan_info_notif data is NULL\n");
-		status = NULL;
-		return;
-	}
-
 	if (len < sizeof(*data)) {
 		IWL_ERR(mvm, "Invalid WoWLAN info notification!\n");
 		status = NULL;
@@ -2620,8 +2454,6 @@ iwl_mvm_parse_wowlan_status_common_ ## _ver(struct iwl_mvm *mvm,	\
 
 iwl_mvm_parse_wowlan_status_common(v6)
 iwl_mvm_parse_wowlan_status_common(v7)
-iwl_mvm_parse_wowlan_status_common(v9)
-iwl_mvm_parse_wowlan_status_common(v12)
 
 static struct iwl_wowlan_status_data *
 iwl_mvm_send_wowlan_get_status(struct iwl_mvm *mvm, u8 sta_id)
@@ -2677,7 +2509,8 @@ iwl_mvm_send_wowlan_get_status(struct iwl_mvm *mvm, u8 sta_id)
 		       v6->gtk.tkip_mic_key,
 		       sizeof(v6->gtk.tkip_mic_key));
 
-		iwl_mvm_convert_key_counters(status, &v6->gtk.rsc.all_tsc_rsc);
+		iwl_mvm_convert_key_counters(status, &v6->gtk.rsc.all_tsc_rsc,
+					     v6->gtk.key_index);
 
 		/* hardcode the key length to 16 since v6 only supports 16 */
 		status->gtk[0].len = 16;
@@ -2688,6 +2521,7 @@ iwl_mvm_send_wowlan_get_status(struct iwl_mvm *mvm, u8 sta_id)
 		 * currently used key.
 		 */
 		status->gtk[0].flags = v6->gtk.key_index | BIT(7);
+		status->gtk[0].id = v6->gtk.key_index;
 	} else if (notif_ver == 7) {
 		struct iwl_wowlan_status_v7 *v7 = (void *)cmd.resp_pkt->data;
 
@@ -2695,36 +2529,10 @@ iwl_mvm_send_wowlan_get_status(struct iwl_mvm *mvm, u8 sta_id)
 		if (!status)
 			goto out_free_resp;
 
-		iwl_mvm_convert_key_counters(status, &v7->gtk[0].rsc.all_tsc_rsc);
+		iwl_mvm_convert_key_counters(status, &v7->gtk[0].rsc.all_tsc_rsc,
+					     v7->gtk[0].key_flags & IWL_WOWLAN_GTK_IDX_MASK);
 		iwl_mvm_convert_gtk_v2(status, &v7->gtk[0]);
 		iwl_mvm_convert_igtk(status, &v7->igtk[0]);
-	} else if (notif_ver == 9 || notif_ver == 10 || notif_ver == 11) {
-		struct iwl_wowlan_status_v9 *v9 = (void *)cmd.resp_pkt->data;
-
-		/* these three command versions have same layout and size, the
-		 * difference is only in a few not used (reserved) fields.
-		 */
-		status = iwl_mvm_parse_wowlan_status_common_v9(mvm, v9, len);
-		if (!status)
-			goto out_free_resp;
-
-		iwl_mvm_convert_key_counters(status, &v9->gtk[0].rsc.all_tsc_rsc);
-		iwl_mvm_convert_gtk_v2(status, &v9->gtk[0]);
-		iwl_mvm_convert_igtk(status, &v9->igtk[0]);
-
-		status->tid_tear_down = v9->tid_tear_down;
-	} else if (notif_ver == 12) {
-		struct iwl_wowlan_status_v12 *v12 = (void *)cmd.resp_pkt->data;
-
-		status = iwl_mvm_parse_wowlan_status_common_v12(mvm, v12, len);
-		if (!status)
-			goto out_free_resp;
-
-		iwl_mvm_convert_key_counters_v5(status, &v12->gtk[0].sc);
-		iwl_mvm_convert_gtk_v3(status, v12->gtk);
-		iwl_mvm_convert_igtk(status, &v12->igtk[0]);
-
-		status->tid_tear_down = v12->tid_tear_down;
 	} else {
 		IWL_ERR(mvm,
 			"Firmware advertises unknown WoWLAN status response %d!\n",
@@ -2748,6 +2556,10 @@ static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	int link_id = vif->active_links ? __ffs(vif->active_links) : 0;
 	struct iwl_mvm_vif_link_info *mvm_link = mvmvif->link[link_id];
+	int wowlan_info_ver = iwl_fw_lookup_notif_ver(mvm->fw,
+						      PROT_OFFLOAD_GROUP,
+						      WOWLAN_INFO_NOTIFICATION,
+						      IWL_FW_CMD_VER_UNKNOWN);
 
 	if (WARN_ON(!mvm_link))
 		goto out_unlock;
@@ -2762,14 +2574,17 @@ static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm,
 	if (!mvm_ap_sta)
 		goto out_unlock;
 
-	for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
-		u16 seq = status->qos_seq_ctr[i];
-		/* firmware stores last-used value, we store next value */
-		seq += 0x10;
-		mvm_ap_sta->tid_data[i].seq_number = seq;
+	/* firmware stores last-used value, we store next value */
+	if (wowlan_info_ver >= 5) {
+		mvm_ap_sta->tid_data[status->tid_offloaded_tx].seq_number =
+			status->qos_seq_ctr[status->tid_offloaded_tx] + 0x10;
+	} else {
+		for (i = 0; i < IWL_MAX_TID_COUNT; i++)
+			mvm_ap_sta->tid_data[i].seq_number =
+				status->qos_seq_ctr[i] + 0x10;
 	}
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
 		i = mvm->offload_tid;
 		iwl_trans_set_q_ptrs(mvm->trans,
 				     mvm_ap_sta->tid_data[i].txq_id,
@@ -2873,6 +2688,7 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
 				    int idx)
 {
 	int i;
+	int n_channels = 0;
 
 	if (fw_has_api(&mvm->fw->ucode_capa,
 		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
@@ -2881,7 +2697,7 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
 
 		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN * 8; i++)
 			if (matches[idx].matching_channels[i / 8] & (BIT(i % 8)))
-				match->channels[match->n_channels++] =
+				match->channels[n_channels++] =
 					mvm->nd_channels[i]->center_freq;
 	} else {
 		struct iwl_scan_offload_profile_match_v1 *matches =
@@ -2889,9 +2705,11 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
 
 		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V1 * 8; i++)
 			if (matches[idx].matching_channels[i / 8] & (BIT(i % 8)))
-				match->channels[match->n_channels++] =
+				match->channels[n_channels++] =
 					mvm->nd_channels[i]->center_freq;
 	}
+	/* We may have ended up with fewer channels than we allocated. */
+	match->n_channels = n_channels;
 }
 
 /**
@@ -2972,6 +2790,8 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm,
 			     GFP_KERNEL);
 	if (!net_detect || !n_matches)
 		goto out_report_nd;
+	net_detect->n_matches = n_matches;
+	n_matches = 0;
 
 	for_each_set_bit(i, &matched_profiles, mvm->n_nd_match_sets) {
 		struct cfg80211_wowlan_nd_match *match;
@@ -2985,8 +2805,9 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm,
 				GFP_KERNEL);
 		if (!match)
 			goto out_report_nd;
+		match->n_channels = n_channels;
 
-		net_detect->matches[net_detect->n_matches++] = match;
+		net_detect->matches[n_matches++] = match;
 
 		/* We inverted the order of the SSIDs in the scan
 		 * request, so invert the index here.
@@ -3001,6 +2822,8 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm,
 
 		iwl_mvm_query_set_freqs(mvm, d3_data->nd_results, match, i);
 	}
+	/* We may have fewer matches than we allocated. */
+	net_detect->n_matches = n_matches;
 
 out_report_nd:
 	wakeup.net_detect = net_detect;
@@ -3028,55 +2851,50 @@ static void iwl_mvm_d3_disconnect_iter(void *data, u8 *mac,
 		ieee80211_resume_disconnect(vif);
 }
 
-static bool iwl_mvm_rt_status(struct iwl_trans *trans, u32 base, u32 *err_id)
-{
-	struct error_table_start {
-		/* cf. struct iwl_error_event_table */
-		u32 valid;
-		__le32 err_id;
-	} err_info;
-
-	if (!base)
-		return false;
-
-	iwl_trans_read_mem_bytes(trans, base,
-				 &err_info, sizeof(err_info));
-	if (err_info.valid && err_id)
-		*err_id = le32_to_cpu(err_info.err_id);
-
-	return !!err_info.valid;
-}
+enum rt_status {
+	FW_ALIVE,
+	FW_NEEDS_RESET,
+	FW_ERROR,
+};
 
-static bool iwl_mvm_check_rt_status(struct iwl_mvm *mvm,
-				   struct ieee80211_vif *vif)
+static enum rt_status iwl_mvm_check_rt_status(struct iwl_mvm *mvm,
+					      struct ieee80211_vif *vif)
 {
 	u32 err_id;
 
 	/* check for lmac1 error */
-	if (iwl_mvm_rt_status(mvm->trans,
-			      mvm->trans->dbg.lmac_error_event_table[0],
-			      &err_id)) {
-		if (err_id == RF_KILL_INDICATOR_FOR_WOWLAN && vif) {
-			struct cfg80211_wowlan_wakeup wakeup = {
-				.rfkill_release = true,
-			};
-			ieee80211_report_wowlan_wakeup(vif, &wakeup,
-						       GFP_KERNEL);
+	if (iwl_fwrt_read_err_table(mvm->trans,
+				    mvm->trans->dbg.lmac_error_event_table[0],
+				    &err_id)) {
+		if (err_id == RF_KILL_INDICATOR_FOR_WOWLAN) {
+			IWL_WARN(mvm, "Rfkill was toggled during suspend\n");
+			if (vif) {
+				struct cfg80211_wowlan_wakeup wakeup = {
+					.rfkill_release = true,
+				};
+
+				ieee80211_report_wowlan_wakeup(vif, &wakeup,
+							       GFP_KERNEL);
+			}
+
+			return FW_NEEDS_RESET;
 		}
-		return true;
+		return FW_ERROR;
 	}
 
 	/* check if we have lmac2 set and check for error */
-	if (iwl_mvm_rt_status(mvm->trans,
-			      mvm->trans->dbg.lmac_error_event_table[1], NULL))
-		return true;
+	if (iwl_fwrt_read_err_table(mvm->trans,
+				    mvm->trans->dbg.lmac_error_event_table[1],
+				    NULL))
+		return FW_ERROR;
 
 	/* check for umac error */
-	if (iwl_mvm_rt_status(mvm->trans,
-			      mvm->trans->dbg.umac_error_event_table, NULL))
-		return true;
+	if (iwl_fwrt_read_err_table(mvm->trans,
+				    mvm->trans->dbg.umac_error_event_table,
+				    NULL))
+		return FW_ERROR;
 
-	return false;
+	return FW_ALIVE;
 }
 
 /*
@@ -3094,7 +2912,7 @@ iwl_mvm_choose_query_wakeup_reasons(struct iwl_mvm *mvm,
 	/* if FW uses status notification, status shouldn't be NULL here */
 	if (!d3_data->status) {
 		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-		u8 sta_id = mvm->net_detect ? IWL_MVM_INVALID_STA :
+		u8 sta_id = mvm->net_detect ? IWL_INVALID_STA :
 					      mvmvif->deflink.ap_sta_id;
 
 		/* bug - FW with MLO has status notification */
@@ -3241,38 +3059,30 @@ static bool iwl_mvm_wait_d3_notif(struct iwl_notif_wait_data *notif_wait,
 			break;
 		}
 
-		if (wowlan_info_ver < 2) {
+		if (wowlan_info_ver == 1) {
 			struct iwl_wowlan_info_notif_v1 *notif_v1 =
 				(void *)pkt->data;
-			struct iwl_wowlan_info_notif_v2 *notif_v2;
-
-			notif_v2 = kmemdup(notif_v1, sizeof(*notif_v2), GFP_ATOMIC);
-
-			if (!notif_v2)
-				return false;
 
-			notif_v2->tid_tear_down = notif_v1->tid_tear_down;
-			notif_v2->station_id = notif_v1->station_id;
-			memset_after(notif_v2, 0, station_id);
-			iwl_mvm_parse_wowlan_info_notif_v2(mvm, notif_v2,
+			iwl_mvm_parse_wowlan_info_notif_v1(mvm, notif_v1,
 							   d3_data->status,
 							   len);
-			kfree(notif_v2);
-
-		} else if (wowlan_info_ver == 2) {
-			struct iwl_wowlan_info_notif_v2 *notif_v2 =
+		} else if (wowlan_info_ver == 3) {
+			struct iwl_wowlan_info_notif_v3 *notif =
 				(void *)pkt->data;
 
-			iwl_mvm_parse_wowlan_info_notif_v2(mvm, notif_v2,
-							   d3_data->status,
-							   len);
-		} else {
+			iwl_mvm_parse_wowlan_info_notif_v3(mvm, notif,
+							   d3_data->status, len);
+		} else if (wowlan_info_ver == 5) {
 			struct iwl_wowlan_info_notif *notif =
 				(void *)pkt->data;
 
 			iwl_mvm_parse_wowlan_info_notif(mvm, notif,
-							d3_data->status, len,
-							wowlan_info_ver > 3);
+							d3_data->status, len);
+		} else {
+			IWL_FW_CHECK(mvm, 1,
+				     "Firmware advertises unknown WoWLAN info notification %d!\n",
+				     wowlan_info_ver);
+			return false;
 		}
 
 		d3_data->notif_received |= IWL_D3_NOTIF_WOWLAN_INFO;
@@ -3323,7 +3133,7 @@ static bool iwl_mvm_wait_d3_notif(struct iwl_notif_wait_data *notif_wait,
 		break;
 	}
 	case WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION): {
-		struct iwl_mvm_d3_end_notif *notif = (void *)pkt->data;
+		struct iwl_d3_end_notif *notif = (void *)pkt->data;
 
 		d3_data->d3_end_flags = __le32_to_cpu(notif->flags);
 		d3_data->notif_received |= IWL_D3_NOTIF_D3_END_NOTIF;
@@ -3342,9 +3152,9 @@ static int iwl_mvm_resume_firmware(struct iwl_mvm *mvm, bool test)
 	int ret;
 	enum iwl_d3_status d3_status;
 	struct iwl_host_cmd cmd = {
-			.id = D0I3_END_CMD,
-			.flags = CMD_WANT_SKB | CMD_SEND_IN_D3,
-		};
+		.id = D0I3_END_CMD,
+		.flags = CMD_WANT_SKB,
+	};
 	bool reset = fw_has_capa(&mvm->fw->ucode_capa,
 				 IWL_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG);
 
@@ -3362,7 +3172,7 @@ static int iwl_mvm_resume_firmware(struct iwl_mvm *mvm, bool test)
 	 * AX210 and above don't need the command since they have
 	 * the doorbell interrupt.
 	 */
-	if (mvm->trans->trans_cfg->device_family <= IWL_DEVICE_FAMILY_22000 &&
+	if (mvm->trans->mac_cfg->device_family <= IWL_DEVICE_FAMILY_22000 &&
 	    fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_D0I3_END_FIRST)) {
 		ret = iwl_mvm_send_cmd(mvm, &cmd);
 		if (ret < 0)
@@ -3439,6 +3249,7 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
 	bool d0i3_first = fw_has_capa(&mvm->fw->ucode_capa,
 				      IWL_UCODE_TLV_CAPA_D0I3_END_FIRST);
 	bool resume_notif_based = iwl_mvm_d3_resume_notif_based(mvm);
+	enum rt_status rt_status;
 	bool keep = false;
 
 	mutex_lock(&mvm->mutex);
@@ -3462,13 +3273,19 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
 
 	iwl_fw_dbg_read_d3_debug_data(&mvm->fwrt);
 
-	if (iwl_mvm_check_rt_status(mvm, vif)) {
+	rt_status = iwl_mvm_check_rt_status(mvm, vif);
+	if (rt_status != FW_ALIVE) {
 		set_bit(STATUS_FW_ERROR, &mvm->trans->status);
-		iwl_mvm_dump_nic_error_log(mvm);
-		iwl_dbg_tlv_time_point(&mvm->fwrt,
-				       IWL_FW_INI_TIME_POINT_FW_ASSERT, NULL);
-		iwl_fw_dbg_collect_desc(&mvm->fwrt, &iwl_dump_desc_assert,
-					false, 0);
+		if (rt_status == FW_ERROR) {
+			IWL_ERR(mvm, "FW Error occurred during suspend. Restarting.\n");
+			iwl_mvm_dump_nic_error_log(mvm);
+			iwl_dbg_tlv_time_point(&mvm->fwrt,
+					       IWL_FW_INI_TIME_POINT_FW_ASSERT,
+					       NULL);
+			iwl_fw_dbg_collect_desc(&mvm->fwrt,
+						&iwl_dump_desc_assert,
+						false, 0);
+		}
 		ret = 1;
 		goto err;
 	}
@@ -3492,9 +3309,6 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
 
 	iwl_mvm_unblock_esr(mvm, vif, IWL_MVM_ESR_BLOCKED_WOWLAN);
 
-	/* after the successful handshake, we're out of D3 */
-	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
-
 	/* when reset is required we can't send these following commands */
 	if (d3_data.d3_end_flags & IWL_D0I3_RESET_REQUIRE)
 		goto query_wakeup_reasons;
@@ -3567,9 +3381,6 @@ out:
 	 */
 	set_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status);
 
-	/* regardless of what happened, we're now out of D3 */
-	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
-
 	return 1;
 }
 
@@ -3607,15 +3418,12 @@ void iwl_mvm_fast_suspend(struct iwl_mvm *mvm)
 	set_bit(IWL_MVM_STATUS_IN_D3, &mvm->status);
 
 	WARN_ON(iwl_mvm_power_update_device(mvm));
-	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_D3;
-	ret = iwl_mvm_send_cmd_pdu(mvm, D3_CONFIG_CMD, CMD_SEND_IN_D3,
+	ret = iwl_mvm_send_cmd_pdu(mvm, D3_CONFIG_CMD, 0,
 				   sizeof(d3_cfg_cmd_data), &d3_cfg_cmd_data);
 	if (ret)
 		IWL_ERR(mvm,
 			"fast suspend: couldn't send D3_CONFIG_CMD %d\n", ret);
 
-	WARN_ON(iwl_mvm_power_update_mac(mvm));
-
 	ret = iwl_trans_d3_suspend(mvm->trans, false, false);
 	if (ret)
 		IWL_ERR(mvm, "fast suspend: trans_d3_suspend failed %d\n", ret);
@@ -3627,6 +3435,7 @@ int iwl_mvm_fast_resume(struct iwl_mvm *mvm)
 		.notif_expected =
 			IWL_D3_NOTIF_D3_END_NOTIF,
 	};
+	enum rt_status rt_status;
 	int ret;
 
 	lockdep_assert_held(&mvm->mutex);
@@ -3636,22 +3445,35 @@ int iwl_mvm_fast_resume(struct iwl_mvm *mvm)
 	mvm->last_reset_or_resume_time_jiffies = jiffies;
 	iwl_fw_dbg_read_d3_debug_data(&mvm->fwrt);
 
-	if (iwl_mvm_check_rt_status(mvm, NULL)) {
+	rt_status = iwl_mvm_check_rt_status(mvm, NULL);
+	if (rt_status != FW_ALIVE) {
 		set_bit(STATUS_FW_ERROR, &mvm->trans->status);
-		iwl_mvm_dump_nic_error_log(mvm);
-		iwl_dbg_tlv_time_point(&mvm->fwrt,
-				       IWL_FW_INI_TIME_POINT_FW_ASSERT, NULL);
-		iwl_fw_dbg_collect_desc(&mvm->fwrt, &iwl_dump_desc_assert,
-					false, 0);
-		return -ENODEV;
+		if (rt_status == FW_ERROR) {
+			IWL_ERR(mvm,
+				"iwl_mvm_check_rt_status failed, device is gone during suspend\n");
+			iwl_mvm_dump_nic_error_log(mvm);
+			iwl_dbg_tlv_time_point(&mvm->fwrt,
+					       IWL_FW_INI_TIME_POINT_FW_ASSERT,
+					       NULL);
+			iwl_fw_dbg_collect_desc(&mvm->fwrt,
+						&iwl_dump_desc_assert,
+						false, 0);
+		}
+		mvm->trans->state = IWL_TRANS_NO_FW;
+		ret = -ENODEV;
+
+		goto out;
 	}
 	ret = iwl_mvm_d3_notif_wait(mvm, &d3_data);
-	clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status);
-	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
-	mvm->fast_resume = false;
 
-	if (ret)
+	if (ret) {
 		IWL_ERR(mvm, "Couldn't get the d3 notif %d\n", ret);
+		mvm->trans->state = IWL_TRANS_NO_FW;
+	}
+
+out:
+	clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status);
+	mvm->fast_resume = false;
 
 	return ret;
 }
@@ -3780,7 +3602,6 @@ static int iwl_mvm_d3_test_release(struct inode *inode, struct file *file)
 }
 
 const struct file_operations iwl_dbgfs_d3_test_ops = {
-	.llseek = no_llseek,
 	.open = iwl_mvm_d3_test_open,
 	.read = iwl_mvm_d3_test_read,
 	.release = iwl_mvm_d3_test_release,
diff --git a/sys/contrib/dev/iwlwifi/mvm/debugfs-vif.c b/sys/contrib/dev/iwlwifi/mvm/debugfs-vif.c
index 0b3bc62f39a7..f1303440d3dd 100644
--- a/sys/contrib/dev/iwlwifi/mvm/debugfs-vif.c
+++ b/sys/contrib/dev/iwlwifi/mvm/debugfs-vif.c
@@ -224,7 +224,7 @@ static ssize_t iwl_dbgfs_mac_params_read(struct file *file,
 				 mvmvif->deflink.queue_params[i].uapsd);
 
 	if (vif->type == NL80211_IFTYPE_STATION &&
-	    ap_sta_id != IWL_MVM_INVALID_STA) {
+	    ap_sta_id != IWL_INVALID_STA) {
 		struct iwl_mvm_sta *mvm_sta;
 
 		mvm_sta = iwl_mvm_sta_from_staid_protected(mvm, ap_sta_id);
@@ -466,11 +466,13 @@ static ssize_t iwl_dbgfs_os_device_timediff_read(struct file *file,
 	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
 }
 
-static ssize_t iwl_dbgfs_low_latency_write(struct ieee80211_vif *vif, char *buf,
-					   size_t count, loff_t *ppos)
+static ssize_t
+iwl_dbgfs_low_latency_write_handle(struct wiphy *wiphy, struct file *file,
+				   char *buf, size_t count, void *data)
 {
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm *mvm = mvmvif->mvm;
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct ieee80211_vif *vif = data;
 	u8 value;
 	int ret;
 
@@ -487,12 +489,28 @@ static ssize_t iwl_dbgfs_low_latency_write(struct ieee80211_vif *vif, char *buf,
 	return count;
 }
 
-static ssize_t
-iwl_dbgfs_low_latency_force_write(struct ieee80211_vif *vif, char *buf,
-				  size_t count, loff_t *ppos)
+static ssize_t iwl_dbgfs_low_latency_write(struct file *file,
+					   const char __user *user_buf,
+					   size_t count, loff_t *ppos)
 {
+	struct ieee80211_vif *vif = file->private_data;
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm *mvm = mvmvif->mvm;
+	char buf[10] = {};
+
+	return wiphy_locked_debugfs_write(mvm->hw->wiphy, file,
+					  buf, sizeof(buf), user_buf, count,
+					  iwl_dbgfs_low_latency_write_handle,
+					  vif);
+}
+
+static ssize_t
+iwl_dbgfs_low_latency_force_write_handle(struct wiphy *wiphy, struct file *file,
+					 char *buf, size_t count, void *data)
+{
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct ieee80211_vif *vif = data;
 	u8 value;
 	int ret;
 
@@ -520,6 +538,22 @@ iwl_dbgfs_low_latency_force_write(struct ieee80211_vif *vif, char *buf,
 	return count;
 }
 
+static ssize_t
+iwl_dbgfs_low_latency_force_write(struct file *file,
+				  const char __user *user_buf,
+				  size_t count, loff_t *ppos)
+{
+	struct ieee80211_vif *vif = file->private_data;
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm *mvm = mvmvif->mvm;
+	char buf[10] = {};
+
+	return wiphy_locked_debugfs_write(mvm->hw->wiphy, file,
+					  buf, sizeof(buf), user_buf, count,
+					  iwl_dbgfs_low_latency_force_write_handle,
+					  vif);
+}
+
 static ssize_t iwl_dbgfs_low_latency_read(struct file *file,
 					  char __user *user_buf,
 					  size_t count, loff_t *ppos)
@@ -838,8 +872,20 @@ MVM_DEBUGFS_READ_FILE_OPS(mac_params);
 MVM_DEBUGFS_READ_FILE_OPS(tx_pwr_lmt);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(pm_params, 32);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(bf_params, 256);
-MVM_DEBUGFS_READ_WRITE_FILE_OPS(low_latency, 10);
-MVM_DEBUGFS_WRITE_FILE_OPS(low_latency_force, 10);
+
+static const struct file_operations iwl_dbgfs_low_latency_ops = {
+	.write = iwl_dbgfs_low_latency_write,
+	.read = iwl_dbgfs_low_latency_read,
+	.open = simple_open,
+	.llseek = generic_file_llseek,
+};
+
+static const struct file_operations iwl_dbgfs_low_latency_force_ops = {
+	.write = iwl_dbgfs_low_latency_force_write,
+	.open = simple_open,
+	.llseek = generic_file_llseek,
+};
+
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(uapsd_misbehaving, 20);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(rx_phyinfo, 10);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(quota_min, 32);
diff --git a/sys/contrib/dev/iwlwifi/mvm/debugfs.c b/sys/contrib/dev/iwlwifi/mvm/debugfs.c
index 24bcffe67235..eb8ae6d574fa 100644
--- a/sys/contrib/dev/iwlwifi/mvm/debugfs.c
+++ b/sys/contrib/dev/iwlwifi/mvm/debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2023 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2023, 2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -16,6 +16,7 @@
 #include "debugfs.h"
 #include "iwl-modparams.h"
 #include "iwl-drv.h"
+#include "iwl-utils.h"
 #include "fw/error-dump.h"
 #include "fw/api/phy-ctxt.h"
 
@@ -462,7 +463,6 @@ static ssize_t iwl_dbgfs_amsdu_len_write(struct ieee80211_link_sta *link_sta,
 	if (amsdu_len) {
 		mvm_link_sta->orig_amsdu_len = link_sta->agg.max_amsdu_len;
 		link_sta->agg.max_amsdu_len = amsdu_len;
-		link_sta->agg.max_amsdu_len = amsdu_len;
 		for (i = 0; i < ARRAY_SIZE(link_sta->agg.max_tid_amsdu_len); i++)
 			link_sta->agg.max_tid_amsdu_len[i] = amsdu_len;
 	} else {
@@ -537,47 +537,12 @@ static ssize_t iwl_dbgfs_disable_power_off_write(struct iwl_mvm *mvm, char *buf,
 	return ret ?: count;
 }
 
-static ssize_t iwl_dbgfs_fw_ver_read(struct file *file, char __user *user_buf,
-				     size_t count, loff_t *ppos)
-{
-	struct iwl_mvm *mvm = file->private_data;
-	char *buff, *pos, *endpos;
-	static const size_t bufsz = 1024;
-	int ret;
-
-	buff = kmalloc(bufsz, GFP_KERNEL);
-	if (!buff)
-		return -ENOMEM;
-
-	pos = buff;
-	endpos = pos + bufsz;
-
-	pos += scnprintf(pos, endpos - pos, "FW id: %s\n",
-			 mvm->fwrt.fw->fw_version);
-	pos += scnprintf(pos, endpos - pos, "FW: %s\n",
-			 mvm->fwrt.fw->human_readable);
-	pos += scnprintf(pos, endpos - pos, "Device: %s\n",
-			 mvm->fwrt.trans->name);
-	pos += scnprintf(pos, endpos - pos, "Bus: %s\n",
-#if defined(__linux__)
-			 mvm->fwrt.dev->bus->name);
-#elif defined(__FreeBSD__)
-			"<bus>");
-#endif
-
-	ret = simple_read_from_buffer(user_buf, count, ppos, buff, pos - buff);
-	kfree(buff);
-
-	return ret;
-}
-
 static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 					     char __user *user_buf,
 					     size_t count, loff_t *ppos)
 {
 	struct iwl_mvm *mvm = file->private_data;
-	struct iwl_mvm_tas_status_resp tas_rsp;
-	struct iwl_mvm_tas_status_resp *rsp = &tas_rsp;
+	struct iwl_tas_status_resp *rsp = NULL;
 	static const size_t bufsz = 1024;
 	char *buff, *pos, *endpos;
 	const char * const tas_dis_reason[TAS_DISABLED_REASON_MAX] = {
@@ -587,6 +552,8 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 			"Due To SAR Limit Less Than 6 dBm",
 		[TAS_DISABLED_REASON_INVALID] =
 			"N/A",
+		[TAS_DISABLED_DUE_TO_TABLE_SOURCE_INVALID] =
+			"Due to table source invalid",
 	};
 	const char * const tas_current_status[TAS_DYNA_STATUS_MAX] = {
 		[TAS_DYNA_INACTIVE] = "INACTIVE",
@@ -613,6 +580,10 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 	if (!iwl_mvm_firmware_running(mvm))
 		return -ENODEV;
 
+	if (iwl_fw_lookup_notif_ver(mvm->fw, DEBUG_GROUP, GET_TAS_STATUS,
+				    0) != 3)
+		return -EOPNOTSUPP;
+
 	mutex_lock(&mvm->mutex);
 	ret = iwl_mvm_send_cmd(mvm, &hcmd);
 	mutex_unlock(&mvm->mutex);
@@ -629,23 +600,19 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 
 	pos += scnprintf(pos, endpos - pos, "TAS Conclusion:\n");
 	for (i = 0; i < rsp->in_dual_radio + 1; i++) {
-		if (rsp->tas_status_mac[i].band != TAS_LMAC_BAND_INVALID &&
-		    rsp->tas_status_mac[i].dynamic_status & BIT(TAS_DYNA_ACTIVE)) {
+		if (rsp->tas_status_mac[i].dynamic_status &
+		    BIT(TAS_DYNA_ACTIVE)) {
 			pos += scnprintf(pos, endpos - pos, "\tON for ");
 			switch (rsp->tas_status_mac[i].band) {
-			case TAS_LMAC_BAND_HB:
+			case PHY_BAND_5:
 				pos += scnprintf(pos, endpos - pos, "HB\n");
 				break;
-			case TAS_LMAC_BAND_LB:
+			case PHY_BAND_24:
 				pos += scnprintf(pos, endpos - pos, "LB\n");
 				break;
-			case TAS_LMAC_BAND_UHB:
+			case PHY_BAND_6:
 				pos += scnprintf(pos, endpos - pos, "UHB\n");
 				break;
-			case TAS_LMAC_BAND_INVALID:
-				pos += scnprintf(pos, endpos - pos,
-						 "INVALID BAND\n");
-				break;
 			default:
 				pos += scnprintf(pos, endpos - pos,
 						 "Unsupported band (%d)\n",
@@ -663,6 +630,14 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 			 rsp->tas_fw_version);
 	pos += scnprintf(pos, endpos - pos, "Is UHB enabled for USA?: %s\n",
 			 rsp->is_uhb_for_usa_enable ? "True" : "False");
+
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_UHB_CANADA_TAS_SUPPORT))
+		pos += scnprintf(pos, endpos - pos,
+				 "Is UHB enabled for CANADA?: %s\n",
+				 rsp->uhb_allowed_flags &
+				 TAS_UHB_ALLOWED_CANADA ? "True" : "False");
+
 	pos += scnprintf(pos, endpos - pos, "Current MCC: 0x%x\n",
 			 le16_to_cpu(rsp->curr_mcc));
 
@@ -691,20 +666,16 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 
 		pos += scnprintf(pos, endpos - pos, "TAS status for ");
 		switch (rsp->tas_status_mac[i].band) {
-		case TAS_LMAC_BAND_HB:
+		case PHY_BAND_5:
 			pos += scnprintf(pos, endpos - pos, "High band\n");
 			break;
-		case TAS_LMAC_BAND_LB:
+		case PHY_BAND_24:
 			pos += scnprintf(pos, endpos - pos, "Low band\n");
 			break;
-		case TAS_LMAC_BAND_UHB:
+		case PHY_BAND_6:
 			pos += scnprintf(pos, endpos - pos,
 					 "Ultra high band\n");
 			break;
-		case TAS_LMAC_BAND_INVALID:
-			pos += scnprintf(pos, endpos - pos,
-					 "INVALID band\n");
-			break;
 		default:
 			pos += scnprintf(pos, endpos - pos,
 					 "Unsupported band (%d)\n",
@@ -727,11 +698,9 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 
 		pos += scnprintf(pos, endpos - pos, "Dynamic status:\n");
 		dyn_status = (rsp->tas_status_mac[i].dynamic_status);
-		for_each_set_bit(tmp, &dyn_status, sizeof(dyn_status)) {
-			if (tmp >= 0 && tmp < TAS_DYNA_STATUS_MAX)
-				pos += scnprintf(pos, endpos - pos,
-						 "\t%s (%d)\n",
-						 tas_current_status[tmp], tmp);
+		for_each_set_bit(tmp, &dyn_status, TAS_DYNA_STATUS_MAX) {
+			pos += scnprintf(pos, endpos - pos, "\t%s (%d)\n",
+					 tas_current_status[tmp], tmp);
 		}
 
 		pos += scnprintf(pos, endpos - pos,
@@ -1163,13 +1132,9 @@ static ssize_t iwl_dbgfs_fw_restart_write(struct iwl_mvm *mvm, char *buf,
 
 	mutex_lock(&mvm->mutex);
 
-	/* allow one more restart that we're provoking here */
-	if (mvm->fw_restart >= 0)
-		mvm->fw_restart++;
-
 	if (count == 6 && !strcmp(buf, "nolog\n")) {
 		set_bit(IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE, &mvm->status);
-		set_bit(STATUS_SUPPRESS_CMD_ERROR_ONCE, &mvm->trans->status);
+		iwl_trans_suppress_cmd_error_once(mvm->trans);
 	}
 
 	/* take the return value to make compiler happy - it will fail anyway */
@@ -1312,7 +1277,7 @@ static ssize_t iwl_dbgfs_inject_packet_write(struct iwl_mvm *mvm,
 		return -EIO;
 
 	/* supporting only MQ RX */
-	if (!mvm->trans->trans_cfg->mq_rx_supported)
+	if (!mvm->trans->mac_cfg->mq_rx_supported)
 		return -EOPNOTSUPP;
 
 	rxb._page = alloc_pages(GFP_ATOMIC, 0);
@@ -1413,9 +1378,9 @@ static int _iwl_dbgfs_inject_beacon_ie(struct iwl_mvm *mvm, char *bin, int len)
 
 		if (iwl_fw_lookup_cmd_ver(mvm->fw,
 					  BEACON_TEMPLATE_CMD, 0) >= 14) {
-			u32 offset = iwl_mvm_find_ie_offset(beacon->data,
-							    WLAN_EID_S1G_TWT,
-							    beacon->len);
+			u32 offset = iwl_find_ie_offset(beacon->data,
+							WLAN_EID_S1G_TWT,
+							beacon->len);
 
 			beacon_cmd.btwt_offset = cpu_to_le32(offset);
 		}
@@ -1499,29 +1464,20 @@ static ssize_t iwl_dbgfs_fw_dbg_conf_write(struct iwl_mvm *mvm,
 	return ret ?: count;
 }
 
-static ssize_t iwl_dbgfs_fw_dbg_collect_write(struct iwl_mvm *mvm,
-					      char *buf, size_t count,
-					      loff_t *ppos)
-{
-	if (count == 0)
-		return 0;
-
-	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_POINT_USER_TRIGGER,
-			       NULL);
-
-	iwl_fw_dbg_collect(&mvm->fwrt, FW_DBG_TRIGGER_USER, buf,
-			   (count - 1), NULL);
-
-	return count;
-}
-
 static ssize_t iwl_dbgfs_fw_dbg_clear_write(struct iwl_mvm *mvm,
 					    char *buf, size_t count,
 					    loff_t *ppos)
 {
-	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000)
+	if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
 		return -EOPNOTSUPP;
 
+	/*
+	 * If the firmware is not running, silently succeed since there is
+	 * no data to clear.
+	 */
+	if (!iwl_mvm_firmware_running(mvm))
+		return count;
+
 	mutex_lock(&mvm->mutex);
 	iwl_fw_dbg_clear_monitor_buf(&mvm->fwrt);
 	mutex_unlock(&mvm->mutex);
@@ -1968,14 +1924,12 @@ MVM_DEBUGFS_READ_WRITE_FILE_OPS(disable_power_off, 64);
 MVM_DEBUGFS_READ_FILE_OPS(fw_rx_stats);
 MVM_DEBUGFS_READ_FILE_OPS(drv_rx_stats);
 MVM_DEBUGFS_READ_FILE_OPS(fw_system_stats);
-MVM_DEBUGFS_READ_FILE_OPS(fw_ver);
 MVM_DEBUGFS_READ_FILE_OPS(phy_integration_ver);
 MVM_DEBUGFS_READ_FILE_OPS(tas_get_status);
 MVM_DEBUGFS_WRITE_FILE_OPS(fw_restart, 10);
 MVM_DEBUGFS_WRITE_FILE_OPS(fw_nmi, 10);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(scan_ant_rxchain, 8);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(fw_dbg_conf, 8);
-MVM_DEBUGFS_WRITE_FILE_OPS(fw_dbg_collect, 64);
 MVM_DEBUGFS_WRITE_FILE_OPS(fw_dbg_clear, 64);
 MVM_DEBUGFS_WRITE_FILE_OPS(dbg_time_point, 64);
 MVM_DEBUGFS_WRITE_FILE_OPS(indirection_tbl,
@@ -2168,7 +2122,6 @@ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm)
 	MVM_DEBUGFS_ADD_FILE(force_ctkill, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(stations, mvm->debugfs_dir, 0400);
 	MVM_DEBUGFS_ADD_FILE(disable_power_off, mvm->debugfs_dir, 0600);
-	MVM_DEBUGFS_ADD_FILE(fw_ver, mvm->debugfs_dir, 0400);
 	MVM_DEBUGFS_ADD_FILE(fw_rx_stats, mvm->debugfs_dir, 0400);
 	MVM_DEBUGFS_ADD_FILE(drv_rx_stats, mvm->debugfs_dir, 0400);
 	MVM_DEBUGFS_ADD_FILE(fw_system_stats, mvm->debugfs_dir, 0400);
@@ -2177,7 +2130,6 @@ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm)
 	MVM_DEBUGFS_ADD_FILE(scan_ant_rxchain, mvm->debugfs_dir, 0600);
 	MVM_DEBUGFS_ADD_FILE(prph_reg, mvm->debugfs_dir, 0600);
 	MVM_DEBUGFS_ADD_FILE(fw_dbg_conf, mvm->debugfs_dir, 0600);
-	MVM_DEBUGFS_ADD_FILE(fw_dbg_collect, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(fw_dbg_clear, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(dbg_time_point, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(send_echo_cmd, mvm->debugfs_dir, 0200);
diff --git a/sys/contrib/dev/iwlwifi/mvm/ftm-initiator.c b/sys/contrib/dev/iwlwifi/mvm/ftm-initiator.c
index b0371acb98ca..83ae969db0da 100644
--- a/sys/contrib/dev/iwlwifi/mvm/ftm-initiator.c
+++ b/sys/contrib/dev/iwlwifi/mvm/ftm-initiator.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include <linux/etherdevice.h>
 #include <linux/math64.h>
@@ -46,107 +46,6 @@ struct iwl_mvm_ftm_iter_data {
 	u8 *tk;
 };
 
-int iwl_mvm_ftm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			     u8 *addr, u32 cipher, u8 *tk, u32 tk_len,
-			     u8 *hltk, u32 hltk_len)
-{
-	struct iwl_mvm_ftm_pasn_entry *pasn = kzalloc(sizeof(*pasn),
-						      GFP_KERNEL);
-	u32 expected_tk_len;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	if (!pasn)
-		return -ENOBUFS;
-
-	iwl_mvm_ftm_remove_pasn_sta(mvm, addr);
-
-	pasn->cipher = iwl_mvm_cipher_to_location_cipher(cipher);
-
-	switch (pasn->cipher) {
-	case IWL_LOCATION_CIPHER_CCMP_128:
-	case IWL_LOCATION_CIPHER_GCMP_128:
-		expected_tk_len = WLAN_KEY_LEN_CCMP;
-		break;
-	case IWL_LOCATION_CIPHER_GCMP_256:
-		expected_tk_len = WLAN_KEY_LEN_GCMP_256;
-		break;
-	default:
-		goto out;
-	}
-
-	/*
-	 * If associated to this AP and already have security context,
-	 * the TK is already configured for this station, so it
-	 * shouldn't be set again here.
-	 */
-	if (vif->cfg.assoc) {
-		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-		struct ieee80211_bss_conf *link_conf;
-		unsigned int link_id;
-		struct ieee80211_sta *sta;
-		u8 sta_id;
-
-		rcu_read_lock();
-		for_each_vif_active_link(vif, link_conf, link_id) {
-			if (memcmp(addr, link_conf->bssid, ETH_ALEN))
-				continue;
-
-			sta_id = mvmvif->link[link_id]->ap_sta_id;
-			sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]);
-			if (!IS_ERR_OR_NULL(sta) && sta->mfp)
-				expected_tk_len = 0;
-			break;
-		}
-		rcu_read_unlock();
-	}
-
-	if (tk_len != expected_tk_len ||
-	    (hltk_len && hltk_len != sizeof(pasn->hltk))) {
-		IWL_ERR(mvm, "Invalid key length: tk_len=%u hltk_len=%u\n",
-			tk_len, hltk_len);
-		goto out;
-	}
-
-	if (!expected_tk_len && !hltk_len) {
-		IWL_ERR(mvm, "TK and HLTK not set\n");
-		goto out;
-	}
-
-	memcpy(pasn->addr, addr, sizeof(pasn->addr));
-
-	if (hltk_len) {
-		memcpy(pasn->hltk, hltk, sizeof(pasn->hltk));
-		pasn->flags |= IWL_MVM_PASN_FLAG_HAS_HLTK;
-	}
-
-	if (tk && tk_len)
-		memcpy(pasn->tk, tk, sizeof(pasn->tk));
-
-	list_add_tail(&pasn->list, &mvm->ftm_initiator.pasn_list);
-	return 0;
-out:
-	kfree(pasn);
-	return -EINVAL;
-}
-
-void iwl_mvm_ftm_remove_pasn_sta(struct iwl_mvm *mvm, u8 *addr)
-{
-	struct iwl_mvm_ftm_pasn_entry *entry, *prev;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	list_for_each_entry_safe(entry, prev, &mvm->ftm_initiator.pasn_list,
-				 list) {
-		if (memcmp(entry->addr, addr, sizeof(entry->addr)))
-			continue;
-
-		list_del(&entry->list);
-		kfree(entry);
-		return;
-	}
-}
-
 static void iwl_mvm_ftm_reset(struct iwl_mvm *mvm)
 {
 	struct iwl_mvm_loc_entry *e, *t;
@@ -563,12 +462,12 @@ static int iwl_mvm_ftm_set_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 		if (mvmvif->ftm_unprotected) {
-			*sta_id = IWL_MVM_INVALID_STA;
+			*sta_id = IWL_INVALID_STA;
 			*flags &= ~cpu_to_le32(IWL_INITIATOR_AP_FLAGS_PMF);
 		}
 #endif
 	} else {
-		*sta_id = IWL_MVM_INVALID_STA;
+		*sta_id = IWL_INVALID_STA;
 	}
 
 	return 0;
@@ -776,7 +675,12 @@ iwl_mvm_ftm_set_secured_ranging(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			struct iwl_mvm_ftm_iter_data target;
 
 			target.bssid = bssid;
+			target.cipher = cipher;
+			target.tk = NULL;
 			ieee80211_iter_keys(mvm->hw, vif, iter, &target);
+
+			if (!WARN_ON(!target.tk))
+				memcpy(tk, target.tk, TK_11AZ_LEN);
 		} else {
 			memcpy(tk, entry->tk, sizeof(entry->tk));
 		}
@@ -952,7 +856,7 @@ static int iwl_mvm_ftm_start_v13(struct iwl_mvm *mvm,
 static int
 iwl_mvm_ftm_put_target_v10(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			   struct cfg80211_pmsr_request_peer *peer,
-			   struct iwl_tof_range_req_ap_entry_v10 *target)
+			   struct iwl_tof_range_req_ap_entry *target)
 {
 	u32 i2r_max_sts, flags;
 	int ret;
@@ -1024,7 +928,7 @@ static int iwl_mvm_ftm_start_v14(struct iwl_mvm *mvm,
 				 struct ieee80211_vif *vif,
 				 struct cfg80211_pmsr_request *req)
 {
-	struct iwl_tof_range_req_cmd_v14 cmd;
+	struct iwl_tof_range_req_cmd cmd;
 	struct iwl_host_cmd hcmd = {
 		.id = WIDE_ID(LOCATION_GROUP, TOF_RANGE_REQ_CMD),
 		.dataflags[0] = IWL_HCMD_DFL_DUP,
@@ -1038,7 +942,7 @@ static int iwl_mvm_ftm_start_v14(struct iwl_mvm *mvm,
 
 	for (i = 0; i < cmd.num_of_ap; i++) {
 		struct cfg80211_pmsr_request_peer *peer = &req->peers[i];
-		struct iwl_tof_range_req_ap_entry_v10 *target = &cmd.ap[i];
+		struct iwl_tof_range_req_ap_entry *target = &cmd.ap[i];
 
 		err = iwl_mvm_ftm_put_target_v10(mvm, vif, peer, target);
 		if (err)
@@ -1066,6 +970,8 @@ int iwl_mvm_ftm_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 						   IWL_FW_CMD_VER_UNKNOWN);
 
 		switch (cmd_ver) {
+		case 15:
+			/* Version 15 has the same struct as 14 */
 		case 14:
 			err = iwl_mvm_ftm_start_v14(mvm, vif, req);
 			break;
@@ -1302,7 +1208,7 @@ static void iwl_mvm_debug_range_resp(struct iwl_mvm *mvm, u8 index,
 
 static void
 iwl_mvm_ftm_pasn_update_pn(struct iwl_mvm *mvm,
-			   struct iwl_tof_range_rsp_ap_entry_ntfy_v6 *fw_ap)
+			   struct iwl_tof_range_rsp_ap_entry_ntfy *fw_ap)
 {
 	struct iwl_mvm_ftm_pasn_entry *entry;
 
@@ -1340,7 +1246,7 @@ static bool iwl_mvm_ftm_resp_size_validation(u8 ver, unsigned int pkt_len)
 	switch (ver) {
 	case 9:
 	case 8:
-		return pkt_len == sizeof(struct iwl_tof_range_rsp_ntfy_v8);
+		return pkt_len == sizeof(struct iwl_tof_range_rsp_ntfy);
 	case 7:
 		return pkt_len == sizeof(struct iwl_tof_range_rsp_ntfy_v7);
 	case 6:
@@ -1360,7 +1266,7 @@ void iwl_mvm_ftm_range_resp(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	struct iwl_tof_range_rsp_ntfy_v5 *fw_resp_v5 = (void *)pkt->data;
 	struct iwl_tof_range_rsp_ntfy_v6 *fw_resp_v6 = (void *)pkt->data;
 	struct iwl_tof_range_rsp_ntfy_v7 *fw_resp_v7 = (void *)pkt->data;
-	struct iwl_tof_range_rsp_ntfy_v8 *fw_resp_v8 = (void *)pkt->data;
+	struct iwl_tof_range_rsp_ntfy *fw_resp_v8 = (void *)pkt->data;
 	int i;
 	bool new_api = fw_has_api(&mvm->fw->ucode_capa,
 				  IWL_UCODE_TLV_API_FTM_NEW_RANGE_REQ);
@@ -1396,9 +1302,9 @@ void iwl_mvm_ftm_range_resp(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	IWL_DEBUG_INFO(mvm, "request id: %lld, num of entries: %u\n",
 		       mvm->ftm_initiator.req->cookie, num_of_aps);
 
-	for (i = 0; i < num_of_aps && i < IWL_MVM_TOF_MAX_APS; i++) {
+	for (i = 0; i < num_of_aps && i < IWL_TOF_MAX_APS; i++) {
 		struct cfg80211_pmsr_result result = {};
-		struct iwl_tof_range_rsp_ap_entry_ntfy_v6 *fw_ap;
+		struct iwl_tof_range_rsp_ap_entry_ntfy *fw_ap;
 		int peer_idx;
 
 		if (new_api) {
diff --git a/sys/contrib/dev/iwlwifi/mvm/ftm-responder.c b/sys/contrib/dev/iwlwifi/mvm/ftm-responder.c
index 3e989f3141b5..83f6e508a094 100644
--- a/sys/contrib/dev/iwlwifi/mvm/ftm-responder.c
+++ b/sys/contrib/dev/iwlwifi/mvm/ftm-responder.c
@@ -131,7 +131,7 @@ iwl_mvm_ftm_responder_cmd(struct iwl_mvm *mvm,
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (cmd_ver == 10) {
+	if (cmd_ver >= 10) {
 		cmd.band =
 			iwl_mvm_phy_band_from_nl80211(chandef->chan->band);
 	}
@@ -324,94 +324,6 @@ static void iwl_mvm_resp_del_pasn_sta(struct iwl_mvm *mvm,
 	kfree(sta);
 }
 
-#if defined(__linux__)
-int iwl_mvm_ftm_respoder_add_pasn_sta(struct iwl_mvm *mvm,
-				      struct ieee80211_vif *vif,
-				      u8 *addr, u32 cipher, u8 *tk, u32 tk_len,
-				      u8 *hltk, u32 hltk_len)
-{
-	int ret;
-	struct iwl_mvm_pasn_sta *sta = NULL;
-	struct iwl_mvm_pasn_hltk_data hltk_data = {
-		.addr = addr,
-		.hltk = hltk,
-	};
-	struct iwl_mvm_pasn_hltk_data *hltk_data_ptr = NULL;
-
-	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw,
-					   WIDE_ID(LOCATION_GROUP, TOF_RESPONDER_DYN_CONFIG_CMD),
-					   2);
-
-	lockdep_assert_held(&mvm->mutex);
-
-	if (cmd_ver < 3) {
-		IWL_ERR(mvm, "Adding PASN station not supported by FW\n");
-		return -EOPNOTSUPP;
-	}
-
-	if ((!hltk || !hltk_len) && (!tk || !tk_len)) {
-		IWL_ERR(mvm, "TK and HLTK not set\n");
-		return -EINVAL;
-	}
-
-	if (hltk && hltk_len) {
-		if (!fw_has_capa(&mvm->fw->ucode_capa,
-				 IWL_UCODE_TLV_CAPA_SECURE_LTF_SUPPORT)) {
-			IWL_ERR(mvm, "No support for secure LTF measurement\n");
-			return -EINVAL;
-		}
-
-		hltk_data.cipher = iwl_mvm_cipher_to_location_cipher(cipher);
-		if (hltk_data.cipher == IWL_LOCATION_CIPHER_INVALID) {
-			IWL_ERR(mvm, "invalid cipher: %u\n", cipher);
-			return -EINVAL;
-		}
-
-		hltk_data_ptr = &hltk_data;
-	}
-
-	if (tk && tk_len) {
-		sta = kzalloc(sizeof(*sta) + tk_len, GFP_KERNEL);
-		if (!sta)
-			return -ENOBUFS;
-
-		ret = iwl_mvm_add_pasn_sta(mvm, vif, &sta->int_sta, addr,
-					   cipher, tk, tk_len, &sta->keyconf);
-		if (ret) {
-			kfree(sta);
-			return ret;
-		}
-
-		memcpy(sta->addr, addr, ETH_ALEN);
-		list_add_tail(&sta->list, &mvm->resp_pasn_list);
-	}
-
-	ret = iwl_mvm_ftm_responder_dyn_cfg_v3(mvm, vif, NULL, hltk_data_ptr);
-	if (ret && sta)
-		iwl_mvm_resp_del_pasn_sta(mvm, vif, sta);
-
-	return ret;
-}
-
-int iwl_mvm_ftm_resp_remove_pasn_sta(struct iwl_mvm *mvm,
-				     struct ieee80211_vif *vif, u8 *addr)
-{
-	struct iwl_mvm_pasn_sta *sta, *prev;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	list_for_each_entry_safe(sta, prev, &mvm->resp_pasn_list, list) {
-		if (!memcmp(sta->addr, addr, ETH_ALEN)) {
-			iwl_mvm_resp_del_pasn_sta(mvm, vif, sta);
-			return 0;
-		}
-	}
-
-	IWL_ERR(mvm, "FTM: PASN station %pM not found\n", addr);
-	return -EINVAL;
-}
-#endif
-
 int iwl_mvm_ftm_start_responder(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				struct ieee80211_bss_conf *bss_conf)
 {
diff --git a/sys/contrib/dev/iwlwifi/mvm/fw.c b/sys/contrib/dev/iwlwifi/mvm/fw.c
index 08c4898c8f1a..d931c6eaf12f 100644
--- a/sys/contrib/dev/iwlwifi/mvm/fw.c
+++ b/sys/contrib/dev/iwlwifi/mvm/fw.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -29,8 +29,8 @@
 #define MVM_UCODE_CALIB_TIMEOUT	(2 * HZ)
 
 struct iwl_mvm_alive_data {
+	__le32 sku_id[3];
 	bool valid;
-	u32 scd_base_addr;
 };
 
 static int iwl_send_tx_ant_cfg(struct iwl_mvm *mvm, u8 valid_tx_ant)
@@ -57,13 +57,13 @@ static int iwl_send_rss_cfg_cmd(struct iwl_mvm *mvm)
 			     BIT(IWL_RSS_HASH_TYPE_IPV6_PAYLOAD),
 	};
 
-	if (mvm->trans->num_rx_queues == 1)
+	if (mvm->trans->info.num_rxqs == 1)
 		return 0;
 
 	/* Do not direct RSS traffic to Q 0 which is our fallback queue */
 	for (i = 0; i < ARRAY_SIZE(cmd.indirection_table); i++)
 		cmd.indirection_table[i] =
-			1 + (i % (mvm->trans->num_rx_queues - 1));
+			1 + (i % (mvm->trans->info.num_rxqs - 1));
 	netdev_rss_key_fill(cmd.secret_key, sizeof(cmd.secret_key));
 
 	return iwl_mvm_send_cmd_pdu(mvm, RSS_CONFIG_CMD, 0, sizeof(cmd), &cmd);
@@ -114,13 +114,29 @@ static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
 	u32 i;
 
 
-	if (version == 6) {
+	if (version >= 6) {
 		struct iwl_alive_ntf_v6 *palive;
 
 		if (pkt_len < sizeof(*palive))
 			return false;
 
 		palive = (void *)pkt->data;
+
+		umac = &palive->umac_data;
+		lmac1 = &palive->lmac_data[0];
+		lmac2 = &palive->lmac_data[1];
+		status = le16_to_cpu(palive->status);
+
+		BUILD_BUG_ON(sizeof(palive->sku_id.data) !=
+			     sizeof(alive_data->sku_id));
+		memcpy(alive_data->sku_id, palive->sku_id.data,
+		       sizeof(palive->sku_id.data));
+
+		IWL_DEBUG_FW(mvm, "Got sku_id: 0x0%x 0x0%x 0x0%x\n",
+			     le32_to_cpu(alive_data->sku_id[0]),
+			     le32_to_cpu(alive_data->sku_id[1]),
+			     le32_to_cpu(alive_data->sku_id[2]));
+
 		mvm->trans->dbg.imr_data.imr_enable =
 			le32_to_cpu(palive->imr.enabled);
 		mvm->trans->dbg.imr_data.imr_size =
@@ -157,39 +173,17 @@ static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
 				}
 			}
 		}
-	}
 
-	if (version >= 5) {
-		struct iwl_alive_ntf_v5 *palive;
+		if (version >= 8) {
+			const struct iwl_alive_ntf *palive_v8 =
+				(void *)pkt->data;
 
-		if (pkt_len < sizeof(*palive))
-			return false;
-
-		palive = (void *)pkt->data;
-		umac = &palive->umac_data;
-		lmac1 = &palive->lmac_data[0];
-		lmac2 = &palive->lmac_data[1];
-		status = le16_to_cpu(palive->status);
+			if (pkt_len < sizeof(*palive_v8))
+				return false;
 
-		mvm->trans->sku_id[0] = le32_to_cpu(palive->sku_id.data[0]);
-		mvm->trans->sku_id[1] = le32_to_cpu(palive->sku_id.data[1]);
-		mvm->trans->sku_id[2] = le32_to_cpu(palive->sku_id.data[2]);
-
-		IWL_DEBUG_FW(mvm, "Got sku_id: 0x0%x 0x0%x 0x0%x\n",
-			     mvm->trans->sku_id[0],
-			     mvm->trans->sku_id[1],
-			     mvm->trans->sku_id[2]);
-	} else if (iwl_rx_packet_payload_len(pkt) == sizeof(struct iwl_alive_ntf_v4)) {
-		struct iwl_alive_ntf_v4 *palive;
-
-		if (pkt_len < sizeof(*palive))
-			return false;
-
-		palive = (void *)pkt->data;
-		umac = &palive->umac_data;
-		lmac1 = &palive->lmac_data[0];
-		lmac2 = &palive->lmac_data[1];
-		status = le16_to_cpu(palive->status);
+			IWL_DEBUG_FW(mvm, "platform id: 0x%llx\n",
+				     palive_v8->platform_id);
+		}
 	} else if (iwl_rx_packet_payload_len(pkt) ==
 		   sizeof(struct iwl_alive_ntf_v3)) {
 		struct iwl_alive_ntf_v3 *palive3;
@@ -221,7 +215,7 @@ static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
 
 	if (umac_error_table) {
 		if (umac_error_table >=
-		    mvm->trans->cfg->min_umac_error_event_table) {
+		    mvm->trans->mac_cfg->base->min_umac_error_event_table) {
 			iwl_fw_umac_set_alive_err_table(mvm->trans,
 							umac_error_table);
 		} else {
@@ -233,7 +227,6 @@ static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
 		}
 	}
 
-	alive_data->scd_base_addr = le32_to_cpu(lmac1->dbg_ptrs.scd_base_ptr);
 	alive_data->valid = status == IWL_ALIVE_STATUS_OK;
 
 	IWL_DEBUG_FW(mvm,
@@ -282,7 +275,7 @@ static void iwl_mvm_print_pd_notification(struct iwl_mvm *mvm)
 	IWL_ERR(mvm, #reg_name ": 0x%x\n", iwl_read_umac_prph(trans, reg_name))
 
 	struct iwl_trans *trans = mvm->trans;
-	enum iwl_device_family device_family = trans->trans_cfg->device_family;
+	enum iwl_device_family device_family = trans->mac_cfg->device_family;
 
 	if (device_family < IWL_DEVICE_FAMILY_8000)
 		return;
@@ -304,7 +297,6 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 {
 	struct iwl_notification_wait alive_wait;
 	struct iwl_mvm_alive_data alive_data = {};
-	const struct fw_img *fw;
 	int ret;
 	enum iwl_ucode_type old_type = mvm->fwrt.cur_fw_img;
 	static const u16 alive_cmd[] = { UCODE_ALIVE_NTFY };
@@ -317,11 +309,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	    iwl_fw_dbg_conf_usniffer(mvm->fw, FW_DBG_START_FROM_ALIVE) &&
 	    !(fw_has_capa(&mvm->fw->ucode_capa,
 			  IWL_UCODE_TLV_CAPA_USNIFFER_UNIFIED)))
-		fw = iwl_get_ucode_image(mvm->fw, IWL_UCODE_REGULAR_USNIFFER);
-	else
-		fw = iwl_get_ucode_image(mvm->fw, ucode_type);
-	if (WARN_ON(!fw))
-		return -EINVAL;
+		ucode_type = IWL_UCODE_REGULAR_USNIFFER;
 	iwl_fw_set_current_image(&mvm->fwrt, ucode_type);
 	clear_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
 
@@ -334,7 +322,8 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	 * For the unified firmware case, the ucode_type is not
 	 * INIT, but we still need to run it.
 	 */
-	ret = iwl_trans_start_fw(mvm->trans, fw, run_in_rfkill);
+	ret = iwl_trans_start_fw(mvm->trans, mvm->fw, ucode_type,
+				 run_in_rfkill);
 	if (ret) {
 		iwl_fw_set_current_image(&mvm->fwrt, old_type);
 		iwl_remove_notification(&mvm->notif_wait, &alive_wait);
@@ -348,7 +337,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	ret = iwl_wait_notification(&mvm->notif_wait, &alive_wait,
 				    MVM_UCODE_ALIVE_TIMEOUT);
 
-	if (mvm->trans->trans_cfg->device_family ==
+	if (mvm->trans->mac_cfg->device_family ==
 	    IWL_DEVICE_FAMILY_AX210) {
 		/* print these registers regardless of alive fail/success */
 		IWL_INFO(mvm, "WFPM_UMAC_PD_NOTIFICATION: 0x%x\n",
@@ -365,14 +354,14 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 		struct iwl_trans *trans = mvm->trans;
 
 		/* SecBoot info */
-		if (trans->trans_cfg->device_family >=
+		if (trans->mac_cfg->device_family >=
 					IWL_DEVICE_FAMILY_22000) {
 			IWL_ERR(mvm,
 				"SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
 				iwl_read_umac_prph(trans, UMAG_SB_CPU_1_STATUS),
 				iwl_read_umac_prph(trans,
 						   UMAG_SB_CPU_2_STATUS));
-		} else if (trans->trans_cfg->device_family >=
+		} else if (trans->mac_cfg->device_family >=
 			   IWL_DEVICE_FAMILY_8000) {
 			IWL_ERR(mvm,
 				"SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
@@ -383,7 +372,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 		iwl_mvm_print_pd_notification(mvm);
 
 		/* LMAC/UMAC PC info */
-		if (trans->trans_cfg->device_family >=
+		if (trans->mac_cfg->device_family >=
 					IWL_DEVICE_FAMILY_22000) {
 			pc_data = trans->dbg.pc_data;
 			for (count = 0; count < trans->dbg.num_pc;
@@ -391,7 +380,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 				IWL_ERR(mvm, "%s: 0x%x\n",
 					pc_data->pc_name,
 					pc_data->pc_address);
-		} else if (trans->trans_cfg->device_family >=
+		} else if (trans->mac_cfg->device_family >=
 					IWL_DEVICE_FAMILY_9000) {
 			IWL_ERR(mvm, "UMAC PC: 0x%x\n",
 				iwl_read_umac_prph(trans,
@@ -422,16 +411,16 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	/* if reached this point, Alive notification was received */
 	iwl_mei_alive_notif(true);
 
+	iwl_trans_fw_alive(mvm->trans);
+
 	ret = iwl_pnvm_load(mvm->trans, &mvm->notif_wait,
-			    &mvm->fw->ucode_capa);
+			    mvm->fw, alive_data.sku_id);
 	if (ret) {
 		IWL_ERR(mvm, "Timeout waiting for PNVM load!\n");
 		iwl_fw_set_current_image(&mvm->fwrt, old_type);
 		return ret;
 	}
 
-	iwl_trans_fw_alive(mvm->trans, alive_data.scd_base_addr);
-
 	/*
 	 * Note: all the queues are enabled as part of the interface
 	 * initialization, but in firmware restart scenarios they
@@ -473,7 +462,7 @@ static void iwl_mvm_phy_filter_init(struct iwl_mvm *mvm,
 				    struct iwl_phy_specific_cfg *phy_filters)
 {
 #ifdef CONFIG_ACPI
-	*phy_filters = mvm->phy_filters;
+	*phy_filters = mvm->fwrt.phy_filters;
 #endif /* CONFIG_ACPI */
 }
 
@@ -490,7 +479,7 @@ static void iwl_mvm_uats_init(struct iwl_mvm *mvm)
 		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
 	};
 
-	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
+	if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
 		IWL_DEBUG_RADIO(mvm, "UATS feature is not supported\n");
 		return;
 	}
@@ -504,11 +493,10 @@ static void iwl_mvm_uats_init(struct iwl_mvm *mvm)
 		return;
 	}
 
-	ret = iwl_uefi_get_uats_table(mvm->trans, &mvm->fwrt);
-	if (ret < 0) {
-		IWL_DEBUG_FW(mvm, "failed to read UATS table (%d)\n", ret);
+	iwl_uefi_get_uats_table(mvm->trans, &mvm->fwrt);
+
+	if (!mvm->fwrt.uats_valid)
 		return;
-	}
 
 	ret = iwl_mvm_send_cmd(mvm, &cmd);
 	if (ret < 0)
@@ -578,7 +566,7 @@ static int iwl_send_phy_cfg_cmd(struct iwl_mvm *mvm)
 
 	/* set flags extra PHY configuration flags from the device's cfg */
 	phy_cfg_cmd.phy_cfg |=
-		cpu_to_le32(mvm->trans->trans_cfg->extra_phy_cfg_flags);
+		cpu_to_le32(mvm->trans->mac_cfg->extra_phy_cfg_flags);
 
 	phy_cfg_cmd.calib_control.event_trigger =
 		mvm->fw->default_calib[ucode_type].event_trigger;
@@ -617,7 +605,7 @@ static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm)
 
 	mvm->rfkill_safe_init_done = false;
 
-	if (mvm->trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210) {
+	if (mvm->trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_AX210) {
 		sb_cfg = iwl_read_umac_prph(mvm->trans, SB_MODIFY_CFG_FLAG);
 		/* if needed, we'll reset this on our way out later */
 		mvm->fw_product_reset = sb_cfg == SB_CFG_RESIDES_IN_ROM;
@@ -642,7 +630,8 @@ static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm)
 		/* if we needed reset then fail here, but notify and remove */
 		if (mvm->fw_product_reset) {
 			iwl_mei_alive_notif(false);
-			iwl_trans_pcie_remove(mvm->trans, true);
+			iwl_trans_pcie_reset(mvm->trans,
+					     IWL_RESET_MODE_RESCAN);
 		}
 
 		goto error;
@@ -650,11 +639,6 @@ static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm)
 	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_POINT_AFTER_ALIVE,
 			       NULL);
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
-		mvm->trans->step_urm = !!(iwl_read_umac_prph(mvm->trans,
-							     CNVI_PMU_STEP_FLOW) &
-						CNVI_PMU_STEP_FLOW_FORCE_URM);
-
 	/* Send init config command to mark that we are sending NVM access
 	 * commands
 	 */
@@ -755,7 +739,7 @@ int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm)
 		goto remove_notif;
 	}
 
-	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000) {
+	if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_8000) {
 		ret = iwl_mvm_send_bt_init_conf(mvm);
 		if (ret)
 			goto remove_notif;
@@ -863,7 +847,10 @@ static int iwl_mvm_config_ltr(struct iwl_mvm *mvm)
 int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b)
 {
 	u32 cmd_id = REDUCE_TX_POWER_CMD;
-	struct iwl_dev_tx_power_cmd cmd = {
+	struct iwl_dev_tx_power_cmd_v3_v8 cmd = {
+		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_CHAINS),
+	};
+	struct iwl_dev_tx_power_cmd cmd_v9_v10 = {
 		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_CHAINS),
 	};
 	__le16 *per_chain;
@@ -871,18 +858,23 @@ int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b)
 	u16 len = 0;
 	u32 n_subbands;
 	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 3);
+	void *cmd_data = &cmd;
 
-	if (cmd_ver >= 7) {
-		len = sizeof(cmd.v7);
+	if (cmd_ver == 10) {
+		len = sizeof(cmd_v9_v10.v10);
 		n_subbands = IWL_NUM_SUB_BANDS_V2;
-		per_chain = cmd.v7.per_chain[0][0];
-		cmd.v7.flags = cpu_to_le32(mvm->fwrt.reduced_power_flags);
-		if (cmd_ver == 8)
-			len = sizeof(cmd.v8);
-	} else if (cmd_ver == 6) {
-		len = sizeof(cmd.v6);
+		per_chain = &cmd_v9_v10.v10.per_chain[0][0][0];
+		cmd_v9_v10.v10.flags =
+			cpu_to_le32(mvm->fwrt.reduced_power_flags);
+	} else if (cmd_ver == 9) {
+		len = sizeof(cmd_v9_v10.v9);
+		n_subbands = IWL_NUM_SUB_BANDS_V1;
+		per_chain = &cmd_v9_v10.v9.per_chain[0][0];
+	} else if (cmd_ver == 8) {
+		len = sizeof(cmd.v8);
 		n_subbands = IWL_NUM_SUB_BANDS_V2;
-		per_chain = cmd.v6.per_chain[0][0];
+		per_chain = cmd.v8.per_chain[0][0];
+		cmd.v8.flags = cpu_to_le32(mvm->fwrt.reduced_power_flags);
 	} else if (fw_has_api(&mvm->fw->ucode_capa,
 			      IWL_UCODE_TLV_API_REDUCE_TX_POWER)) {
 		len = sizeof(cmd.v5);
@@ -899,9 +891,14 @@ int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b)
 		per_chain = cmd.v3.per_chain[0][0];
 	}
 
-	/* all structs have the same common part, add it */
+	/* all structs have the same common part, add its length */
 	len += sizeof(cmd.common);
 
+	if (cmd_ver < 9)
+		len += sizeof(cmd.per_band);
+	else
+		cmd_data = &cmd_v9_v10;
+
 	ret = iwl_sar_fill_profile(&mvm->fwrt, per_chain,
 				   IWL_NUM_CHAIN_TABLES,
 				   n_subbands, prof_a, prof_b);
@@ -913,7 +910,7 @@ int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b)
 	iwl_mei_set_power_limit(per_chain);
 
 	IWL_DEBUG_RADIO(mvm, "Sending REDUCE_TX_POWER_CMD per chain\n");
-	return iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, len, &cmd);
+	return iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, len, cmd_data);
 }
 
 int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm)
@@ -1074,36 +1071,24 @@ static int iwl_mvm_ppag_init(struct iwl_mvm *mvm)
 	return iwl_mvm_ppag_send_cmd(mvm);
 }
 
-static bool iwl_mvm_add_to_tas_block_list(__le32 *list, __le32 *le_size, unsigned int mcc)
-{
-	int i;
-	u32 size = le32_to_cpu(*le_size);
-
-	/* Verify that there is room for another country */
-	if (size >= IWL_WTAS_BLACK_LIST_MAX)
-		return false;
-
-	for (i = 0; i < size; i++) {
-		if (list[i] == cpu_to_le32(mcc))
-			return true;
-	}
-
-	list[size++] = cpu_to_le32(mcc);
-	*le_size = cpu_to_le32(size);
-	return true;
-}
-
 static void iwl_mvm_tas_init(struct iwl_mvm *mvm)
 {
 	u32 cmd_id = WIDE_ID(REGULATORY_AND_NVM_GROUP, TAS_CONFIG);
-	int ret;
+	int fw_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id,
+					   IWL_FW_CMD_VER_UNKNOWN);
+	struct iwl_tas_selection_data selection_data = {};
+	struct iwl_tas_config_cmd_v2_v4 cmd_v2_v4 = {};
+	struct iwl_tas_config_cmd cmd_v5 = {};
 	struct iwl_tas_data data = {};
-	struct iwl_tas_config_cmd cmd = {};
-	int cmd_size, fw_ver;
+	void *cmd_data = &cmd_v2_v4;
+	int cmd_size;
+	int ret;
 
 	BUILD_BUG_ON(ARRAY_SIZE(data.block_list_array) !=
 		     IWL_WTAS_BLACK_LIST_MAX);
-	BUILD_BUG_ON(ARRAY_SIZE(cmd.common.block_list_array) !=
+	BUILD_BUG_ON(ARRAY_SIZE(cmd_v2_v4.common.block_list_array) !=
+		     IWL_WTAS_BLACK_LIST_MAX);
+	BUILD_BUG_ON(ARRAY_SIZE(cmd_v5.block_list_array) !=
 		     IWL_WTAS_BLACK_LIST_MAX);
 
 	if (!fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_TAS_CFG)) {
@@ -1119,17 +1104,17 @@ static void iwl_mvm_tas_init(struct iwl_mvm *mvm)
 		return;
 	}
 
-	if (ret == 0)
+	if (ret == 0 && fw_ver < 5)
 		return;
 
 	if (!iwl_is_tas_approved()) {
 		IWL_DEBUG_RADIO(mvm,
 				"System vendor '%s' is not in the approved list, disabling TAS in US and Canada.\n",
 				dmi_get_system_info(DMI_SYS_VENDOR) ?: "<unknown>");
-		if ((!iwl_mvm_add_to_tas_block_list(data.block_list_array,
+		if ((!iwl_add_mcc_to_tas_block_list(data.block_list_array,
 						    &data.block_list_size,
 						    IWL_MCC_US)) ||
-		    (!iwl_mvm_add_to_tas_block_list(data.block_list_array,
+		    (!iwl_add_mcc_to_tas_block_list(data.block_list_array,
 						    &data.block_list_size,
 						    IWL_MCC_CANADA))) {
 			IWL_DEBUG_RADIO(mvm,
@@ -1142,63 +1127,53 @@ static void iwl_mvm_tas_init(struct iwl_mvm *mvm)
 				dmi_get_system_info(DMI_SYS_VENDOR) ?: "<unknown>");
 	}
 
-	fw_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id,
-				       IWL_FW_CMD_VER_UNKNOWN);
-
-	memcpy(&cmd.common, &data, sizeof(struct iwl_tas_config_cmd_common));
-
-	/* Set v3 or v4 specific parts. will be trunctated for fw_ver < 3 */
-	if (fw_ver == 4) {
-		cmd.v4.override_tas_iec = data.override_tas_iec;
-		cmd.v4.enable_tas_iec = data.enable_tas_iec;
-		cmd.v4.usa_tas_uhb_allowed = data.usa_tas_uhb_allowed;
+	if (fw_ver < 5) {
+		selection_data = iwl_parse_tas_selection(data.tas_selection,
+							 data.table_revision);
+		cmd_v2_v4.common.block_list_size =
+			cpu_to_le32(data.block_list_size);
+		for (u8 i = 0; i < data.block_list_size; i++)
+			cmd_v2_v4.common.block_list_array[i] =
+				cpu_to_le32(data.block_list_array[i]);
+	}
+
+	if (fw_ver == 5) {
+		cmd_size = sizeof(cmd_v5);
+		cmd_data = &cmd_v5;
+		cmd_v5.block_list_size = cpu_to_le16(data.block_list_size);
+		for (u16 i = 0; i < data.block_list_size; i++)
+			cmd_v5.block_list_array[i] =
+				cpu_to_le16(data.block_list_array[i]);
+		cmd_v5.tas_config_info.table_source = data.table_source;
+		cmd_v5.tas_config_info.table_revision = data.table_revision;
+		cmd_v5.tas_config_info.value = cpu_to_le32(data.tas_selection);
+	} else if (fw_ver == 4) {
+		cmd_size = sizeof(cmd_v2_v4.common) + sizeof(cmd_v2_v4.v4);
+		cmd_v2_v4.v4.override_tas_iec = selection_data.override_tas_iec;
+		cmd_v2_v4.v4.enable_tas_iec = selection_data.enable_tas_iec;
+		cmd_v2_v4.v4.usa_tas_uhb_allowed =
+			selection_data.usa_tas_uhb_allowed;
+		if (fw_has_capa(&mvm->fw->ucode_capa,
+				IWL_UCODE_TLV_CAPA_UHB_CANADA_TAS_SUPPORT) &&
+		    selection_data.canada_tas_uhb_allowed)
+			cmd_v2_v4.v4.uhb_allowed_flags = TAS_UHB_ALLOWED_CANADA;
+	} else if (fw_ver == 3) {
+		cmd_size = sizeof(cmd_v2_v4.common) + sizeof(cmd_v2_v4.v3);
+		cmd_v2_v4.v3.override_tas_iec =
+			cpu_to_le16(selection_data.override_tas_iec);
+		cmd_v2_v4.v3.enable_tas_iec =
+			cpu_to_le16(selection_data.enable_tas_iec);
+	} else if (fw_ver == 2) {
+		cmd_size = sizeof(cmd_v2_v4.common);
 	} else {
-		cmd.v3.override_tas_iec = cpu_to_le16(data.override_tas_iec);
-		cmd.v3.enable_tas_iec = cpu_to_le16(data.enable_tas_iec);
+		return;
 	}
 
-	cmd_size = sizeof(struct iwl_tas_config_cmd_common);
-	if (fw_ver >= 3)
-		/* v4 is the same size as v3 */
-		cmd_size += sizeof(struct iwl_tas_config_cmd_v3);
-
-	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, cmd_size, &cmd);
+	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, cmd_size, cmd_data);
 	if (ret < 0)
 		IWL_DEBUG_RADIO(mvm, "failed to send TAS_CONFIG (%d)\n", ret);
 }
 
-static bool iwl_mvm_eval_dsm_rfi(struct iwl_mvm *mvm)
-{
-	u32 value = 0;
-	/* default behaviour is disabled */
-	bool bios_enable_rfi = false;
-	int ret = iwl_bios_get_dsm(&mvm->fwrt, DSM_FUNC_RFI_CONFIG, &value);
-
-
-	if (ret < 0) {
-		IWL_DEBUG_RADIO(mvm, "Failed to get DSM RFI, ret=%d\n", ret);
-		return bios_enable_rfi;
-	}
-
-	value &= DSM_VALUE_RFI_DISABLE;
-	/* RFI BIOS CONFIG value can be 0 or 3 only.
-	 * i.e 0 means DDR and DLVR enabled. 3 means DDR and DLVR disabled.
-	 * 1 and 2 are invalid BIOS configurations, So, it's not possible to
-	 * disable ddr/dlvr separately.
-	 */
-	if (!value) {
-		IWL_DEBUG_RADIO(mvm, "DSM RFI is evaluated to enable\n");
-		bios_enable_rfi = true;
-	} else if (value == DSM_VALUE_RFI_DISABLE) {
-		IWL_DEBUG_RADIO(mvm, "DSM RFI is evaluated to disable\n");
-	} else {
-		IWL_DEBUG_RADIO(mvm,
-				"DSM RFI got invalid value, value=%d\n", value);
-	}
-
-	return bios_enable_rfi;
-}
-
 static void iwl_mvm_lari_cfg(struct iwl_mvm *mvm)
 {
 	struct iwl_lari_config_change_cmd cmd;
@@ -1272,7 +1247,7 @@ void iwl_mvm_get_bios_tables(struct iwl_mvm *mvm)
 		}
 	}
 
-	iwl_acpi_get_phy_filters(&mvm->fwrt, &mvm->phy_filters);
+	iwl_acpi_get_phy_filters(&mvm->fwrt);
 
 	if (iwl_bios_get_eckv(&mvm->fwrt, &mvm->ext_clock_valid))
 		IWL_DEBUG_RADIO(mvm, "ECKV table doesn't exist in BIOS\n");
@@ -1288,8 +1263,8 @@ static void iwl_mvm_disconnect_iterator(void *data, u8 *mac,
 void iwl_mvm_send_recovery_cmd(struct iwl_mvm *mvm, u32 flags)
 {
 	u32 error_log_size = mvm->fw->ucode_capa.error_log_size;
+	u32 status = 0;
 	int ret;
-	u32 resp;
 
 	struct iwl_fw_error_recovery_cmd recovery_cmd = {
 		.flags = cpu_to_le32(flags),
@@ -1297,7 +1272,6 @@ void iwl_mvm_send_recovery_cmd(struct iwl_mvm *mvm, u32 flags)
 	};
 	struct iwl_host_cmd host_cmd = {
 		.id = WIDE_ID(SYSTEM_GROUP, FW_ERROR_RECOVERY_CMD),
-		.flags = CMD_WANT_SKB,
 		.data = {&recovery_cmd, },
 		.len = {sizeof(recovery_cmd), },
 	};
@@ -1317,7 +1291,7 @@ void iwl_mvm_send_recovery_cmd(struct iwl_mvm *mvm, u32 flags)
 		recovery_cmd.buf_size = cpu_to_le32(error_log_size);
 	}
 
-	ret = iwl_mvm_send_cmd(mvm, &host_cmd);
+	ret = iwl_mvm_send_cmd_status(mvm, &host_cmd, &status);
 	kfree(mvm->error_recovery_buf);
 	mvm->error_recovery_buf = NULL;
 
@@ -1328,11 +1302,10 @@ void iwl_mvm_send_recovery_cmd(struct iwl_mvm *mvm, u32 flags)
 
 	/* skb respond is only relevant in ERROR_RECOVERY_UPDATE_DB */
 	if (flags & ERROR_RECOVERY_UPDATE_DB) {
-		resp = le32_to_cpu(*(__le32 *)host_cmd.resp_pkt->data);
-		if (resp) {
+		if (status) {
 			IWL_ERR(mvm,
 				"Failed to send recovery cmd blob was invalid %d\n",
-				resp);
+				status);
 
 			ieee80211_iterate_interfaces(mvm->hw, 0,
 						     iwl_mvm_disconnect_iterator,
@@ -1384,6 +1357,7 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	int ret, i;
 	struct ieee80211_supported_band *sband = NULL;
 
+	lockdep_assert_wiphy(mvm->hw->wiphy);
 	lockdep_assert_held(&mvm->mutex);
 
 	ret = iwl_trans_start_hw(mvm->trans);
@@ -1464,10 +1438,7 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 		RCU_INIT_POINTER(mvm->fw_id_to_link_sta[i], NULL);
 	}
 
-	for (i = 0; i < IWL_MVM_FW_MAX_LINK_ID + 1; i++)
-		RCU_INIT_POINTER(mvm->link_id_to_link_conf[i], NULL);
-
-	mvm->tdls_cs.peer.sta_id = IWL_MVM_INVALID_STA;
+	mvm->tdls_cs.peer.sta_id = IWL_INVALID_STA;
 
 	/* reset quota debouncing buffer - 0xff will yield invalid data */
 	memset(&mvm->last_quota_cmd, 0xff, sizeof(mvm->last_quota_cmd));
@@ -1586,7 +1557,7 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	iwl_mvm_uats_init(mvm);
 
 	if (iwl_rfi_supported(mvm)) {
-		if (iwl_mvm_eval_dsm_rfi(mvm))
+		if (iwl_rfi_is_enabled_in_bios(&mvm->fwrt))
 			iwl_rfi_send_config_cmd(mvm, NULL);
 	}
 
@@ -1603,6 +1574,7 @@ int iwl_mvm_load_d3_fw(struct iwl_mvm *mvm)
 {
 	int ret, i;
 
+	lockdep_assert_wiphy(mvm->hw->wiphy);
 	lockdep_assert_held(&mvm->mutex);
 
 	ret = iwl_trans_start_hw(mvm->trans);
diff --git a/sys/contrib/dev/iwlwifi/mvm/led.c b/sys/contrib/dev/iwlwifi/mvm/led.c
index 1ea7c44250d4..c3cc1ea3ccc9 100644
--- a/sys/contrib/dev/iwlwifi/mvm/led.c
+++ b/sys/contrib/dev/iwlwifi/mvm/led.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2019 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2025 Intel Corporation
  * Copyright (C) 2017 Intel Deutschland GmbH
  */
 #include <linux/leds.h>
@@ -102,7 +102,7 @@ void iwl_mvm_leds_sync(struct iwl_mvm *mvm)
 	 * if we control through the register, we're doing it
 	 * even when the firmware isn't up, so no need to sync
 	 */
-	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
+	if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_8000)
 		return;
 
 	iwl_mvm_led_set(mvm, mvm->led.brightness > 0);
diff --git a/sys/contrib/dev/iwlwifi/mvm/link.c b/sys/contrib/dev/iwlwifi/mvm/link.c
index a9929aa49913..2269acc55c0e 100644
--- a/sys/contrib/dev/iwlwifi/mvm/link.c
+++ b/sys/contrib/dev/iwlwifi/mvm/link.c
@@ -12,6 +12,7 @@
 	HOW(BLOCKED_FW)			\
 	HOW(BLOCKED_NON_BSS)		\
 	HOW(BLOCKED_ROC)		\
+	HOW(BLOCKED_TMP_NON_BSS)	\
 	HOW(EXIT_MISSED_BEACON)		\
 	HOW(EXIT_LOW_RSSI)		\
 	HOW(EXIT_COEX)			\
@@ -48,20 +49,6 @@ static void iwl_mvm_print_esr_state(struct iwl_mvm *mvm, u32 mask)
 #undef NAME_PR
 }
 
-static u32 iwl_mvm_get_free_fw_link_id(struct iwl_mvm *mvm,
-				       struct iwl_mvm_vif *mvm_vif)
-{
-	u32 i;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	for (i = 0; i < ARRAY_SIZE(mvm->link_id_to_link_conf); i++)
-		if (!rcu_access_pointer(mvm->link_id_to_link_conf[i]))
-			return i;
-
-	return IWL_MVM_FW_LINK_ID_INVALID;
-}
-
 static int iwl_mvm_link_cmd_send(struct iwl_mvm *mvm,
 				 struct iwl_link_config_cmd *cmd,
 				 enum iwl_ctxt_action action)
@@ -78,25 +65,15 @@ static int iwl_mvm_link_cmd_send(struct iwl_mvm *mvm,
 	return ret;
 }
 
-int iwl_mvm_set_link_mapping(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			     struct ieee80211_bss_conf *link_conf)
+void iwl_mvm_set_link_fw_id(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			    struct ieee80211_bss_conf *link_conf)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_vif_link_info *link_info =
 		mvmvif->link[link_conf->link_id];
 
-	if (link_info->fw_link_id == IWL_MVM_FW_LINK_ID_INVALID) {
-		link_info->fw_link_id = iwl_mvm_get_free_fw_link_id(mvm,
-								    mvmvif);
-		if (link_info->fw_link_id >=
-		    ARRAY_SIZE(mvm->link_id_to_link_conf))
-			return -EINVAL;
-
-		rcu_assign_pointer(mvm->link_id_to_link_conf[link_info->fw_link_id],
-				   link_conf);
-	}
-
-	return 0;
+	if (link_info->fw_link_id == IWL_MVM_FW_LINK_ID_INVALID)
+		link_info->fw_link_id = mvmvif->id;
 }
 
 int iwl_mvm_add_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
@@ -108,14 +85,11 @@ int iwl_mvm_add_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	struct iwl_link_config_cmd cmd = {};
 	unsigned int cmd_id = WIDE_ID(MAC_CONF_GROUP, LINK_CONFIG_CMD);
 	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 1);
-	int ret;
 
 	if (WARN_ON_ONCE(!link_info))
 		return -EINVAL;
 
-	ret = iwl_mvm_set_link_mapping(mvm, vif, link_conf);
-	if (ret)
-		return ret;
+	iwl_mvm_set_link_fw_id(mvm, vif, link_conf);
 
 	/* Update SF - Disable if needed. if this fails, SF might still be on
 	 * while many macs are bound, which is forbidden - so fail the binding.
@@ -233,10 +207,15 @@ int iwl_mvm_link_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		WARN_ON_ONCE(active == link_info->active);
 
 		/* When deactivating a link session protection should
-		 * be stopped
+		 * be stopped. Also let the firmware know if we can't Tx.
 		 */
-		if (!active && vif->type == NL80211_IFTYPE_STATION)
+		if (!active && vif->type == NL80211_IFTYPE_STATION) {
 			iwl_mvm_stop_session_protection(mvm, vif);
+			if (link_info->csa_block_tx) {
+				cmd.block_tx = 1;
+				link_info->csa_block_tx = false;
+			}
+		}
 	}
 
 	cmd.link_id = cpu_to_le32(link_info->fw_link_id);
@@ -258,7 +237,7 @@ int iwl_mvm_link_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (vif->type == NL80211_IFTYPE_ADHOC && link_conf->bssid)
 		memcpy(cmd.ibss_bssid_addr, link_conf->bssid, ETH_ALEN);
 
-	iwl_mvm_set_fw_basic_rates(mvm, vif, link_conf,
+	iwl_mvm_set_fw_basic_rates(mvm, vif, link_info,
 				   &cmd.cck_rates, &cmd.ofdm_rates);
 
 	cmd.cck_short_preamble = cpu_to_le32(link_conf->use_short_preamble);
@@ -293,6 +272,17 @@ int iwl_mvm_link_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			(link_conf->uora_ocw_range >> 3) & 0x7;
 	}
 
+	/* ap_sta may be NULL if we're disconnecting */
+	if (changes & LINK_CONTEXT_MODIFY_HE_PARAMS && mvmvif->ap_sta) {
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_check(mvmvif->ap_sta, link_id);
+
+		if (!WARN_ON(!link_sta) && link_sta->he_cap.has_he &&
+		    link_sta->he_cap.he_cap_elem.mac_cap_info[5] &
+		    IEEE80211_HE_MAC_CAP5_OM_CTRL_UL_MU_DATA_DIS_RX)
+			cmd.ul_mu_data_disable = 1;
+	}
+
 	/* TODO  how to set ndp_fdbk_buff_th_exp? */
 
 	if (iwl_mvm_set_fw_mu_edca_params(mvm, mvmvif->link[link_id],
@@ -343,7 +333,8 @@ int iwl_mvm_link_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 send_cmd:
 	cmd.modify_mask = cpu_to_le32(changes);
 	cmd.flags = cpu_to_le32(flags);
-	cmd.flags_mask = cpu_to_le32(flags_mask);
+	if (cmd_ver < 6)
+		cmd.flags_mask = cpu_to_le32(flags_mask);
 	cmd.spec_link_id = link_conf->link_id;
 	if (cmd_ver < 2)
 		cmd.listen_lmac = cpu_to_le32(link_info->listen_lmac);
@@ -355,24 +346,6 @@ send_cmd:
 	return ret;
 }
 
-int iwl_mvm_unset_link_mapping(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			       struct ieee80211_bss_conf *link_conf)
-{
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm_vif_link_info *link_info =
-		mvmvif->link[link_conf->link_id];
-
-	/* mac80211 thought we have the link, but it was never configured */
-	if (WARN_ON(!link_info ||
-		    link_info->fw_link_id >=
-		    ARRAY_SIZE(mvm->link_id_to_link_conf)))
-		return -EINVAL;
-
-	RCU_INIT_POINTER(mvm->link_id_to_link_conf[link_info->fw_link_id],
-			 NULL);
-	return 0;
-}
-
 int iwl_mvm_remove_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			struct ieee80211_bss_conf *link_conf)
 {
@@ -382,10 +355,6 @@ int iwl_mvm_remove_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	struct iwl_link_config_cmd cmd = {};
 	int ret;
 
-	ret = iwl_mvm_unset_link_mapping(mvm, vif, link_conf);
-	if (ret)
-		return 0;
-
 	cmd.link_id = cpu_to_le32(link_info->fw_link_id);
 	link_info->fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
 	cmd.spec_link_id = link_conf->link_id;
@@ -393,9 +362,8 @@ int iwl_mvm_remove_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	ret = iwl_mvm_link_cmd_send(mvm, &cmd, FW_CTXT_ACTION_REMOVE);
 
-	if (!ret)
-		if (iwl_mvm_sf_update(mvm, vif, true))
-			IWL_ERR(mvm, "Failed to update SF state\n");
+	if (!ret && iwl_mvm_sf_update(mvm, vif, true))
+		IWL_ERR(mvm, "Failed to update SF state\n");
 
 	return ret;
 }
@@ -743,9 +711,8 @@ bool iwl_mvm_mld_valid_link_pair(struct ieee80211_vif *vif,
 	    iwl_mvm_esr_disallowed_with_link(mvm, vif, b, false))
 		return false;
 
-	if (a->chandef->width != b->chandef->width ||
-	    !(a->chandef->chan->band == NL80211_BAND_6GHZ &&
-	      b->chandef->chan->band == NL80211_BAND_5GHZ))
+	if (a->chandef->chan->band == b->chandef->chan->band ||
+	    a->chandef->width != b->chandef->width)
 		ret |= IWL_MVM_ESR_EXIT_BANDWIDTH;
 
 	if (ret) {
@@ -1148,3 +1115,14 @@ void iwl_mvm_unblock_esr(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (!mvmvif->esr_disable_reason)
 		iwl_mvm_esr_unblocked(mvm, vif);
 }
+
+void iwl_mvm_init_link(struct iwl_mvm_vif_link_info *link)
+{
+	link->bcast_sta.sta_id = IWL_INVALID_STA;
+	link->mcast_sta.sta_id = IWL_INVALID_STA;
+	link->ap_sta_id = IWL_INVALID_STA;
+
+	for (int r = 0; r < NUM_IWL_MVM_SMPS_REQ; r++)
+		link->smps_requests[r] =
+			IEEE80211_SMPS_AUTOMATIC;
+}
diff --git a/sys/contrib/dev/iwlwifi/mvm/mac-ctxt.c b/sys/contrib/dev/iwlwifi/mvm/mac-ctxt.c
index 24bfdb054a63..a3ff2f083aef 100644
--- a/sys/contrib/dev/iwlwifi/mvm/mac-ctxt.c
+++ b/sys/contrib/dev/iwlwifi/mvm/mac-ctxt.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -12,6 +12,7 @@
 #include "fw-api.h"
 #include "mvm.h"
 #include "time-event.h"
+#include "iwl-utils.h"
 
 const u8 iwl_mvm_ac_to_tx_fifo[] = {
 	IWL_MVM_TX_FIFO_VO,
@@ -216,7 +217,7 @@ int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		.preferred_tsf = NUM_TSF_IDS,
 		.found_vif = false,
 	};
-	int ret, i;
+	int ret;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -298,11 +299,9 @@ int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	mvmvif->time_event_data.id = TE_MAX;
 	mvmvif->roc_activity = ROC_NUM_ACTIVITIES;
 
-	mvmvif->deflink.bcast_sta.sta_id = IWL_MVM_INVALID_STA;
-	mvmvif->deflink.mcast_sta.sta_id = IWL_MVM_INVALID_STA;
-	mvmvif->deflink.ap_sta_id = IWL_MVM_INVALID_STA;
+	iwl_mvm_init_link(&mvmvif->deflink);
 
-	/* No need to allocate data queues to P2P Device MAC and NAN.*/
+	/* No need to allocate data queues to P2P Device MAC */
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
 		return 0;
 
@@ -316,9 +315,6 @@ int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		mvmvif->deflink.cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
 	}
 
-	for (i = 0; i < NUM_IWL_MVM_SMPS_REQ; i++)
-		mvmvif->deflink.smps_requests[i] = IEEE80211_SMPS_AUTOMATIC;
-
 	return 0;
 
 exit_fail:
@@ -413,19 +409,18 @@ static void iwl_mvm_ack_rates(struct iwl_mvm *mvm,
 }
 
 void iwl_mvm_set_fw_basic_rates(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-				struct ieee80211_bss_conf *link_conf,
+				struct iwl_mvm_vif_link_info *link_info,
 				__le32 *cck_rates, __le32 *ofdm_rates)
 {
-	struct ieee80211_chanctx_conf *chanctx;
+	struct iwl_mvm_phy_ctxt *phy_ctxt;
 	u8 cck_ack_rates = 0, ofdm_ack_rates = 0;
+	enum nl80211_band band = NL80211_BAND_2GHZ;
 
-	rcu_read_lock();
-	chanctx = rcu_dereference(link_conf->chanctx_conf);
-	iwl_mvm_ack_rates(mvm, vif, chanctx ? chanctx->def.chan->band
-					    : NL80211_BAND_2GHZ,
-			  &cck_ack_rates, &ofdm_ack_rates);
+	phy_ctxt = link_info->phy_ctxt;
+	if (phy_ctxt && phy_ctxt->channel)
+		band = phy_ctxt->channel->band;
 
-	rcu_read_unlock();
+	iwl_mvm_ack_rates(mvm, vif, band, &cck_ack_rates, &ofdm_ack_rates);
 
 	*cck_rates = cpu_to_le32((u32)cck_ack_rates);
 	*ofdm_rates = cpu_to_le32((u32)ofdm_ack_rates);
@@ -563,7 +558,7 @@ static void iwl_mvm_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 	else
 		eth_broadcast_addr(cmd->bssid_addr);
 
-	iwl_mvm_set_fw_basic_rates(mvm, vif, &vif->bss_conf, &cmd->cck_rates,
+	iwl_mvm_set_fw_basic_rates(mvm, vif, &mvmvif->deflink, &cmd->cck_rates,
 				   &cmd->ofdm_rates);
 
 	cmd->cck_short_preamble =
@@ -874,23 +869,6 @@ void iwl_mvm_mac_ctxt_set_tim(struct iwl_mvm *mvm,
 	}
 }
 
-u32 iwl_mvm_find_ie_offset(u8 *beacon, u8 eid, u32 frame_size)
-{
-	struct ieee80211_mgmt *mgmt = (void *)beacon;
-	const u8 *ie;
-
-	if (WARN_ON_ONCE(frame_size <= (mgmt->u.beacon.variable - beacon)))
-		return 0;
-
-	frame_size -= mgmt->u.beacon.variable - beacon;
-
-	ie = cfg80211_find_ie(eid, mgmt->u.beacon.variable, frame_size);
-	if (!ie)
-		return 0;
-
-	return ie - beacon;
-}
-
 u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
 				    struct ieee80211_tx_info *info,
 				    struct ieee80211_vif *vif)
@@ -960,12 +938,19 @@ u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
 
 u16 iwl_mvm_mac_ctxt_get_beacon_flags(const struct iwl_fw *fw, u8 rate_idx)
 {
-	u16 flags = iwl_mvm_mac80211_idx_to_hwrate(fw, rate_idx);
 	bool is_new_rate = iwl_fw_lookup_cmd_ver(fw, BEACON_TEMPLATE_CMD, 0) > 10;
+	u16 flags, cck_flag;
+
+	if (is_new_rate) {
+		flags = iwl_mvm_mac80211_idx_to_hwrate(fw, rate_idx);
+		cck_flag = IWL_MAC_BEACON_CCK;
+	} else {
+		cck_flag = IWL_MAC_BEACON_CCK_V1;
+		flags = iwl_fw_rate_idx_to_plcp(rate_idx);
+	}
 
-	if (rate_idx <= IWL_FIRST_CCK_RATE)
-		flags |= is_new_rate ? IWL_MAC_BEACON_CCK
-			  : IWL_MAC_BEACON_CCK_V1;
+	if (rate_idx <= IWL_LAST_CCK_RATE)
+		flags |= cck_flag;
 
 	return flags;
 }
@@ -991,7 +976,7 @@ u8 iwl_mvm_mac_ctxt_get_beacon_rate(struct iwl_mvm *mvm,
 static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
 				    struct ieee80211_vif *vif,
 				    struct sk_buff *beacon,
-				    struct iwl_tx_cmd *tx)
+				    struct iwl_tx_cmd_v6_params *tx_params)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct ieee80211_tx_info *info;
@@ -1001,30 +986,30 @@ static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
 	info = IEEE80211_SKB_CB(beacon);
 
 	/* Set up TX command fields */
-	tx->len = cpu_to_le16((u16)beacon->len);
-	tx->sta_id = mvmvif->deflink.bcast_sta.sta_id;
-	tx->life_time = cpu_to_le32(TX_CMD_LIFE_TIME_INFINITE);
+	tx_params->len = cpu_to_le16((u16)beacon->len);
+	tx_params->sta_id = mvmvif->deflink.bcast_sta.sta_id;
+	tx_params->life_time = cpu_to_le32(TX_CMD_LIFE_TIME_INFINITE);
 	tx_flags = TX_CMD_FLG_SEQ_CTL | TX_CMD_FLG_TSF;
 	tx_flags |=
 		iwl_mvm_bt_coex_tx_prio(mvm, (void *)beacon->data, info, 0) <<
 						TX_CMD_FLG_BT_PRIO_POS;
-	tx->tx_flags = cpu_to_le32(tx_flags);
+	tx_params->tx_flags = cpu_to_le32(tx_flags);
 
 	if (!fw_has_capa(&mvm->fw->ucode_capa,
 			 IWL_UCODE_TLV_CAPA_BEACON_ANT_SELECTION)) {
 		iwl_mvm_toggle_tx_ant(mvm, &mvm->mgmt_last_antenna_idx);
 
-		tx->rate_n_flags =
+		tx_params->rate_n_flags =
 			cpu_to_le32(BIT(mvm->mgmt_last_antenna_idx) <<
 				    RATE_MCS_ANT_POS);
 	}
 
 	rate = iwl_mvm_mac_ctxt_get_beacon_rate(mvm, info, vif);
 
-	tx->rate_n_flags |=
+	tx_params->rate_n_flags |=
 		cpu_to_le32(iwl_mvm_mac80211_idx_to_hwrate(mvm->fw, rate));
 	if (rate == IWL_FIRST_CCK_RATE)
-		tx->rate_n_flags |= cpu_to_le32(RATE_MCS_CCK_MSK_V1);
+		tx_params->rate_n_flags |= cpu_to_le32(RATE_MCS_CCK_MSK_V1);
 
 }
 
@@ -1084,22 +1069,23 @@ static int iwl_mvm_mac_ctxt_send_beacon_v7(struct iwl_mvm *mvm,
 					 beacon->data, beacon->len);
 
 	beacon_cmd.csa_offset =
-		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
-						   WLAN_EID_CHANNEL_SWITCH,
-						   beacon->len));
+		cpu_to_le32(iwl_find_ie_offset(beacon->data,
+					       WLAN_EID_CHANNEL_SWITCH,
+					       beacon->len));
 	beacon_cmd.ecsa_offset =
-		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
-						   WLAN_EID_EXT_CHANSWITCH_ANN,
-						   beacon->len));
+		cpu_to_le32(iwl_find_ie_offset(beacon->data,
+					       WLAN_EID_EXT_CHANSWITCH_ANN,
+					       beacon->len));
 
 	return iwl_mvm_mac_ctxt_send_beacon_cmd(mvm, beacon, &beacon_cmd,
 						sizeof(beacon_cmd));
 }
 
 bool iwl_mvm_enable_fils(struct iwl_mvm *mvm,
+			 struct ieee80211_vif *vif,
 			 struct ieee80211_chanctx_conf *ctx)
 {
-	if (IWL_MVM_DISABLE_AP_FILS)
+	if (vif->type != NL80211_IFTYPE_AP || IWL_MVM_DISABLE_AP_FILS)
 		return false;
 
 	if (cfg80211_channel_is_psc(ctx->def.chan))
@@ -1128,7 +1114,7 @@ static int iwl_mvm_mac_ctxt_send_beacon_v9(struct iwl_mvm *mvm,
 	ctx = rcu_dereference(link_conf->chanctx_conf);
 	channel = ieee80211_frequency_to_channel(ctx->def.chan->center_freq);
 	WARN_ON(channel == 0);
-	if (iwl_mvm_enable_fils(mvm, ctx)) {
+	if (iwl_mvm_enable_fils(mvm, vif, ctx)) {
 		flags |= iwl_fw_lookup_cmd_ver(mvm->fw, BEACON_TEMPLATE_CMD,
 					       0) > 10 ?
 			IWL_MAC_BEACON_FILS :
@@ -1157,20 +1143,20 @@ static int iwl_mvm_mac_ctxt_send_beacon_v9(struct iwl_mvm *mvm,
 					 beacon->data, beacon->len);
 
 	beacon_cmd.csa_offset =
-		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
-						   WLAN_EID_CHANNEL_SWITCH,
-						   beacon->len));
+		cpu_to_le32(iwl_find_ie_offset(beacon->data,
+					       WLAN_EID_CHANNEL_SWITCH,
+					       beacon->len));
 	beacon_cmd.ecsa_offset =
-		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
-						   WLAN_EID_EXT_CHANSWITCH_ANN,
-						   beacon->len));
+		cpu_to_le32(iwl_find_ie_offset(beacon->data,
+					       WLAN_EID_EXT_CHANSWITCH_ANN,
+					       beacon->len));
 
 	if (vif->type == NL80211_IFTYPE_AP &&
 	    iwl_fw_lookup_cmd_ver(mvm->fw, BEACON_TEMPLATE_CMD, 0) >= 14)
 		beacon_cmd.btwt_offset =
-			cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
-							   WLAN_EID_S1G_TWT,
-							   beacon->len));
+			cpu_to_le32(iwl_find_ie_offset(beacon->data,
+						       WLAN_EID_S1G_TWT,
+						       beacon->len));
 
 	return iwl_mvm_mac_ctxt_send_beacon_cmd(mvm, beacon, &beacon_cmd,
 						sizeof(beacon_cmd));
@@ -1528,7 +1514,7 @@ void iwl_mvm_rx_beacon_notif(struct iwl_mvm *mvm,
 	mvm->ap_last_beacon_gp2 = le32_to_cpu(beacon->gp2);
 
 	if (!iwl_mvm_is_short_beacon_notif_supported(mvm)) {
-		struct iwl_mvm_tx_resp *beacon_notify_hdr =
+		struct iwl_tx_resp *beacon_notify_hdr =
 			&beacon_v5->beacon_notify_hdr;
 
 		if (unlikely(pkt_len < sizeof(*beacon_v5)))
@@ -1586,11 +1572,11 @@ void iwl_mvm_rx_beacon_notif(struct iwl_mvm *mvm,
 	}
 }
 
-void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
-				     struct iwl_rx_cmd_buffer *rxb)
+static void
+iwl_mvm_handle_missed_beacons_notif(struct iwl_mvm *mvm,
+				    const struct iwl_missed_beacons_notif *mb,
+				    struct iwl_rx_packet *pkt)
 {
-	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_missed_beacons_notif *mb = (void *)pkt->data;
 	struct iwl_fw_dbg_trigger_missed_bcon *bcon_trig;
 	struct iwl_fw_dbg_trigger_tlv *trigger;
 	u32 stop_trig_missed_bcon, stop_trig_missed_bcon_since_rx;
@@ -1600,37 +1586,39 @@ void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
 	u32 id = le32_to_cpu(mb->link_id);
 	union iwl_dbg_tlv_tp_data tp_data = { .fw_pkt = pkt };
 	u32 mac_type;
-	int link_id = -1;
+	int link_id;
 	u8 notif_ver = iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
 					       MISSED_BEACONS_NOTIFICATION,
 					       0);
-
-	/* before version four the ID in the notification refers to mac ID */
-	if (notif_ver < 4) {
-		vif = iwl_mvm_rcu_dereference_vif_id(mvm, id, false);
-	} else {
-		struct ieee80211_bss_conf *bss_conf =
-			iwl_mvm_rcu_fw_link_id_to_link_conf(mvm, id, false);
-
-		if (!bss_conf)
-			return;
-
-		vif = bss_conf->vif;
-		link_id = bss_conf->link_id;
-	}
+	u8 new_notif_ver = iwl_fw_lookup_notif_ver(mvm->fw, MAC_CONF_GROUP,
+						   MISSED_BEACONS_NOTIF, 0);
+	struct ieee80211_bss_conf *bss_conf;
+
+	/* If the firmware uses the new notification (from MAC_CONF_GROUP),
+	 * refer to that notification's version.
+	 * Note that the new notification from MAC_CONF_GROUP starts from
+	 * version 5.
+	 */
+	if (new_notif_ver)
+		notif_ver = new_notif_ver;
 
 	IWL_DEBUG_INFO(mvm,
-		       "missed bcn %s_id=%u, consecutive=%u (%u, %u, %u)\n",
+		       "missed bcn %s_id=%u, consecutive=%u (%u)\n",
 		       notif_ver < 4 ? "mac" : "link",
 		       id,
 		       le32_to_cpu(mb->consec_missed_beacons),
-		       le32_to_cpu(mb->consec_missed_beacons_since_last_rx),
-		       le32_to_cpu(mb->num_recvd_beacons),
-		       le32_to_cpu(mb->num_expected_beacons));
+		       le32_to_cpu(mb->consec_missed_beacons_since_last_rx));
 
+	/*
+	 * starting from version 4 the ID is link ID, but driver
+	 * uses link ID == MAC ID, so always treat as MAC ID
+	 */
+	vif = iwl_mvm_rcu_dereference_vif_id(mvm, id, false);
 	if (!vif)
 		return;
 
+	bss_conf = &vif->bss_conf;
+	link_id = bss_conf->link_id;
 	mac_type = iwl_mvm_get_mac_type(vif);
 
 	IWL_DEBUG_INFO(mvm, "missed beacon mac_type=%u,\n", mac_type);
@@ -1656,15 +1644,41 @@ void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
 				 "missed_beacons:%d, missed_beacons_since_rx:%d\n",
 				 rx_missed_bcon, rx_missed_bcon_since_rx);
 		}
-	} else if (rx_missed_bcon >= IWL_MVM_MISSED_BEACONS_EXIT_ESR_THRESH &&
-		   link_id >= 0 && hweight16(vif->active_links) > 1) {
-		iwl_mvm_exit_esr(mvm, vif, IWL_MVM_ESR_EXIT_MISSED_BEACON,
-				 iwl_mvm_get_other_link(vif, link_id));
+	} else if (link_id >= 0 && hweight16(vif->active_links) > 1) {
+		u32 bss_param_ch_cnt_link_id =
+			bss_conf->bss_param_ch_cnt_link_id;
+		u32 scnd_lnk_bcn_lost = 0;
+
+		if (notif_ver >= 5 &&
+		    !IWL_FW_CHECK(mvm,
+				  le32_to_cpu(mb->other_link_id) == IWL_MVM_FW_LINK_ID_INVALID,
+				  "No data for other link id but we are in EMLSR active_links: 0x%x\n",
+				  vif->active_links))
+			scnd_lnk_bcn_lost =
+				le32_to_cpu(mb->consec_missed_beacons_other_link);
+
+		/* Exit EMLSR if we lost more than
+		 * IWL_MVM_MISSED_BEACONS_EXIT_ESR_THRESH beacons on boths links
+		 * OR more than IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH on any link.
+		 * OR more than IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED
+		 * and the link's bss_param_ch_count has changed.
+		 */
+		if ((rx_missed_bcon >= IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS &&
+		     scnd_lnk_bcn_lost >= IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS) ||
+		    rx_missed_bcon >= IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH ||
+		    (bss_param_ch_cnt_link_id != link_id &&
+		     rx_missed_bcon >= IWL_MVM_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED))
+			iwl_mvm_exit_esr(mvm, vif,
+					 IWL_MVM_ESR_EXIT_MISSED_BEACON,
+					 iwl_mvm_get_primary_link(vif));
 	} else if (rx_missed_bcon_since_rx > IWL_MVM_MISSED_BEACONS_THRESHOLD) {
 		if (!iwl_mvm_has_new_tx_api(mvm))
 			ieee80211_beacon_loss(vif);
 		else
 			ieee80211_cqm_beacon_loss_notify(vif, GFP_ATOMIC);
+
+		/* try to switch links, no-op if we don't have MLO */
+		iwl_mvm_int_mlo_scan(mvm, vif);
 	}
 
 	iwl_dbg_tlv_time_point(&mvm->fwrt,
@@ -1691,6 +1705,31 @@ void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
 #endif
 }
 
+void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
+				     struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+
+	iwl_mvm_handle_missed_beacons_notif(mvm, (const void *)pkt->data, pkt);
+}
+
+void iwl_mvm_rx_missed_beacons_notif_legacy(struct iwl_mvm *mvm,
+					    struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	const struct iwl_missed_beacons_notif_v4 *mb_v4 =
+		(const void *)pkt->data;
+	struct iwl_missed_beacons_notif mb = {
+		.link_id = mb_v4->link_id,
+		.consec_missed_beacons = mb_v4->consec_missed_beacons,
+		.consec_missed_beacons_since_last_rx =
+			mb_v4->consec_missed_beacons_since_last_rx,
+		.other_link_id = cpu_to_le32(IWL_MVM_FW_LINK_ID_INVALID),
+	};
+
+	iwl_mvm_handle_missed_beacons_notif(mvm, &mb, pkt);
+}
+
 void iwl_mvm_rx_stored_beacon_notif(struct iwl_mvm *mvm,
 				    struct iwl_rx_cmd_buffer *rxb)
 {
@@ -1717,7 +1756,7 @@ void iwl_mvm_rx_stored_beacon_notif(struct iwl_mvm *mvm,
 
 		data = sb_v2->data;
 	} else {
-		struct iwl_stored_beacon_notif_v3 *sb_v3 = (void *)pkt->data;
+		struct iwl_stored_beacon_notif *sb_v3 = (void *)pkt->data;
 
 		if (pkt_len < struct_size(sb_v3, data, size))
 			return;
@@ -1833,16 +1872,15 @@ void iwl_mvm_channel_switch_start_notif(struct iwl_mvm *mvm,
 	} else {
 		struct iwl_channel_switch_start_notif *notif = (void *)pkt->data;
 		u32 link_id = le32_to_cpu(notif->link_id);
-		struct ieee80211_bss_conf *bss_conf =
-			iwl_mvm_rcu_fw_link_id_to_link_conf(mvm, link_id, true);
 
-		if (!bss_conf)
+		/* we use link ID == MAC ID */
+		vif = iwl_mvm_rcu_dereference_vif_id(mvm, link_id, true);
+		if (!vif)
 			goto out_unlock;
 
 		id = link_id;
-		mac_link_id = bss_conf->link_id;
-		vif = bss_conf->vif;
-		csa_active = bss_conf->csa_active;
+		mac_link_id = vif->bss_conf.link_id;
+		csa_active = vif->bss_conf.csa_active;
 	}
 
 	mvmvif = iwl_mvm_vif_from_mac80211(vif);
@@ -1925,26 +1963,3 @@ void iwl_mvm_channel_switch_error_notif(struct iwl_mvm *mvm,
 		ieee80211_channel_switch_disconnect(vif);
 	rcu_read_unlock();
 }
-
-void iwl_mvm_rx_missed_vap_notif(struct iwl_mvm *mvm,
-				 struct iwl_rx_cmd_buffer *rxb)
-{
-	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_missed_vap_notif *mb = (void *)pkt->data;
-	struct ieee80211_vif *vif;
-	u32 id = le32_to_cpu(mb->mac_id);
-
-	IWL_DEBUG_INFO(mvm,
-		       "missed_vap notify mac_id=%u, num_beacon_intervals_elapsed=%u, profile_periodicity=%u\n",
-		       le32_to_cpu(mb->mac_id),
-		       mb->num_beacon_intervals_elapsed,
-		       mb->profile_periodicity);
-
-	rcu_read_lock();
-
-	vif = iwl_mvm_rcu_dereference_vif_id(mvm, id, true);
-	if (vif)
-		iwl_mvm_connection_loss(mvm, vif, "missed vap beacon");
-
-	rcu_read_unlock();
-}
diff --git a/sys/contrib/dev/iwlwifi/mvm/mac80211.c b/sys/contrib/dev/iwlwifi/mvm/mac80211.c
index a2f455616fbe..f32398213ab8 100644
--- a/sys/contrib/dev/iwlwifi/mvm/mac80211.c
+++ b/sys/contrib/dev/iwlwifi/mvm/mac80211.c
@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
 #include <linux/kernel.h>
+#include <linux/fips.h>
 #include <linux/slab.h>
 #include <linux/skbuff.h>
 #include <linux/netdevice.h>
@@ -78,7 +79,7 @@ static const struct ieee80211_iface_combination iwl_mvm_iface_combinations[] = {
 };
 
 static const struct cfg80211_pmsr_capabilities iwl_mvm_pmsr_capa = {
-	.max_peers = IWL_MVM_TOF_MAX_APS,
+	.max_peers = IWL_TOF_MAX_APS,
 	.report_ap_tsf = 1,
 	.randomize_mac_addr = 1,
 
@@ -154,7 +155,7 @@ struct ieee80211_regdomain *iwl_mvm_get_regdomain(struct wiphy *wiphy,
 					   MCC_UPDATE_CMD, 0);
 	IWL_DEBUG_LAR(mvm, "MCC update response version: %d\n", resp_ver);
 
-	regd = iwl_parse_nvm_mcc_info(mvm->trans->dev, mvm->cfg,
+	regd = iwl_parse_nvm_mcc_info(mvm->trans,
 				      __le32_to_cpu(resp->n_channels),
 				      resp->channels,
 				      __le16_to_cpu(resp->mcc),
@@ -173,16 +174,6 @@ struct ieee80211_regdomain *iwl_mvm_get_regdomain(struct wiphy *wiphy,
 	mvm->lar_regdom_set = true;
 	mvm->mcc_src = src_id;
 
-	/* Some kind of regulatory mess means we need to currently disallow
-	 * puncturing in the US and Canada. Do that here, at least until we
-	 * figure out the new chanctx APIs for puncturing.
-	 */
-	if (resp->mcc == cpu_to_le16(IWL_MCC_US) ||
-	    resp->mcc == cpu_to_le16(IWL_MCC_CANADA))
-		ieee80211_hw_set(mvm->hw, DISALLOW_PUNCTURING);
-	else
-		__clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING, mvm->hw->flags);
-
 	iwl_mei_set_country_code(__le16_to_cpu(resp->mcc));
 
 out:
@@ -284,9 +275,10 @@ static const u8 tm_if_types_ext_capa_sta[] = {
 					__bf_shf(IEEE80211_EML_CAP_EMLSR_PADDING_DELAY) | \
 				 IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US << \
 					__bf_shf(IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY))
-#define IWL_MVM_MLD_CAPA_OPS FIELD_PREP_CONST( \
+#define IWL_MVM_MLD_CAPA_OPS (FIELD_PREP_CONST( \
 			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP, \
-			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME)
+			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME) | \
+			IEEE80211_MLD_CAP_OP_LINK_RECONF_SUPPORT)
 
 static const struct wiphy_iftype_ext_capab add_iftypes_ext_capa[] = {
 	{
@@ -309,7 +301,8 @@ static const struct wiphy_iftype_ext_capab add_iftypes_ext_capa[] = {
 	},
 };
 
-int iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant)
+int iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, int radio_idx,
+			   u32 *tx_ant, u32 *rx_ant)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	*tx_ant = iwl_mvm_get_valid_tx_ant(mvm);
@@ -317,13 +310,15 @@ int iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant)
 	return 0;
 }
 
-int iwl_mvm_op_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
+int iwl_mvm_op_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant,
+			   u32 rx_ant)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
 	/* This has been tested on those devices only */
-	if (mvm->trans->trans_cfg->device_family != IWL_DEVICE_FAMILY_9000 &&
-	    mvm->trans->trans_cfg->device_family != IWL_DEVICE_FAMILY_22000)
+	if (mvm->trans->mac_cfg->device_family != IWL_DEVICE_FAMILY_9000 &&
+	    mvm->trans->mac_cfg->device_family != IWL_DEVICE_FAMILY_22000 &&
+	    mvm->trans->mac_cfg->device_family != IWL_DEVICE_FAMILY_AX210)
 		return -EOPNOTSUPP;
 
 	if (!mvm->nvm_data)
@@ -402,7 +397,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	 * for older devices. We also don't see this issue on any newer
 	 * devices.
 	 */
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_9000)
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_9000)
 		ieee80211_hw_set(hw, TX_AMSDU);
 	ieee80211_hw_set(hw, TX_FRAG_LIST);
 
@@ -413,7 +408,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 
 	/* We want to use the mac80211's reorder buffer for 9000 */
 	if (iwl_mvm_has_new_rx_api(mvm) &&
-	    mvm->trans->trans_cfg->device_family > IWL_DEVICE_FAMILY_9000)
+	    mvm->trans->mac_cfg->device_family > IWL_DEVICE_FAMILY_9000)
 		ieee80211_hw_set(hw, SUPPORTS_REORDERING_BUFFER);
 
 	if (fw_has_capa(&mvm->fw->ucode_capa,
@@ -428,10 +423,10 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		return -EINVAL;
 	}
 
-	if (mvm->trans->num_rx_queues > 1)
+	if (mvm->trans->info.num_rxqs > 1)
 		ieee80211_hw_set(hw, USES_RSS);
 
-	if (mvm->trans->max_skb_frags)
+	if (mvm->trans->info.max_skb_frags)
 		hw->netdev_features = NETIF_F_HIGHDMA | NETIF_F_SG;
 
 	hw->queues = IEEE80211_NUM_ACS;
@@ -452,7 +447,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 
 	hw->uapsd_queues = IWL_MVM_UAPSD_QUEUES;
 	hw->uapsd_max_sp_len = IWL_UAPSD_MAX_SP;
-	hw->max_tx_fragments = mvm->trans->max_skb_frags;
+	hw->max_tx_fragments = mvm->trans->info.max_skb_frags;
 
 	BUILD_BUG_ON(ARRAY_SIZE(mvm->ciphers) < ARRAY_SIZE(mvm_ciphers) + 6);
 	memcpy(mvm->ciphers, mvm_ciphers, sizeof(mvm_ciphers));
@@ -475,7 +470,9 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		IWL_ERR(mvm,
 			"iwlmvm doesn't allow to disable BT Coex, check bt_coex_active module parameter\n");
 
-	ieee80211_hw_set(hw, MFP_CAPABLE);
+	if (!fips_enabled)
+		ieee80211_hw_set(hw, MFP_CAPABLE);
+
 	mvm->ciphers[hw->wiphy->n_cipher_suites] = WLAN_CIPHER_SUITE_AES_CMAC;
 	hw->wiphy->n_cipher_suites++;
 	if (iwl_mvm_has_new_rx_api(mvm)) {
@@ -499,12 +496,17 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		hw->wiphy->pmsr_capa = &iwl_mvm_pmsr_capa;
 	}
 
-	if (sec_key_ver &&
+	/*
+	 * beacon protection must be handled by firmware,
+	 * so cannot be done with fips_enabled
+	 */
+	if (!fips_enabled && sec_key_ver &&
 	    fw_has_capa(&mvm->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_BIGTK_TX_SUPPORT))
 		wiphy_ext_feature_set(hw->wiphy,
 				      NL80211_EXT_FEATURE_BEACON_PROTECTION);
-	else if (fw_has_capa(&mvm->fw->ucode_capa,
+	else if (!fips_enabled &&
+		 fw_has_capa(&mvm->fw->ucode_capa,
 			     IWL_UCODE_TLV_CAPA_BIGTK_SUPPORT))
 		wiphy_ext_feature_set(hw->wiphy,
 				      NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT);
@@ -555,7 +557,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		hw->wiphy->regulatory_flags |= REGULATORY_CUSTOM_REG |
 					       REGULATORY_DISABLE_BEACON_HINTS;
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		wiphy_ext_feature_set(hw->wiphy,
 				      NL80211_EXT_FEATURE_DFS_CONCURRENT);
 
@@ -621,7 +623,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		hw->wiphy->bands[NL80211_BAND_6GHZ] =
 			&mvm->nvm_data->bands[NL80211_BAND_6GHZ];
 
-	hw->wiphy->hw_version = mvm->trans->hw_id;
+	hw->wiphy->hw_version = mvm->trans->info.hw_id;
 
 	if (iwlmvm_mod_params.power_scheme != IWL_POWER_SCHEME_CAM)
 		hw->wiphy->flags |= WIPHY_FLAG_PS_ON_BY_DEFAULT;
@@ -647,10 +649,15 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 			       NL80211_FEATURE_LOW_PRIORITY_SCAN |
 			       NL80211_FEATURE_P2P_GO_OPPPS |
 			       NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE |
-			       NL80211_FEATURE_DYNAMIC_SMPS |
-			       NL80211_FEATURE_STATIC_SMPS |
 			       NL80211_FEATURE_SUPPORTS_WMM_ADMISSION;
 
+	/* when firmware supports RLC/SMPS offload, do not set these
+	 * driver features, since it's no longer supported by driver.
+	 */
+	if (!iwl_mvm_has_rlc_offload(mvm))
+		hw->wiphy->features |= NL80211_FEATURE_STATIC_SMPS |
+				       NL80211_FEATURE_DYNAMIC_SMPS;
+
 	if (fw_has_capa(&mvm->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_TXPOWER_INSERTION_SUPPORT))
 		hw->wiphy->features |= NL80211_FEATURE_TX_POWER_INSERTION;
@@ -737,11 +744,9 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 
 	mvm->rts_threshold = IEEE80211_MAX_RTS_THRESHOLD;
 
-	ieee80211_hw_set(hw, DISALLOW_PUNCTURING_5GHZ);
-
 #ifdef CONFIG_PM_SLEEP
 	if ((unified || mvm->fw->img[IWL_UCODE_WOWLAN].num_sec) &&
-	    device_can_wakeup(mvm->trans->dev)) {
+	    device_can_wakeup(mvm->trans->dev) && !fips_enabled) {
 		mvm->wowlan.flags |= WIPHY_WOWLAN_MAGIC_PKT |
 				     WIPHY_WOWLAN_DISCONNECT |
 				     WIPHY_WOWLAN_EAP_IDENTITY_REQ |
@@ -777,7 +782,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		hw->wiphy->features |= NL80211_FEATURE_TDLS_CHANNEL_SWITCH;
 	}
 
-	hw->netdev_features |= mvm->cfg->features;
+	hw->netdev_features |= mvm->trans->mac_cfg->base->features;
 	if (!iwl_mvm_is_csum_supported(mvm))
 		hw->netdev_features &= ~IWL_CSUM_NETIF_FLAGS_MASK;
 
@@ -846,20 +851,10 @@ void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
 	if (ieee80211_is_mgmt(hdr->frame_control))
 		sta = NULL;
 
-	/* If there is no sta, and it's not offchannel - send through AP */
+	/* this shouldn't even happen: just drop */
 	if (!sta && info->control.vif->type == NL80211_IFTYPE_STATION &&
-	    !offchannel) {
-		struct iwl_mvm_vif *mvmvif =
-			iwl_mvm_vif_from_mac80211(info->control.vif);
-		u8 ap_sta_id = READ_ONCE(mvmvif->deflink.ap_sta_id);
-
-		if (ap_sta_id < mvm->fw->ucode_capa.num_stations) {
-			/* mac80211 holds rcu read lock */
-			sta = rcu_dereference(mvm->fw_id_to_mac_id[ap_sta_id]);
-			if (IS_ERR_OR_NULL(sta))
-				goto drop;
-		}
-	}
+	    !offchannel)
+		goto drop;
 
 	if (tmp_sta && !sta && link_id != IEEE80211_LINK_UNSPECIFIED &&
 	    !ieee80211_is_probe_resp(hdr->frame_control)) {
@@ -1126,7 +1121,7 @@ static void iwl_mvm_cleanup_iterator(void *data, u8 *mac,
 	vif->driver_flags &= ~IEEE80211_VIF_EML_ACTIVE;
 
 	for_each_mvm_vif_valid_link(mvmvif, link_id) {
-		mvmvif->link[link_id]->ap_sta_id = IWL_MVM_INVALID_STA;
+		mvmvif->link[link_id]->ap_sta_id = IWL_INVALID_STA;
 		mvmvif->link[link_id]->fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
 		mvmvif->link[link_id]->phy_ctxt = NULL;
 		mvmvif->link[link_id]->active = 0;
@@ -1170,7 +1165,7 @@ static void iwl_mvm_cleanup_sta_iterator(void *data, struct ieee80211_sta *sta)
 			 * Delete the stale data to avoid issues later on.
 			 */
 			iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_link_sta,
-						  link_id, false);
+						  link_id);
 		}
 	}
 }
@@ -1249,11 +1244,12 @@ int __iwl_mvm_mac_start(struct iwl_mvm *mvm)
 		mvm->nvm_data = NULL;
 	}
 
-#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
 	/* fast_resume will be cleared by iwl_mvm_fast_resume */
 	fast_resume = mvm->fast_resume;
 
 	if (fast_resume) {
+		iwl_mvm_mei_device_state(mvm, true);
 		ret = iwl_mvm_fast_resume(mvm);
 		if (ret) {
 			iwl_mvm_stop_device(mvm);
@@ -1271,7 +1267,7 @@ int __iwl_mvm_mac_start(struct iwl_mvm *mvm)
 			set_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
 		}
 	}
-#endif /* CONFIG_PM */
+#endif /* CONFIG_PM_SLEEP */
 
 	if (test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status)) {
 		/*
@@ -1310,21 +1306,22 @@ int iwl_mvm_mac_start(struct ieee80211_hw *hw)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
+	int retry, max_retry = 0;
 
 	mutex_lock(&mvm->mutex);
 
 	/* we are starting the mac not in error flow, and restart is enabled */
 	if (!test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status) &&
-	    iwlwifi_mod_params.fw_restart) {
-		/*
-		 * This will prevent mac80211 recovery flows to trigger during
-		 * init failures
-		 */
-		set_bit(IWL_MVM_STATUS_STARTING, &mvm->status);
-	}
+	    iwlwifi_mod_params.fw_restart)
+		max_retry = IWL_MAX_INIT_RETRY;
+
+	for (retry = 0; retry <= max_retry; retry++) {
+		ret = __iwl_mvm_mac_start(mvm);
+		if (ret != -ETIMEDOUT)
+			break;
 
-	ret = __iwl_mvm_mac_start(mvm);
-	clear_bit(IWL_MVM_STATUS_STARTING, &mvm->status);
+		IWL_ERR(mvm, "mac start retry %d\n", retry);
+	}
 
 	mutex_unlock(&mvm->mutex);
 
@@ -1353,6 +1350,13 @@ static void iwl_mvm_restart_complete(struct iwl_mvm *mvm)
 	 * of packets the FW sent out, so we must reconnect.
 	 */
 	iwl_mvm_teardown_tdls_peers(mvm);
+
+	IWL_INFO(mvm, "restart completed\n");
+	iwl_trans_finish_sw_reset(mvm->trans);
+
+	/* no need to lock, adding in parallel would schedule too */
+	if (!list_empty(&mvm->add_stream_txqs))
+		schedule_work(&mvm->add_stream_wk);
 }
 
 void iwl_mvm_mac_reconfig_complete(struct ieee80211_hw *hw,
@@ -1386,10 +1390,13 @@ void __iwl_mvm_mac_stop(struct iwl_mvm *mvm, bool suspend)
 		iwl_mvm_rm_aux_sta(mvm);
 
 	if (suspend &&
-	    mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	    mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
 		iwl_mvm_fast_suspend(mvm);
-	else
+		/* From this point on, we won't touch the device */
+		iwl_mvm_mei_device_state(mvm, false);
+	} else {
 		iwl_mvm_stop_device(mvm);
+	}
 
 	iwl_mvm_async_handlers_purge(mvm);
 	/* async_handlers_list is empty and will stay empty: HW is stopped */
@@ -1483,29 +1490,47 @@ struct iwl_mvm_phy_ctxt *iwl_mvm_get_free_phy_ctxt(struct iwl_mvm *mvm)
 	return NULL;
 }
 
-int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+int iwl_mvm_set_tx_power(struct iwl_mvm *mvm,
+			 struct ieee80211_bss_conf *link_conf,
 			 s16 tx_power)
 {
 	u32 cmd_id = REDUCE_TX_POWER_CMD;
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(link_conf->vif);
+	u32 mac_id = mvmvif->id;
 	int len;
-	struct iwl_dev_tx_power_cmd cmd = {
-		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_MAC),
-		.common.mac_context_id =
-			cpu_to_le32(iwl_mvm_vif_from_mac80211(vif)->id),
-		.common.pwr_restriction = cpu_to_le16(8 * tx_power),
+	struct iwl_dev_tx_power_cmd_v3_v8 cmd = {
+		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_LINK),
+		.common.link_id = cpu_to_le32(mac_id),
 	};
-	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id,
-					   IWL_FW_CMD_VER_UNKNOWN);
+	struct iwl_dev_tx_power_cmd cmd_v9_v10;
+	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 3);
+	u16 u_tx_power = tx_power == IWL_DEFAULT_MAX_TX_POWER ?
+		IWL_DEV_MAX_TX_POWER : 8 * tx_power;
+	void *cmd_data = &cmd;
+
+	cmd.common.pwr_restriction = cpu_to_le16(u_tx_power);
+
+	if (cmd_ver > 8) {
+		u32 link_id;
+
+		if (WARN_ON(!mvmvif->link[link_conf->link_id]))
+			return -ENODEV;
 
-	if (tx_power == IWL_DEFAULT_MAX_TX_POWER)
-		cmd.common.pwr_restriction = cpu_to_le16(IWL_DEV_MAX_TX_POWER);
+		link_id = mvmvif->link[link_conf->link_id]->fw_link_id;
 
-	if (cmd_ver == 8)
+		/* Those fields sit on the same place for v9 and v10 */
+		cmd_v9_v10.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_LINK);
+		cmd_v9_v10.common.link_id = cpu_to_le32(link_id);
+		cmd_v9_v10.common.pwr_restriction = cpu_to_le16(u_tx_power);
+		cmd_data = &cmd_v9_v10;
+	}
+
+	if (cmd_ver == 10)
+		len = sizeof(cmd_v9_v10.v10);
+	else if (cmd_ver == 9)
+		len = sizeof(cmd_v9_v10.v9);
+	else if (cmd_ver == 8)
 		len = sizeof(cmd.v8);
-	else if (cmd_ver == 7)
-		len = sizeof(cmd.v7);
-	else if (cmd_ver == 6)
-		len = sizeof(cmd.v6);
 	else if (fw_has_api(&mvm->fw->ucode_capa,
 			    IWL_UCODE_TLV_API_REDUCE_TX_POWER))
 		len = sizeof(cmd.v5);
@@ -1515,10 +1540,14 @@ int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	else
 		len = sizeof(cmd.v3);
 
-	/* all structs have the same common part, add it */
+	/* all structs have the same common part, add its length */
 	len += sizeof(cmd.common);
 
-	return iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, len, &cmd);
+	if (cmd_ver < 9)
+		len += sizeof(cmd.per_band);
+
+	return iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, len, cmd_data);
+
 }
 
 static void iwl_mvm_post_csa_tx(void *data, struct ieee80211_sta *sta)
@@ -1730,6 +1759,21 @@ static void iwl_mvm_unblock_esr_tpt(struct wiphy *wiphy, struct wiphy_work *wk)
 	iwl_mvm_unblock_esr(mvm, vif, IWL_MVM_ESR_BLOCKED_TPT);
 }
 
+static void iwl_mvm_unblock_esr_tmp_non_bss(struct wiphy *wiphy,
+					    struct wiphy_work *wk)
+{
+	struct iwl_mvm_vif *mvmvif =
+		container_of(wk, struct iwl_mvm_vif,
+			     unblock_esr_tmp_non_bss_wk.work);
+	struct iwl_mvm *mvm = mvmvif->mvm;
+	struct ieee80211_vif *vif =
+		container_of((void *)mvmvif, struct ieee80211_vif, drv_priv);
+
+	mutex_lock(&mvm->mutex);
+	iwl_mvm_unblock_esr(mvm, vif, IWL_MVM_ESR_BLOCKED_TMP_NON_BSS);
+	mutex_unlock(&mvm->mutex);
+}
+
 void iwl_mvm_mac_init_mvmvif(struct iwl_mvm *mvm, struct iwl_mvm_vif *mvmvif)
 {
 	lockdep_assert_held(&mvm->mutex);
@@ -1737,6 +1781,8 @@ void iwl_mvm_mac_init_mvmvif(struct iwl_mvm *mvm, struct iwl_mvm_vif *mvmvif)
 	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
 		return;
 
+	mvmvif->deflink.average_beacon_energy = 0;
+
 	INIT_DELAYED_WORK(&mvmvif->csa_work,
 			  iwl_mvm_channel_switch_disconnect_wk);
 
@@ -1748,6 +1794,9 @@ void iwl_mvm_mac_init_mvmvif(struct iwl_mvm *mvm, struct iwl_mvm_vif *mvmvif)
 
 	wiphy_work_init(&mvmvif->unblock_esr_tpt_wk,
 			iwl_mvm_unblock_esr_tpt);
+
+	wiphy_delayed_work_init(&mvmvif->unblock_esr_tmp_non_bss_wk,
+				iwl_mvm_unblock_esr_tmp_non_bss);
 }
 
 static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
@@ -1769,9 +1818,9 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 	mvmvif->deflink.active = 0;
 	mvmvif->link[0] = &mvmvif->deflink;
 
-	ret = iwl_mvm_set_link_mapping(mvm, vif, &vif->bss_conf);
-	if (ret)
-		goto out;
+	vif->driver_flags = IEEE80211_VIF_REMOVE_AP_AFTER_DISASSOC;
+
+	iwl_mvm_set_link_fw_id(mvm, vif, &vif->bss_conf);
 
 	/*
 	 * Not much to do here. The stack will not allow interface
@@ -1792,12 +1841,6 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 
 	rcu_assign_pointer(mvm->vif_id_to_mac[mvmvif->id], vif);
 
-	/* Currently not much to do for NAN */
-	if (vif->type == NL80211_IFTYPE_NAN) {
-		ret = 0;
-		goto out;
-	}
-
 	/*
 	 * The AP binding flow can be done only after the beacon
 	 * template is configured (which happens only in the mac80211
@@ -1898,6 +1941,8 @@ void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
 				  &mvmvif->mlo_int_scan_wk);
 
 	wiphy_work_cancel(mvm->hw->wiphy, &mvmvif->unblock_esr_tpt_wk);
+	wiphy_delayed_work_cancel(mvm->hw->wiphy,
+				  &mvmvif->unblock_esr_tmp_non_bss_wk);
 
 	cancel_delayed_work_sync(&mvmvif->csa_work);
 }
@@ -1944,15 +1989,8 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 	 * interface is be handled as part of the stop_ap flow.
 	 */
 	if (vif->type == NL80211_IFTYPE_AP ||
-	    vif->type == NL80211_IFTYPE_ADHOC) {
-#ifdef CONFIG_NL80211_TESTMODE
-		if (vif == mvm->noa_vif) {
-			mvm->noa_vif = NULL;
-			mvm->noa_duration = 0;
-		}
-#endif
+	    vif->type == NL80211_IFTYPE_ADHOC)
 		goto out;
-	}
 
 	iwl_mvm_power_update_mac(mvm);
 
@@ -1969,7 +2007,6 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 		mvm->p2p_device_vif = NULL;
 	}
 
-	iwl_mvm_unset_link_mapping(mvm, vif, &vif->bss_conf);
 	iwl_mvm_mac_ctxt_remove(mvm, vif);
 
 	RCU_INIT_POINTER(mvm->vif_id_to_mac[mvmvif->id], NULL);
@@ -2937,7 +2974,7 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 						    IWL_MVM_SMPS_REQ_PROT,
 						    IEEE80211_SMPS_DYNAMIC, 0);
 			}
-		} else if (mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA) {
+		} else if (mvmvif->deflink.ap_sta_id != IWL_INVALID_STA) {
 			iwl_mvm_mei_host_disassociated(mvm);
 			/*
 			 * If update fails - SF might be running in associated
@@ -2949,33 +2986,6 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 					    &mvm->status),
 				  "Failed to update SF upon disassociation\n");
 
-			/*
-			 * If we get an assert during the connection (after the
-			 * station has been added, but before the vif is set
-			 * to associated), mac80211 will re-add the station and
-			 * then configure the vif. Since the vif is not
-			 * associated, we would remove the station here and
-			 * this would fail the recovery.
-			 */
-			if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART,
-				      &mvm->status)) {
-				/* first remove remaining keys */
-				iwl_mvm_sec_key_remove_ap(mvm, vif,
-							  &mvmvif->deflink, 0);
-
-				/*
-				 * Remove AP station now that
-				 * the MAC is unassoc
-				 */
-				ret = iwl_mvm_rm_sta_id(mvm, vif,
-							mvmvif->deflink.ap_sta_id);
-				if (ret)
-					IWL_ERR(mvm,
-						"failed to remove AP station\n");
-
-				mvmvif->deflink.ap_sta_id = IWL_MVM_INVALID_STA;
-			}
-
 			/* remove quota for this interface */
 			ret = iwl_mvm_update_quotas(mvm, false, NULL);
 			if (ret)
@@ -3059,7 +3069,7 @@ static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
 	 * context. For the newer, the beacon is a resource that belongs to a
 	 * MAC, so need to send beacon template after adding the mac.
 	 */
-	if (mvm->trans->trans_cfg->device_family > IWL_DEVICE_FAMILY_22000) {
+	if (mvm->trans->mac_cfg->device_family > IWL_DEVICE_FAMILY_22000) {
 		/* Add the mac context */
 		ret = iwl_mvm_mac_ctxt_add(mvm, vif);
 		if (ret)
@@ -3319,7 +3329,7 @@ static void iwl_mvm_bss_info_changed(struct ieee80211_hw *hw,
 	if (changes & BSS_CHANGED_TXPOWER) {
 		IWL_DEBUG_CALIB(mvm, "Changing TX Power to %d dBm\n",
 				bss_conf->txpower);
-		iwl_mvm_set_tx_power(mvm, vif, bss_conf->txpower);
+		iwl_mvm_set_tx_power(mvm, bss_conf, bss_conf->txpower);
 	}
 }
 
@@ -3347,7 +3357,7 @@ void iwl_mvm_mac_cancel_hw_scan(struct ieee80211_hw *hw,
 	 * us to stop a hw_scan when it's already stopped.  This can
 	 * happen, for instance, if we stopped the scan ourselves,
 	 * called ieee80211_scan_completed() and the userspace called
-	 * cancel scan scan before ieee80211_scan_work() could run.
+	 * cancel scan before ieee80211_scan_work() could run.
 	 * To handle that, simply return if the scan is not running.
 	*/
 	if (mvm->scan_status & IWL_MVM_SCAN_REGULAR)
@@ -3430,7 +3440,7 @@ static void __iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw,
 		 */
 		break;
 	case STA_NOTIFY_AWAKE:
-		if (WARN_ON(mvmsta->deflink.sta_id == IWL_MVM_INVALID_STA))
+		if (WARN_ON(mvmsta->deflink.sta_id == IWL_INVALID_STA))
 			break;
 
 		if (txqs)
@@ -3510,6 +3520,8 @@ void iwl_mvm_sta_pre_rcu_remove(struct ieee80211_hw *hw,
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	unsigned int link_id;
 
+	lockdep_assert_wiphy(mvm->hw->wiphy);
+
 	/*
 	 * This is called before mac80211 does RCU synchronisation,
 	 * so here we already invalidate our internal RCU-protected
@@ -3893,7 +3905,7 @@ iwl_mvm_sta_state_notexist_to_none(struct iwl_mvm *mvm,
 
 	if (sta->tdls &&
 	    (vif->p2p ||
-	     iwl_mvm_tdls_sta_count(mvm, NULL) == IWL_MVM_TDLS_STA_COUNT ||
+	     iwl_mvm_tdls_sta_count(mvm, NULL) == IWL_TDLS_STA_COUNT ||
 	     iwl_mvm_phy_ctx_count(mvm) > 1)) {
 		IWL_DEBUG_MAC80211(mvm, "refusing TDLS sta\n");
 		return -EBUSY;
@@ -4091,15 +4103,28 @@ iwl_mvm_sta_state_authorized_to_assoc(struct iwl_mvm *mvm,
 					  &mvmvif->mlo_int_scan_wk);
 
 		wiphy_work_cancel(mvm->hw->wiphy, &mvmvif->unblock_esr_tpt_wk);
-
-		/* No need for the periodic statistics anymore */
-		if (ieee80211_vif_is_mld(vif) && mvmvif->esr_active)
-			iwl_mvm_request_periodic_system_statistics(mvm, false);
+		wiphy_delayed_work_cancel(mvm->hw->wiphy,
+					  &mvmvif->unblock_esr_tmp_non_bss_wk);
 	}
 
 	return 0;
 }
 
+void iwl_mvm_smps_workaround(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     bool update)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+	if (!iwl_mvm_has_rlc_offload(mvm) ||
+	    iwl_fw_lookup_cmd_ver(mvm->fw, MAC_PM_POWER_TABLE, 0) >= 2)
+		return;
+
+	mvmvif->ps_disabled = !vif->cfg.ps;
+
+	if (update)
+		iwl_mvm_power_update_mac(mvm);
+}
+
 /* Common part for MLD and non-MLD modes */
 int iwl_mvm_mac_sta_state_common(struct ieee80211_hw *hw,
 				 struct ieee80211_vif *vif,
@@ -4192,6 +4217,7 @@ int iwl_mvm_mac_sta_state_common(struct ieee80211_hw *hw,
 		   new_state == IEEE80211_STA_AUTHORIZED) {
 		ret = iwl_mvm_sta_state_assoc_to_authorized(mvm, vif, sta,
 							    callbacks);
+		iwl_mvm_smps_workaround(mvm, vif, true);
 	} else if (old_state == IEEE80211_STA_AUTHORIZED &&
 		   new_state == IEEE80211_STA_ASSOC) {
 		ret = iwl_mvm_sta_state_authorized_to_assoc(mvm, vif, sta,
@@ -4242,7 +4268,8 @@ int iwl_mvm_mac_sta_state_common(struct ieee80211_hw *hw,
 	return ret;
 }
 
-int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
+int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				  u32 value)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -4252,8 +4279,9 @@ int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
 }
 
 void iwl_mvm_sta_rc_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-			   struct ieee80211_sta *sta, u32 changed)
+			   struct ieee80211_link_sta *link_sta, u32 changed)
 {
+	struct ieee80211_sta *sta = link_sta->sta;
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
 	if (changed & (IEEE80211_RC_BW_CHANGED |
@@ -4342,7 +4370,7 @@ int iwl_mvm_mac_sched_scan_stop(struct ieee80211_hw *hw,
 	 * us to stop a sched_scan when it's already stopped.  This
 	 * can happen, for instance, if we stopped the scan ourselves,
 	 * called ieee80211_sched_scan_stopped() and the userspace called
-	 * stop sched scan scan before ieee80211_sched_scan_stopped_work()
+	 * stop sched scan before ieee80211_sched_scan_stopped_work()
 	 * could run.  To handle this, simply return if the scan is
 	 * not running.
 	*/
@@ -4379,7 +4407,7 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 
 	switch (key->cipher) {
 	case WLAN_CIPHER_SUITE_TKIP:
-		if (!mvm->trans->trans_cfg->gen2) {
+		if (!mvm->trans->mac_cfg->gen2) {
 			key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIC;
 			key->flags |= IEEE80211_KEY_FLAG_PUT_IV_SPACE;
 		} else if (vif->type == NL80211_IFTYPE_STATION) {
@@ -4496,7 +4524,7 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 
 			WARN_ON(rcu_access_pointer(mvmsta->ptk_pn[keyidx]));
 			ptk_pn = kzalloc(struct_size(ptk_pn, q,
-						     mvm->trans->num_rx_queues),
+						     mvm->trans->info.num_rxqs),
 					 GFP_KERNEL);
 			if (!ptk_pn) {
 				ret = -ENOMEM;
@@ -4505,7 +4533,7 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 
 			for (tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
 				ieee80211_get_key_rx_seq(key, tid, &seq);
-				for (q = 0; q < mvm->trans->num_rx_queues; q++)
+				for (q = 0; q < mvm->trans->info.num_rxqs; q++)
 					memcpy(ptk_pn->q[q].pn[tid],
 					       seq.ccmp.pn,
 					       IEEE80211_CCMP_PN_LEN);
@@ -4606,6 +4634,10 @@ int iwl_mvm_mac_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
+	/* When resuming from wowlan, FW already knows about the newest keys */
+	if (test_bit(IWL_MVM_STATUS_IN_D3, &mvm->status))
+		return 0;
+
 	guard(mvm)(mvm);
 	return __iwl_mvm_mac_set_key(hw, cmd, vif, sta, key);
 }
@@ -4981,34 +5013,46 @@ int iwl_mvm_cancel_roc(struct ieee80211_hw *hw,
 	return 0;
 }
 
-struct iwl_mvm_ftm_responder_iter_data {
-	bool responder;
+struct iwl_mvm_chanctx_usage_data {
+	struct iwl_mvm *mvm;
 	struct ieee80211_chanctx_conf *ctx;
+	bool use_def;
 };
 
-static void iwl_mvm_ftm_responder_chanctx_iter(void *_data, u8 *mac,
-					       struct ieee80211_vif *vif)
+static void iwl_mvm_chanctx_usage_iter(void *_data, u8 *mac,
+				       struct ieee80211_vif *vif)
 {
-	struct iwl_mvm_ftm_responder_iter_data *data = _data;
+	struct iwl_mvm_chanctx_usage_data *data = _data;
+	struct ieee80211_bss_conf *link_conf;
+	int link_id;
+
+	for_each_vif_active_link(vif, link_conf, link_id) {
+		if (rcu_access_pointer(link_conf->chanctx_conf) != data->ctx)
+			continue;
+
+		if (iwl_mvm_enable_fils(data->mvm, vif, data->ctx))
+			data->use_def = true;
 
-	if (rcu_access_pointer(vif->bss_conf.chanctx_conf) == data->ctx &&
-	    vif->type == NL80211_IFTYPE_AP && vif->bss_conf.ftmr_params)
-		data->responder = true;
+		if (vif->type == NL80211_IFTYPE_AP && link_conf->ftmr_params)
+			data->use_def = true;
+	}
 }
 
-bool iwl_mvm_is_ftm_responder_chanctx(struct iwl_mvm *mvm,
-				      struct ieee80211_chanctx_conf *ctx)
+struct cfg80211_chan_def *
+iwl_mvm_chanctx_def(struct iwl_mvm *mvm, struct ieee80211_chanctx_conf *ctx)
 {
-	struct iwl_mvm_ftm_responder_iter_data data = {
-		.responder = false,
+	struct iwl_mvm_chanctx_usage_data data = {
+		.mvm = mvm,
 		.ctx = ctx,
+		.use_def = false,
 	};
 
 	ieee80211_iterate_active_interfaces_atomic(mvm->hw,
-					IEEE80211_IFACE_ITER_NORMAL,
-					iwl_mvm_ftm_responder_chanctx_iter,
-					&data);
-	return data.responder;
+						   IEEE80211_IFACE_ITER_NORMAL,
+						   iwl_mvm_chanctx_usage_iter,
+						   &data);
+
+	return data.use_def ? &ctx->def : &ctx->min_def;
 }
 
 static int __iwl_mvm_add_chanctx(struct iwl_mvm *mvm,
@@ -5083,7 +5127,7 @@ void iwl_mvm_change_chanctx(struct ieee80211_hw *hw,
 		      (changed & ~(IEEE80211_CHANCTX_CHANGE_WIDTH |
 				   IEEE80211_CHANCTX_CHANGE_RX_CHAINS |
 				   IEEE80211_CHANCTX_CHANGE_RADAR |
-				   IEEE80211_CHANCTX_CHANGE_MIN_WIDTH)),
+				   IEEE80211_CHANCTX_CHANGE_MIN_DEF)),
 		      "Cannot change PHY. Ref=%d, changed=0x%X\n",
 		      phy_ctxt->ref, changed))
 		return;
@@ -5091,7 +5135,7 @@ void iwl_mvm_change_chanctx(struct ieee80211_hw *hw,
 	guard(mvm)(mvm);
 
 	/* we are only changing the min_width, may be a noop */
-	if (changed == IEEE80211_CHANCTX_CHANGE_MIN_WIDTH) {
+	if (changed == IEEE80211_CHANCTX_CHANGE_MIN_DEF) {
 		if (phy_ctxt->width == def->width)
 			return;
 
@@ -5392,7 +5436,7 @@ out_reassign:
 
 out_restart:
 	/* things keep failing, better restart the hw */
-	iwl_mvm_nic_restart(mvm, false);
+	iwl_force_nmi(mvm->trans);
 	return ret;
 }
 
@@ -5428,7 +5472,7 @@ out_reassign:
 
 out_restart:
 	/* things keep failing, better restart the hw */
-	iwl_mvm_nic_restart(mvm, false);
+	iwl_force_nmi(mvm->trans);
 	return ret;
 }
 
@@ -5497,70 +5541,6 @@ static int iwl_mvm_set_tim(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
 					       &mvm_sta->vif->bss_conf);
 }
 
-#ifdef CONFIG_NL80211_TESTMODE
-static const struct nla_policy iwl_mvm_tm_policy[IWL_MVM_TM_ATTR_MAX + 1] = {
-	[IWL_MVM_TM_ATTR_CMD] = { .type = NLA_U32 },
-	[IWL_MVM_TM_ATTR_NOA_DURATION] = { .type = NLA_U32 },
-	[IWL_MVM_TM_ATTR_BEACON_FILTER_STATE] = { .type = NLA_U32 },
-};
-
-static int __iwl_mvm_mac_testmode_cmd(struct iwl_mvm *mvm,
-				      struct ieee80211_vif *vif,
-				      void *data, int len)
-{
-	struct nlattr *tb[IWL_MVM_TM_ATTR_MAX + 1];
-	int err;
-	u32 noa_duration;
-
-	err = nla_parse_deprecated(tb, IWL_MVM_TM_ATTR_MAX, data, len,
-				   iwl_mvm_tm_policy, NULL);
-	if (err)
-		return err;
-
-	if (!tb[IWL_MVM_TM_ATTR_CMD])
-		return -EINVAL;
-
-	switch (nla_get_u32(tb[IWL_MVM_TM_ATTR_CMD])) {
-	case IWL_MVM_TM_CMD_SET_NOA:
-		if (!vif || vif->type != NL80211_IFTYPE_AP || !vif->p2p ||
-		    !vif->bss_conf.enable_beacon ||
-		    !tb[IWL_MVM_TM_ATTR_NOA_DURATION])
-			return -EINVAL;
-
-		noa_duration = nla_get_u32(tb[IWL_MVM_TM_ATTR_NOA_DURATION]);
-		if (noa_duration >= vif->bss_conf.beacon_int)
-			return -EINVAL;
-
-		mvm->noa_duration = noa_duration;
-		mvm->noa_vif = vif;
-
-		return iwl_mvm_update_quotas(mvm, true, NULL);
-	case IWL_MVM_TM_CMD_SET_BEACON_FILTER:
-		/* must be associated client vif - ignore authorized */
-		if (!vif || vif->type != NL80211_IFTYPE_STATION ||
-		    !vif->cfg.assoc || !vif->bss_conf.dtim_period ||
-		    !tb[IWL_MVM_TM_ATTR_BEACON_FILTER_STATE])
-			return -EINVAL;
-
-		if (nla_get_u32(tb[IWL_MVM_TM_ATTR_BEACON_FILTER_STATE]))
-			return iwl_mvm_enable_beacon_filter(mvm, vif);
-		return iwl_mvm_disable_beacon_filter(mvm, vif);
-	}
-
-	return -EOPNOTSUPP;
-}
-
-int iwl_mvm_mac_testmode_cmd(struct ieee80211_hw *hw,
-			     struct ieee80211_vif *vif,
-			     void *data, int len)
-{
-	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
-
-	guard(mvm)(mvm);
-	return __iwl_mvm_mac_testmode_cmd(mvm, vif, data, len);
-}
-#endif
-
 void iwl_mvm_channel_switch(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			    struct ieee80211_channel_switch *chsw)
 {
@@ -6112,12 +6092,12 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 		break;
 	}
 
-	if (format == RATE_MCS_CCK_MSK ||
-	    format == RATE_MCS_LEGACY_OFDM_MSK) {
+	if (format == RATE_MCS_MOD_TYPE_CCK ||
+	    format == RATE_MCS_MOD_TYPE_LEGACY_OFDM) {
 		int rate = u32_get_bits(rate_n_flags, RATE_LEGACY_RATE_MSK);
 
 		/* add the offset needed to get to the legacy ofdm indices */
-		if (format == RATE_MCS_LEGACY_OFDM_MSK)
+		if (format == RATE_MCS_MOD_TYPE_LEGACY_OFDM)
 			rate += IWL_FIRST_OFDM_RATE;
 
 		switch (rate) {
@@ -6162,7 +6142,7 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 
 	rinfo->nss = u32_get_bits(rate_n_flags,
 				  RATE_MCS_NSS_MSK) + 1;
-	rinfo->mcs = format == RATE_MCS_HT_MSK ?
+	rinfo->mcs = format == RATE_MCS_MOD_TYPE_HT ?
 		RATE_HT_MCS_INDEX(rate_n_flags) :
 		u32_get_bits(rate_n_flags, RATE_MCS_CODE_MSK);
 
@@ -6170,11 +6150,11 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 		rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
 
 	switch (format) {
-	case RATE_MCS_EHT_MSK:
+	case RATE_MCS_MOD_TYPE_EHT:
 		/* TODO: GI/LTF/RU. How does the firmware encode them? */
 		rinfo->flags |= RATE_INFO_FLAGS_EHT_MCS;
 		break;
-	case RATE_MCS_HE_MSK:
+	case RATE_MCS_MOD_TYPE_HE:
 		gi_ltf = u32_get_bits(rate_n_flags, RATE_MCS_HE_GI_LTF_MSK);
 
 		rinfo->flags |= RATE_INFO_FLAGS_HE_MCS;
@@ -6215,10 +6195,10 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 		if (rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK)
 			rinfo->he_dcm = 1;
 		break;
-	case RATE_MCS_HT_MSK:
+	case RATE_MCS_MOD_TYPE_HT:
 		rinfo->flags |= RATE_INFO_FLAGS_MCS;
 		break;
-	case RATE_MCS_VHT_MSK:
+	case RATE_MCS_MOD_TYPE_VHT:
 		rinfo->flags |= RATE_INFO_FLAGS_VHT_MCS;
 		break;
 	}
@@ -6255,7 +6235,7 @@ void iwl_mvm_mac_sta_statistics(struct ieee80211_hw *hw,
 
 	guard(mvm)(mvm);
 
-	if (mvmvif->deflink.ap_sta_id != mvmsta->deflink.sta_id)
+	if (sta != mvmvif->ap_sta)
 		return;
 
 	if (iwl_mvm_request_statistics(mvm, false))
@@ -6398,37 +6378,36 @@ void iwl_mvm_sync_rx_queues_internal(struct iwl_mvm *mvm,
 				     bool sync,
 				     const void *data, u32 size)
 {
-	struct {
-		struct iwl_rxq_sync_cmd cmd;
-		struct iwl_mvm_internal_rxq_notif notif;
-	} __packed cmd = {
-		.cmd.rxq_mask = cpu_to_le32(BIT(mvm->trans->num_rx_queues) - 1),
-		.cmd.count =
-			cpu_to_le32(sizeof(struct iwl_mvm_internal_rxq_notif) +
-				    size),
-		.notif.type = type,
-		.notif.sync = sync,
-	};
+	DEFINE_RAW_FLEX(struct iwl_rxq_sync_cmd, cmd, payload,
+			sizeof(struct iwl_mvm_internal_rxq_notif));
+	struct iwl_mvm_internal_rxq_notif *notif =
+			(struct iwl_mvm_internal_rxq_notif *)cmd->payload;
 	struct iwl_host_cmd hcmd = {
 		.id = WIDE_ID(DATA_PATH_GROUP, TRIGGER_RX_QUEUES_NOTIF_CMD),
-		.data[0] = &cmd,
-		.len[0] = sizeof(cmd),
+		.data[0] = cmd,
+		.len[0] = __struct_size(cmd),
 		.data[1] = data,
 		.len[1] = size,
 		.flags = CMD_SEND_IN_RFKILL | (sync ? 0 : CMD_ASYNC),
 	};
 	int ret;
 
+	cmd->rxq_mask = cpu_to_le32(BIT(mvm->trans->info.num_rxqs) - 1);
+	cmd->count = cpu_to_le32(sizeof(struct iwl_mvm_internal_rxq_notif) +
+				 size);
+	notif->type = type;
+	notif->sync = sync;
+
 	/* size must be a multiple of DWORD */
-	if (WARN_ON(cmd.cmd.count & cpu_to_le32(3)))
+	if (WARN_ON(cmd->count & cpu_to_le32(3)))
 		return;
 
 	if (!iwl_mvm_has_new_rx_api(mvm))
 		return;
 
 	if (sync) {
-		cmd.notif.cookie = mvm->queue_sync_cookie;
-		mvm->queue_sync_state = (1 << mvm->trans->num_rx_queues) - 1;
+		notif->cookie = mvm->queue_sync_cookie;
+		mvm->queue_sync_state = (1 << mvm->trans->info.num_rxqs) - 1;
 	}
 
 	ret = iwl_mvm_send_cmd(mvm, &hcmd);
@@ -6578,7 +6557,7 @@ const struct ieee80211_ops iwl_mvm_hw_ops = {
 	.allow_buffered_frames = iwl_mvm_mac_allow_buffered_frames,
 	.release_buffered_frames = iwl_mvm_mac_release_buffered_frames,
 	.set_rts_threshold = iwl_mvm_mac_set_rts_threshold,
-	.sta_rc_update = iwl_mvm_sta_rc_update,
+	.link_sta_rc_update = iwl_mvm_sta_rc_update,
 	.conf_tx = iwl_mvm_mac_conf_tx,
 	.mgd_prepare_tx = iwl_mvm_mac_mgd_prepare_tx,
 	.mgd_complete_tx = iwl_mvm_mac_mgd_complete_tx,
@@ -6621,8 +6600,6 @@ const struct ieee80211_ops iwl_mvm_hw_ops = {
 
 	.sync_rx_queues = iwl_mvm_sync_rx_queues,
 
-	CFG80211_TESTMODE_CMD(iwl_mvm_mac_testmode_cmd)
-
 #ifdef CONFIG_PM_SLEEP
 	/* look at d3.c */
 	.suspend = iwl_mvm_suspend,
diff --git a/sys/contrib/dev/iwlwifi/mvm/mld-key.c b/sys/contrib/dev/iwlwifi/mvm/mld-key.c
index 8a38fc4b0b0f..ef0be44207e1 100644
--- a/sys/contrib/dev/iwlwifi/mvm/mld-key.c
+++ b/sys/contrib/dev/iwlwifi/mvm/mld-key.c
@@ -144,7 +144,7 @@ static void iwl_mvm_mld_update_sta_key(struct ieee80211_hw *hw,
 	if (sta != data->sta || key->link_id >= 0)
 		return;
 
-	err = iwl_mvm_send_cmd_pdu(mvm, cmd_id, CMD_ASYNC, sizeof(cmd), &cmd);
+	err = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cmd), &cmd);
 
 	if (err)
 		data->err = err;
@@ -162,8 +162,8 @@ int iwl_mvm_mld_update_sta_keys(struct iwl_mvm *mvm,
 		.new_sta_mask = new_sta_mask,
 	};
 
-	ieee80211_iter_keys_rcu(mvm->hw, vif, iwl_mvm_mld_update_sta_key,
-				&data);
+	ieee80211_iter_keys(mvm->hw, vif, iwl_mvm_mld_update_sta_key,
+			    &data);
 	return data.err;
 }
 
@@ -396,13 +396,13 @@ void iwl_mvm_sec_key_remove_ap(struct iwl_mvm *mvm,
 	u8 sec_key_ver = iwl_fw_lookup_cmd_ver(mvm->fw, sec_key_id, 0);
 
 	if (WARN_ON_ONCE(vif->type != NL80211_IFTYPE_STATION ||
-			 link->ap_sta_id == IWL_MVM_INVALID_STA))
+			 link->ap_sta_id == IWL_INVALID_STA))
 		return;
 
 	if (!sec_key_ver)
 		return;
 
-	ieee80211_iter_keys_rcu(mvm->hw, vif,
-				iwl_mvm_sec_key_remove_ap_iter,
-				(void *)(uintptr_t)link_id);
+	ieee80211_iter_keys(mvm->hw, vif,
+			    iwl_mvm_sec_key_remove_ap_iter,
+			    (void *)(uintptr_t)link_id);
 }
diff --git a/sys/contrib/dev/iwlwifi/mvm/mld-mac.c b/sys/contrib/dev/iwlwifi/mvm/mld-mac.c
index bb7851042177..2d116a41913c 100644
--- a/sys/contrib/dev/iwlwifi/mvm/mld-mac.c
+++ b/sys/contrib/dev/iwlwifi/mvm/mld-mac.c
@@ -1,17 +1,25 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2022 - 2024 Intel Corporation
+ * Copyright (C) 2022 - 2025 Intel Corporation
  */
 #include "mvm.h"
 
 static void iwl_mvm_mld_set_he_support(struct iwl_mvm *mvm,
 				       struct ieee80211_vif *vif,
-				       struct iwl_mac_config_cmd *cmd)
+				       struct iwl_mac_config_cmd *cmd,
+				       int cmd_ver)
 {
-	if (vif->type == NL80211_IFTYPE_AP)
-		cmd->he_ap_support = cpu_to_le16(1);
-	else
-		cmd->he_support = cpu_to_le16(1);
+	if (vif->type == NL80211_IFTYPE_AP) {
+		if (cmd_ver == 2)
+			cmd->wifi_gen_v2.he_ap_support = cpu_to_le16(1);
+		else
+			cmd->wifi_gen.he_ap_support = 1;
+	} else {
+		if (cmd_ver == 2)
+			cmd->wifi_gen_v2.he_support = cpu_to_le16(1);
+		else
+			cmd->wifi_gen.he_support = 1;
+	}
 }
 
 static void iwl_mvm_mld_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
@@ -22,6 +30,12 @@ static void iwl_mvm_mld_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct ieee80211_bss_conf *link_conf;
 	unsigned int link_id;
+	int cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw,
+					    WIDE_ID(MAC_CONF_GROUP,
+						    MAC_CONFIG_CMD), 1);
+
+	if (WARN_ON(cmd_ver > 3))
+		return;
 
 	cmd->id_and_color = cpu_to_le32(mvmvif->id);
 	cmd->action = cpu_to_le32(action);
@@ -30,8 +44,8 @@ static void iwl_mvm_mld_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 
 	memcpy(cmd->local_mld_addr, vif->addr, ETH_ALEN);
 
-	cmd->he_support = 0;
-	cmd->eht_support = 0;
+	cmd->wifi_gen_v2.he_support = 0;
+	cmd->wifi_gen_v2.eht_support = 0;
 
 	/* should be set by specific context type handler */
 	cmd->filter_flags = 0;
@@ -51,8 +65,11 @@ static void iwl_mvm_mld_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 	 * and enable both when we have MLO.
 	 */
 	if (ieee80211_vif_is_mld(vif)) {
-		iwl_mvm_mld_set_he_support(mvm, vif, cmd);
-		cmd->eht_support = cpu_to_le32(1);
+		iwl_mvm_mld_set_he_support(mvm, vif, cmd, cmd_ver);
+		if (cmd_ver == 2)
+			cmd->wifi_gen_v2.eht_support = cpu_to_le32(1);
+		else
+			cmd->wifi_gen.eht_support = 1;
 		return;
 	}
 
@@ -63,16 +80,19 @@ static void iwl_mvm_mld_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 			continue;
 
 		if (link_conf->he_support)
-			iwl_mvm_mld_set_he_support(mvm, vif, cmd);
+			iwl_mvm_mld_set_he_support(mvm, vif, cmd, cmd_ver);
 
-		/* it's not reasonable to have EHT without HE and FW API doesn't
+		/* It's not reasonable to have EHT without HE and FW API doesn't
 		 * support it. Ignore EHT in this case.
 		 */
 		if (!link_conf->he_support && link_conf->eht_support)
 			continue;
 
 		if (link_conf->eht_support) {
-			cmd->eht_support = cpu_to_le32(1);
+			if (cmd_ver == 2)
+				cmd->wifi_gen_v2.eht_support = cpu_to_le32(1);
+			else
+				cmd->wifi_gen.eht_support = 1;
 			break;
 		}
 	}
@@ -262,9 +282,6 @@ int iwl_mvm_mld_mac_ctxt_add(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	int ret;
 
-	if (WARN_ON_ONCE(vif->type == NL80211_IFTYPE_NAN))
-		return -EOPNOTSUPP;
-
 	if (WARN_ONCE(mvmvif->uploaded, "Adding active MAC %pM/%d\n",
 		      vif->addr, ieee80211_vif_type_p2p(vif)))
 		return -EIO;
@@ -287,9 +304,6 @@ int iwl_mvm_mld_mac_ctxt_changed(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	if (WARN_ON_ONCE(vif->type == NL80211_IFTYPE_NAN))
-		return -EOPNOTSUPP;
-
 	if (WARN_ONCE(!mvmvif->uploaded, "Changing inactive MAC %pM/%d\n",
 		      vif->addr, ieee80211_vif_type_p2p(vif)))
 		return -EIO;
@@ -307,9 +321,6 @@ int iwl_mvm_mld_mac_ctxt_remove(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	};
 	int ret;
 
-	if (WARN_ON_ONCE(vif->type == NL80211_IFTYPE_NAN))
-		return -EOPNOTSUPP;
-
 	if (WARN_ONCE(!mvmvif->uploaded, "Removing inactive MAC %pM/%d\n",
 		      vif->addr, ieee80211_vif_type_p2p(vif)))
 		return -EIO;
diff --git a/sys/contrib/dev/iwlwifi/mvm/mld-mac80211.c b/sys/contrib/dev/iwlwifi/mvm/mld-mac80211.c
index 3c99396ad369..bf24f8cb673e 100644
--- a/sys/contrib/dev/iwlwifi/mvm/mld-mac80211.c
+++ b/sys/contrib/dev/iwlwifi/mvm/mld-mac80211.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2022-2024 Intel Corporation
+ * Copyright (C) 2022-2025 Intel Corporation
  */
 #include "mvm.h"
 
@@ -18,6 +18,8 @@ static int iwl_mvm_mld_mac_add_interface(struct ieee80211_hw *hw,
 
 	mvmvif->mvm = mvm;
 
+	vif->driver_flags |= IEEE80211_VIF_REMOVE_AP_AFTER_DISASSOC;
+
 	/* Not much to do here. The stack will not allow interface
 	 * types or combinations that we didn't advertise, so we
 	 * don't really have to check the types.
@@ -41,8 +43,6 @@ static int iwl_mvm_mld_mac_add_interface(struct ieee80211_hw *hw,
 	/* reset deflink MLO parameters */
 	mvmvif->deflink.fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
 	mvmvif->deflink.active = 0;
-	/* the first link always points to the default one */
-	mvmvif->link[0] = &mvmvif->deflink;
 
 	ret = iwl_mvm_mld_mac_ctxt_add(mvm, vif);
 	if (ret)
@@ -60,9 +60,19 @@ static int iwl_mvm_mld_mac_add_interface(struct ieee80211_hw *hw,
 				     IEEE80211_VIF_SUPPORTS_CQM_RSSI;
 	}
 
-	ret = iwl_mvm_add_link(mvm, vif, &vif->bss_conf);
-	if (ret)
-		goto out_free_bf;
+	/* We want link[0] to point to the default link, unless we have MLO and
+	 * in this case this will be modified later by .change_vif_links()
+	 * If we are in the restart flow with an MLD connection, we will wait
+	 * to .change_vif_links() to setup the links.
+	 */
+	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) ||
+	    !ieee80211_vif_is_mld(vif)) {
+		mvmvif->link[0] = &mvmvif->deflink;
+
+		ret = iwl_mvm_add_link(mvm, vif, &vif->bss_conf);
+		if (ret)
+			goto out_free_bf;
+	}
 
 	/* Save a pointer to p2p device vif, so it can later be used to
 	 * update the p2p device MAC when a GO is started/stopped
@@ -140,19 +150,6 @@ static void iwl_mvm_mld_mac_remove_interface(struct ieee80211_hw *hw,
 
 	iwl_mvm_vif_dbgfs_rm_link(mvm, vif);
 
-	/* For AP/GO interface, the tear down of the resources allocated to the
-	 * interface is be handled as part of the stop_ap flow.
-	 */
-	if (vif->type == NL80211_IFTYPE_AP ||
-	    vif->type == NL80211_IFTYPE_ADHOC) {
-#ifdef CONFIG_NL80211_TESTMODE
-		if (vif == mvm->noa_vif) {
-			mvm->noa_vif = NULL;
-			mvm->noa_duration = 0;
-		}
-#endif
-	}
-
 	iwl_mvm_power_update_mac(mvm);
 
 	/* Before the interface removal, mac80211 would cancel the ROC, and the
@@ -200,32 +197,6 @@ static unsigned int iwl_mvm_mld_count_active_links(struct iwl_mvm_vif *mvmvif)
 	return n_active;
 }
 
-static void iwl_mvm_restart_mpdu_count(struct iwl_mvm *mvm,
-				       struct iwl_mvm_vif *mvmvif)
-{
-	struct ieee80211_sta *ap_sta = mvmvif->ap_sta;
-	struct iwl_mvm_sta *mvmsta;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	if (!ap_sta)
-		return;
-
-	mvmsta = iwl_mvm_sta_from_mac80211(ap_sta);
-	if (!mvmsta->mpdu_counters)
-		return;
-
-	for (int q = 0; q < mvm->trans->num_rx_queues; q++) {
-		spin_lock_bh(&mvmsta->mpdu_counters[q].lock);
-		memset(mvmsta->mpdu_counters[q].per_link, 0,
-		       sizeof(mvmsta->mpdu_counters[q].per_link));
-		mvmsta->mpdu_counters[q].window_start = jiffies;
-		spin_unlock_bh(&mvmsta->mpdu_counters[q].lock);
-	}
-
-	IWL_DEBUG_STATS(mvm, "MPDU counters are cleared\n");
-}
-
 static int iwl_mvm_esr_mode_active(struct iwl_mvm *mvm,
 				   struct ieee80211_vif *vif)
 {
@@ -259,15 +230,8 @@ static int iwl_mvm_esr_mode_active(struct iwl_mvm *mvm,
 	else
 		mvmvif->primary_link = __ffs(vif->active_links);
 
-	/* Needed for tracking RSSI */
-	iwl_mvm_request_periodic_system_statistics(mvm, true);
-
-	/*
-	 * Restart the MPDU counters and the counting window, so when the
-	 * statistics arrive (which is where we look at the counters) we
-	 * will be at the end of the window.
-	 */
-	iwl_mvm_restart_mpdu_count(mvm, mvmvif);
+	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_ESR_LINK_UP,
+			       NULL);
 
 	return ret;
 }
@@ -312,7 +276,6 @@ __iwl_mvm_mld_assign_vif_chanctx(struct iwl_mvm *mvm,
 		ret = iwl_mvm_esr_mode_active(mvm, vif);
 		if (ret) {
 			IWL_ERR(mvm, "failed to activate ESR mode (%d)\n", ret);
-			iwl_mvm_request_periodic_system_statistics(mvm, false);
 			goto out;
 		}
 	}
@@ -328,23 +291,18 @@ __iwl_mvm_mld_assign_vif_chanctx(struct iwl_mvm *mvm,
 	if (ret)
 		goto out;
 
-	/* Initialize rate control for the AP station, since we might be
-	 * doing a link switch here - we cannot initialize it before since
-	 * this needs the phy context assigned (and in FW?), and we cannot
-	 * do it later because it needs to be initialized as soon as we're
-	 * able to TX on the link, i.e. when active.
+	/*
+	 * if link switching (link not active yet) we'll activate it in
+	 * firmware later on link-info change, which mac80211 guarantees
+	 * for link switch after the stations are set up
 	 */
-	if (mvmvif->ap_sta) {
-		struct ieee80211_link_sta *link_sta;
-
-		rcu_read_lock();
-		link_sta = rcu_dereference(mvmvif->ap_sta->link[link_id]);
-
-		if (!WARN_ON_ONCE(!link_sta))
-			iwl_mvm_rs_rate_init(mvm, vif, mvmvif->ap_sta,
-					     link_conf, link_sta,
-					     phy_ctxt->channel->band);
-		rcu_read_unlock();
+	if (ieee80211_vif_link_active(vif, link_conf->link_id)) {
+		ret = iwl_mvm_link_changed(mvm, vif, link_conf,
+					   LINK_CONTEXT_MODIFY_ACTIVE |
+					   LINK_CONTEXT_MODIFY_RATES_INFO,
+					   true);
+		if (ret)
+			goto out;
 	}
 
 	if (vif->type == NL80211_IFTYPE_STATION)
@@ -352,14 +310,6 @@ __iwl_mvm_mld_assign_vif_chanctx(struct iwl_mvm *mvm,
 							link_conf,
 							false);
 
-	/* then activate */
-	ret = iwl_mvm_link_changed(mvm, vif, link_conf,
-				   LINK_CONTEXT_MODIFY_ACTIVE |
-				   LINK_CONTEXT_MODIFY_RATES_INFO,
-				   true);
-	if (ret)
-		goto out;
-
 	/*
 	 * Power state must be updated before quotas,
 	 * otherwise fw will complain.
@@ -451,10 +401,8 @@ static int iwl_mvm_esr_mode_inactive(struct iwl_mvm *mvm,
 			break;
 	}
 
-	iwl_mvm_request_periodic_system_statistics(mvm, false);
-
-	/* Start a new counting window */
-	iwl_mvm_restart_mpdu_count(mvm, mvmvif);
+	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_ESR_LINK_DOWN,
+			       NULL);
 
 	return ret;
 }
@@ -769,6 +717,11 @@ iwl_mvm_mld_link_info_changed_station(struct iwl_mvm *mvm,
 	if (WARN_ON_ONCE(!mvmvif->link[link_conf->link_id]))
 		return;
 
+	/* not yet marked active in vif means during link switch */
+	if (!ieee80211_vif_link_active(vif, link_conf->link_id) &&
+	    vif->cfg.assoc && mvmvif->link[link_conf->link_id]->phy_ctxt)
+		link_changes |= LINK_CONTEXT_MODIFY_ACTIVE;
+
 	has_he = link_conf->he_support && !iwlwifi_mod_params.disable_11ax;
 	has_eht = link_conf->eht_support && !iwlwifi_mod_params.disable_11be;
 
@@ -818,37 +771,13 @@ static bool iwl_mvm_mld_vif_have_valid_ap_sta(struct iwl_mvm_vif *mvmvif)
 	int i;
 
 	for_each_mvm_vif_valid_link(mvmvif, i) {
-		if (mvmvif->link[i]->ap_sta_id != IWL_MVM_INVALID_STA)
+		if (mvmvif->link[i]->ap_sta_id != IWL_INVALID_STA)
 			return true;
 	}
 
 	return false;
 }
 
-static void iwl_mvm_mld_vif_delete_all_stas(struct iwl_mvm *mvm,
-					    struct ieee80211_vif *vif)
-{
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	int i, ret;
-
-	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
-		return;
-
-	for_each_mvm_vif_valid_link(mvmvif, i) {
-		struct iwl_mvm_vif_link_info *link = mvmvif->link[i];
-
-		if (!link)
-			continue;
-
-		iwl_mvm_sec_key_remove_ap(mvm, vif, link, i);
-		ret = iwl_mvm_mld_rm_sta_id(mvm, link->ap_sta_id);
-		if (ret)
-			IWL_ERR(mvm, "failed to remove AP station\n");
-
-		link->ap_sta_id = IWL_MVM_INVALID_STA;
-	}
-}
-
 static void iwl_mvm_mld_vif_cfg_changed_station(struct iwl_mvm *mvm,
 						struct ieee80211_vif *vif,
 						u64 changes)
@@ -875,8 +804,13 @@ static void iwl_mvm_mld_vif_cfg_changed_station(struct iwl_mvm *mvm,
 		if (vif->cfg.assoc) {
 			mvmvif->session_prot_connection_loss = false;
 
-			/* clear statistics to get clean beacon counter */
+			/*
+			 * Clear statistics to get clean beacon counter, and ask for
+			 * periodic statistics, as they are needed for link
+			 * selection and RX OMI decisions.
+			 */
 			iwl_mvm_request_statistics(mvm, true);
+			iwl_mvm_request_periodic_system_statistics(mvm, true);
 			iwl_mvm_sf_update(mvm, vif, false);
 			iwl_mvm_power_vif_assoc(mvm, vif);
 
@@ -924,6 +858,8 @@ static void iwl_mvm_mld_vif_cfg_changed_station(struct iwl_mvm *mvm,
 		} else if (iwl_mvm_mld_vif_have_valid_ap_sta(mvmvif)) {
 			iwl_mvm_mei_host_disassociated(mvm);
 
+			iwl_mvm_request_periodic_system_statistics(mvm, false);
+
 			/* If update fails - SF might be running in associated
 			 * mode while disassociated - which is forbidden.
 			 */
@@ -932,21 +868,13 @@ static void iwl_mvm_mld_vif_cfg_changed_station(struct iwl_mvm *mvm,
 				  !test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED,
 					    &mvm->status),
 				  "Failed to update SF upon disassociation\n");
-
-			/* If we get an assert during the connection (after the
-			 * station has been added, but before the vif is set
-			 * to associated), mac80211 will re-add the station and
-			 * then configure the vif. Since the vif is not
-			 * associated, we would remove the station here and
-			 * this would fail the recovery.
-			 */
-			iwl_mvm_mld_vif_delete_all_stas(mvm, vif);
 		}
 
 		iwl_mvm_bss_info_changed_station_assoc(mvm, vif, changes);
 	}
 
 	if (changes & BSS_CHANGED_PS) {
+		iwl_mvm_smps_workaround(mvm, vif, false);
 		ret = iwl_mvm_power_update_mac(mvm);
 		if (ret)
 			IWL_ERR(mvm, "failed to update power mode\n");
@@ -1032,7 +960,7 @@ static void iwl_mvm_mld_link_info_changed(struct ieee80211_hw *hw,
 	if (changes & BSS_CHANGED_TXPOWER) {
 		IWL_DEBUG_CALIB(mvm, "Changing TX Power to %d dBm\n",
 				link_conf->txpower);
-		iwl_mvm_set_tx_power(mvm, vif, link_conf->txpower);
+		iwl_mvm_set_tx_power(mvm, link_conf, link_conf->txpower);
 	}
 }
 
@@ -1163,8 +1091,6 @@ iwl_mvm_mld_change_vif_links(struct ieee80211_hw *hw,
 	int err, i;
 
 	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
-		int r;
-
 		if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
 			break;
 
@@ -1176,19 +1102,17 @@ iwl_mvm_mld_change_vif_links(struct ieee80211_hw *hw,
 			goto free;
 		}
 
-		new_link[i]->bcast_sta.sta_id = IWL_MVM_INVALID_STA;
-		new_link[i]->mcast_sta.sta_id = IWL_MVM_INVALID_STA;
-		new_link[i]->ap_sta_id = IWL_MVM_INVALID_STA;
 		new_link[i]->fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
-
-		for (r = 0; r < NUM_IWL_MVM_SMPS_REQ; r++)
-			new_link[i]->smps_requests[r] =
-				IEEE80211_SMPS_AUTOMATIC;
+		iwl_mvm_init_link(new_link[i]);
 	}
 
 	mutex_lock(&mvm->mutex);
 
-	if (old_links == 0) {
+	/* If we're in RESTART flow, the default link wasn't added in
+         * drv_add_interface(), and link[0] doesn't point to it.
+	 */
+	if (old_links == 0 && !test_bit(IWL_MVM_STATUS_IN_HW_RESTART,
+					&mvm->status)) {
 		err = iwl_mvm_disable_link(mvm, vif, &vif->bss_conf);
 		if (err)
 			goto out_err;
@@ -1335,6 +1259,22 @@ iwl_mvm_mld_mac_pre_channel_switch(struct ieee80211_hw *hw,
 		else
 			selected = primary;
 
+		/*
+		 * remembers to tell the firmware that this link can't tx
+		 * Note that this logic seems to be unrelated to esr, but it
+		 * really is needed only when esr is active. When we have a
+		 * single link, the firmware will handle all this on its own.
+		 * In multi-link scenarios, we can learn about the CSA from
+		 * another link and this logic is too complex for the firmware
+		 * to track.
+		 * Since we want to de-activate the link that got a CSA, we
+		 * need to tell the firmware not to send any frame on that link
+		 * as the firmware may not be aware that link is under a CSA
+		 * with mode=1 (no Tx allowed).
+		 */
+		if (chsw->block_tx && mvmvif->link[chsw->link_id])
+			mvmvif->link[chsw->link_id]->csa_block_tx = true;
+
 		iwl_mvm_exit_esr(mvm, vif, IWL_MVM_ESR_EXIT_CSA, selected);
 		mutex_unlock(&mvm->mutex);
 
@@ -1354,6 +1294,36 @@ iwl_mvm_mld_mac_pre_channel_switch(struct ieee80211_hw *hw,
 	return ret;
 }
 
+#define IWL_MVM_MLD_UNBLOCK_ESR_NON_BSS_TIMEOUT (5 * HZ)
+
+static void iwl_mvm_mld_prep_add_interface(struct ieee80211_hw *hw,
+					   enum nl80211_iftype type)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct ieee80211_vif *bss_vif = iwl_mvm_get_bss_vif(mvm);
+	struct iwl_mvm_vif *mvmvif;
+	int ret;
+
+	IWL_DEBUG_MAC80211(mvm, "prep_add_interface: type=%u\n",
+			   type);
+
+	if (IS_ERR_OR_NULL(bss_vif) ||
+	    !(type == NL80211_IFTYPE_AP ||
+	      type == NL80211_IFTYPE_P2P_GO ||
+	      type == NL80211_IFTYPE_P2P_CLIENT))
+		return;
+
+	mvmvif = iwl_mvm_vif_from_mac80211(bss_vif);
+	ret = iwl_mvm_block_esr_sync(mvm, bss_vif,
+				     IWL_MVM_ESR_BLOCKED_TMP_NON_BSS);
+	if (ret)
+		return;
+
+	wiphy_delayed_work_queue(mvmvif->mvm->hw->wiphy,
+				 &mvmvif->unblock_esr_tmp_non_bss_wk,
+				 IWL_MVM_MLD_UNBLOCK_ESR_NON_BSS_TIMEOUT);
+}
+
 const struct ieee80211_ops iwl_mvm_mld_hw_ops = {
 	.tx = iwl_mvm_mac_tx,
 	.wake_tx_queue = iwl_mvm_mac_wake_tx_queue,
@@ -1379,7 +1349,7 @@ const struct ieee80211_ops iwl_mvm_mld_hw_ops = {
 	.allow_buffered_frames = iwl_mvm_mac_allow_buffered_frames,
 	.release_buffered_frames = iwl_mvm_mac_release_buffered_frames,
 	.set_rts_threshold = iwl_mvm_mac_set_rts_threshold,
-	.sta_rc_update = iwl_mvm_sta_rc_update,
+	.link_sta_rc_update = iwl_mvm_sta_rc_update,
 	.conf_tx = iwl_mvm_mld_mac_conf_tx,
 	.mgd_prepare_tx = iwl_mvm_mac_mgd_prepare_tx,
 	.mgd_complete_tx = iwl_mvm_mac_mgd_complete_tx,
@@ -1420,8 +1390,6 @@ const struct ieee80211_ops iwl_mvm_mld_hw_ops = {
 
 	.sync_rx_queues = iwl_mvm_sync_rx_queues,
 
-	CFG80211_TESTMODE_CMD(iwl_mvm_mac_testmode_cmd)
-
 #ifdef CONFIG_PM_SLEEP
 	/* look at d3.c */
 	.suspend = iwl_mvm_suspend,
@@ -1450,4 +1418,5 @@ const struct ieee80211_ops iwl_mvm_mld_hw_ops = {
 	.change_sta_links = iwl_mvm_mld_change_sta_links,
 	.can_activate_links = iwl_mvm_mld_can_activate_links,
 	.can_neg_ttlm = iwl_mvm_mld_can_neg_ttlm,
+	.prep_add_interface = iwl_mvm_mld_prep_add_interface,
 };
diff --git a/sys/contrib/dev/iwlwifi/mvm/mld-sta.c b/sys/contrib/dev/iwlwifi/mvm/mld-sta.c
index d5a204e52076..e1010521c3ea 100644
--- a/sys/contrib/dev/iwlwifi/mvm/mld-sta.c
+++ b/sys/contrib/dev/iwlwifi/mvm/mld-sta.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2022-2024 Intel Corporation
+ * Copyright (C) 2022-2025 Intel Corporation
  */
 #include "mvm.h"
 #include "time-sync.h"
@@ -46,11 +46,15 @@ u32 iwl_mvm_sta_fw_id_mask(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 }
 
 static int iwl_mvm_mld_send_sta_cmd(struct iwl_mvm *mvm,
-				    struct iwl_mvm_sta_cfg_cmd *cmd)
+				    struct iwl_sta_cfg_cmd *cmd)
 {
+	u32 cmd_id = WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD);
+	int cmd_len = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 0) > 1 ?
+		      sizeof(*cmd) :
+		      sizeof(struct iwl_sta_cfg_cmd_v1);
 	int ret = iwl_mvm_send_cmd_pdu(mvm,
 				       WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD),
-				       0, sizeof(*cmd), cmd);
+				       0, cmd_len, cmd);
 	if (ret)
 		IWL_ERR(mvm, "STA_CONFIG_CMD send failed, ret=0x%x\n", ret);
 	return ret;
@@ -63,7 +67,7 @@ static int iwl_mvm_mld_add_int_sta_to_fw(struct iwl_mvm *mvm,
 					 struct iwl_mvm_int_sta *sta,
 					 const u8 *addr, int link_id)
 {
-	struct iwl_mvm_sta_cfg_cmd cmd;
+	struct iwl_sta_cfg_cmd cmd;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -94,7 +98,7 @@ static int iwl_mvm_mld_add_int_sta_to_fw(struct iwl_mvm *mvm,
  */
 static int iwl_mvm_mld_rm_sta_from_fw(struct iwl_mvm *mvm, u32 sta_id)
 {
-	struct iwl_mvm_remove_sta_cmd rm_sta_cmd = {
+	struct iwl_remove_sta_cmd rm_sta_cmd = {
 		.sta_id = cpu_to_le32(sta_id),
 	};
 	int ret;
@@ -121,7 +125,7 @@ static int iwl_mvm_add_aux_sta_to_fw(struct iwl_mvm *mvm,
 {
 	int ret;
 
-	struct iwl_mvm_aux_sta_cmd cmd = {
+	struct iwl_aux_sta_cmd cmd = {
 		.sta_id = cpu_to_le32(sta->sta_id),
 		.lmac_id = cpu_to_le32(lmac_id),
 	};
@@ -144,9 +148,9 @@ int iwl_mvm_mld_add_int_sta_with_queue(struct iwl_mvm *mvm,
 {
 	int ret, txq;
 	unsigned int wdg_timeout = _wdg_timeout ? *_wdg_timeout :
-		mvm->trans->trans_cfg->base_params->wd_timeout;
+		mvm->trans->mac_cfg->base->wd_timeout;
 
-	if (WARN_ON_ONCE(sta->sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(sta->sta_id == IWL_INVALID_STA))
 		return -ENOSPC;
 
 	if (sta->type == STATION_TYPE_AUX)
@@ -216,7 +220,7 @@ int iwl_mvm_mld_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	static const u8 _baddr[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
 	const u8 *baddr = _baddr;
 	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, vif, false, false);
+		iwl_mvm_get_wd_timeout(mvm, vif);
 	u16 *queue;
 
 	lockdep_assert_held(&mvm->mutex);
@@ -254,7 +258,7 @@ int iwl_mvm_mld_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	struct iwl_mvm_int_sta *msta = &mvm_link->mcast_sta;
 	static const u8 _maddr[] = {0x03, 0x00, 0x00, 0x00, 0x00, 0x00};
 	const u8 *maddr = _maddr;
-	unsigned int timeout = iwl_mvm_get_wd_timeout(mvm, vif, false, false);
+	unsigned int timeout = iwl_mvm_get_wd_timeout(mvm, vif);
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -346,7 +350,7 @@ static int iwl_mvm_mld_rm_int_sta(struct iwl_mvm *mvm,
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (WARN_ON_ONCE(int_sta->sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(int_sta->sta_id == IWL_INVALID_STA))
 		return -EINVAL;
 
 	if (flush)
@@ -438,7 +442,7 @@ static int iwl_mvm_mld_cfg_sta(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	struct iwl_mvm_vif *mvm_vif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_vif_link_info *link_info =
 					mvm_vif->link[link_conf->link_id];
-	struct iwl_mvm_sta_cfg_cmd cmd = {
+	struct iwl_sta_cfg_cmd cmd = {
 		.sta_id = cpu_to_le32(mvm_link_sta->sta_id),
 		.station_type = cpu_to_le32(mvm_sta->sta_type),
 	};
@@ -518,11 +522,12 @@ static int iwl_mvm_mld_cfg_sta(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 void iwl_mvm_mld_free_sta_link(struct iwl_mvm *mvm,
 			       struct iwl_mvm_sta *mvm_sta,
 			       struct iwl_mvm_link_sta *mvm_sta_link,
-			       unsigned int link_id,
-			       bool is_in_fw)
+			       unsigned int link_id)
 {
-	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta_link->sta_id],
-			 is_in_fw ? ERR_PTR(-EINVAL) : NULL);
+	lockdep_assert_wiphy(mvm->hw->wiphy);
+	lockdep_assert_held(&mvm->mutex);
+
+	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta_link->sta_id], NULL);
 	RCU_INIT_POINTER(mvm->fw_id_to_link_sta[mvm_sta_link->sta_id], NULL);
 	RCU_INIT_POINTER(mvm_sta->link[link_id], NULL);
 
@@ -543,7 +548,7 @@ static void iwl_mvm_mld_sta_rm_all_sta_links(struct iwl_mvm *mvm,
 		if (!link)
 			continue;
 
-		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, link, link_id, false);
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, link, link_id);
 	}
 }
 
@@ -559,7 +564,10 @@ static int iwl_mvm_mld_alloc_sta_link(struct iwl_mvm *mvm,
 	u32 sta_id = iwl_mvm_find_free_sta_id(mvm,
 					  ieee80211_vif_type_p2p(vif));
 
-	if (sta_id == IWL_MVM_INVALID_STA)
+	lockdep_assert_wiphy(mvm->hw->wiphy);
+	lockdep_assert_held(&mvm->mutex);
+
+	if (sta_id == IWL_INVALID_STA)
 		return -ENOSPC;
 
 	if (rcu_access_pointer(sta->link[link_id]) == &sta->deflink) {
@@ -612,10 +620,10 @@ static void iwl_mvm_mld_set_ap_sta_id(struct ieee80211_sta *sta,
 				      struct iwl_mvm_link_sta *sta_link)
 {
 	if (!sta->tdls) {
-		WARN_ON(vif_link->ap_sta_id != IWL_MVM_INVALID_STA);
+		WARN_ON(vif_link->ap_sta_id != IWL_INVALID_STA);
 		vif_link->ap_sta_id = sta_link->sta_id;
 	} else {
-		WARN_ON(vif_link->ap_sta_id == IWL_MVM_INVALID_STA);
+		WARN_ON(vif_link->ap_sta_id == IWL_INVALID_STA);
 	}
 }
 
@@ -631,6 +639,9 @@ static int iwl_mvm_alloc_sta_after_restart(struct iwl_mvm *mvm,
 	int ret = -EINVAL;
 	int sta_id;
 
+	lockdep_assert_wiphy(mvm->hw->wiphy);
+	lockdep_assert_held(&mvm->mutex);
+
 	/* First add an empty station since allocating a queue requires
 	 * a valid station. Since we need a link_id to allocate a station,
 	 * pick up the first valid one.
@@ -686,7 +697,7 @@ int iwl_mvm_mld_add_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 		spin_lock_init(&mvm_sta->lock);
 
-		ret = iwl_mvm_sta_init(mvm, vif, sta, IWL_MVM_INVALID_STA,
+		ret = iwl_mvm_sta_init(mvm, vif, sta, IWL_INVALID_STA,
 				       STATION_TYPE_PEER);
 	} else {
 		ret = iwl_mvm_alloc_sta_after_restart(mvm, vif, sta);
@@ -835,18 +846,11 @@ int iwl_mvm_mld_rm_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		struct iwl_mvm_link_sta *mvm_link_sta =
 			rcu_dereference_protected(mvm_sta->link[link_id],
 						  lockdep_is_held(&mvm->mutex));
-		bool stay_in_fw;
+		iwl_mvm_sta_del(mvm, vif, sta, link_sta);
 
-		stay_in_fw = iwl_mvm_sta_del(mvm, vif, sta, link_sta, &ret);
-		if (ret)
-			break;
+		ret = iwl_mvm_mld_rm_sta_from_fw(mvm, mvm_link_sta->sta_id);
 
-		if (!stay_in_fw)
-			ret = iwl_mvm_mld_rm_sta_from_fw(mvm,
-							 mvm_link_sta->sta_id);
-
-		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_link_sta,
-					  link_id, stay_in_fw);
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_link_sta, link_id);
 	}
 	kfree(mvm_sta->mpdu_counters);
 	mvm_sta->mpdu_counters = NULL;
@@ -858,9 +862,10 @@ int iwl_mvm_mld_rm_sta_id(struct iwl_mvm *mvm, u8 sta_id)
 {
 	int ret;
 
+	lockdep_assert_wiphy(mvm->hw->wiphy);
 	lockdep_assert_held(&mvm->mutex);
 
-	if (WARN_ON(sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON(sta_id == IWL_INVALID_STA))
 		return 0;
 
 	ret = iwl_mvm_mld_rm_sta_from_fw(mvm, sta_id);
@@ -1064,6 +1069,7 @@ int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
 	unsigned int link_id;
 	int ret;
 
+	lockdep_assert_wiphy(mvm->hw->wiphy);
 	lockdep_assert_held(&mvm->mutex);
 
 	for_each_set_bit(link_id, &old_links_long,
@@ -1109,10 +1115,9 @@ int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
 			goto err;
 
 		if (vif->type == NL80211_IFTYPE_STATION)
-			mvm_vif_link->ap_sta_id = IWL_MVM_INVALID_STA;
+			mvm_vif_link->ap_sta_id = IWL_INVALID_STA;
 
-		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id,
-					  false);
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id);
 	}
 
 	for_each_set_bit(link_id, &links_to_add, IEEE80211_MLD_MAX_NUM_LINKS) {
@@ -1182,6 +1187,9 @@ int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
 		link_sta_added_to_fw |= BIT(link_id);
 
 		iwl_mvm_rs_add_sta_link(mvm, mvm_sta_link);
+
+		iwl_mvm_rs_rate_init(mvm, vif, sta, link_conf, link_sta,
+				     link_conf->chanreq.oper.chan->band);
 	}
 
 	if (sta_mask_added) {
@@ -1213,8 +1221,7 @@ err:
 			rcu_dereference_protected(mvm_sta->link[link_id],
 						  lockdep_is_held(&mvm->mutex));
 
-		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id,
-					  false);
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id);
 	}
 
 	return ret;
diff --git a/sys/contrib/dev/iwlwifi/mvm/mvm.h b/sys/contrib/dev/iwlwifi/mvm/mvm.h
index 6278c8be3950..41125adf4fd1 100644
--- a/sys/contrib/dev/iwlwifi/mvm/mvm.h
+++ b/sys/contrib/dev/iwlwifi/mvm/mvm.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -108,6 +108,7 @@ struct iwl_mvm_phy_ctxt {
 	u32 center_freq1;
 	bool rlc_disabled;
 	u32 channel_load_by_us;
+	u32 channel_load_not_by_us;
 };
 
 struct iwl_mvm_time_event_data {
@@ -130,7 +131,7 @@ struct iwl_mvm_time_event_data {
  /* Power management */
 
 /**
- * enum iwl_power_scheme
+ * enum iwl_power_scheme - iwl power schemes
  * @IWL_POWER_SCHEME_CAM: Continuously Active Mode
  * @IWL_POWER_SCHEME_BPS: Balanced Power Save (default)
  * @IWL_POWER_SCHEME_LP: Low Power
@@ -304,9 +305,12 @@ struct iwl_probe_resp_data {
  * @active: indicates the link is active in FW (for sanity checking)
  * @cab_queue: content-after-beacon (multicast) queue
  * @listen_lmac: indicates this link is allocated to the listen LMAC
+ * @csa_block_tx: we got CSA with mode=1
  * @mcast_sta: multicast station
  * @phy_ctxt: phy context allocated to this link, if any
  * @bf_data: beacon filtering data
+ * @average_beacon_energy: average beacon energy for beacons received during
+ *	client connections
  */
 struct iwl_mvm_vif_link_info {
 	u8 bssid[ETH_ALEN];
@@ -329,6 +333,7 @@ struct iwl_mvm_vif_link_info {
 	bool he_ru_2mhz_block;
 	bool active;
 	bool listen_lmac;
+	bool csa_block_tx;
 
 	u16 cab_queue;
 	/* Assigned while mac80211 has the link in a channel context,
@@ -344,6 +349,7 @@ struct iwl_mvm_vif_link_info {
 	u16 mgmt_queue;
 
 	struct iwl_mvm_link_bf_data bf_data;
+	u32 average_beacon_energy;
 };
 
 /**
@@ -364,6 +370,9 @@ struct iwl_mvm_vif_link_info {
  * @IWL_MVM_ESR_BLOCKED_NON_BSS: An active non-BSS interface's link is
  *	preventing EMLSR
  * @IWL_MVM_ESR_BLOCKED_ROC: remain-on-channel is preventing EMLSR
+ * @IWL_MVM_ESR_BLOCKED_TMP_NON_BSS: An expected active non-BSS interface's link
+ *      is preventing EMLSR. This is a temporary blocking that is set when there
+ *      is an indication that a non-BSS interface is to be added.
  * @IWL_MVM_ESR_EXIT_MISSED_BEACON: exited EMLSR due to missed beacons
  * @IWL_MVM_ESR_EXIT_LOW_RSSI: link is deactivated/not allowed for EMLSR
  *	due to low RSSI.
@@ -381,6 +390,7 @@ enum iwl_mvm_esr_state {
 	IWL_MVM_ESR_BLOCKED_FW		= 0x8,
 	IWL_MVM_ESR_BLOCKED_NON_BSS	= 0x10,
 	IWL_MVM_ESR_BLOCKED_ROC		= 0x20,
+	IWL_MVM_ESR_BLOCKED_TMP_NON_BSS	= 0x40,
 	IWL_MVM_ESR_EXIT_MISSED_BEACON	= 0x10000,
 	IWL_MVM_ESR_EXIT_LOW_RSSI	= 0x20000,
 	IWL_MVM_ESR_EXIT_COEX		= 0x40000,
@@ -453,6 +463,8 @@ struct iwl_mvm_esr_exit {
  * @prevent_esr_done_wk: work that should be done when esr prevention ends.
  * @mlo_int_scan_wk: work for the internal MLO scan.
  * @unblock_esr_tpt_wk: work for unblocking EMLSR when tpt is high enough.
+ * @unblock_esr_tmp_non_bss_wk: work for removing the
+ *      IWL_MVM_ESR_BLOCKED_TMP_NON_BSS blocking for EMLSR.
  * @roc_activity: currently running ROC activity for this vif (or
  *	ROC_NUM_ACTIVITIES if no activity is running).
  * @session_prot_connection_loss: the connection was lost due to session
@@ -513,7 +525,7 @@ struct iwl_mvm_vif {
 	bool bf_enabled;
 	bool ba_enabled;
 
-#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
 	/* WoWLAN GTK rekey data */
 	struct {
 		u8 kck[NL80211_KCK_EXT_LEN];
@@ -589,6 +601,7 @@ struct iwl_mvm_vif {
 	struct wiphy_delayed_work prevent_esr_done_wk;
 	struct wiphy_delayed_work mlo_int_scan_wk;
 	struct wiphy_work unblock_esr_tpt_wk;
+	struct wiphy_delayed_work unblock_esr_tmp_non_bss_wk;
 
 	struct iwl_mvm_vif_link_info deflink;
 	struct iwl_mvm_vif_link_info *link[IEEE80211_MLD_MAX_NUM_LINKS];
@@ -657,6 +670,8 @@ enum iwl_mvm_sched_scan_pass_all_states {
  * @min_backoff: The minimal tx backoff due to power restrictions
  * @params: Parameters to configure the thermal throttling algorithm.
  * @throttle: Is thermal throttling is active?
+ * @power_budget_mw: maximum cTDP power budget as defined for this system and
+ *	device
  */
 struct iwl_mvm_tt_mgmt {
 	struct delayed_work ct_kill_exit;
@@ -665,6 +680,8 @@ struct iwl_mvm_tt_mgmt {
 	u32 min_backoff;
 	struct iwl_tt_params params;
 	bool throttle;
+
+	u32 power_budget_mw;
 };
 
 #ifdef CONFIG_THERMAL
@@ -774,8 +791,6 @@ struct iwl_mvm_tcm {
  * @head_sn: reorder window head sn
  * @num_stored: number of mpdus stored in the buffer
  * @queue: queue of this reorder buffer
- * @last_amsdu: track last ASMDU SN for duplication detection
- * @last_sub_index: track ASMDU sub frame index for duplication detection
  * @valid: reordering is valid for this queue
  * @lock: protect reorder buffer internal state
  */
@@ -783,8 +798,6 @@ struct iwl_mvm_reorder_buffer {
 	u16 head_sn;
 	u16 num_stored;
 	int queue;
-	u16 last_amsdu;
-	u8 last_sub_index;
 	bool valid;
 	spinlock_t lock;
 } ____cacheline_aligned_in_smp;
@@ -996,7 +1009,7 @@ struct iwl_mvm {
 
 	struct iwl_trans *trans;
 	const struct iwl_fw *fw;
-	const struct iwl_cfg *cfg;
+	const struct iwl_rf_cfg *cfg;
 	struct iwl_phy_db *phy_db;
 	struct ieee80211_hw *hw;
 
@@ -1030,6 +1043,8 @@ struct iwl_mvm {
 
 	u8 cca_40mhz_workaround;
 
+	u8 fw_rates_ver;
+
 	u32 ampdu_ref;
 	bool ampdu_toggle;
 
@@ -1079,8 +1094,9 @@ struct iwl_mvm {
 
 	/* data related to data path */
 	struct iwl_rx_phy_info last_phy_info;
-	struct ieee80211_sta __rcu *fw_id_to_mac_id[IWL_MVM_STATION_COUNT_MAX];
-	struct ieee80211_link_sta __rcu *fw_id_to_link_sta[IWL_MVM_STATION_COUNT_MAX];
+	struct ieee80211_sta __rcu *fw_id_to_mac_id[IWL_STATION_COUNT_MAX];
+	/* note: fw_id_to_link_sta must be protected by wiphy and mvm mutexes */
+	struct ieee80211_link_sta __rcu *fw_id_to_link_sta[IWL_STATION_COUNT_MAX];
 	u8 rx_ba_sessions;
 
 	/* configured by mac80211 */
@@ -1169,10 +1185,6 @@ struct iwl_mvm {
 
 	struct ieee80211_vif __rcu *vif_id_to_mac[NUM_MAC_INDEX_DRIVER];
 
-	struct ieee80211_bss_conf __rcu *link_id_to_link_conf[IWL_MVM_FW_MAX_LINK_ID + 1];
-
-	/* -1 for always, 0 for never, >0 for that many times */
-	s8 fw_restart;
 	u8 *error_recovery_buf;
 
 #ifdef CONFIG_IWLWIFI_LEDS
@@ -1181,7 +1193,7 @@ struct iwl_mvm {
 
 	struct ieee80211_vif *p2p_device_vif;
 
-#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
 	struct wiphy_wowlan_support wowlan;
 	int gtk_ivlen, gtk_icvlen, ptk_ivlen, ptk_icvlen;
 
@@ -1205,8 +1217,11 @@ struct iwl_mvm {
 
 	wait_queue_head_t rx_sync_waitq;
 
-	/* BT-Coex */
-	struct iwl_bt_coex_profile_notif last_bt_notif;
+	/* BT-Coex - only one of those will be used */
+	union {
+		struct iwl_bt_coex_prof_old_notif last_bt_notif;
+		struct iwl_bt_coex_profile_notif last_bt_wifi_loss;
+	};
 	struct iwl_bt_coex_ci_cmd last_bt_ci_cmd;
 
 	u8 bt_tx_prio;
@@ -1241,11 +1256,6 @@ struct iwl_mvm {
 
 	struct iwl_time_quota_cmd last_quota_cmd;
 
-#ifdef CONFIG_NL80211_TESTMODE
-	u32 noa_duration;
-	struct ieee80211_vif *noa_vif;
-#endif
-
 	/* Tx queues */
 	u16 aux_queue;
 	u16 snif_queue;
@@ -1305,7 +1315,7 @@ struct iwl_mvm {
 		struct cfg80211_pmsr_request *req;
 		struct wireless_dev *req_wdev;
 		struct list_head loc_list;
-		int responses[IWL_MVM_TOF_MAX_APS];
+		int responses[IWL_TOF_MAX_APS];
 		struct {
 			struct list_head resp;
 		} smooth;
@@ -1320,7 +1330,6 @@ struct iwl_mvm {
 		u8 range_resp;
 	} cmd_ver;
 
-	struct ieee80211_vif *nan_vif;
 	struct iwl_mvm_baid_data __rcu *baid_map[IWL_MAX_BAID];
 
 	/*
@@ -1343,10 +1352,6 @@ struct iwl_mvm {
 	__le16 cur_aid;
 	u8 cur_bssid[ETH_ALEN];
 
-#ifdef CONFIG_ACPI
-	struct iwl_phy_specific_cfg phy_filters;
-#endif
-
 	/* report rx timestamp in ptp clock time */
 	bool rx_ts_ptp;
 
@@ -1394,8 +1399,6 @@ DEFINE_GUARD(mvm, struct iwl_mvm *, mutex_lock(&_T->mutex), mutex_unlock(&_T->mu
  * @IWL_MVM_STATUS_IN_D3: in D3 (or at least about to go into it)
  * @IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE: suppress one error log
  *	if this is set, when intentionally triggered
- * @IWL_MVM_STATUS_STARTING: starting mac,
- *	used to disable restart flow while in STARTING state
  */
 enum iwl_mvm_status {
 	IWL_MVM_STATUS_HW_RFKILL,
@@ -1407,7 +1410,6 @@ enum iwl_mvm_status {
 	IWL_MVM_STATUS_FIRMWARE_RUNNING,
 	IWL_MVM_STATUS_IN_D3,
 	IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE,
-	IWL_MVM_STATUS_STARTING,
 };
 
 struct iwl_mvm_csme_conn_info {
@@ -1488,20 +1490,6 @@ iwl_mvm_rcu_dereference_vif_id(struct iwl_mvm *mvm, u8 vif_id, bool rcu)
 					 lockdep_is_held(&mvm->mutex));
 }
 
-static inline struct ieee80211_bss_conf *
-iwl_mvm_rcu_fw_link_id_to_link_conf(struct iwl_mvm *mvm, u8 link_id, bool rcu)
-{
-	if (IWL_FW_CHECK(mvm, link_id >= ARRAY_SIZE(mvm->link_id_to_link_conf),
-			 "erroneous FW link ID: %d\n", link_id))
-		return NULL;
-
-	if (rcu)
-		return rcu_dereference(mvm->link_id_to_link_conf[link_id]);
-
-	return rcu_dereference_protected(mvm->link_id_to_link_conf[link_id],
-					 lockdep_is_held(&mvm->mutex));
-}
-
 static inline bool iwl_mvm_is_adaptive_dwell_supported(struct iwl_mvm *mvm)
 {
 	return fw_has_api(&mvm->fw->ucode_capa,
@@ -1559,7 +1547,7 @@ static inline bool iwl_mvm_is_lar_supported(struct iwl_mvm *mvm)
 	 * Enable LAR only if it is supported by the FW (TLV) &&
 	 * enabled in the NVM
 	 */
-	if (mvm->cfg->nvm_type == IWL_NVM_EXT)
+	if (mvm->trans->cfg->nvm_type == IWL_NVM_EXT)
 		return nvm_lar && tlv_lar;
 	else
 		return tlv_lar;
@@ -1583,8 +1571,7 @@ static inline bool iwl_mvm_bt_is_rrc_supported(struct iwl_mvm *mvm)
 static inline bool iwl_mvm_is_csum_supported(struct iwl_mvm *mvm)
 {
 	return fw_has_capa(&mvm->fw->ucode_capa,
-			   IWL_UCODE_TLV_CAPA_CSUM_SUPPORT) &&
-		!IWL_MVM_HW_CSUM_DISABLE;
+			   IWL_UCODE_TLV_CAPA_CSUM_SUPPORT);
 }
 
 static inline bool iwl_mvm_is_mplut_supported(struct iwl_mvm *mvm)
@@ -1624,13 +1611,13 @@ static inline bool iwl_mvm_has_new_station_api(const struct iwl_fw *fw)
 static inline bool iwl_mvm_has_new_tx_api(struct iwl_mvm *mvm)
 {
 	/* TODO - replace with TLV once defined */
-	return mvm->trans->trans_cfg->gen2;
+	return mvm->trans->mac_cfg->gen2;
 }
 
 static inline bool iwl_mvm_has_unified_ucode(struct iwl_mvm *mvm)
 {
 	/* TODO - better define this */
-	return mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000;
+	return mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000;
 }
 
 static inline bool iwl_mvm_is_cdb_supported(struct iwl_mvm *mvm)
@@ -1655,7 +1642,7 @@ static inline bool iwl_mvm_cdb_scan_api(struct iwl_mvm *mvm)
 	 * but then there's a little bit of code in scan that won't make
 	 * any sense...
 	 */
-	return mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000;
+	return mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000;
 }
 
 static inline bool iwl_mvm_is_scan_ext_chan_supported(struct iwl_mvm *mvm)
@@ -1705,9 +1692,9 @@ static inline struct agg_tx_status *
 iwl_mvm_get_agg_status(struct iwl_mvm *mvm, void *tx_resp)
 {
 	if (iwl_mvm_has_new_tx_api(mvm))
-		return &((struct iwl_mvm_tx_resp *)tx_resp)->status;
+		return &((struct iwl_tx_resp *)tx_resp)->status;
 	else
-		return ((struct iwl_mvm_tx_resp_v3 *)tx_resp)->status;
+		return ((struct iwl_tx_resp_v3 *)tx_resp)->status;
 }
 
 static inline bool iwl_mvm_is_tt_in_fw(struct iwl_mvm *mvm)
@@ -1730,12 +1717,19 @@ static inline bool iwl_mvm_is_ctdp_supported(struct iwl_mvm *mvm)
 
 static inline bool iwl_mvm_is_esr_supported(struct iwl_trans *trans)
 {
-	if ((CSR_HW_RFID_TYPE(trans->hw_rf_id) == IWL_CFG_RF_TYPE_FM) &&
-	    !CSR_HW_RFID_IS_CDB(trans->hw_rf_id))
-		/* Step A doesn't support eSR */
-		return CSR_HW_RFID_STEP(trans->hw_rf_id);
+	if (CSR_HW_RFID_IS_CDB(trans->info.hw_rf_id))
+		return false;
 
-	return false;
+	switch (CSR_HW_RFID_TYPE(trans->info.hw_rf_id)) {
+	case IWL_CFG_RF_TYPE_FM:
+		/* Step A doesn't support eSR */
+		return CSR_HW_RFID_STEP(trans->info.hw_rf_id);
+	case IWL_CFG_RF_TYPE_WH:
+	case IWL_CFG_RF_TYPE_PE:
+		return true;
+	default:
+		return false;
+	}
 }
 
 static inline int iwl_mvm_max_active_links(struct iwl_mvm *mvm,
@@ -1748,9 +1742,9 @@ static inline int iwl_mvm_max_active_links(struct iwl_mvm *mvm,
 
 	/* Check if HW supports eSR or STR */
 	if (iwl_mvm_is_esr_supported(trans) ||
-	    (CSR_HW_RFID_TYPE(trans->hw_rf_id) == IWL_CFG_RF_TYPE_FM &&
-	     CSR_HW_RFID_IS_CDB(trans->hw_rf_id)))
-		return IWL_MVM_FW_MAX_ACTIVE_LINKS_NUM;
+	    (CSR_HW_RFID_TYPE(trans->info.hw_rf_id) == IWL_CFG_RF_TYPE_FM &&
+	     CSR_HW_RFID_IS_CDB(trans->info.hw_rf_id)))
+		return IWL_FW_MAX_ACTIVE_LINKS_NUM;
 
 	return 1;
 }
@@ -1762,13 +1756,20 @@ extern const u8 iwl_mvm_ac_to_bz_tx_fifo[];
 static inline u8 iwl_mvm_mac_ac_to_tx_fifo(struct iwl_mvm *mvm,
 					   enum ieee80211_ac_numbers ac)
 {
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		return iwl_mvm_ac_to_bz_tx_fifo[ac];
 	if (iwl_mvm_has_new_tx_api(mvm))
 		return iwl_mvm_ac_to_gen2_tx_fifo[ac];
 	return iwl_mvm_ac_to_tx_fifo[ac];
 }
 
+static inline bool iwl_mvm_has_rlc_offload(struct iwl_mvm *mvm)
+{
+	return iwl_fw_lookup_cmd_ver(mvm->fw,
+				     WIDE_ID(DATA_PATH_GROUP, RLC_CONFIG_CMD),
+				     0) >= 3;
+}
+
 struct iwl_rate_info {
 	u8 plcp;	/* uCode API:  IWL_RATE_6M_PLCP, etc. */
 	u8 plcp_siso;	/* uCode API:  IWL_RATE_SISO_6M_PLCP, etc. */
@@ -1794,9 +1795,6 @@ int iwl_mvm_legacy_rate_to_mac80211_idx(u32 rate_n_flags,
 void iwl_mvm_hwrate_to_tx_rate(u32 rate_n_flags,
 			       enum nl80211_band band,
 			       struct ieee80211_tx_rate *r);
-void iwl_mvm_hwrate_to_tx_rate_v1(u32 rate_n_flags,
-				  enum nl80211_band band,
-				  struct ieee80211_tx_rate *r);
 u8 iwl_mvm_mac80211_idx_to_hwrate(const struct iwl_fw *fw, int rate_idx);
 u8 iwl_mvm_mac80211_ac_to_ucode_ac(enum ieee80211_ac_numbers ac);
 bool iwl_mvm_is_nic_ack_enabled(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
@@ -1811,7 +1809,6 @@ u8 iwl_mvm_next_antenna(struct iwl_mvm *mvm, u8 valid, u8 last_idx);
 void iwl_mvm_get_sync_time(struct iwl_mvm *mvm, int clock_type, u32 *gp2,
 			   u64 *boottime, ktime_t *realtime);
 u32 iwl_mvm_get_systime(struct iwl_mvm *mvm);
-u32 iwl_mvm_find_ie_offset(u8 *beacon, u8 eid, u32 frame_size);
 
 /* Tx / Host Commands */
 int __must_check iwl_mvm_send_cmd(struct iwl_mvm *mvm,
@@ -1828,11 +1825,12 @@ int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb,
 		       struct ieee80211_sta *sta);
 int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb);
 void iwl_mvm_set_tx_cmd(struct iwl_mvm *mvm, struct sk_buff *skb,
-			struct iwl_tx_cmd *tx_cmd,
+			struct iwl_tx_cmd_v6_params *tx_cmd_params,
 			struct ieee80211_tx_info *info, u8 sta_id);
-void iwl_mvm_set_tx_cmd_rate(struct iwl_mvm *mvm, struct iwl_tx_cmd *tx_cmd,
-			    struct ieee80211_tx_info *info,
-			    struct ieee80211_sta *sta, __le16 fc);
+void iwl_mvm_set_tx_cmd_rate(struct iwl_mvm *mvm,
+			     struct iwl_tx_cmd_v6_params *tx_cmd_params,
+			     struct ieee80211_tx_info *info,
+			     struct ieee80211_sta *sta, __le16 fc);
 void iwl_mvm_mac_itxq_xmit(struct ieee80211_hw *hw, struct ieee80211_txq *txq);
 unsigned int iwl_mvm_max_amsdu_size(struct iwl_mvm *mvm,
 				    struct ieee80211_sta *sta,
@@ -1861,12 +1859,12 @@ int iwl_mvm_set_sta_pkt_ext(struct iwl_mvm *mvm,
 void iwl_mvm_async_handlers_purge(struct iwl_mvm *mvm);
 
 static inline void iwl_mvm_set_tx_cmd_ccmp(struct ieee80211_tx_info *info,
-					   struct iwl_tx_cmd *tx_cmd)
+					   struct iwl_tx_cmd_v6_params *tx_cmd_params)
 {
 	struct ieee80211_key_conf *keyconf = info->control.hw_key;
 
-	tx_cmd->sec_ctl = TX_CMD_SEC_CCM;
-	memcpy(tx_cmd->key, keyconf->key, keyconf->keylen);
+	tx_cmd_params->sec_ctl = TX_CMD_SEC_CCM;
+	memcpy(tx_cmd_params->key, keyconf->key, keyconf->keylen);
 }
 
 static inline void iwl_mvm_wait_for_async_handlers(struct iwl_mvm *mvm)
@@ -2008,7 +2006,7 @@ int iwl_mvm_phy_send_rlc(struct iwl_mvm *mvm, struct iwl_mvm_phy_ctxt *ctxt,
 void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
 				 struct ieee80211_vif *vif);
 void iwl_mvm_set_fw_basic_rates(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-				struct ieee80211_bss_conf *link_conf,
+				struct iwl_mvm_vif_link_info *link_info,
 				__le32 *cck_rates, __le32 *ofdm_rates);
 void iwl_mvm_set_fw_protection_flags(struct iwl_mvm *mvm,
 				     struct ieee80211_vif *vif,
@@ -2067,6 +2065,8 @@ void iwl_mvm_rx_beacon_notif(struct iwl_mvm *mvm,
 			     struct iwl_rx_cmd_buffer *rxb);
 void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
 				     struct iwl_rx_cmd_buffer *rxb);
+void iwl_mvm_rx_missed_beacons_notif_legacy(struct iwl_mvm *mvm,
+					    struct iwl_rx_cmd_buffer *rxb);
 void iwl_mvm_rx_stored_beacon_notif(struct iwl_mvm *mvm,
 				    struct iwl_rx_cmd_buffer *rxb);
 void iwl_mvm_mu_mimo_grp_notif(struct iwl_mvm *mvm,
@@ -2078,27 +2078,27 @@ void iwl_mvm_mac_ctxt_recalc_tsf_id(struct iwl_mvm *mvm,
 				    struct ieee80211_vif *vif);
 void iwl_mvm_probe_resp_data_notif(struct iwl_mvm *mvm,
 				   struct iwl_rx_cmd_buffer *rxb);
-void iwl_mvm_rx_missed_vap_notif(struct iwl_mvm *mvm,
-				 struct iwl_rx_cmd_buffer *rxb);
 void iwl_mvm_channel_switch_start_notif(struct iwl_mvm *mvm,
 					struct iwl_rx_cmd_buffer *rxb);
 void iwl_mvm_channel_switch_error_notif(struct iwl_mvm *mvm,
 					struct iwl_rx_cmd_buffer *rxb);
+void iwl_mvm_rx_beacon_filter_notif(struct iwl_mvm *mvm,
+				    struct iwl_rx_cmd_buffer *rxb);
+
 /* Bindings */
 int iwl_mvm_binding_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_binding_remove_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 u32 iwl_mvm_get_lmac_id(struct iwl_mvm *mvm, enum nl80211_band band);
 
 /* Links */
-int iwl_mvm_set_link_mapping(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			     struct ieee80211_bss_conf *link_conf);
+void iwl_mvm_init_link(struct iwl_mvm_vif_link_info *link);
+void iwl_mvm_set_link_fw_id(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			    struct ieee80211_bss_conf *link_conf);
 int iwl_mvm_add_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		     struct ieee80211_bss_conf *link_conf);
 int iwl_mvm_link_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			 struct ieee80211_bss_conf *link_conf,
 			 u32 changes, bool active);
-int iwl_mvm_unset_link_mapping(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			       struct ieee80211_bss_conf *link_conf);
 int iwl_mvm_remove_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			struct ieee80211_bss_conf *link_conf);
 int iwl_mvm_disable_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
@@ -2122,6 +2122,9 @@ bool iwl_mvm_mld_valid_link_pair(struct ieee80211_vif *vif,
 				 const struct iwl_mvm_link_sel_data *b);
 
 s8 iwl_mvm_average_dbm_values(const struct iwl_umac_scan_channel_survey_notif *notif);
+
+extern const struct iwl_hcmd_arr iwl_mvm_groups[];
+extern const unsigned int iwl_mvm_groups_size;
 #endif
 
 /* AP and IBSS */
@@ -2296,7 +2299,7 @@ void iwl_mvm_ipv6_addr_change(struct ieee80211_hw *hw,
 void iwl_mvm_set_default_unicast_key(struct ieee80211_hw *hw,
 				     struct ieee80211_vif *vif, int idx);
 extern const struct file_operations iwl_dbgfs_d3_test_ops;
-#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
 void iwl_mvm_set_last_nonqos_seq(struct iwl_mvm *mvm,
 				 struct ieee80211_vif *vif);
 void iwl_mvm_fast_suspend(struct iwl_mvm *mvm);
@@ -2317,7 +2320,7 @@ static inline int iwl_mvm_fast_resume(struct iwl_mvm *mvm)
 }
 #endif
 void iwl_mvm_set_wowlan_qos_seq(struct iwl_mvm_sta *mvm_ap_sta,
-				struct iwl_wowlan_config_cmd *cmd);
+				struct iwl_wowlan_config_cmd_v6 *cmd);
 int iwl_mvm_send_proto_offload(struct iwl_mvm *mvm,
 			       struct ieee80211_vif *vif,
 			       bool disable_offloading,
@@ -2327,6 +2330,8 @@ int iwl_mvm_send_proto_offload(struct iwl_mvm *mvm,
 
 /* BT Coex */
 int iwl_mvm_send_bt_init_conf(struct iwl_mvm *mvm);
+void iwl_mvm_rx_bt_coex_old_notif(struct iwl_mvm *mvm,
+				  struct iwl_rx_cmd_buffer *rxb);
 void iwl_mvm_rx_bt_coex_notif(struct iwl_mvm *mvm,
 			      struct iwl_rx_cmd_buffer *rxb);
 void iwl_mvm_bt_rssi_event(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
@@ -2441,7 +2446,7 @@ void iwl_mvm_vif_set_low_latency(struct iwl_mvm_vif *mvmvif, bool set,
  */
 static inline u32 iwl_mvm_flushable_queues(struct iwl_mvm *mvm)
 {
-	return ((BIT(mvm->trans->trans_cfg->base_params->num_of_queues) - 1) &
+	return ((BIT(mvm->trans->mac_cfg->base->num_of_queues) - 1) &
 		~BIT(IWL_MVM_DQA_CMD_QUEUE));
 }
 
@@ -2500,14 +2505,6 @@ void iwl_mvm_ftm_restart_responder(struct iwl_mvm *mvm,
 				   struct ieee80211_bss_conf *bss_conf);
 void iwl_mvm_ftm_responder_stats(struct iwl_mvm *mvm,
 				 struct iwl_rx_cmd_buffer *rxb);
-#if defined(__linux__)
-int iwl_mvm_ftm_resp_remove_pasn_sta(struct iwl_mvm *mvm,
-				     struct ieee80211_vif *vif, u8 *addr);
-int iwl_mvm_ftm_respoder_add_pasn_sta(struct iwl_mvm *mvm,
-				      struct ieee80211_vif *vif,
-				      u8 *addr, u32 cipher, u8 *tk, u32 tk_len,
-				      u8 *hltk, u32 hltk_len);
-#endif
 void iwl_mvm_ftm_responder_clear(struct iwl_mvm *mvm,
 				 struct ieee80211_vif *vif);
 
@@ -2522,10 +2519,6 @@ int iwl_mvm_ftm_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 void iwl_mvm_ftm_abort(struct iwl_mvm *mvm, struct cfg80211_pmsr_request *req);
 void iwl_mvm_ftm_initiator_smooth_config(struct iwl_mvm *mvm);
 void iwl_mvm_ftm_initiator_smooth_stop(struct iwl_mvm *mvm);
-int iwl_mvm_ftm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			     u8 *addr, u32 cipher, u8 *tk, u32 tk_len,
-			     u8 *hltk, u32 hltk_len);
-void iwl_mvm_ftm_remove_pasn_sta(struct iwl_mvm *mvm, u8 *addr);
 
 /* TDLS */
 
@@ -2575,10 +2568,8 @@ void iwl_mvm_tcm_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 void iwl_mvm_tcm_rm_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 u8 iwl_mvm_tcm_load_percentage(u32 airtime, u32 elapsed);
 
-void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error);
 unsigned int iwl_mvm_get_wd_timeout(struct iwl_mvm *mvm,
-				    struct ieee80211_vif *vif,
-				    bool tdls, bool cmd_q);
+				    struct ieee80211_vif *vif);
 void iwl_mvm_connection_loss(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			     const char *errmsg);
 void iwl_mvm_event_frame_timeout_callback(struct iwl_mvm *mvm,
@@ -2864,13 +2855,16 @@ void iwl_mvm_mac_wake_tx_queue(struct ieee80211_hw *hw,
 int iwl_mvm_mac_ampdu_action(struct ieee80211_hw *hw,
 			     struct ieee80211_vif *vif,
 			     struct ieee80211_ampdu_params *params);
-int iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant);
-int iwl_mvm_op_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant);
+int iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, int radio_idx, u32 *tx_ant,
+			   u32 *rx_ant);
+int iwl_mvm_op_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant,
+			   u32 rx_ant);
 int iwl_mvm_mac_start(struct ieee80211_hw *hw);
 void iwl_mvm_mac_reconfig_complete(struct ieee80211_hw *hw,
 				   enum ieee80211_reconfig_type reconfig_type);
 void iwl_mvm_mac_stop(struct ieee80211_hw *hw, bool suspend);
-static inline int iwl_mvm_mac_config(struct ieee80211_hw *hw, u32 changed)
+static inline int iwl_mvm_mac_config(struct ieee80211_hw *hw, int radio_idx,
+				     u32 changed)
 {
 	return 0;
 }
@@ -2903,9 +2897,10 @@ iwl_mvm_mac_release_buffered_frames(struct ieee80211_hw *hw,
 				    int num_frames,
 				    enum ieee80211_frame_release_type reason,
 				    bool more_data);
-int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, u32 value);
+int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				  u32 value);
 void iwl_mvm_sta_rc_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-			   struct ieee80211_sta *sta, u32 changed);
+			   struct ieee80211_link_sta *link_sta, u32 changed);
 void iwl_mvm_mac_mgd_prepare_tx(struct ieee80211_hw *hw,
 				struct ieee80211_vif *vif,
 				struct ieee80211_prep_tx_info *info);
@@ -2973,25 +2968,19 @@ void iwl_mvm_abort_pmsr(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 bool iwl_mvm_have_links_same_channel(struct iwl_mvm_vif *vif1,
 				     struct iwl_mvm_vif *vif2);
 bool iwl_mvm_vif_is_active(struct iwl_mvm_vif *mvmvif);
-int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+int iwl_mvm_set_tx_power(struct iwl_mvm *mvm,
+			 struct ieee80211_bss_conf *bss_conf,
 			 s16 tx_power);
 int iwl_mvm_set_hw_timestamp(struct ieee80211_hw *hw,
 			     struct ieee80211_vif *vif,
 			     struct cfg80211_set_hw_timestamp *hwts);
 int iwl_mvm_update_mu_groups(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 bool iwl_mvm_enable_fils(struct iwl_mvm *mvm,
+			 struct ieee80211_vif *vif,
 			 struct ieee80211_chanctx_conf *ctx);
-bool iwl_mvm_is_ftm_responder_chanctx(struct iwl_mvm *mvm,
-				      struct ieee80211_chanctx_conf *ctx);
-
-static inline struct cfg80211_chan_def *
-iwl_mvm_chanctx_def(struct iwl_mvm *mvm, struct ieee80211_chanctx_conf *ctx)
-{
-	bool use_def = iwl_mvm_is_ftm_responder_chanctx(mvm, ctx) ||
-		iwl_mvm_enable_fils(mvm, ctx);
 
-	return use_def ? &ctx->def : &ctx->min_def;
-}
+struct cfg80211_chan_def *
+iwl_mvm_chanctx_def(struct iwl_mvm *mvm, struct ieee80211_chanctx_conf *ctx);
 
 void iwl_mvm_roc_duration_and_delay(struct ieee80211_vif *vif,
 				    u32 duration_ms,
@@ -3033,4 +3022,12 @@ iwl_mvm_send_ap_tx_power_constraint_cmd(struct iwl_mvm *mvm,
 					struct ieee80211_vif *vif,
 					struct ieee80211_bss_conf *bss_conf,
 					bool is_ap);
+
+void iwl_mvm_smps_workaround(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     bool update);
+
+/* rate_n_flags conversion */
+u32 iwl_mvm_v3_rate_from_fw(__le32 rate, u8 rate_ver);
+__le32 iwl_mvm_v3_rate_to_fw(u32 rate, u8 rate_ver);
+
 #endif /* __IWL_MVM_H__ */
diff --git a/sys/contrib/dev/iwlwifi/mvm/nvm.c b/sys/contrib/dev/iwlwifi/mvm/nvm.c
index 9083089f2e68..f01d5836fce6 100644
--- a/sys/contrib/dev/iwlwifi/mvm/nvm.c
+++ b/sys/contrib/dev/iwlwifi/mvm/nvm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2019, 2021-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2021-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -122,7 +122,7 @@ static int iwl_nvm_read_chunk(struct iwl_mvm *mvm, u16 section,
 		} else {
 			IWL_DEBUG_EEPROM(mvm->trans->dev,
 					 "NVM access command failed with status %d (device: %s)\n",
-					 ret, mvm->trans->name);
+					 ret, mvm->trans->info.name);
 			ret = -ENODATA;
 		}
 		goto exit;
@@ -193,7 +193,7 @@ static int iwl_nvm_read_section(struct iwl_mvm *mvm, u16 section,
 	while (ret == length) {
 		/* Check no memory assumptions fail and cause an overflow */
 		if ((size_read + offset + length) >
-		    mvm->trans->trans_cfg->base_params->eeprom_size) {
+		    mvm->trans->mac_cfg->base->eeprom_size) {
 			IWL_ERR(mvm, "EEPROM size is too small for NVM\n");
 			return -ENOBUFS;
 		}
@@ -208,7 +208,7 @@ static int iwl_nvm_read_section(struct iwl_mvm *mvm, u16 section,
 		offset += ret;
 	}
 
-	iwl_nvm_fixups(mvm->trans->hw_id, section, data, offset);
+	iwl_nvm_fixups(mvm->trans->info.hw_id, section, data, offset);
 
 	IWL_DEBUG_EEPROM(mvm->trans->dev,
 			 "NVM section %d read completed\n", section);
@@ -228,7 +228,7 @@ iwl_parse_nvm_sections(struct iwl_mvm *mvm)
 	/* Checking for required sections */
 	if (mvm->trans->cfg->nvm_type == IWL_NVM) {
 		if (!mvm->nvm_sections[NVM_SECTION_TYPE_SW].data ||
-		    !mvm->nvm_sections[mvm->cfg->nvm_hw_section_num].data) {
+		    !mvm->nvm_sections[mvm->trans->mac_cfg->base->nvm_hw_section_num].data) {
 			IWL_ERR(mvm, "Can't parse empty OTP/NVM sections\n");
 			return NULL;
 		}
@@ -246,7 +246,7 @@ iwl_parse_nvm_sections(struct iwl_mvm *mvm)
 			return NULL;
 		}
 		/* MAC_OVERRIDE or at least HW section must exist */
-		if (!mvm->nvm_sections[mvm->cfg->nvm_hw_section_num].data &&
+		if (!mvm->nvm_sections[mvm->trans->mac_cfg->base->nvm_hw_section_num].data &&
 		    !mvm->nvm_sections[NVM_SECTION_TYPE_MAC_OVERRIDE].data) {
 			IWL_ERR(mvm,
 				"Can't parse mac_address, empty sections\n");
@@ -262,7 +262,7 @@ iwl_parse_nvm_sections(struct iwl_mvm *mvm)
 		}
 	}
 
-	hw = (const __be16 *)sections[mvm->cfg->nvm_hw_section_num].data;
+	hw = (const __be16 *)sections[mvm->trans->mac_cfg->base->nvm_hw_section_num].data;
 	sw = (const __le16 *)sections[NVM_SECTION_TYPE_SW].data;
 	calib = (const __le16 *)sections[NVM_SECTION_TYPE_CALIBRATION].data;
 	mac_override =
@@ -310,16 +310,15 @@ int iwl_nvm_init(struct iwl_mvm *mvm)
 	int ret, section;
 	u32 size_read = 0;
 	u8 *nvm_buffer, *temp;
-	const char *nvm_file_C = mvm->cfg->default_nvm_file_C_step;
 
-	if (WARN_ON_ONCE(mvm->cfg->nvm_hw_section_num >= NVM_MAX_NUM_SECTIONS))
+	if (WARN_ON_ONCE(mvm->trans->mac_cfg->base->nvm_hw_section_num >= NVM_MAX_NUM_SECTIONS))
 		return -EINVAL;
 
 	/* load NVM values from nic */
 	/* Read From FW NVM */
 	IWL_DEBUG_EEPROM(mvm->trans->dev, "Read from NVM\n");
 
-	nvm_buffer = kmalloc(mvm->trans->trans_cfg->base_params->eeprom_size,
+	nvm_buffer = kmalloc(mvm->trans->mac_cfg->base->eeprom_size,
 			     GFP_KERNEL);
 	if (!nvm_buffer)
 		return -ENOMEM;
@@ -340,7 +339,7 @@ int iwl_nvm_init(struct iwl_mvm *mvm)
 			break;
 		}
 
-		iwl_nvm_fixups(mvm->trans->hw_id, section, temp, ret);
+		iwl_nvm_fixups(mvm->trans->info.hw_id, section, temp, ret);
 
 		mvm->nvm_sections[section].data = temp;
 		mvm->nvm_sections[section].length = ret;
@@ -369,7 +368,7 @@ int iwl_nvm_init(struct iwl_mvm *mvm)
 			mvm->nvm_reg_blob.size  = ret;
 			break;
 		default:
-			if (section == mvm->cfg->nvm_hw_section_num) {
+			if (section == mvm->trans->mac_cfg->base->nvm_hw_section_num) {
 				mvm->nvm_hw_blob.data = temp;
 				mvm->nvm_hw_blob.size = ret;
 				break;
@@ -386,21 +385,8 @@ int iwl_nvm_init(struct iwl_mvm *mvm)
 		/* read External NVM file from the mod param */
 		ret = iwl_read_external_nvm(mvm->trans, mvm->nvm_file_name,
 					    mvm->nvm_sections);
-		if (ret) {
-			mvm->nvm_file_name = nvm_file_C;
-
-			if ((ret == -EFAULT || ret == -ENOENT) &&
-			    mvm->nvm_file_name) {
-				/* in case nvm file was failed try again */
-				ret = iwl_read_external_nvm(mvm->trans,
-							    mvm->nvm_file_name,
-							    mvm->nvm_sections);
-				if (ret)
-					return ret;
-			} else {
-				return ret;
-			}
-		}
+		if (ret)
+			return ret;
 	}
 
 	/* parse the relevant nvm sections */
@@ -556,7 +542,7 @@ int iwl_mvm_init_mcc(struct iwl_mvm *mvm)
 	struct ieee80211_regdomain *regd;
 	char mcc[3];
 
-	if (mvm->cfg->nvm_type == IWL_NVM_EXT) {
+	if (mvm->trans->cfg->nvm_type == IWL_NVM_EXT) {
 		tlv_lar = fw_has_capa(&mvm->fw->ucode_capa,
 				      IWL_UCODE_TLV_CAPA_LAR_SUPPORT);
 		nvm_lar = mvm->nvm_data->lar_enabled;
@@ -613,6 +599,7 @@ void iwl_mvm_rx_chub_update_mcc(struct iwl_mvm *mvm,
 	char mcc[3];
 	struct ieee80211_regdomain *regd;
 	int wgds_tbl_idx;
+	bool changed = false;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -632,10 +619,15 @@ void iwl_mvm_rx_chub_update_mcc(struct iwl_mvm *mvm,
 	IWL_DEBUG_LAR(mvm,
 		      "RX: received chub update mcc cmd (mcc '%s' src %d)\n",
 		      mcc, src);
-	regd = iwl_mvm_get_regdomain(mvm->hw->wiphy, mcc, src, NULL);
+	regd = iwl_mvm_get_regdomain(mvm->hw->wiphy, mcc, src, &changed);
 	if (IS_ERR_OR_NULL(regd))
 		return;
 
+	if (!changed) {
+		IWL_DEBUG_LAR(mvm, "RX: No change in the regulatory data\n");
+		goto out;
+	}
+
 	wgds_tbl_idx = iwl_mvm_get_sar_geo_profile(mvm);
 	if (wgds_tbl_idx < 1)
 		IWL_DEBUG_INFO(mvm,
@@ -646,5 +638,7 @@ void iwl_mvm_rx_chub_update_mcc(struct iwl_mvm *mvm,
 			       wgds_tbl_idx);
 
 	regulatory_set_wiphy_regd(mvm->hw->wiphy, regd);
+
+out:
 	kfree(regd);
 }
diff --git a/sys/contrib/dev/iwlwifi/mvm/offloading.c b/sys/contrib/dev/iwlwifi/mvm/offloading.c
index 1eb21fe861e5..15d4369678a2 100644
--- a/sys/contrib/dev/iwlwifi/mvm/offloading.c
+++ b/sys/contrib/dev/iwlwifi/mvm/offloading.c
@@ -10,7 +10,7 @@
 #include "mvm.h"
 
 void iwl_mvm_set_wowlan_qos_seq(struct iwl_mvm_sta *mvm_ap_sta,
-				struct iwl_wowlan_config_cmd *cmd)
+				struct iwl_wowlan_config_cmd_v6 *cmd)
 {
 	int i;
 
diff --git a/sys/contrib/dev/iwlwifi/mvm/ops.c b/sys/contrib/dev/iwlwifi/mvm/ops.c
index 2d684f8e9030..912fb6677a0d 100644
--- a/sys/contrib/dev/iwlwifi/mvm/ops.c
+++ b/sys/contrib/dev/iwlwifi/mvm/ops.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -43,7 +43,7 @@ MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION(DRV_DESCRIPTION);
 MODULE_LICENSE("BSD");
 #endif
-MODULE_IMPORT_NS(IWLWIFI);
+MODULE_IMPORT_NS("IWLWIFI");
 
 static const struct iwl_op_mode_ops iwl_mvm_ops;
 static const struct iwl_op_mode_ops iwl_mvm_ops_mq;
@@ -74,8 +74,10 @@ static int __init iwl_mvm_init(void)
 	}
 
 	ret = iwl_opmode_register("iwlmvm", &iwl_mvm_ops);
-	if (ret)
+	if (ret) {
 		pr_err("Unable to register MVM op_mode: %d\n", ret);
+		iwl_mvm_rate_control_unregister();
+	}
 
 	return ret;
 }
@@ -109,11 +111,11 @@ static void iwl_mvm_nic_config(struct iwl_op_mode *op_mode)
 	IWL_DEBUG_INFO(mvm, "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
 		       radio_cfg_step, radio_cfg_dash);
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		return;
 
 	/* SKU control */
-	reg_val = CSR_HW_REV_STEP_DASH(mvm->trans->hw_rev);
+	reg_val = CSR_HW_REV_STEP_DASH(mvm->trans->info.hw_rev);
 
 	/* radio configuration */
 	reg_val |= radio_cfg_type << CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
@@ -131,7 +133,7 @@ static void iwl_mvm_nic_config(struct iwl_op_mode *op_mode)
 	 * unrelated errors. Need to further investigate this, but for now
 	 * we'll separate cases.
 	 */
-	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
+	if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_8000)
 		reg_val |= CSR_HW_IF_CONFIG_REG_BIT_RADIO_SI;
 
 	if (iwl_fw_dbg_is_d3_debug_enabled(&mvm->fwrt))
@@ -152,7 +154,7 @@ static void iwl_mvm_nic_config(struct iwl_op_mode *op_mode)
 	 * (PCIe power is lost before PERST# is asserted), causing ME FW
 	 * to lose ownership and not being able to obtain it back.
 	 */
-	if (!mvm->trans->cfg->apmg_not_supported)
+	if (!mvm->trans->mac_cfg->base->apmg_not_supported)
 		iwl_set_bits_mask_prph(mvm->trans, APMG_PS_CTRL_REG,
 				       APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
 				       ~APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
@@ -162,7 +164,7 @@ static void iwl_mvm_rx_esr_mode_notif(struct iwl_mvm *mvm,
 				      struct iwl_rx_cmd_buffer *rxb)
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_mvm_esr_mode_notif *notif = (void *)pkt->data;
+	struct iwl_esr_mode_notif *notif = (void *)pkt->data;
 	struct ieee80211_vif *vif = iwl_mvm_get_bss_vif(mvm);
 
 	/* FW recommendations is only for entering EMLSR */
@@ -188,7 +190,8 @@ static void iwl_mvm_rx_monitor_notif(struct iwl_mvm *mvm,
 	if (notif->type != cpu_to_le32(IWL_DP_MON_NOTIF_TYPE_EXT_CCA))
 		return;
 
-	vif = iwl_mvm_get_vif_by_macid(mvm, notif->mac_id);
+	/* FIXME: should fetch the link and not the vif */
+	vif = iwl_mvm_get_vif_by_macid(mvm, notif->link_id);
 	if (!vif || vif->type != NL80211_IFTYPE_STATION)
 		return;
 
@@ -278,6 +281,12 @@ static void iwl_mvm_rx_thermal_dual_chain_req(struct iwl_mvm *mvm,
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_thermal_dual_chain_request *req = (void *)pkt->data;
 
+	/* firmware is expected to handle that in RLC offload mode */
+	if (IWL_FW_CHECK(mvm, iwl_mvm_has_rlc_offload(mvm),
+			 "Got THERMAL_DUAL_CHAIN_REQUEST (0x%x) in RLC offload mode\n",
+			 req->event))
+		return;
+
 	/*
 	 * We could pass it to the iterator data, but also need to remember
 	 * it for new interfaces that are added while in this state.
@@ -342,7 +351,7 @@ struct iwl_rx_handlers {
  */
 static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 	RX_HANDLER(TX_CMD, iwl_mvm_rx_tx_cmd, RX_HANDLER_SYNC,
-		   struct iwl_mvm_tx_resp),
+		   struct iwl_tx_resp),
 	RX_HANDLER(BA_NOTIF, iwl_mvm_rx_ba_notif, RX_HANDLER_SYNC,
 		   struct iwl_mvm_ba_notif),
 
@@ -350,9 +359,12 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 		       iwl_mvm_tlc_update_notif, RX_HANDLER_SYNC,
 		       struct iwl_tlc_update_notif),
 
-	RX_HANDLER(BT_PROFILE_NOTIFICATION, iwl_mvm_rx_bt_coex_notif,
+	RX_HANDLER(BT_PROFILE_NOTIFICATION, iwl_mvm_rx_bt_coex_old_notif,
 		   RX_HANDLER_ASYNC_LOCKED_WIPHY,
-		   struct iwl_bt_coex_profile_notif),
+		   struct iwl_bt_coex_prof_old_notif),
+	RX_HANDLER_GRP(BT_COEX_GROUP, PROFILE_NOTIF, iwl_mvm_rx_bt_coex_notif,
+		       RX_HANDLER_ASYNC_LOCKED_WIPHY,
+		       struct iwl_bt_coex_profile_notif),
 	RX_HANDLER_NO_SIZE(BEACON_NOTIFICATION, iwl_mvm_rx_beacon_notif,
 			   RX_HANDLER_ASYNC_LOCKED),
 	RX_HANDLER_NO_SIZE(STATISTICS_NOTIFICATION, iwl_mvm_rx_statistics,
@@ -379,7 +391,7 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 		   RX_HANDLER_SYNC, struct iwl_time_event_notif),
 	RX_HANDLER_GRP(MAC_CONF_GROUP, SESSION_PROTECTION_NOTIF,
 		       iwl_mvm_rx_session_protect_notif, RX_HANDLER_SYNC,
-		       struct iwl_mvm_session_prot_notif),
+		       struct iwl_session_prot_notif),
 	RX_HANDLER(MCC_CHUB_UPDATE_CMD, iwl_mvm_rx_chub_update_mcc,
 		   RX_HANDLER_ASYNC_LOCKED, struct iwl_mcc_chub_notif),
 
@@ -402,10 +414,15 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 		   iwl_mvm_rx_umac_scan_iter_complete_notif, RX_HANDLER_SYNC,
 		   struct iwl_umac_scan_iter_complete_notif),
 
-	RX_HANDLER(MISSED_BEACONS_NOTIFICATION, iwl_mvm_rx_missed_beacons_notif,
+	RX_HANDLER(MISSED_BEACONS_NOTIFICATION,
+		   iwl_mvm_rx_missed_beacons_notif_legacy,
 		   RX_HANDLER_ASYNC_LOCKED_WIPHY,
-		   struct iwl_missed_beacons_notif),
+		   struct iwl_missed_beacons_notif_v4),
 
+	RX_HANDLER_GRP(MAC_CONF_GROUP, MISSED_BEACONS_NOTIF,
+		       iwl_mvm_rx_missed_beacons_notif,
+		       RX_HANDLER_ASYNC_LOCKED_WIPHY,
+		       struct iwl_missed_beacons_notif),
 	RX_HANDLER(REPLY_ERROR, iwl_mvm_rx_fw_error, RX_HANDLER_SYNC,
 		   struct iwl_error_resp),
 	RX_HANDLER(PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION,
@@ -460,7 +477,7 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 	RX_HANDLER_GRP(DATA_PATH_GROUP, ESR_MODE_NOTIF,
 		       iwl_mvm_rx_esr_mode_notif,
 		       RX_HANDLER_ASYNC_LOCKED_WIPHY,
-		       struct iwl_mvm_esr_mode_notif),
+		       struct iwl_esr_mode_notif),
 
 	RX_HANDLER_GRP(DATA_PATH_GROUP, MONITOR_NOTIF,
 		       iwl_mvm_rx_monitor_notif, RX_HANDLER_ASYNC_LOCKED,
@@ -489,6 +506,11 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 	RX_HANDLER_GRP(SCAN_GROUP, CHANNEL_SURVEY_NOTIF,
 		       iwl_mvm_rx_channel_survey_notif, RX_HANDLER_ASYNC_LOCKED,
 		       struct iwl_umac_scan_channel_survey_notif),
+	RX_HANDLER_GRP(DATA_PATH_GROUP, BEACON_FILTER_IN_NOTIF,
+		       iwl_mvm_rx_beacon_filter_notif,
+		       RX_HANDLER_ASYNC_LOCKED,
+		       /* same size as v1 */
+		       struct iwl_beacon_filter_notif),
 };
 #undef RX_HANDLER
 #undef RX_HANDLER_GRP
@@ -659,6 +681,7 @@ static const struct iwl_hcmd_names iwl_mvm_data_path_names[] = {
 	HCMD_NAME(ESR_MODE_NOTIF),
 	HCMD_NAME(MONITOR_NOTIF),
 	HCMD_NAME(THERMAL_DUAL_CHAIN_REQUEST),
+	HCMD_NAME(BEACON_FILTER_IN_NOTIF),
 	HCMD_NAME(STA_PM_NOTIF),
 	HCMD_NAME(MU_GROUP_MGMT_NOTIF),
 	HCMD_NAME(RX_QUEUES_NOTIFICATION),
@@ -730,7 +753,15 @@ static const struct iwl_hcmd_names iwl_mvm_regulatory_and_nvm_names[] = {
 	HCMD_NAME(TAS_CONFIG),
 };
 
-static const struct iwl_hcmd_arr iwl_mvm_groups[] = {
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
+static const struct iwl_hcmd_names iwl_mvm_bt_coex_names[] = {
+	HCMD_NAME(PROFILE_NOTIF),
+};
+
+VISIBLE_IF_IWLWIFI_KUNIT
+const struct iwl_hcmd_arr iwl_mvm_groups[] = {
 	[LEGACY_GROUP] = HCMD_ARR(iwl_mvm_legacy_names),
 	[LONG_GROUP] = HCMD_ARR(iwl_mvm_legacy_names),
 	[SYSTEM_GROUP] = HCMD_ARR(iwl_mvm_system_names),
@@ -739,12 +770,18 @@ static const struct iwl_hcmd_arr iwl_mvm_groups[] = {
 	[DATA_PATH_GROUP] = HCMD_ARR(iwl_mvm_data_path_names),
 	[SCAN_GROUP] = HCMD_ARR(iwl_mvm_scan_names),
 	[LOCATION_GROUP] = HCMD_ARR(iwl_mvm_location_names),
+	[BT_COEX_GROUP] = HCMD_ARR(iwl_mvm_bt_coex_names),
 	[PROT_OFFLOAD_GROUP] = HCMD_ARR(iwl_mvm_prot_offload_names),
 	[REGULATORY_AND_NVM_GROUP] =
 		HCMD_ARR(iwl_mvm_regulatory_and_nvm_names),
 	[DEBUG_GROUP] = HCMD_ARR(iwl_mvm_debug_names),
 	[STATISTICS_GROUP] = HCMD_ARR(iwl_mvm_statistics_names),
 };
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mvm_groups);
+#if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
+const unsigned int iwl_mvm_groups_size = ARRAY_SIZE(iwl_mvm_groups);
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mvm_groups_size);
+#endif
 
 /* this forward declaration can avoid to export the function */
 static void iwl_mvm_async_handlers_wk(struct work_struct *wk);
@@ -1228,13 +1265,12 @@ static void iwl_mvm_trig_link_selection(struct wiphy *wiphy,
 }
 
 static struct iwl_op_mode *
-iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
+iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 		      const struct iwl_fw *fw, struct dentry *dbgfs_dir)
 {
 	struct ieee80211_hw *hw;
 	struct iwl_op_mode *op_mode;
 	struct iwl_mvm *mvm;
-	struct iwl_trans_config trans_cfg = {};
 	static const u8 no_reclaim_cmds[] = {
 		TX_CMD,
 	};
@@ -1242,14 +1278,16 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	size_t scan_size;
 	u32 min_backoff;
 	struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused;
+	int ratecheck;
+	int err;
 
 	/*
-	 * We use IWL_MVM_STATION_COUNT_MAX to check the validity of the station
+	 * We use IWL_STATION_COUNT_MAX to check the validity of the station
 	 * index all over the driver - check that its value corresponds to the
 	 * array size.
 	 */
 	BUILD_BUG_ON(ARRAY_SIZE(mvm->fw_id_to_mac_id) !=
-		     IWL_MVM_STATION_COUNT_MAX);
+		     IWL_STATION_COUNT_MAX);
 
 	/********************************
 	 * 1. Allocating and configuring HW data
@@ -1259,20 +1297,15 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 				iwl_mvm_has_mld_api(fw) ? &iwl_mvm_mld_hw_ops :
 				&iwl_mvm_hw_ops);
 	if (!hw)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		max_agg = 512;
 	else
 		max_agg = IEEE80211_MAX_AMPDU_BUF_HE;
 
 	hw->max_rx_aggregation_subframes = max_agg;
 
-	if (cfg->max_tx_agg_size)
-		hw->max_tx_aggregation_subframes = cfg->max_tx_agg_size;
-	else
-		hw->max_tx_aggregation_subframes = max_agg;
-
 	op_mode = hw->priv;
 
 	mvm = IWL_OP_MODE_GET_MVM(op_mode);
@@ -1288,27 +1321,67 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	iwl_mvm_get_bios_tables(mvm);
 	iwl_uefi_get_sgom_table(trans, &mvm->fwrt);
 	iwl_uefi_get_step_table(trans);
+	iwl_bios_setup_step(trans, &mvm->fwrt);
 
 	mvm->init_status = 0;
 
+	/* start with v1 rates */
+	mvm->fw_rates_ver = 1;
+
+	/* check for rates version 2 */
+	ratecheck =
+		(iwl_fw_lookup_cmd_ver(mvm->fw, TX_CMD, 0) >= 8) +
+		(iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP,
+					 TLC_MNG_UPDATE_NOTIF, 0) >= 3) +
+		(iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
+					 REPLY_RX_MPDU_CMD, 0) >= 4) +
+		(iwl_fw_lookup_notif_ver(mvm->fw, LONG_GROUP, TX_CMD, 0) >= 6);
+	if (ratecheck != 0 && ratecheck != 4) {
+		IWL_ERR(mvm, "Firmware has inconsistent rates\n");
+		err = -EINVAL;
+		goto out_free;
+	}
+	if (ratecheck == 4)
+		mvm->fw_rates_ver = 2;
+
+	/* check for rates version 3 */
+	ratecheck =
+		(iwl_fw_lookup_cmd_ver(mvm->fw, TX_CMD, 0) >= 11) +
+		(iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP,
+					 TLC_MNG_UPDATE_NOTIF, 0) >= 4) +
+		(iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
+					 REPLY_RX_MPDU_CMD, 0) >= 6) +
+		(iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP,
+					 RX_NO_DATA_NOTIF, 0) >= 4) +
+		(iwl_fw_lookup_notif_ver(mvm->fw, LONG_GROUP, TX_CMD, 0) >= 9);
+	if (ratecheck != 0 && ratecheck != 5) {
+		IWL_ERR(mvm, "Firmware has inconsistent rates\n");
+		err = -EINVAL;
+		goto out_free;
+	}
+	if (ratecheck == 5)
+		mvm->fw_rates_ver = 3;
+
+	trans->conf.rx_mpdu_cmd = REPLY_RX_MPDU_CMD;
+
 	if (iwl_mvm_has_new_rx_api(mvm)) {
 		op_mode->ops = &iwl_mvm_ops_mq;
-		trans->rx_mpdu_cmd_hdr_size =
-			(trans->trans_cfg->device_family >=
+		trans->conf.rx_mpdu_cmd_hdr_size =
+			(trans->mac_cfg->device_family >=
 			 IWL_DEVICE_FAMILY_AX210) ?
 			sizeof(struct iwl_rx_mpdu_desc) :
 			IWL_RX_DESC_SIZE_V1;
 	} else {
 		op_mode->ops = &iwl_mvm_ops;
-		trans->rx_mpdu_cmd_hdr_size =
+		trans->conf.rx_mpdu_cmd_hdr_size =
 			sizeof(struct iwl_rx_mpdu_res_start);
 
-		if (WARN_ON(trans->num_rx_queues > 1))
+		if (WARN_ON(trans->info.num_rxqs > 1)) {
+			err = -EINVAL;
 			goto out_free;
+		}
 	}
 
-	mvm->fw_restart = iwlwifi_mod_params.fw_restart ? -1 : 0;
-
 	if (iwl_mvm_has_new_tx_api(mvm)) {
 		/*
 		 * If we have the new TX/queue allocation API initialize them
@@ -1380,66 +1453,58 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 		iwl_fw_lookup_notif_ver(mvm->fw, LOCATION_GROUP,
 					TOF_RANGE_RESPONSE_NOTIF, 5);
 	/* we only support up to version 9 */
-	if (WARN_ON_ONCE(mvm->cmd_ver.range_resp > 9))
+	if (WARN_ON_ONCE(mvm->cmd_ver.range_resp > 9)) {
+		err = -EINVAL;
 		goto out_free;
+	}
 
 	/*
 	 * Populate the state variables that the transport layer needs
 	 * to know about.
 	 */
-	trans_cfg.op_mode = op_mode;
-	trans_cfg.no_reclaim_cmds = no_reclaim_cmds;
-	trans_cfg.n_no_reclaim_cmds = ARRAY_SIZE(no_reclaim_cmds);
-
-	trans_cfg.rx_buf_size = iwl_amsdu_size_to_rxb_size();
+	BUILD_BUG_ON(sizeof(no_reclaim_cmds) >
+		     sizeof(trans->conf.no_reclaim_cmds));
+	memcpy(trans->conf.no_reclaim_cmds, no_reclaim_cmds,
+	       sizeof(no_reclaim_cmds));
+	trans->conf.n_no_reclaim_cmds = ARRAY_SIZE(no_reclaim_cmds);
 
-	trans->wide_cmd_header = true;
-	trans_cfg.bc_table_dword =
-		mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210;
+	trans->conf.rx_buf_size = iwl_amsdu_size_to_rxb_size();
 
-	trans_cfg.command_groups = iwl_mvm_groups;
-	trans_cfg.command_groups_size = ARRAY_SIZE(iwl_mvm_groups);
+	trans->conf.wide_cmd_header = true;
 
-	trans_cfg.cmd_queue = IWL_MVM_DQA_CMD_QUEUE;
-	trans_cfg.cmd_fifo = IWL_MVM_TX_FIFO_CMD;
-	trans_cfg.scd_set_active = true;
+	trans->conf.command_groups = iwl_mvm_groups;
+	trans->conf.command_groups_size = ARRAY_SIZE(iwl_mvm_groups);
 
-	trans_cfg.cb_data_offs = offsetof(struct ieee80211_tx_info,
-					  driver_data[2]);
+	trans->conf.cmd_queue = IWL_MVM_DQA_CMD_QUEUE;
+	trans->conf.cmd_fifo = IWL_MVM_TX_FIFO_CMD;
+	trans->conf.scd_set_active = true;
 
-	/* Set a short watchdog for the command queue */
-	trans_cfg.cmd_q_wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, NULL, false, true);
+	trans->conf.cb_data_offs = offsetof(struct ieee80211_tx_info,
+					    driver_data[2]);
 
 	snprintf(mvm->hw->wiphy->fw_version,
 		 sizeof(mvm->hw->wiphy->fw_version),
 		 "%.31s", fw->fw_version);
 
-	trans_cfg.fw_reset_handshake = fw_has_capa(&mvm->fw->ucode_capa,
-						   IWL_UCODE_TLV_CAPA_FW_RESET_HANDSHAKE);
+	trans->conf.fw_reset_handshake =
+		fw_has_capa(&mvm->fw->ucode_capa,
+			    IWL_UCODE_TLV_CAPA_FW_RESET_HANDSHAKE);
 
-	trans_cfg.queue_alloc_cmd_ver =
+	trans->conf.queue_alloc_cmd_ver =
 		iwl_fw_lookup_cmd_ver(mvm->fw,
 				      WIDE_ID(DATA_PATH_GROUP,
 					      SCD_QUEUE_CONFIG_CMD),
 				      0);
 	mvm->sta_remove_requires_queue_remove =
-		trans_cfg.queue_alloc_cmd_ver > 0;
+		trans->conf.queue_alloc_cmd_ver > 0;
 
 	mvm->mld_api_is_used = iwl_mvm_has_mld_api(mvm->fw);
 
 	/* Configure transport layer */
-	iwl_trans_configure(mvm->trans, &trans_cfg);
+	iwl_trans_op_mode_enter(mvm->trans, op_mode);
 
-	trans->rx_mpdu_cmd = REPLY_RX_MPDU_CMD;
 	trans->dbg.dest_tlv = mvm->fw->dbg.dest_tlv;
 	trans->dbg.n_dest_reg = mvm->fw->dbg.n_dest_reg;
-	memcpy(trans->dbg.conf_tlv, mvm->fw->dbg.conf_tlv,
-	       sizeof(trans->dbg.conf_tlv));
-	trans->dbg.trigger_tlv = mvm->fw->dbg.trigger_tlv;
-
-	trans->iml = mvm->fw->iml;
-	trans->iml_len = mvm->fw->iml_len;
 
 	/* set up notification wait support */
 	iwl_notification_wait_init(&mvm->notif_wait);
@@ -1448,6 +1513,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	mvm->phy_db = iwl_phy_db_init(trans);
 	if (!mvm->phy_db) {
 		IWL_ERR(mvm, "Cannot init phy_db\n");
+		err = -ENOMEM;
 		goto out_free;
 	}
 
@@ -1460,13 +1526,15 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	scan_size = iwl_mvm_scan_size(mvm);
 
 	mvm->scan_cmd = kmalloc(scan_size, GFP_KERNEL);
-	if (!mvm->scan_cmd)
+	if (!mvm->scan_cmd) {
+		err = -ENOMEM;
 		goto out_free;
+	}
 	mvm->scan_cmd_size = scan_size;
 
 	/* invalidate ids to prevent accidental removal of sta_id 0 */
-	mvm->aux_sta.sta_id = IWL_MVM_INVALID_STA;
-	mvm->snif_sta.sta_id = IWL_MVM_INVALID_STA;
+	mvm->aux_sta.sta_id = IWL_INVALID_STA;
+	mvm->snif_sta.sta_id = IWL_INVALID_STA;
 
 	/* Set EBS as successful as long as not stated otherwise by the FW. */
 	mvm->last_ebs_successful = true;
@@ -1494,7 +1562,8 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 
 	iwl_mvm_mei_scan_filter_init(&mvm->mei_scan_filter);
 
-	if (iwl_mvm_start_get_nvm(mvm)) {
+	err = iwl_mvm_start_get_nvm(mvm);
+	if (err) {
 		/*
 		 * Getting NVM failed while CSME is the owner, but we are
 		 * registered to MEI, we'll get the NVM later when it'll be
@@ -1507,7 +1576,8 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	}
 
 
-	if (iwl_mvm_start_post_nvm(mvm))
+	err = iwl_mvm_start_post_nvm(mvm);
+	if (err)
 		goto out_thermal_exit;
 
 	return op_mode;
@@ -1527,7 +1597,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	iwl_trans_op_mode_leave(trans);
 
 	ieee80211_free_hw(mvm->hw);
-	return NULL;
+	return ERR_PTR(err);
 }
 
 void iwl_mvm_stop_device(struct iwl_mvm *mvm)
@@ -1962,27 +2032,62 @@ static void iwl_mvm_free_skb(struct iwl_op_mode *op_mode, struct sk_buff *skb)
 	ieee80211_free_txskb(mvm->hw, skb);
 }
 
-struct iwl_mvm_reprobe {
-	struct device *dev;
-	struct work_struct work;
-};
+static void iwl_mvm_nic_error(struct iwl_op_mode *op_mode,
+			      enum iwl_fw_error_type type)
+{
+	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
+
+	iwl_abort_notification_waits(&mvm->notif_wait);
+	iwl_dbg_tlv_del_timers(mvm->trans);
+
+	if (type == IWL_ERR_TYPE_CMD_QUEUE_FULL)
+		IWL_ERR(mvm, "Command queue full!\n");
+	else if (!iwl_trans_is_dead(mvm->trans) &&
+		 !test_and_clear_bit(IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE,
+				     &mvm->status))
+		iwl_mvm_dump_nic_error_log(mvm);
+
+	/*
+	 * This should be first thing before trying to collect any
+	 * data to avoid endless loops if any HW error happens while
+	 * collecting debug data.
+	 * It might not actually be true that we'll restart, but the
+	 * setting of the bit doesn't matter if we're going to be
+	 * unbound either.
+	 */
+	if (type != IWL_ERR_TYPE_RESET_HS_TIMEOUT)
+		set_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status);
+}
 
-static void iwl_mvm_reprobe_wk(struct work_struct *wk)
+static void iwl_mvm_dump_error(struct iwl_op_mode *op_mode,
+			       struct iwl_fw_error_dump_mode *mode)
 {
-	struct iwl_mvm_reprobe *reprobe;
-
-	reprobe = container_of(wk, struct iwl_mvm_reprobe, work);
-	if (device_reprobe(reprobe->dev))
-		dev_err(reprobe->dev, "reprobe failed!\n");
-	put_device(reprobe->dev);
-	kfree(reprobe);
-	module_put(THIS_MODULE);
+	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
+
+	/* if we come in from opmode we have the mutex held */
+	if (mode->context == IWL_ERR_CONTEXT_FROM_OPMODE) {
+		lockdep_assert_held(&mvm->mutex);
+		iwl_fw_error_collect(&mvm->fwrt);
+	} else {
+		mutex_lock(&mvm->mutex);
+		if (mode->context != IWL_ERR_CONTEXT_ABORT)
+			iwl_fw_error_collect(&mvm->fwrt);
+		mutex_unlock(&mvm->mutex);
+	}
 }
 
-void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
+static bool iwl_mvm_sw_reset(struct iwl_op_mode *op_mode,
+			     enum iwl_fw_error_type type)
 {
-	iwl_abort_notification_waits(&mvm->notif_wait);
-	iwl_dbg_tlv_del_timers(mvm->trans);
+	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
+
+	/*
+	 * If the firmware crashes while we're already considering it
+	 * to be dead then don't ask for a restart, that cannot do
+	 * anything useful anyway.
+	 */
+	if (!test_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status))
+		return false;
 
 	/*
 	 * This is a bit racy, but worst case we tell mac80211 about
@@ -1997,52 +2102,11 @@ void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
 	iwl_mvm_report_scan_aborted(mvm);
 
 	/*
-	 * If we're restarting already, don't cycle restarts.
 	 * If INIT fw asserted, it will likely fail again.
 	 * If WoWLAN fw asserted, don't restart either, mac80211
 	 * can't recover this since we're already half suspended.
 	 */
-	if (!mvm->fw_restart && fw_error) {
-		iwl_fw_error_collect(&mvm->fwrt, false);
-	} else if (test_bit(IWL_MVM_STATUS_STARTING,
-			    &mvm->status)) {
-		IWL_ERR(mvm, "Starting mac, retry will be triggered anyway\n");
-	} else if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
-		struct iwl_mvm_reprobe *reprobe;
-
-		IWL_ERR(mvm,
-			"Firmware error during reconfiguration - reprobe!\n");
-
-		/*
-		 * get a module reference to avoid doing this while unloading
-		 * anyway and to avoid scheduling a work with code that's
-		 * being removed.
-		 */
-		if (!try_module_get(THIS_MODULE)) {
-			IWL_ERR(mvm, "Module is being unloaded - abort\n");
-			return;
-		}
-
-		reprobe = kzalloc(sizeof(*reprobe), GFP_ATOMIC);
-		if (!reprobe) {
-			module_put(THIS_MODULE);
-			return;
-		}
-		reprobe->dev = get_device(mvm->trans->dev);
-		INIT_WORK(&reprobe->work, iwl_mvm_reprobe_wk);
-		schedule_work(&reprobe->work);
-	} else if (test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED,
-			    &mvm->status)) {
-		IWL_ERR(mvm, "HW restart already requested, but not started\n");
-	} else if (mvm->fwrt.cur_fw_img == IWL_UCODE_REGULAR &&
-		   mvm->hw_registered &&
-		   !test_bit(STATUS_TRANS_DEAD, &mvm->trans->status)) {
-		/* This should be first thing before trying to collect any
-		 * data to avoid endless loops if any HW error happens while
-		 * collecting debug data.
-		 */
-		set_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status);
-
+	if (mvm->fwrt.cur_fw_img == IWL_UCODE_REGULAR && mvm->hw_registered) {
 		if (mvm->fw->ucode_capa.error_log_size) {
 			u32 src_size = mvm->fw->ucode_capa.error_log_size;
 			u32 src_addr = mvm->fw->ucode_capa.error_log_addr;
@@ -2057,81 +2121,55 @@ void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
 			}
 		}
 
-		iwl_fw_error_collect(&mvm->fwrt, false);
-
-		if (fw_error && mvm->fw_restart > 0) {
-			mvm->fw_restart--;
-			ieee80211_restart_hw(mvm->hw);
-		} else if (mvm->fwrt.trans->dbg.restart_required) {
+		if (mvm->fwrt.trans->dbg.restart_required) {
 			IWL_DEBUG_INFO(mvm, "FW restart requested after debug collection\n");
 			mvm->fwrt.trans->dbg.restart_required = false;
 			ieee80211_restart_hw(mvm->hw);
-		} else if (mvm->trans->trans_cfg->device_family <= IWL_DEVICE_FAMILY_8000) {
+			return true;
+		} else if (mvm->trans->mac_cfg->device_family <= IWL_DEVICE_FAMILY_8000) {
 			ieee80211_restart_hw(mvm->hw);
+			return true;
 		}
 	}
-}
-
-static void iwl_mvm_nic_error(struct iwl_op_mode *op_mode, bool sync)
-{
-	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
-
-	if (!test_bit(STATUS_TRANS_DEAD, &mvm->trans->status) &&
-	    !test_and_clear_bit(IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE,
-				&mvm->status))
-		iwl_mvm_dump_nic_error_log(mvm);
-
-	if (sync) {
-		iwl_fw_error_collect(&mvm->fwrt, true);
-		/*
-		 * Currently, the only case for sync=true is during
-		 * shutdown, so just stop in this case. If/when that
-		 * changes, we need to be a bit smarter here.
-		 */
-		return;
-	}
 
-	/*
-	 * If the firmware crashes while we're already considering it
-	 * to be dead then don't ask for a restart, that cannot do
-	 * anything useful anyway.
-	 */
-	if (!test_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status))
-		return;
-
-	iwl_mvm_nic_restart(mvm, false);
+	return false;
 }
 
-static void iwl_mvm_cmd_queue_full(struct iwl_op_mode *op_mode)
+static void iwl_op_mode_mvm_time_point(struct iwl_op_mode *op_mode,
+				       enum iwl_fw_ini_time_point tp_id,
+				       union iwl_dbg_tlv_tp_data *tp_data)
 {
 	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
 
-	WARN_ON(1);
-	iwl_mvm_nic_restart(mvm, true);
+	iwl_dbg_tlv_time_point(&mvm->fwrt, tp_id, tp_data);
 }
 
-static void iwl_op_mode_mvm_time_point(struct iwl_op_mode *op_mode,
-				       enum iwl_fw_ini_time_point tp_id,
-				       union iwl_dbg_tlv_tp_data *tp_data)
+static void iwl_mvm_dump(struct iwl_op_mode *op_mode)
 {
 	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
+	struct iwl_fw_runtime *fwrt = &mvm->fwrt;
 
-	iwl_dbg_tlv_time_point(&mvm->fwrt, tp_id, tp_data);
+	if (!iwl_trans_fw_running(fwrt->trans))
+		return;
+
+	iwl_dbg_tlv_time_point(fwrt, IWL_FW_INI_TIME_POINT_USER_TRIGGER, NULL);
 }
 
+#ifdef CONFIG_PM_SLEEP
 static void iwl_op_mode_mvm_device_powered_off(struct iwl_op_mode *op_mode)
 {
 	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
 
 	mutex_lock(&mvm->mutex);
 	clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status);
-	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
 	iwl_mvm_stop_device(mvm);
-#ifdef CONFIG_PM
 	mvm->fast_resume = false;
-#endif
 	mutex_unlock(&mvm->mutex);
 }
+#else
+static void iwl_op_mode_mvm_device_powered_off(struct iwl_op_mode *op_mode)
+{}
+#endif
 
 #define IWL_MVM_COMMON_OPS					\
 	/* these could be differentiated */			\
@@ -2140,7 +2178,8 @@ static void iwl_op_mode_mvm_device_powered_off(struct iwl_op_mode *op_mode)
 	.hw_rf_kill = iwl_mvm_set_hw_rfkill_state,		\
 	.free_skb = iwl_mvm_free_skb,				\
 	.nic_error = iwl_mvm_nic_error,				\
-	.cmd_queue_full = iwl_mvm_cmd_queue_full,		\
+	.dump_error = iwl_mvm_dump_error,			\
+	.sw_reset = iwl_mvm_sw_reset,				\
 	.nic_config = iwl_mvm_nic_config,			\
 	/* as we only register one, these MUST be common! */	\
 	.start = iwl_op_mode_mvm_start,				\
@@ -2162,7 +2201,7 @@ static void iwl_mvm_rx_mq_rss(struct iwl_op_mode *op_mode,
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
 
-	if (unlikely(queue >= mvm->trans->num_rx_queues))
+	if (unlikely(queue >= mvm->trans->info.num_rxqs))
 		return;
 
 	if (unlikely(cmd == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE)))
@@ -2178,4 +2217,5 @@ static const struct iwl_op_mode_ops iwl_mvm_ops_mq = {
 	IWL_MVM_COMMON_OPS,
 	.rx = iwl_mvm_rx_mq,
 	.rx_rss = iwl_mvm_rx_mq_rss,
+	.dump = iwl_mvm_dump,
 };
diff --git a/sys/contrib/dev/iwlwifi/mvm/phy-ctxt.c b/sys/contrib/dev/iwlwifi/mvm/phy-ctxt.c
index ce264b386029..5e7e2926be0c 100644
--- a/sys/contrib/dev/iwlwifi/mvm/phy-ctxt.c
+++ b/sys/contrib/dev/iwlwifi/mvm/phy-ctxt.c
@@ -31,7 +31,7 @@ u8 iwl_mvm_get_channel_width(const struct cfg80211_chan_def *chandef)
 
 /*
  * Maps the driver specific control channel position (relative to the center
- * freq) definitions to the the fw values
+ * freq) definitions to the fw values
  */
 u8 iwl_mvm_get_ctrl_pos(const struct cfg80211_chan_def *chandef)
 {
@@ -159,7 +159,11 @@ int iwl_mvm_phy_send_rlc(struct iwl_mvm *mvm, struct iwl_mvm_phy_ctxt *ctxt,
 		.phy_id = cpu_to_le32(ctxt->id),
 	};
 
-	if (ctxt->rlc_disabled)
+	/* From version 3, RLC is offloaded to firmware, so the driver no
+	 * longer needs to send cmd.rlc, note that we are not using any
+	 * other fields in the command - don't send it.
+	 */
+	if (iwl_mvm_has_rlc_offload(mvm) || ctxt->rlc_disabled)
 		return 0;
 
 	if (iwl_fw_lookup_cmd_ver(mvm->fw, WIDE_ID(DATA_PATH_GROUP,
diff --git a/sys/contrib/dev/iwlwifi/mvm/power.c b/sys/contrib/dev/iwlwifi/mvm/power.c
index bc363e8427e4..610de29b7be0 100644
--- a/sys/contrib/dev/iwlwifi/mvm/power.c
+++ b/sys/contrib/dev/iwlwifi/mvm/power.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2019, 2021-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2021-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -231,7 +231,6 @@ static void iwl_mvm_allow_uapsd_iterator(void *_data, u8 *mac,
 	switch (vif->type) {
 	case NL80211_IFTYPE_AP:
 	case NL80211_IFTYPE_ADHOC:
-	case NL80211_IFTYPE_NAN:
 		data->allow_uapsd = false;
 		break;
 	case NL80211_IFTYPE_STATION:
@@ -376,6 +375,9 @@ static void iwl_mvm_power_build_cmd(struct iwl_mvm *mvm,
 	if (!vif->cfg.ps || !mvmvif->pm_enabled)
 		return;
 
+	if (iwl_fw_lookup_cmd_ver(mvm->fw, MAC_PM_POWER_TABLE, 0) >= 2)
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_ENABLE_SMPS_MSK);
+
 	if (iwl_mvm_vif_low_latency(mvmvif) && vif->p2p &&
 	    (!fw_has_capa(&mvm->fw->ucode_capa,
 			 IWL_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS) ||
@@ -567,7 +569,7 @@ struct iwl_power_vifs {
 	bool monitor_active;
 };
 
-static void iwl_mvm_power_disable_pm_iterator(void *_data, u8* mac,
+static void iwl_mvm_power_disable_pm_iterator(void *_data, u8 *mac,
 					      struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
@@ -575,7 +577,7 @@ static void iwl_mvm_power_disable_pm_iterator(void *_data, u8* mac,
 	mvmvif->pm_enabled = false;
 }
 
-static void iwl_mvm_power_ps_disabled_iterator(void *_data, u8* mac,
+static void iwl_mvm_power_ps_disabled_iterator(void *_data, u8 *mac,
 					       struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
diff --git a/sys/contrib/dev/iwlwifi/mvm/ptp.c b/sys/contrib/dev/iwlwifi/mvm/ptp.c
index e89259de6f4c..06a4c9f74797 100644
--- a/sys/contrib/dev/iwlwifi/mvm/ptp.c
+++ b/sys/contrib/dev/iwlwifi/mvm/ptp.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2021 - 2023 Intel Corporation
+ * Copyright (C) 2021 - 2023, 2025 Intel Corporation
  */
 
 #include "mvm.h"
@@ -298,9 +298,9 @@ void iwl_mvm_ptp_init(struct iwl_mvm *mvm)
 			PTR_ERR(mvm->ptp_data.ptp_clock));
 		mvm->ptp_data.ptp_clock = NULL;
 	} else if (mvm->ptp_data.ptp_clock) {
-		IWL_INFO(mvm, "Registered PHC clock: %s, with index: %d\n",
-			 mvm->ptp_data.ptp_clock_info.name,
-			 ptp_clock_index(mvm->ptp_data.ptp_clock));
+		IWL_DEBUG_INFO(mvm, "Registered PHC clock: %s, with index: %d\n",
+			       mvm->ptp_data.ptp_clock_info.name,
+			       ptp_clock_index(mvm->ptp_data.ptp_clock));
 	}
 }
 
@@ -312,9 +312,9 @@ void iwl_mvm_ptp_init(struct iwl_mvm *mvm)
 void iwl_mvm_ptp_remove(struct iwl_mvm *mvm)
 {
 	if (mvm->ptp_data.ptp_clock) {
-		IWL_INFO(mvm, "Unregistering PHC clock: %s, with index: %d\n",
-			 mvm->ptp_data.ptp_clock_info.name,
-			 ptp_clock_index(mvm->ptp_data.ptp_clock));
+		IWL_DEBUG_INFO(mvm, "Unregistering PHC clock: %s, with index: %d\n",
+			       mvm->ptp_data.ptp_clock_info.name,
+			       ptp_clock_index(mvm->ptp_data.ptp_clock));
 
 		ptp_clock_unregister(mvm->ptp_data.ptp_clock);
 		mvm->ptp_data.ptp_clock = NULL;
diff --git a/sys/contrib/dev/iwlwifi/mvm/quota.c b/sys/contrib/dev/iwlwifi/mvm/quota.c
index aad2614af9ad..798a7e4bea83 100644
--- a/sys/contrib/dev/iwlwifi/mvm/quota.c
+++ b/sys/contrib/dev/iwlwifi/mvm/quota.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018, 2021-2022 Intel Corporation
+ * Copyright (C) 2012-2014, 2018, 2021-2022, 2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -86,45 +86,6 @@ static void iwl_mvm_quota_iterator(void *_data, u8 *mac,
 	}
 }
 
-static void iwl_mvm_adjust_quota_for_noa(struct iwl_mvm *mvm,
-					 struct iwl_time_quota_cmd *cmd)
-{
-#ifdef CONFIG_NL80211_TESTMODE
-	struct iwl_mvm_vif *mvmvif;
-	int i, phy_id = -1, beacon_int = 0;
-
-	if (!mvm->noa_duration || !mvm->noa_vif)
-		return;
-
-	mvmvif = iwl_mvm_vif_from_mac80211(mvm->noa_vif);
-	if (!mvmvif->ap_ibss_active)
-		return;
-
-	phy_id = mvmvif->deflink.phy_ctxt->id;
-	beacon_int = mvm->noa_vif->bss_conf.beacon_int;
-
-	for (i = 0; i < MAX_BINDINGS; i++) {
-		struct iwl_time_quota_data *data =
-					iwl_mvm_quota_cmd_get_quota(mvm, cmd,
-								    i);
-		u32 id_n_c = le32_to_cpu(data->id_and_color);
-		u32 id = (id_n_c & FW_CTXT_ID_MSK) >> FW_CTXT_ID_POS;
-		u32 quota = le32_to_cpu(data->quota);
-
-		if (id != phy_id)
-			continue;
-
-		quota *= (beacon_int - mvm->noa_duration);
-		quota /= beacon_int;
-
-		IWL_DEBUG_QUOTA(mvm, "quota: adjust for NoA from %d to %d\n",
-				le32_to_cpu(data->quota), quota);
-
-		data->quota = cpu_to_le32(quota);
-	}
-#endif
-}
-
 int iwl_mvm_update_quotas(struct iwl_mvm *mvm,
 			  bool force_update,
 			  struct ieee80211_vif *disabled_vif)
@@ -260,8 +221,6 @@ int iwl_mvm_update_quotas(struct iwl_mvm *mvm,
 		}
 	}
 
-	iwl_mvm_adjust_quota_for_noa(mvm, &cmd);
-
 	/* check that we have non-zero quota for all valid bindings */
 	for (i = 0; i < MAX_BINDINGS; i++) {
 		qdata = iwl_mvm_quota_cmd_get_quota(mvm, &cmd, i);
diff --git a/sys/contrib/dev/iwlwifi/mvm/rs-fw.c b/sys/contrib/dev/iwlwifi/mvm/rs-fw.c
index 74c2cbf60c1b..23a9f1a59ad3 100644
--- a/sys/contrib/dev/iwlwifi/mvm/rs-fw.c
+++ b/sys/contrib/dev/iwlwifi/mvm/rs-fw.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include "rs.h"
 #include "fw-api.h"
@@ -72,7 +72,7 @@ static u16 rs_fw_get_config_flags(struct iwl_mvm *mvm,
 	u16 flags = 0;
 
 	/* get STBC flags */
-	if (mvm->cfg->ht_params->stbc &&
+	if (mvm->cfg->ht_params.stbc &&
 	    (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) > 1)) {
 		if (he_cap->has_he && he_cap->he_cap_elem.phy_cap_info[2] &
 				      IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ)
@@ -83,7 +83,7 @@ static u16 rs_fw_get_config_flags(struct iwl_mvm *mvm,
 			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
 	}
 
-	if (mvm->cfg->ht_params->ldpc &&
+	if (mvm->cfg->ht_params.ldpc &&
 	    ((ht_cap->cap & IEEE80211_HT_CAP_LDPC_CODING) ||
 	     (vht_ena && (vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC))))
 		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
@@ -440,12 +440,6 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 
 	mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
-	if (!mvmsta) {
-		IWL_ERR(mvm, "Invalid sta id (%d) in FW TLC notification\n",
-			notif->sta_id);
-		goto out;
-	}
-
 	flags = le32_to_cpu(notif->flags);
 
 	mvm_link_sta = rcu_dereference(mvmsta->link[link_sta->link_id]);
@@ -460,22 +454,11 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 	if (flags & IWL_TLC_NOTIF_FLAG_RATE) {
 		char pretty_rate[100];
 
-		if (iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP,
-					    TLC_MNG_UPDATE_NOTIF, 0) < 3) {
-			rs_pretty_print_rate_v1(pretty_rate,
-						sizeof(pretty_rate),
-						le32_to_cpu(notif->rate));
-			IWL_DEBUG_RATE(mvm,
-				       "Got rate in old format. Rate: %s. Converting.\n",
-				       pretty_rate);
-			lq_sta->last_rate_n_flags =
-				iwl_new_rate_from_v1(le32_to_cpu(notif->rate));
-		} else {
-			lq_sta->last_rate_n_flags = le32_to_cpu(notif->rate);
-		}
+		lq_sta->last_rate_n_flags =
+			iwl_mvm_v3_rate_from_fw(notif->rate, mvm->fw_rates_ver);
 		rs_pretty_print_rate(pretty_rate, sizeof(pretty_rate),
 				     lq_sta->last_rate_n_flags);
-		IWL_DEBUG_RATE(mvm, "new rate: %s\n", pretty_rate);
+		IWL_DEBUG_RATE(mvm, "rate: %s\n", pretty_rate);
 	}
 
 	if (flags & IWL_TLC_NOTIF_FLAG_AMSDU && !mvm_link_sta->orig_amsdu_len) {
@@ -618,11 +601,8 @@ void iwl_mvm_rs_fw_rate_init(struct iwl_mvm *mvm,
 	int cmd_ver;
 	int ret;
 
-	/* Enable external EHT LTF only for GL device and if there's
-	 * mutual support by AP and client
-	 */
-	if (CSR_HW_REV_TYPE(mvm->trans->hw_rev) == IWL_CFG_MAC_TYPE_GL &&
-	    sband_eht_cap &&
+	/* Enable extra EHT LTF if there's mutual support by AP and client */
+	if (sband_eht_cap &&
 	    sband_eht_cap->eht_cap_elem.phy_cap_info[5] &
 		IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF &&
 	    link_sta->eht_cap.has_eht &&
diff --git a/sys/contrib/dev/iwlwifi/mvm/rx.c b/sys/contrib/dev/iwlwifi/mvm/rx.c
index 0802887fb261..f6127c0b5344 100644
--- a/sys/contrib/dev/iwlwifi/mvm/rx.c
+++ b/sys/contrib/dev/iwlwifi/mvm/rx.c
@@ -1,10 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
-#include <asm/unaligned.h>
+#include <linux/unaligned.h>
 #include <linux/etherdevice.h>
 #include <linux/skbuff.h>
 #include "iwl-trans.h"
@@ -499,8 +499,6 @@ void iwl_mvm_rx_rx_mpdu(struct iwl_mvm *mvm, struct napi_struct *napi,
 	if (!(rate_n_flags & RATE_MCS_CCK_MSK_V1) &&
 	    rate_n_flags & RATE_MCS_SGI_MSK_V1)
 		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
-	if (rate_n_flags & RATE_HT_MCS_GF_MSK)
-		rx_status->enc_flags |= RX_ENC_FLAG_HT_GF;
 	if (rate_n_flags & RATE_MCS_LDPC_MSK_V1)
 		rx_status->enc_flags |= RX_ENC_FLAG_LDPC;
 	if (rate_n_flags & RATE_MCS_HT_MSK_V1) {
@@ -569,7 +567,8 @@ static void iwl_mvm_update_link_sig(struct ieee80211_vif *vif, int sig,
 				    struct iwl_mvm_vif_link_info *link_info,
 				    struct ieee80211_bss_conf *bss_conf)
 {
-	struct iwl_mvm *mvm = iwl_mvm_vif_from_mac80211(vif)->mvm;
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm *mvm = mvmvif->mvm;
 	int thold = bss_conf->cqm_rssi_thold;
 	int hyst = bss_conf->cqm_rssi_hyst;
 	int last_event;
@@ -634,6 +633,13 @@ static void iwl_mvm_update_link_sig(struct ieee80211_vif *vif, int sig,
 	if (!vif->cfg.assoc || !ieee80211_vif_is_mld(vif))
 		return;
 
+	/* We're not in EMLSR and our signal is bad, try to switch link maybe */
+	if (sig < IWL_MVM_LOW_RSSI_MLO_SCAN_THRESH && !mvmvif->esr_active) {
+		iwl_mvm_int_mlo_scan(mvm, vif);
+		return;
+	}
+
+	/* We are in EMLSR, check if we need to exit */
 	exit_esr_thresh =
 		iwl_mvm_get_esr_rssi_thresh(mvm,
 					    &bss_conf->chanreq.oper,
@@ -751,8 +757,8 @@ static void iwl_mvm_stats_energy_iter(void *_data,
 	u8 *energy = _data;
 	u32 sta_id = mvmsta->deflink.sta_id;
 
-	if (WARN_ONCE(sta_id >= IWL_MVM_STATION_COUNT_MAX, "sta_id %d >= %d",
-		      sta_id, IWL_MVM_STATION_COUNT_MAX))
+	if (WARN_ONCE(sta_id >= IWL_STATION_COUNT_MAX, "sta_id %d >= %d",
+		      sta_id, IWL_STATION_COUNT_MAX))
 		return;
 
 	if (energy[sta_id])
@@ -794,6 +800,8 @@ static void iwl_mvm_handle_per_phy_stats(struct iwl_mvm *mvm,
 			continue;
 		mvm->phy_ctxts[i].channel_load_by_us =
 			le32_to_cpu(per_phy[i].channel_load_by_us);
+		mvm->phy_ctxts[i].channel_load_not_by_us =
+			le32_to_cpu(per_phy[i].channel_load_not_by_us);
 	}
 }
 
@@ -882,28 +890,28 @@ iwl_mvm_stat_iterator_all_links(struct iwl_mvm *mvm,
 	u32 rx_bytes[MAC_INDEX_AUX] = {};
 	int fw_link_id;
 
-	for (fw_link_id = 0; fw_link_id < ARRAY_SIZE(mvm->link_id_to_link_conf);
+	/* driver uses link ID == MAC ID */
+	for (fw_link_id = 0; fw_link_id < ARRAY_SIZE(mvm->vif_id_to_mac);
 	     fw_link_id++) {
 		struct iwl_stats_ntfy_per_link *link_stats;
-		struct ieee80211_bss_conf *bss_conf;
-		struct iwl_mvm_vif *mvmvif;
 		struct iwl_mvm_vif_link_info *link_info;
+		struct iwl_mvm_vif *mvmvif;
+		struct ieee80211_vif *vif;
 		int link_id;
 		int sig;
 
-		bss_conf = iwl_mvm_rcu_fw_link_id_to_link_conf(mvm, fw_link_id,
-							       false);
-		if (!bss_conf)
+		vif = iwl_mvm_rcu_dereference_vif_id(mvm, fw_link_id, false);
+		if (!vif)
 			continue;
 
-		if (bss_conf->vif->type != NL80211_IFTYPE_STATION)
+		if (vif->type != NL80211_IFTYPE_STATION)
 			continue;
 
-		link_id = bss_conf->link_id;
+		link_id = vif->bss_conf.link_id;
 		if (link_id >= ARRAY_SIZE(mvmvif->link))
 			continue;
 
-		mvmvif = iwl_mvm_vif_from_mac80211(bss_conf->vif);
+		mvmvif = iwl_mvm_vif_from_mac80211(vif);
 		link_info = mvmvif->link[link_id];
 		if (!link_info)
 			continue;
@@ -921,8 +929,7 @@ iwl_mvm_stat_iterator_all_links(struct iwl_mvm *mvm,
 
 		if (link_info->phy_ctxt &&
 		    link_info->phy_ctxt->channel->band == NL80211_BAND_2GHZ)
-			iwl_mvm_bt_coex_update_link_esr(mvm, bss_conf->vif,
-							link_id);
+			iwl_mvm_bt_coex_update_link_esr(mvm, vif, link_id);
 
 		/* make sure that beacon statistics don't go backwards with TCM
 		 * request to clear statistics
@@ -932,8 +939,7 @@ iwl_mvm_stat_iterator_all_links(struct iwl_mvm *mvm,
 				mvmvif->link[link_id]->beacon_stats.num_beacons;
 
 		sig = -le32_to_cpu(link_stats->beacon_filter_average_energy);
-		iwl_mvm_update_link_sig(bss_conf->vif, sig, link_info,
-					bss_conf);
+		iwl_mvm_update_link_sig(vif, sig, link_info, &vif->bss_conf);
 
 		if (WARN_ONCE(mvmvif->id >= MAC_INDEX_AUX,
 			      "invalid mvmvif id: %d", mvmvif->id))
@@ -967,6 +973,9 @@ iwl_mvm_stat_iterator_all_links(struct iwl_mvm *mvm,
 #define SEC_LINK_MIN_TX 3000
 #define SEC_LINK_MIN_RX 400
 
+/* Accept a ~20% short window to avoid issues due to jitter */
+#define IWL_MVM_TPT_MIN_COUNT_WINDOW (IWL_MVM_TPT_COUNT_WINDOW_SEC * HZ * 4 / 5)
+
 static void iwl_mvm_update_esr_mode_tpt(struct iwl_mvm *mvm)
 {
 	struct ieee80211_vif *bss_vif = iwl_mvm_get_bss_vif(mvm);
@@ -976,6 +985,7 @@ static void iwl_mvm_update_esr_mode_tpt(struct iwl_mvm *mvm)
 	unsigned long sec_link_tx = 0, sec_link_rx = 0;
 	u8 sec_link_tx_perc, sec_link_rx_perc;
 	u8 sec_link;
+	bool skip = false;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -1000,11 +1010,11 @@ static void iwl_mvm_update_esr_mode_tpt(struct iwl_mvm *mvm)
 	sec_link = mvmvif->link[sec_link]->fw_link_id;
 
 	/* Sum up RX and TX MPDUs from the different queues/links */
-	for (int q = 0; q < mvm->trans->num_rx_queues; q++) {
+	for (int q = 0; q < mvm->trans->info.num_rxqs; q++) {
 		spin_lock_bh(&mvmsta->mpdu_counters[q].lock);
 
 		/* The link IDs that doesn't exist will contain 0 */
-		for (int link = 0; link < IWL_MVM_FW_MAX_LINK_ID; link++) {
+		for (int link = 0; link < IWL_FW_MAX_LINK_ID; link++) {
 			total_tx += mvmsta->mpdu_counters[q].per_link[link].tx;
 			total_rx += mvmsta->mpdu_counters[q].per_link[link].rx;
 		}
@@ -1015,15 +1025,27 @@ static void iwl_mvm_update_esr_mode_tpt(struct iwl_mvm *mvm)
 		/*
 		 * In EMLSR we have statistics every 5 seconds, so we can reset
 		 * the counters upon every statistics notification.
+		 * The FW sends the notification regularly, but it will be
+		 * misaligned at the start. Skipping the measurement if it is
+		 * short will synchronize us.
 		 */
+		if (jiffies - mvmsta->mpdu_counters[q].window_start <
+		    IWL_MVM_TPT_MIN_COUNT_WINDOW)
+			skip = true;
+		mvmsta->mpdu_counters[q].window_start = jiffies;
 		memset(mvmsta->mpdu_counters[q].per_link, 0,
 		       sizeof(mvmsta->mpdu_counters[q].per_link));
 
 		spin_unlock_bh(&mvmsta->mpdu_counters[q].lock);
 	}
 
-	IWL_DEBUG_STATS(mvm, "total Tx MPDUs: %ld. total Rx MPDUs: %ld\n",
-			total_tx, total_rx);
+	if (skip) {
+		IWL_DEBUG_INFO(mvm, "MPDU statistics window was short\n");
+		return;
+	}
+
+	IWL_DEBUG_INFO(mvm, "total Tx MPDUs: %ld. total Rx MPDUs: %ld\n",
+		       total_tx, total_rx);
 
 	/* If we don't have enough MPDUs - exit EMLSR */
 	if (total_tx < IWL_MVM_ENTER_ESR_TPT_THRESH &&
@@ -1033,6 +1055,9 @@ static void iwl_mvm_update_esr_mode_tpt(struct iwl_mvm *mvm)
 		return;
 	}
 
+	IWL_DEBUG_INFO(mvm, "Secondary Link %d: Tx MPDUs: %ld. Rx MPDUs: %ld\n",
+		       sec_link, sec_link_tx, sec_link_rx);
+
 	/* Calculate the percentage of the secondary link TX/RX */
 	sec_link_tx_perc = total_tx ? sec_link_tx * 100 / total_tx : 0;
 	sec_link_rx_perc = total_rx ? sec_link_rx * 100 / total_rx : 0;
@@ -1052,7 +1077,7 @@ static void iwl_mvm_update_esr_mode_tpt(struct iwl_mvm *mvm)
 void iwl_mvm_handle_rx_system_oper_stats(struct iwl_mvm *mvm,
 					 struct iwl_rx_cmd_buffer *rxb)
 {
-	u8 average_energy[IWL_MVM_STATION_COUNT_MAX];
+	u8 average_energy[IWL_STATION_COUNT_MAX];
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_system_statistics_notif_oper *stats;
 	int i;
@@ -1111,7 +1136,7 @@ static void
 iwl_mvm_handle_rx_statistics_tlv(struct iwl_mvm *mvm,
 				 struct iwl_rx_packet *pkt)
 {
-	u8 average_energy[IWL_MVM_STATION_COUNT_MAX];
+	u8 average_energy[IWL_STATION_COUNT_MAX];
 	__le32 air_time[MAC_INDEX_AUX];
 	__le32 rx_bytes[MAC_INDEX_AUX];
 	__le32 flags = 0;
diff --git a/sys/contrib/dev/iwlwifi/mvm/rxmq.c b/sys/contrib/dev/iwlwifi/mvm/rxmq.c
index cb2f6b347303..71210b8b9edd 100644
--- a/sys/contrib/dev/iwlwifi/mvm/rxmq.c
+++ b/sys/contrib/dev/iwlwifi/mvm/rxmq.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -174,7 +174,7 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 			      shdr->type != htons(ETH_P_PAE) &&
 			      shdr->type != htons(ETH_P_TDLS))))
 			skb->ip_summed = CHECKSUM_NONE;
-		else if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
+		else if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
 			/* mac80211 assumes full CSUM including SNAP header */
 			skb_postpush_rcsum(skb, shdr, sizeof(*shdr));
 	}
@@ -249,13 +249,62 @@ static void iwl_mvm_pass_packet_to_mac80211(struct iwl_mvm *mvm,
 	ieee80211_rx_napi(mvm->hw, sta, skb, napi);
 }
 
+static bool iwl_mvm_used_average_energy(struct iwl_mvm *mvm,
+					struct iwl_rx_mpdu_desc *desc,
+					struct ieee80211_hdr *hdr,
+					struct ieee80211_rx_status *rx_status)
+{
+	struct iwl_mvm_vif *mvm_vif;
+	struct ieee80211_vif *vif;
+	u32 id;
+
+	if (unlikely(!hdr || !desc))
+		return false;
+
+	if (likely(!ieee80211_is_beacon(hdr->frame_control)))
+		return false;
+
+	/* for the link conf lookup */
+	guard(rcu)();
+
+	/* MAC or link ID depending on FW, but driver has them equal */
+	id = u8_get_bits(desc->mac_phy_band,
+			 IWL_RX_MPDU_MAC_PHY_BAND_MAC_MASK);
+
+	/* >= means AUX MAC/link ID, no energy correction needed then */
+	if (id >= ARRAY_SIZE(mvm->vif_id_to_mac))
+		return false;
+
+	vif = iwl_mvm_rcu_dereference_vif_id(mvm, id, true);
+	if (!vif)
+		return false;
+
+	mvm_vif = iwl_mvm_vif_from_mac80211(vif);
+
+	/*
+	 * If we know the MAC by MAC or link ID then the frame was
+	 * received for the link, so by filtering it means it was
+	 * from the AP the link is connected to.
+	 */
+
+	/* skip also in case we don't have it (yet) */
+	if (!mvm_vif->deflink.average_beacon_energy)
+		return false;
+
+	IWL_DEBUG_STATS(mvm, "energy override by average %d\n",
+			mvm_vif->deflink.average_beacon_energy);
+	rx_status->signal = -mvm_vif->deflink.average_beacon_energy;
+	return true;
+}
+
 static void iwl_mvm_get_signal_strength(struct iwl_mvm *mvm,
+					struct iwl_rx_mpdu_desc *desc,
+					struct ieee80211_hdr *hdr,
 					struct ieee80211_rx_status *rx_status,
 					u32 rate_n_flags, int energy_a,
 					int energy_b)
 {
 	int max_energy;
-	u32 rate_flags = rate_n_flags;
 
 	energy_a = energy_a ? -energy_a : S8_MIN;
 	energy_b = energy_b ? -energy_b : S8_MIN;
@@ -264,9 +313,11 @@ static void iwl_mvm_get_signal_strength(struct iwl_mvm *mvm,
 	IWL_DEBUG_STATS(mvm, "energy In A %d B %d, and max %d\n",
 			energy_a, energy_b, max_energy);
 
+	if (iwl_mvm_used_average_energy(mvm, desc, hdr, rx_status))
+		return;
+
 	rx_status->signal = max_energy;
-	rx_status->chains =
-		(rate_flags & RATE_MCS_ANT_AB_MSK) >> RATE_MCS_ANT_POS;
+	rx_status->chains = u32_get_bits(rate_n_flags, RATE_MCS_ANT_AB_MSK);
 	rx_status->chain_signal[0] = energy_a;
 	rx_status->chain_signal[1] = energy_b;
 }
@@ -418,7 +469,7 @@ static int iwl_mvm_rx_crypto(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		    !(status & IWL_RX_MPDU_RES_STATUS_TTAK_OK))
 			return 0;
 
-		if (mvm->trans->trans_cfg->gen2 &&
+		if (mvm->trans->mac_cfg->gen2 &&
 		    !(status & RX_MPDU_RES_STATUS_MIC_OK))
 			stats->flag |= RX_FLAG_MMIC_ERROR;
 
@@ -435,7 +486,7 @@ static int iwl_mvm_rx_crypto(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 
 		if (pkt_flags & FH_RSCSR_RADA_EN) {
 			stats->flag |= RX_FLAG_ICV_STRIPPED;
-			if (mvm->trans->trans_cfg->gen2)
+			if (mvm->trans->mac_cfg->gen2)
 				stats->flag |= RX_FLAG_MMIC_STRIPPED;
 		}
 
@@ -475,7 +526,7 @@ static void iwl_mvm_rx_csum(struct iwl_mvm *mvm,
 {
 	struct iwl_rx_mpdu_desc *desc = (void *)pkt->data;
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		if (pkt->len_n_flags & cpu_to_le32(FH_RSCSR_RPA_EN)) {
 			u16 hwsum = be16_to_cpu(desc->v3.raw_xsum);
 
@@ -649,15 +700,21 @@ static void iwl_mvm_release_frames_from_notif(struct iwl_mvm *mvm,
 	IWL_DEBUG_HT(mvm, "Frame release notification for BAID %u, NSSN %d\n",
 		     baid, nssn);
 
-	if (WARN_ON_ONCE(baid == IWL_RX_REORDER_DATA_INVALID_BAID ||
-			 baid >= ARRAY_SIZE(mvm->baid_map)))
+	if (IWL_FW_CHECK(mvm,
+			 baid == IWL_RX_REORDER_DATA_INVALID_BAID ||
+			 baid >= ARRAY_SIZE(mvm->baid_map),
+			 "invalid BAID from FW: %d\n", baid))
 		return;
 
 	rcu_read_lock();
 
 	ba_data = rcu_dereference(mvm->baid_map[baid]);
-	if (WARN(!ba_data, "BAID %d not found in map\n", baid))
+	if (!ba_data) {
+		IWL_DEBUG_RX(mvm,
+			     "Got valid BAID %d but not allocated, invalid frame release!\n",
+			     baid);
 		goto out;
+	}
 
 	/* pick any STA ID to find the pointer */
 	sta_id = ffs(ba_data->sta_mask) - 1;
@@ -746,8 +803,6 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 #elif defined(__FreeBSD__)
 	u8 tid;
 #endif
-	u8 sub_frame_idx = desc->amsdu_info &
-			   IWL_RX_MPDU_AMSDU_SUBFRAME_IDX_MASK;
 	struct iwl_mvm_reorder_buf_entry *entries;
 	u32 sta_mask;
 	int index;
@@ -757,7 +812,7 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 	baid = (reorder & IWL_RX_MPDU_REORDER_BAID_MASK) >>
 		IWL_RX_MPDU_REORDER_BAID_SHIFT;
 
-	if (mvm->trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_9000)
+	if (mvm->trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_9000)
 		return false;
 
 	/*
@@ -795,9 +850,7 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 #if defined(__FreeBSD__)
 	tid = ieee80211_get_tid(hdr);
 #endif
-	rcu_read_lock();
 	sta_mask = iwl_mvm_sta_fw_id_mask(mvm, sta, -1);
-	rcu_read_unlock();
 
 	if (IWL_FW_CHECK(mvm,
 			 tid != baid_data->tid ||
@@ -836,7 +889,7 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 	if (!buffer->num_stored && ieee80211_sn_less(sn, nssn)) {
 		if (!amsdu || last_subframe)
 			buffer->head_sn = nssn;
-		/* No need to update AMSDU last SN - we are moving the head */
+
 		spin_unlock_bh(&buffer->lock);
 		return false;
 	}
@@ -853,7 +906,6 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 		if (!amsdu || last_subframe)
 			buffer->head_sn = ieee80211_sn_inc(buffer->head_sn);
 
-		/* No need to update AMSDU last SN - we are moving the head */
 		spin_unlock_bh(&buffer->lock);
 		return false;
 	}
@@ -863,11 +915,6 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 	__skb_queue_tail(&entries[index].frames, skb);
 	buffer->num_stored++;
 
-	if (amsdu) {
-		buffer->last_amsdu = sn;
-		buffer->last_sub_index = sub_frame_idx;
-	}
-
 	/*
 	 * We cannot trust NSSN for AMSDU sub-frames that are not the last.
 	 * The reason is that NSSN advances on the first sub-frame, and may
@@ -878,10 +925,15 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 	 * already ahead and it will be dropped.
 	 * If the last sub-frame is not on this queue - we will get frame
 	 * release notification with up to date NSSN.
+	 * If this is the first frame that is stored in the buffer, the head_sn
+	 * may be outdated. Update it based on the last NSSN to make sure it
+	 * will be released when the frame release notification arrives.
 	 */
 	if (!amsdu || last_subframe)
 		iwl_mvm_release_frames(mvm, sta, napi, baid_data,
 				       buffer, nssn);
+	else if (buffer->num_stored == 1)
+		buffer->head_sn = nssn;
 
 	spin_unlock_bh(&buffer->lock);
 	return true;
@@ -1019,7 +1071,7 @@ iwl_mvm_decode_he_phy_ru_alloc(struct iwl_mvm_rx_phy_data *phy_data,
 	 */
 	u8 ru = le32_get_bits(phy_data->d1, IWL_RX_PHY_DATA1_HE_RU_ALLOC_MASK);
 	u32 rate_n_flags = phy_data->rate_n_flags;
-	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK_V1;
+	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK;
 	u8 offs = 0;
 
 	rx_status->bw = RATE_INFO_BW_HE_RU;
@@ -1074,13 +1126,13 @@ iwl_mvm_decode_he_phy_ru_alloc(struct iwl_mvm_rx_phy_data *phy_data,
 
 	if (he_mu)
 		he_mu->flags2 |=
-			le16_encode_bits(FIELD_GET(RATE_MCS_CHAN_WIDTH_MSK_V1,
+			le16_encode_bits(FIELD_GET(RATE_MCS_CHAN_WIDTH_MSK,
 						   rate_n_flags),
 					 IEEE80211_RADIOTAP_HE_MU_FLAGS2_BW_FROM_SIG_A_BW);
-	else if (he_type == RATE_MCS_HE_TYPE_TRIG_V1)
+	else if (he_type == RATE_MCS_HE_TYPE_TRIG)
 		he->data6 |=
 			cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA6_TB_PPDU_BW_KNOWN) |
-			le16_encode_bits(FIELD_GET(RATE_MCS_CHAN_WIDTH_MSK_V1,
+			le16_encode_bits(FIELD_GET(RATE_MCS_CHAN_WIDTH_MSK,
 						   rate_n_flags),
 					 IEEE80211_RADIOTAP_HE_DATA6_TB_PPDU_BW);
 }
@@ -1930,8 +1982,11 @@ static void iwl_mvm_rx_get_sta_block_tx(void *data, struct ieee80211_sta *sta)
 /*
  * Note: requires also rx_status->band to be prefilled, as well
  * as phy_data (apart from phy_data->info_type)
+ * Note: desc/hdr may be NULL
  */
 static void iwl_mvm_rx_fill_status(struct iwl_mvm *mvm,
+				   struct iwl_rx_mpdu_desc *desc,
+				   struct ieee80211_hdr *hdr,
 				   struct sk_buff *skb,
 				   struct iwl_mvm_rx_phy_data *phy_data,
 				   int queue)
@@ -1968,7 +2023,7 @@ static void iwl_mvm_rx_fill_status(struct iwl_mvm *mvm,
 	}
 
 	/* must be before L-SIG data */
-	if (format == RATE_MCS_HE_MSK)
+	if (format == RATE_MCS_MOD_TYPE_HE)
 		iwl_mvm_rx_he(mvm, skb, phy_data, queue);
 
 	iwl_mvm_decode_lsig(skb, phy_data);
@@ -1986,49 +2041,49 @@ static void iwl_mvm_rx_fill_status(struct iwl_mvm *mvm,
 
 	rx_status->freq = ieee80211_channel_to_frequency(phy_data->channel,
 							 rx_status->band);
-	iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags,
+	iwl_mvm_get_signal_strength(mvm, desc, hdr, rx_status, rate_n_flags,
 				    phy_data->energy_a, phy_data->energy_b);
 
 	/* using TLV format and must be after all fixed len fields */
-	if (format == RATE_MCS_EHT_MSK)
+	if (format == RATE_MCS_MOD_TYPE_EHT)
 		iwl_mvm_rx_eht(mvm, skb, phy_data, queue);
 
 	if (unlikely(mvm->monitor_on))
 		iwl_mvm_add_rtap_sniffer_config(mvm, skb);
 
-	is_sgi = format == RATE_MCS_HE_MSK ?
+	is_sgi = format == RATE_MCS_MOD_TYPE_HE ?
 		iwl_he_is_sgi(rate_n_flags) :
 		rate_n_flags & RATE_MCS_SGI_MSK;
 
-	if (!(format == RATE_MCS_CCK_MSK) && is_sgi)
+	if (!(format == RATE_MCS_MOD_TYPE_CCK) && is_sgi)
 		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
 
 	if (rate_n_flags & RATE_MCS_LDPC_MSK)
 		rx_status->enc_flags |= RX_ENC_FLAG_LDPC;
 
 	switch (format) {
-	case RATE_MCS_VHT_MSK:
+	case RATE_MCS_MOD_TYPE_VHT:
 		rx_status->encoding = RX_ENC_VHT;
 		break;
-	case RATE_MCS_HE_MSK:
+	case RATE_MCS_MOD_TYPE_HE:
 		rx_status->encoding = RX_ENC_HE;
 		rx_status->he_dcm =
 			!!(rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK);
 		break;
-	case RATE_MCS_EHT_MSK:
+	case RATE_MCS_MOD_TYPE_EHT:
 		rx_status->encoding = RX_ENC_EHT;
 		break;
 	}
 
 	switch (format) {
-	case RATE_MCS_HT_MSK:
+	case RATE_MCS_MOD_TYPE_HT:
 		rx_status->encoding = RX_ENC_HT;
 		rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags);
 		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
 		break;
-	case RATE_MCS_VHT_MSK:
-	case RATE_MCS_HE_MSK:
-	case RATE_MCS_EHT_MSK:
+	case RATE_MCS_MOD_TYPE_VHT:
+	case RATE_MCS_MOD_TYPE_HE:
+	case RATE_MCS_MOD_TYPE_EHT:
 		rx_status->nss =
 			u32_get_bits(rate_n_flags, RATE_MCS_NSS_MSK) + 1;
 		rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK;
@@ -2072,7 +2127,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	if (unlikely(test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)))
 		return;
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		desc_size = sizeof(*desc);
 	else
 		desc_size = IWL_RX_DESC_SIZE_V1;
@@ -2082,8 +2137,10 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		return;
 	}
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
-		phy_data.rate_n_flags = le32_to_cpu(desc->v3.rate_n_flags);
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+		phy_data.rate_n_flags =
+			iwl_mvm_v3_rate_from_fw(desc->v3.rate_n_flags,
+						mvm->fw_rates_ver);
 		phy_data.channel = desc->v3.channel;
 		phy_data.gp2_on_air_rise = le32_to_cpu(desc->v3.gp2_on_air_rise);
 		phy_data.energy_a = desc->v3.energy_a;
@@ -2096,7 +2153,9 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		phy_data.eht_d4 = desc->phy_eht_data4;
 		phy_data.d5 = desc->v3.phy_data5;
 	} else {
-		phy_data.rate_n_flags = le32_to_cpu(desc->v1.rate_n_flags);
+		phy_data.rate_n_flags =
+			iwl_mvm_v3_rate_from_fw(desc->v1.rate_n_flags,
+						mvm->fw_rates_ver);
 		phy_data.channel = desc->v1.channel;
 		phy_data.gp2_on_air_rise = le32_to_cpu(desc->v1.gp2_on_air_rise);
 		phy_data.energy_a = desc->v1.energy_a;
@@ -2108,13 +2167,6 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		phy_data.d3 = desc->v1.phy_data3;
 	}
 
-	if (iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
-				    REPLY_RX_MPDU_CMD, 0) < 4) {
-		phy_data.rate_n_flags = iwl_new_rate_from_v1(phy_data.rate_n_flags);
-		IWL_DEBUG_DROP(mvm, "Got old format rate, converting. New rate: 0x%x\n",
-			       phy_data.rate_n_flags);
-	}
-
 	format = phy_data.rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
 
 	len = le16_to_cpu(desc->mpdu_len);
@@ -2162,14 +2214,14 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	}
 
 	/* set the preamble flag if appropriate */
-	if (format == RATE_MCS_CCK_MSK &&
+	if (format == RATE_MCS_MOD_TYPE_CCK &&
 	    phy_data.phy_info & IWL_RX_MPDU_PHY_SHORT_PREAMBLE)
 		rx_status->enc_flags |= RX_ENC_FLAG_SHORTPRE;
 
 	if (likely(!(phy_data.phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD))) {
 		u64 tsf_on_air_rise;
 
-		if (mvm->trans->trans_cfg->device_family >=
+		if (mvm->trans->mac_cfg->device_family >=
 		    IWL_DEVICE_FAMILY_AX210)
 			tsf_on_air_rise = le64_to_cpu(desc->v3.tsf_on_air_rise);
 		else
@@ -2181,7 +2233,8 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	}
 
 	if (iwl_mvm_is_band_in_rx_supported(mvm)) {
-		u8 band = BAND_IN_RX_STATUS(desc->mac_phy_idx);
+		u8 band = u8_get_bits(desc->mac_phy_band,
+				      IWL_RX_MPDU_MAC_PHY_BAND_BAND_MASK);
 
 		rx_status->band = iwl_mvm_nl80211_band_from_phy(band);
 	} else {
@@ -2241,7 +2294,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		goto out;
 	}
 
-	iwl_mvm_rx_fill_status(mvm, skb, &phy_data, queue);
+	iwl_mvm_rx_fill_status(mvm, desc, hdr, skb, &phy_data, queue);
 
 	if (sta) {
 		struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
@@ -2331,7 +2384,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 
 			*qc &= ~IEEE80211_QOS_CTL_A_MSDU_PRESENT;
 
-			if (mvm->trans->trans_cfg->device_family ==
+			if (mvm->trans->mac_cfg->device_family ==
 			    IWL_DEVICE_FAMILY_9000) {
 				iwl_mvm_flip_address(hdr->addr3);
 
@@ -2377,7 +2430,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	if (!iwl_mvm_reorder(mvm, napi, queue, sta, skb, desc) &&
 	    likely(!iwl_mvm_time_sync_frame(mvm, skb, hdr->addr2)) &&
 	    likely(!iwl_mvm_mei_filter_scan(mvm, skb))) {
-		if (mvm->trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_9000 &&
+		if (mvm->trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_9000 &&
 		    (desc->mac_flags2 & IWL_RX_MPDU_MFLG2_AMSDU) &&
 		    !(desc->amsdu_info & IWL_RX_MPDU_AMSDU_LAST_SUBFRAME))
 			rx_status->flag |= RX_FLAG_AMSDU_MORE;
@@ -2411,7 +2464,6 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	phy_data.d1 = desc->phy_info[1];
 	phy_data.phy_info = IWL_RX_MPDU_PHY_TSF_OVERLOAD;
 	phy_data.gp2_on_air_rise = le32_to_cpu(desc->on_air_rise_time);
-	phy_data.rate_n_flags = le32_to_cpu(desc->rate);
 	phy_data.energy_a = u32_get_bits(rssi, RX_NO_DATA_CHAIN_A_MSK);
 	phy_data.energy_b = u32_get_bits(rssi, RX_NO_DATA_CHAIN_B_MSK);
 	phy_data.channel = u32_get_bits(rssi, RX_NO_DATA_CHANNEL_MSK);
@@ -2419,14 +2471,8 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	phy_data.rx_vec[0] = desc->rx_vec[0];
 	phy_data.rx_vec[1] = desc->rx_vec[1];
 
-	if (iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP,
-				    RX_NO_DATA_NOTIF, 0) < 2) {
-		IWL_DEBUG_DROP(mvm, "Got an old rate format. Old rate: 0x%x\n",
-			       phy_data.rate_n_flags);
-		phy_data.rate_n_flags = iwl_new_rate_from_v1(phy_data.rate_n_flags);
-		IWL_DEBUG_DROP(mvm, " Rate after conversion to the new format: 0x%x\n",
-			       phy_data.rate_n_flags);
-	}
+	phy_data.rate_n_flags = iwl_mvm_v3_rate_from_fw(desc->rate,
+							mvm->fw_rates_ver);
 
 	format = phy_data.rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
 
@@ -2439,7 +2485,7 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 		phy_data.rx_vec[2] = desc->rx_vec[2];
 		phy_data.rx_vec[3] = desc->rx_vec[3];
 	} else {
-		if (format == RATE_MCS_EHT_MSK)
+		if (format == RATE_MCS_MOD_TYPE_EHT)
 			/* no support for EHT before version 3 API */
 			return;
 	}
@@ -2482,7 +2528,7 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	rx_status->band = phy_data.channel > 14 ? NL80211_BAND_5GHZ :
 		NL80211_BAND_2GHZ;
 
-	iwl_mvm_rx_fill_status(mvm, skb, &phy_data, queue);
+	iwl_mvm_rx_fill_status(mvm, NULL, NULL, skb, &phy_data, queue);
 
 	/* no more radio tap info should be put after this point.
 	 *
@@ -2500,17 +2546,17 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	 * may be up to 8 spatial streams.
 	 */
 	switch (format) {
-	case RATE_MCS_VHT_MSK:
+	case RATE_MCS_MOD_TYPE_VHT:
 		rx_status->nss =
 			le32_get_bits(desc->rx_vec[0],
 				      RX_NO_DATA_RX_VEC0_VHT_NSTS_MSK) + 1;
 		break;
-	case RATE_MCS_HE_MSK:
+	case RATE_MCS_MOD_TYPE_HE:
 		rx_status->nss =
 			le32_get_bits(desc->rx_vec[0],
 				      RX_NO_DATA_RX_VEC0_HE_NSTS_MSK) + 1;
 		break;
-	case RATE_MCS_EHT_MSK:
+	case RATE_MCS_MOD_TYPE_EHT:
 		rx_status->nss =
 			le32_get_bits(desc->rx_vec[2],
 				      RX_NO_DATA_RX_VEC2_EHT_NSTS_MSK) + 1;
@@ -2540,19 +2586,24 @@ void iwl_mvm_rx_bar_frame_release(struct iwl_mvm *mvm, struct napi_struct *napi,
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_bar_frame_release *release = (void *)pkt->data;
-	unsigned int baid = le32_get_bits(release->ba_info,
-					  IWL_BAR_FRAME_RELEASE_BAID_MASK);
-	unsigned int nssn = le32_get_bits(release->ba_info,
-					  IWL_BAR_FRAME_RELEASE_NSSN_MASK);
-	unsigned int sta_id = le32_get_bits(release->sta_tid,
-					    IWL_BAR_FRAME_RELEASE_STA_MASK);
-	unsigned int tid = le32_get_bits(release->sta_tid,
-					 IWL_BAR_FRAME_RELEASE_TID_MASK);
 	struct iwl_mvm_baid_data *baid_data;
+	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
+	unsigned int baid, nssn, sta_id, tid;
 
-	if (unlikely(iwl_rx_packet_payload_len(pkt) < sizeof(*release)))
+	if (IWL_FW_CHECK(mvm, pkt_len < sizeof(*release),
+			 "Unexpected frame release notif size %d (expected %zu)\n",
+			 pkt_len, sizeof(*release)))
 		return;
 
+	baid = le32_get_bits(release->ba_info,
+			     IWL_BAR_FRAME_RELEASE_BAID_MASK);
+	nssn = le32_get_bits(release->ba_info,
+			     IWL_BAR_FRAME_RELEASE_NSSN_MASK);
+	sta_id = le32_get_bits(release->sta_tid,
+			       IWL_BAR_FRAME_RELEASE_STA_MASK);
+	tid = le32_get_bits(release->sta_tid,
+			    IWL_BAR_FRAME_RELEASE_TID_MASK);
+
 	if (WARN_ON_ONCE(baid == IWL_RX_REORDER_DATA_INVALID_BAID ||
 			 baid >= ARRAY_SIZE(mvm->baid_map)))
 		return;
@@ -2566,7 +2617,7 @@ void iwl_mvm_rx_bar_frame_release(struct iwl_mvm *mvm, struct napi_struct *napi,
 		goto out;
 	}
 
-	if (WARN(tid != baid_data->tid || sta_id > IWL_MVM_STATION_COUNT_MAX ||
+	if (WARN(tid != baid_data->tid || sta_id > IWL_STATION_COUNT_MAX ||
 		 !(baid_data->sta_mask & BIT(sta_id)),
 		 "baid 0x%x is mapped to sta_mask:0x%x tid:%d, but BAR release received for sta:%d tid:%d\n",
 		 baid, baid_data->sta_mask, baid_data->tid, sta_id,
@@ -2580,3 +2631,28 @@ void iwl_mvm_rx_bar_frame_release(struct iwl_mvm *mvm, struct napi_struct *napi,
 out:
 	rcu_read_unlock();
 }
+
+void iwl_mvm_rx_beacon_filter_notif(struct iwl_mvm *mvm,
+				    struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	/* MAC or link ID in v1/v2, but driver has the IDs equal */
+	struct iwl_beacon_filter_notif *notif = (void *)pkt->data;
+	u32 id = le32_to_cpu(notif->link_id);
+	struct iwl_mvm_vif *mvm_vif;
+	struct ieee80211_vif *vif;
+
+	/* >= means AUX MAC/link ID, no energy correction needed then */
+	if (IWL_FW_CHECK(mvm, id >= ARRAY_SIZE(mvm->vif_id_to_mac),
+			 "invalid link ID %d\n", id))
+		return;
+
+	vif = iwl_mvm_rcu_dereference_vif_id(mvm, id, false);
+	if (!vif)
+		return;
+
+	mvm_vif = iwl_mvm_vif_from_mac80211(vif);
+
+	mvm_vif->deflink.average_beacon_energy =
+		le32_to_cpu(notif->average_energy);
+}
diff --git a/sys/contrib/dev/iwlwifi/mvm/scan.c b/sys/contrib/dev/iwlwifi/mvm/scan.c
index 0b705e497343..9ce1ce0dab34 100644
--- a/sys/contrib/dev/iwlwifi/mvm/scan.c
+++ b/sys/contrib/dev/iwlwifi/mvm/scan.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -11,6 +11,7 @@
 #include "mvm.h"
 #include "fw/api/scan.h"
 #include "iwl-io.h"
+#include "iwl-utils.h"
 
 #define IWL_DENSE_EBS_SCAN_RATIO 5
 #define IWL_SPARSE_EBS_SCAN_RATIO 1
@@ -462,11 +463,7 @@ static int iwl_ssid_exist(u8 *ssid, u8 ssid_len, struct iwl_ssid_ie *ssid_list)
 		if (!ssid_list[i].len)
 			break;
 		if (ssid_list[i].len == ssid_len &&
-#if defined(__linux__)
-		    !memcmp(ssid_list->ssid, ssid, ssid_len))
-#elif defined(__FreeBSD__)
 		    !memcmp(ssid_list[i].ssid, ssid, ssid_len))
-#endif
 			return i;
 	}
 	return -1;
@@ -839,7 +836,7 @@ static inline bool iwl_mvm_scan_fits(struct iwl_mvm *mvm, int n_ssids,
 				     int n_channels)
 {
 	return ((n_ssids <= PROBE_OPTION_MAX) &&
-		(n_channels <= mvm->fw->ucode_capa.n_scan_channels) &
+		(n_channels <= mvm->fw->ucode_capa.n_scan_channels) &&
 		(ies->common_ie_len +
 		 ies->len[NL80211_BAND_2GHZ] + ies->len[NL80211_BAND_5GHZ] +
 		 ies->len[NL80211_BAND_6GHZ] <=
@@ -1598,7 +1595,7 @@ iwl_mvm_umac_scan_cfg_channels(struct iwl_mvm *mvm,
 
 	for (i = 0; i < n_channels; i++) {
 		channel_cfg[i].flags = cpu_to_le32(flags);
-		channel_cfg[i].v1.channel_num = channels[i]->hw_value;
+		channel_cfg[i].channel_num = channels[i]->hw_value;
 		if (iwl_mvm_is_scan_ext_chan_supported(mvm)) {
 			enum nl80211_band band = channels[i]->band;
 
@@ -1630,13 +1627,13 @@ iwl_mvm_umac_scan_cfg_channels_v4(struct iwl_mvm *mvm,
 			&cp->channel_config[i];
 
 		cfg->flags = cpu_to_le32(flags);
-		cfg->v2.channel_num = channels[i]->hw_value;
+		cfg->channel_num = channels[i]->hw_value;
 		cfg->v2.band = iwl_mvm_phy_band_from_nl80211(band);
 		cfg->v2.iter_count = 1;
 		cfg->v2.iter_interval = 0;
 
 		iwl_mvm_scan_ch_add_n_aps_override(vif_type,
-						   cfg->v2.channel_num,
+						   cfg->channel_num,
 						   cfg->v2.band, bitmap,
 						   bitmap_n_entries);
 	}
@@ -1660,7 +1657,7 @@ iwl_mvm_umac_scan_cfg_channels_v7(struct iwl_mvm *mvm,
 		u8 iwl_band = iwl_mvm_phy_band_from_nl80211(band);
 
 		cfg->flags = cpu_to_le32(flags | n_aps_flag);
-		cfg->v2.channel_num = channels[i]->hw_value;
+		cfg->channel_num = channels[i]->hw_value;
 		if (cfg80211_channel_is_psc(channels[i]))
 			cfg->flags = 0;
 
@@ -1778,7 +1775,7 @@ iwl_mvm_umac_scan_cfg_channels_v7_6g(struct iwl_mvm *mvm,
 			&cp->channel_config[ch_cnt];
 
 		u32 s_ssid_bitmap = 0, bssid_bitmap = 0, flags = 0;
-		u8 j, k, n_s_ssids = 0, n_bssids = 0;
+		u8 k, n_s_ssids = 0, n_bssids = 0;
 		u8 max_s_ssids, max_bssids;
 		bool force_passive = false, found = false, allow_passive = true,
 		     unsolicited_probe_on_chan = false, psc_no_listen = false;
@@ -1793,7 +1790,7 @@ iwl_mvm_umac_scan_cfg_channels_v7_6g(struct iwl_mvm *mvm,
 		    !params->n_6ghz_params && params->n_ssids)
 			continue;
 
-		cfg->v1.channel_num = params->channels[i]->hw_value;
+		cfg->channel_num = params->channels[i]->hw_value;
 		if (version < 17)
 			cfg->v2.band = PHY_BAND_6;
 		else
@@ -1803,7 +1800,7 @@ iwl_mvm_umac_scan_cfg_channels_v7_6g(struct iwl_mvm *mvm,
 		cfg->v5.iter_count = 1;
 		cfg->v5.iter_interval = 0;
 
-		for (j = 0; j < params->n_6ghz_params; j++) {
+		for (u32 j = 0; j < params->n_6ghz_params; j++) {
 			s8 tmp_psd_20;
 
 			if (!(scan_6ghz_params[j].channel_idx == i))
@@ -1877,7 +1874,7 @@ iwl_mvm_umac_scan_cfg_channels_v7_6g(struct iwl_mvm *mvm,
 		 * SSID.
 		 * TODO: improve this logic
 		 */
-		for (j = 0; j < params->n_6ghz_params; j++) {
+		for (u32 j = 0; j < params->n_6ghz_params; j++) {
 			if (!(scan_6ghz_params[j].channel_idx == i))
 				continue;
 
@@ -2481,7 +2478,7 @@ iwl_mvm_scan_umac_fill_ch_p_v7(struct iwl_mvm *mvm,
 			if (!cfg80211_channel_is_psc(channel))
 				continue;
 
-			cfg->v5.channel_num = channel->hw_value;
+			cfg->channel_num = channel->hw_value;
 			cfg->v5.iter_count = 1;
 			cfg->v5.iter_interval = 0;
 
@@ -3317,13 +3314,23 @@ void iwl_mvm_rx_umac_scan_iter_complete_notif(struct iwl_mvm *mvm,
 		       mvm->scan_start);
 }
 
-static int iwl_mvm_umac_scan_abort(struct iwl_mvm *mvm, int type)
+static int iwl_mvm_umac_scan_abort(struct iwl_mvm *mvm, int type, bool *wait)
 {
-	struct iwl_umac_scan_abort cmd = {};
+	struct iwl_umac_scan_abort abort_cmd = {};
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(IWL_ALWAYS_LONG_GROUP, SCAN_ABORT_UMAC),
+		.len = { sizeof(abort_cmd), },
+		.data = { &abort_cmd, },
+		.flags = CMD_SEND_IN_RFKILL,
+	};
+
 	int uid, ret;
+	u32 status = IWL_UMAC_SCAN_ABORT_STATUS_NOT_FOUND;
 
 	lockdep_assert_held(&mvm->mutex);
 
+	*wait = true;
+
 	/* We should always get a valid index here, because we already
 	 * checked that this type of scan was running in the generic
 	 * code.
@@ -3332,17 +3339,28 @@ static int iwl_mvm_umac_scan_abort(struct iwl_mvm *mvm, int type)
 	if (WARN_ON_ONCE(uid < 0))
 		return uid;
 
-	cmd.uid = cpu_to_le32(uid);
+	abort_cmd.uid = cpu_to_le32(uid);
 
 	IWL_DEBUG_SCAN(mvm, "Sending scan abort, uid %u\n", uid);
 
-	ret = iwl_mvm_send_cmd_pdu(mvm,
-				   WIDE_ID(IWL_ALWAYS_LONG_GROUP, SCAN_ABORT_UMAC),
-				   CMD_SEND_IN_RFKILL, sizeof(cmd), &cmd);
+	ret = iwl_mvm_send_cmd_status(mvm, &cmd, &status);
+
+	IWL_DEBUG_SCAN(mvm, "Scan abort: ret=%d, status=%u\n", ret, status);
 	if (!ret)
 		mvm->scan_uid_status[uid] = type << IWL_MVM_SCAN_STOPPING_SHIFT;
 
-	IWL_DEBUG_SCAN(mvm, "Scan abort: ret=%d\n", ret);
+	/* Handle the case that the FW is no longer familiar with the scan that
+	 * is to be stopped. In such a case, it is expected that the scan
+	 * complete notification was already received but not yet processed.
+	 * In such a case, there is no need to wait for a scan complete
+	 * notification and the flow should continue similar to the case that
+	 * the scan was really aborted.
+	 */
+	if (status == IWL_UMAC_SCAN_ABORT_STATUS_NOT_FOUND) {
+		mvm->scan_uid_status[uid] = type << IWL_MVM_SCAN_STOPPING_SHIFT;
+		*wait = false;
+	}
+
 	return ret;
 }
 
@@ -3352,6 +3370,7 @@ static int iwl_mvm_scan_stop_wait(struct iwl_mvm *mvm, int type)
 	static const u16 scan_done_notif[] = { SCAN_COMPLETE_UMAC,
 					      SCAN_OFFLOAD_COMPLETE, };
 	int ret;
+	bool wait = true;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -3363,7 +3382,7 @@ static int iwl_mvm_scan_stop_wait(struct iwl_mvm *mvm, int type)
 	IWL_DEBUG_SCAN(mvm, "Preparing to stop scan, type %x\n", type);
 
 	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN))
-		ret = iwl_mvm_umac_scan_abort(mvm, type);
+		ret = iwl_mvm_umac_scan_abort(mvm, type, &wait);
 	else
 		ret = iwl_mvm_lmac_scan_abort(mvm);
 
@@ -3371,6 +3390,10 @@ static int iwl_mvm_scan_stop_wait(struct iwl_mvm *mvm, int type)
 		IWL_DEBUG_SCAN(mvm, "couldn't stop scan type %d\n", type);
 		iwl_remove_notification(&mvm->notif_wait, &wait_scan_done);
 		return ret;
+	} else if (!wait) {
+		IWL_DEBUG_SCAN(mvm, "no need to wait for scan type %d\n", type);
+		iwl_remove_notification(&mvm->notif_wait, &wait_scan_done);
+		return 0;
 	}
 
 	return iwl_wait_notification(&mvm->notif_wait, &wait_scan_done,
@@ -3455,7 +3478,7 @@ void iwl_mvm_report_scan_aborted(struct iwl_mvm *mvm)
 			 * restart_hw, so do not report if FW is about to be
 			 * restarted.
 			 */
-			if (!mvm->fw_restart)
+			if (!iwlwifi_mod_params.fw_restart)
 				ieee80211_sched_scan_stopped(mvm->hw);
 			mvm->sched_scan_pass_all = SCHED_SCAN_PASS_ALL_DISABLED;
 			mvm->scan_uid_status[uid] = 0;
@@ -3506,7 +3529,7 @@ void iwl_mvm_report_scan_aborted(struct iwl_mvm *mvm)
 		 * restarted.
 		 */
 		if ((mvm->scan_status & IWL_MVM_SCAN_SCHED) &&
-		    !mvm->fw_restart) {
+		    !iwlwifi_mod_params.fw_restart) {
 			ieee80211_sched_scan_stopped(mvm->hw);
 			mvm->sched_scan_pass_all = SCHED_SCAN_PASS_ALL_DISABLED;
 		}
@@ -3524,7 +3547,7 @@ int iwl_mvm_scan_stop(struct iwl_mvm *mvm, int type, bool notify)
 	if (!(mvm->scan_status & type))
 		return 0;
 
-	if (!test_bit(STATUS_DEVICE_ENABLED, &mvm->trans->status)) {
+	if (!iwl_trans_device_enabled(mvm->trans)) {
 		ret = 0;
 		goto out;
 	}
@@ -3575,7 +3598,8 @@ static int iwl_mvm_int_mlo_scan_start(struct iwl_mvm *mvm,
 	IWL_DEBUG_SCAN(mvm, "Starting Internal MLO scan: n_channels=%zu\n",
 		       n_channels);
 
-	if (!vif->cfg.assoc || !ieee80211_vif_is_mld(vif))
+	if (!vif->cfg.assoc || !ieee80211_vif_is_mld(vif) ||
+	    hweight16(vif->valid_links) == 1)
 		return -EINVAL;
 
 	size = struct_size(req, channels, n_channels);
@@ -3662,117 +3686,6 @@ static int iwl_mvm_chanidx_from_phy(struct iwl_mvm *mvm,
 	return -EINVAL;
 }
 
-static u32 iwl_mvm_div_by_db(u32 value, u8 db)
-{
-	/*
-	 * 2^32 * 10**(i / 10) for i = [1, 10], skipping 0 and simply stopping
-	 * at 10 dB and looping instead of using a much larger table.
-	 *
-	 * Using 64 bit math is overkill, but means the helper does not require
-	 * a limit on the input range.
-	 */
-	static const u32 db_to_val[] = {
-		0xcb59185e, 0xa1866ba8, 0x804dce7a, 0x65ea59fe, 0x50f44d89,
-		0x404de61f, 0x331426af, 0x2892c18b, 0x203a7e5b, 0x1999999a,
-	};
-
-	while (value && db > 0) {
-		u8 change = min_t(u8, db, ARRAY_SIZE(db_to_val));
-
-		value = (((u64)value) * db_to_val[change - 1]) >> 32;
-
-		db -= change;
-	}
-
-	return value;
-}
-
-VISIBLE_IF_IWLWIFI_KUNIT s8
-iwl_mvm_average_dbm_values(const struct iwl_umac_scan_channel_survey_notif *notif)
-{
-	s8 average_magnitude;
-	u32 average_factor;
-	s8 sum_magnitude = -128;
-	u32 sum_factor = 0;
-	int i, count = 0;
-
-	/*
-	 * To properly average the decibel values (signal values given in dBm)
-	 * we need to do the math in linear space.  Doing a linear average of
-	 * dB (dBm) values is a bit annoying though due to the large range of
-	 * at least -10 to -110 dBm that will not fit into a 32 bit integer.
-	 *
-	 * A 64 bit integer should be sufficient, but then we still have the
-	 * problem that there are no directly usable utility functions
-	 * available.
-	 *
-	 * So, lets not deal with that and instead do much of the calculation
-	 * with a 16.16 fixed point integer along with a base in dBm. 16.16 bit
-	 * gives us plenty of head-room for adding up a few values and even
-	 * doing some math on it. And the tail should be accurate enough too
-	 * (1/2^16 is somewhere around -48 dB, so effectively zero).
-	 *
-	 * i.e. the real value of sum is:
-	 *      sum = sum_factor / 2^16 * 10^(sum_magnitude / 10) mW
-	 *
-	 * However, that does mean we need to be able to bring two values to
-	 * a common base, so we need a helper for that.
-	 *
-	 * Note that this function takes an input with unsigned negative dBm
-	 * values but returns a signed dBm (i.e. a negative value).
-	 */
-
-	for (i = 0; i < ARRAY_SIZE(notif->noise); i++) {
-		s8 val_magnitude;
-		u32 val_factor;
-
-		if (notif->noise[i] == 0xff)
-			continue;
-
-		val_factor = 0x10000;
-		val_magnitude = -notif->noise[i];
-
-		if (val_magnitude <= sum_magnitude) {
-			u8 div_db = sum_magnitude - val_magnitude;
-
-			val_factor = iwl_mvm_div_by_db(val_factor, div_db);
-			val_magnitude = sum_magnitude;
-		} else {
-			u8 div_db = val_magnitude - sum_magnitude;
-
-			sum_factor = iwl_mvm_div_by_db(sum_factor, div_db);
-			sum_magnitude = val_magnitude;
-		}
-
-		sum_factor += val_factor;
-		count++;
-	}
-
-	/* No valid noise measurement, return a very high noise level */
-	if (count == 0)
-		return 0;
-
-	average_magnitude = sum_magnitude;
-	average_factor = sum_factor / count;
-
-	/*
-	 * average_factor will be a number smaller than 1.0 (0x10000) at this
-	 * point. What we need to do now is to adjust average_magnitude so that
-	 * average_factor is between -0.5 dB and 0.5 dB.
-	 *
-	 * Just do -1 dB steps and find the point where
-	 *   -0.5 dB * -i dB = 0x10000 * 10^(-0.5/10) / i dB
-	 *                   = div_by_db(0xe429, i)
-	 * is smaller than average_factor.
-	 */
-	for (i = 0; average_factor < iwl_mvm_div_by_db(0xe429, i); i++) {
-		/* nothing */
-	}
-
-	return average_magnitude - i;
-}
-EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mvm_average_dbm_values);
-
 void iwl_mvm_rx_channel_survey_notif(struct iwl_mvm *mvm,
 				     struct iwl_rx_cmd_buffer *rxb)
 {
@@ -3830,5 +3743,6 @@ void iwl_mvm_rx_channel_survey_notif(struct iwl_mvm *mvm,
 	info->time_busy = le32_to_cpu(notif->busy_time);
 	info->time_rx = le32_to_cpu(notif->rx_time);
 	info->time_tx = le32_to_cpu(notif->tx_time);
-	info->noise = iwl_mvm_average_dbm_values(notif);
+	info->noise =
+		iwl_average_neg_dbm(notif->noise, ARRAY_SIZE(notif->noise));
 }
diff --git a/sys/contrib/dev/iwlwifi/mvm/sta.c b/sys/contrib/dev/iwlwifi/mvm/sta.c
index f47b3f6b8f64..3fc774c2ca39 100644
--- a/sys/contrib/dev/iwlwifi/mvm/sta.c
+++ b/sys/contrib/dev/iwlwifi/mvm/sta.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2015, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2015, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -32,7 +32,7 @@ int iwl_mvm_find_free_sta_id(struct iwl_mvm *mvm, enum nl80211_iftype iftype)
 	int sta_id;
 	u32 reserved_ids = 0;
 
-	BUILD_BUG_ON(IWL_MVM_STATION_COUNT_MAX > 32);
+	BUILD_BUG_ON(IWL_STATION_COUNT_MAX > 32);
 	WARN_ON_ONCE(test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status));
 
 	lockdep_assert_held(&mvm->mutex);
@@ -50,7 +50,7 @@ int iwl_mvm_find_free_sta_id(struct iwl_mvm *mvm, enum nl80211_iftype iftype)
 					       lockdep_is_held(&mvm->mutex)))
 			return sta_id;
 	}
-	return IWL_MVM_INVALID_STA;
+	return IWL_INVALID_STA;
 }
 
 /* Calculate the ampdu density and max size */
@@ -256,7 +256,7 @@ int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 static void iwl_mvm_rx_agg_session_expired(struct timer_list *t)
 {
 	struct iwl_mvm_baid_data *data =
-		from_timer(data, t, session_timer);
+		timer_container_of(data, t, session_timer);
 	struct iwl_mvm_baid_data __rcu **rcu_ptr = data->rcu_ptr;
 	struct iwl_mvm_baid_data *ba_data;
 	struct ieee80211_sta *sta;
@@ -794,10 +794,10 @@ static int iwl_mvm_find_free_queue(struct iwl_mvm *mvm, u8 sta_id,
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (WARN(maxq >= mvm->trans->trans_cfg->base_params->num_of_queues,
+	if (WARN(maxq >= mvm->trans->mac_cfg->base->num_of_queues,
 		 "max queue %d >= num_of_queues (%d)", maxq,
-		 mvm->trans->trans_cfg->base_params->num_of_queues))
-		maxq = mvm->trans->trans_cfg->base_params->num_of_queues - 1;
+		 mvm->trans->mac_cfg->base->num_of_queues))
+		maxq = mvm->trans->mac_cfg->base->num_of_queues - 1;
 
 	/* This should not be hit with new TX path */
 	if (WARN_ON(iwl_mvm_has_new_tx_api(mvm)))
@@ -855,7 +855,7 @@ int iwl_mvm_tvqm_enable_txq(struct iwl_mvm *mvm,
 	if (tid == IWL_MAX_TID_COUNT) {
 		tid = IWL_MGMT_TID;
 		size = max_t(u32, IWL_MGMT_QUEUE_SIZE,
-			     mvm->trans->cfg->min_txq_size);
+			     mvm->trans->mac_cfg->base->min_txq_size);
 	} else {
 		size = iwl_mvm_get_queue_size(sta);
 	}
@@ -903,7 +903,7 @@ static int iwl_mvm_sta_alloc_queue_tvqm(struct iwl_mvm *mvm,
 	struct iwl_mvm_txq *mvmtxq =
 		iwl_mvm_txq_from_tid(sta, tid);
 	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, mvmsta->vif, false, false);
+		iwl_mvm_get_wd_timeout(mvm, mvmsta->vif);
 	int queue = -1;
 
 	lockdep_assert_held(&mvm->mutex);
@@ -1083,7 +1083,7 @@ static void iwl_mvm_unshare_queue(struct iwl_mvm *mvm, int queue)
 		return;
 
 	mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	wdg_timeout = iwl_mvm_get_wd_timeout(mvm, mvmsta->vif, false, false);
+	wdg_timeout = iwl_mvm_get_wd_timeout(mvm, mvmsta->vif);
 
 	ssn = IEEE80211_SEQ_TO_SN(mvmsta->tid_data[tid].seq_number);
 
@@ -1219,7 +1219,7 @@ static bool iwl_mvm_remove_inactive_tids(struct iwl_mvm *mvm,
  * can be unshared and finding one (and only one) that can be
  * reused.
  * This function is also invoked as a sort of clean-up task,
- * in which case @alloc_for_sta is IWL_MVM_INVALID_STA.
+ * in which case @alloc_for_sta is IWL_INVALID_STA.
  *
  * Returns the queue number, or -ENOSPC.
  */
@@ -1312,7 +1312,7 @@ static int iwl_mvm_inactivity_check(struct iwl_mvm *mvm, u8 alloc_for_sta)
 
 	rcu_read_unlock();
 
-	if (free_queue >= 0 && alloc_for_sta != IWL_MVM_INVALID_STA) {
+	if (free_queue >= 0 && alloc_for_sta != IWL_INVALID_STA) {
 		ret = iwl_mvm_free_inactive_queue(mvm, free_queue, queue_owner,
 						  alloc_for_sta);
 		if (ret)
@@ -1333,7 +1333,7 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 		.frame_limit = IWL_FRAME_LIMIT,
 	};
 	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, mvmsta->vif, false, false);
+		iwl_mvm_get_wd_timeout(mvm, mvmsta->vif);
 	int queue = -1;
 	u16 queue_tmp;
 	unsigned long disable_agg_tids = 0;
@@ -1523,9 +1523,14 @@ void iwl_mvm_add_new_dqa_stream_wk(struct work_struct *wk)
 	struct iwl_mvm *mvm = container_of(wk, struct iwl_mvm,
 					   add_stream_wk);
 
-	mutex_lock(&mvm->mutex);
+	guard(mvm)(mvm);
 
-	iwl_mvm_inactivity_check(mvm, IWL_MVM_INVALID_STA);
+	/* will reschedule to run after restart */
+	if (test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status) ||
+	    test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
+		return;
+
+	iwl_mvm_inactivity_check(mvm, IWL_INVALID_STA);
 
 	while (!list_empty(&mvm->add_stream_txqs)) {
 		struct iwl_mvm_txq *mvmtxq;
@@ -1567,8 +1572,6 @@ void iwl_mvm_add_new_dqa_stream_wk(struct work_struct *wk)
 		iwl_mvm_mac_itxq_xmit(mvm->hw, txq);
 		local_bh_enable();
 	}
-
-	mutex_unlock(&mvm->mutex);
 }
 
 static int iwl_mvm_reserve_sta_stream(struct iwl_mvm *mvm,
@@ -1583,7 +1586,7 @@ static int iwl_mvm_reserve_sta_stream(struct iwl_mvm *mvm,
 		return 0;
 
 	/* run the general cleanup/unsharing of queues */
-	iwl_mvm_inactivity_check(mvm, IWL_MVM_INVALID_STA);
+	iwl_mvm_inactivity_check(mvm, IWL_INVALID_STA);
 
 	/* Make sure we have free resources for this STA */
 	if (vif_type == NL80211_IFTYPE_STATION && !sta->tdls &&
@@ -1625,7 +1628,7 @@ void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	unsigned int wdg =
-		iwl_mvm_get_wd_timeout(mvm, mvm_sta->vif, false, false);
+		iwl_mvm_get_wd_timeout(mvm, mvm_sta->vif);
 	int i;
 	struct iwl_trans_txq_scd_cfg cfg = {
 		.sta_id = mvm_sta->deflink.sta_id,
@@ -1759,13 +1762,13 @@ int iwl_mvm_sta_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 * this function
 	 */
 	if (!mvm->mld_api_is_used) {
-		if (WARN_ON(sta_id == IWL_MVM_INVALID_STA))
+		if (WARN_ON(sta_id == IWL_INVALID_STA))
 			return -EINVAL;
 
 		mvm_sta->deflink.sta_id = sta_id;
 		rcu_assign_pointer(mvm_sta->link[0], &mvm_sta->deflink);
 
-		if (!mvm->trans->trans_cfg->gen2)
+		if (!mvm->trans->mac_cfg->gen2)
 			mvm_sta->deflink.lq_sta.rs_drv.pers.max_agg_bufsize =
 				LINK_QUAL_AGG_FRAME_LIMIT_DEF;
 		else
@@ -1798,7 +1801,7 @@ int iwl_mvm_sta_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (iwl_mvm_has_new_rx_api(mvm)) {
 		int q;
 
-		dup_data = kcalloc(mvm->trans->num_rx_queues,
+		dup_data = kcalloc(mvm->trans->info.num_rxqs,
 				   sizeof(*dup_data), GFP_KERNEL);
 		if (!dup_data)
 			return -ENOMEM;
@@ -1811,7 +1814,7 @@ int iwl_mvm_sta_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		 * This thus allows receiving a packet with seqno 0 and the
 		 * retry bit set as the very first packet on a new TID.
 		 */
-		for (q = 0; q < mvm->trans->num_rx_queues; q++)
+		for (q = 0; q < mvm->trans->info.num_rxqs; q++)
 			memset(dup_data[q].last_seq, 0xff,
 			       sizeof(dup_data[q].last_seq));
 		mvm_sta->dup_data = dup_data;
@@ -1839,11 +1842,11 @@ int iwl_mvm_sta_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (vif->type == NL80211_IFTYPE_STATION && !vif->p2p &&
 	    !sta->tdls && ieee80211_vif_is_mld(vif)) {
 		mvm_sta->mpdu_counters =
-			kcalloc(mvm->trans->num_rx_queues,
+			kcalloc(mvm->trans->info.num_rxqs,
 				sizeof(*mvm_sta->mpdu_counters),
 				GFP_KERNEL);
 		if (mvm_sta->mpdu_counters)
-			for (int q = 0; q < mvm->trans->num_rx_queues; q++)
+			for (int q = 0; q < mvm->trans->info.num_rxqs; q++)
 				spin_lock_init(&mvm_sta->mpdu_counters[q].lock);
 	}
 
@@ -1868,7 +1871,7 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 	else
 		sta_id = mvm_sta->deflink.sta_id;
 
-	if (sta_id == IWL_MVM_INVALID_STA)
+	if (sta_id == IWL_INVALID_STA)
 		return -ENOSPC;
 
 	spin_lock_init(&mvm_sta->lock);
@@ -1906,10 +1909,10 @@ update_fw:
 
 	if (vif->type == NL80211_IFTYPE_STATION) {
 		if (!sta->tdls) {
-			WARN_ON(mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA);
+			WARN_ON(mvmvif->deflink.ap_sta_id != IWL_INVALID_STA);
 			mvmvif->deflink.ap_sta_id = sta_id;
 		} else {
-			WARN_ON(mvmvif->deflink.ap_sta_id == IWL_MVM_INVALID_STA);
+			WARN_ON(mvmvif->deflink.ap_sta_id == IWL_INVALID_STA);
 		}
 	}
 
@@ -2053,9 +2056,9 @@ int iwl_mvm_wait_sta_queues_empty(struct iwl_mvm *mvm,
  * Returns if we're done with removing the station, either
  * with error or success
  */
-bool iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+void iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		     struct ieee80211_sta *sta,
-		     struct ieee80211_link_sta *link_sta, int *ret)
+		     struct ieee80211_link_sta *link_sta)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_vif_link_info *mvm_link =
@@ -2071,39 +2074,13 @@ bool iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 					  lockdep_is_held(&mvm->mutex));
 	sta_id = mvm_link_sta->sta_id;
 
-	/* If there is a TXQ still marked as reserved - free it */
-	if (mvm_sta->reserved_queue != IEEE80211_INVAL_HW_QUEUE) {
-		u8 reserved_txq = mvm_sta->reserved_queue;
-		enum iwl_mvm_queue_status *status;
-
-		/*
-		 * If no traffic has gone through the reserved TXQ - it
-		 * is still marked as IWL_MVM_QUEUE_RESERVED, and
-		 * should be manually marked as free again
-		 */
-		status = &mvm->queue_info[reserved_txq].status;
-		if (WARN((*status != IWL_MVM_QUEUE_RESERVED) &&
-			 (*status != IWL_MVM_QUEUE_FREE),
-			 "sta_id %d reserved txq %d status %d",
-			 sta_id, reserved_txq, *status)) {
-			*ret = -EINVAL;
-			return true;
-		}
-
-		*status = IWL_MVM_QUEUE_FREE;
-	}
-
 	if (vif->type == NL80211_IFTYPE_STATION &&
 	    mvm_link->ap_sta_id == sta_id) {
-		/* if associated - we can't remove the AP STA now */
-		if (vif->cfg.assoc)
-			return true;
-
 		/* first remove remaining keys */
-		iwl_mvm_sec_key_remove_ap(mvm, vif, mvm_link, 0);
+		iwl_mvm_sec_key_remove_ap(mvm, vif, mvm_link,
+					  link_sta->link_id);
 
-		/* unassoc - go ahead - remove the AP STA now */
-		mvm_link->ap_sta_id = IWL_MVM_INVALID_STA;
+		mvm_link->ap_sta_id = IWL_INVALID_STA;
 	}
 
 	/*
@@ -2111,11 +2088,9 @@ bool iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 * before the STA is removed.
 	 */
 	if (WARN_ON_ONCE(mvm->tdls_cs.peer.sta_id == sta_id)) {
-		mvm->tdls_cs.peer.sta_id = IWL_MVM_INVALID_STA;
+		mvm->tdls_cs.peer.sta_id = IWL_INVALID_STA;
 		cancel_delayed_work(&mvm->tdls_cs.dwork);
 	}
-
-	return false;
 }
 
 int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
@@ -2151,8 +2126,27 @@ int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 
 	iwl_mvm_disable_sta_queues(mvm, vif, sta);
 
-	if (iwl_mvm_sta_del(mvm, vif, sta, &sta->deflink, &ret))
-		return ret;
+	/* If there is a TXQ still marked as reserved - free it */
+	if (mvm_sta->reserved_queue != IEEE80211_INVAL_HW_QUEUE) {
+		u8 reserved_txq = mvm_sta->reserved_queue;
+		enum iwl_mvm_queue_status *status;
+
+		/*
+		 * If no traffic has gone through the reserved TXQ - it
+		 * is still marked as IWL_MVM_QUEUE_RESERVED, and
+		 * should be manually marked as free again
+		 */
+		status = &mvm->queue_info[reserved_txq].status;
+		if (WARN((*status != IWL_MVM_QUEUE_RESERVED) &&
+			 (*status != IWL_MVM_QUEUE_FREE),
+			 "sta_id %d reserved txq %d status %d",
+			 mvm_sta->deflink.sta_id, reserved_txq, *status))
+			return -EINVAL;
+
+		*status = IWL_MVM_QUEUE_FREE;
+	}
+
+	iwl_mvm_sta_del(mvm, vif, sta, &sta->deflink);
 
 	ret = iwl_mvm_rm_sta_common(mvm, mvm_sta->deflink.sta_id);
 	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta->deflink.sta_id], NULL);
@@ -2178,9 +2172,9 @@ int iwl_mvm_allocate_int_sta(struct iwl_mvm *mvm,
 			     u8 type)
 {
 	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) ||
-	    sta->sta_id == IWL_MVM_INVALID_STA) {
+	    sta->sta_id == IWL_INVALID_STA) {
 		sta->sta_id = iwl_mvm_find_free_sta_id(mvm, iftype);
-		if (WARN_ON_ONCE(sta->sta_id == IWL_MVM_INVALID_STA))
+		if (WARN_ON_ONCE(sta->sta_id == IWL_INVALID_STA))
 			return -ENOSPC;
 	}
 
@@ -2196,14 +2190,14 @@ void iwl_mvm_dealloc_int_sta(struct iwl_mvm *mvm, struct iwl_mvm_int_sta *sta)
 {
 	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[sta->sta_id], NULL);
 	memset(sta, 0, sizeof(struct iwl_mvm_int_sta));
-	sta->sta_id = IWL_MVM_INVALID_STA;
+	sta->sta_id = IWL_INVALID_STA;
 }
 
 static void iwl_mvm_enable_aux_snif_queue(struct iwl_mvm *mvm, u16 queue,
 					  u8 sta_id, u8 fifo)
 {
 	unsigned int wdg_timeout =
-		mvm->trans->trans_cfg->base_params->wd_timeout;
+		mvm->trans->mac_cfg->base->wd_timeout;
 	struct iwl_trans_txq_scd_cfg cfg = {
 		.fifo = fifo,
 		.sta_id = sta_id,
@@ -2220,7 +2214,7 @@ static void iwl_mvm_enable_aux_snif_queue(struct iwl_mvm *mvm, u16 queue,
 static int iwl_mvm_enable_aux_snif_queue_tvqm(struct iwl_mvm *mvm, u8 sta_id)
 {
 	unsigned int wdg_timeout =
-		mvm->trans->trans_cfg->base_params->wd_timeout;
+		mvm->trans->mac_cfg->base->wd_timeout;
 
 	WARN_ON(!iwl_mvm_has_new_tx_api(mvm));
 
@@ -2314,7 +2308,7 @@ int iwl_mvm_rm_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (WARN_ON_ONCE(mvm->snif_sta.sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(mvm->snif_sta.sta_id == IWL_INVALID_STA))
 		return -EINVAL;
 
 	iwl_mvm_disable_txq(mvm, NULL, mvm->snif_sta.sta_id,
@@ -2332,7 +2326,7 @@ int iwl_mvm_rm_aux_sta(struct iwl_mvm *mvm)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (WARN_ON_ONCE(mvm->aux_sta.sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(mvm->aux_sta.sta_id == IWL_INVALID_STA))
 		return -EINVAL;
 
 	iwl_mvm_disable_txq(mvm, NULL, mvm->aux_sta.sta_id,
@@ -2367,7 +2361,7 @@ int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	int queue;
 	int ret;
 	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, vif, false, false);
+		iwl_mvm_get_wd_timeout(mvm, vif);
 	struct iwl_trans_txq_scd_cfg cfg = {
 		.fifo = IWL_MVM_TX_FIFO_VO,
 		.sta_id = mvmvif->deflink.bcast_sta.sta_id,
@@ -2397,7 +2391,7 @@ int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	if (vif->type == NL80211_IFTYPE_ADHOC)
 		baddr = vif->bss_conf.bssid;
 
-	if (WARN_ON_ONCE(bsta->sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(bsta->sta_id == IWL_INVALID_STA))
 		return -ENOSPC;
 
 	ret = iwl_mvm_add_int_sta_common(mvm, bsta, baddr,
@@ -2475,7 +2469,7 @@ void iwl_mvm_free_bcast_sta_queues(struct iwl_mvm *mvm,
 	mvmvif->deflink.bcast_sta.tfd_queue_msk &= ~BIT(queue);
 }
 
-/* Send the FW a request to remove the station from it's internal data
+/* Send the FW a request to remove the station from its internal data
  * structures, but DO NOT remove the entry from the local data structures. */
 int iwl_mvm_send_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
@@ -2538,7 +2532,7 @@ void iwl_mvm_dealloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 }
 
 /*
- * Send the FW a request to remove the station from it's internal data
+ * Send the FW a request to remove the station from its internal data
  * structures, and in addition remove it from the local data structure.
  */
 int iwl_mvm_rm_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
@@ -2576,7 +2570,7 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		.aggregate = false,
 		.frame_limit = IWL_FRAME_LIMIT,
 	};
-	unsigned int timeout = iwl_mvm_get_wd_timeout(mvm, vif, false, false);
+	unsigned int timeout = iwl_mvm_get_wd_timeout(mvm, vif);
 	int ret;
 
 	lockdep_assert_held(&mvm->mutex);
@@ -2652,7 +2646,7 @@ static int __iwl_mvm_remove_sta_key(struct iwl_mvm *mvm, u8 sta_id,
 	u32 status;
 
 	/* This is a valid situation for GTK removal */
-	if (sta_id == IWL_MVM_INVALID_STA)
+	if (sta_id == IWL_INVALID_STA)
 		return 0;
 
 	key_flags = cpu_to_le16((keyconf->keyidx << STA_KEY_FLG_KEYID_POS) &
@@ -2691,7 +2685,7 @@ static int __iwl_mvm_remove_sta_key(struct iwl_mvm *mvm, u8 sta_id,
 }
 
 /*
- * Send the FW a request to remove the station from it's internal data
+ * Send the FW a request to remove the station from its internal data
  * structures, and in addition remove it from the local data structure.
  */
 int iwl_mvm_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
@@ -2731,7 +2725,7 @@ static void iwl_mvm_free_reorder(struct iwl_mvm *mvm,
 
 	iwl_mvm_sync_rxq_del_ba(mvm, data->baid);
 
-	for (i = 0; i < mvm->trans->num_rx_queues; i++) {
+	for (i = 0; i < mvm->trans->info.num_rxqs; i++) {
 		int j;
 		struct iwl_mvm_reorder_buffer *reorder_buf =
 			&data->reorder_buf[i];
@@ -2764,7 +2758,7 @@ static void iwl_mvm_init_reorder_buffer(struct iwl_mvm *mvm,
 {
 	int i;
 
-	for (i = 0; i < mvm->trans->num_rx_queues; i++) {
+	for (i = 0; i < mvm->trans->info.num_rxqs; i++) {
 		struct iwl_mvm_reorder_buffer *reorder_buf =
 			&data->reorder_buf[i];
 		struct iwl_mvm_reorder_buf_entry *entries =
@@ -2920,7 +2914,7 @@ int iwl_mvm_sta_rx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		/*
 		 * The division below will be OK if either the cache line size
 		 * can be divided by the entry size (ALIGN will round up) or if
-		 * if the entry size can be divided by the cache line size, in
+		 * the entry size can be divided by the cache line size, in
 		 * which case the ALIGN() will do nothing.
 		 */
 		BUILD_BUG_ON(SMP_CACHE_BYTES % sizeof(baid_data->entries[0]) &&
@@ -2939,7 +2933,7 @@ int iwl_mvm_sta_rx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		 * before starting the BA session in the firmware
 		 */
 		baid_data = kzalloc(sizeof(*baid_data) +
-				    mvm->trans->num_rx_queues *
+				    mvm->trans->info.num_rxqs *
 				    reorder_buf_size,
 				    GFP_KERNEL);
 		if (!baid_data)
@@ -3191,7 +3185,7 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 * to align the wrap around of ssn so we compare relevant values.
 	 */
 	normalized_ssn = tid_data->ssn;
-	if (mvm->trans->trans_cfg->gen2)
+	if (mvm->trans->mac_cfg->gen2)
 		normalized_ssn &= 0xff;
 
 	if (normalized_ssn == tid_data->next_reclaimed) {
@@ -3215,7 +3209,7 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_tid_data *tid_data = &mvmsta->tid_data[tid];
 	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, vif, sta->tdls, false);
+		iwl_mvm_get_wd_timeout(mvm, vif);
 	int queue, ret;
 	bool alloc_queue = true;
 	enum iwl_mvm_queue_status queue_status;
@@ -3522,7 +3516,7 @@ static struct iwl_mvm_sta *iwl_mvm_get_key_sta(struct iwl_mvm *mvm,
 	 * station ID, then use AP's station ID.
 	 */
 	if (vif->type == NL80211_IFTYPE_STATION &&
-	    mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA) {
+	    mvmvif->deflink.ap_sta_id != IWL_INVALID_STA) {
 		u8 sta_id = mvmvif->deflink.ap_sta_id;
 
 		sta = rcu_dereference_check(mvm->fw_id_to_mac_id[sta_id],
@@ -3577,7 +3571,7 @@ static int iwl_mvm_send_sta_key(struct iwl_mvm *mvm,
 	int api_ver = iwl_fw_lookup_cmd_ver(mvm->fw, ADD_STA_KEY,
 					    new_api ? 2 : 1);
 
-	if (sta_id == IWL_MVM_INVALID_STA)
+	if (sta_id == IWL_INVALID_STA)
 		return -EINVAL;
 
 	keyidx = (key->keyidx << STA_KEY_FLG_KEYID_POS) &
@@ -3736,7 +3730,7 @@ static int iwl_mvm_send_sta_igtk(struct iwl_mvm *mvm,
 
 	if (remove_key) {
 		/* This is a valid situation for IGTK */
-		if (sta_id == IWL_MVM_INVALID_STA)
+		if (sta_id == IWL_INVALID_STA)
 			return 0;
 
 		igtk_cmd.ctrl_flags |= cpu_to_le32(STA_KEY_NOT_VALID);
@@ -3803,7 +3797,7 @@ static inline u8 *iwl_mvm_get_mac_addr(struct iwl_mvm *mvm,
 		return sta->addr;
 
 	if (vif->type == NL80211_IFTYPE_STATION &&
-	    mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA) {
+	    mvmvif->deflink.ap_sta_id != IWL_INVALID_STA) {
 		u8 sta_id = mvmvif->deflink.ap_sta_id;
 		sta = rcu_dereference_protected(mvm->fw_id_to_mac_id[sta_id],
 						lockdep_is_held(&mvm->mutex));
@@ -3873,7 +3867,7 @@ int iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 {
 	bool mcast = !(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE);
 	struct iwl_mvm_sta *mvm_sta;
-	u8 sta_id = IWL_MVM_INVALID_STA;
+	u8 sta_id = IWL_INVALID_STA;
 	int ret;
 	static const u8 __maybe_unused zero_addr[ETH_ALEN] = {0};
 
@@ -3974,7 +3968,7 @@ int iwl_mvm_remove_sta_key(struct iwl_mvm *mvm,
 {
 	bool mcast = !(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE);
 	struct iwl_mvm_sta *mvm_sta;
-	u8 sta_id = IWL_MVM_INVALID_STA;
+	u8 sta_id = IWL_INVALID_STA;
 	int ret, i;
 
 	lockdep_assert_held(&mvm->mutex);
@@ -4281,7 +4275,7 @@ void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 		return;
 
 	/* Need to block/unblock also multicast station */
-	if (mvmvif->deflink.mcast_sta.sta_id != IWL_MVM_INVALID_STA)
+	if (mvmvif->deflink.mcast_sta.sta_id != IWL_INVALID_STA)
 		iwl_mvm_int_sta_modify_disable_tx(mvm, mvmvif,
 						  &mvmvif->deflink.mcast_sta,
 						  disable);
@@ -4290,7 +4284,7 @@ void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 	 * Only unblock the broadcast station (FW blocks it for immediate
 	 * quiet, not the driver)
 	 */
-	if (!disable && mvmvif->deflink.bcast_sta.sta_id != IWL_MVM_INVALID_STA)
+	if (!disable && mvmvif->deflink.bcast_sta.sta_id != IWL_INVALID_STA)
 		iwl_mvm_int_sta_modify_disable_tx(mvm, mvmvif,
 						  &mvmvif->deflink.bcast_sta,
 						  disable);
@@ -4319,72 +4313,12 @@ u16 iwl_mvm_tid_queued(struct iwl_mvm *mvm, struct iwl_mvm_tid_data *tid_data)
 	 * In 22000 HW, the next_reclaimed index is only 8 bit, so we'll need
 	 * to align the wrap around of ssn so we compare relevant values.
 	 */
-	if (mvm->trans->trans_cfg->gen2)
+	if (mvm->trans->mac_cfg->gen2)
 		sn &= 0xff;
 
 	return ieee80211_sn_sub(sn, tid_data->next_reclaimed);
 }
 
-#if defined(__linux__)
-int iwl_mvm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			 struct iwl_mvm_int_sta *sta, u8 *addr, u32 cipher,
-			 u8 *key, u32 key_len,
-			 struct ieee80211_key_conf *keyconf)
-{
-	int ret;
-	u16 queue;
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, vif, false, false);
-	bool mld = iwl_mvm_has_mld_api(mvm->fw);
-	u32 type = mld ? STATION_TYPE_PEER : IWL_STA_LINK;
-
-	ret = iwl_mvm_allocate_int_sta(mvm, sta, 0,
-				       NL80211_IFTYPE_UNSPECIFIED, type);
-	if (ret)
-		return ret;
-
-	if (mld)
-		ret = iwl_mvm_mld_add_int_sta_with_queue(mvm, sta, addr,
-							 mvmvif->deflink.fw_link_id,
-							 &queue,
-							 IWL_MAX_TID_COUNT,
-							 &wdg_timeout);
-	else
-		ret = iwl_mvm_add_int_sta_with_queue(mvm, mvmvif->id,
-						     mvmvif->color, addr, sta,
-						     &queue,
-						     IWL_MVM_TX_FIFO_BE);
-	if (ret)
-		goto out;
-
-	keyconf->cipher = cipher;
-	memcpy(keyconf->key, key, key_len);
-	keyconf->keylen = key_len;
-	keyconf->flags = IEEE80211_KEY_FLAG_PAIRWISE;
-
-	if (mld) {
-		/* The MFP flag is set according to the station mfp field. Since
-		 * we don't have a station, set it manually.
-		 */
-		u32 key_flags =
-			iwl_mvm_get_sec_flags(mvm, vif, NULL, keyconf) |
-			IWL_SEC_KEY_FLAG_MFP;
-		u32 sta_mask = BIT(sta->sta_id);
-
-		ret = iwl_mvm_mld_send_key(mvm, sta_mask, key_flags, keyconf);
-	} else {
-		ret = iwl_mvm_send_sta_key(mvm, sta->sta_id, keyconf, false,
-					   0, NULL, 0, 0, true);
-	}
-
-out:
-	if (ret)
-		iwl_mvm_dealloc_int_sta(mvm, sta);
-	return ret;
-}
-#endif
-
 void iwl_mvm_cancel_channel_switch(struct iwl_mvm *mvm,
 				   struct ieee80211_vif *vif,
 				   u32 id)
@@ -4465,10 +4399,10 @@ void iwl_mvm_count_mpdu(struct iwl_mvm_sta *mvm_sta, u8 fw_sta_id, u32 count,
 		       sizeof(queue_counter->per_link));
 		queue_counter->window_start = jiffies;
 
-		IWL_DEBUG_STATS(mvm, "MPDU counters are cleared\n");
+		IWL_DEBUG_INFO(mvm, "MPDU counters are cleared\n");
 	}
 
-	for (int i = 0; i < IWL_MVM_FW_MAX_LINK_ID; i++)
+	for (int i = 0; i < IWL_FW_MAX_LINK_ID; i++)
 		total_mpdus += tx ? queue_counter->per_link[i].tx :
 				    queue_counter->per_link[i].rx;
 
diff --git a/sys/contrib/dev/iwlwifi/mvm/sta.h b/sys/contrib/dev/iwlwifi/mvm/sta.h
index c1050fed0cc6..f6906061510b 100644
--- a/sys/contrib/dev/iwlwifi/mvm/sta.h
+++ b/sys/contrib/dev/iwlwifi/mvm/sta.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2016 Intel Deutschland GmbH
  */
@@ -12,7 +12,7 @@
 #include <linux/wait.h>
 
 #include "iwl-trans.h" /* for IWL_MAX_TID_COUNT */
-#include "fw-api.h" /* IWL_MVM_STATION_COUNT_MAX */
+#include "fw-api.h" /* IWL_STATION_COUNT_MAX */
 #include "rs.h"
 
 struct iwl_mvm;
@@ -133,7 +133,7 @@ struct iwl_mvm_vif;
  * and no TID data as this is also not needed.
  * One thing to note, is that these stations have an ID in the fw, but not
  * in mac80211. In order to "reserve" them a sta_id in %fw_id_to_mac_id
- * we fill ERR_PTR(EINVAL) in this mapping and all other dereferencing of
+ * we fill ERR_PTR(-EINVAL) in this mapping and all other dereferencing of
  * pointers from this mapping need to check that the value is not error
  * or NULL.
  *
@@ -214,7 +214,7 @@ struct iwl_mvm_vif;
  */
 
 /**
- * enum iwl_mvm_agg_state
+ * enum iwl_mvm_agg_state - aggregation session state
  *
  * The state machine of the BA agreement establishment / tear down.
  * These states relate to a specific RA / TID.
@@ -225,8 +225,6 @@ struct iwl_mvm_vif;
  * @IWL_AGG_ON: aggregation session is up
  * @IWL_EMPTYING_HW_QUEUE_ADDBA: establishing a BA session - waiting for the
  *	HW queue to be empty from packets for this RA /TID.
- * @IWL_EMPTYING_HW_QUEUE_DELBA: tearing down a BA session - waiting for the
- *	HW queue to be empty from packets for this RA /TID.
  */
 enum iwl_mvm_agg_state {
 	IWL_AGG_OFF = 0,
@@ -234,7 +232,6 @@ enum iwl_mvm_agg_state {
 	IWL_AGG_STARTING,
 	IWL_AGG_ON,
 	IWL_EMPTYING_HW_QUEUE_ADDBA,
-	IWL_EMPTYING_HW_QUEUE_DELBA,
 };
 
 /**
@@ -262,7 +259,7 @@ struct iwl_mvm_tid_data {
 	u16 seq_number;
 	u16 next_reclaimed;
 	/* The rest is Tx AGG related */
-	u32 rate_n_flags;
+	__le32 rate_n_flags;
 	u8 lq_color;
 	bool amsdu_in_ampdu_allowed;
 	enum iwl_mvm_agg_state state;
@@ -361,7 +358,7 @@ struct iwl_mvm_mpdu_counter {
  */
 struct iwl_mvm_tpt_counter {
 	spinlock_t lock;
-	struct iwl_mvm_mpdu_counter per_link[IWL_MVM_FW_MAX_LINK_ID];
+	struct iwl_mvm_mpdu_counter per_link[IWL_FW_MAX_LINK_ID];
 	unsigned long window_start;
 } ____cacheline_aligned_in_smp;
 
@@ -486,6 +483,7 @@ struct iwl_mvm_int_sta {
  *	about. Otherwise (if this is a new STA), this should be false.
  * @flags: if update==true, this marks what is being changed via ORs of values
  *	from enum iwl_sta_modify_flag. Otherwise, this is ignored.
+ * Return: negative error code or 0 on success
  */
 int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			   bool update, unsigned int flags);
@@ -507,9 +505,9 @@ void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
 					  struct ieee80211_sta *sta);
 int iwl_mvm_wait_sta_queues_empty(struct iwl_mvm *mvm,
 				  struct iwl_mvm_sta *mvm_sta);
-bool iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+void iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		     struct ieee80211_sta *sta,
-		     struct ieee80211_link_sta *link_sta, int *ret);
+		     struct ieee80211_link_sta *link_sta);
 int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 		   struct ieee80211_vif *vif,
 		   struct ieee80211_sta *sta);
@@ -597,12 +595,6 @@ void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 void iwl_mvm_csa_client_absent(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_sta_ensure_queue(struct iwl_mvm *mvm, struct ieee80211_txq *txq);
 void iwl_mvm_add_new_dqa_stream_wk(struct work_struct *wk);
-#if defined(__linux__)
-int iwl_mvm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-			 struct iwl_mvm_int_sta *sta, u8 *addr, u32 cipher,
-			 u8 *key, u32 key_len,
-			 struct ieee80211_key_conf *key_conf_out);
-#endif
 void iwl_mvm_cancel_channel_switch(struct iwl_mvm *mvm,
 				   struct ieee80211_vif *vif,
 				   u32 id);
@@ -667,8 +659,7 @@ int iwl_mvm_mld_rm_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 void iwl_mvm_mld_free_sta_link(struct iwl_mvm *mvm,
 			       struct iwl_mvm_sta *mvm_sta,
 			       struct iwl_mvm_link_sta *mvm_sta_link,
-			       unsigned int link_id,
-			       bool is_in_fw);
+			       unsigned int link_id);
 int iwl_mvm_mld_rm_sta_id(struct iwl_mvm *mvm, u8 sta_id);
 int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
 				 struct ieee80211_vif *vif,
diff --git a/sys/contrib/dev/iwlwifi/mvm/tdls.c b/sys/contrib/dev/iwlwifi/mvm/tdls.c
index 5977aa8b51bb..9a250b407f3a 100644
--- a/sys/contrib/dev/iwlwifi/mvm/tdls.c
+++ b/sys/contrib/dev/iwlwifi/mvm/tdls.c
@@ -27,7 +27,7 @@ void iwl_mvm_teardown_tdls_peers(struct iwl_mvm *mvm)
 	for (i = 0; i < mvm->fw->ucode_capa.num_stations; i++) {
 		sta = rcu_dereference_protected(mvm->fw_id_to_mac_id[i],
 						lockdep_is_held(&mvm->mutex));
-		if (!sta || IS_ERR(sta) || !sta->tdls)
+		if (IS_ERR_OR_NULL(sta) || !sta->tdls)
 			continue;
 
 		mvmsta = iwl_mvm_sta_from_mac80211(sta);
@@ -50,7 +50,7 @@ int iwl_mvm_tdls_sta_count(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	for (i = 0; i < mvm->fw->ucode_capa.num_stations; i++) {
 		sta = rcu_dereference_protected(mvm->fw_id_to_mac_id[i],
 						lockdep_is_held(&mvm->mutex));
-		if (!sta || IS_ERR(sta) || !sta->tdls)
+		if (IS_ERR_OR_NULL(sta) || !sta->tdls)
 			continue;
 
 		if (vif) {
@@ -199,7 +199,7 @@ static void iwl_mvm_tdls_update_cs_state(struct iwl_mvm *mvm,
 		mvm->tdls_cs.peer.sent_timestamp = iwl_mvm_get_systime(mvm);
 
 	if (state == IWL_MVM_TDLS_SW_IDLE)
-		mvm->tdls_cs.cur_sta_id = IWL_MVM_INVALID_STA;
+		mvm->tdls_cs.cur_sta_id = IWL_INVALID_STA;
 }
 
 void iwl_mvm_rx_tdls_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
@@ -253,7 +253,7 @@ iwl_mvm_tdls_check_action(struct iwl_mvm *mvm,
 
 	/* get the existing peer if it's there */
 	if (mvm->tdls_cs.state != IWL_MVM_TDLS_SW_IDLE &&
-	    mvm->tdls_cs.cur_sta_id != IWL_MVM_INVALID_STA) {
+	    mvm->tdls_cs.cur_sta_id != IWL_INVALID_STA) {
 		struct ieee80211_sta *sta = rcu_dereference_protected(
 				mvm->fw_id_to_mac_id[mvm->tdls_cs.cur_sta_id],
 				lockdep_is_held(&mvm->mutex));
@@ -468,14 +468,14 @@ void iwl_mvm_tdls_ch_switch_work(struct work_struct *work)
 	iwl_mvm_tdls_update_cs_state(mvm, IWL_MVM_TDLS_SW_IDLE);
 
 	/* station might be gone, in that case do nothing */
-	if (mvm->tdls_cs.peer.sta_id == IWL_MVM_INVALID_STA)
+	if (mvm->tdls_cs.peer.sta_id == IWL_INVALID_STA)
 		return;
 
 	sta = rcu_dereference_protected(
 				mvm->fw_id_to_mac_id[mvm->tdls_cs.peer.sta_id],
 				lockdep_is_held(&mvm->mutex));
 	/* the station may not be here, but if it is, it must be a TDLS peer */
-	if (!sta || IS_ERR(sta) || WARN_ON(!sta->tdls))
+	if (IS_ERR_OR_NULL(sta) || WARN_ON(!sta->tdls))
 		return;
 
 	mvmsta = iwl_mvm_sta_from_mac80211(sta);
@@ -515,7 +515,7 @@ iwl_mvm_tdls_channel_switch(struct ieee80211_hw *hw,
 		       sta->addr, chandef->chan->center_freq, chandef->width);
 
 	/* we only support a single peer for channel switching */
-	if (mvm->tdls_cs.peer.sta_id != IWL_MVM_INVALID_STA) {
+	if (mvm->tdls_cs.peer.sta_id != IWL_INVALID_STA) {
 		IWL_DEBUG_TDLS(mvm,
 			       "Existing peer. Can't start switch with %pM\n",
 			       sta->addr);
@@ -569,7 +569,7 @@ void iwl_mvm_tdls_cancel_channel_switch(struct ieee80211_hw *hw,
 	IWL_DEBUG_TDLS(mvm, "TDLS cancel channel switch with %pM\n", sta->addr);
 
 	/* we only support a single peer for channel switching */
-	if (mvm->tdls_cs.peer.sta_id == IWL_MVM_INVALID_STA) {
+	if (mvm->tdls_cs.peer.sta_id == IWL_INVALID_STA) {
 		IWL_DEBUG_TDLS(mvm, "No ch switch peer - %pM\n", sta->addr);
 		goto out;
 	}
@@ -590,7 +590,7 @@ void iwl_mvm_tdls_cancel_channel_switch(struct ieee80211_hw *hw,
 	    mvm->tdls_cs.state != IWL_MVM_TDLS_SW_IDLE)
 		wait_for_phy = true;
 
-	mvm->tdls_cs.peer.sta_id = IWL_MVM_INVALID_STA;
+	mvm->tdls_cs.peer.sta_id = IWL_INVALID_STA;
 	dev_kfree_skb(mvm->tdls_cs.peer.skb);
 	mvm->tdls_cs.peer.skb = NULL;
 
@@ -637,7 +637,7 @@ iwl_mvm_tdls_recv_channel_switch(struct ieee80211_hw *hw,
 	if (params->action_code == WLAN_TDLS_CHANNEL_SWITCH_RESPONSE &&
 	    params->status != 0 &&
 	    mvm->tdls_cs.state == IWL_MVM_TDLS_SW_REQ_SENT &&
-	    mvm->tdls_cs.cur_sta_id != IWL_MVM_INVALID_STA) {
+	    mvm->tdls_cs.cur_sta_id != IWL_INVALID_STA) {
 		struct ieee80211_sta *cur_sta;
 
 		/* make sure it's the same peer */
diff --git a/sys/contrib/dev/iwlwifi/mvm/tests/hcmd.c b/sys/contrib/dev/iwlwifi/mvm/tests/hcmd.c
new file mode 100644
index 000000000000..1fee0320c756
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/mvm/tests/hcmd.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for channel helper functions
+ *
+ * Copyright (C) 2025 Intel Corporation
+ */
+#include <kunit/test.h>
+
+#include <iwl-trans.h>
+#include "../mvm.h"
+
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+static void test_hcmd_names_sorted(struct kunit *test)
+{
+	for (int i = 0; i < iwl_mvm_groups_size; i++) {
+		const struct iwl_hcmd_arr *arr = &iwl_mvm_groups[i];
+
+		if (!arr->arr)
+			continue;
+
+		for (int j = 0; j < arr->size - 1; j++)
+			KUNIT_EXPECT_LE(test, arr->arr[j].cmd_id,
+					arr->arr[j + 1].cmd_id);
+	}
+}
+
+static struct kunit_case hcmd_names_cases[] = {
+	KUNIT_CASE(test_hcmd_names_sorted),
+	{},
+};
+
+static struct kunit_suite hcmd_names = {
+	.name = "iwlmvm-hcmd-names",
+	.test_cases = hcmd_names_cases,
+};
+
+kunit_test_suite(hcmd_names);
diff --git a/sys/contrib/dev/iwlwifi/mvm/time-event.c b/sys/contrib/dev/iwlwifi/mvm/time-event.c
index a8c42ce3b630..aa653782d6d7 100644
--- a/sys/contrib/dev/iwlwifi/mvm/time-event.c
+++ b/sys/contrib/dev/iwlwifi/mvm/time-event.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2017 Intel Deutschland GmbH
  */
@@ -114,16 +114,14 @@ static void iwl_mvm_cleanup_roc(struct iwl_mvm *mvm)
 		iwl_mvm_flush_sta(mvm, mvm->aux_sta.sta_id,
 				  mvm->aux_sta.tfd_queue_msk);
 
-		if (mvm->mld_api_is_used) {
-			iwl_mvm_mld_rm_aux_sta(mvm);
-			mutex_unlock(&mvm->mutex);
-			return;
-		}
-
 		/* In newer version of this command an aux station is added only
 		 * in cases of dedicated tx queue and need to be removed in end
-		 * of use */
-		if (iwl_mvm_has_new_station_api(mvm->fw))
+		 * of use. For the even newer mld api, use the appropriate
+		 * function.
+		 */
+		if (mvm->mld_api_is_used)
+			iwl_mvm_mld_rm_aux_sta(mvm);
+		else if (iwl_mvm_has_new_station_api(mvm->fw))
 			iwl_mvm_rm_aux_sta(mvm);
 	}
 
@@ -753,7 +751,7 @@ static void iwl_mvm_cancel_session_protection(struct iwl_mvm *mvm,
 					      u32 id, s8 link_id)
 {
 	int mac_link_id = iwl_mvm_get_session_prot_id(mvm, vif, link_id);
-	struct iwl_mvm_session_prot_cmd cmd = {
+	struct iwl_session_prot_cmd cmd = {
 		.id_and_color = cpu_to_le32(mac_link_id),
 		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
 		.conf_id = cpu_to_le32(id),
@@ -777,11 +775,13 @@ static void iwl_mvm_roc_rm_cmd(struct iwl_mvm *mvm, u32 activity)
 		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
 		.activity = cpu_to_le32(activity),
 	};
+	u8 ver = iwl_fw_lookup_cmd_ver(mvm->fw, WIDE_ID(MAC_CONF_GROUP, ROC_CMD), 0);
+	u16 cmd_len = ver < 6 ? sizeof(struct iwl_roc_req_v5) : sizeof(roc_cmd);
 	int ret;
 
 	lockdep_assert_held(&mvm->mutex);
 	ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(MAC_CONF_GROUP, ROC_CMD), 0,
-				   sizeof(roc_cmd), &roc_cmd);
+				   cmd_len, &roc_cmd);
 	if (ret)
 		IWL_ERR(mvm, "Couldn't send the ROC_CMD: %d\n", ret);
 }
@@ -957,41 +957,20 @@ void iwl_mvm_rx_session_protect_notif(struct iwl_mvm *mvm,
 				      struct iwl_rx_cmd_buffer *rxb)
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_mvm_session_prot_notif *notif = (void *)pkt->data;
-	unsigned int ver =
-		iwl_fw_lookup_notif_ver(mvm->fw, MAC_CONF_GROUP,
-					SESSION_PROTECTION_NOTIF, 2);
+	struct iwl_session_prot_notif *notif = (void *)pkt->data;
 	int id = le32_to_cpu(notif->mac_link_id);
 	struct ieee80211_vif *vif;
 	struct iwl_mvm_vif *mvmvif;
-	unsigned int notif_link_id;
 
 	rcu_read_lock();
 
-	if (ver <= 2) {
-		vif = iwl_mvm_rcu_dereference_vif_id(mvm, id, true);
-	} else {
-		struct ieee80211_bss_conf *link_conf =
-			iwl_mvm_rcu_fw_link_id_to_link_conf(mvm, id, true);
-
-		if (!link_conf)
-			goto out_unlock;
-
-		notif_link_id = link_conf->link_id;
-		vif = link_conf->vif;
-	}
-
+	/* note we use link ID == MAC ID */
+	vif = iwl_mvm_rcu_dereference_vif_id(mvm, id, true);
 	if (!vif)
 		goto out_unlock;
 
 	mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	if (WARN(ver > 2 && mvmvif->time_event_data.link_id >= 0 &&
-		 mvmvif->time_event_data.link_id != notif_link_id,
-		 "SESSION_PROTECTION_NOTIF was received for link %u, while the current time event is on link %u\n",
-		 notif_link_id, mvmvif->time_event_data.link_id))
-		goto out_unlock;
-
 	/* The vif is not a P2P_DEVICE, maintain its time_event_data */
 	if (vif->type != NL80211_IFTYPE_P2P_DEVICE) {
 		struct iwl_mvm_time_event_data *te_data =
@@ -1032,6 +1011,8 @@ void iwl_mvm_rx_session_protect_notif(struct iwl_mvm *mvm,
 		/* End TE, notify mac80211 */
 		mvmvif->time_event_data.id = SESSION_PROTECT_CONF_MAX_ID;
 		mvmvif->time_event_data.link_id = -1;
+		/* set the bit so the ROC cleanup will actually clean up */
+		set_bit(IWL_MVM_STATUS_ROC_P2P_RUNNING, &mvm->status);
 		iwl_mvm_roc_finished(mvm);
 		ieee80211_remain_on_channel_expired(mvm->hw);
 	} else if (le32_to_cpu(notif->start)) {
@@ -1107,6 +1088,8 @@ int iwl_mvm_roc_add_cmd(struct iwl_mvm *mvm,
 		.activity = cpu_to_le32(activity),
 		.sta_id = cpu_to_le32(mvm->aux_sta.sta_id),
 	};
+	u8 ver = iwl_fw_lookup_cmd_ver(mvm->fw, WIDE_ID(MAC_CONF_GROUP, ROC_CMD), 0);
+	u16 cmd_len = ver < 6 ? sizeof(struct iwl_roc_req_v5) : sizeof(roc_req);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
 	lockdep_assert_held(&mvm->mutex);
@@ -1136,7 +1119,7 @@ int iwl_mvm_roc_add_cmd(struct iwl_mvm *mvm,
 	memcpy(roc_req.node_addr, vif->addr, ETH_ALEN);
 
 	res = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(MAC_CONF_GROUP, ROC_CMD),
-				   0, sizeof(roc_req), &roc_req);
+				   0, cmd_len, &roc_req);
 	if (!res)
 		mvmvif->roc_activity = activity;
 
@@ -1150,7 +1133,7 @@ iwl_mvm_start_p2p_roc_session_protection(struct iwl_mvm *mvm,
 					 enum ieee80211_roc_type type)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm_session_prot_cmd cmd = {
+	struct iwl_session_prot_cmd cmd = {
 		.id_and_color =
 			cpu_to_le32(iwl_mvm_get_session_prot_id(mvm, vif, 0)),
 		.action = cpu_to_le32(FW_CTXT_ACTION_ADD),
@@ -1419,7 +1402,7 @@ static bool iwl_mvm_session_prot_notif(struct iwl_notif_wait_data *notif_wait,
 {
 	struct iwl_mvm *mvm =
 		container_of(notif_wait, struct iwl_mvm, notif_wait);
-	struct iwl_mvm_session_prot_notif *resp;
+	struct iwl_session_prot_notif *resp;
 	int resp_len = iwl_rx_packet_payload_len(pkt);
 
 	if (WARN_ON(pkt->hdr.cmd != SESSION_PROTECTION_NOTIF ||
@@ -1451,7 +1434,7 @@ void iwl_mvm_schedule_session_protection(struct iwl_mvm *mvm,
 	const u16 notif[] = { WIDE_ID(MAC_CONF_GROUP, SESSION_PROTECTION_NOTIF) };
 	struct iwl_notification_wait wait_notif;
 	int mac_link_id = iwl_mvm_get_session_prot_id(mvm, vif, (s8)link_id);
-	struct iwl_mvm_session_prot_cmd cmd = {
+	struct iwl_session_prot_cmd cmd = {
 		.id_and_color = cpu_to_le32(mac_link_id),
 		.action = cpu_to_le32(FW_CTXT_ACTION_ADD),
 		.conf_id = cpu_to_le32(SESSION_PROTECT_CONF_ASSOC),
diff --git a/sys/contrib/dev/iwlwifi/mvm/time-event.h b/sys/contrib/dev/iwlwifi/mvm/time-event.h
index 49256ba4cf58..1ef8768756db 100644
--- a/sys/contrib/dev/iwlwifi/mvm/time-event.h
+++ b/sys/contrib/dev/iwlwifi/mvm/time-event.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2019-2020, 2023 Intel Corporation
+ * Copyright (C) 2012-2014, 2019-2020, 2023, 2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  */
 #ifndef __time_event_h__
@@ -124,6 +124,8 @@ void iwl_mvm_rx_roc_notif(struct iwl_mvm *mvm,
  * ROC request, it will issue a notification to the driver that it is on the
  * requested channel. Once the FW completes the ROC request it will issue
  * another notification to the driver.
+ *
+ * Return: negative error code or 0 on success
  */
 int iwl_mvm_start_p2p_roc(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			  int duration, enum ieee80211_roc_type type);
@@ -179,6 +181,8 @@ void iwl_mvm_remove_csa_period(struct iwl_mvm *mvm,
  *
  * This function is used to schedule NoA time event and is used to perform
  * the channel switch flow.
+ *
+ * Return: negative error code or 0 on success
  */
 int iwl_mvm_schedule_csa_period(struct iwl_mvm *mvm,
 				struct ieee80211_vif *vif,
@@ -188,7 +192,7 @@ int iwl_mvm_schedule_csa_period(struct iwl_mvm *mvm,
  * iwl_mvm_te_scheduled - check if the fw received the TE cmd
  * @te_data: the time event data that corresponds to that time event
  *
- * This function returns true iff this TE is added to the fw.
+ * Return: %true if this TE is added to the fw, %false otherwise
  */
 static inline bool
 iwl_mvm_te_scheduled(struct iwl_mvm_time_event_data *te_data)
diff --git a/sys/contrib/dev/iwlwifi/mvm/tt.c b/sys/contrib/dev/iwlwifi/mvm/tt.c
index 95a5856cd239..1958f4ca4773 100644
--- a/sys/contrib/dev/iwlwifi/mvm/tt.c
+++ b/sys/contrib/dev/iwlwifi/mvm/tt.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2019-2022, 2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2019-2022, 2024-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2016 Intel Deutschland GmbH
  */
@@ -10,6 +10,9 @@
 
 #include "mvm.h"
 
+#define IWL_MVM_NUM_CTDP_STEPS		20
+#define IWL_MVM_MIN_CTDP_BUDGET_MW	150
+
 #define IWL_MVM_TEMP_NOTIF_WAIT_TIMEOUT	HZ
 
 void iwl_mvm_enter_ctkill(struct iwl_mvm *mvm)
@@ -107,7 +110,7 @@ static bool iwl_mvm_temp_notif_wait(struct iwl_notif_wait_data *notif_wait,
 void iwl_mvm_temp_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_dts_measurement_notif_v2 *notif_v2;
+	struct iwl_dts_measurement_notif *notif_v2;
 	int len = iwl_rx_packet_payload_len(pkt);
 	int temp;
 	u32 ths_crossed;
@@ -481,43 +484,28 @@ static const struct iwl_tt_params iwl_mvm_default_tt_params = {
 	.support_tx_backoff = true,
 };
 
-/* budget in mWatt */
-static const u32 iwl_mvm_cdev_budgets[] = {
-	2400,	/* cooling state 0 */
-	2000,	/* cooling state 1 */
-	1800,	/* cooling state 2 */
-	1600,	/* cooling state 3 */
-	1400,	/* cooling state 4 */
-	1200,	/* cooling state 5 */
-	1000,	/* cooling state 6 */
-	900,	/* cooling state 7 */
-	800,	/* cooling state 8 */
-	700,	/* cooling state 9 */
-	650,	/* cooling state 10 */
-	600,	/* cooling state 11 */
-	550,	/* cooling state 12 */
-	500,	/* cooling state 13 */
-	450,	/* cooling state 14 */
-	400,	/* cooling state 15 */
-	350,	/* cooling state 16 */
-	300,	/* cooling state 17 */
-	250,	/* cooling state 18 */
-	200,	/* cooling state 19 */
-	150,	/* cooling state 20 */
-};
-
 int iwl_mvm_ctdp_command(struct iwl_mvm *mvm, u32 op, u32 state)
 {
-	struct iwl_mvm_ctdp_cmd cmd = {
+	struct iwl_ctdp_cmd cmd = {
 		.operation = cpu_to_le32(op),
-		.budget = cpu_to_le32(iwl_mvm_cdev_budgets[state]),
 		.window_size = 0,
 	};
+	u32 budget;
 	int ret;
 	u32 status;
 
 	lockdep_assert_held(&mvm->mutex);
 
+	/* Do a linear scale from IWL_MVM_MIN_CTDP_BUDGET_MW to the configured
+	 * maximum in the predefined number of steps.
+	 */
+	budget = ((mvm->thermal_throttle.power_budget_mw -
+		   IWL_MVM_MIN_CTDP_BUDGET_MW) *
+		  (IWL_MVM_NUM_CTDP_STEPS - 1 - state)) /
+		 (IWL_MVM_NUM_CTDP_STEPS - 1) +
+		 IWL_MVM_MIN_CTDP_BUDGET_MW;
+	cmd.budget = cpu_to_le32(budget);
+
 	status = 0;
 	ret = iwl_mvm_send_cmd_pdu_status(mvm, WIDE_ID(PHY_OPS_GROUP,
 						       CTDP_CONFIG_CMD),
@@ -554,8 +542,8 @@ int iwl_mvm_ctdp_command(struct iwl_mvm *mvm, u32 op, u32 state)
 #ifdef CONFIG_THERMAL
 static int compare_temps(const void *a, const void *b)
 {
-	return ((s16)le16_to_cpu(*(__le16 *)a) -
-		(s16)le16_to_cpu(*(__le16 *)b));
+	return ((s16)le16_to_cpu(*(const __le16 *)a) -
+		(s16)le16_to_cpu(*(const __le16 *)b));
 }
 
 struct iwl_trip_walk_data {
@@ -709,7 +697,7 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm)
 static int iwl_mvm_tcool_get_max_state(struct thermal_cooling_device *cdev,
 				       unsigned long *state)
 {
-	*state = ARRAY_SIZE(iwl_mvm_cdev_budgets) - 1;
+	*state = IWL_MVM_NUM_CTDP_STEPS - 1;
 
 	return 0;
 }
@@ -735,7 +723,7 @@ static int iwl_mvm_tcool_set_cur_state(struct thermal_cooling_device *cdev,
 	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR)
 		return -EIO;
 
-	if (new_state >= ARRAY_SIZE(iwl_mvm_cdev_budgets))
+	if (new_state >= IWL_MVM_NUM_CTDP_STEPS)
 		return -EINVAL;
 
 	return iwl_mvm_ctdp_command(mvm, CTDP_CMD_OPERATION_START,
@@ -796,6 +784,47 @@ static void iwl_mvm_cooling_device_unregister(struct iwl_mvm *mvm)
 }
 #endif /* CONFIG_THERMAL */
 
+static u32 iwl_mvm_ctdp_get_max_budget(struct iwl_mvm *mvm)
+{
+	u64 bios_power_budget = 0;
+	u32 default_power_budget;
+
+	switch (CSR_HW_RFID_TYPE(mvm->trans->info.hw_rf_id)) {
+	case IWL_CFG_RF_TYPE_JF2:
+	case IWL_CFG_RF_TYPE_JF1:
+		default_power_budget = 2000;
+		break;
+	case IWL_CFG_RF_TYPE_HR2:
+	case IWL_CFG_RF_TYPE_HR1:
+		default_power_budget = 2400;
+		break;
+	case IWL_CFG_RF_TYPE_GF:
+		/* dual-radio devices have a higher budget */
+		if (CSR_HW_RFID_IS_CDB(mvm->trans->info.hw_rf_id))
+			default_power_budget = 5200;
+		else
+			default_power_budget = 2880;
+		break;
+	case IWL_CFG_RF_TYPE_FM:
+		default_power_budget = 3450;
+		break;
+	default:
+		default_power_budget = 5550;
+		break;
+	}
+
+	iwl_bios_get_pwr_limit(&mvm->fwrt, &bios_power_budget);
+
+	/* 32bit in UEFI, 16bit in ACPI; use BIOS value if it is in range */
+	if (bios_power_budget &&
+	    bios_power_budget != 0xffff && bios_power_budget != 0xffffffff &&
+	    bios_power_budget >= IWL_MVM_MIN_CTDP_BUDGET_MW &&
+	    bios_power_budget <= default_power_budget)
+		return (u32)bios_power_budget;
+
+	return default_power_budget;
+}
+
 void iwl_mvm_thermal_initialize(struct iwl_mvm *mvm, u32 min_backoff)
 {
 	struct iwl_mvm_tt_mgmt *tt = &mvm->thermal_throttle;
@@ -807,6 +836,8 @@ void iwl_mvm_thermal_initialize(struct iwl_mvm *mvm, u32 min_backoff)
 	else
 		tt->params = iwl_mvm_default_tt_params;
 
+	tt->power_budget_mw = iwl_mvm_ctdp_get_max_budget(mvm);
+	IWL_DEBUG_TEMP(mvm, "cTDP power budget: %d mW\n", tt->power_budget_mw);
 	tt->throttle = false;
 	tt->dynamic_smps = false;
 	tt->min_backoff = min_backoff;
diff --git a/sys/contrib/dev/iwlwifi/mvm/tx.c b/sys/contrib/dev/iwlwifi/mvm/tx.c
index fbbed0bd7678..2b6052a6f90a 100644
--- a/sys/contrib/dev/iwlwifi/mvm/tx.c
+++ b/sys/contrib/dev/iwlwifi/mvm/tx.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -13,6 +13,7 @@
 
 #include "iwl-trans.h"
 #include "iwl-nvm-utils.h"
+#include "iwl-utils.h"
 #include "mvm.h"
 #include "sta.h"
 #include "time-sync.h"
@@ -147,12 +148,12 @@ out:
  * Sets most of the Tx cmd's fields
  */
 void iwl_mvm_set_tx_cmd(struct iwl_mvm *mvm, struct sk_buff *skb,
-			struct iwl_tx_cmd *tx_cmd,
+			struct iwl_tx_cmd_v6_params *tx_cmd_params,
 			struct ieee80211_tx_info *info, u8 sta_id)
 {
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	__le16 fc = hdr->frame_control;
-	u32 tx_flags = le32_to_cpu(tx_cmd->tx_flags);
+	u32 tx_flags = le32_to_cpu(tx_cmd_params->tx_flags);
 	u32 len = skb->len + FCS_LEN;
 	bool amsdu = false;
 	u8 ac;
@@ -172,7 +173,7 @@ void iwl_mvm_set_tx_cmd(struct iwl_mvm *mvm, struct sk_buff *skb,
 
 	if (ieee80211_is_data_qos(fc)) {
 		u8 *qc = ieee80211_get_qos_ctl(hdr);
-		tx_cmd->tid_tspec = qc[0] & 0xf;
+		tx_cmd_params->tid_tspec = qc[0] & 0xf;
 		tx_flags &= ~TX_CMD_FLG_SEQ_CTL;
 		amsdu = *qc & IEEE80211_QOS_CTL_A_MSDU_PRESENT;
 	} else if (ieee80211_is_back_req(fc)) {
@@ -181,17 +182,17 @@ void iwl_mvm_set_tx_cmd(struct iwl_mvm *mvm, struct sk_buff *skb,
 		u16 ssn = le16_to_cpu(bar->start_seq_num);
 
 		tx_flags |= TX_CMD_FLG_ACK | TX_CMD_FLG_BAR;
-		tx_cmd->tid_tspec = (control &
+		tx_cmd_params->tid_tspec = (control &
 				     IEEE80211_BAR_CTRL_TID_INFO_MASK) >>
 			IEEE80211_BAR_CTRL_TID_INFO_SHIFT;
-		WARN_ON_ONCE(tx_cmd->tid_tspec >= IWL_MAX_TID_COUNT);
-		iwl_mvm_bar_check_trigger(mvm, bar->ra, tx_cmd->tid_tspec,
+		WARN_ON_ONCE(tx_cmd_params->tid_tspec >= IWL_MAX_TID_COUNT);
+		iwl_mvm_bar_check_trigger(mvm, bar->ra, tx_cmd_params->tid_tspec,
 					  ssn);
 	} else {
 		if (ieee80211_is_data(fc))
-			tx_cmd->tid_tspec = IWL_TID_NON_QOS;
+			tx_cmd_params->tid_tspec = IWL_TID_NON_QOS;
 		else
-			tx_cmd->tid_tspec = IWL_MAX_TID_COUNT;
+			tx_cmd_params->tid_tspec = IWL_MAX_TID_COUNT;
 
 		if (info->flags & IEEE80211_TX_CTL_ASSIGN_SEQ)
 			tx_flags |= TX_CMD_FLG_SEQ_CTL;
@@ -200,8 +201,8 @@ void iwl_mvm_set_tx_cmd(struct iwl_mvm *mvm, struct sk_buff *skb,
 	}
 
 	/* Default to 0 (BE) when tid_spec is set to IWL_MAX_TID_COUNT */
-	if (tx_cmd->tid_tspec < IWL_MAX_TID_COUNT)
-		ac = tid_to_mac80211_ac[tx_cmd->tid_tspec];
+	if (tx_cmd_params->tid_tspec < IWL_MAX_TID_COUNT)
+		ac = tid_to_mac80211_ac[tx_cmd_params->tid_tspec];
 	else
 		ac = tid_to_mac80211_ac[0];
 
@@ -210,20 +211,20 @@ void iwl_mvm_set_tx_cmd(struct iwl_mvm *mvm, struct sk_buff *skb,
 
 	if (ieee80211_is_mgmt(fc)) {
 		if (ieee80211_is_assoc_req(fc) || ieee80211_is_reassoc_req(fc))
-			tx_cmd->pm_frame_timeout = cpu_to_le16(PM_FRAME_ASSOC);
+			tx_cmd_params->pm_frame_timeout = cpu_to_le16(PM_FRAME_ASSOC);
 		else if (ieee80211_is_action(fc))
-			tx_cmd->pm_frame_timeout = cpu_to_le16(PM_FRAME_NONE);
+			tx_cmd_params->pm_frame_timeout = cpu_to_le16(PM_FRAME_NONE);
 		else
-			tx_cmd->pm_frame_timeout = cpu_to_le16(PM_FRAME_MGMT);
+			tx_cmd_params->pm_frame_timeout = cpu_to_le16(PM_FRAME_MGMT);
 
 		/* The spec allows Action frames in A-MPDU, we don't support
 		 * it
 		 */
 		WARN_ON_ONCE(info->flags & IEEE80211_TX_CTL_AMPDU);
 	} else if (info->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO) {
-		tx_cmd->pm_frame_timeout = cpu_to_le16(PM_FRAME_MGMT);
+		tx_cmd_params->pm_frame_timeout = cpu_to_le16(PM_FRAME_MGMT);
 	} else {
-		tx_cmd->pm_frame_timeout = cpu_to_le16(PM_FRAME_NONE);
+		tx_cmd_params->pm_frame_timeout = cpu_to_le16(PM_FRAME_NONE);
 	}
 
 	if (ieee80211_is_data(fc) && len > mvm->rts_threshold &&
@@ -235,13 +236,13 @@ void iwl_mvm_set_tx_cmd(struct iwl_mvm *mvm, struct sk_buff *skb,
 	    ieee80211_action_contains_tpc(skb))
 		tx_flags |= TX_CMD_FLG_WRITE_TX_POWER;
 
-	tx_cmd->tx_flags = cpu_to_le32(tx_flags);
+	tx_cmd_params->tx_flags = cpu_to_le32(tx_flags);
 	/* Total # bytes to be transmitted - PCIe code will adjust for A-MSDU */
-	tx_cmd->len = cpu_to_le16((u16)skb->len);
-	tx_cmd->life_time = cpu_to_le32(TX_CMD_LIFE_TIME_INFINITE);
-	tx_cmd->sta_id = sta_id;
+	tx_cmd_params->len = cpu_to_le16((u16)skb->len);
+	tx_cmd_params->life_time = cpu_to_le32(TX_CMD_LIFE_TIME_INFINITE);
+	tx_cmd_params->sta_id = sta_id;
 
-	tx_cmd->offload_assist =
+	tx_cmd_params->offload_assist =
 		cpu_to_le16(iwl_mvm_tx_csum(mvm, skb, info, amsdu));
 }
 
@@ -282,14 +283,10 @@ static u32 iwl_mvm_convert_rate_idx(struct iwl_mvm *mvm,
 		 (rate_idx <= IWL_LAST_CCK_RATE);
 
 	/* Set CCK or OFDM flag */
-	if (iwl_fw_lookup_cmd_ver(mvm->fw, TX_CMD, 0) > 8) {
-		if (!is_cck)
-			rate_flags |= RATE_MCS_LEGACY_OFDM_MSK;
-		else
-			rate_flags |= RATE_MCS_CCK_MSK;
-	} else if (is_cck) {
-		rate_flags |= RATE_MCS_CCK_MSK_V1;
-	}
+	if (!is_cck)
+		rate_flags |= RATE_MCS_MOD_TYPE_LEGACY_OFDM;
+	else
+		rate_flags |= RATE_MCS_MOD_TYPE_CCK;
 
 	return (u32)rate_plcp | rate_flags;
 }
@@ -302,45 +299,35 @@ static u32 iwl_mvm_get_inject_tx_rate(struct iwl_mvm *mvm,
 	struct ieee80211_tx_rate *rate = &info->control.rates[0];
 	u32 result;
 
-	/*
-	 * we only care about legacy/HT/VHT so far, so we can
-	 * build in v1 and use iwl_new_rate_from_v1()
-	 */
-
 	if (rate->flags & IEEE80211_TX_RC_VHT_MCS) {
 		u8 mcs = ieee80211_rate_get_vht_mcs(rate);
 		u8 nss = ieee80211_rate_get_vht_nss(rate);
 
-		result = RATE_MCS_VHT_MSK_V1;
+		result = RATE_MCS_MOD_TYPE_VHT;
 		result |= u32_encode_bits(mcs, RATE_VHT_MCS_RATE_CODE_MSK);
 		result |= u32_encode_bits(nss, RATE_MCS_NSS_MSK);
 		if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
-			result |= RATE_MCS_SGI_MSK_V1;
+			result |= RATE_MCS_SGI_MSK;
 		if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
-			result |= u32_encode_bits(1, RATE_MCS_CHAN_WIDTH_MSK_V1);
+			result |= RATE_MCS_CHAN_WIDTH_40;
 		else if (rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
-			result |= u32_encode_bits(2, RATE_MCS_CHAN_WIDTH_MSK_V1);
+			result |= RATE_MCS_CHAN_WIDTH_80;
 		else if (rate->flags & IEEE80211_TX_RC_160_MHZ_WIDTH)
-			result |= u32_encode_bits(3, RATE_MCS_CHAN_WIDTH_MSK_V1);
-
-		if (iwl_fw_lookup_notif_ver(mvm->fw, LONG_GROUP, TX_CMD, 0) > 6)
-			result = iwl_new_rate_from_v1(result);
+			result |= RATE_MCS_CHAN_WIDTH_160;
 	} else if (rate->flags & IEEE80211_TX_RC_MCS) {
-		result = RATE_MCS_HT_MSK_V1;
-		result |= u32_encode_bits(rate->idx,
-					  RATE_HT_MCS_RATE_CODE_MSK_V1 |
-					  RATE_HT_MCS_NSS_MSK_V1);
+		result = RATE_MCS_MOD_TYPE_HT;
+		result |= u32_encode_bits(rate->idx & 0x7,
+					  RATE_HT_MCS_CODE_MSK);
+		result |= u32_encode_bits(rate->idx >> 3,
+					  RATE_MCS_NSS_MSK);
 		if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
-			result |= RATE_MCS_SGI_MSK_V1;
+			result |= RATE_MCS_SGI_MSK;
 		if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
-			result |= u32_encode_bits(1, RATE_MCS_CHAN_WIDTH_MSK_V1);
+			result |= RATE_MCS_CHAN_WIDTH_40;
 		if (info->flags & IEEE80211_TX_CTL_LDPC)
-			result |= RATE_MCS_LDPC_MSK_V1;
+			result |= RATE_MCS_LDPC_MSK;
 		if (u32_get_bits(info->flags, IEEE80211_TX_CTL_STBC))
 			result |= RATE_MCS_STBC_MSK;
-
-		if (iwl_fw_lookup_notif_ver(mvm->fw, LONG_GROUP, TX_CMD, 0) > 6)
-			result = iwl_new_rate_from_v1(result);
 	} else {
 		int rate_idx = info->control.rates[0].idx;
 
@@ -390,36 +377,41 @@ static u32 iwl_mvm_get_tx_rate(struct iwl_mvm *mvm,
 	return iwl_mvm_convert_rate_idx(mvm, info, rate_idx);
 }
 
-static u32 iwl_mvm_get_tx_rate_n_flags(struct iwl_mvm *mvm,
-				       struct ieee80211_tx_info *info,
-				       struct ieee80211_sta *sta, __le16 fc)
+static __le32 iwl_mvm_get_tx_rate_n_flags(struct iwl_mvm *mvm,
+					  struct ieee80211_tx_info *info,
+					  struct ieee80211_sta *sta, __le16 fc)
 {
+	u32 rate;
+
 	if (unlikely(info->control.flags & IEEE80211_TX_CTRL_RATE_INJECT))
-		return iwl_mvm_get_inject_tx_rate(mvm, info, sta, fc);
+		rate = iwl_mvm_get_inject_tx_rate(mvm, info, sta, fc);
+	else
+		rate = iwl_mvm_get_tx_rate(mvm, info, sta, fc) |
+		       iwl_mvm_get_tx_ant(mvm, info, sta, fc);
 
-	return iwl_mvm_get_tx_rate(mvm, info, sta, fc) |
-		iwl_mvm_get_tx_ant(mvm, info, sta, fc);
+	return iwl_mvm_v3_rate_to_fw(rate, mvm->fw_rates_ver);
 }
 
 /*
  * Sets the fields in the Tx cmd that are rate related
  */
-void iwl_mvm_set_tx_cmd_rate(struct iwl_mvm *mvm, struct iwl_tx_cmd *tx_cmd,
-			    struct ieee80211_tx_info *info,
-			    struct ieee80211_sta *sta, __le16 fc)
+void iwl_mvm_set_tx_cmd_rate(struct iwl_mvm *mvm,
+			     struct iwl_tx_cmd_v6_params *tx_cmd_params,
+			     struct ieee80211_tx_info *info,
+			     struct ieee80211_sta *sta, __le16 fc)
 {
 	/* Set retry limit on RTS packets */
-	tx_cmd->rts_retry_limit = IWL_RTS_DFAULT_RETRY_LIMIT;
+	tx_cmd_params->rts_retry_limit = IWL_RTS_DFAULT_RETRY_LIMIT;
 
 	/* Set retry limit on DATA packets and Probe Responses*/
 	if (ieee80211_is_probe_resp(fc)) {
-		tx_cmd->data_retry_limit = IWL_MGMT_DFAULT_RETRY_LIMIT;
-		tx_cmd->rts_retry_limit =
-			min(tx_cmd->data_retry_limit, tx_cmd->rts_retry_limit);
+		tx_cmd_params->data_retry_limit = IWL_MGMT_DFAULT_RETRY_LIMIT;
+		tx_cmd_params->rts_retry_limit =
+			min(tx_cmd_params->data_retry_limit, tx_cmd_params->rts_retry_limit);
 	} else if (ieee80211_is_back_req(fc)) {
-		tx_cmd->data_retry_limit = IWL_BAR_DFAULT_RETRY_LIMIT;
+		tx_cmd_params->data_retry_limit = IWL_BAR_DFAULT_RETRY_LIMIT;
 	} else {
-		tx_cmd->data_retry_limit = IWL_DEFAULT_TX_RETRY;
+		tx_cmd_params->data_retry_limit = IWL_DEFAULT_TX_RETRY;
 	}
 
 	/*
@@ -432,18 +424,17 @@ void iwl_mvm_set_tx_cmd_rate(struct iwl_mvm *mvm, struct iwl_tx_cmd *tx_cmd,
 		struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
 		if (mvmsta->sta_state >= IEEE80211_STA_AUTHORIZED) {
-			tx_cmd->initial_rate_index = 0;
-			tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_STA_RATE);
+			tx_cmd_params->initial_rate_index = 0;
+			tx_cmd_params->tx_flags |= cpu_to_le32(TX_CMD_FLG_STA_RATE);
 			return;
 		}
 	} else if (ieee80211_is_back_req(fc)) {
-		tx_cmd->tx_flags |=
+		tx_cmd_params->tx_flags |=
 			cpu_to_le32(TX_CMD_FLG_ACK | TX_CMD_FLG_BAR);
 	}
 
 	/* Set the rate in the TX cmd */
-	tx_cmd->rate_n_flags =
-		cpu_to_le32(iwl_mvm_get_tx_rate_n_flags(mvm, info, sta, fc));
+	tx_cmd_params->rate_n_flags = iwl_mvm_get_tx_rate_n_flags(mvm, info, sta, fc);
 }
 
 static inline void iwl_mvm_set_tx_cmd_pn(struct ieee80211_tx_info *info,
@@ -468,7 +459,7 @@ static inline void iwl_mvm_set_tx_cmd_pn(struct ieee80211_tx_info *info,
  */
 static void iwl_mvm_set_tx_cmd_crypto(struct iwl_mvm *mvm,
 				      struct ieee80211_tx_info *info,
-				      struct iwl_tx_cmd *tx_cmd,
+				      struct iwl_tx_cmd_v6_params *tx_cmd_params,
 				      struct sk_buff *skb_frag,
 				      int hdrlen)
 {
@@ -479,26 +470,26 @@ static void iwl_mvm_set_tx_cmd_crypto(struct iwl_mvm *mvm,
 
 	switch (keyconf->cipher) {
 	case WLAN_CIPHER_SUITE_CCMP:
-		iwl_mvm_set_tx_cmd_ccmp(info, tx_cmd);
+		iwl_mvm_set_tx_cmd_ccmp(info, tx_cmd_params);
 		iwl_mvm_set_tx_cmd_pn(info, crypto_hdr);
 		break;
 
 	case WLAN_CIPHER_SUITE_TKIP:
-		tx_cmd->sec_ctl = TX_CMD_SEC_TKIP;
+		tx_cmd_params->sec_ctl = TX_CMD_SEC_TKIP;
 		pn = atomic64_inc_return(&keyconf->tx_pn);
 		ieee80211_tkip_add_iv(crypto_hdr, keyconf, pn);
-		ieee80211_get_tkip_p2k(keyconf, skb_frag, tx_cmd->key);
+		ieee80211_get_tkip_p2k(keyconf, skb_frag, tx_cmd_params->key);
 		break;
 
 	case WLAN_CIPHER_SUITE_WEP104:
-		tx_cmd->sec_ctl |= TX_CMD_SEC_KEY128;
+		tx_cmd_params->sec_ctl |= TX_CMD_SEC_KEY128;
 		fallthrough;
 	case WLAN_CIPHER_SUITE_WEP40:
-		tx_cmd->sec_ctl |= TX_CMD_SEC_WEP |
+		tx_cmd_params->sec_ctl |= TX_CMD_SEC_WEP |
 			((keyconf->keyidx << TX_CMD_SEC_WEP_KEY_IDX_POS) &
 			  TX_CMD_SEC_WEP_KEY_IDX_MSK);
 
-		memcpy(&tx_cmd->key[3], keyconf->key, keyconf->keylen);
+		memcpy(&tx_cmd_params->key[3], keyconf->key, keyconf->keylen);
 		break;
 	case WLAN_CIPHER_SUITE_GCMP:
 	case WLAN_CIPHER_SUITE_GCMP_256:
@@ -511,12 +502,12 @@ static void iwl_mvm_set_tx_cmd_crypto(struct iwl_mvm *mvm,
 		 * one.
 		 * Need to handle this.
 		 */
-		tx_cmd->sec_ctl |= type | TX_CMD_SEC_KEY_FROM_TABLE;
-		tx_cmd->key[0] = keyconf->hw_key_idx;
+		tx_cmd_params->sec_ctl |= type | TX_CMD_SEC_KEY_FROM_TABLE;
+		tx_cmd_params->key[0] = keyconf->hw_key_idx;
 		iwl_mvm_set_tx_cmd_pn(info, crypto_hdr);
 		break;
 	default:
-		tx_cmd->sec_ctl |= TX_CMD_SEC_EXT;
+		tx_cmd_params->sec_ctl |= TX_CMD_SEC_EXT;
 	}
 }
 
@@ -542,7 +533,7 @@ static bool iwl_mvm_use_host_rate(struct iwl_mvm *mvm,
 	 * (since we don't necesarily know the link), but FW rate
 	 * selection was fixed.
 	 */
-	return mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ;
+	return mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ;
 }
 
 static void iwl_mvm_copy_hdr(void *cmd, const void *hdr, int hdrlen,
@@ -566,7 +557,7 @@ iwl_mvm_set_tx_params(struct iwl_mvm *mvm, struct sk_buff *skb,
 {
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct iwl_device_tx_cmd *dev_cmd;
-	struct iwl_tx_cmd *tx_cmd;
+	struct iwl_tx_cmd_v6 *tx_cmd;
 
 	dev_cmd = iwl_trans_alloc_tx_cmd(mvm->trans);
 
@@ -576,7 +567,7 @@ iwl_mvm_set_tx_params(struct iwl_mvm *mvm, struct sk_buff *skb,
 	dev_cmd->hdr.cmd = TX_CMD;
 
 	if (iwl_mvm_has_new_tx_api(mvm)) {
-		u32 rate_n_flags = 0;
+		__le32 rate_n_flags = 0;
 		u16 flags = 0;
 		struct iwl_mvm_sta *mvmsta = sta ?
 			iwl_mvm_sta_from_mac80211(sta) : NULL;
@@ -608,9 +599,9 @@ iwl_mvm_set_tx_params(struct iwl_mvm *mvm, struct sk_buff *skb,
 			flags |= IWL_TX_FLAGS_HIGH_PRI;
 		}
 
-		if (mvm->trans->trans_cfg->device_family >=
+		if (mvm->trans->mac_cfg->device_family >=
 		    IWL_DEVICE_FAMILY_AX210) {
-			struct iwl_tx_cmd_gen3 *cmd = (void *)dev_cmd->payload;
+			struct iwl_tx_cmd *cmd = (void *)dev_cmd->payload;
 			u32 offload_assist = iwl_mvm_tx_csum(mvm, skb,
 							     info, amsdu);
 
@@ -623,9 +614,9 @@ iwl_mvm_set_tx_params(struct iwl_mvm *mvm, struct sk_buff *skb,
 			iwl_mvm_copy_hdr(cmd->hdr, hdr, hdrlen, addr3_override);
 
 			cmd->flags = cpu_to_le16(flags);
-			cmd->rate_n_flags = cpu_to_le32(rate_n_flags);
+			cmd->rate_n_flags = rate_n_flags;
 		} else {
-			struct iwl_tx_cmd_gen2 *cmd = (void *)dev_cmd->payload;
+			struct iwl_tx_cmd_v9 *cmd = (void *)dev_cmd->payload;
 			u16 offload_assist = iwl_mvm_tx_csum(mvm, skb,
 							     info, amsdu);
 
@@ -638,19 +629,19 @@ iwl_mvm_set_tx_params(struct iwl_mvm *mvm, struct sk_buff *skb,
 			iwl_mvm_copy_hdr(cmd->hdr, hdr, hdrlen, addr3_override);
 
 			cmd->flags = cpu_to_le32(flags);
-			cmd->rate_n_flags = cpu_to_le32(rate_n_flags);
+			cmd->rate_n_flags = rate_n_flags;
 		}
 		goto out;
 	}
 
-	tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload;
+	tx_cmd = (struct iwl_tx_cmd_v6 *)dev_cmd->payload;
 
 	if (info->control.hw_key)
-		iwl_mvm_set_tx_cmd_crypto(mvm, info, tx_cmd, skb, hdrlen);
+		iwl_mvm_set_tx_cmd_crypto(mvm, info, &tx_cmd->params, skb, hdrlen);
 
-	iwl_mvm_set_tx_cmd(mvm, skb, tx_cmd, info, sta_id);
+	iwl_mvm_set_tx_cmd(mvm, skb, &tx_cmd->params, info, sta_id);
 
-	iwl_mvm_set_tx_cmd_rate(mvm, tx_cmd, info, sta, hdr->frame_control);
+	iwl_mvm_set_tx_cmd_rate(mvm, &tx_cmd->params, info, sta, hdr->frame_control);
 
 	/* Copy MAC header from skb into command buffer */
 	iwl_mvm_copy_hdr(tx_cmd->hdr, hdr, hdrlen, addr3_override);
@@ -938,78 +929,6 @@ unsigned int iwl_mvm_max_amsdu_size(struct iwl_mvm *mvm,
 
 #ifdef CONFIG_INET
 
-static int
-iwl_mvm_tx_tso_segment(struct sk_buff *skb, unsigned int num_subframes,
-		       netdev_features_t netdev_flags,
-		       struct sk_buff_head *mpdus_skb)
-{
-	struct sk_buff *tmp, *next;
-	struct ieee80211_hdr *hdr = (void *)skb->data;
-	char cb[sizeof(skb->cb)];
-	u16 i = 0;
-	unsigned int tcp_payload_len;
-	unsigned int mss = skb_shinfo(skb)->gso_size;
-	bool ipv4 = (skb->protocol == htons(ETH_P_IP));
-	bool qos = ieee80211_is_data_qos(hdr->frame_control);
-	u16 ip_base_id = ipv4 ? ntohs(ip_hdr(skb)->id) : 0;
-
-	skb_shinfo(skb)->gso_size = num_subframes * mss;
-	memcpy(cb, skb->cb, sizeof(cb));
-
-	next = skb_gso_segment(skb, netdev_flags);
-	skb_shinfo(skb)->gso_size = mss;
-	skb_shinfo(skb)->gso_type = ipv4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
-
-	if (IS_ERR(next) && PTR_ERR(next) == -ENOMEM)
-		return -ENOMEM;
-
-	if (WARN_ONCE(IS_ERR(next),
-		      "skb_gso_segment error: %d\n", (int)PTR_ERR(next)))
-		return PTR_ERR(next);
-
-	if (next)
-		consume_skb(skb);
-
-	skb_list_walk_safe(next, tmp, next) {
-		memcpy(tmp->cb, cb, sizeof(tmp->cb));
-		/*
-		 * Compute the length of all the data added for the A-MSDU.
-		 * This will be used to compute the length to write in the TX
-		 * command. We have: SNAP + IP + TCP for n -1 subframes and
-		 * ETH header for n subframes.
-		 */
-		tcp_payload_len = skb_tail_pointer(tmp) -
-			skb_transport_header(tmp) -
-			tcp_hdrlen(tmp) + tmp->data_len;
-
-		if (ipv4)
-			ip_hdr(tmp)->id = htons(ip_base_id + i * num_subframes);
-
-		if (tcp_payload_len > mss) {
-			skb_shinfo(tmp)->gso_size = mss;
-			skb_shinfo(tmp)->gso_type = ipv4 ? SKB_GSO_TCPV4 :
-							   SKB_GSO_TCPV6;
-		} else {
-			if (qos) {
-				u8 *qc;
-
-				if (ipv4)
-					ip_send_check(ip_hdr(tmp));
-
-				qc = ieee80211_get_qos_ctl((void *)tmp->data);
-				*qc &= ~IEEE80211_QOS_CTL_A_MSDU_PRESENT;
-			}
-			skb_shinfo(tmp)->gso_size = 0;
-		}
-
-		skb_mark_not_on_list(tmp);
-		__skb_queue_tail(mpdus_skb, tmp);
-		i++;
-	}
-
-	return 0;
-}
-
 static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 			  struct ieee80211_tx_info *info,
 			  struct ieee80211_sta *sta,
@@ -1028,7 +947,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	if (!mvmsta->max_amsdu_len ||
 	    !ieee80211_is_data_qos(hdr->frame_control) ||
 	    !mvmsta->amsdu_enabled)
-		return iwl_mvm_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
+		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
 
 	/*
 	 * Do not build AMSDU for IPv6 with extension headers.
@@ -1038,7 +957,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	    ((struct ipv6hdr *)skb_network_header(skb))->nexthdr !=
 	    IPPROTO_TCP) {
 		netdev_flags &= ~NETIF_F_CSUM_MASK;
-		return iwl_mvm_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
+		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
 	}
 
 	tid = ieee80211_get_tid(hdr);
@@ -1052,7 +971,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	if ((info->flags & IEEE80211_TX_CTL_AMPDU &&
 	     !mvmsta->tid_data[tid].amsdu_in_ampdu_allowed) ||
 	    !(mvmsta->amsdu_enabled & BIT(tid)))
-		return iwl_mvm_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
+		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
 
 	/*
 	 * Take the min of ieee80211 station and mvm station
@@ -1094,7 +1013,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 *	1 more for the potential data in the header
 	 */
 	if ((num_subframes * 2 + skb_shinfo(skb)->nr_frags + 1) >
-	    mvm->trans->max_skb_frags)
+	    mvm->trans->info.max_skb_frags)
 		num_subframes = 1;
 
 	if (num_subframes > 1)
@@ -1110,8 +1029,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 * Trick the segmentation function to make it
 	 * create SKBs that can fit into one A-MSDU.
 	 */
-	return iwl_mvm_tx_tso_segment(skb, num_subframes, netdev_flags,
-				      mpdus_skb);
+	return iwl_tx_tso_segment(skb, num_subframes, netdev_flags, mpdus_skb);
 }
 #else /* CONFIG_INET */
 static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
@@ -1203,6 +1121,9 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 	bool is_ampdu = false;
 	int hdrlen;
 
+	if (WARN_ON_ONCE(!sta))
+		return -1;
+
 	mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	fc = hdr->frame_control;
 	hdrlen = ieee80211_hdrlen(fc);
@@ -1210,10 +1131,7 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 	if (IWL_MVM_NON_TRANSMITTING_AP && ieee80211_is_probe_resp(fc))
 		return -1;
 
-	if (WARN_ON_ONCE(!mvmsta))
-		return -1;
-
-	if (WARN_ON_ONCE(mvmsta->deflink.sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(mvmsta->deflink.sta_id == IWL_INVALID_STA))
 		return -1;
 
 	if (unlikely(ieee80211_is_any_nullfunc(fc)) && sta->deflink.he_cap.has_he)
@@ -1257,7 +1175,7 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 		seq_number &= IEEE80211_SCTL_SEQ;
 
 		if (!iwl_mvm_has_new_tx_api(mvm)) {
-			struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload;
+			struct iwl_tx_cmd_v6 *tx_cmd = (void *)dev_cmd->payload;
 
 			hdr->seq_ctrl &= cpu_to_le16(IEEE80211_SCTL_FRAG);
 			hdr->seq_ctrl |= cpu_to_le16(seq_number);
@@ -1349,7 +1267,7 @@ drop:
 int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb,
 		       struct ieee80211_sta *sta)
 {
-	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_mvm_sta *mvmsta;
 	struct ieee80211_tx_info info;
 	struct sk_buff_head mpdus_skbs;
 	struct ieee80211_vif *vif;
@@ -1358,10 +1276,12 @@ int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb,
 	struct sk_buff *orig_skb = skb;
 	const u8 *addr3;
 
-	if (WARN_ON_ONCE(!mvmsta))
+	if (WARN_ON_ONCE(!sta))
 		return -1;
 
-	if (WARN_ON_ONCE(mvmsta->deflink.sta_id == IWL_MVM_INVALID_STA))
+	mvmsta = iwl_mvm_sta_from_mac80211(sta);
+
+	if (WARN_ON_ONCE(mvmsta->deflink.sta_id == IWL_INVALID_STA))
 		return -1;
 
 	memcpy(&info, skb->cb, sizeof(info));
@@ -1448,8 +1368,7 @@ static void iwl_mvm_check_ratid_empty(struct iwl_mvm *mvm,
 
 	lockdep_assert_held(&mvmsta->lock);
 
-	if ((tid_data->state == IWL_AGG_ON ||
-	     tid_data->state == IWL_EMPTYING_HW_QUEUE_DELBA) &&
+	if (tid_data->state == IWL_AGG_ON &&
 	    iwl_mvm_tid_queued(mvm, tid_data) == 0) {
 		/*
 		 * Now that this aggregation or DQA queue is empty tell
@@ -1464,7 +1383,7 @@ static void iwl_mvm_check_ratid_empty(struct iwl_mvm *mvm,
 	 * to align the wrap around of ssn so we compare relevant values.
 	 */
 	normalized_ssn = tid_data->ssn;
-	if (mvm->trans->trans_cfg->gen2)
+	if (mvm->trans->mac_cfg->gen2)
 		normalized_ssn &= 0xff;
 
 	if (normalized_ssn != tid_data->next_reclaimed)
@@ -1478,15 +1397,6 @@ static void iwl_mvm_check_ratid_empty(struct iwl_mvm *mvm,
 		tid_data->state = IWL_AGG_STARTING;
 		ieee80211_start_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		break;
-
-	case IWL_EMPTYING_HW_QUEUE_DELBA:
-		IWL_DEBUG_TX_QUEUES(mvm,
-				    "Can continue DELBA flow ssn = next_recl = %d\n",
-				    tid_data->next_reclaimed);
-		tid_data->state = IWL_AGG_OFF;
-		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
-		break;
-
 	default:
 		break;
 	}
@@ -1553,7 +1463,7 @@ void iwl_mvm_hwrate_to_tx_rate(u32 rate_n_flags,
 			       struct ieee80211_tx_rate *r)
 {
 	u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
-	u32 rate = format == RATE_MCS_HT_MSK ?
+	u32 rate = format == RATE_MCS_MOD_TYPE_HT ?
 		RATE_HT_MCS_INDEX(rate_n_flags) :
 		rate_n_flags & RATE_MCS_CODE_MSK;
 
@@ -1563,68 +1473,51 @@ void iwl_mvm_hwrate_to_tx_rate(u32 rate_n_flags,
 
 	if (rate_n_flags & RATE_MCS_SGI_MSK)
 		r->flags |= IEEE80211_TX_RC_SHORT_GI;
-	if (format ==  RATE_MCS_HT_MSK) {
+	switch (format) {
+	case RATE_MCS_MOD_TYPE_HT:
 		r->flags |= IEEE80211_TX_RC_MCS;
 		r->idx = rate;
-	} else if (format ==  RATE_MCS_VHT_MSK) {
+		break;
+	case RATE_MCS_MOD_TYPE_VHT:
 		ieee80211_rate_set_vht(r, rate,
 				       FIELD_GET(RATE_MCS_NSS_MSK,
 						 rate_n_flags) + 1);
 		r->flags |= IEEE80211_TX_RC_VHT_MCS;
-	} else if (format == RATE_MCS_HE_MSK) {
+		break;
+	case RATE_MCS_MOD_TYPE_HE:
+	case RATE_MCS_MOD_TYPE_EHT:
 		/* mac80211 cannot do this without ieee80211_tx_status_ext()
 		 * but it only matters for radiotap */
 		r->idx = 0;
-	} else {
+		break;
+	default:
 		r->idx = iwl_mvm_legacy_hw_idx_to_mac80211_idx(rate_n_flags,
 							       band);
 	}
 }
 
-void iwl_mvm_hwrate_to_tx_rate_v1(u32 rate_n_flags,
-				  enum nl80211_band band,
-				  struct ieee80211_tx_rate *r)
-{
-	if (rate_n_flags & RATE_HT_MCS_GF_MSK)
-		r->flags |= IEEE80211_TX_RC_GREEN_FIELD;
-
-	r->flags |=
-		iwl_mvm_get_hwrate_chan_width(rate_n_flags &
-					      RATE_MCS_CHAN_WIDTH_MSK_V1);
-
-	if (rate_n_flags & RATE_MCS_SGI_MSK_V1)
-		r->flags |= IEEE80211_TX_RC_SHORT_GI;
-	if (rate_n_flags & RATE_MCS_HT_MSK_V1) {
-		r->flags |= IEEE80211_TX_RC_MCS;
-		r->idx = rate_n_flags & RATE_HT_MCS_INDEX_MSK_V1;
-	} else if (rate_n_flags & RATE_MCS_VHT_MSK_V1) {
-		ieee80211_rate_set_vht(
-			r, rate_n_flags & RATE_VHT_MCS_RATE_CODE_MSK,
-			FIELD_GET(RATE_MCS_NSS_MSK, rate_n_flags) + 1);
-		r->flags |= IEEE80211_TX_RC_VHT_MCS;
-	} else {
-		r->idx = iwl_mvm_legacy_rate_to_mac80211_idx(rate_n_flags,
-							     band);
-	}
-}
-
 /*
  * translate ucode response to mac80211 tx status control values
  */
-static void iwl_mvm_hwrate_to_tx_status(const struct iwl_fw *fw,
-					u32 rate_n_flags,
+static void iwl_mvm_hwrate_to_tx_status(struct iwl_mvm *mvm,
+					__le32 rate_n_flags,
 					struct ieee80211_tx_info *info)
 {
 	struct ieee80211_tx_rate *r = &info->status.rates[0];
+	u32 rate;
 
-	if (iwl_fw_lookup_notif_ver(fw, LONG_GROUP,
-				    TX_CMD, 0) <= 6)
-		rate_n_flags = iwl_new_rate_from_v1(rate_n_flags);
+	/*
+	 * Technically this conversion is incorrect for BA status, however:
+	 *  - we only use the BA notif data for older firmware that have
+	 *    host rate scaling and don't use newer rate formats
+	 *  - the firmware API changed together for BA notif and TX CMD
+	 *    as well
+	 */
+	rate = iwl_mvm_v3_rate_from_fw(rate_n_flags, mvm->fw_rates_ver);
 
 	info->status.antenna =
-		((rate_n_flags & RATE_MCS_ANT_AB_MSK) >> RATE_MCS_ANT_POS);
-	iwl_mvm_hwrate_to_tx_rate(rate_n_flags,
-				  info->band, r);
+		((rate & RATE_MCS_ANT_AB_MSK) >> RATE_MCS_ANT_POS);
+	iwl_mvm_hwrate_to_tx_rate(rate, info->band, r);
 }
 
 static void iwl_mvm_tx_status_check_trigger(struct iwl_mvm *mvm,
@@ -1684,12 +1577,12 @@ static void iwl_mvm_tx_status_check_trigger(struct iwl_mvm *mvm,
  * For 22000-series and lower, this is just 12 bits. For later, 16 bits.
  */
 static inline u32 iwl_mvm_get_scd_ssn(struct iwl_mvm *mvm,
-				      struct iwl_mvm_tx_resp *tx_resp)
+				      struct iwl_tx_resp *tx_resp)
 {
 	u32 val = le32_to_cpup((__le32 *)iwl_mvm_get_agg_status(mvm, tx_resp) +
 			       tx_resp->frame_count);
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		return val & 0xFFFF;
 	return val & 0xFFF;
 }
@@ -1700,10 +1593,10 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 	struct ieee80211_sta *sta;
 	u16 sequence = le16_to_cpu(pkt->hdr.sequence);
 	int txq_id = SEQ_TO_QUEUE(sequence);
-	/* struct iwl_mvm_tx_resp_v3 is almost the same */
-	struct iwl_mvm_tx_resp *tx_resp = (void *)pkt->data;
-	int sta_id = IWL_MVM_TX_RES_GET_RA(tx_resp->ra_tid);
-	int tid = IWL_MVM_TX_RES_GET_TID(tx_resp->ra_tid);
+	/* struct iwl_tx_resp_v3 is almost the same */
+	struct iwl_tx_resp *tx_resp = (void *)pkt->data;
+	int sta_id = IWL_TX_RES_GET_RA(tx_resp->ra_tid);
+	int tid = IWL_TX_RES_GET_TID(tx_resp->ra_tid);
 	struct agg_tx_status *agg_status =
 		iwl_mvm_get_agg_status(mvm, tx_resp);
 	u32 status = le16_to_cpu(agg_status->status);
@@ -1776,9 +1669,7 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 
 		info->status.rates[0].count = tx_resp->failure_frame + 1;
 
-		iwl_mvm_hwrate_to_tx_status(mvm->fw,
-					    le32_to_cpu(tx_resp->initial_rate),
-					    info);
+		iwl_mvm_hwrate_to_tx_status(mvm, tx_resp->initial_rate, info);
 
 		/* Don't assign the converted initial_rate, because driver
 		 * TLC uses this and doesn't support the new FW rate
@@ -1884,7 +1775,9 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 				IWL_DEBUG_TX_REPLY(mvm,
 						   "Next reclaimed packet:%d\n",
 						   next_reclaimed);
-				iwl_mvm_count_mpdu(mvmsta, sta_id, 1, true, 0);
+				if (tid < IWL_MAX_TID_COUNT)
+					iwl_mvm_count_mpdu(mvmsta, sta_id, 1,
+							   true, 0);
 			} else {
 				IWL_DEBUG_TX_REPLY(mvm,
 						   "NDP - don't update next_reclaimed\n");
@@ -1958,7 +1851,7 @@ static const char *iwl_get_agg_tx_status(u16 status)
 static void iwl_mvm_rx_tx_cmd_agg_dbg(struct iwl_mvm *mvm,
 				      struct iwl_rx_packet *pkt)
 {
-	struct iwl_mvm_tx_resp *tx_resp = (void *)pkt->data;
+	struct iwl_tx_resp *tx_resp = (void *)pkt->data;
 	struct agg_tx_status *frame_status =
 		iwl_mvm_get_agg_status(mvm, tx_resp);
 	int i;
@@ -1992,9 +1885,9 @@ static void iwl_mvm_rx_tx_cmd_agg_dbg(struct iwl_mvm *mvm,
 static void iwl_mvm_rx_tx_cmd_agg(struct iwl_mvm *mvm,
 				  struct iwl_rx_packet *pkt)
 {
-	struct iwl_mvm_tx_resp *tx_resp = (void *)pkt->data;
-	int sta_id = IWL_MVM_TX_RES_GET_RA(tx_resp->ra_tid);
-	int tid = IWL_MVM_TX_RES_GET_TID(tx_resp->ra_tid);
+	struct iwl_tx_resp *tx_resp = (void *)pkt->data;
+	int sta_id = IWL_TX_RES_GET_RA(tx_resp->ra_tid);
+	int tid = IWL_TX_RES_GET_TID(tx_resp->ra_tid);
 	u16 sequence = le16_to_cpu(pkt->hdr.sequence);
 	struct iwl_mvm_sta *mvmsta;
 	int queue = SEQ_TO_QUEUE(sequence);
@@ -2018,7 +1911,7 @@ static void iwl_mvm_rx_tx_cmd_agg(struct iwl_mvm *mvm,
 
 	if (!WARN_ON_ONCE(!mvmsta)) {
 		mvmsta->tid_data[tid].rate_n_flags =
-			le32_to_cpu(tx_resp->initial_rate);
+			tx_resp->initial_rate;
 		mvmsta->tid_data[tid].tx_time =
 			le16_to_cpu(tx_resp->wireless_media_time);
 		mvmsta->tid_data[tid].lq_color =
@@ -2033,7 +1926,7 @@ static void iwl_mvm_rx_tx_cmd_agg(struct iwl_mvm *mvm,
 void iwl_mvm_rx_tx_cmd(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_mvm_tx_resp *tx_resp = (void *)pkt->data;
+	struct iwl_tx_resp *tx_resp = (void *)pkt->data;
 
 	if (tx_resp->frame_count == 1)
 		iwl_mvm_rx_tx_cmd_single(mvm, pkt);
@@ -2043,7 +1936,7 @@ void iwl_mvm_rx_tx_cmd(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 
 static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 			       int txq, int index,
-			       struct ieee80211_tx_info *tx_info, u32 rate,
+			       struct ieee80211_tx_info *tx_info, __le32 rate,
 			       bool is_flush)
 {
 	struct sk_buff_head reclaimed_skbs;
@@ -2127,7 +2020,9 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 	tx_info->status.status_driver_data[0] =
 		RS_DRV_DATA_PACK(tid_data->lq_color,
 				 tx_info->status.status_driver_data[0]);
-	tx_info->status.status_driver_data[1] = (void *)(uintptr_t)rate;
+	/* the value is only consumed for old FW that has v1 rates anyway */
+	tx_info->status.status_driver_data[1] =
+		(void *)(uintptr_t)le32_to_cpu(rate);
 
 	skb_queue_walk(&reclaimed_skbs, skb) {
 		struct ieee80211_hdr *hdr = (void *)skb->data;
@@ -2146,7 +2041,7 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 			info->flags |= IEEE80211_TX_STAT_AMPDU;
 			memcpy(&info->status, &tx_info->status,
 			       sizeof(tx_info->status));
-			iwl_mvm_hwrate_to_tx_status(mvm->fw, rate, info);
+			iwl_mvm_hwrate_to_tx_status(mvm, rate, info);
 		}
 	}
 
@@ -2169,7 +2064,7 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 			goto out;
 
 		tx_info->band = chanctx_conf->def.chan->band;
-		iwl_mvm_hwrate_to_tx_status(mvm->fw, rate, tx_info);
+		iwl_mvm_hwrate_to_tx_status(mvm, rate, tx_info);
 
 		IWL_DEBUG_TX_REPLY(mvm, "No reclaim. Update rs directly\n");
 		iwl_mvm_rs_tx_status(mvm, sta, tid, tx_info, false);
@@ -2197,7 +2092,7 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	ba_info.flags = IEEE80211_TX_STAT_AMPDU;
 
 	if (iwl_mvm_has_new_tx_api(mvm)) {
-		struct iwl_mvm_compressed_ba_notif *ba_res =
+		struct iwl_compressed_ba_notif *ba_res =
 			(void *)pkt->data;
 		u8 lq_color = TX_RES_RATE_TABLE_COL_GET(ba_res->tlc_rate_info);
 		u16 tfd_cnt;
@@ -2245,8 +2140,7 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 
 		/* Free per TID */
 		for (i = 0; i < tfd_cnt; i++) {
-			struct iwl_mvm_compressed_ba_tfd *ba_tfd =
-				&ba_res->tfd[i];
+			struct iwl_compressed_ba_tfd *ba_tfd = &ba_res->tfd[i];
 
 			tid = ba_tfd->tid;
 			if (tid == IWL_MGMT_TID)
@@ -2259,7 +2153,7 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 					   (int)(le16_to_cpu(ba_tfd->q_num)),
 					   le16_to_cpu(ba_tfd->tfd_index),
 					   &ba_info,
-					   le32_to_cpu(ba_res->tx_rate), false);
+					   ba_res->tx_rate, false);
 		}
 
 		if (mvmsta) {
diff --git a/sys/contrib/dev/iwlwifi/mvm/utils.c b/sys/contrib/dev/iwlwifi/mvm/utils.c
index 6c507882b08c..e7e24941db15 100644
--- a/sys/contrib/dev/iwlwifi/mvm/utils.c
+++ b/sys/contrib/dev/iwlwifi/mvm/utils.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -145,7 +145,7 @@ int iwl_mvm_legacy_hw_idx_to_mac80211_idx(u32 rate_n_flags,
 	int rate = rate_n_flags & RATE_LEGACY_RATE_MSK;
 	bool is_LB = band == NL80211_BAND_2GHZ;
 
-	if (format == RATE_MCS_LEGACY_OFDM_MSK)
+	if (format == RATE_MCS_MOD_TYPE_LEGACY_OFDM)
 		return is_LB ? rate + IWL_FIRST_OFDM_RATE :
 			rate;
 
@@ -172,15 +172,9 @@ int iwl_mvm_legacy_rate_to_mac80211_idx(u32 rate_n_flags,
 
 u8 iwl_mvm_mac80211_idx_to_hwrate(const struct iwl_fw *fw, int rate_idx)
 {
-	if (iwl_fw_lookup_cmd_ver(fw, TX_CMD, 0) > 8)
-		/* In the new rate legacy rates are indexed:
-		 * 0 - 3 for CCK and 0 - 7 for OFDM.
-		 */
-		return (rate_idx >= IWL_FIRST_OFDM_RATE ?
-			rate_idx - IWL_FIRST_OFDM_RATE :
-			rate_idx);
-
-	return iwl_fw_rate_idx_to_plcp(rate_idx);
+	return (rate_idx >= IWL_FIRST_OFDM_RATE ?
+		rate_idx - IWL_FIRST_OFDM_RATE :
+		rate_idx);
 }
 
 u8 iwl_mvm_mac80211_ac_to_ucode_ac(enum ieee80211_ac_numbers ac)
@@ -264,7 +258,7 @@ int iwl_mvm_send_lq_cmd(struct iwl_mvm *mvm, struct iwl_lq_cmd *lq)
 		.data = { lq, },
 	};
 
-	if (WARN_ON(lq->sta_id == IWL_MVM_INVALID_STA ||
+	if (WARN_ON(lq->sta_id == IWL_INVALID_STA ||
 		    iwl_mvm_has_tlc_offload(mvm)))
 		return -EINVAL;
 
@@ -300,6 +294,10 @@ void iwl_mvm_update_smps(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (vif->type != NL80211_IFTYPE_STATION)
 		return;
 
+	/* SMPS is handled by firmware */
+	if (iwl_mvm_has_rlc_offload(mvm))
+		return;
+
 	mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
 	if (WARN_ON_ONCE(!mvmvif->link[link_id]))
@@ -678,10 +676,8 @@ struct ieee80211_vif *iwl_mvm_get_bss_vif(struct iwl_mvm *mvm)
 		mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
 		iwl_mvm_bss_iface_iterator, &bss_iter_data);
 
-	if (bss_iter_data.error) {
-		IWL_ERR(mvm, "More than one managed interface active!\n");
+	if (bss_iter_data.error)
 		return ERR_PTR(-EINVAL);
-	}
 
 	return bss_iter_data.vif;
 }
@@ -746,58 +742,20 @@ bool iwl_mvm_is_vif_assoc(struct iwl_mvm *mvm)
 }
 
 unsigned int iwl_mvm_get_wd_timeout(struct iwl_mvm *mvm,
-				    struct ieee80211_vif *vif,
-				    bool tdls, bool cmd_q)
+				    struct ieee80211_vif *vif)
 {
-	struct iwl_fw_dbg_trigger_tlv *trigger;
-	struct iwl_fw_dbg_trigger_txq_timer *txq_timer;
-	unsigned int default_timeout = cmd_q ?
-		IWL_DEF_WD_TIMEOUT :
-		mvm->trans->trans_cfg->base_params->wd_timeout;
+	unsigned int default_timeout =
+		mvm->trans->mac_cfg->base->wd_timeout;
 
-	if (!iwl_fw_dbg_trigger_enabled(mvm->fw, FW_DBG_TRIGGER_TXQ_TIMERS)) {
-		/*
-		 * We can't know when the station is asleep or awake, so we
-		 * must disable the queue hang detection.
-		 */
-		if (fw_has_capa(&mvm->fw->ucode_capa,
-				IWL_UCODE_TLV_CAPA_STA_PM_NOTIF) &&
-		    vif && vif->type == NL80211_IFTYPE_AP)
-			return IWL_WATCHDOG_DISABLED;
-		return default_timeout;
-	}
-
-	trigger = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_TXQ_TIMERS);
-	txq_timer = (void *)trigger->data;
-
-	if (tdls)
-		return le32_to_cpu(txq_timer->tdls);
-
-	if (cmd_q)
-		return le32_to_cpu(txq_timer->command_queue);
-
-	if (WARN_ON(!vif))
-		return default_timeout;
-
-	switch (ieee80211_vif_type_p2p(vif)) {
-	case NL80211_IFTYPE_ADHOC:
-		return le32_to_cpu(txq_timer->ibss);
-	case NL80211_IFTYPE_STATION:
-		return le32_to_cpu(txq_timer->bss);
-	case NL80211_IFTYPE_AP:
-		return le32_to_cpu(txq_timer->softap);
-	case NL80211_IFTYPE_P2P_CLIENT:
-		return le32_to_cpu(txq_timer->p2p_client);
-	case NL80211_IFTYPE_P2P_GO:
-		return le32_to_cpu(txq_timer->p2p_go);
-	case NL80211_IFTYPE_P2P_DEVICE:
-		return le32_to_cpu(txq_timer->p2p_device);
-	case NL80211_IFTYPE_MONITOR:
-		return default_timeout;
-	default:
-		WARN_ON(1);
-		return mvm->trans->trans_cfg->base_params->wd_timeout;
-	}
+	/*
+	 * We can't know when the station is asleep or awake, so we
+	 * must disable the queue hang detection.
+	 */
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_STA_PM_NOTIF) &&
+	    vif->type == NL80211_IFTYPE_AP)
+		return IWL_WATCHDOG_DISABLED;
+	return default_timeout;
 }
 
 void iwl_mvm_connection_loss(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
@@ -1226,9 +1184,9 @@ u32 iwl_mvm_get_systime(struct iwl_mvm *mvm)
 {
 	u32 reg_addr = DEVICE_SYSTEM_TIME_REG;
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000 &&
-	    mvm->trans->cfg->gp2_reg_addr)
-		reg_addr = mvm->trans->cfg->gp2_reg_addr;
+	if (mvm->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_22000 &&
+	    mvm->trans->mac_cfg->base->gp2_reg_addr)
+		reg_addr = mvm->trans->mac_cfg->base->gp2_reg_addr;
 
 	return iwl_read_prph(mvm->trans, reg_addr);
 }
@@ -1279,6 +1237,170 @@ bool iwl_mvm_have_links_same_channel(struct iwl_mvm_vif *vif1,
 	return false;
 }
 
+static u32 iwl_legacy_rate_to_fw_idx(u32 rate_n_flags)
+{
+	int rate = rate_n_flags & RATE_LEGACY_RATE_MSK_V1;
+	int idx;
+	bool ofdm = !(rate_n_flags & RATE_MCS_CCK_MSK_V1);
+	int offset = ofdm ? IWL_FIRST_OFDM_RATE : 0;
+	int last = ofdm ? IWL_RATE_COUNT_LEGACY : IWL_FIRST_OFDM_RATE;
+
+	for (idx = offset; idx < last; idx++)
+		if (iwl_fw_rate_idx_to_plcp(idx) == rate)
+			return idx - offset;
+	return IWL_RATE_INVALID;
+}
+
+u32 iwl_mvm_v3_rate_from_fw(__le32 rate, u8 rate_ver)
+{
+	u32 rate_v3 = 0, rate_v1;
+	u32 dup = 0;
+
+	if (rate_ver > 1)
+		return iwl_v3_rate_from_v2_v3(rate, rate_ver >= 3);
+
+	rate_v1 = le32_to_cpu(rate);
+	if (rate_v1 == 0)
+		return rate_v1;
+	/* convert rate */
+	if (rate_v1 & RATE_MCS_HT_MSK_V1) {
+		u32 nss;
+
+		rate_v3 |= RATE_MCS_MOD_TYPE_HT;
+		rate_v3 |=
+			rate_v1 & RATE_HT_MCS_RATE_CODE_MSK_V1;
+		nss = u32_get_bits(rate_v1, RATE_HT_MCS_MIMO2_MSK);
+		rate_v3 |= u32_encode_bits(nss, RATE_MCS_NSS_MSK);
+	} else if (rate_v1 & RATE_MCS_VHT_MSK_V1 ||
+		   rate_v1 & RATE_MCS_HE_MSK_V1) {
+		u32 nss = u32_get_bits(rate_v1, RATE_VHT_MCS_NSS_MSK);
+
+		rate_v3 |= rate_v1 & RATE_VHT_MCS_RATE_CODE_MSK;
+
+		rate_v3 |= u32_encode_bits(nss, RATE_MCS_NSS_MSK);
+
+		if (rate_v1 & RATE_MCS_HE_MSK_V1) {
+			u32 he_type_bits = rate_v1 & RATE_MCS_HE_TYPE_MSK_V1;
+			u32 he_type = he_type_bits >> RATE_MCS_HE_TYPE_POS_V1;
+			u32 he_106t = (rate_v1 & RATE_MCS_HE_106T_MSK_V1) >>
+				RATE_MCS_HE_106T_POS_V1;
+			u32 he_gi_ltf = (rate_v1 & RATE_MCS_HE_GI_LTF_MSK_V1) >>
+				RATE_MCS_HE_GI_LTF_POS;
+
+			if ((he_type_bits == RATE_MCS_HE_TYPE_SU ||
+			     he_type_bits == RATE_MCS_HE_TYPE_EXT_SU) &&
+			    he_gi_ltf == RATE_MCS_HE_SU_4_LTF)
+				/* the new rate have an additional bit to
+				 * represent the value 4 rather then using SGI
+				 * bit for this purpose - as it was done in the
+				 * old rate
+				 */
+				he_gi_ltf += (rate_v1 & RATE_MCS_SGI_MSK_V1) >>
+					RATE_MCS_SGI_POS_V1;
+
+			rate_v3 |= he_gi_ltf << RATE_MCS_HE_GI_LTF_POS;
+			rate_v3 |= he_type << RATE_MCS_HE_TYPE_POS;
+			rate_v3 |= he_106t << RATE_MCS_HE_106T_POS;
+			rate_v3 |= rate_v1 & RATE_HE_DUAL_CARRIER_MODE_MSK;
+			rate_v3 |= RATE_MCS_MOD_TYPE_HE;
+		} else {
+			rate_v3 |= RATE_MCS_MOD_TYPE_VHT;
+		}
+	/* if legacy format */
+	} else {
+		u32 legacy_rate = iwl_legacy_rate_to_fw_idx(rate_v1);
+
+		if (WARN_ON_ONCE(legacy_rate == IWL_RATE_INVALID))
+			legacy_rate = (rate_v1 & RATE_MCS_CCK_MSK_V1) ?
+				IWL_FIRST_CCK_RATE : IWL_FIRST_OFDM_RATE;
+
+		rate_v3 |= legacy_rate;
+		if (!(rate_v1 & RATE_MCS_CCK_MSK_V1))
+			rate_v3 |= RATE_MCS_MOD_TYPE_LEGACY_OFDM;
+	}
+
+	/* convert flags */
+	if (rate_v1 & RATE_MCS_LDPC_MSK_V1)
+		rate_v3 |= RATE_MCS_LDPC_MSK;
+	rate_v3 |= (rate_v1 & RATE_MCS_CHAN_WIDTH_MSK_V1) |
+		(rate_v1 & RATE_MCS_ANT_AB_MSK) |
+		(rate_v1 & RATE_MCS_STBC_MSK) |
+		(rate_v1 & RATE_MCS_BF_MSK);
+
+	dup = (rate_v1 & RATE_MCS_DUP_MSK_V1) >> RATE_MCS_DUP_POS_V1;
+	if (dup) {
+		rate_v3 |= RATE_MCS_DUP_MSK;
+		rate_v3 |= dup << RATE_MCS_CHAN_WIDTH_POS;
+	}
+
+	if ((!(rate_v1 & RATE_MCS_HE_MSK_V1)) &&
+	    (rate_v1 & RATE_MCS_SGI_MSK_V1))
+		rate_v3 |= RATE_MCS_SGI_MSK;
+
+	return rate_v3;
+}
+
+__le32 iwl_mvm_v3_rate_to_fw(u32 rate, u8 rate_ver)
+{
+	u32 result = 0;
+	int rate_idx;
+
+	if (rate_ver > 1)
+		return iwl_v3_rate_to_v2_v3(rate, rate_ver > 2);
+
+	switch (rate & RATE_MCS_MOD_TYPE_MSK) {
+	case RATE_MCS_MOD_TYPE_CCK:
+		result = RATE_MCS_CCK_MSK_V1;
+		fallthrough;
+	case RATE_MCS_MOD_TYPE_LEGACY_OFDM:
+		rate_idx = u32_get_bits(rate, RATE_LEGACY_RATE_MSK);
+		if (!(result & RATE_MCS_CCK_MSK_V1))
+			rate_idx += IWL_FIRST_OFDM_RATE;
+		result |= u32_encode_bits(iwl_fw_rate_idx_to_plcp(rate_idx),
+					  RATE_LEGACY_RATE_MSK_V1);
+		break;
+	case RATE_MCS_MOD_TYPE_HT:
+		result = RATE_MCS_HT_MSK_V1;
+		result |= u32_encode_bits(u32_get_bits(rate,
+						       RATE_HT_MCS_CODE_MSK),
+					  RATE_HT_MCS_RATE_CODE_MSK_V1);
+		result |= u32_encode_bits(u32_get_bits(rate,
+						       RATE_MCS_NSS_MSK),
+					  RATE_HT_MCS_MIMO2_MSK);
+		break;
+	case RATE_MCS_MOD_TYPE_VHT:
+		result = RATE_MCS_VHT_MSK_V1;
+		result |= u32_encode_bits(u32_get_bits(rate,
+						       RATE_VHT_MCS_NSS_MSK),
+					  RATE_MCS_CODE_MSK);
+		result |= u32_encode_bits(u32_get_bits(rate, RATE_MCS_NSS_MSK),
+					  RATE_VHT_MCS_NSS_MSK);
+		break;
+	case RATE_MCS_MOD_TYPE_HE: /* not generated */
+	default:
+		WARN_ONCE(1, "bad modulation type %d\n",
+			  u32_get_bits(rate, RATE_MCS_MOD_TYPE_MSK));
+		return 0;
+	}
+
+	if (rate & RATE_MCS_LDPC_MSK)
+		result |= RATE_MCS_LDPC_MSK_V1;
+	WARN_ON_ONCE(u32_get_bits(rate, RATE_MCS_CHAN_WIDTH_MSK) >
+			RATE_MCS_CHAN_WIDTH_160_VAL);
+	result |= (rate & RATE_MCS_CHAN_WIDTH_MSK_V1) |
+		  (rate & RATE_MCS_ANT_AB_MSK) |
+		  (rate & RATE_MCS_STBC_MSK) |
+		  (rate & RATE_MCS_BF_MSK);
+
+	/* not handling DUP since we don't use it */
+	WARN_ON_ONCE(rate & RATE_MCS_DUP_MSK);
+
+	if (rate & RATE_MCS_SGI_MSK)
+		result |= RATE_MCS_SGI_MSK_V1;
+
+	return cpu_to_le32(result);
+}
+
 bool iwl_mvm_vif_is_active(struct iwl_mvm_vif *mvmvif)
 {
 	unsigned int i;
diff --git a/sys/contrib/dev/iwlwifi/pcie/ctxt-info-gen3.c b/sys/contrib/dev/iwlwifi/pcie/ctxt-info-v2.c
index ae93a72542b2..06be929a3ca5 100644
--- a/sys/contrib/dev/iwlwifi/pcie/ctxt-info-gen3.c
+++ b/sys/contrib/dev/iwlwifi/pcie/ctxt-info-v2.c
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include <linux/dmi.h>
 #include "iwl-trans.h"
 #include "iwl-fh.h"
-#include "iwl-context-info-gen3.h"
-#include "internal.h"
+#include "iwl-context-info-v2.h"
+#include "gen1_2/internal.h"
 #include "iwl-prph.h"
 
 static const struct dmi_system_id dmi_force_scu_active_approved_list[] = {
@@ -97,20 +97,22 @@ out:
 		*control_flags |= IWL_PRPH_SCRATCH_EARLY_DEBUG_EN | dbg_flags;
 }
 
-int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
-				 const struct fw_img *fw)
+int iwl_pcie_ctxt_info_v2_alloc(struct iwl_trans *trans,
+				const struct iwl_fw *fw,
+				const struct fw_img *img)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwl_context_info_gen3 *ctxt_info_gen3;
+	struct iwl_context_info_v2 *ctxt_info_v2;
 	struct iwl_prph_scratch *prph_scratch;
 	struct iwl_prph_scratch_ctrl_cfg *prph_sc_ctrl;
 	struct iwl_prph_info *prph_info;
 	u32 control_flags = 0;
+	u32 control_flags_ext = 0;
 	int ret;
 	int cmdq_size = max_t(u32, IWL_CMD_QUEUE_SIZE,
-			      trans->cfg->min_txq_size);
+			      trans->mac_cfg->base->min_txq_size);
 
-	switch (trans_pcie->rx_buf_size) {
+	switch (trans->conf.rx_buf_size) {
 	case IWL_AMSDU_DEF:
 		return -EINVAL;
 	case IWL_AMSDU_2K:
@@ -130,6 +132,15 @@ int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
 		break;
 	}
 
+	if (trans->conf.dsbr_urm_fw_dependent)
+		control_flags_ext |= IWL_PRPH_SCRATCH_EXT_URM_FW;
+
+	if (trans->conf.dsbr_urm_permanent)
+		control_flags_ext |= IWL_PRPH_SCRATCH_EXT_URM_PERM;
+
+	if (trans->conf.ext_32khz_clock_valid)
+		control_flags_ext |= IWL_PRPH_SCRATCH_EXT_32KHZ_CLK_VALID;
+
 	/* Allocate prph scratch */
 	prph_scratch = dma_alloc_coherent(trans->dev, sizeof(*prph_scratch),
 					  &trans_pcie->prph_scratch_dma_addr,
@@ -141,16 +152,16 @@ int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
 
 	prph_sc_ctrl->version.version = 0;
 	prph_sc_ctrl->version.mac_id =
-		cpu_to_le16((u16)trans->hw_rev);
+		cpu_to_le16((u16)trans->info.hw_rev);
 	prph_sc_ctrl->version.size = cpu_to_le16(sizeof(*prph_scratch) / 4);
 
 	control_flags |= IWL_PRPH_SCRATCH_MTR_MODE;
 	control_flags |= IWL_PRPH_MTR_FORMAT_256B & IWL_PRPH_SCRATCH_MTR_FORMAT;
 
-	if (trans->trans_cfg->imr_enabled)
+	if (trans->mac_cfg->imr_enabled)
 		control_flags |= IWL_PRPH_SCRATCH_IMR_DEBUG_EN;
 
-	if (CSR_HW_REV_TYPE(trans->hw_rev) == IWL_CFG_MAC_TYPE_GL &&
+	if (CSR_HW_REV_TYPE(trans->info.hw_rev) == IWL_CFG_MAC_TYPE_GL &&
 	    iwl_is_force_scu_active_approved()) {
 		control_flags |= IWL_PRPH_SCRATCH_SCU_FORCE_ACTIVE;
 		IWL_DEBUG_FW(trans,
@@ -158,6 +169,11 @@ int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
 			     IWL_PRPH_SCRATCH_SCU_FORCE_ACTIVE);
 	}
 
+	if (trans->do_top_reset) {
+		WARN_ON(trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_SC);
+		control_flags |= IWL_PRPH_SCRATCH_TOP_RESET;
+	}
+
 	/* initialize RX default queue */
 	prph_sc_ctrl->rbd_cfg.free_rbd_addr =
 		cpu_to_le64(trans_pcie->rxq->bd_dma);
@@ -165,17 +181,19 @@ int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
 	iwl_pcie_ctxt_info_dbg_enable(trans, &prph_sc_ctrl->hwm_cfg,
 				      &control_flags);
 	prph_sc_ctrl->control.control_flags = cpu_to_le32(control_flags);
+	prph_sc_ctrl->control.control_flags_ext = cpu_to_le32(control_flags_ext);
 
 	/* initialize the Step equalizer data */
-	prph_sc_ctrl->step_cfg.mbx_addr_0 = cpu_to_le32(trans->mbx_addr_0_step);
-	prph_sc_ctrl->step_cfg.mbx_addr_1 = cpu_to_le32(trans->mbx_addr_1_step);
+	prph_sc_ctrl->step_cfg.mbx_addr_0 =
+		cpu_to_le32(trans->conf.mbx_addr_0_step);
+	prph_sc_ctrl->step_cfg.mbx_addr_1 =
+		cpu_to_le32(trans->conf.mbx_addr_1_step);
 
 	/* allocate ucode sections in dram and set addresses */
-	ret = iwl_pcie_init_fw_sec(trans, fw, &prph_scratch->dram);
+	ret = iwl_pcie_init_fw_sec(trans, img, &prph_scratch->dram.common);
 	if (ret)
 		goto err_free_prph_scratch;
 
-
 	/* Allocate prph information
 	 * currently we don't assign to the prph info anything, but it would get
 	 * assigned later
@@ -195,42 +213,58 @@ int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
 	}
 
 	/* Allocate context info */
-	ctxt_info_gen3 = dma_alloc_coherent(trans->dev,
-					    sizeof(*ctxt_info_gen3),
-					    &trans_pcie->ctxt_info_dma_addr,
-					    GFP_KERNEL);
-	if (!ctxt_info_gen3) {
+	ctxt_info_v2 = dma_alloc_coherent(trans->dev,
+					  sizeof(*ctxt_info_v2),
+					  &trans_pcie->ctxt_info_dma_addr,
+					  GFP_KERNEL);
+	if (!ctxt_info_v2) {
 		ret = -ENOMEM;
 		goto err_free_prph_info;
 	}
 
-	ctxt_info_gen3->prph_info_base_addr =
+	ctxt_info_v2->prph_info_base_addr =
 		cpu_to_le64(trans_pcie->prph_info_dma_addr);
-	ctxt_info_gen3->prph_scratch_base_addr =
+	ctxt_info_v2->prph_scratch_base_addr =
 		cpu_to_le64(trans_pcie->prph_scratch_dma_addr);
-	ctxt_info_gen3->prph_scratch_size =
-		cpu_to_le32(sizeof(*prph_scratch));
-	ctxt_info_gen3->cr_head_idx_arr_base_addr =
+
+	/*
+	 * This code assumes the FSEQ is last and we can make that
+	 * optional; old devices _should_ be fine with a bigger size,
+	 * but in simulation we check the size more precisely.
+	 */
+	BUILD_BUG_ON(offsetofend(typeof(*prph_scratch), dram.common) +
+		     sizeof(prph_scratch->dram.fseq_img) !=
+		     sizeof(*prph_scratch));
+	if (control_flags_ext & IWL_PRPH_SCRATCH_EXT_EXT_FSEQ)
+		ctxt_info_v2->prph_scratch_size =
+			cpu_to_le32(sizeof(*prph_scratch));
+	else
+		ctxt_info_v2->prph_scratch_size =
+			cpu_to_le32(offsetofend(typeof(*prph_scratch),
+						dram.common));
+
+	ctxt_info_v2->cr_head_idx_arr_base_addr =
 		cpu_to_le64(trans_pcie->rxq->rb_stts_dma);
-	ctxt_info_gen3->tr_tail_idx_arr_base_addr =
+	ctxt_info_v2->tr_tail_idx_arr_base_addr =
 		cpu_to_le64(trans_pcie->prph_info_dma_addr + PAGE_SIZE / 2);
-	ctxt_info_gen3->cr_tail_idx_arr_base_addr =
+	ctxt_info_v2->cr_tail_idx_arr_base_addr =
 		cpu_to_le64(trans_pcie->prph_info_dma_addr + 3 * PAGE_SIZE / 4);
-	ctxt_info_gen3->mtr_base_addr =
-		cpu_to_le64(trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id]->dma_addr);
-	ctxt_info_gen3->mcr_base_addr =
+	ctxt_info_v2->mtr_base_addr =
+		cpu_to_le64(trans_pcie->txqs.txq[trans->conf.cmd_queue]->dma_addr);
+	ctxt_info_v2->mcr_base_addr =
 		cpu_to_le64(trans_pcie->rxq->used_bd_dma);
-	ctxt_info_gen3->mtr_size =
+	ctxt_info_v2->mtr_size =
 		cpu_to_le16(TFD_QUEUE_CB_SIZE(cmdq_size));
-	ctxt_info_gen3->mcr_size =
-		cpu_to_le16(RX_QUEUE_CB_SIZE(trans->cfg->num_rbds));
+	ctxt_info_v2->mcr_size =
+		cpu_to_le16(RX_QUEUE_CB_SIZE(iwl_trans_get_num_rbds(trans)));
 
-	trans_pcie->ctxt_info_gen3 = ctxt_info_gen3;
+	trans_pcie->ctxt_info_v2 = ctxt_info_v2;
 	trans_pcie->prph_info = prph_info;
 	trans_pcie->prph_scratch = prph_scratch;
 
 	/* Allocate IML */
-	trans_pcie->iml = dma_alloc_coherent(trans->dev, trans->iml_len,
+	trans_pcie->iml_len = fw->iml_len;
+	trans_pcie->iml = dma_alloc_coherent(trans->dev, fw->iml_len,
 					     &trans_pcie->iml_dma_addr,
 					     GFP_KERNEL);
 	if (!trans_pcie->iml) {
@@ -238,27 +272,15 @@ int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
 		goto err_free_ctxt_info;
 	}
 
-	memcpy(trans_pcie->iml, trans->iml, trans->iml_len);
-
-	iwl_enable_fw_load_int_ctx_info(trans);
-
-	/* kick FW self load */
-	iwl_write64(trans, CSR_CTXT_INFO_ADDR,
-		    trans_pcie->ctxt_info_dma_addr);
-	iwl_write64(trans, CSR_IML_DATA_ADDR,
-		    trans_pcie->iml_dma_addr);
-	iwl_write32(trans, CSR_IML_SIZE_ADDR, trans->iml_len);
-
-	iwl_set_bit(trans, CSR_CTXT_INFO_BOOT_CTRL,
-		    CSR_AUTO_FUNC_BOOT_ENA);
+	memcpy(trans_pcie->iml, fw->iml, fw->iml_len);
 
 	return 0;
 
 err_free_ctxt_info:
-	dma_free_coherent(trans->dev, sizeof(*trans_pcie->ctxt_info_gen3),
-			  trans_pcie->ctxt_info_gen3,
+	dma_free_coherent(trans->dev, sizeof(*trans_pcie->ctxt_info_v2),
+			  trans_pcie->ctxt_info_v2,
 			  trans_pcie->ctxt_info_dma_addr);
-	trans_pcie->ctxt_info_gen3 = NULL;
+	trans_pcie->ctxt_info_v2 = NULL;
 err_free_prph_info:
 	dma_free_coherent(trans->dev, PAGE_SIZE, prph_info,
 			  trans_pcie->prph_info_dma_addr);
@@ -272,14 +294,31 @@ err_free_prph_scratch:
 
 }
 
-void iwl_pcie_ctxt_info_gen3_free(struct iwl_trans *trans, bool alive)
+void iwl_pcie_ctxt_info_v2_kick(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+	iwl_enable_fw_load_int_ctx_info(trans, trans->do_top_reset);
+
+	/* kick FW self load */
+	iwl_write64(trans, CSR_CTXT_INFO_ADDR, trans_pcie->ctxt_info_dma_addr);
+	iwl_write64(trans, CSR_IML_DATA_ADDR, trans_pcie->iml_dma_addr);
+	iwl_write32(trans, CSR_IML_SIZE_ADDR, trans_pcie->iml_len);
+
+	iwl_set_bit(trans, CSR_CTXT_INFO_BOOT_CTRL,
+		    CSR_AUTO_FUNC_BOOT_ENA);
+}
+
+void iwl_pcie_ctxt_info_v2_free(struct iwl_trans *trans, bool alive)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	if (trans_pcie->iml) {
-		dma_free_coherent(trans->dev, trans->iml_len, trans_pcie->iml,
+		dma_free_coherent(trans->dev, trans_pcie->iml_len,
+				  trans_pcie->iml,
 				  trans_pcie->iml_dma_addr);
 		trans_pcie->iml_dma_addr = 0;
+		trans_pcie->iml_len = 0;
 		trans_pcie->iml = NULL;
 	}
 
@@ -288,15 +327,15 @@ void iwl_pcie_ctxt_info_gen3_free(struct iwl_trans *trans, bool alive)
 	if (alive)
 		return;
 
-	if (!trans_pcie->ctxt_info_gen3)
+	if (!trans_pcie->ctxt_info_v2)
 		return;
 
-	/* ctxt_info_gen3 and prph_scratch are still needed for PNVM load */
-	dma_free_coherent(trans->dev, sizeof(*trans_pcie->ctxt_info_gen3),
-			  trans_pcie->ctxt_info_gen3,
+	/* ctxt_info_v2 and prph_scratch are still needed for PNVM load */
+	dma_free_coherent(trans->dev, sizeof(*trans_pcie->ctxt_info_v2),
+			  trans_pcie->ctxt_info_v2,
 			  trans_pcie->ctxt_info_dma_addr);
 	trans_pcie->ctxt_info_dma_addr = 0;
-	trans_pcie->ctxt_info_gen3 = NULL;
+	trans_pcie->ctxt_info_v2 = NULL;
 
 	dma_free_coherent(trans->dev, sizeof(*trans_pcie->prph_scratch),
 			  trans_pcie->prph_scratch,
@@ -311,9 +350,9 @@ void iwl_pcie_ctxt_info_gen3_free(struct iwl_trans *trans, bool alive)
 	trans_pcie->prph_info = NULL;
 }
 
-static int iwl_pcie_load_payloads_continuously(struct iwl_trans *trans,
-					       const struct iwl_pnvm_image *pnvm_data,
-					       struct iwl_dram_data *dram)
+static int iwl_pcie_load_payloads_contig(struct iwl_trans *trans,
+					 const struct iwl_pnvm_image *pnvm_data,
+					 struct iwl_dram_data *dram)
 {
 	u32 len, len0, len1;
 
@@ -352,13 +391,13 @@ static int iwl_pcie_load_payloads_segments
 {
 	struct iwl_dram_data *cur_payload_dram = &dram_regions->drams[0];
 	struct iwl_dram_data *desc_dram = &dram_regions->prph_scratch_mem_desc;
-	struct iwl_prph_scrath_mem_desc_addr_array *addresses;
+	struct iwl_prph_scratch_mem_desc_addr_array *addresses;
 	const void *data;
 	u32 len;
 	int i;
 
 	/* allocate and init DRAM descriptors array */
-	len = sizeof(struct iwl_prph_scrath_mem_desc_addr_array);
+	len = sizeof(struct iwl_prph_scratch_mem_desc_addr_array);
 	desc_dram->block = iwl_pcie_ctxt_info_dma_alloc_coherent
 						(trans,
 						 len,
@@ -400,9 +439,9 @@ static int iwl_pcie_load_payloads_segments
 
 }
 
-int iwl_trans_pcie_ctx_info_gen3_load_pnvm(struct iwl_trans *trans,
-					   const struct iwl_pnvm_image *pnvm_payloads,
-					   const struct iwl_ucode_capabilities *capa)
+int iwl_trans_pcie_ctx_info_v2_load_pnvm(struct iwl_trans *trans,
+					 const struct iwl_pnvm_image *pnvm_payloads,
+					 const struct iwl_ucode_capabilities *capa)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_prph_scratch_ctrl_cfg *prph_sc_ctrl =
@@ -417,7 +456,7 @@ int iwl_trans_pcie_ctx_info_gen3_load_pnvm(struct iwl_trans *trans,
 	if (WARN_ON(prph_sc_ctrl->pnvm_cfg.pnvm_size))
 		return -EBUSY;
 
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		return 0;
 
 	if (!pnvm_payloads->n_chunks) {
@@ -434,10 +473,8 @@ int iwl_trans_pcie_ctx_info_gen3_load_pnvm(struct iwl_trans *trans,
 			trans->pnvm_loaded = true;
 	} else {
 		/* save only in one DRAM section */
-		ret = iwl_pcie_load_payloads_continuously
-						(trans,
-						 pnvm_payloads,
-						 &dram_regions->drams[0]);
+		ret = iwl_pcie_load_payloads_contig(trans, pnvm_payloads,
+						    &dram_regions->drams[0]);
 		if (!ret) {
 			dram_regions->n_regions = 1;
 			trans->pnvm_loaded = true;
@@ -472,7 +509,7 @@ static void iwl_pcie_set_pnvm_segments(struct iwl_trans *trans)
 		cpu_to_le32(iwl_dram_regions_size(dram_regions));
 }
 
-static void iwl_pcie_set_continuous_pnvm(struct iwl_trans *trans)
+static void iwl_pcie_set_contig_pnvm(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_prph_scratch_ctrl_cfg *prph_sc_ctrl =
@@ -484,21 +521,21 @@ static void iwl_pcie_set_continuous_pnvm(struct iwl_trans *trans)
 		cpu_to_le32(trans_pcie->pnvm_data.drams[0].size);
 }
 
-void iwl_trans_pcie_ctx_info_gen3_set_pnvm(struct iwl_trans *trans,
-					   const struct iwl_ucode_capabilities *capa)
+void iwl_trans_pcie_ctx_info_v2_set_pnvm(struct iwl_trans *trans,
+					 const struct iwl_ucode_capabilities *capa)
 {
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		return;
 
 	if (fw_has_capa(capa, IWL_UCODE_TLV_CAPA_FRAGMENTED_PNVM_IMG))
 		iwl_pcie_set_pnvm_segments(trans);
 	else
-		iwl_pcie_set_continuous_pnvm(trans);
+		iwl_pcie_set_contig_pnvm(trans);
 }
 
-int iwl_trans_pcie_ctx_info_gen3_load_reduce_power(struct iwl_trans *trans,
-						   const struct iwl_pnvm_image *payloads,
-						   const struct iwl_ucode_capabilities *capa)
+int iwl_trans_pcie_ctx_info_v2_load_reduce_power(struct iwl_trans *trans,
+						 const struct iwl_pnvm_image *payloads,
+						 const struct iwl_ucode_capabilities *capa)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_prph_scratch_ctrl_cfg *prph_sc_ctrl =
@@ -510,7 +547,7 @@ int iwl_trans_pcie_ctx_info_gen3_load_reduce_power(struct iwl_trans *trans,
 	if (trans->reduce_power_loaded)
 		return 0;
 
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		return 0;
 
 	if (WARN_ON(prph_sc_ctrl->reduce_power_cfg.size))
@@ -530,10 +567,8 @@ int iwl_trans_pcie_ctx_info_gen3_load_reduce_power(struct iwl_trans *trans,
 			trans->reduce_power_loaded = true;
 	} else {
 		/* save only in one DRAM section */
-		ret = iwl_pcie_load_payloads_continuously
-						(trans,
-						 payloads,
-						 &dram_regions->drams[0]);
+		ret = iwl_pcie_load_payloads_contig(trans, payloads,
+						    &dram_regions->drams[0]);
 		if (!ret) {
 			dram_regions->n_regions = 1;
 			trans->reduce_power_loaded = true;
@@ -556,7 +591,7 @@ static void iwl_pcie_set_reduce_power_segments(struct iwl_trans *trans)
 		cpu_to_le32(iwl_dram_regions_size(dram_regions));
 }
 
-static void iwl_pcie_set_continuous_reduce_power(struct iwl_trans *trans)
+static void iwl_pcie_set_contig_reduce_power(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_prph_scratch_ctrl_cfg *prph_sc_ctrl =
@@ -569,15 +604,15 @@ static void iwl_pcie_set_continuous_reduce_power(struct iwl_trans *trans)
 }
 
 void
-iwl_trans_pcie_ctx_info_gen3_set_reduce_power(struct iwl_trans *trans,
-					      const struct iwl_ucode_capabilities *capa)
+iwl_trans_pcie_ctx_info_v2_set_reduce_power(struct iwl_trans *trans,
+					    const struct iwl_ucode_capabilities *capa)
 {
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		return;
 
 	if (fw_has_capa(capa, IWL_UCODE_TLV_CAPA_FRAGMENTED_PNVM_IMG))
 		iwl_pcie_set_reduce_power_segments(trans);
 	else
-		iwl_pcie_set_continuous_reduce_power(trans);
+		iwl_pcie_set_contig_reduce_power(trans);
 }
 
diff --git a/sys/contrib/dev/iwlwifi/pcie/ctxt-info.c b/sys/contrib/dev/iwlwifi/pcie/ctxt-info.c
index 344e4d5a1c6e..0957223c776d 100644
--- a/sys/contrib/dev/iwlwifi/pcie/ctxt-info.c
+++ b/sys/contrib/dev/iwlwifi/pcie/ctxt-info.c
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #include "iwl-trans.h"
 #include "iwl-fh.h"
 #include "iwl-context-info.h"
-#include "internal.h"
+#include "gen1_2/internal.h"
 #include "iwl-prph.h"
 
 static void *_iwl_pcie_ctxt_info_dma_alloc_coherent(struct iwl_trans *trans,
@@ -83,7 +83,7 @@ void iwl_pcie_ctxt_info_free_paging(struct iwl_trans *trans)
 
 int iwl_pcie_init_fw_sec(struct iwl_trans *trans,
 			 const struct fw_img *fw,
-			 struct iwl_context_info_dram *ctxt_dram)
+			 struct iwl_context_info_dram_nonfseq *ctxt_dram)
 {
 	struct iwl_self_init_dram *dram = &trans->init_dram;
 	int i, ret, lmac_cnt, umac_cnt, paging_cnt;
@@ -161,12 +161,12 @@ int iwl_pcie_init_fw_sec(struct iwl_trans *trans,
 }
 
 int iwl_pcie_ctxt_info_init(struct iwl_trans *trans,
-			    const struct fw_img *fw)
+			    const struct fw_img *img)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_context_info *ctxt_info;
 	struct iwl_context_info_rbd_cfg *rx_cfg;
-	u32 control_flags = 0, rb_size;
+	u32 control_flags = 0, rb_size, cb_size;
 	dma_addr_t phys;
 	int ret;
 
@@ -180,11 +180,11 @@ int iwl_pcie_ctxt_info_init(struct iwl_trans *trans,
 
 	ctxt_info->version.version = 0;
 	ctxt_info->version.mac_id =
-		cpu_to_le16((u16)trans->hw_rev);
+		cpu_to_le16((u16)trans->info.hw_rev);
 	/* size is in DWs */
 	ctxt_info->version.size = cpu_to_le16(sizeof(*ctxt_info) / 4);
 
-	switch (trans_pcie->rx_buf_size) {
+	switch (trans->conf.rx_buf_size) {
 	case IWL_AMSDU_2K:
 		rb_size = IWL_CTXT_INFO_RB_SIZE_2K;
 		break;
@@ -202,11 +202,12 @@ int iwl_pcie_ctxt_info_init(struct iwl_trans *trans,
 		rb_size = IWL_CTXT_INFO_RB_SIZE_4K;
 	}
 
-	WARN_ON(RX_QUEUE_CB_SIZE(trans->cfg->num_rbds) > 12);
+	cb_size = RX_QUEUE_CB_SIZE(iwl_trans_get_num_rbds(trans));
+	if (WARN_ON(cb_size > 12))
+		cb_size = 12;
+
 	control_flags = IWL_CTXT_INFO_TFD_FORMAT_LONG;
-	control_flags |=
-		u32_encode_bits(RX_QUEUE_CB_SIZE(trans->cfg->num_rbds),
-				IWL_CTXT_INFO_RB_CB_SIZE);
+	control_flags |= u32_encode_bits(cb_size, IWL_CTXT_INFO_RB_CB_SIZE);
 	control_flags |= u32_encode_bits(rb_size, IWL_CTXT_INFO_RB_SIZE);
 	ctxt_info->control.control_flags = cpu_to_le32(control_flags);
 
@@ -218,12 +219,12 @@ int iwl_pcie_ctxt_info_init(struct iwl_trans *trans,
 
 	/* initialize TX command queue */
 	ctxt_info->hcmd_cfg.cmd_queue_addr =
-		cpu_to_le64(trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id]->dma_addr);
+		cpu_to_le64(trans_pcie->txqs.txq[trans->conf.cmd_queue]->dma_addr);
 	ctxt_info->hcmd_cfg.cmd_queue_size =
 		TFD_QUEUE_CB_SIZE(IWL_CMD_QUEUE_SIZE);
 
 	/* allocate ucode sections in dram and set addresses */
-	ret = iwl_pcie_init_fw_sec(trans, fw, &ctxt_info->dram);
+	ret = iwl_pcie_init_fw_sec(trans, img, &ctxt_info->dram);
 	if (ret) {
 		dma_free_coherent(trans->dev, sizeof(*trans_pcie->ctxt_info),
 				  ctxt_info, trans_pcie->ctxt_info_dma_addr);
@@ -232,7 +233,7 @@ int iwl_pcie_ctxt_info_init(struct iwl_trans *trans,
 
 	trans_pcie->ctxt_info = ctxt_info;
 
-	iwl_enable_fw_load_int_ctx_info(trans);
+	iwl_enable_fw_load_int_ctx_info(trans, false);
 
 	/* Configure debug, if exists */
 	if (iwl_pcie_dbg_on(trans))
diff --git a/sys/contrib/dev/iwlwifi/pcie/drv.c b/sys/contrib/dev/iwlwifi/pcie/drv.c
index 8d28ce1374b8..4deb57058c9d 100644
--- a/sys/contrib/dev/iwlwifi/pcie/drv.c
+++ b/sys/contrib/dev/iwlwifi/pcie/drv.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -18,17 +18,19 @@
 #include "iwl-trans.h"
 #include "iwl-drv.h"
 #include "iwl-prph.h"
-#include "internal.h"
+#include "gen1_2/internal.h"
+
+#if defined(__FreeBSD__)
+#include <sys/rman.h>
+#endif
 
-#define TRANS_CFG_MARKER BIT(0)
 #define _IS_A(cfg, _struct) __builtin_types_compatible_p(typeof(cfg),	\
 							 struct _struct)
 extern int _invalid_type;
-#define _TRANS_CFG_MARKER(cfg)						\
-	(__builtin_choose_expr(_IS_A(cfg, iwl_cfg_trans_params),	\
-			       TRANS_CFG_MARKER,			\
-	 __builtin_choose_expr(_IS_A(cfg, iwl_cfg), 0, _invalid_type)))
-#define _ASSIGN_CFG(cfg) (_TRANS_CFG_MARKER(cfg) + (kernel_ulong_t)&(cfg))
+#define _TRANS_CFG_CHECK(cfg)						\
+	(__builtin_choose_expr(_IS_A(cfg, iwl_mac_cfg),	\
+			       0, _invalid_type))
+#define _ASSIGN_CFG(cfg) (_TRANS_CFG_CHECK(cfg) + (kernel_ulong_t)&(cfg))
 
 #define IWL_PCI_DEVICE(dev, subdev, cfg) \
 	.vendor = PCI_VENDOR_ID_INTEL,  .device = (dev), \
@@ -38,1268 +40,1062 @@ extern int _invalid_type;
 /* Hardware specific file defines the PCI IDs table for that hardware module */
 VISIBLE_IF_IWLWIFI_KUNIT const struct pci_device_id iwl_hw_card_ids[] = {
 #if IS_ENABLED(CONFIG_IWLDVM)
-	{IWL_PCI_DEVICE(0x4232, 0x1201, iwl5100_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1301, iwl5100_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1204, iwl5100_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1304, iwl5100_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1205, iwl5100_bgn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1305, iwl5100_bgn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1206, iwl5100_abg_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1306, iwl5100_abg_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1221, iwl5100_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1321, iwl5100_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1224, iwl5100_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1324, iwl5100_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1225, iwl5100_bgn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1325, iwl5100_bgn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1226, iwl5100_abg_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4232, 0x1326, iwl5100_abg_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1211, iwl5100_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1311, iwl5100_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1214, iwl5100_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1314, iwl5100_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1215, iwl5100_bgn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1315, iwl5100_bgn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1216, iwl5100_abg_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4237, 0x1316, iwl5100_abg_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1201, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1301, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1204, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1304, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1205, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1305, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1206, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1306, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1221, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1321, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1224, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1324, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1225, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1325, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1226, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4232, 0x1326, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1211, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1311, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1214, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1314, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1215, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1315, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1216, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4237, 0x1316, iwl5000_mac_cfg)}, /* Half Mini Card */
 
 /* 5300 Series WiFi */
-	{IWL_PCI_DEVICE(0x4235, 0x1021, iwl5300_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4235, 0x1121, iwl5300_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4235, 0x1024, iwl5300_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4235, 0x1124, iwl5300_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4235, 0x1001, iwl5300_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4235, 0x1101, iwl5300_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4235, 0x1004, iwl5300_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4235, 0x1104, iwl5300_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4236, 0x1011, iwl5300_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4236, 0x1111, iwl5300_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x4236, 0x1014, iwl5300_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x4236, 0x1114, iwl5300_agn_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1021, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1121, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1024, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1124, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1001, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1101, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1004, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4235, 0x1104, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4236, 0x1011, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4236, 0x1111, iwl5000_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x4236, 0x1014, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x4236, 0x1114, iwl5000_mac_cfg)}, /* Half Mini Card */
 
 /* 5350 Series WiFi/WiMax */
-	{IWL_PCI_DEVICE(0x423A, 0x1001, iwl5350_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x423A, 0x1021, iwl5350_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x423B, 0x1011, iwl5350_agn_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423A, 0x1001, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423A, 0x1021, iwl5000_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423B, 0x1011, iwl5000_mac_cfg)}, /* Mini Card */
 
 /* 5150 Series Wifi/WiMax */
-	{IWL_PCI_DEVICE(0x423C, 0x1201, iwl5150_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x423C, 0x1301, iwl5150_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x423C, 0x1206, iwl5150_abg_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x423C, 0x1306, iwl5150_abg_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x423C, 0x1221, iwl5150_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x423C, 0x1321, iwl5150_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x423C, 0x1326, iwl5150_abg_cfg)}, /* Half Mini Card */
-
-	{IWL_PCI_DEVICE(0x423D, 0x1211, iwl5150_agn_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x423D, 0x1311, iwl5150_agn_cfg)}, /* Half Mini Card */
-	{IWL_PCI_DEVICE(0x423D, 0x1216, iwl5150_abg_cfg)}, /* Mini Card */
-	{IWL_PCI_DEVICE(0x423D, 0x1316, iwl5150_abg_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x423C, 0x1201, iwl5150_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423C, 0x1301, iwl5150_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x423C, 0x1206, iwl5150_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423C, 0x1306, iwl5150_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x423C, 0x1221, iwl5150_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423C, 0x1321, iwl5150_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x423C, 0x1326, iwl5150_mac_cfg)}, /* Half Mini Card */
+
+	{IWL_PCI_DEVICE(0x423D, 0x1211, iwl5150_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423D, 0x1311, iwl5150_mac_cfg)}, /* Half Mini Card */
+	{IWL_PCI_DEVICE(0x423D, 0x1216, iwl5150_mac_cfg)}, /* Mini Card */
+	{IWL_PCI_DEVICE(0x423D, 0x1316, iwl5150_mac_cfg)}, /* Half Mini Card */
 
 /* 6x00 Series */
-	{IWL_PCI_DEVICE(0x422B, 0x1101, iwl6000_3agn_cfg)},
-	{IWL_PCI_DEVICE(0x422B, 0x1108, iwl6000_3agn_cfg)},
-	{IWL_PCI_DEVICE(0x422B, 0x1121, iwl6000_3agn_cfg)},
-	{IWL_PCI_DEVICE(0x422B, 0x1128, iwl6000_3agn_cfg)},
-	{IWL_PCI_DEVICE(0x422C, 0x1301, iwl6000i_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x422C, 0x1306, iwl6000i_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x422C, 0x1307, iwl6000i_2bg_cfg)},
-	{IWL_PCI_DEVICE(0x422C, 0x1321, iwl6000i_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x422C, 0x1326, iwl6000i_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x4238, 0x1111, iwl6000_3agn_cfg)},
-	{IWL_PCI_DEVICE(0x4238, 0x1118, iwl6000_3agn_cfg)},
-	{IWL_PCI_DEVICE(0x4239, 0x1311, iwl6000i_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x4239, 0x1316, iwl6000i_2abg_cfg)},
+	{IWL_PCI_DEVICE(0x422B, 0x1101, iwl6000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422B, 0x1108, iwl6000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422B, 0x1121, iwl6000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422B, 0x1128, iwl6000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422C, 0x1301, iwl6000i_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422C, 0x1306, iwl6000i_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422C, 0x1307, iwl6000i_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422C, 0x1321, iwl6000i_mac_cfg)},
+	{IWL_PCI_DEVICE(0x422C, 0x1326, iwl6000i_mac_cfg)},
+	{IWL_PCI_DEVICE(0x4238, 0x1111, iwl6000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x4238, 0x1118, iwl6000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x4239, 0x1311, iwl6000i_mac_cfg)},
+	{IWL_PCI_DEVICE(0x4239, 0x1316, iwl6000i_mac_cfg)},
 
 /* 6x05 Series */
-	{IWL_PCI_DEVICE(0x0082, 0x1301, iwl6005_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x1306, iwl6005_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x1307, iwl6005_2bg_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x1308, iwl6005_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x1321, iwl6005_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x1326, iwl6005_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x1328, iwl6005_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0085, 0x1311, iwl6005_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0085, 0x1318, iwl6005_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0085, 0x1316, iwl6005_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0xC020, iwl6005_2agn_sff_cfg)},
-	{IWL_PCI_DEVICE(0x0085, 0xC220, iwl6005_2agn_sff_cfg)},
-	{IWL_PCI_DEVICE(0x0085, 0xC228, iwl6005_2agn_sff_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x4820, iwl6005_2agn_d_cfg)},
-	{IWL_PCI_DEVICE(0x0082, 0x1304, iwl6005_2agn_mow1_cfg)},/* low 5GHz active */
-	{IWL_PCI_DEVICE(0x0082, 0x1305, iwl6005_2agn_mow2_cfg)},/* high 5GHz active */
-
-/* 6x30 Series */
-	{IWL_PCI_DEVICE(0x008A, 0x5305, iwl1030_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x008A, 0x5307, iwl1030_bg_cfg)},
-	{IWL_PCI_DEVICE(0x008A, 0x5325, iwl1030_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x008A, 0x5327, iwl1030_bg_cfg)},
-	{IWL_PCI_DEVICE(0x008B, 0x5315, iwl1030_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x008B, 0x5317, iwl1030_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0090, 0x5211, iwl6030_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0090, 0x5215, iwl6030_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0090, 0x5216, iwl6030_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x0091, 0x5201, iwl6030_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0091, 0x5205, iwl6030_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0091, 0x5206, iwl6030_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x0091, 0x5207, iwl6030_2bg_cfg)},
-	{IWL_PCI_DEVICE(0x0091, 0x5221, iwl6030_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0091, 0x5225, iwl6030_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0091, 0x5226, iwl6030_2abg_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1301, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1306, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1307, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1308, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1321, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1326, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1328, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0085, 0x1311, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0085, 0x1318, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0085, 0x1316, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0xC020, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0085, 0xC220, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0085, 0xC228, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x4820, iwl6005_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0082, 0x1304, iwl6005_mac_cfg)},/* low 5GHz active */
+	{IWL_PCI_DEVICE(0x0082, 0x1305, iwl6005_mac_cfg)},/* high 5GHz active */
+
+/* 1030/6x30 Series */
+	{IWL_PCI_DEVICE(0x008A, 0x5305, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x008A, 0x5307, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x008A, 0x5325, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x008A, 0x5327, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x008B, 0x5315, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x008B, 0x5317, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0090, 0x5211, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0090, 0x5215, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0090, 0x5216, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0091, 0x5201, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0091, 0x5205, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0091, 0x5206, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0091, 0x5207, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0091, 0x5221, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0091, 0x5225, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0091, 0x5226, iwl6030_mac_cfg)},
 
 /* 6x50 WiFi/WiMax Series */
-	{IWL_PCI_DEVICE(0x0087, 0x1301, iwl6050_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0087, 0x1306, iwl6050_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x0087, 0x1321, iwl6050_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0087, 0x1326, iwl6050_2abg_cfg)},
-	{IWL_PCI_DEVICE(0x0089, 0x1311, iwl6050_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x0089, 0x1316, iwl6050_2abg_cfg)},
+	{IWL_PCI_DEVICE(0x0087, 0x1301, iwl6050_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0087, 0x1306, iwl6050_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0087, 0x1321, iwl6050_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0087, 0x1326, iwl6050_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0089, 0x1311, iwl6050_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0089, 0x1316, iwl6050_mac_cfg)},
 
 /* 6150 WiFi/WiMax Series */
-	{IWL_PCI_DEVICE(0x0885, 0x1305, iwl6150_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0885, 0x1307, iwl6150_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0885, 0x1325, iwl6150_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0885, 0x1327, iwl6150_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0886, 0x1315, iwl6150_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0886, 0x1317, iwl6150_bg_cfg)},
+	{IWL_PCI_DEVICE(0x0885, 0x1305, iwl6150_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0885, 0x1307, iwl6150_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0885, 0x1325, iwl6150_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0885, 0x1327, iwl6150_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0886, 0x1315, iwl6150_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0886, 0x1317, iwl6150_mac_cfg)},
 
 /* 1000 Series WiFi */
-	{IWL_PCI_DEVICE(0x0083, 0x1205, iwl1000_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0083, 0x1305, iwl1000_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0083, 0x1225, iwl1000_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0083, 0x1325, iwl1000_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0084, 0x1215, iwl1000_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0084, 0x1315, iwl1000_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0083, 0x1206, iwl1000_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0083, 0x1306, iwl1000_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0083, 0x1226, iwl1000_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0083, 0x1326, iwl1000_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0084, 0x1216, iwl1000_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0084, 0x1316, iwl1000_bg_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1205, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1305, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1225, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1325, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0084, 0x1215, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0084, 0x1315, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1206, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1306, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1226, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0083, 0x1326, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0084, 0x1216, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0084, 0x1316, iwl1000_mac_cfg)},
 
 /* 100 Series WiFi */
-	{IWL_PCI_DEVICE(0x08AE, 0x1005, iwl100_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x08AE, 0x1007, iwl100_bg_cfg)},
-	{IWL_PCI_DEVICE(0x08AF, 0x1015, iwl100_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x08AF, 0x1017, iwl100_bg_cfg)},
-	{IWL_PCI_DEVICE(0x08AE, 0x1025, iwl100_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x08AE, 0x1027, iwl100_bg_cfg)},
+	{IWL_PCI_DEVICE(0x08AE, 0x1005, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08AE, 0x1007, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08AF, 0x1015, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08AF, 0x1017, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08AE, 0x1025, iwl1000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08AE, 0x1027, iwl1000_mac_cfg)},
 
 /* 130 Series WiFi */
-	{IWL_PCI_DEVICE(0x0896, 0x5005, iwl130_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0896, 0x5007, iwl130_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0897, 0x5015, iwl130_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0897, 0x5017, iwl130_bg_cfg)},
-	{IWL_PCI_DEVICE(0x0896, 0x5025, iwl130_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0896, 0x5027, iwl130_bg_cfg)},
+	{IWL_PCI_DEVICE(0x0896, 0x5005, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0896, 0x5007, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0897, 0x5015, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0897, 0x5017, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0896, 0x5025, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0896, 0x5027, iwl6030_mac_cfg)},
 
 /* 2x00 Series */
-	{IWL_PCI_DEVICE(0x0890, 0x4022, iwl2000_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0891, 0x4222, iwl2000_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0890, 0x4422, iwl2000_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0890, 0x4822, iwl2000_2bgn_d_cfg)},
+	{IWL_PCI_DEVICE(0x0890, 0x4022, iwl2000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0891, 0x4222, iwl2000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0890, 0x4422, iwl2000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0890, 0x4822, iwl2000_mac_cfg)},
 
 /* 2x30 Series */
-	{IWL_PCI_DEVICE(0x0887, 0x4062, iwl2030_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0888, 0x4262, iwl2030_2bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0887, 0x4462, iwl2030_2bgn_cfg)},
+	{IWL_PCI_DEVICE(0x0887, 0x4062, iwl2030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0888, 0x4262, iwl2030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0887, 0x4462, iwl2030_mac_cfg)},
 
 /* 6x35 Series */
-	{IWL_PCI_DEVICE(0x088E, 0x4060, iwl6035_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x088E, 0x406A, iwl6035_2agn_sff_cfg)},
-	{IWL_PCI_DEVICE(0x088F, 0x4260, iwl6035_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x088F, 0x426A, iwl6035_2agn_sff_cfg)},
-	{IWL_PCI_DEVICE(0x088E, 0x4460, iwl6035_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x088E, 0x446A, iwl6035_2agn_sff_cfg)},
-	{IWL_PCI_DEVICE(0x088E, 0x4860, iwl6035_2agn_cfg)},
-	{IWL_PCI_DEVICE(0x088F, 0x5260, iwl6035_2agn_cfg)},
+	{IWL_PCI_DEVICE(0x088E, 0x4060, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x088E, 0x406A, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x088F, 0x4260, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x088F, 0x426A, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x088E, 0x4460, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x088E, 0x446A, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x088E, 0x4860, iwl6030_mac_cfg)},
+	{IWL_PCI_DEVICE(0x088F, 0x5260, iwl6030_mac_cfg)},
 
 /* 105 Series */
-	{IWL_PCI_DEVICE(0x0894, 0x0022, iwl105_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0895, 0x0222, iwl105_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0894, 0x0422, iwl105_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0894, 0x0822, iwl105_bgn_d_cfg)},
+	{IWL_PCI_DEVICE(0x0894, 0x0022, iwl105_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0895, 0x0222, iwl105_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0894, 0x0422, iwl105_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0894, 0x0822, iwl105_mac_cfg)},
 
 /* 135 Series */
-	{IWL_PCI_DEVICE(0x0892, 0x0062, iwl135_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0893, 0x0262, iwl135_bgn_cfg)},
-	{IWL_PCI_DEVICE(0x0892, 0x0462, iwl135_bgn_cfg)},
+	{IWL_PCI_DEVICE(0x0892, 0x0062, iwl135_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0893, 0x0262, iwl135_mac_cfg)},
+	{IWL_PCI_DEVICE(0x0892, 0x0462, iwl135_mac_cfg)},
 #endif /* CONFIG_IWLDVM */
 
 #if IS_ENABLED(CONFIG_IWLMVM)
 /* 7260 Series */
-	{IWL_PCI_DEVICE(0x08B1, 0x4070, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4072, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4170, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4C60, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4C70, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4060, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x406A, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4160, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4062, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4162, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x4270, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x4272, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x4260, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x426A, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x4262, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4470, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4472, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4460, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x446A, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4462, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4870, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x486E, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4A70, iwl7260_2ac_cfg_high_temp)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4A6E, iwl7260_2ac_cfg_high_temp)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4A6C, iwl7260_2ac_cfg_high_temp)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4570, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4560, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x4370, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x4360, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x5070, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x5072, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x5170, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x5770, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4020, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x402A, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0x4220, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0x4420, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC070, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC072, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC170, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC060, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC06A, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC160, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC062, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC162, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC770, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC760, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0xC270, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xCC70, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xCC60, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0xC272, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0xC260, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0xC26A, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0xC262, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC470, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC472, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC460, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC462, iwl7260_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC570, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC560, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0xC370, iwl7260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC360, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC020, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC02A, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B2, 0xC220, iwl7260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B1, 0xC420, iwl7260_2n_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4070, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4072, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4170, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4C60, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4C70, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4060, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x406A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4160, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4062, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4162, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x4270, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x4272, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x4260, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x426A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x4262, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4470, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4472, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4460, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x446A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4462, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4870, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x486E, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4A70, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4A6E, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4A6C, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4570, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4560, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x4370, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x4360, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x5070, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x5072, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x5170, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x5770, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4020, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x402A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0x4220, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0x4420, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC070, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC072, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC170, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC060, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC06A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC160, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC062, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC162, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC770, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC760, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0xC270, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xCC70, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xCC60, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0xC272, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0xC260, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0xC26A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0xC262, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC470, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC472, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC460, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC462, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC570, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC560, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0xC370, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC360, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC020, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC02A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B2, 0xC220, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B1, 0xC420, iwl7000_mac_cfg)},
 
 /* 3160 Series */
-	{IWL_PCI_DEVICE(0x08B3, 0x0070, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x0072, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x0170, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x0172, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x0060, iwl3160_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x0062, iwl3160_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B4, 0x0270, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B4, 0x0272, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x0470, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x0472, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B4, 0x0370, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8070, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8072, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8170, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8172, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8060, iwl3160_2n_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8062, iwl3160_n_cfg)},
-	{IWL_PCI_DEVICE(0x08B4, 0x8270, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B4, 0x8370, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B4, 0x8272, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8470, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x8570, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x1070, iwl3160_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x08B3, 0x1170, iwl3160_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0070, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0072, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0170, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0172, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0060, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0062, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B4, 0x0270, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B4, 0x0272, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0470, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x0472, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B4, 0x0370, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8070, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8072, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8170, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8172, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8060, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8062, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B4, 0x8270, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B4, 0x8370, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B4, 0x8272, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8470, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x8570, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x1070, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x08B3, 0x1170, iwl7000_mac_cfg)},
 
 /* 3165 Series */
-	{IWL_PCI_DEVICE(0x3165, 0x4010, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3165, 0x4012, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3166, 0x4212, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3165, 0x4410, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3165, 0x4510, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3165, 0x4110, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3166, 0x4310, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3166, 0x4210, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3165, 0x8010, iwl3165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x3165, 0x8110, iwl3165_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x3165, 0x4010, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3165, 0x4012, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3166, 0x4212, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3165, 0x4410, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3165, 0x4510, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3165, 0x4110, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3166, 0x4310, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3166, 0x4210, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3165, 0x8010, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3165, 0x8110, iwl7000_mac_cfg)},
 
 /* 3168 Series */
-	{IWL_PCI_DEVICE(0x24FB, 0x2010, iwl3168_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FB, 0x2110, iwl3168_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FB, 0x2050, iwl3168_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FB, 0x2150, iwl3168_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FB, 0x0000, iwl3168_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x24FB, 0x2010, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FB, 0x2110, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FB, 0x2050, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FB, 0x2150, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FB, 0x0000, iwl7000_mac_cfg)},
 
 /* 7265 Series */
-	{IWL_PCI_DEVICE(0x095A, 0x5010, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5110, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5100, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x5310, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x5302, iwl7265_n_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x5210, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5C10, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5012, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5412, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5410, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5510, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5400, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x1010, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5000, iwl7265_2n_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x500A, iwl7265_2n_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x5200, iwl7265_2n_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5002, iwl7265_n_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5102, iwl7265_n_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x5202, iwl7265_n_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9010, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9012, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x900A, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9110, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9112, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x9210, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x9200, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9510, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x9310, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9410, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5020, iwl7265_2n_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x502A, iwl7265_2n_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5420, iwl7265_2n_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5090, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5190, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5590, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x5290, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5490, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x5F10, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x5212, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095B, 0x520A, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9000, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9400, iwl7265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x095A, 0x9E10, iwl7265_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5010, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5110, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5100, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x5310, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x5302, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x5210, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5C10, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5012, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5412, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5410, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5510, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5400, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x1010, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5000, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x500A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x5200, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5002, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5102, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x5202, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9010, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9012, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x900A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9110, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9112, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x9210, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x9200, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9510, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x9310, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9410, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5020, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x502A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5420, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5090, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5190, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5590, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x5290, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5490, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x5F10, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x5212, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095B, 0x520A, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9000, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9400, iwl7000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9E10, iwl7000_mac_cfg)},
 
 /* 8000 Series */
-	{IWL_PCI_DEVICE(0x24F3, 0x0010, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x1010, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x10B0, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0130, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x1130, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0132, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x1132, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0110, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x01F0, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0012, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x1012, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x1110, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0050, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0250, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x1050, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0150, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x1150, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F4, 0x0030, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F4, 0x1030, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0xC010, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0xC110, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0xD010, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0xC050, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0xD050, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0xD0B0, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0xB0B0, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x8010, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x8110, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x9010, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x9110, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F4, 0x8030, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F4, 0x9030, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F4, 0xC030, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F4, 0xD030, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x8130, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x9130, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x8132, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x9132, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x8050, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x8150, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x9050, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x9150, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0004, iwl8260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0044, iwl8260_2n_cfg)},
-	{IWL_PCI_DEVICE(0x24F5, 0x0010, iwl4165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F6, 0x0030, iwl4165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0810, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0910, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0850, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0950, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0930, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x0000, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24F3, 0x4010, iwl8260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0010, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0110, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x1110, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x1130, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0130, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x1010, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x10D0, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0050, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0150, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x9010, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x8110, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x8050, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x8010, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0810, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x9110, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x8130, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0910, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0930, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0950, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0850, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x1014, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x3E02, iwl8275_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x3E01, iwl8275_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x1012, iwl8275_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0012, iwl8275_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x0014, iwl8265_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x24FD, 0x9074, iwl8265_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x1010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x10B0, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0130, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x1130, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0132, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x1132, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x01F0, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0012, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x1012, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x1110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0250, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x1050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0150, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x1150, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F4, 0x0030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F4, 0x1030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xC010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xC110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xD010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xC050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xD050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xD0B0, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xB0B0, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x8010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x8110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x9010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x9110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F4, 0x8030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F4, 0x9030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F4, 0xC030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F4, 0xD030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x8130, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x9130, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x8132, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x9132, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x8050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x8150, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x9050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x9150, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0004, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0044, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F5, 0x0010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F6, 0x0030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0810, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0910, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0850, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0950, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0930, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x0000, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0x4010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xC030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24F3, 0xD030, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x1110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x1130, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0130, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x1010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x10D0, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0150, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x9010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x8110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x8050, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x8010, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0810, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x9110, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x8130, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0910, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0930, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0950, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0850, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x1014, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x3E02, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x3E01, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x1012, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0012, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0014, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x9074, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x1431, iwl8000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x1432, iwl8000_mac_cfg)},
 
 /* 9000 Series */
-	{IWL_PCI_DEVICE(0x2526, PCI_ANY_ID, iwl9000_trans_cfg)},
-	{IWL_PCI_DEVICE(0x271B, PCI_ANY_ID, iwl9000_trans_cfg)},
-	{IWL_PCI_DEVICE(0x271C, PCI_ANY_ID, iwl9000_trans_cfg)},
-	{IWL_PCI_DEVICE(0x30DC, PCI_ANY_ID, iwl9560_long_latency_trans_cfg)},
-	{IWL_PCI_DEVICE(0x31DC, PCI_ANY_ID, iwl9560_shared_clk_trans_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, PCI_ANY_ID, iwl9560_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA370, PCI_ANY_ID, iwl9560_trans_cfg)},
+	{IWL_PCI_DEVICE(0x2526, PCI_ANY_ID, iwl9000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x271B, PCI_ANY_ID, iwl9000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x271C, PCI_ANY_ID, iwl9000_mac_cfg)},
+	{IWL_PCI_DEVICE(0x30DC, PCI_ANY_ID, iwl9560_long_latency_mac_cfg)},
+	{IWL_PCI_DEVICE(0x31DC, PCI_ANY_ID, iwl9560_shared_clk_mac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, PCI_ANY_ID, iwl9560_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA370, PCI_ANY_ID, iwl9560_mac_cfg)},
 
 /* Qu devices */
-	{IWL_PCI_DEVICE(0x02F0, PCI_ANY_ID, iwl_qu_trans_cfg)},
-	{IWL_PCI_DEVICE(0x06F0, PCI_ANY_ID, iwl_qu_trans_cfg)},
+	{IWL_PCI_DEVICE(0x02F0, PCI_ANY_ID, iwl_qu_mac_cfg)},
+	{IWL_PCI_DEVICE(0x06F0, PCI_ANY_ID, iwl_qu_mac_cfg)},
 
-	{IWL_PCI_DEVICE(0x34F0, PCI_ANY_ID, iwl_qu_medium_latency_trans_cfg)},
-	{IWL_PCI_DEVICE(0x3DF0, PCI_ANY_ID, iwl_qu_medium_latency_trans_cfg)},
-	{IWL_PCI_DEVICE(0x4DF0, PCI_ANY_ID, iwl_qu_medium_latency_trans_cfg)},
+	{IWL_PCI_DEVICE(0x34F0, PCI_ANY_ID, iwl_qu_medium_latency_mac_cfg)},
+	{IWL_PCI_DEVICE(0x3DF0, PCI_ANY_ID, iwl_qu_medium_latency_mac_cfg)},
+	{IWL_PCI_DEVICE(0x4DF0, PCI_ANY_ID, iwl_qu_medium_latency_mac_cfg)},
 
-	{IWL_PCI_DEVICE(0x43F0, PCI_ANY_ID, iwl_qu_long_latency_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA0F0, PCI_ANY_ID, iwl_qu_long_latency_trans_cfg)},
+	{IWL_PCI_DEVICE(0x43F0, PCI_ANY_ID, iwl_qu_long_latency_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA0F0, PCI_ANY_ID, iwl_qu_long_latency_mac_cfg)},
 
-	{IWL_PCI_DEVICE(0x2723, PCI_ANY_ID, iwl_ax200_trans_cfg)},
+	{IWL_PCI_DEVICE(0x2723, PCI_ANY_ID, iwl_ax200_mac_cfg)},
 
-/* So devices */
-	{IWL_PCI_DEVICE(0x2725, PCI_ANY_ID, iwl_so_trans_cfg)},
-	{IWL_PCI_DEVICE(0x7A70, PCI_ANY_ID, iwl_so_long_latency_imr_trans_cfg)},
-	{IWL_PCI_DEVICE(0x7AF0, PCI_ANY_ID, iwl_so_trans_cfg)},
-	{IWL_PCI_DEVICE(0x51F0, PCI_ANY_ID, iwl_so_long_latency_trans_cfg)},
-	{IWL_PCI_DEVICE(0x51F1, PCI_ANY_ID, iwl_so_long_latency_imr_trans_cfg)},
-	{IWL_PCI_DEVICE(0x54F0, PCI_ANY_ID, iwl_so_long_latency_trans_cfg)},
-	{IWL_PCI_DEVICE(0x7F70, PCI_ANY_ID, iwl_so_trans_cfg)},
+/* Ty/So devices */
+	{IWL_PCI_DEVICE(0x2725, PCI_ANY_ID, iwl_ty_mac_cfg)},
+	{IWL_PCI_DEVICE(0x7A70, PCI_ANY_ID, iwl_so_long_latency_imr_mac_cfg)},
+	{IWL_PCI_DEVICE(0x7AF0, PCI_ANY_ID, iwl_so_mac_cfg)},
+	{IWL_PCI_DEVICE(0x51F0, PCI_ANY_ID, iwl_so_long_latency_mac_cfg)},
+	{IWL_PCI_DEVICE(0x51F1, PCI_ANY_ID, iwl_so_long_latency_imr_mac_cfg)},
+	{IWL_PCI_DEVICE(0x54F0, PCI_ANY_ID, iwl_so_long_latency_mac_cfg)},
+	{IWL_PCI_DEVICE(0x7F70, PCI_ANY_ID, iwl_so_mac_cfg)},
 
 /* Ma devices */
-	{IWL_PCI_DEVICE(0x2729, PCI_ANY_ID, iwl_ma_trans_cfg)},
-	{IWL_PCI_DEVICE(0x7E40, PCI_ANY_ID, iwl_ma_trans_cfg)},
-
+	{IWL_PCI_DEVICE(0x2729, PCI_ANY_ID, iwl_ma_mac_cfg)},
+	{IWL_PCI_DEVICE(0x7E40, PCI_ANY_ID, iwl_ma_mac_cfg)},
+#endif /* CONFIG_IWLMVM */
+#if IS_ENABLED(CONFIG_IWLMVM) || IS_ENABLED(CONFIG_IWLMLD)
 /* Bz devices */
-	{IWL_PCI_DEVICE(0x2727, PCI_ANY_ID, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0x272D, PCI_ANY_ID, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0x272b, PCI_ANY_ID, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0000, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0090, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0094, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0098, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x009C, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x00C0, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x00C4, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x00E0, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x00E4, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x00E8, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x00EC, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0100, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0110, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0114, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0118, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x011C, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0310, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0314, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0510, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x0A10, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x1671, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x1672, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x1771, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x1772, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x1791, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x1792, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x4090, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x40C4, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x40E0, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x4110, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0xA840, 0x4314, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0x7740, PCI_ANY_ID, iwl_bz_trans_cfg)},
-	{IWL_PCI_DEVICE(0x4D40, PCI_ANY_ID, iwl_bz_trans_cfg)},
+	{IWL_PCI_DEVICE(0x272b, PCI_ANY_ID, iwl_gl_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0000, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0090, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0094, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0098, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x009C, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x00C0, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x00C4, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x00E0, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x00E4, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x00E8, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x00EC, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0100, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0110, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0114, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0118, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x011C, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0310, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0314, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0510, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x0A10, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1671, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1672, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1771, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1772, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1791, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1792, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x4090, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x40C4, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x40E0, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x4110, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x4314, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1775, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1776, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0x7740, PCI_ANY_ID, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0x4D40, PCI_ANY_ID, iwl_bz_mac_cfg)},
 
 /* Sc devices */
-	{IWL_PCI_DEVICE(0xE440, PCI_ANY_ID, iwl_sc_trans_cfg)},
-	{IWL_PCI_DEVICE(0xE340, PCI_ANY_ID, iwl_sc_trans_cfg)},
-	{IWL_PCI_DEVICE(0xD340, PCI_ANY_ID, iwl_sc_trans_cfg)},
-	{IWL_PCI_DEVICE(0x6E70, PCI_ANY_ID, iwl_sc_trans_cfg)},
-#endif /* CONFIG_IWLMVM */
+	{IWL_PCI_DEVICE(0xE440, PCI_ANY_ID, iwl_sc_mac_cfg)},
+	{IWL_PCI_DEVICE(0xE340, PCI_ANY_ID, iwl_sc_mac_cfg)},
+	{IWL_PCI_DEVICE(0xD340, PCI_ANY_ID, iwl_sc_mac_cfg)},
+	{IWL_PCI_DEVICE(0x6E70, PCI_ANY_ID, iwl_sc_mac_cfg)},
+	{IWL_PCI_DEVICE(0xD240, PCI_ANY_ID, iwl_sc_mac_cfg)},
+#endif /* CONFIG_IWLMVM || CONFIG_IWLMLD */
 
 	{0}
 };
 MODULE_DEVICE_TABLE(pci, iwl_hw_card_ids);
 EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_hw_card_ids);
 
-#define _IWL_DEV_INFO(_device, _subdevice, _mac_type, _mac_step, _rf_type, \
-		      _rf_id, _rf_step, _no_160, _cores, _cdb, _cfg, _name) \
-	{ .device = (_device), .subdevice = (_subdevice), .cfg = &(_cfg), \
-	  .name = _name, .mac_type = _mac_type, .rf_type = _rf_type, .rf_step = _rf_step, \
-	  .no_160 = _no_160, .cores = _cores, .rf_id = _rf_id, \
-	  .mac_step = _mac_step, .cdb = _cdb, .jacket = IWL_CFG_ANY }
-
-#define IWL_DEV_INFO(_device, _subdevice, _cfg, _name) \
-	_IWL_DEV_INFO(_device, _subdevice, IWL_CFG_ANY, IWL_CFG_ANY,   \
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,  \
-		      IWL_CFG_ANY, _cfg, _name)
+#define _IWL_DEV_INFO(_cfg, _name, ...) {	\
+	.cfg = &_cfg,				\
+	.name = _name,				\
+	.device = IWL_CFG_ANY,			\
+	.subdevice = IWL_CFG_ANY,		\
+	.subdevice_m_h = 15,			\
+	__VA_ARGS__				\
+}
+#define IWL_DEV_INFO(_cfg, _name, ...)		\
+	_IWL_DEV_INFO(_cfg, _name, __VA_ARGS__)
+
+#define DEVICE(n)		.device = (n)
+#define SUBDEV(n)		.subdevice = (n)
+#define _LOWEST_BIT(n)		(__builtin_ffs(n) - 1)
+#define _BIT_ABOVE_MASK(n)	((n) + (1 << _LOWEST_BIT(n)))
+#define _HIGHEST_BIT(n)		(__builtin_ffs(_BIT_ABOVE_MASK(n)) - 2)
+#define _IS_POW2(n)		(((n) & ((n) - 1)) == 0)
+#define _IS_CONTIG(n)		_IS_POW2(_BIT_ABOVE_MASK(n))
+#define _CHECK_MASK(m)		BUILD_BUG_ON_ZERO(!_IS_CONTIG(m))
+#define SUBDEV_MASKED(v, m)	.subdevice = (v) + _CHECK_MASK(m),	\
+				.subdevice_m_l = _LOWEST_BIT(m),	\
+				.subdevice_m_h = _HIGHEST_BIT(m)
+#define RF_TYPE(n)		.match_rf_type = 1,			\
+				.rf_type = IWL_CFG_RF_TYPE_##n
+#define DISCRETE		.match_discrete = 1,			\
+				.discrete = 1
+#define INTEGRATED		.match_discrete = 1,			\
+				.discrete = 0
+#define RF_ID(n)		.match_rf_id = 1,			\
+				.rf_id = IWL_CFG_RF_ID_##n
+#define NO_CDB			.match_cdb = 1, .cdb = 0
+#define CDB			.match_cdb = 1, .cdb = 1
+#define BW_NOT_LIMITED		.match_bw_limit = 1, .bw_limit = 0
+#define BW_LIMITED		.match_bw_limit = 1, .bw_limit = 1
 
 VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info iwl_dev_info_table[] = {
-#if IS_ENABLED(CONFIG_IWLMVM)
-/* 9000 */
-	IWL_DEV_INFO(0x2526, 0x1550, iwl9260_2ac_cfg, iwl9260_killer_1550_name),
-	IWL_DEV_INFO(0x2526, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_name),
-	IWL_DEV_INFO(0x2526, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_name),
-	IWL_DEV_INFO(0x30DC, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_name),
-	IWL_DEV_INFO(0x30DC, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_name),
-	IWL_DEV_INFO(0x31DC, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_name),
-	IWL_DEV_INFO(0x31DC, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_name),
-	IWL_DEV_INFO(0xA370, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_name),
-	IWL_DEV_INFO(0xA370, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_name),
-	IWL_DEV_INFO(0x54F0, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_160_name),
-	IWL_DEV_INFO(0x54F0, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_name),
-	IWL_DEV_INFO(0x51F0, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_160_name),
-	IWL_DEV_INFO(0x51F0, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_160_name),
-	IWL_DEV_INFO(0x51F0, 0x1691, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690s_name),
-	IWL_DEV_INFO(0x51F0, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name),
-	IWL_DEV_INFO(0x51F1, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name),
-	IWL_DEV_INFO(0x54F0, 0x1691, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690s_name),
-	IWL_DEV_INFO(0x54F0, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name),
-	IWL_DEV_INFO(0x7A70, 0x1691, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690s_name),
-	IWL_DEV_INFO(0x7A70, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name),
-	IWL_DEV_INFO(0x7AF0, 0x1691, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690s_name),
-	IWL_DEV_INFO(0x7AF0, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name),
-
-	IWL_DEV_INFO(0x271C, 0x0214, iwl9260_2ac_cfg, iwl9260_1_name),
-	IWL_DEV_INFO(0x7E40, 0x1691, iwl_cfg_ma, iwl_ax411_killer_1690s_name),
-	IWL_DEV_INFO(0x7E40, 0x1692, iwl_cfg_ma, iwl_ax411_killer_1690i_name),
-
-/* AX200 */
-	IWL_DEV_INFO(0x2723, IWL_CFG_ANY, iwl_ax200_cfg_cc, iwl_ax200_name),
-	IWL_DEV_INFO(0x2723, 0x1653, iwl_ax200_cfg_cc, iwl_ax200_killer_1650w_name),
-	IWL_DEV_INFO(0x2723, 0x1654, iwl_ax200_cfg_cc, iwl_ax200_killer_1650x_name),
-
-	/* Qu with Hr */
-	IWL_DEV_INFO(0x43F0, 0x0070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x43F0, 0x0074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x43F0, 0x0078, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x43F0, 0x007C, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x43F0, 0x1651, killer1650s_2ax_cfg_qu_b0_hr_b0, iwl_ax201_killer_1650s_name),
-	IWL_DEV_INFO(0x43F0, 0x1652, killer1650i_2ax_cfg_qu_b0_hr_b0, iwl_ax201_killer_1650i_name),
-	IWL_DEV_INFO(0x43F0, 0x2074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x43F0, 0x4070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x0070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x0074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x0078, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x007C, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x0A10, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x1651, killer1650s_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x1652, killer1650i_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x2074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x4070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0xA0F0, 0x6074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x0070, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x0074, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x6074, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x0078, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x007C, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x0310, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x1651, iwl_ax1650s_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x1652, iwl_ax1650i_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x2074, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x02F0, 0x4070, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x0070, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x0074, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x0078, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x007C, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x0310, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x1651, iwl_ax1650s_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x1652, iwl_ax1650i_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x2074, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x06F0, 0x4070, iwl_ax201_cfg_quz_hr, NULL),
-	IWL_DEV_INFO(0x34F0, 0x0070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x34F0, 0x0074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x34F0, 0x0078, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x34F0, 0x007C, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x34F0, 0x0310, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x34F0, 0x1651, killer1650s_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0x34F0, 0x1652, killer1650i_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0x34F0, 0x2074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x34F0, 0x4070, iwl_ax201_cfg_qu_hr, NULL),
-
-	IWL_DEV_INFO(0x3DF0, 0x0070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x0074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x0078, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x007C, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x0310, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x1651, killer1650s_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x1652, killer1650i_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x2074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x3DF0, 0x4070, iwl_ax201_cfg_qu_hr, NULL),
-
-	IWL_DEV_INFO(0x4DF0, 0x0070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x0074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x0078, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x007C, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x0310, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x1651, killer1650s_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x1652, killer1650i_2ax_cfg_qu_b0_hr_b0, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x2074, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x4070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x4DF0, 0x6074, iwl_ax201_cfg_qu_hr, NULL),
-
-	/* So with HR */
-	IWL_DEV_INFO(0x2725, 0x0090, iwlax211_2ax_cfg_so_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x0020, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x2020, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x0024, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x0310, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x0510, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x0A10, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0xE020, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0xE024, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x4020, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x6020, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x6024, iwlax210_2ax_cfg_ty_gf_a0, NULL),
-	IWL_DEV_INFO(0x2725, 0x1673, iwlax210_2ax_cfg_ty_gf_a0, iwl_ax210_killer_1675w_name),
-	IWL_DEV_INFO(0x2725, 0x1674, iwlax210_2ax_cfg_ty_gf_a0, iwl_ax210_killer_1675x_name),
-	IWL_DEV_INFO(0x7A70, 0x0090, iwlax211_2ax_cfg_so_gf_a0_long, NULL),
-	IWL_DEV_INFO(0x7A70, 0x0098, iwlax211_2ax_cfg_so_gf_a0_long, NULL),
-	IWL_DEV_INFO(0x7A70, 0x00B0, iwlax411_2ax_cfg_so_gf4_a0_long, NULL),
-	IWL_DEV_INFO(0x7A70, 0x0310, iwlax211_2ax_cfg_so_gf_a0_long, NULL),
-	IWL_DEV_INFO(0x7A70, 0x0510, iwlax211_2ax_cfg_so_gf_a0_long, NULL),
-	IWL_DEV_INFO(0x7A70, 0x0A10, iwlax211_2ax_cfg_so_gf_a0_long, NULL),
-	IWL_DEV_INFO(0x7AF0, 0x0090, iwlax211_2ax_cfg_so_gf_a0, NULL),
-	IWL_DEV_INFO(0x7AF0, 0x0098, iwlax211_2ax_cfg_so_gf_a0, NULL),
-	IWL_DEV_INFO(0x7AF0, 0x00B0, iwlax411_2ax_cfg_so_gf4_a0, NULL),
-	IWL_DEV_INFO(0x7AF0, 0x0310, iwlax211_2ax_cfg_so_gf_a0, NULL),
-	IWL_DEV_INFO(0x7AF0, 0x0510, iwlax211_2ax_cfg_so_gf_a0, NULL),
-	IWL_DEV_INFO(0x7AF0, 0x0A10, iwlax211_2ax_cfg_so_gf_a0, NULL),
-
-	/* So with JF */
-	IWL_DEV_INFO(0x7A70, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_160_name),
-	IWL_DEV_INFO(0x7A70, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_160_name),
-	IWL_DEV_INFO(0x7AF0, 0x1551, iwl9560_2ac_cfg_soc, iwl9560_killer_1550s_160_name),
-	IWL_DEV_INFO(0x7AF0, 0x1552, iwl9560_2ac_cfg_soc, iwl9560_killer_1550i_160_name),
-
-	/* SO with GF2 */
-	IWL_DEV_INFO(0x2726, 0x1671, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x2726, 0x1672, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675i_name),
-	IWL_DEV_INFO(0x51F0, 0x1671, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x51F0, 0x1672, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675i_name),
-	IWL_DEV_INFO(0x51F1, 0x1671, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x51F1, 0x1672, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675i_name),
-	IWL_DEV_INFO(0x54F0, 0x1671, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x54F0, 0x1672, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675i_name),
-	IWL_DEV_INFO(0x7A70, 0x1671, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x7A70, 0x1672, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675i_name),
-	IWL_DEV_INFO(0x7AF0, 0x1671, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x7AF0, 0x1672, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675i_name),
-	IWL_DEV_INFO(0x7F70, 0x1671, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x7F70, 0x1672, iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_killer_1675i_name),
-
-	/* MA with GF2 */
-	IWL_DEV_INFO(0x7E40, 0x1671, iwl_cfg_ma, iwl_ax211_killer_1675s_name),
-	IWL_DEV_INFO(0x7E40, 0x1672, iwl_cfg_ma, iwl_ax211_killer_1675i_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_2ac_cfg_soc, iwl9461_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_2ac_cfg_soc, iwl9461_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_2ac_cfg_soc, iwl9462_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_2ac_cfg_soc, iwl9462_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_2ac_cfg_soc, iwl9560_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_2ac_cfg_soc, iwl9560_name),
-
-	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT_GNSS, IWL_CFG_NO_CDB,
-		      iwl9260_2ac_cfg, iwl9270_160_name),
-	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT_GNSS, IWL_CFG_NO_CDB,
-		      iwl9260_2ac_cfg, iwl9270_name),
-
-	_IWL_DEV_INFO(0x271B, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH1, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9260_2ac_cfg, iwl9162_160_name),
-	_IWL_DEV_INFO(0x271B, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH1, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9260_2ac_cfg, iwl9162_name),
-
-	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9260_2ac_cfg, iwl9260_160_name),
-	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9260_2ac_cfg, iwl9260_name),
-
-/* Qu with Jf */
-	/* Qu B step */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9461_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9461_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9462_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9462_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1551,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_killer_1550s_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1552,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_killer_1550i_name),
-
-	/* Qu C step */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9461_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9461_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9462_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9462_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1551,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_killer_1550s_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1552,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_killer_1550i_name),
-
-	/* QuZ */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9461_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9461_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9462_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9462_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1551,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_killer_1550s_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1552,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_killer_1550i_name),
-
-/* Qu with Hr */
-	/* Qu B step */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_qu_b0_hr1_b0, iwl_ax101_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_qu_b0_hr_b0, iwl_ax203_name),
-
-	/* Qu C step */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_qu_c0_hr1_b0, iwl_ax101_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_qu_c0_hr_b0, iwl_ax203_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_qu_c0_hr_b0, iwl_ax201_name),
-
-	/* QuZ */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_quz_a0_hr1_b0, iwl_ax101_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_quz_a0_hr_b0, iwl_ax203_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_QUZ, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_quz_a0_hr_b0, iwl_ax201_name),
-
-/* Ma */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_ma, iwl_ax201_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      iwl_cfg_ma, iwl_ax211_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_ma, iwl_ax231_name),
-
-/* So with Hr */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_so_a0_hr_a0, iwl_ax203_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_so_a0_hr_a0, iwl_ax101_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_so_a0_hr_a0, iwl_ax201_name),
-
-/* So-F with Hr */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_so_a0_hr_a0, iwl_ax203_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_so_a0_hr_a0, iwl_ax101_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_so_a0_hr_a0, iwl_ax201_name),
-
-/* So-F with Gf */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_CDB,
-		      iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_name),
-
-/* SoF with JF2 */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_name),
-
-/* SoF with JF */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_name),
-
-/* So with GF */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_CDB,
-		      iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_name),
-
-/* So with JF2 */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_name),
-
-/* So with JF */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_160_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_name),
-
-/* Bz */
-/* FIXME: need to change the naming according to the actual CRF */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      iwl_cfg_bz, iwl_fm_name),
-
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_BZ_W, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      iwl_cfg_bz, iwl_fm_name),
-
-/* Ga (Gl) */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_320, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_gl, iwl_gl_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_NO_320, IWL_CFG_ANY, IWL_CFG_NO_CDB,
-		      iwl_cfg_gl, iwl_mtp_name),
-
-/* Sc */
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SC, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      iwl_cfg_sc, iwl_sc_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SC2, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      iwl_cfg_sc2, iwl_sc2_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SC2F, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,
-		      iwl_cfg_sc2f, iwl_sc2f_name),
-#endif /* CONFIG_IWLMVM */
-};
-EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_dev_info_table);
+#if IS_ENABLED(CONFIG_IWLDVM)
+	IWL_DEV_INFO(iwl5100_n_cfg, iwl5100_agn_name,
+		     DEVICE(0x4232), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl5100_n_cfg, iwl5100_agn_name,
+		     DEVICE(0x4232), SUBDEV_MASKED(0x4, 0xF)),
+	IWL_DEV_INFO(iwl5100_n_cfg, iwl5100_bgn_name,
+		     DEVICE(0x4232), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl5100_abg_cfg, iwl5100_abg_name,
+		     DEVICE(0x4232), SUBDEV_MASKED(0x6, 0xF)),
+	IWL_DEV_INFO(iwl5100_n_cfg, iwl5100_agn_name,
+		     DEVICE(0x4237), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl5100_n_cfg, iwl5100_agn_name,
+		     DEVICE(0x4237), SUBDEV_MASKED(0x4, 0xF)),
+	IWL_DEV_INFO(iwl5100_n_cfg, iwl5100_bgn_name,
+		     DEVICE(0x4237), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl5100_abg_cfg, iwl5100_abg_name,
+		     DEVICE(0x4237), SUBDEV_MASKED(0x6, 0xF)),
 
-#if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
-const unsigned int iwl_dev_info_table_size = ARRAY_SIZE(iwl_dev_info_table);
-EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_dev_info_table_size);
-#endif
+/* 5300 Series WiFi */
+	IWL_DEV_INFO(iwl5300_agn_cfg, iwl5300_agn_name,
+		     DEVICE(0x4235), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl5300_agn_cfg, iwl5300_agn_name,
+		     DEVICE(0x4235), SUBDEV_MASKED(0x4, 0xF)),
+	IWL_DEV_INFO(iwl5300_agn_cfg, iwl5300_agn_name,
+		     DEVICE(0x4236), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl5300_agn_cfg, iwl5300_agn_name,
+		     DEVICE(0x4236), SUBDEV_MASKED(0x4, 0xF)),
 
-/*
- * Read rf id and cdb info from prph register and store it
- */
-static void get_crf_id(struct iwl_trans *iwl_trans)
-{
-	u32 sd_reg_ver_addr;
-	u32 val = 0;
-	u8 step;
-
-	if (iwl_trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
-		sd_reg_ver_addr = SD_REG_VER_GEN2;
-	else
-		sd_reg_ver_addr = SD_REG_VER;
-
-	/* Enable access to peripheral registers */
-	val = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG);
-	val |= WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK;
-	iwl_write_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG, val);
-
-	/* Read crf info */
-	iwl_trans->hw_crf_id = iwl_read_prph_no_grab(iwl_trans, sd_reg_ver_addr);
-
-	/* Read cnv info */
-	iwl_trans->hw_cnv_id =
-		iwl_read_prph_no_grab(iwl_trans, CNVI_AUX_MISC_CHIP);
-
-	/* For BZ-W, take B step also when A step is indicated */
-	if (CSR_HW_REV_TYPE(iwl_trans->hw_rev) == IWL_CFG_MAC_TYPE_BZ_W)
-		step = SILICON_B_STEP;
-
-	/* In BZ, the MAC step must be read from the CNVI aux register */
-	if (CSR_HW_REV_TYPE(iwl_trans->hw_rev) == IWL_CFG_MAC_TYPE_BZ) {
-		step = CNVI_AUX_MISC_CHIP_MAC_STEP(iwl_trans->hw_cnv_id);
-
-		/* For BZ-U, take B step also when A step is indicated */
-		if ((CNVI_AUX_MISC_CHIP_PROD_TYPE(iwl_trans->hw_cnv_id) ==
-		    CNVI_AUX_MISC_CHIP_PROD_TYPE_BZ_U) &&
-		    step == SILICON_A_STEP)
-			step = SILICON_B_STEP;
-	}
+/* 5350 Series WiFi/WiMax */
+	IWL_DEV_INFO(iwl5350_agn_cfg, iwl5350_agn_name,
+		     DEVICE(0x423A)),
+	IWL_DEV_INFO(iwl5350_agn_cfg, iwl5350_agn_name,
+		     DEVICE(0x423B)),
 
-	if (CSR_HW_REV_TYPE(iwl_trans->hw_rev) == IWL_CFG_MAC_TYPE_BZ ||
-	    CSR_HW_REV_TYPE(iwl_trans->hw_rev) == IWL_CFG_MAC_TYPE_BZ_W) {
-		iwl_trans->hw_rev_step = step;
-		iwl_trans->hw_rev |= step;
-	}
+/* 5150 Series Wifi/WiMax */
+	IWL_DEV_INFO(iwl5150_agn_cfg, iwl5150_agn_name,
+		     DEVICE(0x423C), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl5150_abg_cfg, iwl5150_abg_name,
+		     DEVICE(0x423C), SUBDEV_MASKED(0x6, 0xF)),
 
-	/* Read cdb info (also contains the jacket info if needed in the future */
-	iwl_trans->hw_wfpm_id =
-		iwl_read_umac_prph_no_grab(iwl_trans, WFPM_OTP_CFG1_ADDR);
-	IWL_INFO(iwl_trans, "Detected crf-id 0x%x, cnv-id 0x%x wfpm id 0x%x\n",
-		 iwl_trans->hw_crf_id, iwl_trans->hw_cnv_id,
-		 iwl_trans->hw_wfpm_id);
-}
+	IWL_DEV_INFO(iwl5150_agn_cfg, iwl5150_agn_name,
+		     DEVICE(0x423D), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl5150_abg_cfg, iwl5150_abg_name,
+		     DEVICE(0x423D), SUBDEV_MASKED(0x6, 0xF)),
 
-/*
- * In case that there is no OTP on the NIC, map the rf id and cdb info
- * from the prph registers.
- */
-static int map_crf_id(struct iwl_trans *iwl_trans)
-{
-	int ret = 0;
-	u32 val = iwl_trans->hw_crf_id;
-	u32 step_id = REG_CRF_ID_STEP(val);
-	u32 slave_id = REG_CRF_ID_SLAVE(val);
-	u32 jacket_id_cnv  = REG_CRF_ID_SLAVE(iwl_trans->hw_cnv_id);
-	u32 jacket_id_wfpm  = WFPM_OTP_CFG1_IS_JACKET(iwl_trans->hw_wfpm_id);
-	u32 cdb_id_wfpm  = WFPM_OTP_CFG1_IS_CDB(iwl_trans->hw_wfpm_id);
-
-	/* Map between crf id to rf id */
-	switch (REG_CRF_ID_TYPE(val)) {
-	case REG_CRF_ID_TYPE_JF_1:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_JF1 << 12);
-		break;
-	case REG_CRF_ID_TYPE_JF_2:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_JF2 << 12);
-		break;
-	case REG_CRF_ID_TYPE_HR_NONE_CDB_1X1:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_HR1 << 12);
-		break;
-	case REG_CRF_ID_TYPE_HR_NONE_CDB:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12);
-		break;
-	case REG_CRF_ID_TYPE_HR_CDB:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12);
-		break;
-	case REG_CRF_ID_TYPE_GF:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_GF << 12);
-		break;
-	case REG_CRF_ID_TYPE_FM:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_FM << 12);
-		break;
-	case REG_CRF_ID_TYPE_WHP:
-		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_WH << 12);
-		break;
-	default:
-		ret = -EIO;
-		IWL_ERR(iwl_trans,
-			"Can't find a correct rfid for crf id 0x%x\n",
-			REG_CRF_ID_TYPE(val));
-		goto out;
+/* 6x00 Series */
+	IWL_DEV_INFO(iwl6000_3agn_cfg, iwl6000_3agn_name,
+		     DEVICE(0x422B), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl6000_3agn_cfg, iwl6000_3agn_name,
+		     DEVICE(0x422B), SUBDEV_MASKED(0x8, 0xF)),
+	IWL_DEV_INFO(iwl6000i_2agn_cfg, iwl6000i_2agn_name,
+		     DEVICE(0x422C), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl6000i_non_n_cfg, iwl6000i_2abg_name,
+		     DEVICE(0x422C), SUBDEV_MASKED(0x6, 0xF)),
+	IWL_DEV_INFO(iwl6000i_non_n_cfg, iwl6000i_2bg_name,
+		     DEVICE(0x422C), SUBDEV_MASKED(0x7, 0xF)),
+	IWL_DEV_INFO(iwl6000_3agn_cfg, iwl6000_3agn_name,
+		     DEVICE(0x4238), SUBDEV(0x1111)),
+	IWL_DEV_INFO(iwl6000_3agn_cfg, iwl6000_3agn_name,
+		     DEVICE(0x4238), SUBDEV(0x1118)),
+	IWL_DEV_INFO(iwl6000i_2agn_cfg, iwl6000i_2agn_name,
+		     DEVICE(0x4239), SUBDEV(0x1311)),
+	IWL_DEV_INFO(iwl6000i_non_n_cfg, iwl6000i_2abg_name,
+		     DEVICE(0x4239), SUBDEV(0x1316)),
 
-	}
+/* 6x05 Series */
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_name,
+		     DEVICE(0x0082), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl6005_non_n_cfg, iwl6005_2abg_name,
+		     DEVICE(0x0082), SUBDEV_MASKED(0x6, 0xF)),
+	IWL_DEV_INFO(iwl6005_non_n_cfg, iwl6005_2bg_name,
+		     DEVICE(0x0082), SUBDEV_MASKED(0x7, 0xF)),
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_name,
+		     DEVICE(0x0082), SUBDEV_MASKED(0x8, 0xF)),
+
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_name,
+		     DEVICE(0x0085), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_name,
+		     DEVICE(0x0085), SUBDEV_MASKED(0x8, 0xF)),
+	IWL_DEV_INFO(iwl6005_non_n_cfg, iwl6005_2abg_name,
+		     DEVICE(0x0085), SUBDEV_MASKED(0x6, 0xF)),
+
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_sff_name,
+		     DEVICE(0x0082), SUBDEV_MASKED(0xC000, 0xF000)),
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_sff_name,
+		     DEVICE(0x0085), SUBDEV_MASKED(0xC000, 0xF000)),
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_d_name,
+		     DEVICE(0x0082), SUBDEV(0x4820)),
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_mow1_name,
+		     DEVICE(0x0082), SUBDEV(0x1304)),/* low 5GHz active */
+	IWL_DEV_INFO(iwl6005_n_cfg, iwl6005_2agn_mow2_name,
+		     DEVICE(0x0082), SUBDEV(0x1305)),/* high 5GHz active */
 
-	/* Set Step-id */
-	iwl_trans->hw_rf_id |= (step_id << 8);
+/* 6x30 Series */
+	IWL_DEV_INFO(iwl6030_n_cfg, iwl1030_bgn_name,
+		     DEVICE(0x008A), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl6030_non_n_cfg, iwl1030_bg_name,
+		     DEVICE(0x008A), SUBDEV_MASKED(0x7, 0xF)),
+	IWL_DEV_INFO(iwl6030_n_cfg, iwl1030_bgn_name,
+		     DEVICE(0x008B), SUBDEV(0x5315)),
+	IWL_DEV_INFO(iwl6030_non_n_cfg, iwl1030_bg_name,
+		     DEVICE(0x008B), SUBDEV(0x5317)),
+	IWL_DEV_INFO(iwl6030_n_cfg, iwl6030_2agn_name,
+		     DEVICE(0x0090), SUBDEV(0x5211)),
+	IWL_DEV_INFO(iwl6030_n_cfg, iwl6030_2bgn_name,
+		     DEVICE(0x0090), SUBDEV(0x5215)),
+	IWL_DEV_INFO(iwl6030_non_n_cfg, iwl6030_2abg_name,
+		     DEVICE(0x0090), SUBDEV(0x5216)),
+	IWL_DEV_INFO(iwl6030_n_cfg, iwl6030_2agn_name,
+		     DEVICE(0x0091), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl6030_n_cfg, iwl6030_2bgn_name,
+		     DEVICE(0x0091), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl6030_non_n_cfg, iwl6030_2abg_name,
+		     DEVICE(0x0091), SUBDEV_MASKED(0x6, 0xF)),
+	IWL_DEV_INFO(iwl6030_non_n_cfg, iwl6030_2bg_name,
+		     DEVICE(0x0091), SUBDEV(0x5207)),
 
-	/* Set CDB capabilities */
-	if (cdb_id_wfpm || slave_id) {
-		iwl_trans->hw_rf_id += BIT(28);
-		IWL_INFO(iwl_trans, "Adding cdb to rf id\n");
-	}
+/* 6x50 WiFi/WiMax Series */
+	IWL_DEV_INFO(iwl6050_2agn_cfg, iwl6050_2agn_name,
+		     DEVICE(0x0087), SUBDEV_MASKED(0x1, 0xF)),
+	IWL_DEV_INFO(iwl6050_2abg_cfg, iwl6050_2abg_name,
+		     DEVICE(0x0087), SUBDEV_MASKED(0x6, 0xF)),
+	IWL_DEV_INFO(iwl6050_2agn_cfg, iwl6050_2agn_name,
+		     DEVICE(0x0089), SUBDEV(0x1311)),
+	IWL_DEV_INFO(iwl6050_2abg_cfg, iwl6050_2abg_name,
+		     DEVICE(0x0089), SUBDEV(0x1316)),
 
-	/* Set Jacket capabilities */
-	if (jacket_id_wfpm || jacket_id_cnv) {
-		iwl_trans->hw_rf_id += BIT(29);
-		IWL_INFO(iwl_trans, "Adding jacket to rf id\n");
-	}
+/* 6150 WiFi/WiMax Series */
+	IWL_DEV_INFO(iwl6150_bgn_cfg, iwl6150_bgn_name,
+		     DEVICE(0x0885), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl6150_bg_cfg, iwl6150_bg_name,
+		     DEVICE(0x0885), SUBDEV_MASKED(0x7, 0xF)),
+	IWL_DEV_INFO(iwl6150_bgn_cfg, iwl6150_bgn_name,
+		     DEVICE(0x0886), SUBDEV(0x1315)),
+	IWL_DEV_INFO(iwl6150_bg_cfg, iwl6150_bg_name,
+		     DEVICE(0x0886), SUBDEV(0x1317)),
 
-	IWL_INFO(iwl_trans,
-		 "Detected rf-type 0x%x step-id 0x%x slave-id 0x%x from crf id 0x%x\n",
-		 REG_CRF_ID_TYPE(val), step_id, slave_id, iwl_trans->hw_rf_id);
-	IWL_INFO(iwl_trans,
-		 "Detected cdb-id 0x%x jacket-id 0x%x from wfpm id 0x%x\n",
-		 cdb_id_wfpm, jacket_id_wfpm, iwl_trans->hw_wfpm_id);
-	IWL_INFO(iwl_trans, "Detected jacket-id 0x%x from cnvi id 0x%x\n",
-		 jacket_id_cnv, iwl_trans->hw_cnv_id);
+/* 1000 Series WiFi */
+	IWL_DEV_INFO(iwl1000_bgn_cfg, iwl1000_bgn_name,
+		     DEVICE(0x0083), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl1000_bg_cfg, iwl1000_bg_name,
+		     DEVICE(0x0083), SUBDEV_MASKED(0x6, 0xF)),
+	IWL_DEV_INFO(iwl1000_bgn_cfg, iwl1000_bgn_name,
+		     DEVICE(0x0084), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl1000_bg_cfg, iwl1000_bg_name,
+		     DEVICE(0x0084), SUBDEV_MASKED(0x6, 0xF)),
 
-out:
-	return ret;
-}
+/* 100 Series WiFi */
+	IWL_DEV_INFO(iwl100_bgn_cfg, iwl100_bgn_name,
+		     DEVICE(0x08AE), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl100_bg_cfg, iwl100_bg_name,
+		     DEVICE(0x08AE), SUBDEV_MASKED(0x7, 0xF)),
+	IWL_DEV_INFO(iwl100_bgn_cfg, iwl100_bgn_name,
+		     DEVICE(0x08AF), SUBDEV(0x1015)),
+	IWL_DEV_INFO(iwl100_bg_cfg, iwl100_bg_name,
+		     DEVICE(0x08AF), SUBDEV(0x1017)),
+
+/* 130 Series WiFi */
+	IWL_DEV_INFO(iwl130_bgn_cfg, iwl130_bgn_name,
+		     DEVICE(0x0896), SUBDEV_MASKED(0x5, 0xF)),
+	IWL_DEV_INFO(iwl130_bg_cfg, iwl130_bg_name,
+		     DEVICE(0x0896), SUBDEV_MASKED(0x7, 0xF)),
+	IWL_DEV_INFO(iwl130_bgn_cfg, iwl130_bgn_name,
+		     DEVICE(0x0897), SUBDEV(0x5015)),
+	IWL_DEV_INFO(iwl130_bg_cfg, iwl130_bg_name,
+		     DEVICE(0x0897), SUBDEV(0x5017)),
+
+/* 2x00 Series */
+	IWL_DEV_INFO(iwl2000_2bgn_cfg, iwl2000_2bgn_name,
+		     DEVICE(0x0890), SUBDEV(0x4022)),
+	IWL_DEV_INFO(iwl2000_2bgn_cfg, iwl2000_2bgn_name,
+		     DEVICE(0x0891), SUBDEV(0x4222)),
+	IWL_DEV_INFO(iwl2000_2bgn_cfg, iwl2000_2bgn_name,
+		     DEVICE(0x0890), SUBDEV(0x4422)),
+	IWL_DEV_INFO(iwl2000_2bgn_cfg, iwl2000_2bgn_d_name,
+		     DEVICE(0x0890), SUBDEV(0x4822)),
+
+/* 2x30 Series */
+	IWL_DEV_INFO(iwl2030_2bgn_cfg, iwl2030_2bgn_name,
+		     DEVICE(0x0887)),
+	IWL_DEV_INFO(iwl2030_2bgn_cfg, iwl2030_2bgn_name,
+		     DEVICE(0x0888), SUBDEV(0x4262)),
+
+/* 6x35 Series */
+	IWL_DEV_INFO(iwl6035_2agn_cfg, iwl6035_2agn_name,
+		     DEVICE(0x088E), SUBDEV_MASKED(0x0, 0xF)),
+	IWL_DEV_INFO(iwl6035_2agn_cfg, iwl6035_2agn_sff_name,
+		     DEVICE(0x088E), SUBDEV_MASKED(0xA, 0xF)),
+	IWL_DEV_INFO(iwl6035_2agn_cfg, iwl6035_2agn_name,
+		     DEVICE(0x088F), SUBDEV_MASKED(0x0, 0xF)),
+	IWL_DEV_INFO(iwl6035_2agn_cfg, iwl6035_2agn_sff_name,
+		     DEVICE(0x088F), SUBDEV_MASKED(0xA, 0xF)),
+
+/* 105 Series */
+	IWL_DEV_INFO(iwl105_bgn_cfg, iwl105_bgn_name,
+		     DEVICE(0x0894)),
+	IWL_DEV_INFO(iwl105_bgn_cfg, iwl105_bgn_name,
+		     DEVICE(0x0895), SUBDEV(0x0222)),
+
+/* 135 Series */
+	IWL_DEV_INFO(iwl135_bgn_cfg, iwl135_bgn_name,
+		     DEVICE(0x0892)),
+	IWL_DEV_INFO(iwl135_bgn_cfg, iwl135_bgn_name,
+		     DEVICE(0x0893), SUBDEV(0x0262)),
+#endif /* CONFIG_IWLDVM */
+
+#if IS_ENABLED(CONFIG_IWLMVM)
+/* 7260 Series */
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B1)), // unlisted ones fall through to here
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4060)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x406A)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4160)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4062)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4162)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4460)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x446A)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4462)),
+	IWL_DEV_INFO(iwl7260_high_temp_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B1), SUBDEV(0x4A70)),
+	IWL_DEV_INFO(iwl7260_high_temp_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B1), SUBDEV(0x4A6E)),
+	IWL_DEV_INFO(iwl7260_high_temp_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B1), SUBDEV(0x4A6C)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4560)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4020)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x402A)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0x4420)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC060)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC06A)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC160)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC062)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC162)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC760)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC460)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC462)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC560)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC360)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC020)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC02A)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B1), SUBDEV(0xC420)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B2), SUBDEV(0x4270)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B2), SUBDEV(0x4272)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B2), SUBDEV(0x4260)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B2), SUBDEV(0x426A)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B2), SUBDEV(0x4262)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B2), SUBDEV(0x4370)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B2), SUBDEV(0x4360)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B2), SUBDEV(0x4220)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B2), SUBDEV(0xC270)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B2), SUBDEV(0xC272)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B2), SUBDEV(0xC260)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B2), SUBDEV(0xC26A)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_n_name,
+		     DEVICE(0x08B2), SUBDEV(0xC262)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2ac_name,
+		     DEVICE(0x08B2), SUBDEV(0xC370)),
+	IWL_DEV_INFO(iwl7260_cfg, iwl7260_2n_name,
+		     DEVICE(0x08B2), SUBDEV(0xC220)),
+
+/* 3160 Series */
+	IWL_DEV_INFO(iwl3160_cfg, iwl3160_2ac_name,
+		     DEVICE(0x08B3)),
+
+	IWL_DEV_INFO(iwl3160_cfg, iwl3160_n_name,
+		     DEVICE(0x08B3), SUBDEV_MASKED(0x62, 0xFF)),
+	IWL_DEV_INFO(iwl3160_cfg, iwl3160_2n_name,
+		     DEVICE(0x08B3), SUBDEV_MASKED(0x60, 0xFF)),
+
+	IWL_DEV_INFO(iwl3160_cfg, iwl3160_2ac_name,
+		     DEVICE(0x08B4)),
+
+/* 3165 Series */
+	IWL_DEV_INFO(iwl3165_2ac_cfg, iwl3165_2ac_name,
+		     DEVICE(0x3165)),
+	IWL_DEV_INFO(iwl3165_2ac_cfg, iwl3165_2ac_name,
+		     DEVICE(0x3166)),
+
+/* 3168 Series */
+	IWL_DEV_INFO(iwl3168_2ac_cfg, iwl3168_2ac_name,
+		     DEVICE(0x24FB)),
+
+/* 7265 Series */
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2ac_name,
+		     DEVICE(0x095A)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5000)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x500A)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_n_name,
+		     DEVICE(0x095A), SUBDEV(0x5002)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_n_name,
+		     DEVICE(0x095A), SUBDEV(0x5102)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5020)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x502A)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5090)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5190)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5100)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5400)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5420)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5490)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5C10)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x5590)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x9000)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x900A)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095A), SUBDEV(0x9400)),
+
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2ac_name,
+		     DEVICE(0x095B)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095B), SUBDEV(0x520A)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_n_name,
+		     DEVICE(0x095B), SUBDEV(0x5302)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095B), SUBDEV(0x5200)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_n_name,
+		     DEVICE(0x095B), SUBDEV(0x5202)),
+	IWL_DEV_INFO(iwl7265_cfg, iwl7265_2n_name,
+		     DEVICE(0x095B), SUBDEV(0x9200)),
+
+/* 8000 Series */
+	IWL_DEV_INFO(iwl8260_cfg, iwl8260_2ac_name,
+		     DEVICE(0x24F3)),
+	IWL_DEV_INFO(iwl8260_cfg, iwl8260_2n_name,
+		     DEVICE(0x24F3), SUBDEV(0x0004)),
+	IWL_DEV_INFO(iwl8260_cfg, iwl8260_2n_name,
+		     DEVICE(0x24F3), SUBDEV(0x0044)),
+	IWL_DEV_INFO(iwl8260_cfg, iwl8260_2ac_name,
+		     DEVICE(0x24F4)),
+	IWL_DEV_INFO(iwl8260_cfg, iwl4165_2ac_name,
+		     DEVICE(0x24F5)),
+	IWL_DEV_INFO(iwl8260_cfg, iwl4165_2ac_name,
+		     DEVICE(0x24F6)),
+	IWL_DEV_INFO(iwl8265_cfg, iwl8265_2ac_name,
+		     DEVICE(0x24FD)),
+	IWL_DEV_INFO(iwl8265_cfg, iwl8275_2ac_name,
+		     DEVICE(0x24FD), SUBDEV(0x3E02)),
+	IWL_DEV_INFO(iwl8265_cfg, iwl8275_2ac_name,
+		     DEVICE(0x24FD), SUBDEV(0x3E01)),
+	IWL_DEV_INFO(iwl8265_cfg, iwl8275_2ac_name,
+		     DEVICE(0x24FD), SUBDEV(0x1012)),
+	IWL_DEV_INFO(iwl8265_cfg, iwl8275_2ac_name,
+		     DEVICE(0x24FD), SUBDEV(0x0012)),
+	IWL_DEV_INFO(iwl8265_cfg, iwl_killer_1435i_name,
+		     DEVICE(0x24FD), SUBDEV(0x1431)),
+	IWL_DEV_INFO(iwl8265_cfg, iwl_killer_1434_kix_name,
+		     DEVICE(0x24FD), SUBDEV(0x1432)),
+
+/* JF1 RF */
+	IWL_DEV_INFO(iwl_rf_jf, iwl9461_160_name,
+		     RF_TYPE(JF1)),
+	IWL_DEV_INFO(iwl_rf_jf_80mhz, iwl9461_name,
+		     RF_TYPE(JF1), BW_LIMITED),
+	IWL_DEV_INFO(iwl_rf_jf, iwl9462_160_name,
+		     RF_TYPE(JF1), RF_ID(JF1_DIV)),
+	IWL_DEV_INFO(iwl_rf_jf_80mhz, iwl9462_name,
+		     RF_TYPE(JF1), RF_ID(JF1_DIV), BW_LIMITED),
+/* JF2 RF */
+	IWL_DEV_INFO(iwl_rf_jf, iwl9260_160_name,
+		     RF_TYPE(JF2)),
+	IWL_DEV_INFO(iwl_rf_jf_80mhz, iwl9260_name,
+		     RF_TYPE(JF2), BW_LIMITED),
+	IWL_DEV_INFO(iwl_rf_jf, iwl9560_160_name,
+		     RF_TYPE(JF2), RF_ID(JF)),
+	IWL_DEV_INFO(iwl_rf_jf_80mhz, iwl9560_name,
+		     RF_TYPE(JF2), RF_ID(JF), BW_LIMITED),
+
+/* HR RF */
+	IWL_DEV_INFO(iwl_rf_hr, iwl_ax201_name, RF_TYPE(HR2)),
+	IWL_DEV_INFO(iwl_rf_hr_80mhz, iwl_ax101_name, RF_TYPE(HR1)),
+	IWL_DEV_INFO(iwl_rf_hr_80mhz, iwl_ax203_name, RF_TYPE(HR2), BW_LIMITED),
+	IWL_DEV_INFO(iwl_rf_hr, iwl_ax200_name, DEVICE(0x2723)),
+
+/* GF RF */
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax211_name, RF_TYPE(GF)),
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax411_name, RF_TYPE(GF), CDB),
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax210_name, DEVICE(0x2725)),
+
+/* Killer CRFs */
+	IWL_DEV_INFO(iwl_rf_jf, iwl9260_killer_1550_name, SUBDEV(0x1550)),
+	IWL_DEV_INFO(iwl_rf_jf, iwl9560_killer_1550s_name, SUBDEV(0x1551)),
+	IWL_DEV_INFO(iwl_rf_jf, iwl9560_killer_1550i_name, SUBDEV(0x1552)),
+
+	IWL_DEV_INFO(iwl_rf_hr, iwl_ax201_killer_1650s_name, SUBDEV(0x1651)),
+	IWL_DEV_INFO(iwl_rf_hr, iwl_ax201_killer_1650i_name, SUBDEV(0x1652)),
+
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax211_killer_1675s_name, SUBDEV(0x1671)),
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax211_killer_1675i_name, SUBDEV(0x1672)),
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax210_killer_1675w_name, SUBDEV(0x1673)),
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax210_killer_1675x_name, SUBDEV(0x1674)),
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax411_killer_1690s_name, SUBDEV(0x1691)),
+	IWL_DEV_INFO(iwl_rf_gf, iwl_ax411_killer_1690i_name, SUBDEV(0x1692)),
+
+/* Killer discrete */
+	IWL_DEV_INFO(iwl_rf_hr, iwl_ax200_killer_1650w_name,
+		     DEVICE(0x2723), SUBDEV(0x1653)),
+	IWL_DEV_INFO(iwl_rf_hr, iwl_ax200_killer_1650x_name,
+		     DEVICE(0x2723), SUBDEV(0x1654)),
+#endif /* CONFIG_IWLMVM */
+#if IS_ENABLED(CONFIG_IWLMLD)
+/* FM RF */
+	IWL_DEV_INFO(iwl_rf_fm, iwl_be201_name, RF_TYPE(FM)),
+	IWL_DEV_INFO(iwl_rf_fm, iwl_be401_name, RF_TYPE(FM), CDB),
+	IWL_DEV_INFO(iwl_rf_fm, iwl_be200_name, RF_TYPE(FM),
+		     DEVICE(0x272B), DISCRETE),
+	IWL_DEV_INFO(iwl_rf_fm_160mhz, iwl_be202_name,
+		     RF_TYPE(FM), BW_LIMITED),
+
+/* Killer CRFs */
+	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1750s_name, SUBDEV(0x1771)),
+	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1750i_name, SUBDEV(0x1772)),
+	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1790s_name, SUBDEV(0x1791)),
+	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1790i_name, SUBDEV(0x1792)),
+
+/* Killer discrete */
+	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1750w_name,
+		     DEVICE(0x272B), SUBDEV(0x1773)),
+	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1750x_name,
+		     DEVICE(0x272B), SUBDEV(0x1774)),
+
+/* WH RF */
+	IWL_DEV_INFO(iwl_rf_wh, iwl_be211_name, RF_TYPE(WH)),
+	IWL_DEV_INFO(iwl_rf_wh_160mhz, iwl_be213_name, RF_TYPE(WH), BW_LIMITED),
+
+/* PE RF */
+	IWL_DEV_INFO(iwl_rf_pe, iwl_bn201_name, RF_TYPE(PE)),
+	IWL_DEV_INFO(iwl_rf_pe, iwl_be223_name, RF_TYPE(PE), SUBDEV(0x0524)),
+	IWL_DEV_INFO(iwl_rf_pe, iwl_be221_name, RF_TYPE(PE), SUBDEV(0x0324)),
+
+/* Killer */
+	IWL_DEV_INFO(iwl_rf_wh, iwl_killer_be1775s_name, SUBDEV(0x1776)),
+	IWL_DEV_INFO(iwl_rf_wh, iwl_killer_be1775i_name, SUBDEV(0x1775)),
+
+	IWL_DEV_INFO(iwl_rf_pe, iwl_killer_bn1850w2_name, SUBDEV(0x1851)),
+	IWL_DEV_INFO(iwl_rf_pe, iwl_killer_bn1850i_name, SUBDEV(0x1852)),
+#endif /* CONFIG_IWLMLD */
+};
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_dev_info_table);
+
+#if IS_ENABLED(CONFIG_IWLWIFI_KUNIT_TESTS)
+const unsigned int iwl_dev_info_table_size = ARRAY_SIZE(iwl_dev_info_table);
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_dev_info_table_size);
+#endif
 
 /* PCI registers */
 #define PCI_CFG_RETRY_TIMEOUT	0x041
 
-VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info *
-iwl_pci_find_dev_info(u16 device, u16 subsystem_device,
-		      u16 mac_type, u8 mac_step, u16 rf_type, u8 cdb,
-		      u8 jacket, u8 rf_id, u8 no_160, u8 cores, u8 rf_step)
+const struct iwl_dev_info *
+iwl_pci_find_dev_info(u16 device, u16 subsystem_device, u16 rf_type, u8 cdb,
+		      u8 rf_id, u8 bw_limit, bool discrete)
 {
 	int num_devices = ARRAY_SIZE(iwl_dev_info_table);
 	int i;
@@ -1309,49 +1105,32 @@ iwl_pci_find_dev_info(u16 device, u16 subsystem_device,
 
 	for (i = num_devices - 1; i >= 0; i--) {
 		const struct iwl_dev_info *dev_info = &iwl_dev_info_table[i];
+		u16 subdevice_mask;
 
 		if (dev_info->device != (u16)IWL_CFG_ANY &&
 		    dev_info->device != device)
 			continue;
 
-		if (dev_info->subdevice != (u16)IWL_CFG_ANY &&
-		    dev_info->subdevice != subsystem_device)
-			continue;
-
-		if (dev_info->mac_type != (u16)IWL_CFG_ANY &&
-		    dev_info->mac_type != mac_type)
-			continue;
-
-		if (dev_info->mac_step != (u8)IWL_CFG_ANY &&
-		    dev_info->mac_step != mac_step)
-			continue;
-
-		if (dev_info->rf_type != (u16)IWL_CFG_ANY &&
-		    dev_info->rf_type != rf_type)
-			continue;
+		subdevice_mask = GENMASK(dev_info->subdevice_m_h,
+					 dev_info->subdevice_m_l);
 
-		if (dev_info->cdb != (u8)IWL_CFG_ANY &&
-		    dev_info->cdb != cdb)
+		if (dev_info->subdevice != (u16)IWL_CFG_ANY &&
+		    dev_info->subdevice != (subsystem_device & subdevice_mask))
 			continue;
 
-		if (dev_info->jacket != (u8)IWL_CFG_ANY &&
-		    dev_info->jacket != jacket)
+		if (dev_info->match_rf_type && dev_info->rf_type != rf_type)
 			continue;
 
-		if (dev_info->rf_id != (u8)IWL_CFG_ANY &&
-		    dev_info->rf_id != rf_id)
+		if (dev_info->match_cdb && dev_info->cdb != cdb)
 			continue;
 
-		if (dev_info->no_160 != (u8)IWL_CFG_ANY &&
-		    dev_info->no_160 != no_160)
+		if (dev_info->match_rf_id && dev_info->rf_id != rf_id)
 			continue;
 
-		if (dev_info->cores != (u8)IWL_CFG_ANY &&
-		    dev_info->cores != cores)
+		if (dev_info->match_bw_limit && dev_info->bw_limit != bw_limit)
 			continue;
 
-		if (dev_info->rf_step != (u8)IWL_CFG_ANY &&
-		    dev_info->rf_step != rf_step)
+		if (dev_info->match_discrete && dev_info->discrete != discrete)
 			continue;
 
 		return dev_info;
@@ -1363,206 +1142,65 @@ EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_pci_find_dev_info);
 
 static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
-	const struct iwl_cfg_trans_params *trans;
-	const struct iwl_cfg *cfg_7265d __maybe_unused = NULL;
-	const struct iwl_dev_info *dev_info;
-	struct iwl_trans *iwl_trans;
-	struct iwl_trans_pcie *trans_pcie;
+	const struct iwl_mac_cfg *mac_cfg = (void *)ent->driver_data;
+	u8 __iomem *hw_base;
+	u32 bar0, hw_rev;
 	int ret;
-	const struct iwl_cfg *cfg;
-
-	trans = (void *)(ent->driver_data & ~TRANS_CFG_MARKER);
-
-	/*
-	 * This is needed for backwards compatibility with the old
-	 * tables, so we don't need to change all the config structs
-	 * at the same time.  The cfg is used to compare with the old
-	 * full cfg structs.
-	 */
-	cfg = (void *)(ent->driver_data & ~TRANS_CFG_MARKER);
 
-	/* make sure trans is the first element in iwl_cfg */
-	BUILD_BUG_ON(offsetof(struct iwl_cfg, trans));
-
-	iwl_trans = iwl_trans_pcie_alloc(pdev, ent, trans);
-	if (IS_ERR(iwl_trans))
-		return PTR_ERR(iwl_trans);
-
-	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(iwl_trans);
-
-	/*
-	 * Let's try to grab NIC access early here. Sometimes, NICs may
-	 * fail to initialize, and if that happens it's better if we see
-	 * issues early on (and can reprobe, per the logic inside), than
-	 * first trying to load the firmware etc. and potentially only
-	 * detecting any problems when the first interface is brought up.
-	 */
-	ret = iwl_pcie_prepare_card_hw(iwl_trans);
-	if (!ret) {
-		ret = iwl_finish_nic_init(iwl_trans);
+	/* reassign our BAR 0 if invalid due to possible runtime PM races */
+	pci_read_config_dword(pdev, PCI_BASE_ADDRESS_0, &bar0);
+	if (bar0 == PCI_BASE_ADDRESS_MEM_TYPE_64) {
+		ret = pci_assign_resource(pdev, 0);
 		if (ret)
-			goto out_free_trans;
-		if (iwl_trans_grab_nic_access(iwl_trans)) {
-			get_crf_id(iwl_trans);
-			/* all good */
-			iwl_trans_release_nic_access(iwl_trans);
-		} else {
-			ret = -EIO;
-			goto out_free_trans;
-		}
-	}
-
-	iwl_trans->hw_rf_id = iwl_read32(iwl_trans, CSR_HW_RF_ID);
-
-	/*
-	 * The RF_ID is set to zero in blank OTP so read version to
-	 * extract the RF_ID.
-	 * This is relevant only for family 9000 and up.
-	 */
-	if (iwl_trans->trans_cfg->rf_id &&
-	    iwl_trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_9000 &&
-	    !CSR_HW_RFID_TYPE(iwl_trans->hw_rf_id) && map_crf_id(iwl_trans)) {
-		ret = -EINVAL;
-		goto out_free_trans;
+			return ret;
 	}
 
-	IWL_INFO(iwl_trans, "PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n",
-		 pdev->device, pdev->subsystem_device,
-		 iwl_trans->hw_rev, iwl_trans->hw_rf_id);
-
-	dev_info = iwl_pci_find_dev_info(pdev->device, pdev->subsystem_device,
-					 CSR_HW_REV_TYPE(iwl_trans->hw_rev),
-					 iwl_trans->hw_rev_step,
-					 CSR_HW_RFID_TYPE(iwl_trans->hw_rf_id),
-					 CSR_HW_RFID_IS_CDB(iwl_trans->hw_rf_id),
-					 CSR_HW_RFID_IS_JACKET(iwl_trans->hw_rf_id),
-					 IWL_SUBDEVICE_RF_ID(pdev->subsystem_device),
-					 IWL_SUBDEVICE_NO_160(pdev->subsystem_device),
-					 IWL_SUBDEVICE_CORES(pdev->subsystem_device),
-					 CSR_HW_RFID_STEP(iwl_trans->hw_rf_id));
-	if (dev_info) {
-		iwl_trans->cfg = dev_info->cfg;
-		iwl_trans->name = dev_info->name;
-		iwl_trans->no_160 = dev_info->no_160 == IWL_CFG_NO_160;
-	}
-
-#if IS_ENABLED(CONFIG_IWLMVM)
-	/*
-	 * special-case 7265D, it has the same PCI IDs.
-	 *
-	 * Note that because we already pass the cfg to the transport above,
-	 * all the parameters that the transport uses must, until that is
-	 * changed, be identical to the ones in the 7265D configuration.
-	 */
-	if (cfg == &iwl7265_2ac_cfg)
-		cfg_7265d = &iwl7265d_2ac_cfg;
-	else if (cfg == &iwl7265_2n_cfg)
-		cfg_7265d = &iwl7265d_2n_cfg;
-	else if (cfg == &iwl7265_n_cfg)
-		cfg_7265d = &iwl7265d_n_cfg;
-	if (cfg_7265d &&
-	    (iwl_trans->hw_rev & CSR_HW_REV_TYPE_MSK) == CSR_HW_REV_TYPE_7265D)
-		iwl_trans->cfg = cfg_7265d;
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
 
-	/*
-	 * This is a hack to switch from Qu B0 to Qu C0.  We need to
-	 * do this for all cfgs that use Qu B0, except for those using
-	 * Jf, which have already been moved to the new table.  The
-	 * rest must be removed once we convert Qu with Hr as well.
-	 */
-	if (iwl_trans->hw_rev == CSR_HW_REV_TYPE_QU_C0) {
-		if (iwl_trans->cfg == &iwl_ax201_cfg_qu_hr)
-			iwl_trans->cfg = &iwl_ax201_cfg_qu_c0_hr_b0;
-		else if (iwl_trans->cfg == &killer1650s_2ax_cfg_qu_b0_hr_b0)
-			iwl_trans->cfg = &killer1650s_2ax_cfg_qu_c0_hr_b0;
-		else if (iwl_trans->cfg == &killer1650i_2ax_cfg_qu_b0_hr_b0)
-			iwl_trans->cfg = &killer1650i_2ax_cfg_qu_c0_hr_b0;
-	}
+	pci_set_master(pdev);
 
-	/* same thing for QuZ... */
-	if (iwl_trans->hw_rev == CSR_HW_REV_TYPE_QUZ) {
-		if (iwl_trans->cfg == &iwl_ax201_cfg_qu_hr)
-			iwl_trans->cfg = &iwl_ax201_cfg_quz_hr;
-		else if (iwl_trans->cfg == &killer1650s_2ax_cfg_qu_b0_hr_b0)
-			iwl_trans->cfg = &iwl_ax1650s_cfg_quz_hr;
-		else if (iwl_trans->cfg == &killer1650i_2ax_cfg_qu_b0_hr_b0)
-			iwl_trans->cfg = &iwl_ax1650i_cfg_quz_hr;
+	ret = pcim_request_all_regions(pdev, DRV_NAME);
+	if (ret) {
+		dev_err(&pdev->dev, "Requesting all PCI BARs failed.\n");
+		return ret;
 	}
 
+#if defined(__FreeBSD__)
+	linuxkpi_pcim_want_to_use_bus_functions(pdev);
 #endif
-	/*
-	 * If we didn't set the cfg yet, the PCI ID table entry should have
-	 * been a full config - if yes, use it, otherwise fail.
-	 */
-	if (!iwl_trans->cfg) {
-		if (ent->driver_data & TRANS_CFG_MARKER) {
-			pr_err("No config found for PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n",
-			       pdev->device, pdev->subsystem_device,
-			       iwl_trans->hw_rev, iwl_trans->hw_rf_id);
-			ret = -EINVAL;
-			goto out_free_trans;
-		}
-		iwl_trans->cfg = cfg;
-	}
-
-	/* if we don't have a name yet, copy name from the old cfg */
-	if (!iwl_trans->name)
-		iwl_trans->name = iwl_trans->cfg->name;
-
-	IWL_INFO(iwl_trans, "Detected %s\n", iwl_trans->name);
-
-	if (iwl_trans->trans_cfg->mq_rx_supported) {
-		if (WARN_ON(!iwl_trans->cfg->num_rbds)) {
-			ret = -EINVAL;
-			goto out_free_trans;
-		}
-		trans_pcie->num_rx_bufs = iwl_trans->cfg->num_rbds;
-	} else {
-		trans_pcie->num_rx_bufs = RX_QUEUE_SIZE;
-	}
-
-	if (!iwl_trans->trans_cfg->integrated) {
-		u16 link_status;
-
-		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &link_status);
 
-		iwl_trans->pcie_link_speed =
-			u16_get_bits(link_status, PCI_EXP_LNKSTA_CLS);
+	hw_base = pcim_iomap(pdev, 0, 0);
+	if (!hw_base) {
+		dev_err(&pdev->dev, "Failed to map BAR 0.\n");
+		return -ENOMEM;
 	}
 
-	ret = iwl_trans_init(iwl_trans);
-	if (ret)
-		goto out_free_trans;
-
-	pci_set_drvdata(pdev, iwl_trans);
-
-	/* try to get ownership so that we'll know if we don't own it */
-	iwl_pcie_prepare_card_hw(iwl_trans);
-
-	iwl_trans->drv = iwl_drv_start(iwl_trans);
-
-	if (IS_ERR(iwl_trans->drv)) {
-		ret = PTR_ERR(iwl_trans->drv);
-		goto out_free_trans;
+	/* We can't use iwl_read32 because trans wasn't allocated */
+#if defined(__linux__)
+	hw_rev = readl(hw_base + CSR_HW_REV);
+#elif defined(__FreeBSD__)
+	hw_rev = bus_read_4((struct resource *)hw_base, CSR_HW_REV);
+#endif
+	if (hw_rev == 0xffffffff) {
+		dev_err(&pdev->dev, "HW_REV=0xFFFFFFFF, PCI issues?\n");
+		return -EIO;
 	}
 
-	/* register transport layer debugfs here */
-	iwl_trans_pcie_dbgfs_register(iwl_trans);
-
-	return 0;
-
-out_free_trans:
-	iwl_trans_pcie_free(iwl_trans);
-	return ret;
+	return iwl_pci_gen1_2_probe(pdev, ent, mac_cfg, hw_base, hw_rev);
 }
 
 static void iwl_pci_remove(struct pci_dev *pdev)
 {
 	struct iwl_trans *trans = pci_get_drvdata(pdev);
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	if (!trans)
 		return;
 
+	cancel_delayed_work_sync(&trans_pcie->me_recheck_wk);
+
 	iwl_drv_stop(trans->drv);
 
 	iwl_trans_pcie_free(trans);
@@ -1605,11 +1243,31 @@ static int _iwl_pci_resume(struct device *device, bool restore)
 	 * Scratch value was altered, this means the device was powered off, we
 	 * need to reset it completely.
 	 * Note: MAC (bits 0:7) will be cleared upon suspend even with wowlan,
-	 * so assume that any bits there mean that the device is usable.
+	 * but not bits [15:8]. So if we have bits set in lower word, assume
+	 * the device is alive.
+	 * Alternatively, if the scratch value is 0xFFFFFFFF, then we no longer
+	 * have access to the device and consider it powered off.
+	 * For older devices, just try silently to grab the NIC.
 	 */
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ &&
-	    !iwl_read32(trans, CSR_FUNC_SCRATCH))
-		device_was_powered_off = true;
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+		u32 scratch = iwl_read32(trans, CSR_FUNC_SCRATCH);
+
+		if (!(scratch & CSR_FUNC_SCRATCH_POWER_OFF_MASK) ||
+		    scratch == ~0U)
+			device_was_powered_off = true;
+	} else {
+		/*
+		 * bh are re-enabled by iwl_trans_pcie_release_nic_access,
+		 * so re-enable them if _iwl_trans_pcie_grab_nic_access fails.
+		 */
+		local_bh_disable();
+		if (_iwl_trans_pcie_grab_nic_access(trans, true)) {
+			iwl_trans_pcie_release_nic_access(trans);
+		} else {
+			device_was_powered_off = true;
+			local_bh_enable();
+		}
+	}
 
 	if (restore || device_was_powered_off) {
 		trans->state = IWL_TRANS_NO_FW;
@@ -1668,12 +1326,21 @@ static const struct dev_pm_ops iwl_dev_pm_ops = {
 
 #endif /* CONFIG_PM_SLEEP */
 
+static void iwl_pci_dump(struct device *device)
+{
+	struct pci_dev *pdev = to_pci_dev(device);
+	struct iwl_trans *trans = pci_get_drvdata(pdev);
+
+	iwl_op_mode_dump(trans->op_mode);
+}
+
 static struct pci_driver iwl_pci_driver = {
 	.name = DRV_NAME,
 	.id_table = iwl_hw_card_ids,
 	.probe = iwl_pci_probe,
 	.remove = iwl_pci_remove,
 	.driver.pm = IWL_PM_OPS,
+	.driver.coredump = iwl_pci_dump,
 #if defined(__FreeBSD__)
 	/* Allow iwm(4) to attach for conflicting IDs for now. */
 	.bsd_probe_return = (BUS_PROBE_DEFAULT - 1),
@@ -1713,48 +1380,30 @@ sysctl_iwlwifi_pci_ids_name(SYSCTL_HANDLER_ARGS)
 	id = iwl_hw_card_ids;
 	while (id != NULL && id->vendor != 0) {
 
-		if ((id->driver_data & TRANS_CFG_MARKER) != 0) {
-			/* Skip and print them below. */
-			struct iwl_cfg_trans_params *trans;
+		if (id->driver_data != 0) {
+			const struct iwl_mac_cfg *trans;
 
-			trans = (void *)(id->driver_data & ~TRANS_CFG_MARKER);
-			sbuf_printf(sb, "%#06x/%#06x/%#06x/%#06x\t%s\t%s\t%d\t%s\n",
+			trans = (void *)id->driver_data;
+			sbuf_printf(sb, "%#06x/%#06x/%#06x/%#06x\t%d\t%s\t%d\n",
 			    id->vendor, id->device, id->subvendor, id->subdevice,
-			    "", "", trans->device_family,
-			    iwl_device_family_name(trans->device_family));
-
-		} else if (id->driver_data != 0) {
-			const struct iwl_cfg *cfg;
-
-			cfg = (void *)(id->driver_data & ~TRANS_CFG_MARKER);
-			sbuf_printf(sb, "%#06x/%#06x/%#06x/%#06x\t%s\t%s\t%d\t%s\n",
-			    id->vendor, id->device, id->subvendor, id->subdevice,
-			    cfg->name, cfg->fw_name_pre, cfg->trans.device_family,
-			    iwl_device_family_name(cfg->trans.device_family));
+			    trans->device_family,
+			    iwl_device_family_name(trans->device_family),
+			    trans->gen2);
 		} else {
-			sbuf_printf(sb, "%#06x/%#06x/%#06x/%#06x\t%s\t%s\t%d\t%s\n",
+			sbuf_printf(sb, "%#06x/%#06x/%#06x/%#06x\t%d\t%s\t%d\n",
 			    id->vendor, id->device, id->subvendor, id->subdevice,
-			    "","", IWL_DEVICE_FAMILY_UNDEFINED,
-			    iwl_device_family_name(IWL_DEVICE_FAMILY_UNDEFINED));
+			    IWL_DEVICE_FAMILY_UNDEFINED,
+			    iwl_device_family_name(IWL_DEVICE_FAMILY_UNDEFINED), -1);
 		}
 		id++;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(iwl_dev_info_table); i++) {
 		const struct iwl_dev_info *dev_info = &iwl_dev_info_table[i];
-		const char *name;
-
-		if (dev_info->name)
-			name = dev_info->name;
-		else if (dev_info->cfg && dev_info->cfg->name)
-			name = dev_info->cfg->name;
-		else
-			name = "";
 
-		sbuf_printf(sb, "%#06x/%#06x/%#06x/%#06x\t%s\t%s\t%d\t%s\n",
+		sbuf_printf(sb, "%#06x/%#06x/%#06x/%#06x\t%s\t%s\n",
 		    PCI_VENDOR_ID_INTEL, dev_info->device, PCI_ANY_ID, dev_info->subdevice,
-		    name, dev_info->cfg->fw_name_pre, dev_info->cfg->trans.device_family,
-		    iwl_device_family_name(dev_info->cfg->trans.device_family));
+		    dev_info->name, dev_info->cfg->fw_name_pre);
 	}
 
 	error = sbuf_finish(sb);
diff --git a/sys/contrib/dev/iwlwifi/pcie/internal.h b/sys/contrib/dev/iwlwifi/pcie/gen1_2/internal.h
index 27a7e0b5b3d5..f48aeebb151c 100644
--- a/sys/contrib/dev/iwlwifi/pcie/internal.h
+++ b/sys/contrib/dev/iwlwifi/pcie/gen1_2/internal.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2003-2015, 2018-2024 Intel Corporation
+ * Copyright (C) 2003-2015, 2018-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -22,7 +22,7 @@
 #include "iwl-io.h"
 #include "iwl-op-mode.h"
 #include "iwl-drv.h"
-#include "iwl-context-info.h"
+#include "pcie/iwl-context-info.h"
 
 /*
  * RX related structures and functions
@@ -39,7 +39,7 @@ struct iwl_host_cmd;
  * trans_pcie layer */
 
 /**
- * struct iwl_rx_mem_buffer
+ * struct iwl_rx_mem_buffer - driver-side RX buffer descriptor
  * @page_dma: bus address of rxb page
  * @page: driver's pointer to the rxb page
  * @list: list entry for the membuffer
@@ -190,11 +190,12 @@ struct iwl_rb_allocator {
  * iwl_get_closed_rb_stts - get closed rb stts from different structs
  * @trans: transport pointer (for configuration)
  * @rxq: the rxq to get the rb stts from
+ * Return: last closed RB index
  */
 static inline u16 iwl_get_closed_rb_stts(struct iwl_trans *trans,
 					 struct iwl_rxq *rxq)
 {
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		__le16 *rb_stts = rxq->rb_stts;
 
 		return le16_to_cpu(READ_ONCE(*rb_stts));
@@ -269,6 +270,7 @@ enum iwl_pcie_fw_reset_state {
 	FW_RESET_REQUESTED,
 	FW_RESET_OK,
 	FW_RESET_ERROR,
+	FW_RESET_TOP_REQUESTED,
 };
 
 /**
@@ -288,22 +290,14 @@ enum iwl_pcie_imr_status {
 /**
  * struct iwl_pcie_txqs - TX queues data
  *
- * @bc_table_dword: true if the BC table expects DWORD (as opposed to bytes)
- * @page_offs: offset from skb->cb to mac header page pointer
- * @dev_cmd_offs: offset from skb->cb to iwl_device_tx_cmd pointer
  * @queue_used: bit mask of used queues
  * @queue_stopped: bit mask of stopped queues
  * @txq: array of TXQ data structures representing the TXQs
  * @scd_bc_tbls: gen1 pointer to the byte count table of the scheduler
- * @queue_alloc_cmd_ver: queue allocation command version
  * @bc_pool: bytecount DMA allocations pool
  * @bc_tbl_size: bytecount table size
  * @tso_hdr_page: page allocated (per CPU) for A-MSDU headers when doing TSO
  *	(and similar usage)
- * @cmd: command queue data
- * @cmd.fifo: FIFO number
- * @cmd.q_id: queue ID
- * @cmd.wdg_timeout: watchdog timeout
  * @tfd: TFD data
  * @tfd.max_tbs: max number of buffers per TFD
  * @tfd.size: TFD size
@@ -315,26 +309,15 @@ struct iwl_pcie_txqs {
 	struct iwl_txq *txq[IWL_MAX_TVQM_QUEUES];
 	struct dma_pool *bc_pool;
 	size_t bc_tbl_size;
-	bool bc_table_dword;
-	u8 page_offs;
-	u8 dev_cmd_offs;
 	struct iwl_tso_hdr_page __percpu *tso_hdr_page;
 
 	struct {
-		u8 fifo;
-		u8 q_id;
-		unsigned int wdg_timeout;
-	} cmd;
-
-	struct {
 		u8 max_tbs;
 		u16 size;
 		u8 addr_size;
 	} tfd;
 
 	struct iwl_dma_ptr scd_bc_tbls;
-
-	u8 queue_alloc_cmd_ver;
 };
 
 /**
@@ -344,7 +327,7 @@ struct iwl_pcie_txqs {
  * @global_table: table mapping received VID from hw to rxb
  * @rba: allocator for RX replenishing
  * @ctxt_info: context information for FW self init
- * @ctxt_info_gen3: context information for gen3 devices
+ * @ctxt_info_v2: context information for v1 devices
  * @prph_info: prph info for self init
  * @prph_scratch: prph scratch for self init
  * @ctxt_info_dma_addr: dma addr of context information
@@ -352,6 +335,7 @@ struct iwl_pcie_txqs {
  * @prph_scratch_dma_addr: dma addr of prph scratch
  * @ctxt_info_dma_addr: dma addr of context information
  * @iml: image loader image virtual address
+ * @iml_len: image loader image size
  * @iml_dma_addr: image loader image DMA address
  * @trans: pointer to the generic transport area
  * @scd_base_addr: scheduler sram base address in SRAM
@@ -363,9 +347,6 @@ struct iwl_pcie_txqs {
  * @hw_base: pci hardware address support
  * @ucode_write_complete: indicates that the ucode has been copied.
  * @ucode_write_waitq: wait queue for uCode load
- * @cmd_queue - command queue number
- * @rx_buf_size: Rx buffer size
- * @scd_set_active: should the transport configure the SCD for HCMD queue
  * @rx_page_order: page order for receive buffer size
  * @rx_buf_bytes: RX buffer (RB) size in bytes
  * @reg_lock: protect hw register access
@@ -402,23 +383,23 @@ struct iwl_pcie_txqs {
  * @irq_lock: lock to synchronize IRQ handling
  * @txq_memory: TXQ allocation array
  * @sx_waitq: waitqueue for Sx transitions
- * @sx_complete: completion for Sx transitions
- * @pcie_dbg_dumped_once: indicates PCIe regs were dumped already
+ * @sx_state: state tracking Sx transitions
  * @opmode_down: indicates opmode went away
  * @num_rx_bufs: number of RX buffers to allocate/use
- * @no_reclaim_cmds: special commands not using reclaim flow
- *	(firmware workaround)
- * @n_no_reclaim_cmds: number of special commands not using reclaim flow
  * @affinity_mask: IRQ affinity mask for each RX queue
  * @debug_rfkill: RF-kill debugging state, -1 for unset, 0/1 for radio
  *	enable/disable
- * @fw_reset_handshake: indicates FW reset handshake is needed
  * @fw_reset_state: state of FW reset handshake
  * @fw_reset_waitq: waitqueue for FW reset handshake
  * @is_down: indicates the NIC is down
  * @isr_stats: interrupt statistics
  * @napi_dev: (fake) netdev for NAPI registration
  * @txqs: transport tx queues data.
+ * @me_present: WiAMT/CSME is detected as present (1), not present (0)
+ *	or unknown (-1, so can still use it as a boolean safely)
+ * @me_recheck_wk: worker to recheck WiAMT/CSME presence
+ * @invalid_tx_cmd: invalid TX command buffer
+ * @wait_command_queue: wait queue for sync commands
  */
 struct iwl_trans_pcie {
 	struct iwl_rxq *rxq;
@@ -427,11 +408,12 @@ struct iwl_trans_pcie {
 	struct iwl_rb_allocator rba;
 	union {
 		struct iwl_context_info *ctxt_info;
-		struct iwl_context_info_gen3 *ctxt_info_gen3;
+		struct iwl_context_info_v2 *ctxt_info_v2;
 	};
 	struct iwl_prph_info *prph_info;
 	struct iwl_prph_scratch *prph_scratch;
 	void *iml;
+	size_t iml_len;
 	dma_addr_t ctxt_info_dma_addr;
 	dma_addr_t prph_info_dma_addr;
 	dma_addr_t prph_scratch_dma_addr;
@@ -466,17 +448,17 @@ struct iwl_trans_pcie {
 	u8 __iomem *hw_base;
 
 	bool ucode_write_complete;
-	bool sx_complete;
+	enum {
+		IWL_SX_INVALID = 0,
+		IWL_SX_WAITING,
+		IWL_SX_ERROR,
+		IWL_SX_COMPLETE,
+	} sx_state;
 	wait_queue_head_t ucode_write_waitq;
 	wait_queue_head_t sx_waitq;
 
-	u8 n_no_reclaim_cmds;
-	u8 no_reclaim_cmds[MAX_NO_RECLAIM_CMDS];
 	u16 num_rx_bufs;
 
-	enum iwl_amsdu_size rx_buf_size;
-	bool scd_set_active;
-	bool pcie_dbg_dumped_once;
 	u32 rx_page_order;
 	u32 rx_buf_bytes;
 	u32 supported_dma_mask;
@@ -510,7 +492,6 @@ struct iwl_trans_pcie {
 	void *base_rb_stts;
 	dma_addr_t base_rb_stts_dma;
 
-	bool fw_reset_handshake;
 	enum iwl_pcie_fw_reset_state fw_reset_state;
 	wait_queue_head_t fw_reset_waitq;
 	enum iwl_pcie_imr_status imr_status;
@@ -518,6 +499,13 @@ struct iwl_trans_pcie {
 	char rf_name[32];
 
 	struct iwl_pcie_txqs txqs;
+
+	s8 me_present;
+	struct delayed_work me_recheck_wk;
+
+	struct iwl_dma_ptr invalid_tx_cmd;
+
+	wait_queue_head_t wait_command_queue;
 };
 
 static inline struct iwl_trans_pcie *
@@ -550,18 +538,17 @@ iwl_trans_pcie_get_trans(struct iwl_trans_pcie *trans_pcie)
  * Convention: trans API functions: iwl_trans_pcie_XXX
  *	Other functions: iwl_pcie_XXX
  */
-struct iwl_trans
-*iwl_trans_pcie_alloc(struct pci_dev *pdev,
-		      const struct pci_device_id *ent,
-		      const struct iwl_cfg_trans_params *cfg_trans);
 void iwl_trans_pcie_free(struct iwl_trans *trans);
 void iwl_trans_pcie_free_pnvm_dram_regions(struct iwl_dram_regions *dram_regions,
 					   struct device *dev);
 
-bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans);
-#define _iwl_trans_pcie_grab_nic_access(trans)			\
+bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent);
+#define _iwl_trans_pcie_grab_nic_access(trans, silent)		\
 	__cond_lock(nic_access_nobh,				\
-		    likely(__iwl_trans_pcie_grab_nic_access(trans)))
+		    likely(__iwl_trans_pcie_grab_nic_access(trans, silent)))
+
+void iwl_trans_pcie_check_product_reset_status(struct pci_dev *pdev);
+void iwl_trans_pcie_check_product_reset_mode(struct pci_dev *pdev);
 
 /*****************************************************
 * RX
@@ -620,7 +607,7 @@ struct iwl_tso_page_info {
 				      IWL_TSO_PAGE_DATA_SIZE))
 
 int iwl_pcie_tx_init(struct iwl_trans *trans);
-void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr);
+void iwl_pcie_tx_start(struct iwl_trans *trans);
 int iwl_pcie_tx_stop(struct iwl_trans *trans);
 void iwl_pcie_tx_free(struct iwl_trans *trans);
 bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int queue, u16 ssn,
@@ -643,7 +630,8 @@ dma_addr_t iwl_pcie_get_sgt_tb_phys(struct sg_table *sgt, unsigned int offset,
 				    unsigned int len);
 struct sg_table *iwl_pcie_prep_tso(struct iwl_trans *trans, struct sk_buff *skb,
 				   struct iwl_cmd_meta *cmd_meta,
-				   u8 **hdr, unsigned int hdr_room);
+				   u8 **hdr, unsigned int hdr_room,
+				   unsigned int offset);
 
 void iwl_pcie_free_tso_pages(struct iwl_trans *trans, struct sk_buff *skb,
 			     struct iwl_cmd_meta *cmd_meta);
@@ -675,7 +663,7 @@ static inline void *iwl_txq_get_tfd(struct iwl_trans *trans,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
-	if (trans->trans_cfg->gen2)
+	if (trans->mac_cfg->gen2)
 		idx = iwl_txq_get_cmd_index(txq, idx);
 
 	return (u8 *)txq->tfds + trans_pcie->txqs.tfd.size * idx;
@@ -710,22 +698,24 @@ static inline void iwl_txq_stop(struct iwl_trans *trans, struct iwl_txq *txq)
  * iwl_txq_inc_wrap - increment queue index, wrap back to beginning
  * @trans: the transport (for configuration data)
  * @index: current index
+ * Return: the queue index incremented, subject to wrapping
  */
 static inline int iwl_txq_inc_wrap(struct iwl_trans *trans, int index)
 {
 	return ++index &
-		(trans->trans_cfg->base_params->max_tfd_queue_size - 1);
+		(trans->mac_cfg->base->max_tfd_queue_size - 1);
 }
 
 /**
  * iwl_txq_dec_wrap - decrement queue index, wrap back to end
  * @trans: the transport (for configuration data)
  * @index: current index
+ * Return: the queue index decremented, subject to wrapping
  */
 static inline int iwl_txq_dec_wrap(struct iwl_trans *trans, int index)
 {
 	return --index &
-		(trans->trans_cfg->base_params->max_tfd_queue_size - 1);
+		(trans->mac_cfg->base->max_tfd_queue_size - 1);
 }
 
 void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq);
@@ -748,10 +738,12 @@ int iwl_txq_gen2_set_tb(struct iwl_trans *trans,
 static inline void iwl_txq_set_tfd_invalid_gen2(struct iwl_trans *trans,
 						struct iwl_tfh_tfd *tfd)
 {
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
 	tfd->num_tbs = 0;
 
-	iwl_txq_gen2_set_tb(trans, tfd, trans->invalid_tx_cmd.dma,
-			    trans->invalid_tx_cmd.size);
+	iwl_txq_gen2_set_tb(trans, tfd, trans_pcie->invalid_tx_cmd.dma,
+			    trans_pcie->invalid_tx_cmd.size);
 }
 
 void iwl_txq_gen2_tfd_unmap(struct iwl_trans *trans,
@@ -778,7 +770,7 @@ static inline u16 iwl_txq_gen1_tfd_tb_get_len(struct iwl_trans *trans,
 	struct iwl_tfd *tfd;
 	struct iwl_tfd_tb *tb;
 
-	if (trans->trans_cfg->gen2) {
+	if (trans->mac_cfg->gen2) {
 		struct iwl_tfh_tfd *tfh_tfd = _tfd;
 		struct iwl_tfh_tb *tfh_tb = &tfh_tfd->tbs[idx];
 
@@ -936,11 +928,13 @@ static inline void iwl_enable_fw_load_int(struct iwl_trans *trans)
 	}
 }
 
-static inline void iwl_enable_fw_load_int_ctx_info(struct iwl_trans *trans)
+static inline void iwl_enable_fw_load_int_ctx_info(struct iwl_trans *trans,
+						   bool top_reset)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
-	IWL_DEBUG_ISR(trans, "Enabling ALIVE interrupt only\n");
+	IWL_DEBUG_ISR(trans, "Enabling %s interrupt only\n",
+		      top_reset ? "RESET" : "ALIVE");
 
 	if (!trans_pcie->msix_enabled) {
 		/*
@@ -950,11 +944,20 @@ static inline void iwl_enable_fw_load_int_ctx_info(struct iwl_trans *trans)
 		 * RX interrupt which will allow us to receive the ALIVE
 		 * notification (which is Rx) and continue the flow.
 		 */
-		trans_pcie->inta_mask =  CSR_INT_BIT_ALIVE | CSR_INT_BIT_FH_RX;
+		if (top_reset)
+			trans_pcie->inta_mask =  CSR_INT_BIT_RESET_DONE;
+		else
+			trans_pcie->inta_mask =  CSR_INT_BIT_ALIVE |
+						 CSR_INT_BIT_FH_RX;
 		iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
 	} else {
-		iwl_enable_hw_int_msk_msix(trans,
-					   MSIX_HW_INT_CAUSES_REG_ALIVE);
+		u32 val = top_reset ? MSIX_HW_INT_CAUSES_REG_RESET_DONE
+				    : MSIX_HW_INT_CAUSES_REG_ALIVE;
+
+		iwl_enable_hw_int_msk_msix(trans, val);
+
+		if (top_reset)
+			return;
 		/*
 		 * Leave all the FH causes enabled to get the ALIVE
 		 * notification.
@@ -1001,7 +1004,7 @@ static inline void iwl_enable_rfkill_int(struct iwl_trans *trans)
 					   MSIX_HW_INT_CAUSES_REG_RF_KILL);
 	}
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_9000) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_9000) {
 		/*
 		 * On 9000-series devices this bit isn't enabled by default, so
 		 * when we power down the device we need set the bit to allow it
@@ -1027,40 +1030,12 @@ static inline bool iwl_is_rfkill_set(struct iwl_trans *trans)
 		CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW);
 }
 
-static inline void __iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans,
-						  u32 reg, u32 mask, u32 value)
-{
-	u32 v;
-
-#ifdef CONFIG_IWLWIFI_DEBUG
-	WARN_ON_ONCE(value & ~mask);
-#endif
-
-	v = iwl_read32(trans, reg);
-	v &= ~mask;
-	v |= value;
-	iwl_write32(trans, reg, v);
-}
-
-static inline void __iwl_trans_pcie_clear_bit(struct iwl_trans *trans,
-					      u32 reg, u32 mask)
-{
-	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, 0);
-}
-
-static inline void __iwl_trans_pcie_set_bit(struct iwl_trans *trans,
-					    u32 reg, u32 mask)
-{
-	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, mask);
-}
-
 static inline bool iwl_pcie_dbg_on(struct iwl_trans *trans)
 {
 	return (trans->dbg.dest_tlv || iwl_trans_dbg_ini_valid(trans));
 }
 
 void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state, bool from_irq);
-void iwl_trans_pcie_dump_regs(struct iwl_trans *trans);
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans);
@@ -1072,8 +1047,8 @@ static inline void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans) { }
 void iwl_pcie_rx_allocator_work(struct work_struct *data);
 
 /* common trans ops for all generations transports */
-void iwl_trans_pcie_configure(struct iwl_trans *trans,
-			      const struct iwl_trans_config *trans_cfg);
+void iwl_trans_pcie_op_mode_enter(struct iwl_trans *trans);
+int _iwl_trans_pcie_start_hw(struct iwl_trans *trans);
 int iwl_trans_pcie_start_hw(struct iwl_trans *trans);
 void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans);
 void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val);
@@ -1083,8 +1058,6 @@ u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg);
 void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr, u32 val);
 int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
 			    void *buf, int dwords);
-int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
-			     const void *buf, int dwords);
 int iwl_trans_pcie_sw_reset(struct iwl_trans *trans, bool retake_ownership);
 struct iwl_trans_dump_data *
 iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
@@ -1101,15 +1074,24 @@ void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
 int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
 				 u32 *val);
 bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans);
-void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans);
+void __releases(nic_access_nobh)
+iwl_trans_pcie_release_nic_access(struct iwl_trans *trans);
+void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power);
+int iwl_pci_gen1_2_probe(struct pci_dev *pdev,
+			 const struct pci_device_id *ent,
+			 const struct iwl_mac_cfg *mac_cfg,
+			 u8 __iomem *hw_base, u32 hw_rev);
 
 /* transport gen 1 exported functions */
-void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr);
+void iwl_trans_pcie_fw_alive(struct iwl_trans *trans);
 int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
-			    const struct fw_img *fw, bool run_in_rfkill);
+			    const struct iwl_fw *fw,
+			    const struct fw_img *img,
+			    bool run_in_rfkill);
 void iwl_trans_pcie_stop_device(struct iwl_trans *trans);
 
 /* common functions that are used by gen2 transport */
+void iwl_trans_pcie_gen2_op_mode_leave(struct iwl_trans *trans);
 int iwl_pcie_gen2_apm_init(struct iwl_trans *trans);
 void iwl_pcie_apm_config(struct iwl_trans *trans);
 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans);
@@ -1124,19 +1106,13 @@ int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans,
 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr);
 void iwl_pcie_apply_destination(struct iwl_trans *trans);
 
-/* common functions that are used by gen3 transport */
-void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power);
-
 /* transport gen 2 exported functions */
 int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
-				 const struct fw_img *fw, bool run_in_rfkill);
+				 const struct iwl_fw *fw,
+				 const struct fw_img *img,
+				 bool run_in_rfkill);
 void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans);
-int iwl_trans_pcie_gen2_send_hcmd(struct iwl_trans *trans,
-				  struct iwl_host_cmd *cmd);
 void iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans);
-void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans);
-void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
-				  bool test, bool reset);
 int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 			       struct iwl_host_cmd *cmd);
 int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
diff --git a/sys/contrib/dev/iwlwifi/pcie/rx.c b/sys/contrib/dev/iwlwifi/pcie/gen1_2/rx.c
index 6c7fc89ed14e..340bc56ae842 100644
--- a/sys/contrib/dev/iwlwifi/pcie/rx.c
+++ b/sys/contrib/dev/iwlwifi/pcie/gen1_2/rx.c
@@ -12,7 +12,8 @@
 #include "iwl-io.h"
 #include "internal.h"
 #include "iwl-op-mode.h"
-#include "iwl-context-info-gen3.h"
+#include "pcie/iwl-context-info-v2.h"
+#include "fw/dbg.h"
 
 /******************************************************************************
  *
@@ -143,12 +144,12 @@ static inline __le32 iwl_pcie_dma_addr2rbd_ptr(dma_addr_t dma_addr)
  */
 int iwl_pcie_rx_stop(struct iwl_trans *trans)
 {
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		/* TODO: remove this once fw does it */
-		iwl_write_umac_prph(trans, RFH_RXF_DMA_CFG_GEN3, 0);
-		return iwl_poll_umac_prph_bit(trans, RFH_GEN_STATUS_GEN3,
+		iwl_write_umac_prph(trans, RFH_RXF_DMA_CFG_AX210, 0);
+		return iwl_poll_umac_prph_bit(trans, RFH_GEN_STATUS_AX210,
 					      RXF_DMA_IDLE, RXF_DMA_IDLE, 1000);
-	} else if (trans->trans_cfg->mq_rx_supported) {
+	} else if (trans->mac_cfg->mq_rx_supported) {
 		iwl_write_prph(trans, RFH_RXF_DMA_CFG, 0);
 		return iwl_poll_prph_bit(trans, RFH_GEN_STATUS,
 					   RXF_DMA_IDLE, RXF_DMA_IDLE, 1000);
@@ -175,7 +176,7 @@ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans,
 	 * 1. shadow registers aren't enabled
 	 * 2. there is a chance that the NIC is asleep
 	 */
-	if (!trans->trans_cfg->base_params->shadow_reg_enable &&
+	if (!trans->mac_cfg->base->shadow_reg_enable &&
 	    test_bit(STATUS_TPOWER_PMI, &trans->status)) {
 		reg = iwl_read32(trans, CSR_UCODE_DRV_GP1);
 
@@ -190,9 +191,9 @@ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans,
 	}
 
 	rxq->write_actual = round_down(rxq->write, 8);
-	if (!trans->trans_cfg->mq_rx_supported)
+	if (!trans->mac_cfg->mq_rx_supported)
 		iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual);
-	else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	else if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		iwl_write32(trans, HBUS_TARG_WRPTR, rxq->write_actual |
 			    HBUS_TARG_WRPTR_RX_Q(rxq->id));
 	else
@@ -205,7 +206,7 @@ static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans)
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int i;
 
-	for (i = 0; i < trans->num_rx_queues; i++) {
+	for (i = 0; i < trans->info.num_rxqs; i++) {
 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
 
 		if (!rxq->need_update)
@@ -221,7 +222,7 @@ static void iwl_pcie_restock_bd(struct iwl_trans *trans,
 				struct iwl_rxq *rxq,
 				struct iwl_rx_mem_buffer *rxb)
 {
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		struct iwl_rx_transfer_desc *bd = rxq->bd;
 
 		BUILD_BUG_ON(sizeof(*bd) != 2 * sizeof(u64));
@@ -234,12 +235,8 @@ static void iwl_pcie_restock_bd(struct iwl_trans *trans,
 		bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid);
 	}
 
-#if defined(__linux__)
 	IWL_DEBUG_RX(trans, "Assigned virtual RB ID %u to queue %d index %d\n",
-#elif defined(__FreeBSD__)
-	IWL_DEBUG_PCI_RW(trans, "Assigned virtual RB ID %u to queue %d index %d\n",
 		     (u32)rxb->vid, rxq->id, rxq->write);
-#endif
 }
 
 /*
@@ -352,7 +349,7 @@ static void iwl_pcie_rxsq_restock(struct iwl_trans *trans,
 static
 void iwl_pcie_rxq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq)
 {
-	if (trans->trans_cfg->mq_rx_supported)
+	if (trans->mac_cfg->mq_rx_supported)
 		iwl_pcie_rxmq_restock(trans, rxq);
 	else
 		iwl_pcie_rxsq_restock(trans, rxq);
@@ -366,8 +363,8 @@ static struct page *iwl_pcie_rx_alloc_page(struct iwl_trans *trans,
 					   u32 *offset, gfp_t priority)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	unsigned int rbsize = iwl_trans_get_rb_size(trans_pcie->rx_buf_size);
 	unsigned int allocsize = PAGE_SIZE << trans_pcie->rx_page_order;
+	unsigned int rbsize = trans_pcie->rx_buf_bytes;
 	struct page *page;
 	gfp_t gfp_mask = priority;
 
@@ -661,19 +658,19 @@ void iwl_pcie_rx_allocator_work(struct work_struct *data)
 
 static int iwl_pcie_free_bd_size(struct iwl_trans *trans)
 {
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		return sizeof(struct iwl_rx_transfer_desc);
 
-	return trans->trans_cfg->mq_rx_supported ?
+	return trans->mac_cfg->mq_rx_supported ?
 			sizeof(__le64) : sizeof(__le32);
 }
 
 static int iwl_pcie_used_bd_size(struct iwl_trans *trans)
 {
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		return sizeof(struct iwl_rx_completion_desc_bz);
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		return sizeof(struct iwl_rx_completion_desc);
 
 	return sizeof(__le32);
@@ -705,7 +702,7 @@ static void iwl_pcie_free_rxq_dma(struct iwl_trans *trans,
 
 static size_t iwl_pcie_rb_stts_size(struct iwl_trans *trans)
 {
-	bool use_rx_td = (trans->trans_cfg->device_family >=
+	bool use_rx_td = (trans->mac_cfg->device_family >=
 			  IWL_DEVICE_FAMILY_AX210);
 
 	if (use_rx_td)
@@ -724,8 +721,8 @@ static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans,
 	int free_size;
 
 	spin_lock_init(&rxq->lock);
-	if (trans->trans_cfg->mq_rx_supported)
-		rxq->queue_size = trans->cfg->num_rbds;
+	if (trans->mac_cfg->mq_rx_supported)
+		rxq->queue_size = iwl_trans_get_num_rbds(trans);
 	else
 		rxq->queue_size = RX_QUEUE_SIZE;
 
@@ -740,7 +737,7 @@ static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans,
 	if (!rxq->bd)
 		goto err;
 
-	if (trans->trans_cfg->mq_rx_supported) {
+	if (trans->mac_cfg->mq_rx_supported) {
 		rxq->used_bd = dma_alloc_coherent(dev,
 						  iwl_pcie_used_bd_size(trans) *
 							rxq->queue_size,
@@ -757,7 +754,7 @@ static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans,
 	return 0;
 
 err:
-	for (i = 0; i < trans->num_rx_queues; i++) {
+	for (i = 0; i < trans->info.num_rxqs; i++) {
 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
 
 		iwl_pcie_free_rxq_dma(trans, rxq);
@@ -776,7 +773,7 @@ static int iwl_pcie_rx_alloc(struct iwl_trans *trans)
 	if (WARN_ON(trans_pcie->rxq))
 		return -EINVAL;
 
-	trans_pcie->rxq = kcalloc(trans->num_rx_queues, sizeof(struct iwl_rxq),
+	trans_pcie->rxq = kcalloc(trans->info.num_rxqs, sizeof(struct iwl_rxq),
 				  GFP_KERNEL);
 	trans_pcie->rx_pool = kcalloc(RX_POOL_SIZE(trans_pcie->num_rx_bufs),
 				      sizeof(trans_pcie->rx_pool[0]),
@@ -799,7 +796,7 @@ static int iwl_pcie_rx_alloc(struct iwl_trans *trans)
 	 */
 	trans_pcie->base_rb_stts =
 			dma_alloc_coherent(trans->dev,
-					   rb_stts_size * trans->num_rx_queues,
+					   rb_stts_size * trans->info.num_rxqs,
 					   &trans_pcie->base_rb_stts_dma,
 					   GFP_KERNEL);
 	if (!trans_pcie->base_rb_stts) {
@@ -807,7 +804,7 @@ static int iwl_pcie_rx_alloc(struct iwl_trans *trans)
 		goto err;
 	}
 
-	for (i = 0; i < trans->num_rx_queues; i++) {
+	for (i = 0; i < trans->info.num_rxqs; i++) {
 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
 
 		rxq->id = i;
@@ -820,7 +817,7 @@ static int iwl_pcie_rx_alloc(struct iwl_trans *trans)
 err:
 	if (trans_pcie->base_rb_stts) {
 		dma_free_coherent(trans->dev,
-				  rb_stts_size * trans->num_rx_queues,
+				  rb_stts_size * trans->info.num_rxqs,
 				  trans_pcie->base_rb_stts,
 				  trans_pcie->base_rb_stts_dma);
 		trans_pcie->base_rb_stts = NULL;
@@ -838,11 +835,10 @@ err:
 
 static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq)
 {
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	u32 rb_size;
 	const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */
 
-	switch (trans_pcie->rx_buf_size) {
+	switch (trans->conf.rx_buf_size) {
 	case IWL_AMSDU_4K:
 		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K;
 		break;
@@ -910,7 +906,7 @@ static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans)
 	u32 rb_size, enabled = 0;
 	int i;
 
-	switch (trans_pcie->rx_buf_size) {
+	switch (trans->conf.rx_buf_size) {
 	case IWL_AMSDU_2K:
 		rb_size = RFH_RXF_DMA_RB_SIZE_2K;
 		break;
@@ -936,7 +932,7 @@ static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans)
 	/* disable free amd used rx queue operation */
 	iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, 0);
 
-	for (i = 0; i < trans->num_rx_queues; i++) {
+	for (i = 0; i < trans->info.num_rxqs; i++) {
 		/* Tell device where to find RBD free table in DRAM */
 		iwl_write_prph64_no_grab(trans,
 					 RFH_Q_FRBDCB_BA_LSB(i),
@@ -980,7 +976,7 @@ static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans)
 			       RFH_GEN_CFG_VAL(DEFAULT_RXQ_NUM, 0) |
 			       RFH_GEN_CFG_SERVICE_DMA_SNOOP |
 			       RFH_GEN_CFG_VAL(RB_CHUNK_SIZE,
-					       trans->trans_cfg->integrated ?
+					       trans->mac_cfg->integrated ?
 					       RFH_GEN_CFG_RB_CHUNK_SIZE_64 :
 					       RFH_GEN_CFG_RB_CHUNK_SIZE_128));
 	/* Enable the relevant rx queues */
@@ -1076,7 +1072,7 @@ void iwl_pcie_rx_napi_sync(struct iwl_trans *trans)
 	if (unlikely(!trans_pcie->rxq))
 		return;
 
-	for (i = 0; i < trans->num_rx_queues; i++) {
+	for (i = 0; i < trans->info.num_rxqs; i++) {
 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
 
 		if (rxq && rxq->napi.poll)
@@ -1113,7 +1109,7 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans)
 	for (i = 0; i < RX_QUEUE_SIZE; i++)
 		def_rxq->queue[i] = NULL;
 
-	for (i = 0; i < trans->num_rx_queues; i++) {
+	for (i = 0; i < trans->info.num_rxqs; i++) {
 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
 
 		spin_lock_bh(&rxq->lock);
@@ -1126,7 +1122,7 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans)
 		rxq->write = 0;
 		rxq->write_actual = 0;
 		memset(rxq->rb_stts, 0,
-		       (trans->trans_cfg->device_family >=
+		       (trans->mac_cfg->device_family >=
 			IWL_DEVICE_FAMILY_AX210) ?
 		       sizeof(__le16) : sizeof(struct iwl_rb_status));
 
@@ -1148,9 +1144,9 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans)
 	}
 
 	/* move the pool to the default queue and allocator ownerships */
-	queue_size = trans->trans_cfg->mq_rx_supported ?
+	queue_size = trans->mac_cfg->mq_rx_supported ?
 			trans_pcie->num_rx_bufs - 1 : RX_QUEUE_SIZE;
-	allocator_pool_size = trans->num_rx_queues *
+	allocator_pool_size = trans->info.num_rxqs *
 		(RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC);
 	num_alloc = queue_size + allocator_pool_size;
 
@@ -1179,7 +1175,7 @@ int iwl_pcie_rx_init(struct iwl_trans *trans)
 	if (ret)
 		return ret;
 
-	if (trans->trans_cfg->mq_rx_supported)
+	if (trans->mac_cfg->mq_rx_supported)
 		iwl_pcie_rx_mq_hw_init(trans);
 	else
 		iwl_pcie_rx_hw_init(trans, trans_pcie->rxq);
@@ -1227,14 +1223,14 @@ void iwl_pcie_rx_free(struct iwl_trans *trans)
 
 	if (trans_pcie->base_rb_stts) {
 		dma_free_coherent(trans->dev,
-				  rb_stts_size * trans->num_rx_queues,
+				  rb_stts_size * trans->info.num_rxqs,
 				  trans_pcie->base_rb_stts,
 				  trans_pcie->base_rb_stts_dma);
 		trans_pcie->base_rb_stts = NULL;
 		trans_pcie->base_rb_stts_dma = 0;
 	}
 
-	for (i = 0; i < trans->num_rx_queues; i++) {
+	for (i = 0; i < trans->info.num_rxqs; i++) {
 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
 
 		iwl_pcie_free_rxq_dma(trans, rxq);
@@ -1305,7 +1301,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
 				int i)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwl_txq *txq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
+	struct iwl_txq *txq = trans_pcie->txqs.txq[trans->conf.cmd_queue];
 	bool page_stolen = false;
 	int max_len = trans_pcie->rx_buf_bytes;
 	u32 offset = 0;
@@ -1372,8 +1368,8 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
 		if (reclaim && !pkt->hdr.group_id) {
 			int i;
 
-			for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) {
-				if (trans_pcie->no_reclaim_cmds[i] ==
+			for (i = 0; i < trans->conf.n_no_reclaim_cmds; i++) {
+				if (trans->conf.no_reclaim_cmds[i] ==
 							pkt->hdr.cmd) {
 					reclaim = false;
 					break;
@@ -1412,7 +1408,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
 		}
 
 		page_stolen |= rxcb._page_stolen;
-		if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+		if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 			break;
 	}
 
@@ -1458,18 +1454,18 @@ static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans,
 	BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc) != 32);
 	BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc_bz) != 4);
 
-	if (!trans->trans_cfg->mq_rx_supported) {
+	if (!trans->mac_cfg->mq_rx_supported) {
 		rxb = rxq->queue[i];
 		rxq->queue[i] = NULL;
 		return rxb;
 	}
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		struct iwl_rx_completion_desc_bz *cd = rxq->used_bd;
 
 		vid = le16_to_cpu(cd[i].rbid);
 		*join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED;
-	} else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	} else if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		struct iwl_rx_completion_desc *cd = rxq->used_bd;
 
 		vid = le16_to_cpu(cd[i].rbid);
@@ -1652,7 +1648,7 @@ irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id)
 
 	trace_iwlwifi_dev_irq_msix(trans->dev, entry, false, 0, 0);
 
-	if (WARN_ON(entry->entry >= trans->num_rx_queues))
+	if (WARN_ON(entry->entry >= trans->info.num_rxqs))
 		return IRQ_NONE;
 
 	if (!trans_pcie->rxq) {
@@ -1687,29 +1683,38 @@ static void iwl_pcie_irq_handle_error(struct iwl_trans *trans)
 
 	/* W/A for WiFi/WiMAX coex and WiMAX own the RF */
 	if (trans->cfg->internal_wimax_coex &&
-	    !trans->cfg->apmg_not_supported &&
+	    !trans->mac_cfg->base->apmg_not_supported &&
 	    (!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) &
 			     APMS_CLK_VAL_MRB_FUNC_MODE) ||
 	     (iwl_read_prph(trans, APMG_PS_CTRL_REG) &
 			    APMG_PS_CTRL_VAL_RESET_REQ))) {
 		clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
 		iwl_op_mode_wimax_active(trans->op_mode);
-		wake_up(&trans->wait_command_queue);
+		wake_up(&trans_pcie->wait_command_queue);
 		return;
 	}
 
-	for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) {
+	for (i = 0; i < trans->mac_cfg->base->num_of_queues; i++) {
 		if (!trans_pcie->txqs.txq[i])
 			continue;
-		del_timer(&trans_pcie->txqs.txq[i]->stuck_timer);
+		timer_delete(&trans_pcie->txqs.txq[i]->stuck_timer);
+	}
+
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC) {
+		u32 val = iwl_read32(trans, CSR_IPC_STATE);
+
+		if (val & CSR_IPC_STATE_TOP_RESET_REQ) {
+			IWL_ERR(trans, "FW requested TOP reset for FSEQ\n");
+			trans->do_top_reset = 1;
+		}
 	}
 
 	/* The STATUS_FW_ERROR bit is set in this function. This must happen
 	 * before we wake up the command caller, to ensure a proper cleanup. */
-	iwl_trans_fw_error(trans, false);
+	iwl_trans_fw_error(trans, IWL_ERR_TYPE_IRQ);
 
 	clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
-	wake_up(&trans->wait_command_queue);
+	wake_up(&trans_pcie->wait_command_queue);
 }
 
 static u32 iwl_pcie_int_cause_non_ict(struct iwl_trans *trans)
@@ -1824,7 +1829,7 @@ void iwl_pcie_handle_rfkill_irq(struct iwl_trans *trans, bool from_irq)
 				       &trans->status))
 			IWL_DEBUG_RF_KILL(trans,
 					  "Rfkill while SYNC HCMD in flight\n");
-		wake_up(&trans->wait_command_queue);
+		wake_up(&trans_pcie->wait_command_queue);
 	} else {
 		clear_bit(STATUS_RFKILL_HW, &trans->status);
 		if (trans_pcie->opmode_down)
@@ -1832,6 +1837,59 @@ void iwl_pcie_handle_rfkill_irq(struct iwl_trans *trans, bool from_irq)
 	}
 }
 
+static void iwl_trans_pcie_handle_reset_interrupt(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	u32 state;
+
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC) {
+		u32 val = iwl_read32(trans, CSR_IPC_STATE);
+
+		state = u32_get_bits(val, CSR_IPC_STATE_RESET);
+		IWL_DEBUG_ISR(trans, "IPC state = 0x%x/%d\n", val, state);
+	} else {
+		state = CSR_IPC_STATE_RESET_SW_READY;
+	}
+
+	switch (state) {
+	case CSR_IPC_STATE_RESET_SW_READY:
+		if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) {
+			IWL_DEBUG_ISR(trans, "Reset flow completed\n");
+			trans_pcie->fw_reset_state = FW_RESET_OK;
+			wake_up(&trans_pcie->fw_reset_waitq);
+			break;
+		}
+		fallthrough;
+	case CSR_IPC_STATE_RESET_TOP_READY:
+		if (trans_pcie->fw_reset_state == FW_RESET_TOP_REQUESTED) {
+			IWL_DEBUG_ISR(trans, "TOP Reset continues\n");
+			trans_pcie->fw_reset_state = FW_RESET_OK;
+			wake_up(&trans_pcie->fw_reset_waitq);
+			break;
+		}
+		fallthrough;
+	case CSR_IPC_STATE_RESET_NONE:
+		IWL_FW_CHECK_FAILED(trans,
+				    "Invalid reset interrupt (state=%d)!\n",
+				    state);
+		break;
+	case CSR_IPC_STATE_RESET_TOP_FOLLOWER:
+		if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) {
+			/* if we were in reset, wake that up */
+			IWL_INFO(trans,
+				 "TOP reset from BT while doing reset\n");
+			trans_pcie->fw_reset_state = FW_RESET_OK;
+			wake_up(&trans_pcie->fw_reset_waitq);
+		} else {
+			IWL_INFO(trans, "TOP reset from BT\n");
+			trans->state = IWL_TRANS_NO_FW;
+			iwl_trans_schedule_reset(trans,
+						 IWL_ERR_TYPE_TOP_RESET_BY_BT);
+		}
+		break;
+	}
+}
+
 irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
 {
 	struct iwl_trans *trans = dev_id;
@@ -1944,7 +2002,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
 	if (inta & CSR_INT_BIT_ALIVE) {
 		IWL_DEBUG_ISR(trans, "Alive interrupt\n");
 		isr_stats->alive++;
-		if (trans->trans_cfg->gen2) {
+		if (trans->mac_cfg->gen2) {
 			/*
 			 * We can restock, since firmware configured
 			 * the RFH
@@ -1955,6 +2013,11 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
 		handled |= CSR_INT_BIT_ALIVE;
 	}
 
+	if (inta & CSR_INT_BIT_RESET_DONE) {
+		iwl_trans_pcie_handle_reset_interrupt(trans);
+		handled |= CSR_INT_BIT_RESET_DONE;
+	}
+
 	/* Safely ignore these bits for debug checks below */
 	inta &= ~(CSR_INT_BIT_SCD | CSR_INT_BIT_ALIVE);
 
@@ -1976,7 +2039,12 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
 		IWL_ERR(trans, "Microcode SW error detected. "
 			" Restarting 0x%X.\n", inta);
 		isr_stats->sw++;
-		iwl_pcie_irq_handle_error(trans);
+		if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) {
+			trans_pcie->fw_reset_state = FW_RESET_ERROR;
+			wake_up(&trans_pcie->fw_reset_waitq);
+		} else {
+			iwl_pcie_irq_handle_error(trans);
+		}
 		handled |= CSR_INT_BIT_SW_ERR;
 	}
 
@@ -2082,7 +2150,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
 			iwl_enable_rfkill_int(trans);
 		/* Re-enable the ALIVE / Rx interrupt if it occurred */
 		else if (handled & (CSR_INT_BIT_ALIVE | CSR_INT_BIT_FH_RX))
-			iwl_enable_fw_load_int_ctx_info(trans);
+			iwl_enable_fw_load_int_ctx_info(trans, false);
 		spin_unlock_bh(&trans_pcie->irq_lock);
 	}
 
@@ -2299,7 +2367,7 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
 		}
 	}
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		sw_err = inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ;
 	else
 		sw_err = inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR;
@@ -2307,7 +2375,13 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_TOP_FATAL_ERR) {
 		IWL_ERR(trans, "TOP Fatal error detected, inta_hw=0x%x.\n",
 			inta_hw);
-		/* TODO: PLDR flow required here for >= Bz */
+		if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+			trans->request_top_reset = 1;
+			iwl_op_mode_nic_error(trans->op_mode,
+					      IWL_ERR_TYPE_TOP_FATAL_ERROR);
+			iwl_trans_schedule_reset(trans,
+						 IWL_ERR_TYPE_TOP_FATAL_ERROR);
+		}
 	}
 
 	/* Error detected by uCode */
@@ -2326,6 +2400,11 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
 		} else {
 			iwl_pcie_irq_handle_error(trans);
 		}
+
+		if (trans_pcie->sx_state == IWL_SX_WAITING) {
+			trans_pcie->sx_state = IWL_SX_ERROR;
+			wake_up(&trans_pcie->sx_waitq);
+		}
 	}
 
 	/* After checking FH register check HW register */
@@ -2348,7 +2427,7 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_ALIVE) {
 		IWL_DEBUG_ISR(trans, "Alive interrupt\n");
 		isr_stats->alive++;
-		if (trans->trans_cfg->gen2) {
+		if (trans->mac_cfg->gen2) {
 			/* We can restock, since firmware configured the RFH */
 			iwl_pcie_rxmq_restock(trans, trans_pcie->rxq);
 		}
@@ -2362,13 +2441,20 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP && trans_pcie->prph_info) {
 		u32 sleep_notif =
 			le32_to_cpu(trans_pcie->prph_info->sleep_notif);
+
 		if (sleep_notif == IWL_D3_SLEEP_STATUS_SUSPEND ||
 		    sleep_notif == IWL_D3_SLEEP_STATUS_RESUME) {
 			IWL_DEBUG_ISR(trans,
 				      "Sx interrupt: sleep notification = 0x%x\n",
 				      sleep_notif);
-			trans_pcie->sx_complete = true;
-			wake_up(&trans_pcie->sx_waitq);
+			if (trans_pcie->sx_state == IWL_SX_WAITING) {
+				trans_pcie->sx_state = IWL_SX_COMPLETE;
+				wake_up(&trans_pcie->sx_waitq);
+			} else {
+				IWL_ERR(trans,
+					"unexpected Sx interrupt (0x%x)\n",
+					sleep_notif);
+			}
 		} else {
 			/* uCode wakes up after power-down sleep */
 			IWL_DEBUG_ISR(trans, "Wakeup interrupt\n");
@@ -2398,11 +2484,8 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
 		iwl_pcie_irq_handle_error(trans);
 	}
 
-	if (inta_hw & MSIX_HW_INT_CAUSES_REG_RESET_DONE) {
-		IWL_DEBUG_ISR(trans, "Reset flow completed\n");
-		trans_pcie->fw_reset_state = FW_RESET_OK;
-		wake_up(&trans_pcie->fw_reset_waitq);
-	}
+	if (inta_hw & MSIX_HW_INT_CAUSES_REG_RESET_DONE)
+		iwl_trans_pcie_handle_reset_interrupt(trans);
 
 	if (!polling)
 		iwl_pcie_clear_irq(trans, entry->entry);
diff --git a/sys/contrib/dev/iwlwifi/pcie/trans-gen2.c b/sys/contrib/dev/iwlwifi/pcie/gen1_2/trans-gen2.c
index 96127accc0a0..b27e58c1a00b 100644
--- a/sys/contrib/dev/iwlwifi/pcie/trans-gen2.c
+++ b/sys/contrib/dev/iwlwifi/pcie/gen1_2/trans-gen2.c
@@ -1,15 +1,15 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2025 Intel Corporation
  */
 #if defined(__FreeBSD__)
 #include <linux/delay.h>
 #endif
 #include "iwl-trans.h"
 #include "iwl-prph.h"
-#include "iwl-context-info.h"
-#include "iwl-context-info-gen3.h"
+#include "pcie/iwl-context-info.h"
+#include "pcie/iwl-context-info-v2.h"
 #include "internal.h"
 #include "fw/dbg.h"
 
@@ -46,7 +46,7 @@ int iwl_pcie_gen2_apm_init(struct iwl_trans *trans)
 	 * wake device's PCI Express link L1a -> L0s
 	 */
 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-		    CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
+		    CSR_HW_IF_CONFIG_REG_HAP_WAKE);
 
 	iwl_pcie_apm_config(trans);
 
@@ -71,8 +71,8 @@ static void iwl_pcie_gen2_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 		iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
 			    CSR_RESET_LINK_PWR_MGMT_DISABLED);
 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-			    CSR_HW_IF_CONFIG_REG_PREPARE |
-			    CSR_HW_IF_CONFIG_REG_ENABLE_PME);
+			    CSR_HW_IF_CONFIG_REG_WAKE_ME |
+			    CSR_HW_IF_CONFIG_REG_WAKE_ME_PCIE_OWNER_EN);
 		mdelay(1);
 		iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
 			      CSR_RESET_LINK_PWR_MGMT_DISABLED);
@@ -84,13 +84,13 @@ static void iwl_pcie_gen2_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 	/* Stop device's DMA activity */
 	iwl_pcie_apm_stop_master(trans);
 
-	iwl_trans_sw_reset(trans, false);
+	iwl_trans_pcie_sw_reset(trans, false);
 
 	/*
 	 * Clear "initialization complete" bit to move adapter from
 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
 	 */
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		iwl_clear_bit(trans, CSR_GP_CNTRL,
 			      CSR_GP_CNTRL_REG_FLAG_MAC_INIT);
 	else
@@ -98,17 +98,17 @@ static void iwl_pcie_gen2_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 			      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
 }
 
-static void iwl_trans_pcie_fw_reset_handshake(struct iwl_trans *trans)
+void iwl_trans_pcie_fw_reset_handshake(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int ret;
 
 	trans_pcie->fw_reset_state = FW_RESET_REQUESTED;
 
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
 				    UREG_NIC_SET_NMI_DRIVER_RESET_HANDSHAKE);
-	else if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210)
+	else if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_AX210)
 		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
 				    UREG_DOORBELL_TO_ISR6_RESET_HANDSHAKE);
 	else
@@ -120,20 +120,37 @@ static void iwl_trans_pcie_fw_reset_handshake(struct iwl_trans *trans)
 				 trans_pcie->fw_reset_state != FW_RESET_REQUESTED,
 				 FW_RESET_TIMEOUT);
 	if (!ret || trans_pcie->fw_reset_state == FW_RESET_ERROR) {
-		u32 inta_hw = iwl_read32(trans, CSR_MSIX_HW_INT_CAUSES_AD);
+		bool reset_done;
+		u32 inta_hw;
+
+		if (trans_pcie->msix_enabled) {
+			inta_hw = iwl_read32(trans, CSR_MSIX_HW_INT_CAUSES_AD);
+			reset_done =
+				inta_hw & MSIX_HW_INT_CAUSES_REG_RESET_DONE;
+		} else {
+			inta_hw = iwl_read32(trans, CSR_INT);
+			reset_done = inta_hw & CSR_INT_BIT_RESET_DONE;
+		}
 
 		IWL_ERR(trans,
-			"timeout waiting for FW reset ACK (inta_hw=0x%x)\n",
-			inta_hw);
-
-		if (!(inta_hw & MSIX_HW_INT_CAUSES_REG_RESET_DONE))
-			iwl_trans_fw_error(trans, true);
+			"timeout waiting for FW reset ACK (inta_hw=0x%x, reset_done %d)\n",
+			inta_hw, reset_done);
+
+		if (!reset_done) {
+			struct iwl_fw_error_dump_mode mode = {
+				.type = IWL_ERR_TYPE_RESET_HS_TIMEOUT,
+				.context = IWL_ERR_CONTEXT_FROM_OPMODE,
+			};
+			iwl_op_mode_nic_error(trans->op_mode,
+					      IWL_ERR_TYPE_RESET_HS_TIMEOUT);
+			iwl_op_mode_dump_error(trans->op_mode, &mode);
+		}
 	}
 
 	trans_pcie->fw_reset_state = FW_RESET_IDLE;
 }
 
-void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans)
+static void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
@@ -142,9 +159,15 @@ void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans)
 	if (trans_pcie->is_down)
 		return;
 
-	if (trans->state >= IWL_TRANS_FW_STARTED)
-		if (trans_pcie->fw_reset_handshake)
-			iwl_trans_pcie_fw_reset_handshake(trans);
+	if (trans->state >= IWL_TRANS_FW_STARTED &&
+	    trans->conf.fw_reset_handshake) {
+		/*
+		 * Reset handshake can dump firmware on timeout, but that
+		 * should assume that the firmware is already dead.
+		 */
+		trans->state = IWL_TRANS_NO_FW;
+		iwl_trans_pcie_fw_reset_handshake(trans);
+	}
 
 	trans_pcie->is_down = true;
 
@@ -171,8 +194,8 @@ void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans)
 	}
 
 	iwl_pcie_ctxt_info_free_paging(trans);
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
-		iwl_pcie_ctxt_info_gen3_free(trans, false);
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+		iwl_pcie_ctxt_info_v2_free(trans, false);
 	else
 		iwl_pcie_ctxt_info_free(trans);
 
@@ -180,7 +203,7 @@ void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans)
 	iwl_pcie_gen2_apm_stop(trans, false);
 
 	/* re-take ownership to prevent other users from stealing the device */
-	iwl_trans_sw_reset(trans, true);
+	iwl_trans_pcie_sw_reset(trans, true);
 
 	/*
 	 * Upon stop, the IVAR table gets erased, so msi-x won't
@@ -233,7 +256,7 @@ static int iwl_pcie_gen2_nic_init(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int queue_size = max_t(u32, IWL_CMD_QUEUE_SIZE,
-			       trans->cfg->min_txq_size);
+			       trans->mac_cfg->base->min_txq_size);
 	int ret;
 
 	/* TODO: most of the logic can be removed in A0 - but not in Z0 */
@@ -250,7 +273,7 @@ static int iwl_pcie_gen2_nic_init(struct iwl_trans *trans)
 		return -ENOMEM;
 
 	/* Allocate or reset and init all Tx and Command queues */
-	if (iwl_txq_gen2_init(trans, trans_pcie->txqs.cmd.q_id, queue_size))
+	if (iwl_txq_gen2_init(trans, trans->conf.cmd_queue, queue_size))
 		return -ENOMEM;
 
 	/* enable shadow regs in HW */
@@ -271,7 +294,7 @@ static void iwl_pcie_get_rf_name(struct iwl_trans *trans)
 	if (buf[0])
 		return;
 
-	switch (CSR_HW_RFID_TYPE(trans->hw_rf_id)) {
+	switch (CSR_HW_RFID_TYPE(trans->info.hw_rf_id)) {
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_JF):
 		pos = scnprintf(buf, buflen, "JF");
 		break;
@@ -290,15 +313,12 @@ static void iwl_pcie_get_rf_name(struct iwl_trans *trans)
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_HRCDB):
 		pos = scnprintf(buf, buflen, "HRCDB");
 		break;
-	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_MS):
-		pos = scnprintf(buf, buflen, "MS");
-		break;
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_FM):
 		pos = scnprintf(buf, buflen, "FM");
 		break;
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_WP):
 		if (SILICON_Z_STEP ==
-		    CSR_HW_RFID_STEP(trans->hw_rf_id))
+		    CSR_HW_RFID_STEP(trans->info.hw_rf_id))
 			pos = scnprintf(buf, buflen, "WHTC");
 		else
 			pos = scnprintf(buf, buflen, "WH");
@@ -307,7 +327,7 @@ static void iwl_pcie_get_rf_name(struct iwl_trans *trans)
 		return;
 	}
 
-	switch (CSR_HW_RFID_TYPE(trans->hw_rf_id)) {
+	switch (CSR_HW_RFID_TYPE(trans->info.hw_rf_id)) {
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_HR):
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_HR1):
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_HRCDB):
@@ -330,7 +350,7 @@ static void iwl_pcie_get_rf_name(struct iwl_trans *trans)
 	}
 
 	pos += scnprintf(buf + pos, buflen - pos, ", rfid=0x%x",
-			 trans->hw_rf_id);
+			 trans->info.hw_rf_id);
 
 	IWL_INFO(trans, "Detected RF %s\n", buf);
 
@@ -357,8 +377,8 @@ void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans)
 	/* now that we got alive we can free the fw image & the context info.
 	 * paging memory cannot be freed included since FW will still use it
 	 */
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
-		iwl_pcie_ctxt_info_gen3_free(trans, true);
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+		iwl_pcie_ctxt_info_v2_free(trans, true);
 	else
 		iwl_pcie_ctxt_info_free(trans);
 
@@ -372,6 +392,11 @@ void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans)
 
 	iwl_pcie_get_rf_name(trans);
 	mutex_unlock(&trans_pcie->mutex);
+
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+		trans->step_urm = !!(iwl_read_umac_prph(trans,
+							CNVI_PMU_STEP_FLOW) &
+					CNVI_PMU_STEP_FLOW_FORCE_URM);
 }
 
 static bool iwl_pcie_set_ltr(struct iwl_trans *trans)
@@ -391,21 +416,21 @@ static bool iwl_pcie_set_ltr(struct iwl_trans *trans)
 	 * initialize the LTR to ~250 usec (see ltr_val above).
 	 * The firmware initializes this again later (to a smaller value).
 	 */
-	if ((trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210 ||
-	     trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000) &&
-	    !trans->trans_cfg->integrated) {
+	if ((trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_AX210 ||
+	     trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_22000) &&
+	    !trans->mac_cfg->integrated) {
 		iwl_write32(trans, CSR_LTR_LONG_VAL_AD, ltr_val);
 		return true;
 	}
 
-	if (trans->trans_cfg->integrated &&
-	    trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000) {
+	if (trans->mac_cfg->integrated &&
+	    trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_22000) {
 		iwl_write_prph(trans, HPM_MAC_LTR_CSR, HPM_MAC_LRT_ENABLE_ALL);
 		iwl_write_prph(trans, HPM_UMAC_LTR, ltr_val);
 		return true;
 	}
 
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210) {
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_AX210) {
 		/* First clear the interrupt, just in case */
 		iwl_write32(trans, CSR_MSIX_HW_INT_CAUSES_AD,
 			    MSIX_HW_INT_CAUSES_REG_IML);
@@ -462,16 +487,22 @@ static void iwl_pcie_spin_for_iml(struct iwl_trans *trans)
 }
 
 int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
-				 const struct fw_img *fw, bool run_in_rfkill)
+				 const struct iwl_fw *fw,
+				 const struct fw_img *img,
+				 bool run_in_rfkill)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	bool hw_rfkill, keep_ram_busy;
+	bool top_reset_done = false;
 	int ret;
 
+	mutex_lock(&trans_pcie->mutex);
+again:
 	/* This may fail if AMT took ownership of the device */
 	if (iwl_pcie_prepare_card_hw(trans)) {
 		IWL_WARN(trans, "Exit HW not ready\n");
-		return -EIO;
+		ret = -EIO;
+		goto out;
 	}
 
 	iwl_enable_rfkill_int(trans);
@@ -488,8 +519,6 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 	/* Make sure it finished running */
 	iwl_pcie_synchronize_irqs(trans);
 
-	mutex_lock(&trans_pcie->mutex);
-
 	/* If platform's RF_KILL switch is NOT set to KILL */
 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
 	if (hw_rfkill && !run_in_rfkill) {
@@ -519,20 +548,39 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 		goto out;
 	}
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
-		ret = iwl_pcie_ctxt_info_gen3_init(trans, fw);
-	else
-		ret = iwl_pcie_ctxt_info_init(trans, fw);
-	if (ret)
+	if (WARN_ON(trans->do_top_reset &&
+		    trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_SC)) {
+		ret = -EINVAL;
 		goto out;
+	}
+
+	/* we need to wait later - set state */
+	if (trans->do_top_reset)
+		trans_pcie->fw_reset_state = FW_RESET_TOP_REQUESTED;
+
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+		if (!top_reset_done) {
+			ret = iwl_pcie_ctxt_info_v2_alloc(trans, fw, img);
+			if (ret)
+				goto out;
+		}
+
+		iwl_pcie_ctxt_info_v2_kick(trans);
+	} else {
+		ret = iwl_pcie_ctxt_info_init(trans, img);
+		if (ret)
+			goto out;
+	}
 
 	keep_ram_busy = !iwl_pcie_set_ltr(trans);
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+		IWL_DEBUG_POWER(trans, "function scratch register value is 0x%08x\n",
+				iwl_read32(trans, CSR_FUNC_SCRATCH));
 		iwl_write32(trans, CSR_FUNC_SCRATCH, CSR_FUNC_SCRATCH_INIT_VALUE);
 		iwl_set_bit(trans, CSR_GP_CNTRL,
 			    CSR_GP_CNTRL_REG_FLAG_ROM_START);
-	} else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	} else if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		iwl_write_umac_prph(trans, UREG_CPU_INIT_RUN, 1);
 	} else {
 		iwl_write_prph(trans, UREG_CPU_INIT_RUN, 1);
@@ -541,6 +589,43 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 	if (keep_ram_busy)
 		iwl_pcie_spin_for_iml(trans);
 
+	if (trans->do_top_reset) {
+		trans->do_top_reset = 0;
+
+#define FW_TOP_RESET_TIMEOUT	(HZ / 4)
+		ret = wait_event_timeout(trans_pcie->fw_reset_waitq,
+					 trans_pcie->fw_reset_state != FW_RESET_TOP_REQUESTED,
+					 FW_TOP_RESET_TIMEOUT);
+
+		if (trans_pcie->fw_reset_state != FW_RESET_OK) {
+			if (trans_pcie->fw_reset_state != FW_RESET_TOP_REQUESTED)
+				IWL_ERR(trans,
+					"TOP reset interrupted by error (state %d)!\n",
+					trans_pcie->fw_reset_state);
+			else
+				IWL_ERR(trans, "TOP reset timed out!\n");
+			iwl_op_mode_nic_error(trans->op_mode,
+					      IWL_ERR_TYPE_TOP_RESET_FAILED);
+			iwl_trans_schedule_reset(trans,
+						 IWL_ERR_TYPE_TOP_RESET_FAILED);
+			ret = -EIO;
+			goto out;
+		}
+
+		msleep(10);
+		IWL_INFO(trans, "TOP reset successful, reinit now\n");
+		/* now load the firmware again properly */
+		ret = _iwl_trans_pcie_start_hw(trans);
+		if (ret) {
+			IWL_ERR(trans, "failed to start HW after TOP reset\n");
+			goto out;
+		}
+		trans_pcie->prph_scratch->ctrl_cfg.control.control_flags &=
+			~cpu_to_le32(IWL_PRPH_SCRATCH_TOP_RESET);
+		top_reset_done = true;
+		goto again;
+	}
+
 	/* re-check RF-Kill state since we may have missed the interrupt */
 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
 	if (hw_rfkill && !run_in_rfkill)
@@ -550,3 +635,23 @@ out:
 	mutex_unlock(&trans_pcie->mutex);
 	return ret;
 }
+
+void iwl_trans_pcie_gen2_op_mode_leave(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+	mutex_lock(&trans_pcie->mutex);
+
+	/* disable interrupts - don't enable HW RF kill interrupt */
+	iwl_disable_interrupts(trans);
+
+	iwl_pcie_gen2_apm_stop(trans, true);
+
+	iwl_disable_interrupts(trans);
+
+	iwl_pcie_disable_ict(trans);
+
+	mutex_unlock(&trans_pcie->mutex);
+
+	iwl_pcie_synchronize_irqs(trans);
+}
diff --git a/sys/contrib/dev/iwlwifi/pcie/trans.c b/sys/contrib/dev/iwlwifi/pcie/gen1_2/trans.c
index 30be42af1ae1..340a3dd7055c 100644
--- a/sys/contrib/dev/iwlwifi/pcie/trans.c
+++ b/sys/contrib/dev/iwlwifi/pcie/gen1_2/trans.c
@@ -28,137 +28,22 @@
 #include "fw/error-dump.h"
 #include "fw/dbg.h"
 #include "fw/api/tx.h"
+#include "fw/acpi.h"
+#include "fw/api/tx.h"
 #include "mei/iwl-mei.h"
 #include "internal.h"
 #include "iwl-fh.h"
-#include "iwl-context-info-gen3.h"
+#include "pcie/iwl-context-info-v2.h"
+#include "pcie/utils.h"
 
 /* extended range in FW SRAM */
 #define IWL_FW_MEM_EXTENDED_START	0x40000
 #define IWL_FW_MEM_EXTENDED_END		0x57FFF
 
-void iwl_trans_pcie_dump_regs(struct iwl_trans *trans)
-{
-#define PCI_DUMP_SIZE		352
-#define PCI_MEM_DUMP_SIZE	64
-#define PCI_PARENT_DUMP_SIZE	524
-#define PREFIX_LEN		32
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct pci_dev *pdev = trans_pcie->pci_dev;
-	u32 i, pos, alloc_size, *ptr, *buf;
-	char *prefix;
-
-	if (trans_pcie->pcie_dbg_dumped_once)
-		return;
-
-	/* Should be a multiple of 4 */
-	BUILD_BUG_ON(PCI_DUMP_SIZE > 4096 || PCI_DUMP_SIZE & 0x3);
-	BUILD_BUG_ON(PCI_MEM_DUMP_SIZE > 4096 || PCI_MEM_DUMP_SIZE & 0x3);
-	BUILD_BUG_ON(PCI_PARENT_DUMP_SIZE > 4096 || PCI_PARENT_DUMP_SIZE & 0x3);
-
-	/* Alloc a max size buffer */
-	alloc_size = PCI_ERR_ROOT_ERR_SRC +  4 + PREFIX_LEN;
-	alloc_size = max_t(u32, alloc_size, PCI_DUMP_SIZE + PREFIX_LEN);
-	alloc_size = max_t(u32, alloc_size, PCI_MEM_DUMP_SIZE + PREFIX_LEN);
-	alloc_size = max_t(u32, alloc_size, PCI_PARENT_DUMP_SIZE + PREFIX_LEN);
-
-	buf = kmalloc(alloc_size, GFP_ATOMIC);
-	if (!buf)
-		return;
-	prefix = (char *)buf + alloc_size - PREFIX_LEN;
-
-	IWL_ERR(trans, "iwlwifi transaction failed, dumping registers\n");
-
-	/* Print wifi device registers */
-	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
-	IWL_ERR(trans, "iwlwifi device config registers:\n");
-	for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
-		if (pci_read_config_dword(pdev, i, ptr))
-			goto err_read;
-#if defined(__linux__)
-	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
-#elif defined(__FreeBSD__)
-	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
-#endif
-
-	IWL_ERR(trans, "iwlwifi device memory mapped registers:\n");
-	for (i = 0, ptr = buf; i < PCI_MEM_DUMP_SIZE; i += 4, ptr++)
-		*ptr = iwl_read32(trans, i);
-#if defined(__linux__)
-	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
-#elif defined(__FreeBSD__)
-	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
-#endif
-
-	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
-	if (pos) {
-		IWL_ERR(trans, "iwlwifi device AER capability structure:\n");
-		for (i = 0, ptr = buf; i < PCI_ERR_ROOT_COMMAND; i += 4, ptr++)
-			if (pci_read_config_dword(pdev, pos + i, ptr))
-				goto err_read;
-#if defined(__linux__)
-		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
-			       32, 4, buf, i, 0);
-#elif defined(__FreeBSD__)
-		iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
-#endif
-	}
-
-	/* Print parent device registers next */
-	if (!pdev->bus->self)
-		goto out;
-
-	pdev = pdev->bus->self;
-	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
-
-	IWL_ERR(trans, "iwlwifi parent port (%s) config registers:\n",
-		pci_name(pdev));
-	for (i = 0, ptr = buf; i < PCI_PARENT_DUMP_SIZE; i += 4, ptr++)
-		if (pci_read_config_dword(pdev, i, ptr))
-			goto err_read;
-#if defined(__linux__)
-	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
-#elif defined(__FreeBSD__)
-	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
-#endif
-
-	/* Print root port AER registers */
-	pos = 0;
-	pdev = pcie_find_root_port(pdev);
-	if (pdev)
-		pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
-	if (pos) {
-		IWL_ERR(trans, "iwlwifi root port (%s) AER cap structure:\n",
-			pci_name(pdev));
-		sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
-		for (i = 0, ptr = buf; i <= PCI_ERR_ROOT_ERR_SRC; i += 4, ptr++)
-			if (pci_read_config_dword(pdev, pos + i, ptr))
-				goto err_read;
-#if defined(__linux__)
-		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32,
-			       4, buf, i, 0);
-#elif defined(__FreeBSD__)
-		iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
-#endif
-	}
-	goto out;
-
-err_read:
-#if defined(__linux__)
-	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
-#elif defined(__FreeBSD__)
-	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
-#endif
-	IWL_ERR(trans, "Read failed at 0x%X\n", i);
-out:
-	trans_pcie->pcie_dbg_dumped_once = 1;
-	kfree(buf);
-}
-
 int iwl_trans_pcie_sw_reset(struct iwl_trans *trans, bool retake_ownership)
 {
 	/* Reset entire device - do controller reset (results in SHRD_HW_RST) */
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		iwl_set_bit(trans, CSR_GP_CNTRL,
 			    CSR_GP_CNTRL_REG_FLAG_SW_RESET);
 		usleep_range(10000, 20000);
@@ -264,7 +149,7 @@ static void iwl_trans_pcie_write_shr(struct iwl_trans *trans, u32 reg, u32 val)
 
 static void iwl_pcie_set_pwr(struct iwl_trans *trans, bool vaux)
 {
-	if (trans->cfg->apmg_not_supported)
+	if (trans->mac_cfg->base->apmg_not_supported)
 		return;
 
 	if (vaux && pci_pme_capable(to_pci_dev(trans->dev), PCI_D3cold))
@@ -320,7 +205,7 @@ static int iwl_pcie_apm_init(struct iwl_trans *trans)
 	 */
 
 	/* Disable L0S exit timer (platform NMI Work/Around) */
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_8000)
 		iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
 			    CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
 
@@ -339,12 +224,12 @@ static int iwl_pcie_apm_init(struct iwl_trans *trans)
 	 * wake device's PCI Express link L1a -> L0s
 	 */
 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-		    CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
+		    CSR_HW_IF_CONFIG_REG_HAP_WAKE);
 
 	iwl_pcie_apm_config(trans);
 
 	/* Configure analog phase-lock-loop before activating to D0A */
-	if (trans->trans_cfg->base_params->pll_cfg)
+	if (trans->mac_cfg->base->pll_cfg)
 		iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL);
 
 	ret = iwl_finish_nic_init(trans);
@@ -380,7 +265,7 @@ static int iwl_pcie_apm_init(struct iwl_trans *trans)
 	 * bits do not disable clocks.  This preserves any hardware
 	 * bits already set by default in "CLK_CTRL_REG" after reset.
 	 */
-	if (!trans->cfg->apmg_not_supported) {
+	if (!trans->mac_cfg->base->apmg_not_supported) {
 		iwl_write_prph(trans, APMG_CLK_EN_REG,
 			       APMG_CLK_VAL_DMA_CLK_RQT);
 		udelay(20);
@@ -414,8 +299,8 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 	u32 dl_cfg_reg;
 
 	/* Force XTAL ON */
-	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
-				 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
+	iwl_trans_set_bit(trans, CSR_GP_CNTRL,
+			  CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
 
 	ret = iwl_trans_pcie_sw_reset(trans, true);
 
@@ -424,8 +309,8 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 
 	if (WARN_ON(ret)) {
 		/* Release XTAL ON request */
-		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-					   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
+		iwl_trans_clear_bit(trans, CSR_GP_CNTRL,
+				    CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
 		return;
 	}
 
@@ -467,7 +352,7 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 	 * SHRD_HW_RST is applied in S3.
 	 */
 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-		    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
+		    CSR_HW_IF_CONFIG_REG_PERSISTENCE);
 
 	/*
 	 * Clear "initialization complete" bit to move adapter from
@@ -476,12 +361,12 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
 
 	/* Activates XTAL resources monitor */
-	__iwl_trans_pcie_set_bit(trans, CSR_MONITOR_CFG_REG,
-				 CSR_MONITOR_XTAL_RESOURCES);
+	iwl_trans_set_bit(trans, CSR_MONITOR_CFG_REG,
+			  CSR_MONITOR_XTAL_RESOURCES);
 
 	/* Release XTAL ON request */
-	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
+	iwl_trans_clear_bit(trans, CSR_GP_CNTRL,
+			    CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
 	udelay(10);
 
 	/* Release APMG XTAL */
@@ -496,24 +381,22 @@ void iwl_pcie_apm_stop_master(struct iwl_trans *trans)
 
 	/* stop device's busmaster DMA activity */
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		iwl_set_bit(trans, CSR_GP_CNTRL,
 			    CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_REQ);
 
-		ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS,
-				   CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS,
-				   100);
+		ret = iwl_poll_bits(trans, CSR_GP_CNTRL,
+				    CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS,
+				    100);
 		usleep_range(10000, 20000);
 	} else {
 		iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);
 
-		ret = iwl_poll_bit(trans, CSR_RESET,
-				   CSR_RESET_REG_FLAG_MASTER_DISABLED,
-				   CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
+		ret = iwl_poll_bits(trans, CSR_RESET,
+				    CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
 	}
 
-	if (ret < 0)
+	if (ret)
 		IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");
 
 	IWL_DEBUG_INFO(trans, "stop master\n");
@@ -528,16 +411,16 @@ static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 			iwl_pcie_apm_init(trans);
 
 		/* inform ME that we are leaving */
-		if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000)
+		if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_7000)
 			iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
 					  APMG_PCIDEV_STT_VAL_WAKE_ME);
-		else if (trans->trans_cfg->device_family >=
+		else if (trans->mac_cfg->device_family >=
 			 IWL_DEVICE_FAMILY_8000) {
 			iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
 				    CSR_RESET_LINK_PWR_MGMT_DISABLED);
 			iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-				    CSR_HW_IF_CONFIG_REG_PREPARE |
-				    CSR_HW_IF_CONFIG_REG_ENABLE_PME);
+				    CSR_HW_IF_CONFIG_REG_WAKE_ME |
+				    CSR_HW_IF_CONFIG_REG_WAKE_ME_PCIE_OWNER_EN);
 			mdelay(1);
 			iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
 				      CSR_RESET_LINK_PWR_MGMT_DISABLED);
@@ -592,7 +475,7 @@ static int iwl_pcie_nic_init(struct iwl_trans *trans)
 		return -ENOMEM;
 	}
 
-	if (trans->trans_cfg->base_params->shadow_reg_enable) {
+	if (trans->mac_cfg->base->shadow_reg_enable) {
 		/* enable shadow regs in HW */
 		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
 		IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
@@ -609,18 +492,17 @@ static int iwl_pcie_set_hw_ready(struct iwl_trans *trans)
 	int ret;
 
 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-		    CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
+		    CSR_HW_IF_CONFIG_REG_PCI_OWN_SET);
 
 	/* See if we got it */
-	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
-			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
-			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
-			   HW_READY_TIMEOUT);
+	ret = iwl_poll_bits(trans, CSR_HW_IF_CONFIG_REG,
+			    CSR_HW_IF_CONFIG_REG_PCI_OWN_SET,
+			    HW_READY_TIMEOUT);
 
-	if (ret >= 0)
+	if (!ret)
 		iwl_set_bit(trans, CSR_MBOX_SET_REG, CSR_MBOX_SET_REG_OS_ALIVE);
 
-	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
+	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret ? " not" : "");
 	return ret;
 }
 
@@ -634,7 +516,7 @@ int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
 
 	ret = iwl_pcie_set_hw_ready(trans);
 	/* If the card is ready, exit 0 */
-	if (ret >= 0) {
+	if (!ret) {
 		trans->csme_own = false;
 		return 0;
 	}
@@ -648,11 +530,11 @@ int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
 
 		/* If HW is not ready, prepare the conditions to check again */
 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-			    CSR_HW_IF_CONFIG_REG_PREPARE);
+			    CSR_HW_IF_CONFIG_REG_WAKE_ME);
 
 		do {
 			ret = iwl_pcie_set_hw_ready(trans);
-			if (ret >= 0) {
+			if (!ret) {
 				trans->csme_own = false;
 				return 0;
 			}
@@ -661,7 +543,7 @@ int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
 				IWL_DEBUG_INFO(trans,
 					       "Couldn't prepare the card but SAP is connected\n");
 				trans->csme_own = true;
-				if (trans->trans_cfg->device_family !=
+				if (trans->mac_cfg->device_family !=
 				    IWL_DEVICE_FAMILY_9000)
 					IWL_ERR(trans,
 						"SAP not supported for this NIC family\n");
@@ -731,7 +613,7 @@ static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans,
 				 trans_pcie->ucode_write_complete, 5 * HZ);
 	if (!ret) {
 		IWL_ERR(trans, "Failed to load firmware chunk!\n");
-		iwl_trans_pcie_dump_regs(trans);
+		iwl_trans_pcie_dump_regs(trans, trans_pcie->pci_dev);
 		return -ETIMEDOUT;
 	}
 
@@ -846,7 +728,7 @@ static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
 
 	iwl_enable_interrupts(trans);
 
-	if (trans->trans_cfg->gen2) {
+	if (trans->mac_cfg->gen2) {
 		if (cpu == 1)
 			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
 				       0xFFFF);
@@ -1004,7 +886,7 @@ monitor:
 	if (dest->monitor_mode == EXTERNAL_MODE && fw_mon->size) {
 		iwl_write_prph(trans, le32_to_cpu(dest->base_reg),
 			       fw_mon->physical >> dest->base_shift);
-		if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
+		if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
 			iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
 				       (fw_mon->physical + fw_mon->size -
 					256) >> dest->end_shift);
@@ -1180,7 +1062,7 @@ static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans)
 	 */
 	iwl_pcie_map_list(trans, causes_list_common,
 			  ARRAY_SIZE(causes_list_common), val);
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		iwl_pcie_map_list(trans, causes_list_bz,
 				  ARRAY_SIZE(causes_list_bz), val);
 	else
@@ -1202,7 +1084,7 @@ static void iwl_pcie_map_rx_causes(struct iwl_trans *trans)
 	 * the other (N - 2) interrupt vectors.
 	 */
 	val = BIT(MSIX_FH_INT_CAUSES_Q(0));
-	for (idx = 1; idx < trans->num_rx_queues; idx++) {
+	for (idx = 1; idx < trans->info.num_rxqs; idx++) {
 		iwl_write8(trans, CSR_MSIX_RX_IVAR(idx),
 			   MSIX_FH_INT_CAUSES_Q(idx - offset));
 		val |= BIT(MSIX_FH_INT_CAUSES_Q(idx));
@@ -1223,7 +1105,7 @@ void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie)
 	struct iwl_trans *trans = trans_pcie->trans;
 
 	if (!trans_pcie->msix_enabled) {
-		if (trans->trans_cfg->mq_rx_supported &&
+		if (trans->mac_cfg->mq_rx_supported &&
 		    test_bit(STATUS_DEVICE_ENABLED, &trans->status))
 			iwl_write_umac_prph(trans, UREG_CHICK,
 					    UREG_CHICK_MSI_ENABLE);
@@ -1298,7 +1180,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool from_irq)
 		iwl_pcie_rx_stop(trans);
 
 		/* Power-down device's busmaster DMA clocks */
-		if (!trans->cfg->apmg_not_supported) {
+		if (!trans->mac_cfg->base->apmg_not_supported) {
 			iwl_write_prph(trans, APMG_CLK_DIS_REG,
 				       APMG_CLK_VAL_DMA_CLK_RQT);
 			udelay(5);
@@ -1306,7 +1188,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool from_irq)
 	}
 
 	/* Make sure (redundant) we've released our request to stay awake */
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		iwl_clear_bit(trans, CSR_GP_CNTRL,
 			      CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
 	else
@@ -1364,7 +1246,9 @@ void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
 }
 
 int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
-			    const struct fw_img *fw, bool run_in_rfkill)
+			    const struct iwl_fw *fw,
+			    const struct fw_img *img,
+			    bool run_in_rfkill)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	bool hw_rfkill;
@@ -1435,10 +1319,10 @@ int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
 
 	/* Load the given image to the HW */
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
-		ret = iwl_pcie_load_given_ucode_8000(trans, fw);
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
+		ret = iwl_pcie_load_given_ucode_8000(trans, img);
 	else
-		ret = iwl_pcie_load_given_ucode(trans, fw);
+		ret = iwl_pcie_load_given_ucode(trans, img);
 
 	/* re-check RF-Kill state since we may have missed the interrupt */
 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
@@ -1450,10 +1334,10 @@ out:
 	return ret;
 }
 
-void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr)
+void iwl_trans_pcie_fw_alive(struct iwl_trans *trans)
 {
 	iwl_pcie_reset_ict(trans);
-	iwl_pcie_tx_start(trans, scd_addr);
+	iwl_pcie_tx_start(trans);
 }
 
 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
@@ -1512,12 +1396,12 @@ void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state, bool from_irq)
 	IWL_WARN(trans, "reporting RF_KILL (radio %s)\n",
 		 state ? "disabled" : "enabled");
 	if (iwl_op_mode_hw_rf_kill(trans->op_mode, state) &&
-	    !WARN_ON(trans->trans_cfg->gen2))
+	    !WARN_ON(trans->mac_cfg->gen2))
 		_iwl_trans_pcie_stop_device(trans, from_irq);
 }
 
-void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
-				  bool test, bool reset)
+static void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
+					 bool test, bool reset)
 {
 	iwl_disable_interrupts(trans);
 
@@ -1532,7 +1416,7 @@ void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
 
 	iwl_pcie_synchronize_irqs(trans);
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		iwl_clear_bit(trans, CSR_GP_CNTRL,
 			      CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
 		iwl_clear_bit(trans, CSR_GP_CNTRL,
@@ -1561,30 +1445,41 @@ static int iwl_pcie_d3_handshake(struct iwl_trans *trans, bool suspend)
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int ret;
 
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+		return 0;
+
+	trans_pcie->sx_state = IWL_SX_WAITING;
+
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_AX210)
 		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
 				    suspend ? UREG_DOORBELL_TO_ISR6_SUSPEND :
 					      UREG_DOORBELL_TO_ISR6_RESUME);
-	else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	else
 		iwl_write32(trans, CSR_IPC_SLEEP_CONTROL,
 			    suspend ? CSR_IPC_SLEEP_CONTROL_SUSPEND :
 				      CSR_IPC_SLEEP_CONTROL_RESUME);
-	else
-		return 0;
 
 	ret = wait_event_timeout(trans_pcie->sx_waitq,
-				 trans_pcie->sx_complete, 2 * HZ);
-
-	/* Invalidate it toward next suspend or resume */
-	trans_pcie->sx_complete = false;
-
+				 trans_pcie->sx_state != IWL_SX_WAITING,
+				 2 * HZ);
 	if (!ret) {
 		IWL_ERR(trans, "Timeout %s D3\n",
 			suspend ? "entering" : "exiting");
-		return -ETIMEDOUT;
+		ret = -ETIMEDOUT;
+	} else {
+		ret = 0;
 	}
 
-	return 0;
+	if (trans_pcie->sx_state == IWL_SX_ERROR) {
+		IWL_ERR(trans, "FW error while %s D3\n",
+			suspend ? "entering" : "exiting");
+		ret = -EIO;
+	}
+
+	/* Invalidate it toward next suspend or resume */
+	trans_pcie->sx_state = IWL_SX_INVALID;
+
+	return ret;
 }
 
 int iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test, bool reset)
@@ -1594,7 +1489,7 @@ int iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test, bool reset)
 	if (!reset)
 		/* Enable persistence mode to avoid reset */
 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
-			    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
+			    CSR_HW_IF_CONFIG_REG_PERSISTENCE);
 
 	ret = iwl_pcie_d3_handshake(trans, true);
 	if (ret)
@@ -1620,7 +1515,7 @@ int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
 		goto out;
 	}
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		iwl_set_bit(trans, CSR_GP_CNTRL,
 			    CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
 	else
@@ -1671,6 +1566,8 @@ int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
 out:
 	if (*status == IWL_D3_STATUS_ALIVE)
 		ret = iwl_pcie_d3_handshake(trans, false);
+	else
+		trans->state = IWL_TRANS_NO_FW;
 
 	return ret;
 }
@@ -1678,17 +1575,18 @@ out:
 static void
 iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
 			    struct iwl_trans *trans,
-			    const struct iwl_cfg_trans_params *cfg_trans)
+			    const struct iwl_mac_cfg *mac_cfg,
+			    struct iwl_trans_info *info)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int max_irqs, num_irqs, i, ret;
 	u16 pci_cmd;
 	u32 max_rx_queues = IWL_MAX_RX_HW_QUEUES;
 
-	if (!cfg_trans->mq_rx_supported)
+	if (!mac_cfg->mq_rx_supported)
 		goto enable_msi;
 
-	if (cfg_trans->device_family <= IWL_DEVICE_FAMILY_9000)
+	if (mac_cfg->device_family <= IWL_DEVICE_FAMILY_9000)
 		max_rx_queues = IWL_9000_MAX_RX_HW_QUEUES;
 
 	max_irqs = min_t(u32, num_online_cpus() + 2, max_rx_queues);
@@ -1718,27 +1616,28 @@ iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
 	 * More than two interrupts: we will use fewer RSS queues.
 	 */
 	if (num_irqs <= max_irqs - 2) {
-		trans_pcie->trans->num_rx_queues = num_irqs + 1;
+		info->num_rxqs = num_irqs + 1;
 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
 			IWL_SHARED_IRQ_FIRST_RSS;
 	} else if (num_irqs == max_irqs - 1) {
-		trans_pcie->trans->num_rx_queues = num_irqs;
+		info->num_rxqs = num_irqs;
 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
 	} else {
-		trans_pcie->trans->num_rx_queues = num_irqs - 1;
+		info->num_rxqs = num_irqs - 1;
 	}
 
 	IWL_DEBUG_INFO(trans,
 		       "MSI-X enabled with rx queues %d, vec mask 0x%x\n",
-		       trans_pcie->trans->num_rx_queues, trans_pcie->shared_vec_mask);
+		       info->num_rxqs, trans_pcie->shared_vec_mask);
 
-	WARN_ON(trans_pcie->trans->num_rx_queues > IWL_MAX_RX_HW_QUEUES);
+	WARN_ON(info->num_rxqs > IWL_MAX_RX_HW_QUEUES);
 
 	trans_pcie->alloc_vecs = num_irqs;
 	trans_pcie->msix_enabled = true;
 	return;
 
 enable_msi:
+	info->num_rxqs = 1;
 	ret = pci_enable_msi(pdev);
 	if (ret) {
 		dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret);
@@ -1751,14 +1650,15 @@ enable_msi:
 	}
 }
 
-static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
+static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans,
+				      struct iwl_trans_info *info)
 {
 #if defined(CONFIG_SMP)
 	int iter_rx_q, i, ret, cpu, offset;
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1;
-	iter_rx_q = trans_pcie->trans->num_rx_queues - 1 + i;
+	iter_rx_q = info->num_rxqs - 1 + i;
 	offset = 1 + i;
 	for (; i < iter_rx_q ; i++) {
 		/*
@@ -1778,7 +1678,8 @@ static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
 }
 
 static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
-				      struct iwl_trans_pcie *trans_pcie)
+				      struct iwl_trans_pcie *trans_pcie,
+				      struct iwl_trans_info *info)
 {
 	int i;
 
@@ -1807,7 +1708,7 @@ static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
 			return ret;
 		}
 	}
-	iwl_pcie_irq_set_affinity(trans_pcie->trans);
+	iwl_pcie_irq_set_affinity(trans_pcie->trans, info);
 
 	return 0;
 }
@@ -1816,7 +1717,7 @@ static int iwl_trans_pcie_clear_persistence_bit(struct iwl_trans *trans)
 {
 	u32 hpm, wprot;
 
-	switch (trans->trans_cfg->device_family) {
+	switch (trans->mac_cfg->device_family) {
 	case IWL_DEVICE_FAMILY_9000:
 		wprot = PREG_PRPH_WPROT_9000;
 		break;
@@ -1864,7 +1765,7 @@ static int iwl_pcie_gen2_force_power_gating(struct iwl_trans *trans)
 	return iwl_trans_pcie_sw_reset(trans, true);
 }
 
-static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans)
+int _iwl_trans_pcie_start_hw(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int err;
@@ -1885,8 +1786,8 @@ static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans)
 	if (err)
 		return err;
 
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000 &&
-	    trans->trans_cfg->integrated) {
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_22000 &&
+	    trans->mac_cfg->integrated) {
 		err = iwl_pcie_gen2_force_power_gating(trans);
 		if (err)
 			return err;
@@ -1962,15 +1863,11 @@ u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
 #elif defined(__FreeBSD__)
 void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
 {
-
-	IWL_DEBUG_PCI_RW(trans, "W1 %#010x %#04x\n", ofs, val);
 	bus_write_1((struct resource *)IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base, ofs, val);
 }
 
 void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
 {
-
-	IWL_DEBUG_PCI_RW(trans, "W4 %#010x %#010x\n", ofs, val);
 	bus_write_4((struct resource *)IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base, ofs, val);
 }
 
@@ -1979,14 +1876,13 @@ u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
 	u32 v;
 
 	v = bus_read_4((struct resource *)IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base, ofs);
-	IWL_DEBUG_PCI_RW(trans, "R4 %#010x %#010x\n", ofs, v);
 	return (v);
 }
 #endif
 
 static u32 iwl_trans_pcie_prph_msk(struct iwl_trans *trans)
 {
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 		return 0x00FFFFFF;
 	else
 		return 0x000FFFFF;
@@ -2010,46 +1906,17 @@ void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr, u32 val)
 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
 }
 
-void iwl_trans_pcie_configure(struct iwl_trans *trans,
-			      const struct iwl_trans_config *trans_cfg)
+void iwl_trans_pcie_op_mode_enter(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	/* free all first - we might be reconfigured for a different size */
 	iwl_pcie_free_rbs_pool(trans);
 
-	trans_pcie->txqs.cmd.q_id = trans_cfg->cmd_queue;
-	trans_pcie->txqs.cmd.fifo = trans_cfg->cmd_fifo;
-	trans_pcie->txqs.cmd.wdg_timeout = trans_cfg->cmd_q_wdg_timeout;
-	trans_pcie->txqs.page_offs = trans_cfg->cb_data_offs;
-	trans_pcie->txqs.dev_cmd_offs = trans_cfg->cb_data_offs + sizeof(void *);
-	trans_pcie->txqs.queue_alloc_cmd_ver = trans_cfg->queue_alloc_cmd_ver;
-
-	if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
-		trans_pcie->n_no_reclaim_cmds = 0;
-	else
-		trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
-	if (trans_pcie->n_no_reclaim_cmds)
-		memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
-		       trans_pcie->n_no_reclaim_cmds * sizeof(u8));
-
-	trans_pcie->rx_buf_size = trans_cfg->rx_buf_size;
 	trans_pcie->rx_page_order =
-		iwl_trans_get_rb_size_order(trans_pcie->rx_buf_size);
+		iwl_trans_get_rb_size_order(trans->conf.rx_buf_size);
 	trans_pcie->rx_buf_bytes =
-		iwl_trans_get_rb_size(trans_pcie->rx_buf_size);
-	trans_pcie->supported_dma_mask = DMA_BIT_MASK(12);
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
-		trans_pcie->supported_dma_mask = DMA_BIT_MASK(11);
-
-	trans_pcie->txqs.bc_table_dword = trans_cfg->bc_table_dword;
-	trans_pcie->scd_set_active = trans_cfg->scd_set_active;
-
-	trans->command_groups = trans_cfg->command_groups;
-	trans->command_groups_size = trans_cfg->command_groups_size;
-
-
-	trans_pcie->fw_reset_handshake = trans_cfg->fw_reset_handshake;
+		iwl_trans_get_rb_size(trans->conf.rx_buf_size);
 }
 
 void iwl_trans_pcie_free_pnvm_dram_regions(struct iwl_dram_regions *dram_regions,
@@ -2077,11 +1944,14 @@ void iwl_trans_pcie_free_pnvm_dram_regions(struct iwl_dram_regions *dram_regions
 
 static void iwl_pcie_free_invalid_tx_cmd(struct iwl_trans *trans)
 {
-	iwl_pcie_free_dma_ptr(trans, &trans->invalid_tx_cmd);
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+	iwl_pcie_free_dma_ptr(trans, &trans_pcie->invalid_tx_cmd);
 }
 
 static int iwl_pcie_alloc_invalid_tx_cmd(struct iwl_trans *trans)
 {
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_cmd_header_wide bad_cmd = {
 		.cmd = INVALID_WR_PTR_CMD,
 		.group_id = DEBUG_GROUP,
@@ -2091,11 +1961,11 @@ static int iwl_pcie_alloc_invalid_tx_cmd(struct iwl_trans *trans)
 	};
 	int ret;
 
-	ret = iwl_pcie_alloc_dma_ptr(trans, &trans->invalid_tx_cmd,
+	ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->invalid_tx_cmd,
 				     sizeof(bad_cmd));
 	if (ret)
 		return ret;
-	memcpy(trans->invalid_tx_cmd.addr, &bad_cmd, sizeof(bad_cmd));
+	memcpy(trans_pcie->invalid_tx_cmd.addr, &bad_cmd, sizeof(bad_cmd));
 	return 0;
 }
 
@@ -2106,7 +1976,7 @@ void iwl_trans_pcie_free(struct iwl_trans *trans)
 
 	iwl_pcie_synchronize_irqs(trans);
 
-	if (trans->trans_cfg->gen2)
+	if (trans->mac_cfg->gen2)
 		iwl_txq_gen2_tx_free(trans);
 	else
 		iwl_pcie_tx_free(trans);
@@ -2159,10 +2029,157 @@ void iwl_trans_pcie_free(struct iwl_trans *trans)
 	iwl_trans_free(trans);
 }
 
+static union acpi_object *
+iwl_trans_pcie_call_prod_reset_dsm(struct pci_dev *pdev, u16 cmd, u16 value)
+{
+#ifdef CONFIG_ACPI
+	struct iwl_dsm_internal_product_reset_cmd pldr_arg = {
+		.cmd = cmd,
+		.value = value,
+	};
+	union acpi_object arg = {
+		.buffer.type = ACPI_TYPE_BUFFER,
+		.buffer.length = sizeof(pldr_arg),
+		.buffer.pointer = (void *)&pldr_arg,
+	};
+	static const guid_t dsm_guid = GUID_INIT(0x7266172C, 0x220B, 0x4B29,
+						 0x81, 0x4F, 0x75, 0xE4,
+						 0xDD, 0x26, 0xB5, 0xFD);
+
+	if (!acpi_check_dsm(ACPI_HANDLE(&pdev->dev), &dsm_guid, ACPI_DSM_REV,
+			    DSM_INTERNAL_FUNC_PRODUCT_RESET))
+		return ERR_PTR(-ENODEV);
+
+	return iwl_acpi_get_dsm_object(&pdev->dev, ACPI_DSM_REV,
+				       DSM_INTERNAL_FUNC_PRODUCT_RESET,
+				       &arg, &dsm_guid);
+#else
+	return ERR_PTR(-EOPNOTSUPP);
+#endif
+}
+
+void iwl_trans_pcie_check_product_reset_mode(struct pci_dev *pdev)
+{
+	union acpi_object *res;
+
+	res = iwl_trans_pcie_call_prod_reset_dsm(pdev,
+						 DSM_INTERNAL_PLDR_CMD_GET_MODE,
+						 0);
+	if (IS_ERR(res))
+		return;
+
+	if (res->type != ACPI_TYPE_INTEGER)
+		IWL_ERR_DEV(&pdev->dev,
+			    "unexpected return type from product reset DSM\n");
+	else
+		IWL_DEBUG_DEV_POWER(&pdev->dev,
+				    "product reset mode is 0x%llx\n",
+				    res->integer.value);
+
+	ACPI_FREE(res);
+}
+
+static void iwl_trans_pcie_set_product_reset(struct pci_dev *pdev, bool enable,
+					     bool integrated)
+{
+	union acpi_object *res;
+	u16 mode = enable ? DSM_INTERNAL_PLDR_MODE_EN_PROD_RESET : 0;
+
+	if (!integrated)
+		mode |= DSM_INTERNAL_PLDR_MODE_EN_WIFI_FLR |
+			DSM_INTERNAL_PLDR_MODE_EN_BT_OFF_ON;
+
+	res = iwl_trans_pcie_call_prod_reset_dsm(pdev,
+						 DSM_INTERNAL_PLDR_CMD_SET_MODE,
+						 mode);
+	if (IS_ERR(res)) {
+		if (enable)
+			IWL_ERR_DEV(&pdev->dev,
+				    "ACPI _DSM not available (%d), cannot do product reset\n",
+				    (int)PTR_ERR(res));
+		return;
+	}
+
+	ACPI_FREE(res);
+	IWL_DEBUG_DEV_POWER(&pdev->dev, "%sabled product reset via DSM\n",
+			    enable ? "En" : "Dis");
+	iwl_trans_pcie_check_product_reset_mode(pdev);
+}
+
+void iwl_trans_pcie_check_product_reset_status(struct pci_dev *pdev)
+{
+	union acpi_object *res;
+
+	res = iwl_trans_pcie_call_prod_reset_dsm(pdev,
+						 DSM_INTERNAL_PLDR_CMD_GET_STATUS,
+						 0);
+	if (IS_ERR(res))
+		return;
+
+	if (res->type != ACPI_TYPE_INTEGER)
+		IWL_ERR_DEV(&pdev->dev,
+			    "unexpected return type from product reset DSM\n");
+	else
+		IWL_DEBUG_DEV_POWER(&pdev->dev,
+				    "product reset status is 0x%llx\n",
+				    res->integer.value);
+
+	ACPI_FREE(res);
+}
+
+static void iwl_trans_pcie_call_reset(struct pci_dev *pdev)
+{
+#ifdef CONFIG_ACPI
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *p, *ref;
+	acpi_status status;
+	int ret = -EINVAL;
+
+	status = acpi_evaluate_object(ACPI_HANDLE(&pdev->dev),
+				      "_PRR", NULL, &buffer);
+	if (ACPI_FAILURE(status)) {
+		IWL_DEBUG_DEV_POWER(&pdev->dev, "No _PRR method found\n");
+		goto out;
+	}
+	p = buffer.pointer;
+
+	if (p->type != ACPI_TYPE_PACKAGE || p->package.count != 1) {
+		pci_err(pdev, "Bad _PRR return type\n");
+		goto out;
+	}
+
+	ref = &p->package.elements[0];
+	if (ref->type != ACPI_TYPE_LOCAL_REFERENCE) {
+		pci_err(pdev, "_PRR wasn't a reference\n");
+		goto out;
+	}
+
+	status = acpi_evaluate_object(ref->reference.handle,
+				      "_RST", NULL, NULL);
+	if (ACPI_FAILURE(status)) {
+		pci_err(pdev,
+			"Failed to call _RST on object returned by _PRR (%d)\n",
+			status);
+		goto out;
+	}
+	ret = 0;
+out:
+	kfree(buffer.pointer);
+	if (!ret) {
+		IWL_DEBUG_DEV_POWER(&pdev->dev, "called _RST on _PRR object\n");
+		return;
+	}
+	IWL_DEBUG_DEV_POWER(&pdev->dev,
+			    "No BIOS support, using pci_reset_function()\n");
+#endif
+	pci_reset_function(pdev);
+}
+
 struct iwl_trans_pcie_removal {
 	struct pci_dev *pdev;
 	struct work_struct work;
-	bool rescan;
+	enum iwl_reset_mode mode;
+	bool integrated;
 };
 
 static void iwl_trans_pcie_removal_wk(struct work_struct *wk)
@@ -2180,17 +2197,71 @@ static void iwl_trans_pcie_removal_wk(struct work_struct *wk)
 	if (!bus)
 		goto out;
 
-	dev_err(&pdev->dev, "Device gone - attempting removal\n");
-
 	kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop);
 
+	if (removal->mode == IWL_RESET_MODE_PROD_RESET) {
+		struct pci_dev *bt = NULL;
+
+		if (!removal->integrated) {
+			/* discrete devices have WiFi/BT at function 0/1 */
+			int slot = PCI_SLOT(pdev->devfn);
+			int func = PCI_FUNC(pdev->devfn);
+
+			if (func == 0)
+				bt = pci_get_slot(bus, PCI_DEVFN(slot, 1));
+			else
+				pci_info(pdev, "Unexpected function %d\n",
+					 func);
+		} else {
+			/* on integrated we have to look up by ID (same bus) */
+			static const struct pci_device_id bt_device_ids[] = {
+#define BT_DEV(_id) { PCI_DEVICE(PCI_VENDOR_ID_INTEL, _id) }
+				BT_DEV(0xA876), /* LNL */
+				BT_DEV(0xE476), /* PTL-P */
+				BT_DEV(0xE376), /* PTL-H */
+				BT_DEV(0xD346), /* NVL-H */
+				BT_DEV(0x6E74), /* NVL-S */
+				BT_DEV(0x4D76), /* WCL */
+				BT_DEV(0xD246), /* RZL-H */
+				BT_DEV(0x6C46), /* RZL-M */
+				{}
+			};
+			struct pci_dev *tmp = NULL;
+
+			for_each_pci_dev(tmp) {
+				if (tmp->bus != bus)
+					continue;
+
+				if (pci_match_id(bt_device_ids, tmp)) {
+					bt = tmp;
+					break;
+				}
+			}
+		}
+
+		if (bt) {
+			pci_info(bt, "Removal by WiFi due to product reset\n");
+			pci_stop_and_remove_bus_device(bt);
+			pci_dev_put(bt);
+		}
+	}
+
+	iwl_trans_pcie_set_product_reset(pdev,
+					 removal->mode ==
+						IWL_RESET_MODE_PROD_RESET,
+					 removal->integrated);
+	if (removal->mode >= IWL_RESET_MODE_FUNC_RESET)
+		iwl_trans_pcie_call_reset(pdev);
+
 	pci_stop_and_remove_bus_device(pdev);
 	pci_dev_put(pdev);
 
-	if (removal->rescan) {
+	if (removal->mode >= IWL_RESET_MODE_RESCAN) {
 #if defined(__linux__)
 		if (bus->parent)
 			bus = bus->parent;
+#elif defined(__FreeBSD__)
+		/* XXX-TODO */
 #endif
 		pci_rescan_bus(bus);
 	}
@@ -2202,14 +2273,29 @@ out:
 	module_put(THIS_MODULE);
 }
 
-void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan)
+void iwl_trans_pcie_reset(struct iwl_trans *trans, enum iwl_reset_mode mode)
 {
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_trans_pcie_removal *removal;
+	char _msg = 0, *msg = &_msg;
+
+	if (WARN_ON(mode < IWL_RESET_MODE_REMOVE_ONLY ||
+		    mode == IWL_RESET_MODE_BACKOFF))
+		return;
 
 	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
 		return;
 
-	IWL_ERR(trans, "Device gone - scheduling removal!\n");
+	if (trans_pcie->me_present && mode == IWL_RESET_MODE_PROD_RESET) {
+		mode = IWL_RESET_MODE_FUNC_RESET;
+		if (trans_pcie->me_present < 0)
+			msg = " instead of product reset as ME may be present";
+		else
+			msg = " instead of product reset as ME is present";
+	}
+
+	IWL_INFO(trans, "scheduling reset (mode=%d%s)\n", mode, msg);
+
 	iwl_pcie_dump_csr(trans);
 
 	/*
@@ -2236,18 +2322,19 @@ void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan)
 	set_bit(STATUS_TRANS_DEAD, &trans->status);
 
 	removal->pdev = to_pci_dev(trans->dev);
-	removal->rescan = rescan;
+	removal->mode = mode;
+	removal->integrated = trans->mac_cfg->integrated;
 	INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk);
 	pci_dev_get(removal->pdev);
 	schedule_work(&removal->work);
 }
-EXPORT_SYMBOL(iwl_trans_pcie_remove);
+EXPORT_SYMBOL(iwl_trans_pcie_reset);
 
 /*
  * This version doesn't disable BHs but rather assumes they're
  * already disabled.
  */
-bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
+bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent)
 {
 	int ret;
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
@@ -2264,15 +2351,15 @@ bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
 	if (trans_pcie->cmd_hold_nic_awake)
 		goto out;
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		write = CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ;
 		mask = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
 		poll = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
 	}
 
 	/* this bit wakes up the NIC */
-	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL, write);
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
+	iwl_trans_set_bit(trans, CSR_GP_CNTRL, write);
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
 		udelay(2);
 
 	/*
@@ -2295,18 +2382,24 @@ bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
 	 * 5000 series and later (including 1000 series) have non-volatile SRAM,
 	 * and do not save/restore SRAM when power cycling.
 	 */
-	ret = iwl_poll_bit(trans, CSR_GP_CNTRL, poll, mask, 15000);
-	if (unlikely(ret < 0)) {
+	ret = iwl_poll_bits_mask(trans, CSR_GP_CNTRL, poll, mask, 15000);
+	if (unlikely(ret)) {
 		u32 cntrl = iwl_read32(trans, CSR_GP_CNTRL);
 
+		if (silent) {
+			spin_unlock(&trans_pcie->reg_lock);
+			return false;
+		}
+
 		WARN_ONCE(1,
 			  "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n",
 			  cntrl);
 
-		iwl_trans_pcie_dump_regs(trans);
+		iwl_trans_pcie_dump_regs(trans, trans_pcie->pci_dev);
 
 		if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U)
-			iwl_trans_pcie_remove(trans, false);
+			iwl_trans_pcie_reset(trans,
+					     IWL_RESET_MODE_REMOVE_ONLY);
 		else
 			iwl_write32(trans, CSR_RESET,
 				    CSR_RESET_REG_FLAG_FORCE_NMI);
@@ -2329,7 +2422,7 @@ bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
 	bool ret;
 
 	local_bh_disable();
-	ret = __iwl_trans_pcie_grab_nic_access(trans);
+	ret = __iwl_trans_pcie_grab_nic_access(trans, false);
 	if (ret) {
 		/* keep BHs disabled until iwl_trans_pcie_release_nic_access */
 		return ret;
@@ -2338,7 +2431,8 @@ bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
 	return false;
 }
 
-void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
+void __releases(nic_access_nobh)
+iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
@@ -2352,12 +2446,12 @@ void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
 
 	if (trans_pcie->cmd_hold_nic_awake)
 		goto out;
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
-		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-					   CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+		iwl_trans_clear_bit(trans, CSR_GP_CNTRL,
+				    CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
 	else
-		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-					   CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+		iwl_trans_clear_bit(trans, CSR_GP_CNTRL,
+				    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
 	/*
 	 * Above we read the CSR_GP_CNTRL register, which will flush
 	 * any previous writes, but we need the write that clears the
@@ -2365,6 +2459,7 @@ void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
 	 * scheduled on different CPUs (after we drop reg_lock).
 	 */
 out:
+	__release(nic_access_nobh);
 	spin_unlock_bh(&trans_pcie->reg_lock);
 }
 
@@ -2418,24 +2513,6 @@ int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
 	return 0;
 }
 
-int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
-			     const void *buf, int dwords)
-{
-	int offs, ret = 0;
-	const u32 *vals = buf;
-
-	if (iwl_trans_grab_nic_access(trans)) {
-		iwl_write32(trans, HBUS_TARG_MEM_WADDR, addr);
-		for (offs = 0; offs < dwords; offs++)
-			iwl_write32(trans, HBUS_TARG_MEM_WDAT,
-				    vals ? vals[offs] : 0);
-		iwl_trans_release_nic_access(trans);
-	} else {
-		ret = -EBUSY;
-	}
-	return ret;
-}
-
 int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
 				 u32 *val)
 {
@@ -2450,7 +2527,7 @@ int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
-	if (queue >= trans->num_rx_queues || !trans_pcie->rxq)
+	if (queue >= trans->info.num_rxqs || !trans_pcie->rxq)
 		return -EINVAL;
 
 	data->fr_bd_cb = trans_pcie->rxq[queue].bd_dma;
@@ -2531,10 +2608,10 @@ int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm)
 
 	/* waiting for all the tx frames complete might take a while */
 	for (cnt = 0;
-	     cnt < trans->trans_cfg->base_params->num_of_queues;
+	     cnt < trans->mac_cfg->base->num_of_queues;
 	     cnt++) {
 
-		if (cnt == trans_pcie->txqs.cmd.q_id)
+		if (cnt == trans->conf.cmd_queue)
 			continue;
 		if (!test_bit(cnt, trans_pcie->txqs.queue_used))
 			continue;
@@ -2555,7 +2632,7 @@ void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	spin_lock_bh(&trans_pcie->reg_lock);
-	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, value);
+	_iwl_trans_set_bits_mask(trans, reg, mask, value);
 	spin_unlock_bh(&trans_pcie->reg_lock);
 }
 
@@ -2675,7 +2752,7 @@ static void *iwl_dbgfs_tx_queue_seq_start(struct seq_file *seq, loff_t *pos)
 	struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
 	struct iwl_dbgfs_tx_queue_state *state;
 
-	if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
+	if (*pos >= priv->trans->mac_cfg->base->num_of_queues)
 		return NULL;
 
 	state = kmalloc(sizeof(*state), GFP_KERNEL);
@@ -2693,7 +2770,7 @@ static void *iwl_dbgfs_tx_queue_seq_next(struct seq_file *seq,
 
 	*pos = ++state->pos;
 
-	if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
+	if (*pos >= priv->trans->mac_cfg->base->num_of_queues)
 		return NULL;
 
 	return state;
@@ -2725,7 +2802,7 @@ static int iwl_dbgfs_tx_queue_seq_show(struct seq_file *seq, void *v)
 	else
 		seq_puts(seq, "(unallocated)");
 
-	if (state->pos == trans_pcie->txqs.cmd.q_id)
+	if (state->pos == trans->conf.cmd_queue)
 		seq_puts(seq, " (HCMD)");
 	seq_puts(seq, "\n");
 
@@ -2763,7 +2840,7 @@ static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
 	int pos = 0, i, ret;
 	size_t bufsz;
 
-	bufsz = sizeof(char) * 121 * trans->num_rx_queues;
+	bufsz = sizeof(char) * 121 * trans->info.num_rxqs;
 
 	if (!trans_pcie->rxq)
 		return -EAGAIN;
@@ -2772,9 +2849,11 @@ static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
 	if (!buf)
 		return -ENOMEM;
 
-	for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) {
+	for (i = 0; i < trans->info.num_rxqs && pos < bufsz; i++) {
 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
 
+		spin_lock_bh(&rxq->lock);
+
 		pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n",
 				 i);
 		pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n",
@@ -2795,6 +2874,7 @@ static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
 			pos += scnprintf(buf + pos, bufsz - pos,
 					 "\tclosed_rb_num: Not Allocated\n");
 		}
+		spin_unlock_bh(&rxq->lock);
 	}
 	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
 	kfree(buf);
@@ -3093,12 +3173,58 @@ static ssize_t iwl_dbgfs_rf_read(struct file *file,
 				       strlen(trans_pcie->rf_name));
 }
 
+static ssize_t iwl_dbgfs_reset_write(struct file *file,
+				     const char __user *user_buf,
+				     size_t count, loff_t *ppos)
+{
+	struct iwl_trans *trans = file->private_data;
+	static const char * const modes[] = {
+		[IWL_RESET_MODE_SW_RESET] = "sw",
+		[IWL_RESET_MODE_REPROBE] = "reprobe",
+		[IWL_RESET_MODE_TOP_RESET] = "top",
+		[IWL_RESET_MODE_REMOVE_ONLY] = "remove",
+		[IWL_RESET_MODE_RESCAN] = "rescan",
+		[IWL_RESET_MODE_FUNC_RESET] = "function",
+		[IWL_RESET_MODE_PROD_RESET] = "product",
+	};
+	char buf[10] = {};
+	int mode;
+
+	if (count > sizeof(buf) - 1)
+		return -EINVAL;
+
+	if (copy_from_user(buf, user_buf, count))
+		return -EFAULT;
+
+	mode = sysfs_match_string(modes, buf);
+	if (mode < 0)
+		return mode;
+
+	if (mode < IWL_RESET_MODE_REMOVE_ONLY) {
+		if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
+			return -EINVAL;
+		if (mode == IWL_RESET_MODE_TOP_RESET) {
+			if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_SC)
+				return -EINVAL;
+			trans->request_top_reset = 1;
+		}
+		iwl_op_mode_nic_error(trans->op_mode, IWL_ERR_TYPE_DEBUGFS);
+		iwl_trans_schedule_reset(trans, IWL_ERR_TYPE_DEBUGFS);
+		return count;
+	}
+
+	iwl_trans_pcie_reset(trans, mode);
+
+	return count;
+}
+
 DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
 DEBUGFS_READ_FILE_OPS(fh_reg);
 DEBUGFS_READ_FILE_OPS(rx_queue);
 DEBUGFS_WRITE_FILE_OPS(csr);
 DEBUGFS_READ_WRITE_FILE_OPS(rfkill);
 DEBUGFS_READ_FILE_OPS(rf);
+DEBUGFS_WRITE_FILE_OPS(reset);
 
 static const struct file_operations iwl_dbgfs_tx_queue_ops = {
 	.owner = THIS_MODULE,
@@ -3127,6 +3253,7 @@ void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
 	DEBUGFS_ADD_FILE(rfkill, dir, 0600);
 	DEBUGFS_ADD_FILE(monitor_data, dir, 0400);
 	DEBUGFS_ADD_FILE(rf, dir, 0400);
+	DEBUGFS_ADD_FILE(reset, dir, 0200);
 }
 
 void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans)
@@ -3225,7 +3352,7 @@ static u32 iwl_trans_pcie_fh_regs_dump(struct iwl_trans *trans,
 	(*data)->len = cpu_to_le32(fh_regs_len);
 	val = (void *)(*data)->data;
 
-	if (!trans->trans_cfg->gen2)
+	if (!trans->mac_cfg->gen2)
 		for (i = FH_MEM_LOWER_BOUND; i < FH_MEM_UPPER_BOUND;
 		     i += sizeof(u32))
 			*val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
@@ -3272,7 +3399,7 @@ iwl_trans_pcie_dump_pointers(struct iwl_trans *trans,
 {
 	u32 base, base_high, write_ptr, write_ptr_val, wrap_cnt;
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		base = DBGC_CUR_DBGBUF_BASE_ADDR_LSB;
 		base_high = DBGC_CUR_DBGBUF_BASE_ADDR_MSB;
 		write_ptr = DBGC_CUR_DBGBUF_STATUS;
@@ -3292,7 +3419,7 @@ iwl_trans_pcie_dump_pointers(struct iwl_trans *trans,
 		cpu_to_le32(iwl_read_prph(trans, wrap_cnt));
 	fw_mon_data->fw_mon_base_ptr =
 		cpu_to_le32(iwl_read_prph(trans, base));
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		fw_mon_data->fw_mon_base_high_ptr =
 			cpu_to_le32(iwl_read_prph(trans, base_high));
 		write_ptr_val &= DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK;
@@ -3312,8 +3439,8 @@ iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
 
 	if (trans->dbg.dest_tlv ||
 	    (fw_mon->size &&
-	     (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000 ||
-	      trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))) {
+	     (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_7000 ||
+	      trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))) {
 		struct iwl_fw_error_dump_fw_mon *fw_mon_data;
 
 		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FW_MONITOR);
@@ -3336,14 +3463,14 @@ iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
 					IWL_LDBG_M2S_BUF_BA_MSK) <<
 				       trans->dbg.dest_tlv->base_shift;
 				base *= IWL_M2S_UNIT_SIZE;
-				base += trans->cfg->smem_offset;
+				base += trans->mac_cfg->base->smem_offset;
 			} else {
 				base = iwl_read_prph(trans, base) <<
 				       trans->dbg.dest_tlv->base_shift;
 			}
 
-			iwl_trans_read_mem(trans, base, fw_mon_data->data,
-					   monitor_len / sizeof(u32));
+			iwl_trans_pcie_read_mem(trans, base, fw_mon_data->data,
+						monitor_len / sizeof(u32));
 		} else if (trans->dbg.dest_tlv->monitor_mode == MARBH_MODE) {
 			monitor_len =
 				iwl_trans_pci_dump_marbh_monitor(trans,
@@ -3377,7 +3504,7 @@ static int iwl_trans_get_fw_monitor_len(struct iwl_trans *trans, u32 *len)
 			base = (cfg_reg & IWL_LDBG_M2S_BUF_BA_MSK) <<
 				trans->dbg.dest_tlv->base_shift;
 			base *= IWL_M2S_UNIT_SIZE;
-			base += trans->cfg->smem_offset;
+			base += trans->mac_cfg->base->smem_offset;
 
 			monitor_len =
 				(cfg_reg & IWL_LDBG_M2S_BUF_SIZE_MSK) >>
@@ -3393,7 +3520,7 @@ static int iwl_trans_get_fw_monitor_len(struct iwl_trans *trans, u32 *len)
 			      trans->dbg.dest_tlv->end_shift;
 
 			/* Make "end" point to the actual end */
-			if (trans->trans_cfg->device_family >=
+			if (trans->mac_cfg->device_family >=
 			    IWL_DEVICE_FAMILY_8000 ||
 			    trans->dbg.dest_tlv->monitor_mode == MARBH_MODE)
 				end += (1 << trans->dbg.dest_tlv->end_shift);
@@ -3414,13 +3541,13 @@ iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_fw_error_dump_data *data;
-	struct iwl_txq *cmdq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
+	struct iwl_txq *cmdq = trans_pcie->txqs.txq[trans->conf.cmd_queue];
 	struct iwl_fw_error_dump_txcmd *txcmd;
 	struct iwl_trans_dump_data *dump_data;
 	u32 len, num_rbs = 0, monitor_len = 0;
 	int i, ptr;
 	bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) &&
-			!trans->trans_cfg->mq_rx_supported &&
+			!trans->mac_cfg->mq_rx_supported &&
 			dump_mask & BIT(IWL_FW_ERROR_DUMP_RB);
 
 	if (!dump_mask)
@@ -3445,7 +3572,7 @@ iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
 
 	/* FH registers */
 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS)) {
-		if (trans->trans_cfg->gen2)
+		if (trans->mac_cfg->gen2)
 			len += sizeof(*data) +
 			       (iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2) -
 				iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2));
@@ -3459,15 +3586,18 @@ iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
 		/* Dump RBs is supported only for pre-9000 devices (1 queue) */
 		struct iwl_rxq *rxq = &trans_pcie->rxq[0];
 		/* RBs */
+		spin_lock_bh(&rxq->lock);
 		num_rbs = iwl_get_closed_rb_stts(trans, rxq);
 		num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK;
+		spin_unlock_bh(&rxq->lock);
+
 		len += num_rbs * (sizeof(*data) +
 				  sizeof(struct iwl_fw_error_dump_rb) +
 				  (PAGE_SIZE << trans_pcie->rx_page_order));
 	}
 
 	/* Paged memory for gen2 HW */
-	if (trans->trans_cfg->gen2 && dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING))
+	if (trans->mac_cfg->gen2 && dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING))
 		for (i = 0; i < trans->init_dram.paging_cnt; i++)
 			len += sizeof(*data) +
 			       sizeof(struct iwl_fw_error_dump_paging) +
@@ -3492,7 +3622,7 @@ iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
 			u8 tfdidx;
 			u32 caplen, cmdlen;
 
-			if (trans->trans_cfg->gen2)
+			if (trans->mac_cfg->gen2)
 				tfdidx = idx;
 			else
 				tfdidx = ptr;
@@ -3532,7 +3662,7 @@ iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
 		len += iwl_trans_pcie_dump_rbs(trans, &data, num_rbs);
 
 	/* Paged memory for gen2 HW */
-	if (trans->trans_cfg->gen2 &&
+	if (trans->mac_cfg->gen2 &&
 	    dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING)) {
 		for (i = 0; i < trans->init_dram.paging_cnt; i++) {
 			struct iwl_fw_error_dump_paging *paging;
@@ -3572,7 +3702,7 @@ void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans)
 
 	if (trans_pcie->msix_enabled) {
 		inta_addr = CSR_MSIX_HW_INT_CAUSES_AD;
-		if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+		if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 			sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ;
 		else
 			sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR;
@@ -3584,36 +3714,60 @@ void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans)
 	iwl_trans_sync_nmi_with_addr(trans, inta_addr, sw_err_bit);
 }
 
-struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
-			       const struct pci_device_id *ent,
-			       const struct iwl_cfg_trans_params *cfg_trans)
+static int iwl_trans_pcie_set_txcmd_info(const struct iwl_mac_cfg *mac_cfg,
+					 unsigned int *txcmd_size,
+					 unsigned int *txcmd_align)
+{
+	if (!mac_cfg->gen2) {
+		*txcmd_size = sizeof(struct iwl_tx_cmd_v6);
+		*txcmd_align = sizeof(void *);
+	} else if (mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
+		*txcmd_size = sizeof(struct iwl_tx_cmd_v9);
+		*txcmd_align = 64;
+	} else {
+		*txcmd_size = sizeof(struct iwl_tx_cmd);
+		*txcmd_align = 128;
+	}
+
+	*txcmd_size += sizeof(struct iwl_cmd_header);
+	*txcmd_size += 36; /* biggest possible 802.11 header */
+
+	/* Ensure device TX cmd cannot reach/cross a page boundary in gen2 */
+	if (WARN_ON((mac_cfg->gen2 && *txcmd_size >= *txcmd_align)))
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct iwl_trans *
+iwl_trans_pcie_alloc(struct pci_dev *pdev,
+		     const struct iwl_mac_cfg *mac_cfg,
+		     struct iwl_trans_info *info, u8 __iomem *hw_base)
 {
 	struct iwl_trans_pcie *trans_pcie, **priv;
+	unsigned int txcmd_size, txcmd_align;
 	struct iwl_trans *trans;
+	unsigned int bc_tbl_n_entries;
 	int ret, addr_size;
-	void __iomem * const *table;
-	u32 bar0;
-
-	/* reassign our BAR 0 if invalid due to possible runtime PM races */
-	pci_read_config_dword(pdev, PCI_BASE_ADDRESS_0, &bar0);
-	if (bar0 == PCI_BASE_ADDRESS_MEM_TYPE_64) {
-		ret = pci_assign_resource(pdev, 0);
-		if (ret)
-			return ERR_PTR(ret);
-	}
 
-	ret = pcim_enable_device(pdev);
+	ret = iwl_trans_pcie_set_txcmd_info(mac_cfg, &txcmd_size,
+					    &txcmd_align);
 	if (ret)
 		return ERR_PTR(ret);
 
 	trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie), &pdev->dev,
-				cfg_trans);
+				mac_cfg, txcmd_size, txcmd_align);
 	if (!trans)
 		return ERR_PTR(-ENOMEM);
 
 	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
-	if (trans->trans_cfg->gen2) {
+	trans_pcie->hw_base = hw_base;
+
+	/* Initialize the wait queue for commands */
+	init_waitqueue_head(&trans_pcie->wait_command_queue);
+
+	if (trans->mac_cfg->gen2) {
 		trans_pcie->txqs.tfd.addr_size = 64;
 		trans_pcie->txqs.tfd.max_tbs = IWL_TFH_NUM_TBS;
 		trans_pcie->txqs.tfd.size = sizeof(struct iwl_tfh_tfd);
@@ -3622,7 +3776,12 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		trans_pcie->txqs.tfd.max_tbs = IWL_NUM_OF_TBS;
 		trans_pcie->txqs.tfd.size = sizeof(struct iwl_tfd);
 	}
-	trans->max_skb_frags = IWL_TRANS_PCIE_MAX_FRAGS(trans_pcie);
+
+	trans_pcie->supported_dma_mask = (u32)DMA_BIT_MASK(12);
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+		trans_pcie->supported_dma_mask = (u32)DMA_BIT_MASK(11);
+
+	info->max_skb_frags = IWL_TRANS_PCIE_MAX_FRAGS(trans_pcie);
 
 #ifdef CONFIG_INET
 	trans_pcie->txqs.tso_hdr_page = alloc_percpu(struct iwl_tso_hdr_page);
@@ -3632,20 +3791,21 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 	}
 #endif
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
-		trans_pcie->txqs.bc_tbl_size =
-			sizeof(struct iwl_gen3_bc_tbl_entry) * TFD_QUEUE_BC_SIZE_GEN3_BZ;
-	else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
-		trans_pcie->txqs.bc_tbl_size =
-			sizeof(struct iwl_gen3_bc_tbl_entry) * TFD_QUEUE_BC_SIZE_GEN3_AX210;
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+		bc_tbl_n_entries = TFD_QUEUE_BC_SIZE_BZ;
+	else if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+		bc_tbl_n_entries = TFD_QUEUE_BC_SIZE_AX210;
 	else
-		trans_pcie->txqs.bc_tbl_size = sizeof(struct iwlagn_scd_bc_tbl);
+		bc_tbl_n_entries = TFD_QUEUE_BC_SIZE;
+
+	trans_pcie->txqs.bc_tbl_size =
+		sizeof(struct iwl_bc_tbl_entry) * bc_tbl_n_entries;
 	/*
 	 * For gen2 devices, we use a single allocation for each byte-count
 	 * table, but they're pretty small (1k) so use a DMA pool that we
 	 * allocate here.
 	 */
-	if (trans->trans_cfg->gen2) {
+	if (trans->mac_cfg->gen2) {
 		trans_pcie->txqs.bc_pool =
 			dmam_pool_create("iwlwifi:bc", trans->dev,
 					 trans_pcie->txqs.bc_tbl_size,
@@ -3658,7 +3818,7 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 
 	/* Some things must not change even if the config does */
 	WARN_ON(trans_pcie->txqs.tfd.addr_size !=
-		(trans->trans_cfg->gen2 ? 64 : 36));
+		(trans->mac_cfg->gen2 ? 64 : 36));
 
 	/* Initialize NAPI here - it should be before registering to mac80211
 	 * in the opmode but after the HW struct is allocated.
@@ -3692,7 +3852,7 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 
 	trans_pcie->debug_rfkill = -1;
 
-	if (!cfg_trans->base_params->pcie_l1_allowed) {
+	if (!mac_cfg->base->pcie_l1_allowed) {
 		/*
 		 * W/A - seems to solve weird behavior. We need to remove this
 		 * if we don't want to stay in L1 all the time. This wastes a
@@ -3703,8 +3863,6 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 				       PCIE_LINK_STATE_CLKPM);
 	}
 
-	pci_set_master(pdev);
-
 	addr_size = trans_pcie->txqs.tfd.addr_size;
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_size));
 	if (ret) {
@@ -3716,29 +3874,6 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		}
 	}
 
-	ret = pcim_iomap_regions_request_all(pdev, BIT(0), DRV_NAME);
-	if (ret) {
-		dev_err(&pdev->dev, "pcim_iomap_regions_request_all failed\n");
-		goto out_no_pci;
-	}
-
-#if defined(__FreeBSD__)
-	linuxkpi_pcim_want_to_use_bus_functions(pdev);
-#endif
-	table = pcim_iomap_table(pdev);
-	if (!table) {
-		dev_err(&pdev->dev, "pcim_iomap_table failed\n");
-		ret = -ENOMEM;
-		goto out_no_pci;
-	}
-
-	trans_pcie->hw_base = table[0];
-	if (!trans_pcie->hw_base) {
-		dev_err(&pdev->dev, "couldn't find IO mem in first BAR\n");
-		ret = -ENODEV;
-		goto out_no_pci;
-	}
-
 	/* We disable the RETRY_TIMEOUT register (0x41) to keep
 	 * PCI Tx retries from interfering with C3 CPU state */
 	pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);
@@ -3746,30 +3881,20 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 	trans_pcie->pci_dev = pdev;
 	iwl_disable_interrupts(trans);
 
-	trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
-	if (trans->hw_rev == 0xffffffff) {
-		dev_err(&pdev->dev, "HW_REV=0xFFFFFFFF, PCI issues?\n");
-		ret = -EIO;
-		goto out_no_pci;
-	}
-
 	/*
 	 * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
 	 * changed, and now the revision step also includes bit 0-1 (no more
 	 * "dash" value). To keep hw_rev backwards compatible - we'll store it
 	 * in the old format.
 	 */
-	if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_8000)
-		trans->hw_rev_step = trans->hw_rev & 0xF;
+	if (mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
+		info->hw_rev_step = info->hw_rev & 0xF;
 	else
-		trans->hw_rev_step = (trans->hw_rev & 0xC) >> 2;
+		info->hw_rev_step = (info->hw_rev & 0xC) >> 2;
 
-	IWL_DEBUG_INFO(trans, "HW REV: 0x%0x\n", trans->hw_rev);
+	IWL_DEBUG_INFO(trans, "HW REV: 0x%0x\n", info->hw_rev);
 
-	iwl_pcie_set_interrupt_capa(pdev, trans, cfg_trans);
-	trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
-	snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
-		 "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
+	iwl_pcie_set_interrupt_capa(pdev, trans, mac_cfg, info);
 
 	init_waitqueue_head(&trans_pcie->sx_waitq);
 
@@ -3778,7 +3903,7 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		goto out_no_pci;
 
 	if (trans_pcie->msix_enabled) {
-		ret = iwl_pcie_init_msix_handler(pdev, trans_pcie);
+		ret = iwl_pcie_init_msix_handler(pdev, trans_pcie, info);
 		if (ret)
 			goto out_no_pci;
 	 } else {
@@ -3851,9 +3976,332 @@ int iwl_trans_pcie_copy_imr(struct iwl_trans *trans,
 				 IMR_D2S_REQUESTED, 5 * HZ);
 	if (!ret || trans_pcie->imr_status == IMR_D2S_ERROR) {
 		IWL_ERR(trans, "Failed to copy IMR Memory chunk!\n");
-		iwl_trans_pcie_dump_regs(trans);
+		iwl_trans_pcie_dump_regs(trans, trans_pcie->pci_dev);
 		return -ETIMEDOUT;
 	}
 	trans_pcie->imr_status = IMR_D2S_IDLE;
 	return 0;
 }
+
+/*
+ * Read rf id and cdb info from prph register and store it
+ */
+static void get_crf_id(struct iwl_trans *iwl_trans,
+		       struct iwl_trans_info *info)
+{
+	u32 sd_reg_ver_addr;
+	u32 hw_wfpm_id;
+	u32 val = 0;
+	u8 step;
+
+	if (iwl_trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+		sd_reg_ver_addr = SD_REG_VER_GEN2;
+	else
+		sd_reg_ver_addr = SD_REG_VER;
+
+	/* Enable access to peripheral registers */
+	val = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG);
+	val |= WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK;
+	iwl_write_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG, val);
+
+	/* Read crf info */
+	info->hw_crf_id = iwl_read_prph_no_grab(iwl_trans, sd_reg_ver_addr);
+
+	/* Read cnv info */
+	info->hw_cnv_id = iwl_read_prph_no_grab(iwl_trans, CNVI_AUX_MISC_CHIP);
+
+	/* For BZ-W, take B step also when A step is indicated */
+	if (CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ_W)
+		step = SILICON_B_STEP;
+
+	/* In BZ, the MAC step must be read from the CNVI aux register */
+	if (CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ) {
+		step = CNVI_AUX_MISC_CHIP_MAC_STEP(info->hw_cnv_id);
+
+		/* For BZ-U, take B step also when A step is indicated */
+		if ((CNVI_AUX_MISC_CHIP_PROD_TYPE(info->hw_cnv_id) ==
+		    CNVI_AUX_MISC_CHIP_PROD_TYPE_BZ_U) &&
+		    step == SILICON_A_STEP)
+			step = SILICON_B_STEP;
+	}
+
+	if (CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ ||
+	    CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ_W) {
+		info->hw_rev_step = step;
+		info->hw_rev |= step;
+	}
+
+	/* Read cdb info (also contains the jacket info if needed in the future */
+	hw_wfpm_id = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_OTP_CFG1_ADDR);
+	IWL_INFO(iwl_trans, "Detected crf-id 0x%x, cnv-id 0x%x wfpm id 0x%x\n",
+		 info->hw_crf_id, info->hw_cnv_id, hw_wfpm_id);
+}
+
+/*
+ * In case that there is no OTP on the NIC, map the rf id and cdb info
+ * from the prph registers.
+ */
+static int map_crf_id(struct iwl_trans *iwl_trans,
+		      struct iwl_trans_info *info)
+{
+	int ret = 0;
+	u32 val = info->hw_crf_id;
+	u32 step_id = REG_CRF_ID_STEP(val);
+	u32 slave_id = REG_CRF_ID_SLAVE(val);
+	u32 jacket_id_cnv = REG_CRF_ID_SLAVE(info->hw_cnv_id);
+	u32 hw_wfpm_id = iwl_read_umac_prph_no_grab(iwl_trans,
+						    WFPM_OTP_CFG1_ADDR);
+	u32 jacket_id_wfpm = WFPM_OTP_CFG1_IS_JACKET(hw_wfpm_id);
+	u32 cdb_id_wfpm = WFPM_OTP_CFG1_IS_CDB(hw_wfpm_id);
+
+	/* Map between crf id to rf id */
+	switch (REG_CRF_ID_TYPE(val)) {
+	case REG_CRF_ID_TYPE_JF_1:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_JF1 << 12);
+		break;
+	case REG_CRF_ID_TYPE_JF_2:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_JF2 << 12);
+		break;
+	case REG_CRF_ID_TYPE_HR_NONE_CDB_1X1:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_HR1 << 12);
+		break;
+	case REG_CRF_ID_TYPE_HR_NONE_CDB:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12);
+		break;
+	case REG_CRF_ID_TYPE_HR_CDB:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12);
+		break;
+	case REG_CRF_ID_TYPE_GF:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_GF << 12);
+		break;
+	case REG_CRF_ID_TYPE_FM:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_FM << 12);
+		break;
+	case REG_CRF_ID_TYPE_WHP:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_WH << 12);
+		break;
+	case REG_CRF_ID_TYPE_PE:
+		info->hw_rf_id = (IWL_CFG_RF_TYPE_PE << 12);
+		break;
+	default:
+		ret = -EIO;
+		IWL_ERR(iwl_trans,
+			"Can't find a correct rfid for crf id 0x%x\n",
+			REG_CRF_ID_TYPE(val));
+		goto out;
+	}
+
+	/* Set Step-id */
+	info->hw_rf_id |= (step_id << 8);
+
+	/* Set CDB capabilities */
+	if (cdb_id_wfpm || slave_id) {
+		info->hw_rf_id += BIT(28);
+		IWL_INFO(iwl_trans, "Adding cdb to rf id\n");
+	}
+
+	/* Set Jacket capabilities */
+	if (jacket_id_wfpm || jacket_id_cnv) {
+		info->hw_rf_id += BIT(29);
+		IWL_INFO(iwl_trans, "Adding jacket to rf id\n");
+	}
+
+	IWL_INFO(iwl_trans,
+		 "Detected rf-type 0x%x step-id 0x%x slave-id 0x%x from crf id 0x%x\n",
+		 REG_CRF_ID_TYPE(val), step_id, slave_id, info->hw_rf_id);
+	IWL_INFO(iwl_trans,
+		 "Detected cdb-id 0x%x jacket-id 0x%x from wfpm id 0x%x\n",
+		 cdb_id_wfpm, jacket_id_wfpm, hw_wfpm_id);
+	IWL_INFO(iwl_trans, "Detected jacket-id 0x%x from cnvi id 0x%x\n",
+		 jacket_id_cnv, info->hw_cnv_id);
+
+out:
+	return ret;
+}
+
+static void iwl_pcie_recheck_me_status(struct work_struct *wk)
+{
+	struct iwl_trans_pcie *trans_pcie = container_of(wk,
+							 typeof(*trans_pcie),
+							 me_recheck_wk.work);
+	u32 val;
+
+	val = iwl_read32(trans_pcie->trans, CSR_HW_IF_CONFIG_REG);
+	trans_pcie->me_present = !!(val & CSR_HW_IF_CONFIG_REG_IAMT_UP);
+}
+
+static void iwl_pcie_check_me_status(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	u32 val;
+
+	trans_pcie->me_present = -1;
+
+	INIT_DELAYED_WORK(&trans_pcie->me_recheck_wk,
+			  iwl_pcie_recheck_me_status);
+
+	/* we don't have a good way of determining this until BZ */
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
+		return;
+
+	val = iwl_read_prph(trans, CNVI_SCU_REG_FOR_ECO_1);
+	if (val & CNVI_SCU_REG_FOR_ECO_1_WIAMT_KNOWN) {
+		trans_pcie->me_present =
+			!!(val & CNVI_SCU_REG_FOR_ECO_1_WIAMT_PRESENT);
+		return;
+	}
+
+	val = iwl_read32(trans, CSR_HW_IF_CONFIG_REG);
+	if (val & (CSR_HW_IF_CONFIG_REG_ME_OWN |
+		   CSR_HW_IF_CONFIG_REG_IAMT_UP)) {
+		trans_pcie->me_present = 1;
+		return;
+	}
+
+	/* recheck again later, ME might still be initializing */
+	schedule_delayed_work(&trans_pcie->me_recheck_wk, HZ);
+}
+
+int iwl_pci_gen1_2_probe(struct pci_dev *pdev,
+			 const struct pci_device_id *ent,
+			 const struct iwl_mac_cfg *mac_cfg,
+			 u8 __iomem *hw_base, u32 hw_rev)
+{
+	const struct iwl_dev_info *dev_info;
+	struct iwl_trans_info info = {
+		.hw_id = (pdev->device << 16) + pdev->subsystem_device,
+		.hw_rev = hw_rev,
+	};
+	struct iwl_trans *iwl_trans;
+	struct iwl_trans_pcie *trans_pcie;
+	int ret;
+
+	iwl_trans = iwl_trans_pcie_alloc(pdev, mac_cfg, &info, hw_base);
+	if (IS_ERR(iwl_trans))
+		return PTR_ERR(iwl_trans);
+
+	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(iwl_trans);
+
+	iwl_trans_pcie_check_product_reset_status(pdev);
+	iwl_trans_pcie_check_product_reset_mode(pdev);
+
+	/* set the things we know so far for the grab NIC access */
+	iwl_trans_set_info(iwl_trans, &info);
+
+	/*
+	 * Let's try to grab NIC access early here. Sometimes, NICs may
+	 * fail to initialize, and if that happens it's better if we see
+	 * issues early on (and can reprobe, per the logic inside), than
+	 * first trying to load the firmware etc. and potentially only
+	 * detecting any problems when the first interface is brought up.
+	 */
+	ret = iwl_pcie_prepare_card_hw(iwl_trans);
+	if (!ret) {
+		ret = iwl_finish_nic_init(iwl_trans);
+		if (ret)
+			goto out_free_trans;
+		if (iwl_trans_grab_nic_access(iwl_trans)) {
+			get_crf_id(iwl_trans, &info);
+			/* all good */
+			iwl_trans_release_nic_access(iwl_trans);
+		} else {
+			ret = -EIO;
+			goto out_free_trans;
+		}
+	}
+
+	info.hw_rf_id = iwl_read32(iwl_trans, CSR_HW_RF_ID);
+
+	/*
+	 * The RF_ID is set to zero in blank OTP so read version to
+	 * extract the RF_ID.
+	 * This is relevant only for family 9000 and up.
+	 */
+	if (iwl_trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_9000 &&
+	    !CSR_HW_RFID_TYPE(info.hw_rf_id) && map_crf_id(iwl_trans, &info)) {
+		ret = -EINVAL;
+		goto out_free_trans;
+	}
+
+	IWL_INFO(iwl_trans, "PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n",
+		 pdev->device, pdev->subsystem_device,
+		 info.hw_rev, info.hw_rf_id);
+
+	dev_info = iwl_pci_find_dev_info(pdev->device, pdev->subsystem_device,
+					 CSR_HW_RFID_TYPE(info.hw_rf_id),
+					 CSR_HW_RFID_IS_CDB(info.hw_rf_id),
+					 IWL_SUBDEVICE_RF_ID(pdev->subsystem_device),
+					 IWL_SUBDEVICE_BW_LIM(pdev->subsystem_device),
+					 !iwl_trans->mac_cfg->integrated);
+	if (dev_info) {
+		iwl_trans->cfg = dev_info->cfg;
+		info.name = dev_info->name;
+	}
+
+#if IS_ENABLED(CONFIG_IWLMVM)
+
+	/*
+	 * special-case 7265D, it has the same PCI IDs.
+	 *
+	 * Note that because we already pass the cfg to the transport above,
+	 * all the parameters that the transport uses must, until that is
+	 * changed, be identical to the ones in the 7265D configuration.
+	 */
+	if (iwl_trans->cfg == &iwl7265_cfg &&
+	    (info.hw_rev & CSR_HW_REV_TYPE_MSK) == CSR_HW_REV_TYPE_7265D)
+		iwl_trans->cfg = &iwl7265d_cfg;
+#endif
+	if (!iwl_trans->cfg) {
+		pr_err("No config found for PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n",
+		       pdev->device, pdev->subsystem_device,
+		       info.hw_rev, info.hw_rf_id);
+		ret = -EINVAL;
+		goto out_free_trans;
+	}
+
+	IWL_INFO(iwl_trans, "Detected %s\n", info.name);
+
+	if (iwl_trans->mac_cfg->mq_rx_supported) {
+		if (WARN_ON(!iwl_trans->cfg->num_rbds)) {
+			ret = -EINVAL;
+			goto out_free_trans;
+		}
+		trans_pcie->num_rx_bufs = iwl_trans_get_num_rbds(iwl_trans);
+	} else {
+		trans_pcie->num_rx_bufs = RX_QUEUE_SIZE;
+	}
+
+	if (!iwl_trans->mac_cfg->integrated) {
+		u16 link_status;
+
+		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &link_status);
+
+		info.pcie_link_speed =
+			u16_get_bits(link_status, PCI_EXP_LNKSTA_CLS);
+	}
+
+	iwl_trans_set_info(iwl_trans, &info);
+
+	pci_set_drvdata(pdev, iwl_trans);
+
+	iwl_pcie_check_me_status(iwl_trans);
+
+	/* try to get ownership so that we'll know if we don't own it */
+	iwl_pcie_prepare_card_hw(iwl_trans);
+
+	iwl_trans->drv = iwl_drv_start(iwl_trans);
+
+	if (IS_ERR(iwl_trans->drv)) {
+		ret = PTR_ERR(iwl_trans->drv);
+		goto out_free_trans;
+	}
+
+	/* register transport layer debugfs here */
+	iwl_trans_pcie_dbgfs_register(iwl_trans);
+
+	return 0;
+
+out_free_trans:
+	iwl_trans_pcie_free(iwl_trans);
+	return ret;
+}
diff --git a/sys/contrib/dev/iwlwifi/pcie/tx-gen2.c b/sys/contrib/dev/iwlwifi/pcie/gen1_2/tx-gen2.c
index 72be71184f22..4163d6518ec6 100644
--- a/sys/contrib/dev/iwlwifi/pcie/tx-gen2.c
+++ b/sys/contrib/dev/iwlwifi/pcie/gen1_2/tx-gen2.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2020, 2023-2024 Intel Corporation
+ * Copyright (C) 2018-2020, 2023-2025 Intel Corporation
  */
 #ifdef CONFIG_INET
 #include <net/tso.h>
@@ -20,13 +20,12 @@
 static struct page *get_workaround_page(struct iwl_trans *trans,
 					struct sk_buff *skb)
 {
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_tso_page_info *info;
 	struct page **page_ptr;
 	struct page *ret;
 	dma_addr_t phys;
 
-	page_ptr = (void *)((u8 *)skb->cb + trans_pcie->txqs.page_offs);
+	page_ptr = (void *)((u8 *)skb->cb + trans->conf.cb_data_offs);
 
 	ret = alloc_page(GFP_ATOMIC);
 	if (!ret)
@@ -166,7 +165,7 @@ static int iwl_txq_gen2_build_amsdu(struct iwl_trans *trans,
 				    struct iwl_device_tx_cmd *dev_cmd)
 {
 #ifdef CONFIG_INET
-	struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload;
+	struct iwl_tx_cmd_v9 *tx_cmd = (void *)dev_cmd->payload;
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room;
 	unsigned int mss = skb_shinfo(skb)->gso_size;
@@ -190,7 +189,8 @@ static int iwl_txq_gen2_build_amsdu(struct iwl_trans *trans,
 		(3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr));
 
 	/* Our device supports 9 segments at most, it will fit in 1 page */
-	sgt = iwl_pcie_prep_tso(trans, skb, out_meta, &start_hdr, hdr_room);
+	sgt = iwl_pcie_prep_tso(trans, skb, out_meta, &start_hdr, hdr_room,
+				snap_ip_tcp_hdrlen + hdr_len);
 	if (!sgt)
 		return -ENOMEM;
 
@@ -349,6 +349,7 @@ iwl_tfh_tfd *iwl_txq_gen2_build_tx_amsdu(struct iwl_trans *trans,
 	return tfd;
 
 out_err:
+	iwl_pcie_free_tso_pages(trans, skb, out_meta);
 	iwl_txq_gen2_tfd_unmap(trans, out_meta, tfd);
 	return NULL;
 }
@@ -491,21 +492,21 @@ struct iwl_tfh_tfd *iwl_txq_gen2_build_tfd(struct iwl_trans *trans,
 	bool amsdu;
 
 	/* There must be data left over for TB1 or this code must be changed */
-	BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen2) < IWL_FIRST_TB_SIZE);
+	BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_v9) < IWL_FIRST_TB_SIZE);
 	BUILD_BUG_ON(sizeof(struct iwl_cmd_header) +
-		     offsetofend(struct iwl_tx_cmd_gen2, dram_info) >
+		     offsetofend(struct iwl_tx_cmd_v9, dram_info) >
 		     IWL_FIRST_TB_SIZE);
-	BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen3) < IWL_FIRST_TB_SIZE);
+	BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE);
 	BUILD_BUG_ON(sizeof(struct iwl_cmd_header) +
-		     offsetofend(struct iwl_tx_cmd_gen3, dram_info) >
+		     offsetofend(struct iwl_tx_cmd, dram_info) >
 		     IWL_FIRST_TB_SIZE);
 
 	memset(tfd, 0, sizeof(*tfd));
 
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
-		len = sizeof(struct iwl_tx_cmd_gen2);
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+		len = sizeof(struct iwl_tx_cmd_v9);
 	else
-		len = sizeof(struct iwl_tx_cmd_gen3);
+		len = sizeof(struct iwl_tx_cmd);
 
 	amsdu = ieee80211_is_data_qos(hdr->frame_control) &&
 			(*ieee80211_get_qos_ctl(hdr) &
@@ -536,17 +537,17 @@ int iwl_txq_space(struct iwl_trans *trans, const struct iwl_txq *q)
 	 * If q->n_window is smaller than max_tfd_queue_size, there is no need
 	 * to reserve any queue entries for this purpose.
 	 */
-	if (q->n_window < trans->trans_cfg->base_params->max_tfd_queue_size)
+	if (q->n_window < trans->mac_cfg->base->max_tfd_queue_size)
 		max = q->n_window;
 	else
-		max = trans->trans_cfg->base_params->max_tfd_queue_size - 1;
+		max = trans->mac_cfg->base->max_tfd_queue_size - 1;
 
 	/*
 	 * max_tfd_queue_size is a power of 2, so the following is equivalent to
 	 * modulo by max_tfd_queue_size and is well defined.
 	 */
 	used = (q->write_ptr - q->read_ptr) &
-		(trans->trans_cfg->base_params->max_tfd_queue_size - 1);
+		(trans->mac_cfg->base->max_tfd_queue_size - 1);
 
 	if (WARN_ON(used > max))
 		return 0;
@@ -561,8 +562,8 @@ static void iwl_pcie_gen2_update_byte_tbl(struct iwl_trans *trans,
 					  struct iwl_txq *txq, u16 byte_cnt,
 					  int num_tbs)
 {
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);
+	struct iwl_bc_tbl_entry *scd_bc_tbl = txq->bc_tbl.addr;
 	u8 filled_tfd_size, num_fetch_chunks;
 	u16 len = byte_cnt;
 	__le16 bc_ent;
@@ -582,24 +583,16 @@ static void iwl_pcie_gen2_update_byte_tbl(struct iwl_trans *trans,
 	 */
 	num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1;
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
-		struct iwl_gen3_bc_tbl_entry *scd_bc_tbl_gen3 = txq->bc_tbl.addr;
-
-		/* Starting from AX210, the HW expects bytes */
-		WARN_ON(trans_pcie->txqs.bc_table_dword);
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 		WARN_ON(len > 0x3FFF);
 		bc_ent = cpu_to_le16(len | (num_fetch_chunks << 14));
-		scd_bc_tbl_gen3[idx].tfd_offset = bc_ent;
 	} else {
-		struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr;
-
-		/* Before AX210, the HW expects DW */
-		WARN_ON(!trans_pcie->txqs.bc_table_dword);
 		len = DIV_ROUND_UP(len, 4);
 		WARN_ON(len > 0xFFF);
 		bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12));
-		scd_bc_tbl->tfd_offset[idx] = bc_ent;
 	}
+
+	scd_bc_tbl[idx].tfd_offset = bc_ent;
 }
 
 static u8 iwl_txq_gen2_get_num_tbs(struct iwl_tfh_tfd *tfd)
@@ -756,7 +749,8 @@ int iwl_txq_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
 			struct iwl_device_tx_cmd **dev_cmd_ptr;
 
 			dev_cmd_ptr = (void *)((u8 *)skb->cb +
-					       trans_pcie->txqs.dev_cmd_offs);
+					       trans->conf.cb_data_offs +
+					       sizeof(void *));
 
 			*dev_cmd_ptr = dev_cmd;
 			__skb_queue_tail(&txq->overflow_q, skb);
@@ -785,16 +779,16 @@ int iwl_txq_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
 		return -1;
 	}
 
-	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
-		struct iwl_tx_cmd_gen3 *tx_cmd_gen3 =
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+		struct iwl_tx_cmd *tx_cmd =
 			(void *)dev_cmd->payload;
 
-		cmd_len = le16_to_cpu(tx_cmd_gen3->len);
+		cmd_len = le16_to_cpu(tx_cmd->len);
 	} else {
-		struct iwl_tx_cmd_gen2 *tx_cmd_gen2 =
+		struct iwl_tx_cmd_v9 *tx_cmd_v9 =
 			(void *)dev_cmd->payload;
 
-		cmd_len = le16_to_cpu(tx_cmd_gen2->len);
+		cmd_len = le16_to_cpu(tx_cmd_v9->len);
 	}
 
 	/* Set up entry for this TFD in Tx byte-count array */
@@ -832,7 +826,7 @@ static void iwl_txq_gen2_unmap(struct iwl_trans *trans, int txq_id)
 		IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n",
 				   txq_id, txq->read_ptr);
 
-		if (txq_id != trans_pcie->txqs.cmd.q_id) {
+		if (txq_id != trans->conf.cmd_queue) {
 			int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr);
 			struct iwl_cmd_meta *cmd_meta = &txq->entries[idx].meta;
 			struct sk_buff *skb = txq->entries[idx].skb;
@@ -906,12 +900,12 @@ static void iwl_txq_gen2_free(struct iwl_trans *trans, int txq_id)
 	iwl_txq_gen2_unmap(trans, txq_id);
 
 	/* De-alloc array of command/tx buffers */
-	if (txq_id == trans_pcie->txqs.cmd.q_id)
+	if (txq_id == trans->conf.cmd_queue)
 		for (i = 0; i < txq->n_window; i++) {
 			kfree_sensitive(txq->entries[i].cmd);
 			kfree_sensitive(txq->entries[i].free_buf);
 		}
-	del_timer_sync(&txq->stuck_timer);
+	timer_delete_sync(&txq->stuck_timer);
 
 	iwl_txq_gen2_free_memory(trans, txq);
 
@@ -1007,7 +1001,7 @@ static int iwl_pcie_txq_alloc_response(struct iwl_trans *trans,
 
 	txq->id = qid;
 	trans_pcie->txqs.txq[qid] = txq;
-	wr_ptr &= (trans->trans_cfg->base_params->max_tfd_queue_size - 1);
+	wr_ptr &= (trans->mac_cfg->base->max_tfd_queue_size - 1);
 
 	/* Place first TFD at index corresponding to start sequence number */
 	txq->read_ptr = wr_ptr;
@@ -1043,8 +1037,8 @@ int iwl_txq_dyn_alloc(struct iwl_trans *trans, u32 flags, u32 sta_mask,
 	/* but must be power of 2 values for calculating read/write pointers */
 	size = rounddown_pow_of_two(size);
 
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_BZ &&
-	    trans->hw_rev_step == SILICON_A_STEP) {
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_BZ &&
+	    trans->info.hw_rev_step == SILICON_A_STEP) {
 		size = 4096;
 		txq = iwl_txq_dyn_alloc_dma(trans, size, timeout);
 	} else {
@@ -1064,7 +1058,7 @@ int iwl_txq_dyn_alloc(struct iwl_trans *trans, u32 flags, u32 sta_mask,
 	if (IS_ERR(txq))
 		return PTR_ERR(txq);
 
-	if (trans_pcie->txqs.queue_alloc_cmd_ver == 0) {
+	if (trans->conf.queue_alloc_cmd_ver == 0) {
 		memset(&cmd.old, 0, sizeof(cmd.old));
 		cmd.old.tfdq_addr = cpu_to_le64(txq->dma_addr);
 		cmd.old.byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma);
@@ -1081,7 +1075,7 @@ int iwl_txq_dyn_alloc(struct iwl_trans *trans, u32 flags, u32 sta_mask,
 		hcmd.id = SCD_QUEUE_CFG;
 		hcmd.len[0] = sizeof(cmd.old);
 		hcmd.data[0] = &cmd.old;
-	} else if (trans_pcie->txqs.queue_alloc_cmd_ver == 3) {
+	} else if (trans->conf.queue_alloc_cmd_ver == 3) {
 		memset(&cmd.new, 0, sizeof(cmd.new));
 		cmd.new.operation = cpu_to_le32(IWL_SCD_QUEUE_ADD);
 		cmd.new.u.add.tfdq_dram_addr = cpu_to_le64(txq->dma_addr);
@@ -1176,7 +1170,7 @@ int iwl_txq_gen2_init(struct iwl_trans *trans, int txq_id, int queue_size)
 	}
 
 	ret = iwl_txq_init(trans, queue, queue_size,
-			   (txq_id == trans_pcie->txqs.cmd.q_id));
+			   (txq_id == trans->conf.cmd_queue));
 	if (ret) {
 		IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id);
 		goto error;
@@ -1206,7 +1200,7 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 			       struct iwl_host_cmd *cmd)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwl_txq *txq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
+	struct iwl_txq *txq = trans_pcie->txqs.txq[trans->conf.cmd_queue];
 	struct iwl_device_cmd *out_cmd;
 	struct iwl_cmd_meta *out_meta;
 	void *dup_buf = NULL;
@@ -1300,7 +1294,9 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 		spin_unlock_irqrestore(&txq->lock, flags);
 
 		IWL_ERR(trans, "No space in command queue\n");
-		iwl_op_mode_cmd_queue_full(trans->op_mode);
+		iwl_op_mode_nic_error(trans->op_mode,
+				      IWL_ERR_TYPE_CMD_QUEUE_FULL);
+		iwl_trans_schedule_reset(trans, IWL_ERR_TYPE_CMD_QUEUE_FULL);
 		idx = -ENOSPC;
 		goto free_dup_buf;
 	}
@@ -1321,7 +1317,7 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 		cpu_to_le16(cmd_size - sizeof(struct iwl_cmd_header_wide));
 	out_cmd->hdr_wide.reserved = 0;
 	out_cmd->hdr_wide.sequence =
-		cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->txqs.cmd.q_id) |
+		cpu_to_le16(QUEUE_TO_SEQ(trans->conf.cmd_queue) |
 					 INDEX_TO_SEQ(txq->write_ptr));
 
 	cmd_pos = sizeof(struct iwl_cmd_header_wide);
@@ -1369,7 +1365,7 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 		     "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n",
 		     iwl_get_cmd_string(trans, cmd->id), group_id,
 		     out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence),
-		     cmd_size, txq->write_ptr, idx, trans_pcie->txqs.cmd.q_id);
+		     cmd_size, txq->write_ptr, idx, trans->conf.cmd_queue);
 
 	/* start the TFD with the minimum copy bytes */
 	tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE);
diff --git a/sys/contrib/dev/iwlwifi/pcie/tx.c b/sys/contrib/dev/iwlwifi/pcie/gen1_2/tx.c
index e2b8165aead6..5d62d19ad3cf 100644
--- a/sys/contrib/dev/iwlwifi/pcie/tx.c
+++ b/sys/contrib/dev/iwlwifi/pcie/gen1_2/tx.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2003-2014, 2018-2021, 2023-2024 Intel Corporation
+ * Copyright (C) 2003-2014, 2018-2021, 2023-2025 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -30,6 +30,8 @@
 #include "iwl-op-mode.h"
 #include "internal.h"
 #include "fw/api/tx.h"
+#include "fw/dbg.h"
+#include "pcie/utils.h"
 
 /*************** DMA-QUEUE-GENERAL-FUNCTIONS  *****
  * DMA services
@@ -83,7 +85,6 @@ void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr)
 static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans,
 				    struct iwl_txq *txq)
 {
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	u32 reg = 0;
 	int txq_id = txq->id;
 
@@ -95,8 +96,8 @@ static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans,
 	 * 2. NIC is woken up for CMD regardless of shadow outside this function
 	 * 3. there is a chance that the NIC is asleep
 	 */
-	if (!trans->trans_cfg->base_params->shadow_reg_enable &&
-	    txq_id != trans_pcie->txqs.cmd.q_id &&
+	if (!trans->mac_cfg->base->shadow_reg_enable &&
+	    txq_id != trans->conf.cmd_queue &&
 	    test_bit(STATUS_TPOWER_PMI, &trans->status)) {
 		/*
 		 * wake up nic if it's powered down ...
@@ -130,7 +131,7 @@ void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans)
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int i;
 
-	for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) {
+	for (i = 0; i < trans->mac_cfg->base->num_of_queues; i++) {
 		struct iwl_txq *txq = trans_pcie->txqs.txq[i];
 
 		if (!test_bit(i, trans_pcie->txqs.queue_used))
@@ -198,7 +199,7 @@ static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
-	if (!trans->trans_cfg->base_params->apmg_wake_up_wa)
+	if (!trans->mac_cfg->base->apmg_wake_up_wa)
 		return;
 
 	spin_lock(&trans_pcie->reg_lock);
@@ -209,8 +210,8 @@ static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans)
 	}
 
 	trans_pcie->cmd_hold_nic_awake = false;
-	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+	iwl_trans_clear_bit(trans, CSR_GP_CNTRL,
+			    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
 	spin_unlock(&trans_pcie->reg_lock);
 }
 
@@ -231,11 +232,10 @@ static void iwl_pcie_free_and_unmap_tso_page(struct iwl_trans *trans,
 void iwl_pcie_free_tso_pages(struct iwl_trans *trans, struct sk_buff *skb,
 			     struct iwl_cmd_meta *cmd_meta)
 {
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct page **page_ptr;
 	struct page *next;
 
-	page_ptr = (void *)((u8 *)skb->cb + trans_pcie->txqs.page_offs);
+	page_ptr = (void *)((u8 *)skb->cb + trans->conf.cb_data_offs);
 	next = *page_ptr;
 	*page_ptr = NULL;
 
@@ -285,10 +285,12 @@ iwl_txq_gen1_tfd_tb_get_addr(struct iwl_tfd *tfd, u8 idx)
 static void iwl_txq_set_tfd_invalid_gen1(struct iwl_trans *trans,
 					 struct iwl_tfd *tfd)
 {
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
 	tfd->num_tbs = 0;
 
-	iwl_pcie_gen1_tfd_set_tb(tfd, 0, trans->invalid_tx_cmd.dma,
-				 trans->invalid_tx_cmd.size);
+	iwl_pcie_gen1_tfd_set_tb(tfd, 0, trans_pcie->invalid_tx_cmd.dma,
+				 trans_pcie->invalid_tx_cmd.size);
 }
 
 static void iwl_txq_gen1_tfd_unmap(struct iwl_trans *trans,
@@ -360,7 +362,7 @@ static void iwl_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq,
 	/* We have only q->n_window txq->entries, but we use
 	 * TFD_QUEUE_SIZE_MAX tfds
 	 */
-	if (trans->trans_cfg->gen2)
+	if (trans->mac_cfg->gen2)
 		iwl_txq_gen2_tfd_unmap(trans, &txq->entries[idx].meta,
 				       iwl_txq_get_tfd(trans, txq, read_ptr));
 	else
@@ -399,7 +401,7 @@ static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
 		IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n",
 				   txq_id, txq->read_ptr);
 
-		if (txq_id != trans_pcie->txqs.cmd.q_id) {
+		if (txq_id != trans->conf.cmd_queue) {
 			struct sk_buff *skb = txq->entries[txq->read_ptr].skb;
 			struct iwl_cmd_meta *cmd_meta =
 				&txq->entries[txq->read_ptr].meta;
@@ -413,7 +415,7 @@ static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
 		txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr);
 
 		if (txq->read_ptr == txq->write_ptr &&
-		    txq_id == trans_pcie->txqs.cmd.q_id)
+		    txq_id == trans->conf.cmd_queue)
 			iwl_pcie_clear_cmd_in_flight(trans);
 	}
 
@@ -451,7 +453,7 @@ static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
 	iwl_pcie_txq_unmap(trans, txq_id);
 
 	/* De-alloc array of command/tx buffers */
-	if (txq_id == trans_pcie->txqs.cmd.q_id)
+	if (txq_id == trans->conf.cmd_queue)
 		for (i = 0; i < txq->n_window; i++) {
 			kfree_sensitive(txq->entries[i].cmd);
 			kfree_sensitive(txq->entries[i].free_buf);
@@ -461,7 +463,7 @@ static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
 	if (txq->tfds) {
 		dma_free_coherent(dev,
 				  trans_pcie->txqs.tfd.size *
-				  trans->trans_cfg->base_params->max_tfd_queue_size,
+				  trans->mac_cfg->base->max_tfd_queue_size,
 				  txq->tfds, txq->dma_addr);
 		txq->dma_addr = 0;
 		txq->tfds = NULL;
@@ -474,16 +476,16 @@ static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
 	kfree(txq->entries);
 	txq->entries = NULL;
 
-	del_timer_sync(&txq->stuck_timer);
+	timer_delete_sync(&txq->stuck_timer);
 
 	/* 0-fill queue descriptor structure */
 	memset(txq, 0, sizeof(*txq));
 }
 
-void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr)
+void iwl_pcie_tx_start(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	int nq = trans->trans_cfg->base_params->num_of_queues;
+	int nq = trans->mac_cfg->base->num_of_queues;
 	int chan;
 	u32 reg_val;
 	int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) -
@@ -498,9 +500,6 @@ void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr)
 	trans_pcie->scd_base_addr =
 		iwl_read_prph(trans, SCD_SRAM_BASE_ADDR);
 
-	WARN_ON(scd_base_addr != 0 &&
-		scd_base_addr != trans_pcie->scd_base_addr);
-
 	/* reset context data, TX status and translation data */
 	iwl_trans_write_mem(trans, trans_pcie->scd_base_addr +
 				   SCD_CONTEXT_MEM_LOWER_BOUND,
@@ -512,12 +511,12 @@ void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr)
 	/* The chain extension of the SCD doesn't work well. This feature is
 	 * enabled by default by the HW, so we need to disable it manually.
 	 */
-	if (trans->trans_cfg->base_params->scd_chain_ext_wa)
+	if (trans->mac_cfg->base->scd_chain_ext_wa)
 		iwl_write_prph(trans, SCD_CHAINEXT_EN, 0);
 
-	iwl_trans_ac_txq_enable(trans, trans_pcie->txqs.cmd.q_id,
-				trans_pcie->txqs.cmd.fifo,
-				trans_pcie->txqs.cmd.wdg_timeout);
+	iwl_trans_ac_txq_enable(trans, trans->conf.cmd_queue,
+				trans->conf.cmd_fifo,
+				IWL_DEF_WD_TIMEOUT);
 
 	/* Activate all Tx DMA/FIFO channels */
 	iwl_scd_activate_fifos(trans);
@@ -534,7 +533,7 @@ void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr)
 			   reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
 
 	/* Enable L1-Active */
-	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_8000)
 		iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
 				    APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
 }
@@ -548,13 +547,13 @@ void iwl_trans_pcie_tx_reset(struct iwl_trans *trans)
 	 * we should never get here in gen2 trans mode return early to avoid
 	 * having invalid accesses
 	 */
-	if (WARN_ON_ONCE(trans->trans_cfg->gen2))
+	if (WARN_ON_ONCE(trans->mac_cfg->gen2))
 		return;
 
-	for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
+	for (txq_id = 0; txq_id < trans->mac_cfg->base->num_of_queues;
 	     txq_id++) {
 		struct iwl_txq *txq = trans_pcie->txqs.txq[txq_id];
-		if (trans->trans_cfg->gen2)
+		if (trans->mac_cfg->gen2)
 			iwl_write_direct64(trans,
 					   FH_MEM_CBBC_QUEUE(trans, txq_id),
 					   txq->dma_addr);
@@ -576,7 +575,7 @@ void iwl_trans_pcie_tx_reset(struct iwl_trans *trans)
 	 * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will
 	 * contain garbage.
 	 */
-	iwl_pcie_tx_start(trans, 0);
+	iwl_pcie_tx_start(trans);
 }
 
 static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans)
@@ -597,8 +596,8 @@ static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans)
 	}
 
 	/* Wait for DMA channels to be idle */
-	ret = iwl_poll_bit(trans, FH_TSSR_TX_STATUS_REG, mask, mask, 5000);
-	if (ret < 0)
+	ret = iwl_poll_bits(trans, FH_TSSR_TX_STATUS_REG, mask, 5000);
+	if (ret)
 		IWL_ERR(trans,
 			"Failing on timeout while stopping DMA channel %d [0x%08x]\n",
 			ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG));
@@ -638,7 +637,7 @@ int iwl_pcie_tx_stop(struct iwl_trans *trans)
 		return 0;
 
 	/* Unmap DMA from host system and free skb's */
-	for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
+	for (txq_id = 0; txq_id < trans->mac_cfg->base->num_of_queues;
 	     txq_id++)
 		iwl_pcie_txq_unmap(trans, txq_id);
 
@@ -661,7 +660,7 @@ void iwl_pcie_tx_free(struct iwl_trans *trans)
 	/* Tx queues */
 	if (trans_pcie->txq_memory) {
 		for (txq_id = 0;
-		     txq_id < trans->trans_cfg->base_params->num_of_queues;
+		     txq_id < trans->mac_cfg->base->num_of_queues;
 		     txq_id++) {
 			iwl_pcie_txq_free(trans, txq_id);
 			trans_pcie->txqs.txq[txq_id] = NULL;
@@ -683,7 +682,7 @@ void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq)
 	bool active;
 	u8 fifo;
 
-	if (trans->trans_cfg->gen2) {
+	if (trans->mac_cfg->gen2) {
 		IWL_ERR(trans, "Queue %d is stuck %d %d\n", txq_id,
 			txq->read_ptr, txq->write_ptr);
 		/* TODO: access new SCD registers and dump them */
@@ -700,15 +699,15 @@ void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq)
 		jiffies_to_msecs(txq->wd_timeout),
 		txq->read_ptr, txq->write_ptr,
 		iwl_read_prph(trans, SCD_QUEUE_RDPTR(txq_id)) &
-			(trans->trans_cfg->base_params->max_tfd_queue_size - 1),
+			(trans->mac_cfg->base->max_tfd_queue_size - 1),
 			iwl_read_prph(trans, SCD_QUEUE_WRPTR(txq_id)) &
-			(trans->trans_cfg->base_params->max_tfd_queue_size - 1),
+			(trans->mac_cfg->base->max_tfd_queue_size - 1),
 			iwl_read_direct32(trans, FH_TX_TRB_REG(fifo)));
 }
 
 static void iwl_txq_stuck_timer(struct timer_list *t)
 {
-	struct iwl_txq *txq = from_timer(txq, t, stuck_timer);
+	struct iwl_txq *txq = timer_container_of(txq, t, stuck_timer);
 	struct iwl_trans *trans = txq->trans;
 
 	spin_lock(&txq->lock);
@@ -728,8 +727,8 @@ int iwl_pcie_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq,
 		       int slots_num, bool cmd_queue)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	size_t num_entries = trans->trans_cfg->gen2 ?
-		slots_num : trans->trans_cfg->base_params->max_tfd_queue_size;
+	size_t num_entries = trans->mac_cfg->gen2 ?
+		slots_num : trans->mac_cfg->base->max_tfd_queue_size;
 	size_t tfd_sz;
 	size_t tb0_buf_sz;
 	int i;
@@ -784,7 +783,7 @@ int iwl_pcie_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq,
 	for (i = 0; i < num_entries; i++) {
 		void *tfd = iwl_txq_get_tfd(trans, txq, i);
 
-		if (trans->trans_cfg->gen2)
+		if (trans->mac_cfg->gen2)
 			iwl_txq_set_tfd_invalid_gen2(trans, tfd);
 		else
 			iwl_txq_set_tfd_invalid_gen1(trans, tfd);
@@ -804,6 +803,8 @@ error:
 	return -ENOMEM;
 }
 
+#define BC_TABLE_SIZE	(sizeof(struct iwl_bc_tbl_entry) * TFD_QUEUE_BC_SIZE)
+
 /*
  * iwl_pcie_tx_alloc - allocate TX context
  * Allocate all Tx DMA structures and initialize them
@@ -813,12 +814,12 @@ static int iwl_pcie_tx_alloc(struct iwl_trans *trans)
 	int ret;
 	int txq_id, slots_num;
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	u16 bc_tbls_size = trans->trans_cfg->base_params->num_of_queues;
+	u16 bc_tbls_size = trans->mac_cfg->base->num_of_queues;
 
-	if (WARN_ON(trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))
+	if (WARN_ON(trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))
 		return -EINVAL;
 
-	bc_tbls_size *= sizeof(struct iwlagn_scd_bc_tbl);
+	bc_tbls_size *= BC_TABLE_SIZE;
 
 	/*It is not allowed to alloc twice, so warn when this happens.
 	 * We cannot rely on the previous allocation, so free and fail */
@@ -842,7 +843,7 @@ static int iwl_pcie_tx_alloc(struct iwl_trans *trans)
 	}
 
 	trans_pcie->txq_memory =
-		kcalloc(trans->trans_cfg->base_params->num_of_queues,
+		kcalloc(trans->mac_cfg->base->num_of_queues,
 			sizeof(struct iwl_txq), GFP_KERNEL);
 	if (!trans_pcie->txq_memory) {
 		IWL_ERR(trans, "Not enough memory for txq\n");
@@ -851,16 +852,16 @@ static int iwl_pcie_tx_alloc(struct iwl_trans *trans)
 	}
 
 	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
-	for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
+	for (txq_id = 0; txq_id < trans->mac_cfg->base->num_of_queues;
 	     txq_id++) {
-		bool cmd_queue = (txq_id == trans_pcie->txqs.cmd.q_id);
+		bool cmd_queue = (txq_id == trans->conf.cmd_queue);
 
 		if (cmd_queue)
 			slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE,
-					  trans->cfg->min_txq_size);
+					  trans->mac_cfg->base->min_txq_size);
 		else
 			slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE,
-					  trans->cfg->min_ba_txq_size);
+					  trans->mac_cfg->base->min_ba_txq_size);
 		trans_pcie->txqs.txq[txq_id] = &trans_pcie->txq_memory[txq_id];
 		ret = iwl_pcie_txq_alloc(trans, trans_pcie->txqs.txq[txq_id],
 					 slots_num, cmd_queue);
@@ -910,7 +911,7 @@ int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq,
 		 int slots_num, bool cmd_queue)
 {
 	u32 tfd_queue_max_size =
-		trans->trans_cfg->base_params->max_tfd_queue_size;
+		trans->mac_cfg->base->max_tfd_queue_size;
 	int ret;
 
 	txq->need_update = false;
@@ -970,16 +971,16 @@ int iwl_pcie_tx_init(struct iwl_trans *trans)
 	spin_unlock_bh(&trans_pcie->irq_lock);
 
 	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
-	for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues;
+	for (txq_id = 0; txq_id < trans->mac_cfg->base->num_of_queues;
 	     txq_id++) {
-		bool cmd_queue = (txq_id == trans_pcie->txqs.cmd.q_id);
+		bool cmd_queue = (txq_id == trans->conf.cmd_queue);
 
 		if (cmd_queue)
 			slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE,
-					  trans->cfg->min_txq_size);
+					  trans->mac_cfg->base->min_txq_size);
 		else
 			slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE,
-					  trans->cfg->min_ba_txq_size);
+					  trans->mac_cfg->base->min_ba_txq_size);
 		ret = iwl_txq_init(trans, trans_pcie->txqs.txq[txq_id], slots_num,
 				   cmd_queue);
 		if (ret) {
@@ -998,7 +999,7 @@ int iwl_pcie_tx_init(struct iwl_trans *trans)
 	}
 
 	iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE);
-	if (trans->trans_cfg->base_params->num_of_queues > 20)
+	if (trans->mac_cfg->base->num_of_queues > 20)
 		iwl_set_bits_prph(trans, SCD_GP_CTRL,
 				  SCD_GP_CTRL_ENABLE_31_QUEUES);
 
@@ -1019,7 +1020,7 @@ static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans,
 	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
 		return -ENODEV;
 
-	if (!trans->trans_cfg->base_params->apmg_wake_up_wa)
+	if (!trans->mac_cfg->base->apmg_wake_up_wa)
 		return 0;
 
 	/*
@@ -1028,7 +1029,7 @@ static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans,
 	 * returned. This needs to be done only on NICs that have
 	 * apmg_wake_up_wa set (see above.)
 	 */
-	if (!_iwl_trans_pcie_grab_nic_access(trans))
+	if (!_iwl_trans_pcie_grab_nic_access(trans, false))
 		return -EIO;
 
 	/*
@@ -1061,7 +1062,7 @@ static void iwl_txq_progress(struct iwl_txq *txq)
 	 * since we're making progress on this queue
 	 */
 	if (txq->read_ptr == txq->write_ptr)
-		del_timer(&txq->stuck_timer);
+		timer_delete(&txq->stuck_timer);
 	else
 		mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
 }
@@ -1097,12 +1098,12 @@ static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx)
 	idx = iwl_txq_get_cmd_index(txq, idx);
 	r = iwl_txq_get_cmd_index(txq, txq->read_ptr);
 
-	if (idx >= trans->trans_cfg->base_params->max_tfd_queue_size ||
+	if (idx >= trans->mac_cfg->base->max_tfd_queue_size ||
 	    (!iwl_txq_used(txq, idx, txq->read_ptr, txq->write_ptr))) {
 		WARN_ONCE(test_bit(txq_id, trans_pcie->txqs.queue_used),
 			  "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n",
 			  __func__, txq_id, idx,
-			  trans->trans_cfg->base_params->max_tfd_queue_size,
+			  trans->mac_cfg->base->max_tfd_queue_size,
 			  txq->write_ptr, txq->read_ptr);
 		return;
 	}
@@ -1171,15 +1172,15 @@ bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
 		fifo = cfg->fifo;
 
 		/* Disable the scheduler prior configuring the cmd queue */
-		if (txq_id == trans_pcie->txqs.cmd.q_id &&
-		    trans_pcie->scd_set_active)
+		if (txq_id == trans->conf.cmd_queue &&
+		    trans->conf.scd_set_active)
 			iwl_scd_enable_set_active(trans, 0);
 
 		/* Stop this Tx queue before configuring it */
 		iwl_scd_txq_set_inactive(trans, txq_id);
 
 		/* Set this queue as a chain-building queue unless it is CMD */
-		if (txq_id != trans_pcie->txqs.cmd.q_id)
+		if (txq_id != trans->conf.cmd_queue)
 			iwl_scd_txq_set_chain(trans, txq_id);
 
 		if (cfg->aggregate) {
@@ -1213,7 +1214,7 @@ bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
 		 * this sad hardware issue.
 		 * This bug has been fixed on devices 9000 and up.
 		 */
-		scd_bug = !trans->trans_cfg->mq_rx_supported &&
+		scd_bug = !trans->mac_cfg->mq_rx_supported &&
 			!((ssn - txq->write_ptr) & 0x3f) &&
 			(ssn != txq->write_ptr);
 		if (scd_bug)
@@ -1249,8 +1250,8 @@ bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
 			       SCD_QUEUE_STTS_REG_MSK);
 
 		/* enable the scheduler for this queue (only) */
-		if (txq_id == trans_pcie->txqs.cmd.q_id &&
-		    trans_pcie->scd_set_active)
+		if (txq_id == trans->conf.cmd_queue &&
+		    trans->conf.scd_set_active)
 			iwl_scd_enable_set_active(trans, BIT(txq_id));
 
 		IWL_DEBUG_TX_QUEUES(trans,
@@ -1317,10 +1318,10 @@ static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block)
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int i;
 
-	for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) {
+	for (i = 0; i < trans->mac_cfg->base->num_of_queues; i++) {
 		struct iwl_txq *txq = trans_pcie->txqs.txq[i];
 
-		if (i == trans_pcie->txqs.cmd.q_id)
+		if (i == trans->conf.cmd_queue)
 			continue;
 
 		/* we skip the command queue (obviously) so it's OK to nest */
@@ -1353,7 +1354,7 @@ int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 			  struct iwl_host_cmd *cmd)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwl_txq *txq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
+	struct iwl_txq *txq = trans_pcie->txqs.txq[trans->conf.cmd_queue];
 	struct iwl_device_cmd *out_cmd;
 	struct iwl_cmd_meta *out_meta;
 	void *dup_buf = NULL;
@@ -1368,7 +1369,7 @@ int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 	u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD];
 	unsigned long flags;
 
-	if (WARN(!trans->wide_cmd_header &&
+	if (WARN(!trans->conf.wide_cmd_header &&
 		 group_id > IWL_ALWAYS_LONG_GROUP,
 		 "unsupported wide command %#x\n", cmd->id))
 		return -EINVAL;
@@ -1456,7 +1457,9 @@ int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		spin_unlock_irqrestore(&txq->lock, flags);
 
 		IWL_ERR(trans, "No space in command queue\n");
-		iwl_op_mode_cmd_queue_full(trans->op_mode);
+		iwl_op_mode_nic_error(trans->op_mode,
+				      IWL_ERR_TYPE_CMD_QUEUE_FULL);
+		iwl_trans_schedule_reset(trans, IWL_ERR_TYPE_CMD_QUEUE_FULL);
 		idx = -ENOSPC;
 		goto free_dup_buf;
 	}
@@ -1480,7 +1483,7 @@ int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 				    sizeof(struct iwl_cmd_header_wide));
 		out_cmd->hdr_wide.reserved = 0;
 		out_cmd->hdr_wide.sequence =
-			cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->txqs.cmd.q_id) |
+			cpu_to_le16(QUEUE_TO_SEQ(trans->conf.cmd_queue) |
 						 INDEX_TO_SEQ(txq->write_ptr));
 
 		cmd_pos = sizeof(struct iwl_cmd_header_wide);
@@ -1488,7 +1491,7 @@ int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 	} else {
 		out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id);
 		out_cmd->hdr.sequence =
-			cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->txqs.cmd.q_id) |
+			cpu_to_le16(QUEUE_TO_SEQ(trans->conf.cmd_queue) |
 						 INDEX_TO_SEQ(txq->write_ptr));
 		out_cmd->hdr.group_id = 0;
 
@@ -1539,7 +1542,7 @@ int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		     iwl_get_cmd_string(trans, cmd->id),
 		     group_id, out_cmd->hdr.cmd,
 		     le16_to_cpu(out_cmd->hdr.sequence),
-		     cmd_size, txq->write_ptr, idx, trans_pcie->txqs.cmd.q_id);
+		     cmd_size, txq->write_ptr, idx, trans->conf.cmd_queue);
 
 	/* start the TFD with the minimum copy bytes */
 	tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE);
@@ -1638,18 +1641,16 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
 	struct iwl_device_cmd *cmd;
 	struct iwl_cmd_meta *meta;
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwl_txq *txq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
+	struct iwl_txq *txq = trans_pcie->txqs.txq[trans->conf.cmd_queue];
 
 	/* If a Tx command is being handled and it isn't in the actual
 	 * command queue then there a command routing bug has been introduced
 	 * in the queue management code. */
-	if (WARN(txq_id != trans_pcie->txqs.cmd.q_id,
-		 "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n",
-		 txq_id, trans_pcie->txqs.cmd.q_id, sequence, txq->read_ptr,
-		 txq->write_ptr)) {
-		iwl_print_hex_error(trans, pkt, 32);
+	if (IWL_FW_CHECK(trans, txq_id != trans->conf.cmd_queue,
+			 "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d pkt=%*phN\n",
+			 txq_id, trans->conf.cmd_queue, sequence, txq->read_ptr,
+			 txq->write_ptr, 32, pkt))
 		return;
-	}
 
 	spin_lock_bh(&txq->lock);
 
@@ -1659,7 +1660,7 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
 	group_id = cmd->hdr.group_id;
 	cmd_id = WIDE_ID(group_id, cmd->hdr.cmd);
 
-	if (trans->trans_cfg->gen2)
+	if (trans->mac_cfg->gen2)
 		iwl_txq_gen2_tfd_unmap(trans, meta,
 				       iwl_txq_get_tfd(trans, txq, index));
 	else
@@ -1692,7 +1693,7 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
 		clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
 		IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n",
 			       iwl_get_cmd_string(trans, cmd_id));
-		wake_up(&trans->wait_command_queue);
+		wake_up(&trans_pcie->wait_command_queue);
 	}
 
 	meta->flags = 0;
@@ -1762,7 +1763,7 @@ static void *iwl_pcie_get_page_hdr(struct iwl_trans *trans,
 	dma_addr_t phys;
 	void *ret;
 
-	page_ptr = (void *)((u8 *)skb->cb + trans_pcie->txqs.page_offs);
+	page_ptr = (void *)((u8 *)skb->cb + trans->conf.cb_data_offs);
 
 	if (WARN_ON(*page_ptr))
 		return NULL;
@@ -1864,6 +1865,7 @@ dma_addr_t iwl_pcie_get_sgt_tb_phys(struct sg_table *sgt, unsigned int offset,
  * @cmd_meta: command meta to store the scatter list information for unmapping
  * @hdr: output argument for TSO headers
  * @hdr_room: requested length for TSO headers
+ * @offset: offset into the data from which mapping should start
  *
  * Allocate space for a scatter gather list and TSO headers and map the SKB
  * using the scatter gather list. The SKB is unmapped again when the page is
@@ -1873,9 +1875,12 @@ dma_addr_t iwl_pcie_get_sgt_tb_phys(struct sg_table *sgt, unsigned int offset,
  */
 struct sg_table *iwl_pcie_prep_tso(struct iwl_trans *trans, struct sk_buff *skb,
 				   struct iwl_cmd_meta *cmd_meta,
-				   u8 **hdr, unsigned int hdr_room)
+				   u8 **hdr, unsigned int hdr_room,
+				   unsigned int offset)
 {
 	struct sg_table *sgt;
+	unsigned int n_segments = skb_shinfo(skb)->nr_frags + 1;
+	int orig_nents;
 
 	if (WARN_ON_ONCE(skb_has_frag_list(skb)))
 		return NULL;
@@ -1883,8 +1888,7 @@ struct sg_table *iwl_pcie_prep_tso(struct iwl_trans *trans, struct sk_buff *skb,
 	*hdr = iwl_pcie_get_page_hdr(trans,
 				     hdr_room + __alignof__(struct sg_table) +
 				     sizeof(struct sg_table) +
-				     (skb_shinfo(skb)->nr_frags + 1) *
-				     sizeof(struct scatterlist),
+				     n_segments * sizeof(struct scatterlist),
 				     skb);
 	if (!*hdr)
 		return NULL;
@@ -1892,14 +1896,15 @@ struct sg_table *iwl_pcie_prep_tso(struct iwl_trans *trans, struct sk_buff *skb,
 	sgt = (void *)PTR_ALIGN(*hdr + hdr_room, __alignof__(struct sg_table));
 	sgt->sgl = (void *)(sgt + 1);
 
-	sg_init_table(sgt->sgl, skb_shinfo(skb)->nr_frags + 1);
+	sg_init_table(sgt->sgl, n_segments);
 
 	/* Only map the data, not the header (it is copied to the TSO page) */
-	sgt->orig_nents = skb_to_sgvec(skb, sgt->sgl, skb_headlen(skb),
-				       skb->data_len);
-	if (WARN_ON_ONCE(sgt->orig_nents <= 0))
+	orig_nents = skb_to_sgvec(skb, sgt->sgl, offset, skb->len - offset);
+	if (WARN_ON_ONCE(orig_nents <= 0))
 		return NULL;
 
+	sgt->orig_nents = orig_nents;
+
 	/* And map the entire SKB */
 	if (dma_map_sgtable(trans->dev, sgt, DMA_TO_DEVICE, 0) < 0)
 		return NULL;
@@ -1917,7 +1922,7 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
 				   u16 tb1_len)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload;
+	struct iwl_tx_cmd_v6 *tx_cmd = (void *)dev_cmd->payload;
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room;
 	unsigned int mss = skb_shinfo(skb)->gso_size;
@@ -1948,7 +1953,8 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
 		(3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len;
 
 	/* Our device supports 9 segments at most, it will fit in 1 page */
-	sgt = iwl_pcie_prep_tso(trans, skb, out_meta, &start_hdr, hdr_room);
+	sgt = iwl_pcie_prep_tso(trans, skb, out_meta, &start_hdr, hdr_room,
+				snap_ip_tcp_hdrlen + hdr_len + iv_len);
 	if (!sgt)
 		return -ENOMEM;
 
@@ -1968,7 +1974,7 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
 	 * have in the MPDU by themselves, but that we duplicate into
 	 * all the different MSDUs inside the A-MSDU.
 	 */
-	le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen);
+	le16_add_cpu(&tx_cmd->params.len, -snap_ip_tcp_hdrlen);
 
 	tso_start(skb, &tso);
 
@@ -2011,7 +2017,7 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
 		trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr,
 					hdr_tb_phys, hdr_tb_len);
 		/* add this subframe's headers' length to the tx_cmd */
-		le16_add_cpu(&tx_cmd->len, pos_hdr - subf_hdrs_start);
+		le16_add_cpu(&tx_cmd->params.len, pos_hdr - subf_hdrs_start);
 
 		/* prepare the start_hdr for the next subframe */
 		start_hdr = pos_hdr;
@@ -2071,19 +2077,19 @@ static void iwl_txq_gen1_update_byte_cnt_tbl(struct iwl_trans *trans,
 					     int num_tbs)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwlagn_scd_bc_tbl *scd_bc_tbl;
+	struct iwl_bc_tbl_entry *scd_bc_tbl;
 	int write_ptr = txq->write_ptr;
 	int txq_id = txq->id;
 	u8 sec_ctl = 0;
 	u16 len = byte_cnt + IWL_TX_CRC_SIZE + IWL_TX_DELIMITER_SIZE;
 	__le16 bc_ent;
 	struct iwl_device_tx_cmd *dev_cmd = txq->entries[txq->write_ptr].cmd;
-	struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload;
-	u8 sta_id = tx_cmd->sta_id;
+	struct iwl_tx_cmd_v6 *tx_cmd = (void *)dev_cmd->payload;
+	u8 sta_id = tx_cmd->params.sta_id;
 
 	scd_bc_tbl = trans_pcie->txqs.scd_bc_tbls.addr;
 
-	sec_ctl = tx_cmd->sec_ctl;
+	sec_ctl = tx_cmd->params.sec_ctl;
 
 	switch (sec_ctl & TX_CMD_SEC_MSK) {
 	case TX_CMD_SEC_CCM:
@@ -2096,7 +2102,9 @@ static void iwl_txq_gen1_update_byte_cnt_tbl(struct iwl_trans *trans,
 		len += IEEE80211_WEP_IV_LEN + IEEE80211_WEP_ICV_LEN;
 		break;
 	}
-	if (trans_pcie->txqs.bc_table_dword)
+
+	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_7000 &&
+	    trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		len = DIV_ROUND_UP(len, 4);
 
 	if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX))
@@ -2104,10 +2112,10 @@ static void iwl_txq_gen1_update_byte_cnt_tbl(struct iwl_trans *trans,
 
 	bc_ent = cpu_to_le16(len | (sta_id << 12));
 
-	scd_bc_tbl[txq_id].tfd_offset[write_ptr] = bc_ent;
+	scd_bc_tbl[txq_id * TFD_QUEUE_BC_SIZE + write_ptr].tfd_offset = bc_ent;
 
 	if (write_ptr < TFD_QUEUE_SIZE_BC_DUP)
-		scd_bc_tbl[txq_id].tfd_offset[TFD_QUEUE_SIZE_MAX + write_ptr] =
+		scd_bc_tbl[txq_id * TFD_QUEUE_BC_SIZE + TFD_QUEUE_SIZE_MAX + write_ptr].tfd_offset =
 			bc_ent;
 }
 
@@ -2116,7 +2124,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct ieee80211_hdr *hdr;
-	struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload;
+	struct iwl_tx_cmd_v6 *tx_cmd = (struct iwl_tx_cmd_v6 *)dev_cmd->payload;
 	struct iwl_cmd_meta *out_meta;
 	struct iwl_txq *txq;
 	dma_addr_t tb0_phys, tb1_phys, scratch_phys;
@@ -2157,7 +2165,8 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 			struct iwl_device_tx_cmd **dev_cmd_ptr;
 
 			dev_cmd_ptr = (void *)((u8 *)skb->cb +
-					       trans_pcie->txqs.dev_cmd_offs);
+					       trans->conf.cb_data_offs +
+					       sizeof(void *));
 
 			*dev_cmd_ptr = dev_cmd;
 			__skb_queue_tail(&txq->overflow_q, skb);
@@ -2188,10 +2197,10 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 
 	tb0_phys = iwl_txq_get_first_tb_dma(txq, txq->write_ptr);
 	scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) +
-		       offsetof(struct iwl_tx_cmd, scratch);
+		       offsetof(struct iwl_tx_cmd_v6_params, scratch);
 
-	tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys);
-	tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);
+	tx_cmd->params.dram_lsb_ptr = cpu_to_le32(scratch_phys);
+	tx_cmd->params.dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);
 
 	/* Set up first empty entry in queue's array of Tx/cmd buffers */
 	out_meta = &txq->entries[txq->write_ptr].meta;
@@ -2203,7 +2212,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	 * (This calculation modifies the TX command, so do it before the
 	 * setup of the first TB)
 	 */
-	len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) +
+	len = sizeof(struct iwl_tx_cmd_v6) + sizeof(struct iwl_cmd_header) +
 	      hdr_len - IWL_FIRST_TB_SIZE;
 	/* do not align A-MSDU to dword as the subframe header aligns it */
 	amsdu = ieee80211_is_data_qos(fc) &&
@@ -2213,7 +2222,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 		tb1_len = ALIGN(len, 4);
 		/* Tell NIC about any 2-byte padding after MAC header */
 		if (tb1_len != len)
-			tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD);
+			tx_cmd->params.tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD);
 	} else {
 		tb1_len = len;
 	}
@@ -2226,9 +2235,9 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 			       IWL_FIRST_TB_SIZE, true);
 
 	/* there must be data left over for TB1 or this code must be changed */
-	BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE);
+	BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_v6) < IWL_FIRST_TB_SIZE);
 	BUILD_BUG_ON(sizeof(struct iwl_cmd_header) +
-		     offsetofend(struct iwl_tx_cmd, scratch) >
+		     offsetofend(struct iwl_tx_cmd_v6_params, scratch) >
 		     IWL_FIRST_TB_SIZE);
 
 	/* map the data for TB1 */
@@ -2274,7 +2283,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 
 	tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr);
 	/* Set up entry for this TFD in Tx byte-count array */
-	iwl_txq_gen1_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len),
+	iwl_txq_gen1_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->params.len),
 					 iwl_txq_gen1_tfd_get_num_tbs(tfd));
 
 	wait_write_ptr = ieee80211_has_morefrags(fc);
@@ -2316,24 +2325,24 @@ static void iwl_txq_gen1_inval_byte_cnt_tbl(struct iwl_trans *trans,
 					    int read_ptr)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwlagn_scd_bc_tbl *scd_bc_tbl = trans_pcie->txqs.scd_bc_tbls.addr;
+	struct iwl_bc_tbl_entry *scd_bc_tbl = trans_pcie->txqs.scd_bc_tbls.addr;
 	int txq_id = txq->id;
 	u8 sta_id = 0;
 	__le16 bc_ent;
 	struct iwl_device_tx_cmd *dev_cmd = txq->entries[read_ptr].cmd;
-	struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload;
+	struct iwl_tx_cmd_v6 *tx_cmd = (void *)dev_cmd->payload;
 
 	WARN_ON(read_ptr >= TFD_QUEUE_SIZE_MAX);
 
-	if (txq_id != trans_pcie->txqs.cmd.q_id)
-		sta_id = tx_cmd->sta_id;
+	if (txq_id != trans->conf.cmd_queue)
+		sta_id = tx_cmd->params.sta_id;
 
 	bc_ent = cpu_to_le16(1 | (sta_id << 12));
 
-	scd_bc_tbl[txq_id].tfd_offset[read_ptr] = bc_ent;
+	scd_bc_tbl[txq_id * TFD_QUEUE_BC_SIZE + read_ptr].tfd_offset = bc_ent;
 
 	if (read_ptr < TFD_QUEUE_SIZE_BC_DUP)
-		scd_bc_tbl[txq_id].tfd_offset[TFD_QUEUE_SIZE_MAX + read_ptr] =
+		scd_bc_tbl[txq_id * TFD_QUEUE_BC_SIZE + TFD_QUEUE_SIZE_MAX + read_ptr].tfd_offset =
 			bc_ent;
 }
 
@@ -2347,7 +2356,7 @@ void iwl_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 	int txq_read_ptr, txq_write_ptr;
 
 	/* This function is not meant to release cmd queue*/
-	if (WARN_ON(txq_id == trans_pcie->txqs.cmd.q_id))
+	if (WARN_ON(txq_id == trans->conf.cmd_queue))
 		return;
 
 	if (WARN_ON(!txq))
@@ -2362,6 +2371,10 @@ void iwl_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 	txq_write_ptr = txq->write_ptr;
 	spin_unlock(&txq->lock);
 
+	/* There is nothing to do if we are flushing an empty queue */
+	if (is_flush && txq_write_ptr == txq_read_ptr)
+		goto out;
+
 	read_ptr = iwl_txq_get_cmd_index(txq, txq_read_ptr);
 
 	if (!test_bit(txq_id, trans_pcie->txqs.queue_used)) {
@@ -2385,7 +2398,7 @@ void iwl_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 		IWL_ERR(trans,
 			"%s: Read index for txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n",
 			__func__, txq_id, last_to_free,
-			trans->trans_cfg->base_params->max_tfd_queue_size,
+			trans->mac_cfg->base->max_tfd_queue_size,
 			txq_write_ptr, txq_read_ptr);
 
 		iwl_op_mode_time_point(trans->op_mode,
@@ -2414,7 +2427,7 @@ void iwl_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 
 		txq->entries[read_ptr].skb = NULL;
 
-		if (!trans->trans_cfg->gen2)
+		if (!trans->mac_cfg->gen2)
 			iwl_txq_gen1_inval_byte_cnt_tbl(trans, txq,
 							txq_read_ptr);
 
@@ -2456,7 +2469,8 @@ void iwl_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 			struct iwl_device_tx_cmd *dev_cmd_ptr;
 
 			dev_cmd_ptr = *(void **)((u8 *)skb->cb +
-						 trans_pcie->txqs.dev_cmd_offs);
+						 trans->conf.cb_data_offs +
+						 sizeof(void *));
 
 			/*
 			 * Note that we can very well be overflowing again.
@@ -2529,7 +2543,7 @@ void iwl_pcie_freeze_txq_timer(struct iwl_trans *trans,
 			/* remember how long until the timer fires */
 			txq->frozen_expiry_remainder =
 				txq->stuck_timer.expires - now;
-			del_timer(&txq->stuck_timer);
+			timer_delete(&txq->stuck_timer);
 			goto next_queue;
 		}
 
@@ -2548,11 +2562,11 @@ next_queue:
 #define HOST_COMPLETE_TIMEOUT	(2 * HZ)
 
 static int iwl_trans_pcie_send_hcmd_sync(struct iwl_trans *trans,
-					 struct iwl_host_cmd *cmd)
+					 struct iwl_host_cmd *cmd,
+					 const char *cmd_str)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	const char *cmd_str = iwl_get_cmd_string(trans, cmd->id);
-	struct iwl_txq *txq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
+	struct iwl_txq *txq = trans_pcie->txqs.txq[trans->conf.cmd_queue];
 	int cmd_idx;
 	int ret;
 
@@ -2565,7 +2579,7 @@ static int iwl_trans_pcie_send_hcmd_sync(struct iwl_trans *trans,
 
 	IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", cmd_str);
 
-	if (trans->trans_cfg->gen2)
+	if (trans->mac_cfg->gen2)
 		cmd_idx = iwl_pcie_gen2_enqueue_hcmd(trans, cmd);
 	else
 		cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd);
@@ -2578,7 +2592,7 @@ static int iwl_trans_pcie_send_hcmd_sync(struct iwl_trans *trans,
 		return ret;
 	}
 
-	ret = wait_event_timeout(trans->wait_command_queue,
+	ret = wait_event_timeout(trans_pcie->wait_command_queue,
 				 !test_bit(STATUS_SYNC_HCMD_ACTIVE,
 					   &trans->status),
 				 HOST_COMPLETE_TIMEOUT);
@@ -2594,7 +2608,7 @@ static int iwl_trans_pcie_send_hcmd_sync(struct iwl_trans *trans,
 			       cmd_str);
 		ret = -ETIMEDOUT;
 
-		iwl_trans_sync_nmi(trans);
+		iwl_trans_pcie_sync_nmi(trans);
 		goto cancel;
 	}
 
@@ -2645,6 +2659,8 @@ cancel:
 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans,
 			     struct iwl_host_cmd *cmd)
 {
+	const char *cmd_str = iwl_get_cmd_string(trans, cmd->id);
+
 	/* Make sure the NIC is still alive in the bus */
 	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
 		return -ENODEV;
@@ -2656,20 +2672,16 @@ int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans,
 		return -ERFKILL;
 	}
 
-	if (unlikely(trans->system_pm_mode == IWL_PLAT_PM_MODE_D3 &&
-		     !(cmd->flags & CMD_SEND_IN_D3))) {
-		IWL_DEBUG_WOWLAN(trans, "Dropping CMD 0x%x: D3\n", cmd->id);
-		return -EHOSTDOWN;
-	}
-
 	if (cmd->flags & CMD_ASYNC) {
 		int ret;
 
+		IWL_DEBUG_INFO(trans, "Sending async command %s\n", cmd_str);
+
 		/* An asynchronous command can not expect an SKB to be set. */
 		if (WARN_ON(cmd->flags & CMD_WANT_SKB))
 			return -EINVAL;
 
-		if (trans->trans_cfg->gen2)
+		if (trans->mac_cfg->gen2)
 			ret = iwl_pcie_gen2_enqueue_hcmd(trans, cmd);
 		else
 			ret = iwl_pcie_enqueue_hcmd(trans, cmd);
@@ -2683,6 +2695,5 @@ int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans,
 		return 0;
 	}
 
-	return iwl_trans_pcie_send_hcmd_sync(trans, cmd);
+	return iwl_trans_pcie_send_hcmd_sync(trans, cmd, cmd_str);
 }
-IWL_EXPORT_SYMBOL(iwl_trans_pcie_send_hcmd);
diff --git a/sys/contrib/dev/iwlwifi/iwl-context-info-gen3.h b/sys/contrib/dev/iwlwifi/pcie/iwl-context-info-v2.h
index 5b62933134cf..416baadc5017 100644
--- a/sys/contrib/dev/iwlwifi/iwl-context-info-gen3.h
+++ b/sys/contrib/dev/iwlwifi/pcie/iwl-context-info-v2.h
@@ -1,9 +1,9 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2018, 2020-2024 Intel Corporation
+ * Copyright (C) 2018, 2020-2025 Intel Corporation
  */
-#ifndef __iwl_context_info_file_gen3_h__
-#define __iwl_context_info_file_gen3_h__
+#ifndef __iwl_context_info_file_v2_h__
+#define __iwl_context_info_file_v2_h__
 
 #include "iwl-context-info.h"
 
@@ -58,6 +58,7 @@ enum iwl_prph_scratch_mtr_format {
  * @IWL_PRPH_SCRATCH_RB_SIZE_EXT_16K: 16kB RB size
  * @IWL_PRPH_SCRATCH_SCU_FORCE_ACTIVE: Indicate fw to set SCU_FORCE_ACTIVE
  *	upon reset.
+ * @IWL_PRPH_SCRATCH_TOP_RESET: request TOP reset
  */
 enum iwl_prph_scratch_flags {
 	IWL_PRPH_SCRATCH_IMR_DEBUG_EN		= BIT(1),
@@ -74,9 +75,24 @@ enum iwl_prph_scratch_flags {
 	IWL_PRPH_SCRATCH_RB_SIZE_EXT_12K	= 9 << 20,
 	IWL_PRPH_SCRATCH_RB_SIZE_EXT_16K	= 10 << 20,
 	IWL_PRPH_SCRATCH_SCU_FORCE_ACTIVE	= BIT(29),
+	IWL_PRPH_SCRATCH_TOP_RESET		= BIT(30),
 };
 
-/*
+/**
+ * enum iwl_prph_scratch_ext_flags - PRPH scratch control ext flags
+ * @IWL_PRPH_SCRATCH_EXT_EXT_FSEQ: external FSEQ image provided
+ * @IWL_PRPH_SCRATCH_EXT_URM_FW: switch to URM mode based on fw setting
+ * @IWL_PRPH_SCRATCH_EXT_URM_PERM: switch to permanent URM mode
+ * @IWL_PRPH_SCRATCH_EXT_32KHZ_CLK_VALID: use external 32 KHz clock
+ */
+enum iwl_prph_scratch_ext_flags {
+	IWL_PRPH_SCRATCH_EXT_EXT_FSEQ		= BIT(0),
+	IWL_PRPH_SCRATCH_EXT_URM_FW		= BIT(4),
+	IWL_PRPH_SCRATCH_EXT_URM_PERM		= BIT(5),
+	IWL_PRPH_SCRATCH_EXT_32KHZ_CLK_VALID	= BIT(8),
+};
+
+/**
  * struct iwl_prph_scratch_version - version structure
  * @mac_id: SKU and revision id
  * @version: prph scratch information version id
@@ -90,17 +106,18 @@ struct iwl_prph_scratch_version {
 	__le16 reserved;
 } __packed; /* PERIPH_SCRATCH_VERSION_S */
 
-/*
+/**
  * struct iwl_prph_scratch_control - control structure
  * @control_flags: context information flags see &enum iwl_prph_scratch_flags
- * @reserved: reserved
+ * @control_flags_ext: context information for extended flags,
+ *	see &enum iwl_prph_scratch_ext_flags
  */
 struct iwl_prph_scratch_control {
 	__le32 control_flags;
-	__le32 reserved;
+	__le32 control_flags_ext;
 } __packed; /* PERIPH_SCRATCH_CONTROL_S */
 
-/*
+/**
  * struct iwl_prph_scratch_pnvm_cfg - PNVM scratch
  * @pnvm_base_addr: PNVM start address
  * @pnvm_size: the size of the PNVM image in bytes
@@ -113,14 +130,15 @@ struct iwl_prph_scratch_pnvm_cfg {
 } __packed; /* PERIPH_SCRATCH_PNVM_CFG_S */
 
 /**
- * struct iwl_prph_scrath_mem_desc_addr_array
+ * struct iwl_prph_scratch_mem_desc_addr_array - DRAM
  * @mem_descs: array of dram addresses.
- * Each address is the beggining of a pnvm payload.
+ * Each address is the beginning of a PNVM payload.
  */
-struct iwl_prph_scrath_mem_desc_addr_array {
+struct iwl_prph_scratch_mem_desc_addr_array {
 	__le64 mem_descs[IPC_DRAM_MAP_ENTRY_NUM_MAX];
 } __packed; /* PERIPH_SCRATCH_MEM_DESC_ADDR_ARRAY_S_VER_1 */
-/*
+
+/**
  * struct iwl_prph_scratch_hwm_cfg - hwm config
  * @hwm_base_addr: hwm start address
  * @hwm_size: hwm size in DWs
@@ -132,7 +150,7 @@ struct iwl_prph_scratch_hwm_cfg {
 	__le32 debug_token_config;
 } __packed; /* PERIPH_SCRATCH_HWM_CFG_S */
 
-/*
+/**
  * struct iwl_prph_scratch_rbd_cfg - RBDs configuration
  * @free_rbd_addr: default queue free RB CB base address
  * @reserved: reserved
@@ -142,10 +160,11 @@ struct iwl_prph_scratch_rbd_cfg {
 	__le32 reserved;
 } __packed; /* PERIPH_SCRATCH_RBD_CFG_S */
 
-/*
+/**
  * struct iwl_prph_scratch_uefi_cfg - prph scratch reduce power table
  * @base_addr: reduce power table address
  * @size: the size of the entire power table image
+ * @reserved: (reserved)
  */
 struct iwl_prph_scratch_uefi_cfg {
 	__le64 base_addr;
@@ -153,7 +172,7 @@ struct iwl_prph_scratch_uefi_cfg {
 	__le32 reserved;
 } __packed; /* PERIPH_SCRATCH_UEFI_CFG_S */
 
-/*
+/**
  * struct iwl_prph_scratch_step_cfg - prph scratch step configuration
  * @mbx_addr_0: [0:7] revision,
  *		[8:15] cnvi_to_cnvr length,
@@ -167,13 +186,14 @@ struct iwl_prph_scratch_step_cfg {
 	__le32 mbx_addr_1;
 } __packed;
 
-/*
+/**
  * struct iwl_prph_scratch_ctrl_cfg - prph scratch ctrl and config
  * @version: version information of context info and HW
  * @control: control flags of FH configurations
  * @pnvm_cfg: ror configuration
  * @hwm_cfg: hwm configuration
  * @rbd_cfg: default RX queue configuration
+ * @reduce_power_cfg: UEFI power reduction table
  * @step_cfg: step configuration
  */
 struct iwl_prph_scratch_ctrl_cfg {
@@ -186,7 +206,20 @@ struct iwl_prph_scratch_ctrl_cfg {
 	struct iwl_prph_scratch_step_cfg step_cfg;
 } __packed; /* PERIPH_SCRATCH_CTRL_CFG_S */
 
-/*
+#define IWL_NUM_DRAM_FSEQ_ENTRIES	8
+
+/**
+ * struct iwl_context_info_dram_fseq - images DRAM map (with fseq)
+ * each entry in the map represents a DRAM chunk of up to 32 KB
+ * @common: UMAC/LMAC/virtual images
+ * @fseq_img: FSEQ image DRAM map
+ */
+struct iwl_context_info_dram_fseq {
+	struct iwl_context_info_dram_nonfseq common;
+	__le64 fseq_img[IWL_NUM_DRAM_FSEQ_ENTRIES];
+} __packed; /* PERIPH_SCRATCH_DRAM_MAP_S */
+
+/**
  * struct iwl_prph_scratch - peripheral scratch mapping
  * @ctrl_cfg: control and configuration of prph scratch
  * @dram: firmware images addresses in DRAM
@@ -199,10 +232,10 @@ struct iwl_prph_scratch {
 	__le32 fseq_override;
 	__le32 step_analog_params;
 	__le32 reserved[8];
-	struct iwl_context_info_dram dram;
+	struct iwl_context_info_dram_fseq dram;
 } __packed; /* PERIPH_SCRATCH_S */
 
-/*
+/**
  * struct iwl_prph_info - peripheral information
  * @boot_stage_mirror: reflects the value in the Boot Stage CSR register
  * @ipc_status_mirror: reflects the value in the IPC Status CSR register
@@ -216,8 +249,8 @@ struct iwl_prph_info {
 	__le32 reserved;
 } __packed; /* PERIPH_INFO_S */
 
-/*
- * struct iwl_context_info_gen3 - device INIT configuration
+/**
+ * struct iwl_context_info_v2 - device INIT configuration
  * @version: version of the context information
  * @size: size of context information in DWs
  * @config: context in which the peripheral would execute - a subset of
@@ -260,7 +293,7 @@ struct iwl_prph_info {
  * @prph_scratch_size: the size of the peripheral scratch structure in DWs
  * @reserved: reserved
  */
-struct iwl_context_info_gen3 {
+struct iwl_context_info_v2 {
 	__le16 version;
 	__le16 size;
 	__le32 config;
@@ -290,22 +323,22 @@ struct iwl_context_info_gen3 {
 	__le32 reserved;
 } __packed; /* IPC_CONTEXT_INFO_S */
 
-int iwl_pcie_ctxt_info_gen3_init(struct iwl_trans *trans,
-				 const struct fw_img *fw);
-void iwl_pcie_ctxt_info_gen3_free(struct iwl_trans *trans, bool alive);
+int iwl_pcie_ctxt_info_v2_alloc(struct iwl_trans *trans,
+				const struct iwl_fw *fw,
+				const struct fw_img *img);
+void iwl_pcie_ctxt_info_v2_kick(struct iwl_trans *trans);
+void iwl_pcie_ctxt_info_v2_free(struct iwl_trans *trans, bool alive);
 
-int iwl_trans_pcie_ctx_info_gen3_load_pnvm(struct iwl_trans *trans,
-					   const struct iwl_pnvm_image *pnvm_payloads,
-					   const struct iwl_ucode_capabilities *capa);
-void iwl_trans_pcie_ctx_info_gen3_set_pnvm(struct iwl_trans *trans,
-					   const struct iwl_ucode_capabilities *capa);
+int iwl_trans_pcie_ctx_info_v2_load_pnvm(struct iwl_trans *trans,
+					 const struct iwl_pnvm_image *pnvm_payloads,
+					 const struct iwl_ucode_capabilities *capa);
+void iwl_trans_pcie_ctx_info_v2_set_pnvm(struct iwl_trans *trans,
+					 const struct iwl_ucode_capabilities *capa);
 int
-iwl_trans_pcie_ctx_info_gen3_load_reduce_power(struct iwl_trans *trans,
-					       const struct iwl_pnvm_image *payloads,
-					       const struct iwl_ucode_capabilities *capa);
+iwl_trans_pcie_ctx_info_v2_load_reduce_power(struct iwl_trans *trans,
+					     const struct iwl_pnvm_image *payloads,
+					     const struct iwl_ucode_capabilities *capa);
 void
-iwl_trans_pcie_ctx_info_gen3_set_reduce_power(struct iwl_trans *trans,
-					      const struct iwl_ucode_capabilities *capa);
-int iwl_trans_pcie_ctx_info_gen3_set_step(struct iwl_trans *trans,
-					  u32 mbx_addr_0_step, u32 mbx_addr_1_step);
-#endif /* __iwl_context_info_file_gen3_h__ */
+iwl_trans_pcie_ctx_info_v2_set_reduce_power(struct iwl_trans *trans,
+					    const struct iwl_ucode_capabilities *capa);
+#endif /* __iwl_context_info_file_v2_h__ */
diff --git a/sys/contrib/dev/iwlwifi/iwl-context-info.h b/sys/contrib/dev/iwlwifi/pcie/iwl-context-info.h
index 1a1321db137c..7ae0fbdef208 100644
--- a/sys/contrib/dev/iwlwifi/iwl-context-info.h
+++ b/sys/contrib/dev/iwlwifi/pcie/iwl-context-info.h
@@ -1,12 +1,12 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2020, 2022 Intel Corporation
+ * Copyright (C) 2018-2020, 2022, 2024-2025 Intel Corporation
  */
 #ifndef __iwl_context_info_file_h__
 #define __iwl_context_info_file_h__
 
-/* maximmum number of DRAM map entries supported by FW */
+/* maximum number of DRAM map entries supported by FW */
 #define IWL_MAX_DRAM_ENTRY	64
 #define CSR_CTXT_INFO_BA	0x40
 
@@ -53,11 +53,12 @@ enum iwl_context_info_flags {
 	IWL_CTXT_INFO_RB_SIZE_32K	= 0xe,
 };
 
-/*
+/**
  * struct iwl_context_info_version - version structure
  * @mac_id: SKU and revision id
  * @version: context information version id
  * @size: the size of the context information in DWs
+ * @reserved: (reserved)
  */
 struct iwl_context_info_version {
 	__le16 mac_id;
@@ -66,29 +67,30 @@ struct iwl_context_info_version {
 	__le16 reserved;
 } __packed;
 
-/*
+/**
  * struct iwl_context_info_control - version structure
  * @control_flags: context information flags see &enum iwl_context_info_flags
+ * @reserved: (reserved)
  */
 struct iwl_context_info_control {
 	__le32 control_flags;
 	__le32 reserved;
 } __packed;
 
-/*
- * struct iwl_context_info_dram - images DRAM map
+/**
+ * struct iwl_context_info_dram_nonfseq - images DRAM map
  * each entry in the map represents a DRAM chunk of up to 32 KB
  * @umac_img: UMAC image DRAM map
  * @lmac_img: LMAC image DRAM map
  * @virtual_img: paged image DRAM map
  */
-struct iwl_context_info_dram {
+struct iwl_context_info_dram_nonfseq {
 	__le64 umac_img[IWL_MAX_DRAM_ENTRY];
 	__le64 lmac_img[IWL_MAX_DRAM_ENTRY];
 	__le64 virtual_img[IWL_MAX_DRAM_ENTRY];
 } __packed;
 
-/*
+/**
  * struct iwl_context_info_rbd_cfg - RBDs configuration
  * @free_rbd_addr: default queue free RB CB base address
  * @used_rbd_addr: default queue used RB CB base address
@@ -100,10 +102,11 @@ struct iwl_context_info_rbd_cfg {
 	__le64 status_wr_ptr;
 } __packed;
 
-/*
+/**
  * struct iwl_context_info_hcmd_cfg  - command queue configuration
  * @cmd_queue_addr: address of command queue
  * @cmd_queue_size: number of entries
+ * @reserved: (reserved)
  */
 struct iwl_context_info_hcmd_cfg {
 	__le64 cmd_queue_addr;
@@ -111,10 +114,11 @@ struct iwl_context_info_hcmd_cfg {
 	u8 reserved[7];
 } __packed;
 
-/*
+/**
  * struct iwl_context_info_dump_cfg - Core Dump configuration
  * @core_dump_addr: core dump (debug DRAM address) start address
  * @core_dump_size: size, in DWs
+ * @reserved: (reserved)
  */
 struct iwl_context_info_dump_cfg {
 	__le64 core_dump_addr;
@@ -122,10 +126,11 @@ struct iwl_context_info_dump_cfg {
 	__le32 reserved;
 } __packed;
 
-/*
+/**
  * struct iwl_context_info_pnvm_cfg - platform NVM data configuration
  * @platform_nvm_addr: Platform NVM data start address
  * @platform_nvm_size: size in DWs
+ * @reserved: (reserved)
  */
 struct iwl_context_info_pnvm_cfg {
 	__le64 platform_nvm_addr;
@@ -133,11 +138,12 @@ struct iwl_context_info_pnvm_cfg {
 	__le32 reserved;
 } __packed;
 
-/*
+/**
  * struct iwl_context_info_early_dbg_cfg - early debug configuration for
  *	dumping DRAM addresses
  * @early_debug_addr: early debug start address
  * @early_debug_size: size in DWs
+ * @reserved: (reserved)
  */
 struct iwl_context_info_early_dbg_cfg {
 	__le64 early_debug_addr;
@@ -145,16 +151,20 @@ struct iwl_context_info_early_dbg_cfg {
 	__le32 reserved;
 } __packed;
 
-/*
+/**
  * struct iwl_context_info - device INIT configuration
  * @version: version information of context info and HW
  * @control: control flags of FH configurations
+ * @reserved0: (reserved)
  * @rbd_cfg: default RX queue configuration
  * @hcmd_cfg: command queue configuration
+ * @reserved1: (reserved)
  * @dump_cfg: core dump data
  * @edbg_cfg: early debug configuration
  * @pnvm_cfg: platform nvm configuration
+ * @reserved2: (reserved)
  * @dram: firmware image addresses in DRAM
+ * @reserved3: (reserved)
  */
 struct iwl_context_info {
 	struct iwl_context_info_version version;
@@ -167,16 +177,16 @@ struct iwl_context_info {
 	struct iwl_context_info_early_dbg_cfg edbg_cfg;
 	struct iwl_context_info_pnvm_cfg pnvm_cfg;
 	__le32 reserved2[16];
-	struct iwl_context_info_dram dram;
+	struct iwl_context_info_dram_nonfseq dram;
 	__le32 reserved3[16];
-} __packed;
+} __packed; /* BOOT_LOADER_CONTEXT_INFO_S */
 
-int iwl_pcie_ctxt_info_init(struct iwl_trans *trans, const struct fw_img *fw);
+int iwl_pcie_ctxt_info_init(struct iwl_trans *trans, const struct fw_img *img);
 void iwl_pcie_ctxt_info_free(struct iwl_trans *trans);
 void iwl_pcie_ctxt_info_free_paging(struct iwl_trans *trans);
 int iwl_pcie_init_fw_sec(struct iwl_trans *trans,
 			 const struct fw_img *fw,
-			 struct iwl_context_info_dram *ctxt_dram);
+			 struct iwl_context_info_dram_nonfseq *ctxt_dram);
 void *iwl_pcie_ctxt_info_dma_alloc_coherent(struct iwl_trans *trans,
 					    size_t size,
 					    dma_addr_t *phys);
diff --git a/sys/contrib/dev/iwlwifi/pcie/utils.c b/sys/contrib/dev/iwlwifi/pcie/utils.c
new file mode 100644
index 000000000000..d777e1517cc5
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/pcie/utils.c
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+
+#include <linux/pci.h>
+#include <linux/gfp.h>
+
+#include "iwl-io.h"
+#include "pcie/utils.h"
+
+void iwl_trans_pcie_dump_regs(struct iwl_trans *trans, struct pci_dev *pdev)
+{
+#define PCI_DUMP_SIZE		352
+#define PCI_MEM_DUMP_SIZE	64
+#define PCI_PARENT_DUMP_SIZE	524
+#define PREFIX_LEN		32
+
+	static bool pcie_dbg_dumped_once = 0;
+	u32 i, pos, alloc_size, *ptr, *buf;
+	char *prefix;
+
+	if (pcie_dbg_dumped_once)
+		return;
+
+	/* Should be a multiple of 4 */
+	BUILD_BUG_ON(PCI_DUMP_SIZE > 4096 || PCI_DUMP_SIZE & 0x3);
+	BUILD_BUG_ON(PCI_MEM_DUMP_SIZE > 4096 || PCI_MEM_DUMP_SIZE & 0x3);
+	BUILD_BUG_ON(PCI_PARENT_DUMP_SIZE > 4096 || PCI_PARENT_DUMP_SIZE & 0x3);
+
+	/* Alloc a max size buffer */
+	alloc_size = PCI_ERR_ROOT_ERR_SRC +  4 + PREFIX_LEN;
+	alloc_size = max_t(u32, alloc_size, PCI_DUMP_SIZE + PREFIX_LEN);
+	alloc_size = max_t(u32, alloc_size, PCI_MEM_DUMP_SIZE + PREFIX_LEN);
+	alloc_size = max_t(u32, alloc_size, PCI_PARENT_DUMP_SIZE + PREFIX_LEN);
+
+	buf = kmalloc(alloc_size, GFP_ATOMIC);
+	if (!buf)
+		return;
+	prefix = (char *)buf + alloc_size - PREFIX_LEN;
+
+	IWL_ERR(trans, "iwlwifi transaction failed, dumping registers\n");
+
+	/* Print wifi device registers */
+	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
+	IWL_ERR(trans, "iwlwifi device config registers:\n");
+	for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
+		if (pci_read_config_dword(pdev, i, ptr))
+			goto err_read;
+#if defined(__linux__)
+	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
+#elif defined(__FreeBSD__)
+	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
+#endif
+
+	IWL_ERR(trans, "iwlwifi device memory mapped registers:\n");
+	for (i = 0, ptr = buf; i < PCI_MEM_DUMP_SIZE; i += 4, ptr++)
+		*ptr = iwl_read32(trans, i);
+#if defined(__linux__)
+	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
+#elif defined(__FreeBSD__)
+	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
+#endif
+
+	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
+	if (pos) {
+		IWL_ERR(trans, "iwlwifi device AER capability structure:\n");
+		for (i = 0, ptr = buf; i < PCI_ERR_ROOT_COMMAND; i += 4, ptr++)
+			if (pci_read_config_dword(pdev, pos + i, ptr))
+				goto err_read;
+#if defined(__linux__)
+		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
+			       32, 4, buf, i, 0);
+#elif defined(__FreeBSD__)
+		iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
+#endif
+	}
+
+	/* Print parent device registers next */
+	if (!pdev->bus->self)
+		goto out;
+
+	pdev = pdev->bus->self;
+	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
+
+	IWL_ERR(trans, "iwlwifi parent port (%s) config registers:\n",
+		pci_name(pdev));
+	for (i = 0, ptr = buf; i < PCI_PARENT_DUMP_SIZE; i += 4, ptr++)
+		if (pci_read_config_dword(pdev, i, ptr))
+			goto err_read;
+#if defined(__linux__)
+	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
+#elif defined(__FreeBSD__)
+	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
+#endif
+
+	/* Print root port AER registers */
+	pos = 0;
+	pdev = pcie_find_root_port(pdev);
+	if (pdev)
+		pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
+	if (pos) {
+		IWL_ERR(trans, "iwlwifi root port (%s) AER cap structure:\n",
+			pci_name(pdev));
+		sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
+		for (i = 0, ptr = buf; i <= PCI_ERR_ROOT_ERR_SRC; i += 4, ptr++)
+			if (pci_read_config_dword(pdev, pos + i, ptr))
+				goto err_read;
+#if defined(__linux__)
+		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32,
+			       4, buf, i, 0);
+#elif defined(__FreeBSD__)
+		iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
+#endif
+	}
+	goto out;
+
+err_read:
+#if defined(__linux__)
+	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
+#elif defined(__FreeBSD__)
+	iwl_print_hex_dump(NULL, IWL_DL_ANY, prefix, (u8 *)buf, i);
+#endif
+	IWL_ERR(trans, "Read failed at 0x%X\n", i);
+out:
+	pcie_dbg_dumped_once = 1;
+	kfree(buf);
+}
diff --git a/sys/contrib/dev/iwlwifi/pcie/utils.h b/sys/contrib/dev/iwlwifi/pcie/utils.h
new file mode 100644
index 000000000000..27437d5e099b
--- /dev/null
+++ b/sys/contrib/dev/iwlwifi/pcie/utils.h
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+
+#ifndef __iwl_pcie_utils_h__
+#define __iwl_pcie_utils_h__
+
+#include "iwl-io.h"
+
+void iwl_trans_pcie_dump_regs(struct iwl_trans *trans, struct pci_dev *pdev);
+
+static inline void _iwl_trans_set_bits_mask(struct iwl_trans *trans,
+					    u32 reg, u32 mask, u32 value)
+{
+	u32 v;
+
+#ifdef CONFIG_IWLWIFI_DEBUG
+	WARN_ON_ONCE(value & ~mask);
+#endif
+
+	v = iwl_read32(trans, reg);
+	v &= ~mask;
+	v |= value;
+	iwl_write32(trans, reg, v);
+}
+
+static inline void iwl_trans_clear_bit(struct iwl_trans *trans,
+				       u32 reg, u32 mask)
+{
+	_iwl_trans_set_bits_mask(trans, reg, mask, 0);
+}
+
+static inline void iwl_trans_set_bit(struct iwl_trans *trans,
+				     u32 reg, u32 mask)
+{
+	_iwl_trans_set_bits_mask(trans, reg, mask, mask);
+}
+
+#endif /* __iwl_pcie_utils_h__ */
diff --git a/sys/contrib/dev/iwlwifi/tests/Makefile b/sys/contrib/dev/iwlwifi/tests/Makefile
index 84491488f589..1b49241c578f 100644
--- a/sys/contrib/dev/iwlwifi/tests/Makefile
+++ b/sys/contrib/dev/iwlwifi/tests/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 
-iwlwifi-tests-y += module.o devinfo.o
+iwlwifi-tests-y += module.o devinfo.o utils.o
 
 ccflags-y += -I$(src)/../
 
diff --git a/sys/contrib/dev/iwlwifi/tests/devinfo.c b/sys/contrib/dev/iwlwifi/tests/devinfo.c
index 7361b6d0cdb8..c31bbd4e7a4a 100644
--- a/sys/contrib/dev/iwlwifi/tests/devinfo.c
+++ b/sys/contrib/dev/iwlwifi/tests/devinfo.c
@@ -2,21 +2,59 @@
 /*
  * KUnit tests for the iwlwifi device info table
  *
- * Copyright (C) 2023-2024 Intel Corporation
+ * Copyright (C) 2023-2025 Intel Corporation
  */
 #include <kunit/test.h>
 #include <linux/pci.h>
 #include "iwl-drv.h"
 #include "iwl-config.h"
 
-MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING);
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
 
 static void iwl_pci_print_dev_info(const char *pfx, const struct iwl_dev_info *di)
 {
-	printk(KERN_DEBUG "%sdev=%.4x,subdev=%.4x,mac_type=%.4x,mac_step=%.4x,rf_type=%.4x,cdb=%d,jacket=%d,rf_id=%.2x,no_160=%d,cores=%.2x\n",
-	       pfx, di->device, di->subdevice, di->mac_type, di->mac_step,
-	       di->rf_type, di->cdb, di->jacket, di->rf_id, di->no_160,
-	       di->cores);
+	u16 subdevice_mask = GENMASK(di->subdevice_m_h, di->subdevice_m_l);
+	char buf[100] = {};
+	int pos = 0;
+
+	if (di->match_rf_type)
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " rf_type=%03x", di->rf_type);
+	else
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " rf_type=*");
+
+	if (di->match_bw_limit)
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " bw_limit=%d", di->bw_limit);
+	else
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " bw_limit=*");
+
+	if (di->match_rf_id)
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " rf_id=0x%x", di->rf_id);
+	else
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " rf_id=*");
+
+	if (di->match_cdb)
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " cdb=%d", di->cdb);
+	else
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " cdb=*");
+
+	if (di->match_discrete)
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " discrete=%d",
+				 di->discrete);
+	else
+		pos += scnprintf(buf + pos, sizeof(buf) - pos,
+				 " discrete=*");
+
+	printk(KERN_DEBUG "%sdev=%04x subdev=%04x/%04x%s\n",
+	       pfx, di->device, di->subdevice, subdevice_mask, buf);
 }
 
 static void devinfo_table_order(struct kunit *test)
@@ -28,11 +66,14 @@ static void devinfo_table_order(struct kunit *test)
 		const struct iwl_dev_info *ret;
 
 		ret = iwl_pci_find_dev_info(di->device, di->subdevice,
-					    di->mac_type, di->mac_step,
 					    di->rf_type, di->cdb,
-					    di->jacket, di->rf_id,
-					    di->no_160, di->cores, di->rf_step);
-		if (ret != di) {
+					    di->rf_id, di->bw_limit,
+					    di->discrete);
+		if (!ret) {
+			iwl_pci_print_dev_info("No entry found for: ", di);
+			KUNIT_FAIL(test,
+				   "No entry found for entry at index %d\n", idx);
+		} else if (ret != di) {
 			iwl_pci_print_dev_info("searched: ", di);
 			iwl_pci_print_dev_info("found:    ", ret);
 			KUNIT_FAIL(test,
@@ -42,6 +83,118 @@ static void devinfo_table_order(struct kunit *test)
 	}
 }
 
+static void devinfo_discrete_match(struct kunit *test)
+{
+	/*
+	 * Validate that any entries with discrete/integrated match have
+	 * the same config with the value inverted (if they match at all.)
+	 */
+
+	for (int idx = 0; idx < iwl_dev_info_table_size; idx++) {
+		const struct iwl_dev_info *di = &iwl_dev_info_table[idx];
+		const struct iwl_dev_info *ret;
+
+		if (!di->match_discrete)
+			continue;
+
+		ret = iwl_pci_find_dev_info(di->device, di->subdevice,
+					    di->rf_type, di->cdb,
+					    di->rf_id, di->bw_limit,
+					    !di->discrete);
+		if (!ret)
+			continue;
+		KUNIT_EXPECT_PTR_EQ(test, di->cfg, ret->cfg);
+		/* and check the name is different, that'd be the point of it */
+		KUNIT_EXPECT_NE(test, strcmp(di->name, ret->name), 0);
+	}
+}
+
+static void devinfo_names(struct kunit *test)
+{
+	int idx;
+
+	for (idx = 0; idx < iwl_dev_info_table_size; idx++) {
+		const struct iwl_dev_info *di = &iwl_dev_info_table[idx];
+
+		KUNIT_ASSERT_TRUE(test, di->name);
+	}
+}
+
+static void devinfo_no_cfg_dups(struct kunit *test)
+{
+	for (int i = 0; i < iwl_dev_info_table_size; i++) {
+		const struct iwl_rf_cfg *cfg_i = iwl_dev_info_table[i].cfg;
+
+		for (int j = 0; j < i; j++) {
+			const struct iwl_rf_cfg *cfg_j = iwl_dev_info_table[j].cfg;
+
+			if (cfg_i == cfg_j)
+				continue;
+
+			KUNIT_EXPECT_NE_MSG(test, memcmp(cfg_i, cfg_j,
+							 sizeof(*cfg_i)), 0,
+					    "identical configs: %ps and %ps\n",
+					    cfg_i, cfg_j);
+		}
+	}
+}
+
+static void devinfo_no_name_dups(struct kunit *test)
+{
+	for (int i = 0; i < iwl_dev_info_table_size; i++) {
+		for (int j = 0; j < i; j++) {
+			if (iwl_dev_info_table[i].name == iwl_dev_info_table[j].name)
+				continue;
+
+			KUNIT_EXPECT_NE_MSG(test,
+					    strcmp(iwl_dev_info_table[i].name,
+						   iwl_dev_info_table[j].name),
+					    0,
+					    "name dup: %ps/%ps",
+					    iwl_dev_info_table[i].name,
+					    iwl_dev_info_table[j].name);
+		}
+	}
+}
+
+static void devinfo_check_subdev_match(struct kunit *test)
+{
+	for (int i = 0; i < iwl_dev_info_table_size; i++) {
+		const struct iwl_dev_info *di = &iwl_dev_info_table[i];
+		u16 subdevice_mask = GENMASK(di->subdevice_m_h,
+					     di->subdevice_m_l);
+
+		/* if BW limit bit is matched then must have a limit */
+		if (di->match_bw_limit == 1 && di->bw_limit == 1)
+			KUNIT_EXPECT_NE(test, di->cfg->bw_limit, 0);
+
+		/* if subdevice is ANY we can have RF ID/BW limit */
+		if (di->subdevice == (u16)IWL_CFG_ANY)
+			continue;
+
+		/* same if the subdevice mask doesn't overlap them */
+		if (IWL_SUBDEVICE_RF_ID(subdevice_mask) == 0 &&
+		    IWL_SUBDEVICE_BW_LIM(subdevice_mask) == 0)
+			continue;
+
+		/* but otherwise they shouldn't be used */
+		KUNIT_EXPECT_EQ(test, (int)di->match_rf_id, 0);
+		KUNIT_EXPECT_EQ(test, (int)di->match_bw_limit, 0);
+	}
+}
+
+static void devinfo_check_killer_subdev(struct kunit *test)
+{
+	for (int i = 0; i < iwl_dev_info_table_size; i++) {
+		const struct iwl_dev_info *di = &iwl_dev_info_table[i];
+
+		if (!strstr(di->name, "Killer"))
+			continue;
+
+		KUNIT_EXPECT_NE(test, di->subdevice, (u16)IWL_CFG_ANY);
+	}
+}
+
 static void devinfo_pci_ids(struct kunit *test)
 {
 	struct pci_dev *dev;
@@ -64,9 +217,65 @@ static void devinfo_pci_ids(struct kunit *test)
 	}
 }
 
+static void devinfo_no_mac_cfg_dups(struct kunit *test)
+{
+	for (int i = 0; iwl_hw_card_ids[i].vendor; i++) {
+		const struct iwl_mac_cfg *cfg_i =
+			(void *)iwl_hw_card_ids[i].driver_data;
+
+		for (int j = 0; j < i; j++) {
+			const struct iwl_mac_cfg *cfg_j =
+				(void *)iwl_hw_card_ids[j].driver_data;
+
+			if (cfg_i == cfg_j)
+				continue;
+
+			KUNIT_EXPECT_NE_MSG(test, memcmp(cfg_j, cfg_i,
+							 sizeof(*cfg_i)), 0,
+					    "identical configs: %ps and %ps\n",
+					    cfg_i, cfg_j);
+		}
+	}
+}
+
+static void devinfo_api_range(struct kunit *test)
+{
+	/* Check that all iwl_mac_cfg's have either both min and max set, or neither */
+	for (int i = 0; iwl_hw_card_ids[i].vendor; i++) {
+		const struct iwl_mac_cfg *mac_cfg =
+			(void *)iwl_hw_card_ids[i].driver_data;
+		const struct iwl_family_base_params *base = mac_cfg->base;
+
+		KUNIT_EXPECT_EQ_MSG(test, !!base->ucode_api_min,
+				    !!base->ucode_api_max,
+				    "%ps: ucode_api_min (%u) and ucode_api_min (%u) should be both set or neither.\n",
+				    base, base->ucode_api_min,
+				    base->ucode_api_max);
+	}
+
+	/* Check the same for the iwl_rf_cfg's */
+	for (int i = 0; i < iwl_dev_info_table_size; i++) {
+		const struct iwl_rf_cfg *rf_cfg = iwl_dev_info_table[i].cfg;
+
+		KUNIT_EXPECT_EQ_MSG(test, !!rf_cfg->ucode_api_min,
+				    !!rf_cfg->ucode_api_max,
+				    "%ps: ucode_api_min (%u) and ucode_api_min (%u) should be both set or neither.\n",
+				    rf_cfg, rf_cfg->ucode_api_min,
+				    rf_cfg->ucode_api_max);
+	}
+}
+
 static struct kunit_case devinfo_test_cases[] = {
 	KUNIT_CASE(devinfo_table_order),
+	KUNIT_CASE(devinfo_discrete_match),
+	KUNIT_CASE(devinfo_names),
+	KUNIT_CASE(devinfo_no_cfg_dups),
+	KUNIT_CASE(devinfo_no_name_dups),
+	KUNIT_CASE(devinfo_check_subdev_match),
+	KUNIT_CASE(devinfo_check_killer_subdev),
 	KUNIT_CASE(devinfo_pci_ids),
+	KUNIT_CASE(devinfo_no_mac_cfg_dups),
+	KUNIT_CASE(devinfo_api_range),
 	{}
 };
 
diff --git a/sys/contrib/dev/mediatek/mt76/channel.c b/sys/contrib/dev/mediatek/mt76/channel.c
index 6a35c6ebd823..77b75792eb48 100644
--- a/sys/contrib/dev/mediatek/mt76/channel.c
+++ b/sys/contrib/dev/mediatek/mt76/channel.c
@@ -173,13 +173,13 @@ void mt76_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	if (!mlink)
 		goto out;
 
-	if (link_conf != &vif->bss_conf)
+	if (mlink != (struct mt76_vif_link *)vif->drv_priv)
 		rcu_assign_pointer(mvif->link[link_id], NULL);
 
 	dev->drv->vif_link_remove(phy, vif, link_conf, mlink);
 	mlink->ctx = NULL;
 
-	if (link_conf != &vif->bss_conf)
+	if (mlink != (struct mt76_vif_link *)vif->drv_priv)
 		kfree_rcu(mlink, rcu_head);
 
 out:
@@ -293,6 +293,7 @@ struct mt76_vif_link *mt76_get_vif_phy_link(struct mt76_phy *phy,
 		kfree(mlink);
 		return ERR_PTR(ret);
 	}
+	rcu_assign_pointer(mvif->offchannel_link, mlink);
 
 	return mlink;
 }
@@ -301,10 +302,14 @@ void mt76_put_vif_phy_link(struct mt76_phy *phy, struct ieee80211_vif *vif,
 			   struct mt76_vif_link *mlink)
 {
 	struct mt76_dev *dev = phy->dev;
+	struct mt76_vif_data *mvif;
 
 	if (IS_ERR_OR_NULL(mlink) || !mlink->offchannel)
 		return;
 
+	mvif = mlink->mvif;
+
+	rcu_assign_pointer(mvif->offchannel_link, NULL);
 	dev->drv->vif_link_remove(phy, vif, &vif->bss_conf, mlink);
 	kfree(mlink);
 }
diff --git a/sys/contrib/dev/mediatek/mt76/dma.c b/sys/contrib/dev/mediatek/mt76/dma.c
index 6765e1281ac3..af902a761e42 100644
--- a/sys/contrib/dev/mediatek/mt76/dma.c
+++ b/sys/contrib/dev/mediatek/mt76/dma.c
@@ -6,7 +6,7 @@
 #include <linux/dma-mapping.h>
 #if defined(__FreeBSD__)
 #include <linux/cache.h>
-#include <net/page_pool.h>
+#include <net/page_pool/helpers.h>
 #endif
 #include "mt76.h"
 #include "dma.h"
@@ -647,10 +647,8 @@ mt76_dma_rx_fill_buf(struct mt76_dev *dev, struct mt76_queue *q,
 
 	while (q->queued < q->ndesc - 1) {
 		struct mt76_queue_buf qbuf = {};
-		enum dma_data_direction dir;
-		dma_addr_t addr;
-		int offset;
 		void *buf = NULL;
+		int offset;
 
 		if (mt76_queue_is_wed_rro_ind(q))
 			goto done;
@@ -659,11 +657,8 @@ mt76_dma_rx_fill_buf(struct mt76_dev *dev, struct mt76_queue *q,
 		if (!buf)
 			break;
 
-		addr = page_pool_get_dma_addr(virt_to_head_page(buf)) + offset;
-		dir = page_pool_get_dma_dir(q->page_pool);
-		dma_sync_single_for_device(dev->dma_dev, addr, len, dir);
-
-		qbuf.addr = addr + q->buf_offset;
+		qbuf.addr = page_pool_get_dma_addr(virt_to_head_page(buf)) +
+			    offset + q->buf_offset;
 done:
 		qbuf.len = len - q->buf_offset;
 		qbuf.skip_unmap = false;
@@ -1023,6 +1018,7 @@ void mt76_dma_cleanup(struct mt76_dev *dev)
 	int i;
 
 	mt76_worker_disable(&dev->tx_worker);
+	napi_disable(&dev->tx_napi);
 	netif_napi_del(&dev->tx_napi);
 
 	for (i = 0; i < ARRAY_SIZE(dev->phys); i++) {
diff --git a/sys/contrib/dev/mediatek/mt76/eeprom.c b/sys/contrib/dev/mediatek/mt76/eeprom.c
index eb25879e2021..f2eb2cd6e509 100644
--- a/sys/contrib/dev/mediatek/mt76/eeprom.c
+++ b/sys/contrib/dev/mediatek/mt76/eeprom.c
@@ -101,6 +101,10 @@ int mt76_get_of_data_from_mtd(struct mt76_dev *dev, void *eep, int offset, int l
 
 #ifdef CONFIG_NL80211_TESTMODE
 	dev->test_mtd.name = devm_kstrdup(dev->dev, part, GFP_KERNEL);
+	if (!dev->test_mtd.name) {
+		ret = -ENOMEM;
+		goto out_put_node;
+	}
 	dev->test_mtd.offset = offset;
 #endif
 
diff --git a/sys/contrib/dev/mediatek/mt76/mac80211.c b/sys/contrib/dev/mediatek/mt76/mac80211.c
index f4a714c57f82..927d8519104f 100644
--- a/sys/contrib/dev/mediatek/mt76/mac80211.c
+++ b/sys/contrib/dev/mediatek/mt76/mac80211.c
@@ -459,8 +459,10 @@ mt76_phy_init(struct mt76_phy *phy, struct ieee80211_hw *hw)
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_AIRTIME_FAIRNESS);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_AQL);
 
-	wiphy->available_antennas_tx = phy->antenna_mask;
-	wiphy->available_antennas_rx = phy->antenna_mask;
+	if (!wiphy->available_antennas_tx)
+		wiphy->available_antennas_tx = phy->antenna_mask;
+	if (!wiphy->available_antennas_rx)
+		wiphy->available_antennas_rx = phy->antenna_mask;
 
 	wiphy->sar_capa = &mt76_sar_capa;
 	phy->frp = devm_kcalloc(dev->dev, wiphy->sar_capa->num_freq_ranges,
@@ -848,8 +850,45 @@ void mt76_free_device(struct mt76_dev *dev)
 }
 EXPORT_SYMBOL_GPL(mt76_free_device);
 
-static struct mt76_phy *
-mt76_vif_phy(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+static void mt76_reset_phy(struct mt76_phy *phy)
+{
+	if (!phy)
+		return;
+
+	INIT_LIST_HEAD(&phy->tx_list);
+}
+
+void mt76_reset_device(struct mt76_dev *dev)
+{
+	int i;
+
+	rcu_read_lock();
+	for (i = 0; i < ARRAY_SIZE(dev->wcid); i++) {
+		struct mt76_wcid *wcid;
+
+		wcid = rcu_dereference(dev->wcid[i]);
+		if (!wcid)
+			continue;
+
+		wcid->sta = 0;
+		mt76_wcid_cleanup(dev, wcid);
+		rcu_assign_pointer(dev->wcid[i], NULL);
+	}
+	rcu_read_unlock();
+
+	INIT_LIST_HEAD(&dev->wcid_list);
+	INIT_LIST_HEAD(&dev->sta_poll_list);
+	dev->vif_mask = 0;
+	memset(dev->wcid_mask, 0, sizeof(dev->wcid_mask));
+
+	mt76_reset_phy(&dev->phy);
+	for (i = 0; i < ARRAY_SIZE(dev->phys); i++)
+		mt76_reset_phy(dev->phys[i]);
+}
+EXPORT_SYMBOL_GPL(mt76_reset_device);
+
+struct mt76_phy *mt76_vif_phy(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif)
 {
 	struct mt76_vif_link *mlink = (struct mt76_vif_link *)vif->drv_priv;
 	struct mt76_chanctx *ctx;
@@ -863,6 +902,7 @@ mt76_vif_phy(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 	ctx = (struct mt76_chanctx *)mlink->ctx->drv_priv;
 	return ctx->phy;
 }
+EXPORT_SYMBOL_GPL(mt76_vif_phy);
 
 static void mt76_rx_release_amsdu(struct mt76_phy *phy, enum mt76_rxq_id q)
 {
@@ -1712,6 +1752,10 @@ void mt76_wcid_cleanup(struct mt76_dev *dev, struct mt76_wcid *wcid)
 	skb_queue_splice_tail_init(&wcid->tx_pending, &list);
 	spin_unlock(&wcid->tx_pending.lock);
 
+	spin_lock(&wcid->tx_offchannel.lock);
+	skb_queue_splice_tail_init(&wcid->tx_offchannel, &list);
+	spin_unlock(&wcid->tx_offchannel.lock);
+
 	spin_unlock_bh(&phy->tx_lock);
 
 	while ((skb = __skb_dequeue(&list)) != NULL) {
@@ -1723,7 +1767,7 @@ EXPORT_SYMBOL_GPL(mt76_wcid_cleanup);
 
 void mt76_wcid_add_poll(struct mt76_dev *dev, struct mt76_wcid *wcid)
 {
-	if (test_bit(MT76_MCU_RESET, &dev->phy.state))
+	if (test_bit(MT76_MCU_RESET, &dev->phy.state) || !wcid->sta)
 		return;
 
 	spin_lock_bh(&dev->sta_poll_lock);
@@ -1733,6 +1777,17 @@ void mt76_wcid_add_poll(struct mt76_dev *dev, struct mt76_wcid *wcid)
 }
 EXPORT_SYMBOL_GPL(mt76_wcid_add_poll);
 
+s8 mt76_get_power_bound(struct mt76_phy *phy, s8 txpower)
+{
+	int n_chains = hweight16(phy->chainmask);
+
+	txpower = mt76_get_sar_power(phy, phy->chandef.chan, txpower * 2);
+	txpower -= mt76_tx_power_path_delta(n_chains);
+
+	return txpower;
+}
+EXPORT_SYMBOL_GPL(mt76_get_power_bound);
+
 int mt76_get_txpower(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		     unsigned int link_id, int *dbm)
 {
@@ -1743,7 +1798,7 @@ int mt76_get_txpower(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		return -EINVAL;
 
 	n_chains = hweight16(phy->chainmask);
-	delta = mt76_tx_power_nss_delta(n_chains);
+	delta = mt76_tx_power_path_delta(n_chains);
 	*dbm = DIV_ROUND_UP(phy->txpower_cur + delta, 2);
 
 	return 0;
@@ -1914,7 +1969,8 @@ void mt76_sw_scan_complete(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 }
 EXPORT_SYMBOL_GPL(mt76_sw_scan_complete);
 
-int mt76_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant)
+int mt76_get_antenna(struct ieee80211_hw *hw, int radio_idx, u32 *tx_ant,
+		     u32 *rx_ant)
 {
 	struct mt76_phy *phy = hw->priv;
 	struct mt76_dev *dev = phy->dev;
diff --git a/sys/contrib/dev/mediatek/mt76/mcu.c b/sys/contrib/dev/mediatek/mt76/mcu.c
index f8f47a40d3be..d554eed10986 100644
--- a/sys/contrib/dev/mediatek/mt76/mcu.c
+++ b/sys/contrib/dev/mediatek/mt76/mcu.c
@@ -78,6 +78,10 @@ int mt76_mcu_skb_send_and_get_msg(struct mt76_dev *dev, struct sk_buff *skb,
 	unsigned long expires;
 	int ret, seq;
 
+	if (mt76_is_sdio(dev))
+		if (test_bit(MT76_RESET, &dev->phy.state) && atomic_read(&dev->bus_hung))
+			return -EIO;
+
 	if (ret_skb)
 		*ret_skb = NULL;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt76.h b/sys/contrib/dev/mediatek/mt76/mt76.h
index c54d02346262..0b7686e6c36e 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76.h
+++ b/sys/contrib/dev/mediatek/mt76/mt76.h
@@ -20,7 +20,6 @@
 #include <linux/debugfs.h>
 #include <linux/pci.h>
 #include <linux/interrupt.h>
-#include <net/page_pool.h>
 #endif
 #include <net/mac80211.h>
 #include <net/page_pool/helpers.h>
@@ -170,6 +169,16 @@ enum mt76_dfs_state {
 	MT_DFS_STATE_ACTIVE,
 };
 
+#define MT76_RNR_SCAN_MAX_BSSIDS       16
+struct mt76_scan_rnr_param {
+	u8 bssid[MT76_RNR_SCAN_MAX_BSSIDS][ETH_ALEN];
+	u8 channel[MT76_RNR_SCAN_MAX_BSSIDS];
+	u8 random_mac[ETH_ALEN];
+	u8 seq_num;
+	u8 bssid_num;
+	u32 sreq_flag;
+};
+
 struct mt76_queue_buf {
 	dma_addr_t addr;
 	u16 len:15,
@@ -359,6 +368,7 @@ struct mt76_wcid {
 	u8 hw_key_idx;
 	u8 hw_key_idx2;
 
+	u8 offchannel:1;
 	u8 sta:1;
 	u8 sta_disabled:1;
 	u8 amsdu:1;
@@ -499,6 +509,7 @@ struct mt76_hw_cap {
 #define MT_DRV_RX_DMA_HDR		BIT(3)
 #define MT_DRV_HW_MGMT_TXQ		BIT(4)
 #define MT_DRV_AMSDU_OFFLOAD		BIT(5)
+#define MT_DRV_IGNORE_TXS_FAILED	BIT(6)
 
 struct mt76_driver_ops {
 	u32 drv_flags;
@@ -777,6 +788,7 @@ struct mt76_testmode_data {
 
 struct mt76_vif_link {
 	u8 idx;
+	u8 link_idx;
 	u8 omac_idx;
 	u8 band_idx;
 	u8 wmm_idx;
@@ -794,6 +806,7 @@ struct mt76_vif_link {
 
 struct mt76_vif_data {
 	struct mt76_vif_link __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
+	struct mt76_vif_link __rcu *offchannel_link;
 
 	struct mt76_phy *roc_phy;
 	u16 valid_links;
@@ -945,6 +958,8 @@ struct mt76_dev {
 	char alpha2[3];
 	enum nl80211_dfs_regions region;
 
+	struct mt76_scan_rnr_param rnr;
+
 	u32 debugfs_reg;
 
 	u8 csa_complete;
@@ -975,6 +990,8 @@ struct mt76_dev {
 		struct mt76_usb usb;
 		struct mt76_sdio sdio;
 	};
+
+	atomic_t bus_hung;
 };
 
 /* per-phy stats.  */
@@ -1216,6 +1233,16 @@ static inline int mt76_wed_dma_setup(struct mt76_dev *dev, struct mt76_queue *q,
 #define mt76_dereference(p, dev) \
 	rcu_dereference_protected(p, lockdep_is_held(&(dev)->mutex))
 
+static inline struct mt76_wcid *
+__mt76_wcid_ptr(struct mt76_dev *dev, u16 idx)
+{
+	if (idx >= ARRAY_SIZE(dev->wcid))
+		return NULL;
+	return rcu_dereference(dev->wcid[idx]);
+}
+
+#define mt76_wcid_ptr(dev, idx) __mt76_wcid_ptr(&(dev)->mt76, idx)
+
 struct mt76_dev *mt76_alloc_device(struct device *pdev, unsigned int size,
 				   const struct ieee80211_ops *ops,
 				   const struct mt76_driver_ops *drv_ops);
@@ -1223,6 +1250,7 @@ int mt76_register_device(struct mt76_dev *dev, bool vht,
 			 struct ieee80211_rate *rates, int n_rates);
 void mt76_unregister_device(struct mt76_dev *dev);
 void mt76_free_device(struct mt76_dev *dev);
+void mt76_reset_device(struct mt76_dev *dev);
 void mt76_unregister_phy(struct mt76_phy *phy);
 
 struct mt76_phy *mt76_alloc_radio_phy(struct mt76_dev *dev, unsigned int size,
@@ -1232,6 +1260,8 @@ struct mt76_phy *mt76_alloc_phy(struct mt76_dev *dev, unsigned int size,
 				u8 band_idx);
 int mt76_register_phy(struct mt76_phy *phy, bool vht,
 		      struct ieee80211_rate *rates, int n_rates);
+struct mt76_phy *mt76_vif_phy(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif);
 
 struct dentry *mt76_register_debugfs_fops(struct mt76_phy *phy,
 					  const struct file_operations *ops);
@@ -1388,12 +1418,12 @@ static inline bool mt76_is_skb_pktid(u8 pktid)
 	return pktid >= MT_PACKET_ID_FIRST;
 }
 
-static inline u8 mt76_tx_power_nss_delta(u8 nss)
+static inline u8 mt76_tx_power_path_delta(u8 path)
 {
-	static const u8 nss_delta[4] = { 0, 6, 9, 12 };
-	u8 idx = nss - 1;
+	static const u8 path_delta[5] = { 0, 6, 9, 12, 14 };
+	u8 idx = path - 1;
 
-	return (idx < ARRAY_SIZE(nss_delta)) ? nss_delta[idx] : 0;
+	return (idx < ARRAY_SIZE(path_delta)) ? path_delta[idx] : 0;
 }
 
 static inline bool mt76_testmode_enabled(struct mt76_phy *phy)
@@ -1490,6 +1520,8 @@ void mt76_sta_pre_rcu_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 
 int mt76_get_min_avg_rssi(struct mt76_dev *dev, u8 phy_idx);
 
+s8 mt76_get_power_bound(struct mt76_phy *phy, s8 txpower);
+
 int mt76_get_txpower(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		     unsigned int link_id, int *dbm);
 int mt76_init_sar_power(struct ieee80211_hw *hw,
@@ -1501,7 +1533,8 @@ int mt76_get_sar_power(struct mt76_phy *phy,
 void mt76_csa_check(struct mt76_dev *dev);
 void mt76_csa_finish(struct mt76_dev *dev);
 
-int mt76_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant);
+int mt76_get_antenna(struct ieee80211_hw *hw, int radio_idx, u32 *tx_ant,
+		     u32 *rx_ant);
 int mt76_set_tim(struct ieee80211_hw *hw, struct ieee80211_sta *sta, bool set);
 void mt76_insert_ccmp_hdr(struct sk_buff *skb, u8 key_id);
 int mt76_get_rate(struct mt76_dev *dev,
@@ -1793,7 +1826,8 @@ static inline void mt76_put_page_pool_buf(void *buf, bool allow_direct)
 {
 	struct page *page = virt_to_head_page(buf);
 
-	page_pool_put_full_page(page->pp, page, allow_direct);
+	page_pool_put_full_page(pp_page_to_nmdesc(page)->pp, page,
+				allow_direct);
 }
 
 static inline void *
@@ -1864,6 +1898,9 @@ mt76_vif_link(struct mt76_dev *dev, struct ieee80211_vif *vif, int link_id)
 	struct mt76_vif_link *mlink = (struct mt76_vif_link *)vif->drv_priv;
 	struct mt76_vif_data *mvif = mlink->mvif;
 
+	if (!link_id)
+		return mlink;
+
 	return mt76_dereference(mvif->link[link_id], dev);
 }
 
@@ -1874,7 +1911,7 @@ mt76_vif_conf_link(struct mt76_dev *dev, struct ieee80211_vif *vif,
 	struct mt76_vif_link *mlink = (struct mt76_vif_link *)vif->drv_priv;
 	struct mt76_vif_data *mvif = mlink->mvif;
 
-	if (link_conf == &vif->bss_conf)
+	if (link_conf == &vif->bss_conf || !link_conf->link_id)
 		return mlink;
 
 	return mt76_dereference(mvif->link[link_conf->link_id], dev);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/dma.c b/sys/contrib/dev/mediatek/mt76/mt7603/dma.c
index 863e5770df51..e26cc78fff94 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7603/dma.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7603/dma.c
@@ -44,7 +44,7 @@ mt7603_rx_loopback_skb(struct mt7603_dev *dev, struct sk_buff *skb)
 	if (idx >= MT7603_WTBL_STA - 1)
 		goto free;
 
-	wcid = rcu_dereference(dev->mt76.wcid[idx]);
+	wcid = mt76_wcid_ptr(dev, idx);
 	if (!wcid)
 		goto free;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/mac.c b/sys/contrib/dev/mediatek/mt76/mt7603/mac.c
index 413973d05b43..6387f9e61060 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7603/mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7603/mac.c
@@ -487,10 +487,7 @@ mt7603_rx_get_wcid(struct mt7603_dev *dev, u8 idx, bool unicast)
 	struct mt7603_sta *sta;
 	struct mt76_wcid *wcid;
 
-	if (idx >= MT7603_WTBL_SIZE)
-		return NULL;
-
-	wcid = rcu_dereference(dev->mt76.wcid[idx]);
+	wcid = mt76_wcid_ptr(dev, idx);
 	if (unicast || !wcid)
 		return wcid;
 
@@ -1266,12 +1263,9 @@ void mt7603_mac_add_txs(struct mt7603_dev *dev, void *data)
 	if (pid == MT_PACKET_ID_NO_ACK)
 		return;
 
-	if (wcidx >= MT7603_WTBL_SIZE)
-		return;
-
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[wcidx]);
+	wcid = mt76_wcid_ptr(dev, wcidx);
 	if (!wcid)
 		goto out;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7603/main.c b/sys/contrib/dev/mediatek/mt76/mt7603/main.c
index 3e8b1ec76169..0d7c84941cd0 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7603/main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7603/main.c
@@ -216,7 +216,7 @@ static int mt7603_set_sar_specs(struct ieee80211_hw *hw,
 }
 
 static int
-mt7603_config(struct ieee80211_hw *hw, u32 changed)
+mt7603_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct mt7603_dev *dev = hw->priv;
 	int ret = 0;
@@ -657,7 +657,8 @@ mt7603_sta_rate_tbl_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 }
 
 static void
-mt7603_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
+mt7603_set_coverage_class(struct ieee80211_hw *hw, int radio_idx,
+			  s16 coverage_class)
 {
 	struct mt7603_dev *dev = hw->priv;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/init.c b/sys/contrib/dev/mediatek/mt76/mt7615/init.c
index 1e55e600981b..06d5a3f2fa67 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7615/init.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7615/init.c
@@ -275,7 +275,7 @@ void mt7615_init_txpower(struct mt7615_dev *dev,
 			 struct ieee80211_supported_band *sband)
 {
 	int i, n_chains = hweight8(dev->mphy.antenna_mask), target_chains;
-	int delta_idx, delta = mt76_tx_power_nss_delta(n_chains);
+	int delta_idx, delta = mt76_tx_power_path_delta(n_chains);
 	u8 *eep = (u8 *)dev->mt76.eeprom.data;
 	enum nl80211_band band = sband->band;
 	struct mt76_power_limits limits;
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mac.c b/sys/contrib/dev/mediatek/mt76/mt7615/mac.c
index 994f6f8ccd87..10bf7e5b3acb 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7615/mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7615/mac.c
@@ -93,10 +93,7 @@ static struct mt76_wcid *mt7615_rx_get_wcid(struct mt7615_dev *dev,
 	struct mt7615_sta *sta;
 	struct mt76_wcid *wcid;
 
-	if (idx >= MT7615_WTBL_SIZE)
-		return NULL;
-
-	wcid = rcu_dereference(dev->mt76.wcid[idx]);
+	wcid = mt76_wcid_ptr(dev, idx);
 	if (unicast || !wcid)
 		return wcid;
 
@@ -1507,7 +1504,7 @@ static void mt7615_mac_add_txs(struct mt7615_dev *dev, void *data)
 
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[wcidx]);
+	wcid = mt76_wcid_ptr(dev, wcidx);
 	if (!wcid)
 		goto out;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/main.c b/sys/contrib/dev/mediatek/mt76/mt7615/main.c
index 2e7b05eeef7a..15fe155ac3f3 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7615/main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7615/main.c
@@ -97,7 +97,7 @@ static void mt7615_stop(struct ieee80211_hw *hw, bool suspend)
 	struct mt7615_phy *phy = mt7615_hw_phy(hw);
 
 	cancel_delayed_work_sync(&phy->mt76->mac_work);
-	del_timer_sync(&phy->roc_timer);
+	timer_delete_sync(&phy->roc_timer);
 	cancel_work_sync(&phy->roc_work);
 
 	cancel_delayed_work_sync(&dev->pm.ps_work);
@@ -420,7 +420,7 @@ static int mt7615_set_sar_specs(struct ieee80211_hw *hw,
 	return mt76_update_channel(phy->mt76);
 }
 
-static int mt7615_config(struct ieee80211_hw *hw, u32 changed)
+static int mt7615_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct mt7615_dev *dev = mt7615_hw_dev(hw);
 	struct mt7615_phy *phy = mt7615_hw_phy(hw);
@@ -784,7 +784,8 @@ static void mt7615_tx(struct ieee80211_hw *hw,
 	mt76_connac_pm_queue_skb(hw, &dev->pm, wcid, skb);
 }
 
-static int mt7615_set_rts_threshold(struct ieee80211_hw *hw, u32 val)
+static int mt7615_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				    u32 val)
 {
 	struct mt7615_dev *dev = mt7615_hw_dev(hw);
 	struct mt7615_phy *phy = mt7615_hw_phy(hw);
@@ -972,7 +973,8 @@ mt7615_offset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 }
 
 static void
-mt7615_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
+mt7615_set_coverage_class(struct ieee80211_hw *hw, int radio_idx,
+			  s16 coverage_class)
 {
 	struct mt7615_phy *phy = mt7615_hw_phy(hw);
 	struct mt7615_dev *dev = phy->dev;
@@ -984,7 +986,8 @@ mt7615_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
 }
 
 static int
-mt7615_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
+mt7615_set_antenna(struct ieee80211_hw *hw, int radio_idx,
+		   u32 tx_ant, u32 rx_ant)
 {
 	struct mt7615_dev *dev = mt7615_hw_dev(hw);
 	struct mt7615_phy *phy = mt7615_hw_phy(hw);
@@ -1043,7 +1046,7 @@ void mt7615_roc_work(struct work_struct *work)
 
 void mt7615_roc_timer(struct timer_list *timer)
 {
-	struct mt7615_phy *phy = from_timer(phy, timer, roc_timer);
+	struct mt7615_phy *phy = timer_container_of(phy, timer, roc_timer);
 
 	ieee80211_queue_work(phy->mt76->hw, &phy->roc_work);
 }
@@ -1194,7 +1197,7 @@ static int mt7615_cancel_remain_on_channel(struct ieee80211_hw *hw,
 	if (!test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state))
 		return 0;
 
-	del_timer_sync(&phy->roc_timer);
+	timer_delete_sync(&phy->roc_timer);
 	cancel_work_sync(&phy->roc_work);
 
 	mt7615_mutex_acquire(phy->dev);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c
index ccc36ee0900c..ec2f759d407f 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7615/mcu.c
@@ -2071,7 +2071,7 @@ static void mt7615_mcu_set_txpower_sku(struct mt7615_phy *phy, u8 *sku)
 	};
 
 	tx_power = mt76_get_sar_power(mphy, mphy->chandef.chan, tx_power);
-	tx_power -= mt76_tx_power_nss_delta(n_chains);
+	tx_power -= mt76_tx_power_path_delta(n_chains);
 	tx_power = mt76_get_rate_power_limits(mphy, mphy->chandef.chan,
 					      &limits, tx_power);
 	mphy->txpower_cur = tx_power;
@@ -2088,8 +2088,8 @@ static void mt7615_mcu_set_txpower_sku(struct mt7615_phy *phy, u8 *sku)
 		int delta = 0;
 
 		if (i < n_chains - 1)
-			delta = mt76_tx_power_nss_delta(n_chains) -
-				mt76_tx_power_nss_delta(i + 1);
+			delta = mt76_tx_power_path_delta(n_chains) -
+				mt76_tx_power_path_delta(i + 1);
 		sku[MT_SKU_1SS_DELTA + i] = delta;
 	}
 }
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c b/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c
index 5da2bf332af0..fe8a3d852dbf 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7615/pci_mac.c
@@ -223,12 +223,12 @@ void mt7615_mac_reset_work(struct work_struct *work)
 	set_bit(MT76_MCU_RESET, &dev->mphy.state);
 	wake_up(&dev->mt76.mcu.wait);
 	cancel_delayed_work_sync(&dev->mphy.mac_work);
-	del_timer_sync(&dev->phy.roc_timer);
+	timer_delete_sync(&dev->phy.roc_timer);
 	cancel_work_sync(&dev->phy.roc_work);
 	if (phy2) {
 		set_bit(MT76_RESET, &phy2->mt76->state);
 		cancel_delayed_work_sync(&phy2->mt76->mac_work);
-		del_timer_sync(&phy2->roc_timer);
+		timer_delete_sync(&phy2->roc_timer);
 		cancel_work_sync(&phy2->roc_work);
 	}
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/sdio_mcu.c b/sys/contrib/dev/mediatek/mt76/mt7615/sdio_mcu.c
deleted file mode 100644
index a7b8acb2da83..000000000000
--- a/sys/contrib/dev/mediatek/mt76/mt7615/sdio_mcu.c
+++ /dev/null
@@ -1,180 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright (C) 2020 MediaTek Inc.
- *
- * Author: Felix Fietkau <nbd@nbd.name>
- *	   Lorenzo Bianconi <lorenzo@kernel.org>
- *	   Sean Wang <sean.wang@mediatek.com>
- */
-#include <linux/kernel.h>
-#include <linux/mmc/sdio_func.h>
-#include <linux/module.h>
-#include <linux/iopoll.h>
-
-#include "../sdio.h"
-#include "mt7615.h"
-#include "mac.h"
-#include "mcu.h"
-#include "regs.h"
-
-static int mt7663s_mcu_init_sched(struct mt7615_dev *dev)
-{
-	struct mt76_sdio *sdio = &dev->mt76.sdio;
-	u32 txdwcnt;
-
-	sdio->sched.pse_data_quota = mt76_get_field(dev, MT_PSE_PG_HIF0_GROUP,
-						    MT_HIF0_MIN_QUOTA);
-	sdio->sched.pse_mcu_quota = mt76_get_field(dev, MT_PSE_PG_HIF1_GROUP,
-						   MT_HIF1_MIN_QUOTA);
-	sdio->sched.ple_data_quota = mt76_get_field(dev, MT_PLE_PG_HIF0_GROUP,
-						    MT_HIF0_MIN_QUOTA);
-	sdio->sched.pse_page_size = MT_PSE_PAGE_SZ;
-	txdwcnt = mt76_get_field(dev, MT_PP_TXDWCNT,
-				 MT_PP_TXDWCNT_TX1_ADD_DW_CNT);
-	sdio->sched.deficit = txdwcnt << 2;
-
-	return 0;
-}
-
-static int
-mt7663s_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb,
-			 int cmd, int *seq)
-{
-	struct mt7615_dev *dev = container_of(mdev, struct mt7615_dev, mt76);
-	int ret;
-
-	mt7615_mcu_fill_msg(dev, skb, cmd, seq);
-	ret = mt76_tx_queue_skb_raw(dev, mdev->q_mcu[MT_MCUQ_WM], skb, 0);
-	if (ret)
-		return ret;
-
-	mt76_queue_kick(dev, mdev->q_mcu[MT_MCUQ_WM]);
-
-	return ret;
-}
-
-static int __mt7663s_mcu_drv_pmctrl(struct mt7615_dev *dev)
-{
-	struct sdio_func *func = dev->mt76.sdio.func;
-	struct mt76_phy *mphy = &dev->mt76.phy;
-	struct mt76_connac_pm *pm = &dev->pm;
-	u32 status;
-	int ret;
-
-	sdio_claim_host(func);
-
-	sdio_writel(func, WHLPCR_FW_OWN_REQ_CLR, MCR_WHLPCR, NULL);
-
-	ret = readx_poll_timeout(mt76s_read_pcr, &dev->mt76, status,
-				 status & WHLPCR_IS_DRIVER_OWN, 2000, 1000000);
-	if (ret < 0) {
-		dev_err(dev->mt76.dev, "Cannot get ownership from device");
-	} else {
-		clear_bit(MT76_STATE_PM, &mphy->state);
-
-		pm->stats.last_wake_event = jiffies;
-		pm->stats.doze_time += pm->stats.last_wake_event -
-				       pm->stats.last_doze_event;
-	}
-	sdio_release_host(func);
-
-	return ret;
-}
-
-static int mt7663s_mcu_drv_pmctrl(struct mt7615_dev *dev)
-{
-	struct mt76_phy *mphy = &dev->mt76.phy;
-	int ret = 0;
-
-	mutex_lock(&dev->pm.mutex);
-
-	if (test_bit(MT76_STATE_PM, &mphy->state))
-		ret = __mt7663s_mcu_drv_pmctrl(dev);
-
-	mutex_unlock(&dev->pm.mutex);
-
-	return ret;
-}
-
-static int mt7663s_mcu_fw_pmctrl(struct mt7615_dev *dev)
-{
-	struct sdio_func *func = dev->mt76.sdio.func;
-	struct mt76_phy *mphy = &dev->mt76.phy;
-	struct mt76_connac_pm *pm = &dev->pm;
-	int ret = 0;
-	u32 status;
-
-	mutex_lock(&pm->mutex);
-
-	if (mt76_connac_skip_fw_pmctrl(mphy, pm))
-		goto out;
-
-	sdio_claim_host(func);
-
-	sdio_writel(func, WHLPCR_FW_OWN_REQ_SET, MCR_WHLPCR, NULL);
-
-	ret = readx_poll_timeout(mt76s_read_pcr, &dev->mt76, status,
-				 !(status & WHLPCR_IS_DRIVER_OWN), 2000, 1000000);
-	if (ret < 0) {
-		dev_err(dev->mt76.dev, "Cannot set ownership to device");
-		clear_bit(MT76_STATE_PM, &mphy->state);
-	} else {
-		pm->stats.last_doze_event = jiffies;
-		pm->stats.awake_time += pm->stats.last_doze_event -
-					pm->stats.last_wake_event;
-	}
-
-	sdio_release_host(func);
-out:
-	mutex_unlock(&pm->mutex);
-
-	return ret;
-}
-
-int mt7663s_mcu_init(struct mt7615_dev *dev)
-{
-	static const struct mt76_mcu_ops mt7663s_mcu_ops = {
-		.headroom = sizeof(struct mt7615_mcu_txd),
-		.tailroom = MT_USB_TAIL_SIZE,
-		.mcu_skb_send_msg = mt7663s_mcu_send_message,
-		.mcu_parse_response = mt7615_mcu_parse_response,
-		.mcu_rr = mt76_connac_mcu_reg_rr,
-		.mcu_wr = mt76_connac_mcu_reg_wr,
-	};
-	struct mt7615_mcu_ops *mcu_ops;
-	int ret;
-
-	ret = __mt7663s_mcu_drv_pmctrl(dev);
-	if (ret)
-		return ret;
-
-	dev->mt76.mcu_ops = &mt7663s_mcu_ops;
-
-	ret = mt76_get_field(dev, MT_CONN_ON_MISC, MT_TOP_MISC2_FW_N9_RDY);
-	if (ret) {
-		mt7615_mcu_restart(&dev->mt76);
-		if (!mt76_poll_msec(dev, MT_CONN_ON_MISC,
-				    MT_TOP_MISC2_FW_N9_RDY, 0, 500))
-			return -EIO;
-	}
-
-	ret = __mt7663_load_firmware(dev);
-	if (ret)
-		return ret;
-
-	mcu_ops = devm_kmemdup(dev->mt76.dev, dev->mcu_ops, sizeof(*mcu_ops),
-			       GFP_KERNEL);
-	if (!mcu_ops)
-		return -ENOMEM;
-
-	mcu_ops->set_drv_ctrl = mt7663s_mcu_drv_pmctrl;
-	mcu_ops->set_fw_ctrl = mt7663s_mcu_fw_pmctrl;
-	dev->mcu_ops = mcu_ops;
-
-	ret = mt7663s_mcu_init_sched(dev);
-	if (ret)
-		return ret;
-
-	set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state);
-
-	return 0;
-}
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/usb.c b/sys/contrib/dev/mediatek/mt76/mt7615/usb.c
index 4aa9fa1c4a23..d96e06b4fee1 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7615/usb.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7615/usb.c
@@ -85,7 +85,7 @@ static void mt7663u_stop(struct ieee80211_hw *hw, bool suspend)
 	struct mt7615_dev *dev = hw->priv;
 
 	clear_bit(MT76_STATE_RUNNING, &dev->mphy.state);
-	del_timer_sync(&phy->roc_timer);
+	timer_delete_sync(&phy->roc_timer);
 	cancel_work_sync(&phy->roc_work);
 	cancel_delayed_work_sync(&phy->scan_work);
 	cancel_delayed_work_sync(&phy->mt76->mac_work);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7615/usb_mcu.c b/sys/contrib/dev/mediatek/mt76/mt7615/usb_mcu.c
deleted file mode 100644
index 33c01f8ce8e2..000000000000
--- a/sys/contrib/dev/mediatek/mt76/mt7615/usb_mcu.c
+++ /dev/null
@@ -1,100 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright (C) 2019 MediaTek Inc.
- *
- * Author: Felix Fietkau <nbd@nbd.name>
- *	   Lorenzo Bianconi <lorenzo@kernel.org>
- *	   Sean Wang <sean.wang@mediatek.com>
- */
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-#include "mt7615.h"
-#include "mac.h"
-#include "mcu.h"
-#include "regs.h"
-
-static int
-mt7663u_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb,
-			 int cmd, int *seq)
-{
-	struct mt7615_dev *dev = container_of(mdev, struct mt7615_dev, mt76);
-	int ret, ep, len, pad;
-
-	mt7615_mcu_fill_msg(dev, skb, cmd, seq);
-	if (cmd != MCU_CMD(FW_SCATTER))
-		ep = MT_EP_OUT_INBAND_CMD;
-	else
-		ep = MT_EP_OUT_AC_BE;
-
-	len = skb->len;
-	put_unaligned_le32(len, skb_push(skb, sizeof(len)));
-	pad = round_up(skb->len, 4) + 4 - skb->len;
-	ret = mt76_skb_adjust_pad(skb, pad);
-	if (ret < 0)
-		goto out;
-
-	ret = mt76u_bulk_msg(&dev->mt76, skb->data, skb->len, NULL,
-			     1000, ep);
-
-out:
-	dev_kfree_skb(skb);
-
-	return ret;
-}
-
-int mt7663u_mcu_power_on(struct mt7615_dev *dev)
-{
-	int ret;
-
-	ret = mt76u_vendor_request(&dev->mt76, MT_VEND_POWER_ON,
-				   USB_DIR_OUT | USB_TYPE_VENDOR,
-				   0x0, 0x1, NULL, 0);
-	if (ret)
-		return ret;
-
-	if (!mt76_poll_msec(dev, MT_CONN_ON_MISC,
-			    MT_TOP_MISC2_FW_PWR_ON,
-			    FW_STATE_PWR_ON << 1, 500)) {
-		dev_err(dev->mt76.dev, "Timeout for power on\n");
-		ret = -EIO;
-	}
-
-	return 0;
-}
-
-int mt7663u_mcu_init(struct mt7615_dev *dev)
-{
-	static const struct mt76_mcu_ops mt7663u_mcu_ops = {
-		.headroom = MT_USB_HDR_SIZE + sizeof(struct mt7615_mcu_txd),
-		.tailroom = MT_USB_TAIL_SIZE,
-		.mcu_skb_send_msg = mt7663u_mcu_send_message,
-		.mcu_parse_response = mt7615_mcu_parse_response,
-	};
-	int ret;
-
-	dev->mt76.mcu_ops = &mt7663u_mcu_ops;
-
-	mt76_set(dev, MT_UDMA_TX_QSEL, MT_FW_DL_EN);
-	if (test_and_clear_bit(MT76_STATE_POWER_OFF, &dev->mphy.state)) {
-		ret = mt7615_mcu_restart(&dev->mt76);
-		if (ret)
-			return ret;
-
-		if (!mt76_poll_msec(dev, MT_CONN_ON_MISC,
-				    MT_TOP_MISC2_FW_PWR_ON, 0, 500))
-			return -EIO;
-
-		ret = mt7663u_mcu_power_on(dev);
-		if (ret)
-			return ret;
-	}
-
-	ret = __mt7663_load_firmware(dev);
-	if (ret)
-		return ret;
-
-	mt76_clear(dev, MT_UDMA_TX_QSEL, MT_FW_DL_EN);
-	set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state);
-
-	return 0;
-}
diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac.h b/sys/contrib/dev/mediatek/mt76/mt76_connac.h
index 455979476d11..192dcc374a64 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76_connac.h
+++ b/sys/contrib/dev/mediatek/mt76/mt76_connac.h
@@ -232,9 +232,14 @@ static inline bool is_mt7992(struct mt76_dev *dev)
 	return mt76_chip(dev) == 0x7992;
 }
 
+static inline bool is_mt7990(struct mt76_dev *dev)
+{
+	return mt76_chip(dev) == 0x7993;
+}
+
 static inline bool is_mt799x(struct mt76_dev *dev)
 {
-	return is_mt7996(dev) || is_mt7992(dev);
+	return is_mt7996(dev) || is_mt7992(dev) || is_mt7990(dev);
 }
 
 static inline bool is_mt7622(struct mt76_dev *dev)
diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h b/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h
index db0c29e65185..1013cad57a7f 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h
+++ b/sys/contrib/dev/mediatek/mt76/mt76_connac3_mac.h
@@ -273,6 +273,7 @@ enum tx_frag_idx {
 #define MT_TXD6_TX_RATE			GENMASK(21, 16)
 #define MT_TXD6_TIMESTAMP_OFS_EN	BIT(15)
 #define MT_TXD6_TIMESTAMP_OFS_IDX	GENMASK(14, 10)
+#define MT_TXD6_TID_ADDBA		GENMASK(10, 8)
 #define MT_TXD6_MSDU_CNT		GENMASK(9, 4)
 #define MT_TXD6_MSDU_CNT_V2		GENMASK(15, 10)
 #define MT_TXD6_DIS_MAT			BIT(3)
@@ -314,6 +315,9 @@ enum tx_frag_idx {
 #define MT_TXFREE_INFO_COUNT		GENMASK(27, 24)
 #define MT_TXFREE_INFO_STAT		GENMASK(29, 28)
 
+#define MT_TXS_HDR_SIZE			4 /* Unit: DW */
+#define MT_TXS_SIZE			12 /* Unit: DW */
+
 #define MT_TXS0_BW			GENMASK(31, 29)
 #define MT_TXS0_TID			GENMASK(28, 26)
 #define MT_TXS0_AMPDU			BIT(25)
diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c b/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c
index e9ac8a7317a1..0db00efe88b0 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76_connac_mac.c
@@ -1172,7 +1172,7 @@ void mt76_connac2_txwi_free(struct mt76_dev *dev, struct mt76_txwi_cache *t,
 		wcid_idx = wcid->idx;
 	} else {
 		wcid_idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX);
-		wcid = rcu_dereference(dev->wcid[wcid_idx]);
+		wcid = __mt76_wcid_ptr(dev, wcid_idx);
 
 		if (wcid && wcid->sta) {
 			sta = container_of((void *)wcid, struct ieee80211_sta,
diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c
index 85a8573f2822..ca0d022668e9 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.c
@@ -67,8 +67,7 @@ int mt76_connac_mcu_init_download(struct mt76_dev *dev, u32 addr, u32 len,
 	if ((!is_connac_v1(dev) && addr == MCU_PATCH_ADDRESS) ||
 	    (is_mt7921(dev) && addr == 0x900000) ||
 	    (is_mt7925(dev) && (addr == 0x900000 || addr == 0xe0002800)) ||
-	    (is_mt7996(dev) && addr == 0x900000) ||
-	    (is_mt7992(dev) && addr == 0x900000))
+	    (is_mt799x(dev) && addr == 0x900000))
 		cmd = MCU_CMD(PATCH_START_REQ);
 	else
 		cmd = MCU_CMD(TARGET_ADDRESS_LEN_REQ);
@@ -288,7 +287,7 @@ __mt76_connac_mcu_alloc_sta_req(struct mt76_dev *dev, struct mt76_vif_link *mvif
 
 	mt76_connac_mcu_get_wlan_idx(dev, wcid, &hdr.wlan_idx_lo,
 				     &hdr.wlan_idx_hi);
-	skb = mt76_mcu_msg_alloc(dev, NULL, len);
+	skb = __mt76_mcu_msg_alloc(dev, NULL, len, len, GFP_ATOMIC);
 	if (!skb)
 		return ERR_PTR(-ENOMEM);
 
@@ -1168,7 +1167,7 @@ int mt76_connac_mcu_uni_add_dev(struct mt76_phy *phy,
 			.tag = cpu_to_le16(DEV_INFO_ACTIVE),
 			.len = cpu_to_le16(sizeof(struct req_tlv)),
 			.active = enable,
-			.link_idx = mvif->idx,
+			.link_idx = mvif->link_idx,
 		},
 	};
 	struct {
@@ -1191,7 +1190,7 @@ int mt76_connac_mcu_uni_add_dev(struct mt76_phy *phy,
 			.bmc_tx_wlan_idx = cpu_to_le16(wcid->idx),
 			.sta_idx = cpu_to_le16(wcid->idx),
 			.conn_state = 1,
-			.link_idx = mvif->idx,
+			.link_idx = mvif->link_idx,
 		},
 	};
 	int err, idx, cmd, len;
@@ -1667,6 +1666,44 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy,
 }
 EXPORT_SYMBOL_GPL(mt76_connac_mcu_uni_add_bss);
 
+void mt76_connac_mcu_build_rnr_scan_param(struct mt76_dev *mdev,
+					  struct cfg80211_scan_request *sreq)
+{
+	struct ieee80211_channel **scan_list = sreq->channels;
+	int i, bssid_index = 0;
+
+	/* clear 6G active Scan BSSID table */
+	memset(&mdev->rnr, 0, sizeof(mdev->rnr));
+
+	for (i = 0; i < sreq->n_6ghz_params; i++) {
+		u8 ch_idx = sreq->scan_6ghz_params[i].channel_idx;
+		int k = 0;
+
+		/* Remove the duplicated BSSID */
+		for (k = 0; k < bssid_index; k++) {
+			if (!memcmp(&mdev->rnr.bssid[k],
+				    sreq->scan_6ghz_params[i].bssid,
+				    ETH_ALEN))
+				break;
+		}
+
+		if (k == bssid_index &&
+		    bssid_index < MT76_RNR_SCAN_MAX_BSSIDS) {
+			memcpy(&mdev->rnr.bssid[bssid_index++],
+			       sreq->scan_6ghz_params[i].bssid, ETH_ALEN);
+			mdev->rnr.channel[k] = scan_list[ch_idx]->hw_value;
+		}
+	}
+
+	mdev->rnr.bssid_num = bssid_index;
+
+	if (sreq->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) {
+		memcpy(mdev->rnr.random_mac, sreq->mac_addr, ETH_ALEN);
+		mdev->rnr.sreq_flag = sreq->flags;
+	}
+}
+EXPORT_SYMBOL_GPL(mt76_connac_mcu_build_rnr_scan_param);
+
 #define MT76_CONNAC_SCAN_CHANNEL_TIME		60
 int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 			    struct ieee80211_scan_request *scan_req)
@@ -1703,8 +1740,8 @@ int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 		if (!sreq->ssids[i].ssid_len)
 			continue;
 
-		req->ssids[i].ssid_len = cpu_to_le32(sreq->ssids[i].ssid_len);
-		memcpy(req->ssids[i].ssid, sreq->ssids[i].ssid,
+		req->ssids[n_ssids].ssid_len = cpu_to_le32(sreq->ssids[i].ssid_len);
+		memcpy(req->ssids[n_ssids].ssid, sreq->ssids[i].ssid,
 		       sreq->ssids[i].ssid_len);
 		n_ssids++;
 	}
diff --git a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h
index 18db7bb31730..3ba81ab77e99 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h
+++ b/sys/contrib/dev/mediatek/mt76/mt76_connac_mcu.h
@@ -873,6 +873,7 @@ enum {
 #define NETWORK_WDS			BIT(21)
 
 #define SCAN_FUNC_RANDOM_MAC		BIT(0)
+#define SCAN_FUNC_RNR_SCAN		BIT(3)
 #define SCAN_FUNC_SPLIT_SCAN		BIT(5)
 
 #define CONNECTION_INFRA_STA		(STA_TYPE_STA | NETWORK_INFRA)
@@ -1069,6 +1070,7 @@ enum {
 	MCU_UNI_EVENT_WED_RRO = 0x57,
 	MCU_UNI_EVENT_PER_STA_INFO = 0x6d,
 	MCU_UNI_EVENT_ALL_STA_INFO = 0x6e,
+	MCU_UNI_EVENT_SDO = 0x83,
 };
 
 #define MCU_UNI_CMD_EVENT			BIT(1)
@@ -1186,6 +1188,11 @@ enum {
 #define MCU_UNI_CMD(_t)				(__MCU_CMD_FIELD_UNI |			\
 						 FIELD_PREP(__MCU_CMD_FIELD_ID,		\
 							    MCU_UNI_CMD_##_t))
+
+#define MCU_UNI_QUERY(_t)			(__MCU_CMD_FIELD_UNI | __MCU_CMD_FIELD_QUERY | \
+						 FIELD_PREP(__MCU_CMD_FIELD_ID,		\
+							    MCU_UNI_CMD_##_t))
+
 #define MCU_CE_CMD(_t)				(__MCU_CMD_FIELD_CE |			\
 						 FIELD_PREP(__MCU_CMD_FIELD_ID,		\
 							   MCU_CE_CMD_##_t))
@@ -1291,16 +1298,20 @@ enum {
 	MCU_UNI_CMD_EFUSE_CTRL = 0x2d,
 	MCU_UNI_CMD_RA = 0x2f,
 	MCU_UNI_CMD_MURU = 0x31,
+	MCU_UNI_CMD_TESTMODE_RX_STAT = 0x32,
 	MCU_UNI_CMD_BF = 0x33,
 	MCU_UNI_CMD_CHANNEL_SWITCH = 0x34,
 	MCU_UNI_CMD_THERMAL = 0x35,
 	MCU_UNI_CMD_VOW = 0x37,
 	MCU_UNI_CMD_FIXED_RATE_TABLE = 0x40,
+	MCU_UNI_CMD_TESTMODE_CTRL = 0x46,
 	MCU_UNI_CMD_RRO = 0x57,
 	MCU_UNI_CMD_OFFCH_SCAN_CTRL = 0x58,
 	MCU_UNI_CMD_PER_STA_INFO = 0x6d,
 	MCU_UNI_CMD_ALL_STA_INFO = 0x6e,
 	MCU_UNI_CMD_ASSERT_DUMP = 0x6f,
+	MCU_UNI_CMD_RADIO_STATUS = 0x80,
+	MCU_UNI_CMD_SDO = 0x88,
 };
 
 enum {
@@ -1373,6 +1384,7 @@ enum {
 	UNI_BSS_INFO_OFFLOAD = 25,
 	UNI_BSS_INFO_MLD = 26,
 	UNI_BSS_INFO_PM_DISABLE = 27,
+	UNI_BSS_INFO_EHT = 30,
 };
 
 enum {
@@ -1973,6 +1985,8 @@ int mt76_connac_mcu_start_patch(struct mt76_dev *dev);
 int mt76_connac_mcu_patch_sem_ctrl(struct mt76_dev *dev, bool get);
 int mt76_connac_mcu_start_firmware(struct mt76_dev *dev, u32 addr, u32 option);
 
+void mt76_connac_mcu_build_rnr_scan_param(struct mt76_dev *mdev,
+					  struct cfg80211_scan_request *sreq);
 int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 			    struct ieee80211_scan_request *scan_req);
 int mt76_connac_mcu_cancel_hw_scan(struct mt76_phy *phy,
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c b/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c
index b456ccd00d58..11c16d1fc70f 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x0/pci.c
@@ -156,7 +156,8 @@ mt76x0e_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	static const struct mt76_driver_ops drv_ops = {
 		.txwi_size = sizeof(struct mt76x02_txwi),
 		.drv_flags = MT_DRV_TX_ALIGNED4_SKBS |
-			     MT_DRV_SW_RX_AIRTIME,
+			     MT_DRV_SW_RX_AIRTIME |
+			     MT_DRV_IGNORE_TXS_FAILED,
 		.survey_flags = SURVEY_INFO_TIME_TX,
 		.update_survey = mt76x02_update_channel,
 		.set_channel = mt76x0_set_channel,
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02.h b/sys/contrib/dev/mediatek/mt76/mt76x02.h
index 4cd63bacd742..8d06ef8c7c62 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x02.h
+++ b/sys/contrib/dev/mediatek/mt76/mt76x02.h
@@ -183,8 +183,8 @@ void mt76x02_wdt_work(struct work_struct *work);
 void mt76x02_tx_set_txpwr_auto(struct mt76x02_dev *dev, s8 txpwr);
 void mt76x02_set_tx_ackto(struct mt76x02_dev *dev);
 void mt76x02_set_coverage_class(struct ieee80211_hw *hw,
-				s16 coverage_class);
-int mt76x02_set_rts_threshold(struct ieee80211_hw *hw, u32 val);
+				int radio_idx, s16 coverage_class);
+int mt76x02_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx, u32 val);
 void mt76x02_remove_hdr_pad(struct sk_buff *skb, int len);
 bool mt76x02_tx_status_data(struct mt76_dev *mdev, u8 *update);
 void mt76x02_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
@@ -262,10 +262,7 @@ mt76x02_rx_get_sta(struct mt76_dev *dev, u8 idx)
 {
 	struct mt76_wcid *wcid;
 
-	if (idx >= MT76x02_N_WCIDS)
-		return NULL;
-
-	wcid = rcu_dereference(dev->wcid[idx]);
+	wcid = __mt76_wcid_ptr(dev, idx);
 	if (!wcid)
 		return NULL;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c b/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c
index d5db6ffd6d36..83488b2d6efb 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x02_mac.c
@@ -564,9 +564,7 @@ void mt76x02_send_tx_status(struct mt76x02_dev *dev,
 
 	rcu_read_lock();
 
-	if (stat->wcid < MT76x02_N_WCIDS)
-		wcid = rcu_dereference(dev->mt76.wcid[stat->wcid]);
-
+	wcid = mt76_wcid_ptr(dev, stat->wcid);
 	if (wcid && wcid->sta) {
 		void *priv;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c b/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c
index a82c75ba26e6..a683d53c7ceb 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x02_mmio.c
@@ -174,7 +174,6 @@ static int mt76x02_poll_tx(struct napi_struct *napi, int budget)
 
 int mt76x02_dma_init(struct mt76x02_dev *dev)
 {
-	struct mt76_txwi_cache __maybe_unused *t;
 	int i, ret, fifo_size;
 	struct mt76_queue *q;
 	void *status_fifo;
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c b/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c
index 0e1ede9314d8..4840d0b500b3 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x02_usb_core.c
@@ -264,8 +264,8 @@ void mt76x02u_init_beacon_config(struct mt76x02_dev *dev)
 	};
 	dev->beacon_ops = &beacon_ops;
 
-	hrtimer_init(&dev->pre_tbtt_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-	dev->pre_tbtt_timer.function = mt76x02u_pre_tbtt_interrupt;
+	hrtimer_setup(&dev->pre_tbtt_timer, mt76x02u_pre_tbtt_interrupt, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL);
 	INIT_WORK(&dev->pre_tbtt_work, mt76x02u_pre_tbtt_work);
 
 	mt76x02_init_beacon_config(dev);
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x02_util.c b/sys/contrib/dev/mediatek/mt76/mt76x02_util.c
index 4fb30589fa7a..7dfcb20c692c 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x02_util.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x02_util.c
@@ -548,7 +548,7 @@ void mt76x02_set_tx_ackto(struct mt76x02_dev *dev)
 EXPORT_SYMBOL_GPL(mt76x02_set_tx_ackto);
 
 void mt76x02_set_coverage_class(struct ieee80211_hw *hw,
-				s16 coverage_class)
+				int radio_idx, s16 coverage_class)
 {
 	struct mt76x02_dev *dev = hw->priv;
 
@@ -559,7 +559,7 @@ void mt76x02_set_coverage_class(struct ieee80211_hw *hw,
 }
 EXPORT_SYMBOL_GPL(mt76x02_set_coverage_class);
 
-int mt76x02_set_rts_threshold(struct ieee80211_hw *hw, u32 val)
+int mt76x02_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx, u32 val)
 {
 	struct mt76x02_dev *dev = hw->priv;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c b/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c
index 727bfdd00b40..2303019670e2 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x2/pci.c
@@ -22,7 +22,8 @@ mt76x2e_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	static const struct mt76_driver_ops drv_ops = {
 		.txwi_size = sizeof(struct mt76x02_txwi),
 		.drv_flags = MT_DRV_TX_ALIGNED4_SKBS |
-			     MT_DRV_SW_RX_AIRTIME,
+			     MT_DRV_SW_RX_AIRTIME |
+			     MT_DRV_IGNORE_TXS_FAILED,
 		.survey_flags = SURVEY_INFO_TIME_TX,
 		.update_survey = mt76x02_update_channel,
 		.set_channel = mt76x2e_set_channel,
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c
index eb70130d2711..c5dfb06d81e8 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x2/pci_main.c
@@ -54,7 +54,7 @@ int mt76x2e_set_channel(struct mt76_phy *phy)
 }
 
 static int
-mt76x2_config(struct ieee80211_hw *hw, u32 changed)
+mt76x2_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct mt76x02_dev *dev = hw->priv;
 
@@ -99,8 +99,8 @@ mt76x2_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 {
 }
 
-static int mt76x2_set_antenna(struct ieee80211_hw *hw, u32 tx_ant,
-			      u32 rx_ant)
+static int mt76x2_set_antenna(struct ieee80211_hw *hw, int radio_idx,
+			      u32 tx_ant, u32 rx_ant)
 {
 	struct mt76x02_dev *dev = hw->priv;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c
index e832ad53e239..96cecc576a98 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb.c
@@ -17,11 +17,14 @@ static const struct usb_device_id mt76x2u_device_table[] = {
 	{ USB_DEVICE(0x057c, 0x8503) },	/* Avm FRITZ!WLAN AC860 */
 	{ USB_DEVICE(0x7392, 0xb711) },	/* Edimax EW 7722 UAC */
 	{ USB_DEVICE(0x0e8d, 0x7632) },	/* HC-M7662BU1 */
+	{ USB_DEVICE(0x0471, 0x2126) }, /* LiteOn WN4516R module, nonstandard USB connector */
+	{ USB_DEVICE(0x0471, 0x7600) }, /* LiteOn WN4519R module, nonstandard USB connector */
 	{ USB_DEVICE(0x2c4e, 0x0103) },	/* Mercury UD13 */
 	{ USB_DEVICE(0x0846, 0x9014) },	/* Netgear WNDA3100v3 */
 	{ USB_DEVICE(0x0846, 0x9053) },	/* Netgear A6210 */
 	{ USB_DEVICE(0x045e, 0x02e6) },	/* XBox One Wireless Adapter */
 	{ USB_DEVICE(0x045e, 0x02fe) },	/* XBox One Wireless Adapter */
+	{ USB_DEVICE(0x2357, 0x0137) },	/* TP-Link TL-WDN6200 */
 	{ },
 };
 
@@ -29,7 +32,8 @@ static int mt76x2u_probe(struct usb_interface *intf,
 			 const struct usb_device_id *id)
 {
 	static const struct mt76_driver_ops drv_ops = {
-		.drv_flags = MT_DRV_SW_RX_AIRTIME,
+		.drv_flags = MT_DRV_SW_RX_AIRTIME |
+			     MT_DRV_IGNORE_TXS_FAILED,
 		.survey_flags = SURVEY_INFO_TIME_TX,
 		.update_survey = mt76x02_update_channel,
 		.set_channel = mt76x2u_set_channel,
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c
index 33a14365ec9b..3b5562811511 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_init.c
@@ -191,6 +191,7 @@ int mt76x2u_register_device(struct mt76x02_dev *dev)
 {
 	struct ieee80211_hw *hw = mt76_hw(dev);
 	struct mt76_usb *usb = &dev->mt76.usb;
+	bool vht;
 	int err;
 
 	INIT_DELAYED_WORK(&dev->cal_work, mt76x2u_phy_calibrate);
@@ -217,7 +218,17 @@ int mt76x2u_register_device(struct mt76x02_dev *dev)
 
 	/* check hw sg support in order to enable AMSDU */
 	hw->max_tx_fragments = dev->mt76.usb.sg_en ? MT_TX_SG_MAX_SIZE : 1;
-	err = mt76_register_device(&dev->mt76, true, mt76x02_rates,
+	switch (dev->mt76.rev) {
+	case 0x76320044:
+		/* these ASIC revisions do not support VHT */
+		vht = false;
+		break;
+	default:
+		vht = true;
+		break;
+	}
+
+	err = mt76_register_device(&dev->mt76, vht, mt76x02_rates,
 				   ARRAY_SIZE(mt76x02_rates));
 	if (err)
 		goto fail;
diff --git a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c
index 83e7061b10e2..6671c53faf9f 100644
--- a/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt76x2/usb_main.c
@@ -50,7 +50,7 @@ int mt76x2u_set_channel(struct mt76_phy *mphy)
 }
 
 static int
-mt76x2u_config(struct ieee80211_hw *hw, u32 changed)
+mt76x2u_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct mt76x02_dev *dev = hw->priv;
 	int err = 0;
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c
index 578013884e43..b287b7d9394e 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/debugfs.c
@@ -211,13 +211,28 @@ static const struct file_operations mt7915_sys_recovery_ops = {
 static int
 mt7915_radar_trigger(void *data, u64 val)
 {
-	struct mt7915_dev *dev = data;
+#define RADAR_MAIN_CHAIN	1
+#define RADAR_BACKGROUND	2
+	struct mt7915_phy *phy = data;
+	struct mt7915_dev *dev = phy->dev;
+	int rdd_idx;
+
+	if (!val || val > RADAR_BACKGROUND)
+		return -EINVAL;
 
-	if (val > MT_RX_SEL2)
+	if (val == RADAR_BACKGROUND && !dev->rdd2_phy) {
+		dev_err(dev->mt76.dev, "Background radar is not enabled\n");
 		return -EINVAL;
+	}
+
+	rdd_idx = mt7915_get_rdd_idx(phy, val == RADAR_BACKGROUND);
+	if (rdd_idx < 0) {
+		dev_err(dev->mt76.dev, "No RDD found\n");
+		return -EINVAL;
+	}
 
 	return mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_RADAR_EMULATE,
-				       val, 0, 0);
+				       rdd_idx, 0, 0);
 }
 
 DEFINE_DEBUGFS_ATTRIBUTE(fops_radar_trigger, NULL,
@@ -303,9 +318,9 @@ static int mt7915_muru_stats_show(struct seq_file *file, void *data)
 		   phy->mib.dl_vht_3mu_cnt,
 		   phy->mib.dl_vht_4mu_cnt);
 
-	sub_total_cnt = phy->mib.dl_vht_2mu_cnt +
-			phy->mib.dl_vht_3mu_cnt +
-			phy->mib.dl_vht_4mu_cnt;
+	sub_total_cnt = (u64)phy->mib.dl_vht_2mu_cnt +
+			     phy->mib.dl_vht_3mu_cnt +
+			     phy->mib.dl_vht_4mu_cnt;
 
 	seq_printf(file, "\nTotal non-HE MU-MIMO DL PPDU count: %lld",
 		   sub_total_cnt);
@@ -353,26 +368,27 @@ static int mt7915_muru_stats_show(struct seq_file *file, void *data)
 		   phy->mib.dl_he_9to16ru_cnt,
 		   phy->mib.dl_he_gtr16ru_cnt);
 
-	sub_total_cnt = phy->mib.dl_he_2mu_cnt +
-			phy->mib.dl_he_3mu_cnt +
-			phy->mib.dl_he_4mu_cnt;
+	sub_total_cnt = (u64)phy->mib.dl_he_2mu_cnt +
+			     phy->mib.dl_he_3mu_cnt +
+			     phy->mib.dl_he_4mu_cnt;
 	total_ppdu_cnt = sub_total_cnt;
 
 	seq_printf(file, "\nTotal HE MU-MIMO DL PPDU count: %lld",
 		   sub_total_cnt);
 
-	sub_total_cnt = phy->mib.dl_he_2ru_cnt +
-			phy->mib.dl_he_3ru_cnt +
-			phy->mib.dl_he_4ru_cnt +
-			phy->mib.dl_he_5to8ru_cnt +
-			phy->mib.dl_he_9to16ru_cnt +
-			phy->mib.dl_he_gtr16ru_cnt;
+	sub_total_cnt = (u64)phy->mib.dl_he_2ru_cnt +
+			     phy->mib.dl_he_3ru_cnt +
+			     phy->mib.dl_he_4ru_cnt +
+			     phy->mib.dl_he_5to8ru_cnt +
+			     phy->mib.dl_he_9to16ru_cnt +
+			     phy->mib.dl_he_gtr16ru_cnt;
 	total_ppdu_cnt += sub_total_cnt;
 
 	seq_printf(file, "\nTotal HE OFDMA DL PPDU count: %lld",
 		   sub_total_cnt);
 
-	total_ppdu_cnt += phy->mib.dl_he_su_cnt + phy->mib.dl_he_ext_su_cnt;
+	total_ppdu_cnt += (u64)phy->mib.dl_he_su_cnt +
+			       phy->mib.dl_he_ext_su_cnt;
 
 	seq_printf(file, "\nAll HE DL PPDU count: %lld", total_ppdu_cnt);
 
@@ -404,20 +420,20 @@ static int mt7915_muru_stats_show(struct seq_file *file, void *data)
 		   phy->mib.ul_hetrig_9to16ru_cnt,
 		   phy->mib.ul_hetrig_gtr16ru_cnt);
 
-	sub_total_cnt = phy->mib.ul_hetrig_2mu_cnt +
-			phy->mib.ul_hetrig_3mu_cnt +
-			phy->mib.ul_hetrig_4mu_cnt;
+	sub_total_cnt = (u64)phy->mib.ul_hetrig_2mu_cnt +
+			     phy->mib.ul_hetrig_3mu_cnt +
+			     phy->mib.ul_hetrig_4mu_cnt;
 	total_ppdu_cnt = sub_total_cnt;
 
 	seq_printf(file, "\nTotal HE MU-MIMO UL TB PPDU count: %lld",
 		   sub_total_cnt);
 
-	sub_total_cnt = phy->mib.ul_hetrig_2ru_cnt +
-			phy->mib.ul_hetrig_3ru_cnt +
-			phy->mib.ul_hetrig_4ru_cnt +
-			phy->mib.ul_hetrig_5to8ru_cnt +
-			phy->mib.ul_hetrig_9to16ru_cnt +
-			phy->mib.ul_hetrig_gtr16ru_cnt;
+	sub_total_cnt = (u64)phy->mib.ul_hetrig_2ru_cnt +
+			     phy->mib.ul_hetrig_3ru_cnt +
+			     phy->mib.ul_hetrig_4ru_cnt +
+			     phy->mib.ul_hetrig_5to8ru_cnt +
+			     phy->mib.ul_hetrig_9to16ru_cnt +
+			     phy->mib.ul_hetrig_gtr16ru_cnt;
 	total_ppdu_cnt += sub_total_cnt;
 
 	seq_printf(file, "\nTotal HE OFDMA UL TB PPDU count: %lld",
@@ -444,6 +460,11 @@ mt7915_rdd_monitor(struct seq_file *s, void *data)
 
 	mutex_lock(&dev->mt76.mutex);
 
+	if (!mt7915_eeprom_has_background_radar(dev)) {
+		seq_puts(s, "no background radar capability\n");
+		goto out;
+	}
+
 	if (!cfg80211_chandef_valid(chandef)) {
 		ret = -EINVAL;
 		goto out;
@@ -1084,13 +1105,13 @@ mt7915_rate_txpower_set(struct file *file, const char __user *user_buf,
 		return -EINVAL;
 
 	if (pwr160)
-		pwr160 = mt7915_get_power_bound(phy, pwr160);
+		pwr160 = mt76_get_power_bound(mphy, pwr160);
 	if (pwr80)
-		pwr80 = mt7915_get_power_bound(phy, pwr80);
+		pwr80 = mt76_get_power_bound(mphy, pwr80);
 	if (pwr40)
-		pwr40 = mt7915_get_power_bound(phy, pwr40);
+		pwr40 = mt76_get_power_bound(mphy, pwr40);
 	if (pwr20)
-		pwr20 = mt7915_get_power_bound(phy, pwr20);
+		pwr20 = mt76_get_power_bound(mphy, pwr20);
 
 	if (pwr160 < 0 || pwr80 < 0 || pwr40 < 0 || pwr20 < 0)
 		return -EINVAL;
@@ -1241,7 +1262,7 @@ int mt7915_init_debugfs(struct mt7915_phy *phy)
 	if (!dev->dbdc_support || phy->mt76->band_idx) {
 		debugfs_create_u32("dfs_hw_pattern", 0400, dir,
 				   &dev->hw_pattern);
-		debugfs_create_file("radar_trigger", 0200, dir, dev,
+		debugfs_create_file("radar_trigger", 0200, dir, phy,
 				    &fops_radar_trigger);
 		debugfs_create_devm_seqfile(dev->mt76.dev, "rdd_monitor", dir,
 					    mt7915_rdd_monitor);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c
index c772d2e06e8c..0df2bd93c82e 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.c
@@ -147,7 +147,7 @@ static int mt7915_eeprom_load(struct mt7915_dev *dev)
 		/* read eeprom data from efuse */
 		block_num = DIV_ROUND_UP(eeprom_size, eeprom_blk_size);
 		for (i = 0; i < block_num; i++) {
-			ret = mt7915_mcu_get_eeprom(dev, i * eeprom_blk_size);
+			ret = mt7915_mcu_get_eeprom(dev, i * eeprom_blk_size, NULL);
 			if (ret < 0)
 				return ret;
 		}
@@ -365,6 +365,37 @@ s8 mt7915_eeprom_get_power_delta(struct mt7915_dev *dev, int band)
 	return val & MT_EE_RATE_DELTA_SIGN ? delta : -delta;
 }
 
+bool
+mt7915_eeprom_has_background_radar(struct mt7915_dev *dev)
+{
+	u8 val, buf[MT7915_EEPROM_BLOCK_SIZE];
+	u8 band_sel, tx_path, rx_path;
+	int offs = MT_EE_WIFI_CONF + 1;
+
+	switch (mt76_chip(&dev->mt76)) {
+	case 0x7915:
+		return true;
+	case 0x7906:
+		/* read efuse to check background radar capability */
+		if (mt7915_mcu_get_eeprom(dev, offs, buf))
+			break;
+
+		val = buf[offs % MT7915_EEPROM_BLOCK_SIZE];
+		band_sel = u8_get_bits(val, MT_EE_WIFI_CONF0_BAND_SEL);
+		tx_path = u8_get_bits(val, MT_EE_WIFI_CONF0_TX_PATH);
+		rx_path = u8_get_bits(val, MT_EE_WIFI_CONF0_RX_PATH);
+
+		return (band_sel == MT_EE_V2_BAND_SEL_5GHZ &&
+			tx_path == rx_path && rx_path == 2);
+	case 0x7981:
+	case 0x7986:
+	default:
+		break;
+	}
+
+	return false;
+}
+
 const u8 mt7915_sku_group_len[] = {
 	[SKU_CCK] = 4,
 	[SKU_OFDM] = 8,
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h
index 509fb43d8a68..31aec0f40232 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/eeprom.h
@@ -55,6 +55,7 @@ enum mt7915_eeprom_field {
 #define MT_EE_CAL_DPD_SIZE_V2_7981		(102 * MT_EE_CAL_UNIT)	/* no 6g dpd data */
 
 #define MT_EE_WIFI_CONF0_TX_PATH		GENMASK(2, 0)
+#define MT_EE_WIFI_CONF0_RX_PATH		GENMASK(5, 3)
 #define MT_EE_WIFI_CONF0_BAND_SEL		GENMASK(7, 6)
 #define MT_EE_WIFI_CONF1_BAND_SEL		GENMASK(7, 6)
 #define MT_EE_WIFI_CONF_STREAM_NUM		GENMASK(7, 5)
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/init.c b/sys/contrib/dev/mediatek/mt76/mt7915/init.c
index 33928fe48165..6c0f1cdc4987 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/init.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/init.c
@@ -300,7 +300,7 @@ static void __mt7915_init_txpower(struct mt7915_phy *phy,
 {
 	struct mt7915_dev *dev = phy->dev;
 	int i, n_chains = hweight16(phy->mt76->chainmask);
-	int nss_delta = mt76_tx_power_nss_delta(n_chains);
+	int path_delta = mt76_tx_power_path_delta(n_chains);
 	int pwr_delta = mt7915_eeprom_get_power_delta(dev, sband->band);
 	struct mt76_power_limits limits;
 
@@ -320,7 +320,7 @@ static void __mt7915_init_txpower(struct mt7915_phy *phy,
 		target_power = mt76_get_rate_power_limits(phy->mt76, chan,
 							  &limits,
 							  target_power);
-		target_power += nss_delta;
+		target_power += path_delta;
 		target_power = DIV_ROUND_UP(target_power, 2);
 		chan->max_power = min_t(int, chan->max_reg_power,
 					target_power);
@@ -407,10 +407,13 @@ mt7915_init_wiphy(struct mt7915_phy *phy)
 	if (!is_mt7915(&dev->mt76))
 		wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_STA_TX_PWR);
 
+	if (mt7915_eeprom_has_background_radar(phy->dev) &&
 #if defined(CONFIG_OF)
-	if (!mdev->dev->of_node ||
-	    !of_property_read_bool(mdev->dev->of_node,
-				   "mediatek,disable-radar-background"))
+	    (!mdev->dev->of_node ||
+	     !of_property_read_bool(mdev->dev->of_node,
+				    "mediatek,disable-radar-background")))
+#else
+	    1)
 #endif
 		wiphy_ext_feature_set(wiphy,
 				      NL80211_EXT_FEATURE_RADAR_BACKGROUND);
@@ -951,8 +954,7 @@ mt7915_set_stream_he_txbf_caps(struct mt7915_phy *phy,
 
 	c = IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US;
 	if (!is_mt7915(&dev->mt76))
-		c |= IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-		     IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO;
+		c |= IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO;
 	elem->phy_cap_info[2] |= c;
 
 	c = IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE |
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mac.c b/sys/contrib/dev/mediatek/mt76/mt7915/mac.c
index 53b241bd2338..dcdbde3ee5b7 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/mac.c
@@ -60,10 +60,7 @@ static struct mt76_wcid *mt7915_rx_get_wcid(struct mt7915_dev *dev,
 	struct mt7915_sta *sta;
 	struct mt76_wcid *wcid;
 
-	if (idx >= ARRAY_SIZE(dev->mt76.wcid))
-		return NULL;
-
-	wcid = rcu_dereference(dev->mt76.wcid[idx]);
+	wcid = mt76_wcid_ptr(dev, idx);
 	if (unicast || !wcid)
 		return wcid;
 
@@ -938,7 +935,7 @@ mt7915_mac_tx_free(struct mt7915_dev *dev, void *data, int len)
 			u16 idx;
 
 			idx = FIELD_GET(MT_TX_FREE_WLAN_ID, info);
-			wcid = rcu_dereference(dev->mt76.wcid[idx]);
+			wcid = mt76_wcid_ptr(dev, idx);
 			sta = wcid_to_sta(wcid);
 			if (!sta)
 				continue;
@@ -1043,12 +1040,9 @@ static void mt7915_mac_add_txs(struct mt7915_dev *dev, void *data)
 	if (pid < MT_PACKET_ID_WED)
 		return;
 
-	if (wcidx >= mt7915_wtbl_size(dev))
-		return;
-
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[wcidx]);
+	wcid = mt76_wcid_ptr(dev, wcidx);
 	if (!wcid)
 		goto out;
 
@@ -1500,17 +1494,15 @@ mt7915_mac_full_reset(struct mt7915_dev *dev)
 	if (i == 10)
 		dev_err(dev->mt76.dev, "chip full reset failed\n");
 
-	spin_lock_bh(&dev->mt76.sta_poll_lock);
-	while (!list_empty(&dev->mt76.sta_poll_list))
-		list_del_init(dev->mt76.sta_poll_list.next);
-	spin_unlock_bh(&dev->mt76.sta_poll_lock);
-
-	memset(dev->mt76.wcid_mask, 0, sizeof(dev->mt76.wcid_mask));
-	dev->mt76.vif_mask = 0;
 	dev->phy.omac_mask = 0;
 	if (phy2)
 		phy2->omac_mask = 0;
 
+	mt76_reset_device(&dev->mt76);
+
+	INIT_LIST_HEAD(&dev->sta_rc_list);
+	INIT_LIST_HEAD(&dev->twt_list);
+
 	i = mt76_wcid_alloc(dev->mt76.wcid_mask, MT7915_WTBL_STA);
 	dev->mt76.global_wcid.idx = i;
 	dev->recovery.hw_full_reset = false;
@@ -2073,16 +2065,15 @@ void mt7915_mac_work(struct work_struct *work)
 static void mt7915_dfs_stop_radar_detector(struct mt7915_phy *phy)
 {
 	struct mt7915_dev *dev = phy->dev;
+	int rdd_idx = mt7915_get_rdd_idx(phy, false);
+
+	if (rdd_idx < 0)
+		return;
 
-	if (phy->rdd_state & BIT(0))
-		mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_STOP, 0,
-					MT_RX_SEL0, 0);
-	if (phy->rdd_state & BIT(1))
-		mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_STOP, 1,
-					MT_RX_SEL0, 0);
+	mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_STOP, rdd_idx, 0, 0);
 }
 
-static int mt7915_dfs_start_rdd(struct mt7915_dev *dev, int chain)
+static int mt7915_dfs_start_rdd(struct mt7915_dev *dev, int rdd_idx)
 {
 	int err, region;
 
@@ -2099,52 +2090,38 @@ static int mt7915_dfs_start_rdd(struct mt7915_dev *dev, int chain)
 		break;
 	}
 
-	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_START, chain,
-				      MT_RX_SEL0, region);
+	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_START, rdd_idx, 0, region);
 	if (err < 0)
 		return err;
 
 	if (is_mt7915(&dev->mt76)) {
-		err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_SET_WF_ANT, chain,
+		err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_SET_WF_ANT, rdd_idx,
 					      0, dev->dbdc_support ? 2 : 0);
 		if (err < 0)
 			return err;
 	}
 
-	return mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_DET_MODE, chain,
-				       MT_RX_SEL0, 1);
+	return mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_DET_MODE, rdd_idx, 0, 1);
 }
 
 static int mt7915_dfs_start_radar_detector(struct mt7915_phy *phy)
 {
-	struct cfg80211_chan_def *chandef = &phy->mt76->chandef;
 	struct mt7915_dev *dev = phy->dev;
-	int err;
+	int err, rdd_idx;
+
+	rdd_idx = mt7915_get_rdd_idx(phy, false);
+	if (rdd_idx < 0)
+		return -EINVAL;
 
 	/* start CAC */
-	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_CAC_START,
-				      phy->mt76->band_idx, MT_RX_SEL0, 0);
+	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_CAC_START, rdd_idx, 0, 0);
 	if (err < 0)
 		return err;
 
-	err = mt7915_dfs_start_rdd(dev, phy->mt76->band_idx);
+	err = mt7915_dfs_start_rdd(dev, rdd_idx);
 	if (err < 0)
 		return err;
 
-	phy->rdd_state |= BIT(phy->mt76->band_idx);
-
-	if (!is_mt7915(&dev->mt76))
-		return 0;
-
-	if (chandef->width == NL80211_CHAN_WIDTH_160 ||
-	    chandef->width == NL80211_CHAN_WIDTH_80P80) {
-		err = mt7915_dfs_start_rdd(dev, 1);
-		if (err < 0)
-			return err;
-
-		phy->rdd_state |= BIT(1);
-	}
-
 	return 0;
 }
 
@@ -2186,12 +2163,12 @@ int mt7915_dfs_init_radar_detector(struct mt7915_phy *phy)
 {
 	struct mt7915_dev *dev = phy->dev;
 	enum mt76_dfs_state dfs_state, prev_state;
-	int err;
+	int err, rdd_idx = mt7915_get_rdd_idx(phy, false);
 
 	prev_state = phy->mt76->dfs_state;
 	dfs_state = mt76_phy_dfs_state(phy->mt76);
 
-	if (prev_state == dfs_state)
+	if (prev_state == dfs_state || rdd_idx < 0)
 		return 0;
 
 	if (prev_state == MT_DFS_STATE_UNKNOWN)
@@ -2215,8 +2192,7 @@ int mt7915_dfs_init_radar_detector(struct mt7915_phy *phy)
 	if (dfs_state == MT_DFS_STATE_CAC)
 		return 0;
 
-	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_CAC_END,
-				      phy->mt76->band_idx, MT_RX_SEL0, 0);
+	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_CAC_END, rdd_idx, 0, 0);
 	if (err < 0) {
 		phy->mt76->dfs_state = MT_DFS_STATE_UNKNOWN;
 		return err;
@@ -2226,15 +2202,13 @@ int mt7915_dfs_init_radar_detector(struct mt7915_phy *phy)
 	return 0;
 
 stop:
-	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_NORMAL_START,
-				      phy->mt76->band_idx, MT_RX_SEL0, 0);
+	err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_NORMAL_START, rdd_idx, 0, 0);
 	if (err < 0)
 		return err;
 
 	if (is_mt7915(&dev->mt76)) {
 		err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_SET_WF_ANT,
-					      phy->mt76->band_idx, 0,
-					      dev->dbdc_support ? 2 : 0);
+					      rdd_idx, 0, dev->dbdc_support ? 2 : 0);
 		if (err < 0)
 			return err;
 	}
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/main.c b/sys/contrib/dev/mediatek/mt76/mt7915/main.c
index 3aa31c5cefa6..fe0639c14bf9 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/main.c
@@ -449,7 +449,8 @@ out:
 	return err;
 }
 
-static int mt7915_config(struct ieee80211_hw *hw, u32 changed)
+static int mt7915_config(struct ieee80211_hw *hw, int radio_idx,
+			 u32 changed)
 {
 	struct mt7915_dev *dev = mt7915_hw_dev(hw);
 	struct mt7915_phy *phy = mt7915_hw_phy(hw);
@@ -906,7 +907,8 @@ static void mt7915_tx(struct ieee80211_hw *hw,
 	mt76_tx(mphy, control->sta, wcid, skb);
 }
 
-static int mt7915_set_rts_threshold(struct ieee80211_hw *hw, u32 val)
+static int mt7915_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				    u32 val)
 {
 	struct mt7915_dev *dev = mt7915_hw_dev(hw);
 	struct mt7915_phy *phy = mt7915_hw_phy(hw);
@@ -1102,7 +1104,8 @@ mt7915_offset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 }
 
 static void
-mt7915_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
+mt7915_set_coverage_class(struct ieee80211_hw *hw, int radio_idx,
+			  s16 coverage_class)
 {
 	struct mt7915_phy *phy = mt7915_hw_phy(hw);
 	struct mt7915_dev *dev = phy->dev;
@@ -1114,7 +1117,7 @@ mt7915_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
 }
 
 static int
-mt7915_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
+mt7915_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_ant)
 {
 	struct mt7915_dev *dev = mt7915_hw_dev(hw);
 	struct mt7915_phy *phy = mt7915_hw_phy(hw);
@@ -1655,7 +1658,7 @@ mt7915_twt_teardown_request(struct ieee80211_hw *hw,
 }
 
 static int
-mt7915_set_frag_threshold(struct ieee80211_hw *hw, u32 val)
+mt7915_set_frag_threshold(struct ieee80211_hw *hw, int radio_idx, u32 val)
 {
 	return 0;
 }
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c
index 03dfe2ed4682..f83f4eaa8a24 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.c
@@ -197,6 +197,8 @@ mt7915_mcu_parse_response(struct mt76_dev *mdev, int cmd,
 static void
 mt7915_mcu_set_timeout(struct mt76_dev *mdev, int cmd)
 {
+	mdev->mcu.timeout = 5 * HZ;
+
 	if ((cmd & __MCU_CMD_FIELD_ID) != MCU_CMD_EXT_CID)
 		return;
 
@@ -208,6 +210,9 @@ mt7915_mcu_set_timeout(struct mt76_dev *mdev, int cmd)
 	case MCU_EXT_CMD_BSS_INFO_UPDATE:
 		mdev->mcu.timeout = 2 * HZ;
 		return;
+	case MCU_EXT_CMD_EFUSE_BUFFER_MODE:
+		mdev->mcu.timeout = 10 * HZ;
+		return;
 	default:
 		break;
 	}
@@ -303,17 +308,35 @@ mt7915_mcu_rx_radar_detected(struct mt7915_dev *dev, struct sk_buff *skb)
 {
 	struct mt76_phy *mphy = &dev->mt76.phy;
 	struct mt7915_mcu_rdd_report *r;
+	u32 sku;
 
 	r = (struct mt7915_mcu_rdd_report *)skb->data;
 
-	if (r->band_idx > MT_RX_SEL2)
+	switch (r->rdd_idx) {
+	case MT_RDD_IDX_BAND0:
+		break;
+	case MT_RDD_IDX_BAND1:
+		sku = mt7915_check_adie(dev, true);
+		/* the main phy is bound to band 1 for this sku */
+		if (is_mt7986(&dev->mt76) &&
+		    (sku == MT7975_ONE_ADIE || sku == MT7976_ONE_ADIE))
+			break;
+		mphy = dev->mt76.phys[MT_BAND1];
+		break;
+	case MT_RDD_IDX_BACKGROUND:
+		if (!dev->rdd2_phy)
+			return;
+		mphy = dev->rdd2_phy->mt76;
+		break;
+	default:
+		dev_err(dev->mt76.dev, "Unknown RDD idx %d\n", r->rdd_idx);
 		return;
+	}
 
-	if ((r->band_idx && !dev->phy.mt76->band_idx) &&
-	    dev->mt76.phys[MT_BAND1])
-		mphy = dev->mt76.phys[MT_BAND1];
+	if (!mphy)
+		return;
 
-	if (r->band_idx == MT_RX_SEL2)
+	if (r->rdd_idx == MT_RDD_IDX_BACKGROUND)
 		cfg80211_background_radar_event(mphy->hw->wiphy,
 						&dev->rdd2_chandef,
 						GFP_ATOMIC);
@@ -2098,16 +2121,21 @@ static int mt7915_load_firmware(struct mt7915_dev *dev)
 {
 	int ret;
 
-	/* make sure fw is download state */
-	if (mt7915_firmware_state(dev, false)) {
-		/* restart firmware once */
-		mt76_connac_mcu_restart(&dev->mt76);
-		ret = mt7915_firmware_state(dev, false);
-		if (ret) {
-			dev_err(dev->mt76.dev,
-				"Firmware is not ready for download\n");
-			return ret;
-		}
+	/* Release Semaphore if taken by previous failed attempt */
+	ret = mt76_connac_mcu_patch_sem_ctrl(&dev->mt76, false);
+	if (ret != PATCH_REL_SEM_SUCCESS) {
+		dev_err(dev->mt76.dev, "Could not release semaphore\n");
+		/* Continue anyways */
+	}
+
+	/* Always restart MCU firmware */
+	mt76_connac_mcu_restart(&dev->mt76);
+
+	/* Check if MCU is ready */
+	ret = mt7915_firmware_state(dev, false);
+	if (ret) {
+		dev_err(dev->mt76.dev, "Firmware did not enter download state\n");
+		return ret;
 	}
 
 	ret = mt76_connac2_load_patch(&dev->mt76, fw_name_var(dev, ROM_PATCH));
@@ -2703,11 +2731,14 @@ int mt7915_mcu_rdd_background_enable(struct mt7915_phy *phy,
 				     struct cfg80211_chan_def *chandef)
 {
 	struct mt7915_dev *dev = phy->dev;
-	int err, region;
+	int err, region, rdd_idx;
+
+	rdd_idx = mt7915_get_rdd_idx(phy, true);
+	if (rdd_idx < 0)
+		return -EINVAL;
 
 	if (!chandef) { /* disable offchain */
-		err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_STOP, MT_RX_SEL2,
-					      0, 0);
+		err = mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_STOP, rdd_idx, 0, 0);
 		if (err)
 			return err;
 
@@ -2733,8 +2764,7 @@ int mt7915_mcu_rdd_background_enable(struct mt7915_phy *phy,
 		break;
 	}
 
-	return mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_START, MT_RX_SEL2,
-				       0, region);
+	return mt76_connac_mcu_rdd_cmd(&dev->mt76, RDD_START, rdd_idx, 0, region);
 }
 
 int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd)
@@ -2865,7 +2895,7 @@ int mt7915_mcu_set_eeprom(struct mt7915_dev *dev)
 				 &req, sizeof(req), true);
 }
 
-int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset)
+int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset, u8 *read_buf)
 {
 	struct mt7915_mcu_eeprom_info req = {
 		.addr = cpu_to_le32(round_down(offset,
@@ -2873,8 +2903,8 @@ int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset)
 	};
 	struct mt7915_mcu_eeprom_info *res;
 	struct sk_buff *skb;
+	u8 *buf = read_buf;
 	int ret;
-	u8 *buf;
 
 	ret = mt76_mcu_send_and_get_msg(&dev->mt76,
 					MCU_EXT_QUERY(EFUSE_ACCESS),
@@ -2883,12 +2913,14 @@ int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset)
 		return ret;
 
 	res = (struct mt7915_mcu_eeprom_info *)skb->data;
+	if (!buf)
 #if defined(__linux__)
-	buf = dev->mt76.eeprom.data + le32_to_cpu(res->addr);
+		buf = dev->mt76.eeprom.data + le32_to_cpu(res->addr);
 #elif defined(__FreeBSD__)
-	buf = (u8 *)dev->mt76.eeprom.data + le32_to_cpu(res->addr);
+		buf = (u8 *)dev->mt76.eeprom.data + le32_to_cpu(res->addr);
 #endif
 	memcpy(buf, res->data, MT7915_EEPROM_BLOCK_SIZE);
+
 	dev_kfree_skb(skb);
 
 	return 0;
@@ -3333,7 +3365,7 @@ int mt7915_mcu_set_txpower_frame(struct mt7915_phy *phy,
 	if (ret)
 		return ret;
 
-	txpower = mt7915_get_power_bound(phy, txpower);
+	txpower = mt76_get_power_bound(mphy, txpower);
 	if (txpower > mphy->txpower_cur || txpower < 0)
 		return -EINVAL;
 
@@ -3383,7 +3415,7 @@ int mt7915_mcu_set_txpower_sku(struct mt7915_phy *phy)
 	int i, idx;
 	int tx_power;
 
-	tx_power = mt7915_get_power_bound(phy, hw->conf.power_level);
+	tx_power = mt76_get_power_bound(mphy, hw->conf.power_level);
 	tx_power = mt76_get_rate_power_limits(mphy, mphy->chandef.chan,
 					      &limits_array, tx_power);
 	mphy->txpower_cur = tx_power;
@@ -3974,7 +4006,7 @@ int mt7915_mcu_wed_wa_tx_stats(struct mt7915_dev *dev, u16 wlan_idx)
 
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[wlan_idx]);
+	wcid = mt76_wcid_ptr(dev, wlan_idx);
 	if (wcid)
 		wcid->stats.tx_packets += le32_to_cpu(res->tx_packets);
 	else
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h
index 49476a4182fd..086ad89ecd91 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/mcu.h
@@ -57,7 +57,7 @@ struct mt7915_mcu_bcc_notify {
 struct mt7915_mcu_rdd_report {
 	struct mt76_connac2_mcu_rxd_hdr rxd;
 
-	u8 band_idx;
+	u8 rdd_idx;
 	u8 long_detected;
 	u8 constant_prf_detected;
 	u8 staggered_prf_detected;
@@ -515,16 +515,4 @@ enum {
 					 sizeof(struct bss_info_bmc_rate) +\
 					 sizeof(struct bss_info_ext_bss))
 
-static inline s8
-mt7915_get_power_bound(struct mt7915_phy *phy, s8 txpower)
-{
-	struct mt76_phy *mphy = phy->mt76;
-	int n_chains = hweight16(mphy->chainmask);
-
-	txpower = mt76_get_sar_power(mphy, mphy->chandef.chan, txpower * 2);
-	txpower -= mt76_tx_power_nss_delta(n_chains);
-
-	return txpower;
-}
-
 #endif
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c b/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c
index 8530043d3232..83a828b7c578 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/mmio.c
@@ -595,12 +595,9 @@ static void mt7915_mmio_wed_update_rx_stats(struct mtk_wed_device *wed,
 
 	dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed);
 
-	if (idx >= mt7915_wtbl_size(dev))
-		return;
-
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[idx]);
+	wcid = mt76_wcid_ptr(dev, idx);
 	if (wcid) {
 		wcid->stats.rx_bytes += le32_to_cpu(stats->rx_byte_cnt);
 		wcid->stats.rx_packets += le32_to_cpu(stats->rx_pkt_cnt);
@@ -659,6 +656,9 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr,
 		wed->wlan.base = devm_ioremap(dev->mt76.dev,
 					      pci_resource_start(pci_dev, 0),
 					      pci_resource_len(pci_dev, 0));
+		if (!wed->wlan.base)
+			return -ENOMEM;
+
 		wed->wlan.phy_base = pci_resource_start(pci_dev, 0);
 		wed->wlan.wpdma_int = pci_resource_start(pci_dev, 0) +
 				      MT_INT_WED_SOURCE_CSR;
@@ -686,6 +686,9 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr,
 		wed->wlan.bus_type = MTK_WED_BUS_AXI;
 		wed->wlan.base = devm_ioremap(dev->mt76.dev, res->start,
 					      resource_size(res));
+		if (!wed->wlan.base)
+			return -ENOMEM;
+
 		wed->wlan.phy_base = res->start;
 		wed->wlan.wpdma_int = res->start + MT_INT_SOURCE_CSR;
 		wed->wlan.wpdma_mask = res->start + MT_INT_MASK_CSR;
diff --git a/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h b/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h
index afec39e8703b..d674186488f8 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7915/mt7915.h
@@ -218,8 +218,6 @@ struct mt7915_phy {
 	s16 coverage_class;
 	u8 slottime;
 
-	u8 rdd_state;
-
 	u32 trb_ts;
 
 	u32 rx_ampdu_ts;
@@ -334,10 +332,10 @@ enum {
 	__MT_WFDMA_MAX,
 };
 
-enum {
-	MT_RX_SEL0,
-	MT_RX_SEL1,
-	MT_RX_SEL2, /* monitor chain */
+enum rdd_idx {
+	MT_RDD_IDX_BAND0,	/* RDD idx for band idx 0 (single-band) */
+	MT_RDD_IDX_BAND1,	/* RDD idx for band idx 1 */
+	MT_RDD_IDX_BACKGROUND,	/* RDD idx for background chain */
 };
 
 enum mt7915_rdd_cmd {
@@ -357,6 +355,18 @@ enum mt7915_rdd_cmd {
 	RDD_IRQ_OFF,
 };
 
+static inline int
+mt7915_get_rdd_idx(struct mt7915_phy *phy, bool is_background)
+{
+	if (!phy->mt76->cap.has_5ghz)
+		return -1;
+
+	if (is_background)
+		return MT_RDD_IDX_BACKGROUND;
+
+	return phy->mt76->band_idx;
+}
+
 static inline struct mt7915_phy *
 mt7915_hw_phy(struct ieee80211_hw *hw)
 {
@@ -428,6 +438,7 @@ int mt7915_eeprom_get_target_power(struct mt7915_dev *dev,
 				   struct ieee80211_channel *chan,
 				   u8 chain_idx);
 s8 mt7915_eeprom_get_power_delta(struct mt7915_dev *dev, int band);
+bool mt7915_eeprom_has_background_radar(struct mt7915_dev *dev);
 int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2);
 void mt7915_dma_prefetch(struct mt7915_dev *dev);
 void mt7915_dma_cleanup(struct mt7915_dev *dev);
@@ -476,7 +487,7 @@ int mt7915_mcu_set_fixed_rate_ctrl(struct mt7915_dev *dev,
 				   struct ieee80211_sta *sta,
 				   void *data, u32 field);
 int mt7915_mcu_set_eeprom(struct mt7915_dev *dev);
-int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset);
+int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset, u8 *read_buf);
 int mt7915_mcu_get_eeprom_free_block(struct mt7915_dev *dev, u8 *block_num);
 int mt7915_mcu_set_mac(struct mt7915_dev *dev, int band, bool enable,
 		       bool hdr_trans);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/mac.c b/sys/contrib/dev/mediatek/mt76/mt7921/mac.c
index 25010d2ea442..577b27d9faa0 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7921/mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7921/mac.c
@@ -472,7 +472,7 @@ void mt7921_mac_add_txs(struct mt792x_dev *dev, void *data)
 
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[wcidx]);
+	wcid = mt76_wcid_ptr(dev, wcidx);
 	if (!wcid)
 		goto out;
 
@@ -531,7 +531,7 @@ static void mt7921_mac_tx_free(struct mt792x_dev *dev, void *data, int len)
 
 			count++;
 			idx = FIELD_GET(MT_TX_FREE_WLAN_ID, info);
-			wcid = rcu_dereference(dev->mt76.wcid[idx]);
+			wcid = mt76_wcid_ptr(dev, idx);
 			sta = wcid_to_sta(wcid);
 			if (!sta)
 				continue;
@@ -690,6 +690,8 @@ void mt7921_mac_reset_work(struct work_struct *work)
 		if (!ret)
 			break;
 	}
+	if (mt76_is_sdio(&dev->mt76) && atomic_read(&dev->mt76.bus_hung))
+		return;
 
 	if (i == 10)
 		dev_err(dev->mt76.dev, "chip reset failed\n");
@@ -831,7 +833,7 @@ void mt7921_usb_sdio_tx_complete_skb(struct mt76_dev *mdev,
 	u16 idx;
 
 	idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX);
-	wcid = rcu_dereference(mdev->wcid[idx]);
+	wcid = __mt76_wcid_ptr(mdev, idx);
 	sta = wcid_to_sta(wcid);
 
 	if (sta && likely(e->skb->protocol != cpu_to_be16(ETH_P_PAE)))
diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/main.c b/sys/contrib/dev/mediatek/mt76/mt7921/main.c
index 13e58c328aff..5881040ac195 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7921/main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7921/main.c
@@ -83,6 +83,11 @@ mt7921_init_he_caps(struct mt792x_phy *phy, enum nl80211_band band,
 			he_cap_elem->phy_cap_info[9] |=
 				IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU |
 				IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU;
+
+			if (is_mt7922(phy->mt76->dev)) {
+				he_cap_elem->phy_cap_info[0] |=
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
+			}
 			break;
 		case NL80211_IFTYPE_STATION:
 			he_cap_elem->mac_cap_info[1] |=
@@ -364,7 +369,7 @@ void mt7921_roc_abort_sync(struct mt792x_dev *dev)
 {
 	struct mt792x_phy *phy = &dev->phy;
 
-	del_timer_sync(&phy->roc_timer);
+	timer_delete_sync(&phy->roc_timer);
 	cancel_work_sync(&phy->roc_work);
 	if (test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state))
 		ieee80211_iterate_interfaces(mt76_hw(dev),
@@ -395,7 +400,7 @@ static int mt7921_abort_roc(struct mt792x_phy *phy, struct mt792x_vif *vif)
 {
 	int err = 0;
 
-	del_timer_sync(&phy->roc_timer);
+	timer_delete_sync(&phy->roc_timer);
 	cancel_work_sync(&phy->roc_work);
 
 	mt792x_mutex_acquire(phy->dev);
@@ -619,7 +624,7 @@ void mt7921_set_runtime_pm(struct mt792x_dev *dev)
 	mt76_connac_mcu_set_deep_sleep(&dev->mt76, pm->ds_enable);
 }
 
-static int mt7921_config(struct ieee80211_hw *hw, u32 changed)
+static int mt7921_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 	struct mt792x_phy *phy = mt792x_hw_phy(hw);
@@ -811,6 +816,7 @@ int mt7921_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif,
 	msta->deflink.wcid.phy_idx = mvif->bss_conf.mt76.band_idx;
 	msta->deflink.wcid.tx_info |= MT_WCID_TX_INFO_SET;
 	msta->deflink.last_txs = jiffies;
+	msta->deflink.sta = msta;
 
 	ret = mt76_connac_pm_wake(&dev->mphy, &dev->pm);
 	if (ret)
@@ -901,7 +907,8 @@ void mt7921_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif,
 }
 EXPORT_SYMBOL_GPL(mt7921_mac_sta_remove);
 
-static int mt7921_set_rts_threshold(struct ieee80211_hw *hw, u32 val)
+static int mt7921_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				    u32 val)
 {
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 
@@ -1082,7 +1089,8 @@ mt7921_stop_sched_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 }
 
 static int
-mt7921_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
+mt7921_set_antenna(struct ieee80211_hw *hw, int radio_idx,
+		   u32 tx_ant, u32 rx_ant)
 {
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 	struct mt792x_phy *phy = mt792x_hw_phy(hw);
@@ -1174,6 +1182,9 @@ static void mt7921_sta_set_decap_offload(struct ieee80211_hw *hw,
 	struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv;
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 
+	if (!msta->deflink.wcid.sta)
+		return;
+
 	mt792x_mutex_acquire(dev);
 
 	if (enabled)
@@ -1448,11 +1459,8 @@ static int mt7921_pre_channel_switch(struct ieee80211_hw *hw,
 	if (vif->type != NL80211_IFTYPE_STATION || !vif->cfg.assoc)
 		return -EOPNOTSUPP;
 
-	/* Avoid beacon loss due to the CAC(Channel Availability Check) time
-	 * of the AP.
-	 */
 	if (!cfg80211_chandef_usable(hw->wiphy, &chsw->chandef,
-				     IEEE80211_CHAN_RADAR))
+				     IEEE80211_CHAN_DISABLED))
 		return -EOPNOTSUPP;
 
 	return 0;
@@ -1475,7 +1483,7 @@ static void mt7921_abort_channel_switch(struct ieee80211_hw *hw,
 {
 	struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv;
 
-	del_timer_sync(&mvif->csa_timer);
+	timer_delete_sync(&mvif->csa_timer);
 	cancel_work_sync(&mvif->csa_work);
 }
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c b/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c
index 45b9f35aab17..d8d36b3c3068 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7921/sdio.c
@@ -150,6 +150,8 @@ static int mt7921s_probe(struct sdio_func *func,
 	if (ret)
 		goto error;
 
+	atomic_set(&mdev->bus_hung, false);
+
 	mdev->rev = (mt76_rr(dev, MT_HW_CHIPID) << 16) |
 		    (mt76_rr(dev, MT_HW_REV) & 0xff);
 	dev_dbg(mdev->dev, "ASIC revision: %04x\n", mdev->rev);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c b/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c
index 1f77cf71ca70..a9eb6252a904 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7921/sdio_mac.c
@@ -6,6 +6,8 @@
 #include "mt7921.h"
 #include "../mt76_connac2_mac.h"
 #include "../sdio.h"
+#include <linux/mmc/host.h>
+#include <linux/kallsyms.h>
 
 static void mt7921s_enable_irq(struct mt76_dev *dev)
 {
@@ -35,6 +37,9 @@ int mt7921s_wfsys_reset(struct mt792x_dev *dev)
 	struct mt76_sdio *sdio = &dev->mt76.sdio;
 	u32 val, status;
 
+	if (atomic_read(&dev->mt76.bus_hung))
+		return 0;
+
 	mt7921s_mcu_drv_pmctrl(dev);
 
 	sdio_claim_host(sdio->func);
@@ -91,11 +96,64 @@ int mt7921s_init_reset(struct mt792x_dev *dev)
 	return 0;
 }
 
+static struct mt76_sdio *msdio;
+static void mt7921s_card_reset(struct work_struct *work)
+{
+	struct mmc_host *sdio_host = msdio->func->card->host;
+
+	sdio_claim_host(msdio->func);
+	sdio_release_irq(msdio->func);
+	sdio_release_host(msdio->func);
+
+	mmc_remove_host(sdio_host);
+	msleep(50);
+	mmc_add_host(sdio_host);
+}
+
+static DECLARE_WORK(sdio_reset_work, mt7921s_card_reset);
+static int mt7921s_check_bus(struct mt76_dev *dev)
+{
+	struct mt76_sdio *sdio = &dev->sdio;
+	int err;
+
+	sdio_claim_host(sdio->func);
+	sdio_readl(dev->sdio.func, MCR_WHCR, &err);
+	sdio_release_host(sdio->func);
+
+	return err;
+}
+
+static int mt7921s_host_reset(struct mt792x_dev *dev)
+{
+	struct mt76_dev *mdev = &dev->mt76;
+	int err = -1;
+
+	if (!atomic_read(&mdev->bus_hung))
+		err = mt7921s_check_bus(&dev->mt76);
+
+	if (err) {
+		atomic_set(&mdev->bus_hung, true);
+		msdio = &dev->mt76.sdio;
+		dev_err(mdev->dev, "SDIO bus problem detected(%d), resetting card!!\n", err);
+		schedule_work(&sdio_reset_work);
+		return err;
+	}
+
+	atomic_set(&mdev->bus_hung, false);
+
+	return 0;
+}
+
 int mt7921s_mac_reset(struct mt792x_dev *dev)
 {
 	int err;
 
 	mt76_connac_free_pending_tx_skbs(&dev->pm, NULL);
+
+	mt7921s_host_reset(dev);
+	if (atomic_read(&dev->mt76.bus_hung))
+		return 0;
+
 	mt76_txq_schedule_all(&dev->mphy);
 	mt76_worker_disable(&dev->mt76.tx_worker);
 	set_bit(MT76_MCU_RESET, &dev->mphy.state);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/Makefile b/sys/contrib/dev/mediatek/mt76/mt7925/Makefile
index d321e4ed732f..ade5e647c941 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/Makefile
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/Makefile
@@ -5,5 +5,6 @@ obj-$(CONFIG_MT7925E) += mt7925e.o
 obj-$(CONFIG_MT7925U) += mt7925u.o
 
 mt7925-common-y := mac.o mcu.o main.o init.o debugfs.o
+mt7925-common-$(CONFIG_NL80211_TESTMODE) += testmode.o
 mt7925e-y := pci.o pci_mac.o pci_mcu.o
 mt7925u-y := usb.o
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/init.c b/sys/contrib/dev/mediatek/mt76/mt7925/init.c
index c7ea94e9c803..a5b893b39568 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/init.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/init.c
@@ -53,6 +53,8 @@ static int mt7925_thermal_init(struct mt792x_phy *phy)
 
 	name = devm_kasprintf(&wiphy->dev, GFP_KERNEL, "mt7925_%s",
 			      wiphy_name(wiphy));
+	if (!name)
+		return -ENOMEM;
 
 	hwmon = devm_hwmon_device_register_with_groups(&wiphy->dev, name, phy,
 						       mt7925_hwmon_groups);
@@ -60,15 +62,110 @@ static int mt7925_thermal_init(struct mt792x_phy *phy)
 }
 #endif
 
+void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2)
+{
+	struct mt792x_phy *phy = &dev->phy;
+	struct mt7925_clc_rule_v2 *rule;
+	struct mt7925_clc *clc;
+	bool old = dev->has_eht, new = true;
+	u32 mtcl_conf = mt792x_acpi_get_mtcl_conf(&dev->phy, alpha2);
+	u8 *pos;
+
+	if (mtcl_conf != MT792X_ACPI_MTCL_INVALID &&
+	    (((mtcl_conf >> 4) & 0x3) == 0)) {
+		new = false;
+		goto out;
+	}
+
+	if (!phy->clc[MT792x_CLC_BE_CTRL])
+		goto out;
+
+	clc = (struct mt7925_clc *)phy->clc[MT792x_CLC_BE_CTRL];
+	pos = clc->data;
+
+	while (1) {
+		rule = (struct mt7925_clc_rule_v2 *)pos;
+
+		if (rule->alpha2[0] == alpha2[0] &&
+		    rule->alpha2[1] == alpha2[1]) {
+			new = false;
+			break;
+		}
+
+		/* Check the last one */
+		if (rule->flag & BIT(0))
+			break;
+
+		pos += sizeof(*rule);
+	}
+
+out:
+	if (old == new)
+		return;
+
+	dev->has_eht = new;
+	mt7925_set_stream_he_eht_caps(phy);
+}
+
+static void
+mt7925_regd_channel_update(struct wiphy *wiphy, struct mt792x_dev *dev)
+{
+#define IS_UNII_INVALID(idx, sfreq, efreq, cfreq) \
+	(!(dev->phy.clc_chan_conf & BIT(idx)) && (cfreq) >= (sfreq) && (cfreq) <= (efreq))
+#define MT7925_UNII_59G_IS_VALID	0x1
+#define MT7925_UNII_6G_IS_VALID	0x1e
+	struct ieee80211_supported_band *sband;
+	struct mt76_dev *mdev = &dev->mt76;
+	struct ieee80211_channel *ch;
+	u32 mtcl_conf = mt792x_acpi_get_mtcl_conf(&dev->phy, mdev->alpha2);
+	int i;
+
+	if (mtcl_conf != MT792X_ACPI_MTCL_INVALID) {
+		if ((mtcl_conf & 0x3) == 0)
+			dev->phy.clc_chan_conf &= ~MT7925_UNII_59G_IS_VALID;
+		if (((mtcl_conf >> 2) & 0x3) == 0)
+			dev->phy.clc_chan_conf &= ~MT7925_UNII_6G_IS_VALID;
+	}
+
+	sband = wiphy->bands[NL80211_BAND_5GHZ];
+	if (!sband)
+		return;
+
+	for (i = 0; i < sband->n_channels; i++) {
+		ch = &sband->channels[i];
+
+		/* UNII-4 */
+		if (IS_UNII_INVALID(0, 5845, 5925, ch->center_freq))
+			ch->flags |= IEEE80211_CHAN_DISABLED;
+	}
+
+	sband = wiphy->bands[NL80211_BAND_6GHZ];
+	if (!sband)
+		return;
+
+	for (i = 0; i < sband->n_channels; i++) {
+		ch = &sband->channels[i];
+
+		/* UNII-5/6/7/8 */
+		if (IS_UNII_INVALID(1, 5925, 6425, ch->center_freq) ||
+		    IS_UNII_INVALID(2, 6425, 6525, ch->center_freq) ||
+		    IS_UNII_INVALID(3, 6525, 6875, ch->center_freq) ||
+		    IS_UNII_INVALID(4, 6875, 7125, ch->center_freq))
+			ch->flags |= IEEE80211_CHAN_DISABLED;
+	}
+}
+
 void mt7925_regd_update(struct mt792x_dev *dev)
 {
 	struct mt76_dev *mdev = &dev->mt76;
 	struct ieee80211_hw *hw = mdev->hw;
+	struct wiphy *wiphy = hw->wiphy;
 
 	if (!dev->regd_change)
 		return;
 
 	mt7925_mcu_set_clc(dev, mdev->alpha2, dev->country_ie_env);
+	mt7925_regd_channel_update(wiphy, dev);
 	mt7925_mcu_set_channel_domain(hw->priv);
 	mt7925_set_tx_sar_pwr(hw, NULL);
 	dev->regd_change = false;
@@ -233,6 +330,12 @@ static void mt7925_init_work(struct work_struct *work)
 	}
 #endif
 
+	ret = mt7925_mcu_set_thermal_protect(dev);
+	if (ret) {
+		dev_err(dev->mt76.dev, "thermal protection enable failed\n");
+		return;
+	}
+
 	/* we support chip reset now */
 	dev->hw_init_done = true;
 
@@ -250,6 +353,7 @@ int mt7925_register_device(struct mt792x_dev *dev)
 	dev->mt76.tx_worker.fn = mt792x_tx_worker;
 
 	INIT_DELAYED_WORK(&dev->pm.ps_work, mt792x_pm_power_save_work);
+	INIT_DELAYED_WORK(&dev->mlo_pm_work, mt7925_mlo_pm_work);
 	INIT_WORK(&dev->pm.wake_work, mt792x_pm_wake_work);
 	spin_lock_init(&dev->pm.wake.lock);
 	mutex_init(&dev->pm.mutex);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mac.c b/sys/contrib/dev/mediatek/mt76/mt7925/mac.c
index e00e7407410f..63995bf9c5d4 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/mac.c
@@ -1040,7 +1040,7 @@ void mt7925_mac_add_txs(struct mt792x_dev *dev, void *data)
 
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[wcidx]);
+	wcid = mt76_wcid_ptr(dev, wcidx);
 	if (!wcid)
 		goto out;
 
@@ -1126,7 +1126,7 @@ mt7925_mac_tx_free(struct mt792x_dev *dev, void *data, int len)
 			u16 idx;
 
 			idx = FIELD_GET(MT_TXFREE_INFO_WLAN_ID, info);
-			wcid = rcu_dereference(dev->mt76.wcid[idx]);
+			wcid = mt76_wcid_ptr(dev, idx);
 			sta = wcid_to_sta(wcid);
 			if (!sta)
 				continue;
@@ -1449,11 +1449,11 @@ void mt7925_usb_sdio_tx_complete_skb(struct mt76_dev *mdev,
 	u16 idx;
 
 	idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX);
-	wcid = rcu_dereference(mdev->wcid[idx]);
+	wcid = __mt76_wcid_ptr(mdev, idx);
 	sta = wcid_to_sta(wcid);
 
 	if (sta && likely(e->skb->protocol != cpu_to_be16(ETH_P_PAE)))
-		mt76_connac2_tx_check_aggr(sta, txwi);
+		mt7925_tx_check_aggr(sta, e->skb, wcid);
 
 	skb_pull(e->skb, headroom);
 	mt76_tx_complete_skb(mdev, e->wcid, e->skb);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/main.c b/sys/contrib/dev/mediatek/mt76/mt7925/main.c
index 98daf80ac131..b0e053b15227 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/main.c
@@ -251,12 +251,12 @@ int mt7925_init_mlo_caps(struct mt792x_phy *phy)
 		},
 	};
 
-	if (!(phy->chip_cap & MT792x_CHIP_CAP_MLO_EVT_EN))
+	if (!(phy->chip_cap & MT792x_CHIP_CAP_MLO_EN))
 		return 0;
 
 	ext_capab[0].eml_capabilities = phy->eml_cap;
 	ext_capab[0].mld_capa_and_ops =
-		u16_encode_bits(1, IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS);
+		u16_encode_bits(0, IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS);
 
 	wiphy->flags |= WIPHY_FLAG_SUPPORTS_MLO;
 	wiphy->iftype_ext_capab = ext_capab;
@@ -334,6 +334,9 @@ int __mt7925_start(struct mt792x_phy *phy)
 	ieee80211_queue_delayed_work(mphy->hw, &mphy->mac_work,
 				     MT792x_WATCHDOG_TIME);
 
+	if (phy->chip_cap & MT792x_CHIP_CAP_WF_RF_PIN_CTRL_EVT_EN)
+		wiphy_rfkill_start_polling(mphy->hw->wiphy);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(__mt7925_start);
@@ -360,10 +363,15 @@ static int mt7925_mac_link_bss_add(struct mt792x_dev *dev,
 	struct mt76_txq *mtxq;
 	int idx, ret = 0;
 
-	mconf->mt76.idx = __ffs64(~dev->mt76.vif_mask);
-	if (mconf->mt76.idx >= MT792x_MAX_INTERFACES) {
-		ret = -ENOSPC;
-		goto out;
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
+		mconf->mt76.idx = MT792x_MAX_INTERFACES;
+	} else {
+		mconf->mt76.idx = __ffs64(~dev->mt76.vif_mask);
+
+		if (mconf->mt76.idx >= MT792x_MAX_INTERFACES) {
+			ret = -ENOSPC;
+			goto out;
+		}
 	}
 
 	mconf->mt76.omac_idx = ieee80211_vif_is_mld(vif) ?
@@ -371,6 +379,7 @@ static int mt7925_mac_link_bss_add(struct mt792x_dev *dev,
 	mconf->mt76.band_idx = 0xff;
 	mconf->mt76.wmm_idx = ieee80211_vif_is_mld(vif) ?
 			      0 : mconf->mt76.idx % MT76_CONNAC_MAX_WMM_SETS;
+	mconf->mt76.link_idx = hweight16(mvif->valid_links);
 
 	if (mvif->phy->mt76->chandef.chan->band != NL80211_BAND_2GHZ)
 		mconf->mt76.basic_rates_idx = MT792x_BASIC_RATES_TBL + 4;
@@ -421,6 +430,7 @@ mt7925_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 	mvif->bss_conf.vif = mvif;
 	mvif->sta.vif = mvif;
 	mvif->deflink_id = IEEE80211_LINK_UNSPECIFIED;
+	mvif->mlo_pm_state = MT792x_MLO_LINK_DISASSOC;
 
 	ret = mt7925_mac_link_bss_add(dev, &vif->bss_conf, &mvif->sta.deflink);
 	if (ret < 0)
@@ -446,7 +456,7 @@ void mt7925_roc_abort_sync(struct mt792x_dev *dev)
 {
 	struct mt792x_phy *phy = &dev->phy;
 
-	del_timer_sync(&phy->roc_timer);
+	timer_delete_sync(&phy->roc_timer);
 	cancel_work_sync(&phy->roc_work);
 	if (test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state))
 		ieee80211_iterate_interfaces(mt76_hw(dev),
@@ -478,7 +488,7 @@ static int mt7925_abort_roc(struct mt792x_phy *phy,
 {
 	int err = 0;
 
-	del_timer_sync(&phy->roc_timer);
+	timer_delete_sync(&phy->roc_timer);
 	cancel_work_sync(&phy->roc_work);
 
 	mt792x_mutex_acquire(phy->dev);
@@ -747,7 +757,7 @@ void mt7925_set_runtime_pm(struct mt792x_dev *dev)
 	mt7925_mcu_set_deep_sleep(dev, pm->ds_enable);
 }
 
-static int mt7925_config(struct ieee80211_hw *hw, u32 changed)
+static int mt7925_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 	int ret = 0;
@@ -1149,7 +1159,12 @@ static void mt7925_mac_link_sta_remove(struct mt76_dev *mdev,
 		struct mt792x_bss_conf *mconf;
 
 		mconf = mt792x_link_conf_to_mconf(link_conf);
-		mt792x_mac_link_bss_remove(dev, mconf, mlink);
+
+		if (ieee80211_vif_is_mld(vif))
+			mt792x_mac_link_bss_remove(dev, mconf, mlink);
+		else
+			mt7925_mcu_add_bss_info(&dev->phy, mconf->mt76.ctx, link_conf,
+						link_sta, false);
 	}
 
 	spin_lock_bh(&mdev->sta_poll_lock);
@@ -1169,6 +1184,34 @@ mt7925_mac_sta_remove_links(struct mt792x_dev *dev, struct ieee80211_vif *vif,
 	struct mt76_wcid *wcid;
 	unsigned int link_id;
 
+	/* clean up bss before starec */
+	for_each_set_bit(link_id, &old_links, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_link_sta *link_sta;
+		struct ieee80211_bss_conf *link_conf;
+		struct mt792x_bss_conf *mconf;
+		struct mt792x_link_sta *mlink;
+
+		if (vif->type == NL80211_IFTYPE_AP)
+			break;
+
+		link_sta = mt792x_sta_to_link_sta(vif, sta, link_id);
+		if (!link_sta)
+			continue;
+
+		mlink = mt792x_sta_to_link(msta, link_id);
+		if (!mlink)
+			continue;
+
+		link_conf = mt792x_vif_to_bss_conf(vif, link_id);
+		if (!link_conf)
+			continue;
+
+		mconf = mt792x_link_conf_to_mconf(link_conf);
+
+		mt7925_mcu_add_bss_info(&dev->phy, mconf->mt76.ctx, link_conf,
+					link_sta, false);
+	}
+
 	for_each_set_bit(link_id, &old_links, IEEE80211_MLD_MAX_NUM_LINKS) {
 		struct ieee80211_link_sta *link_sta;
 		struct mt792x_link_sta *mlink;
@@ -1206,55 +1249,27 @@ void mt7925_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif,
 {
 	struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76);
 	struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv;
-	struct {
-		struct {
-			u8 omac_idx;
-			u8 band_idx;
-			__le16 pad;
-		} __packed hdr;
-		struct req_tlv {
-			__le16 tag;
-			__le16 len;
-			u8 active;
-			u8 link_idx; /* hw link idx */
-			u8 omac_addr[ETH_ALEN];
-		} __packed tlv;
-	} dev_req = {
-		.hdr = {
-			.omac_idx = 0,
-			.band_idx = 0,
-		},
-		.tlv = {
-			.tag = cpu_to_le16(DEV_INFO_ACTIVE),
-			.len = cpu_to_le16(sizeof(struct req_tlv)),
-			.active = true,
-		},
-	};
+	struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv;
 	unsigned long rem;
 
 	rem = ieee80211_vif_is_mld(vif) ? msta->valid_links : BIT(0);
 
 	mt7925_mac_sta_remove_links(dev, vif, sta, rem);
 
-	if (ieee80211_vif_is_mld(vif)) {
-		mt7925_mcu_set_dbdc(&dev->mphy, false);
-
-		/* recovery omac address for the legacy interface */
-		memcpy(dev_req.tlv.omac_addr, vif->addr, ETH_ALEN);
-		mt76_mcu_send_msg(mdev, MCU_UNI_CMD(DEV_INFO_UPDATE),
-				  &dev_req, sizeof(dev_req), true);
-	}
+	if (ieee80211_vif_is_mld(vif))
+		mt7925_mcu_del_dev(mdev, vif);
 
 	if (vif->type == NL80211_IFTYPE_STATION) {
-		struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv;
-
 		mvif->wep_sta = NULL;
 		ewma_rssi_init(&mvif->bss_conf.rssi);
 	}
+
+	mvif->mlo_pm_state = MT792x_MLO_LINK_DISASSOC;
 }
 EXPORT_SYMBOL_GPL(mt7925_mac_sta_remove);
 
-static int mt7925_set_rts_threshold(struct ieee80211_hw *hw, u32 val)
+static int mt7925_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				    u32 val)
 {
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 
@@ -1289,22 +1304,22 @@ mt7925_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	case IEEE80211_AMPDU_RX_START:
 		mt76_rx_aggr_start(&dev->mt76, &msta->deflink.wcid, tid, ssn,
 				   params->buf_size);
-		mt7925_mcu_uni_rx_ba(dev, vif, params, true);
+		mt7925_mcu_uni_rx_ba(dev, params, true);
 		break;
 	case IEEE80211_AMPDU_RX_STOP:
 		mt76_rx_aggr_stop(&dev->mt76, &msta->deflink.wcid, tid);
-		mt7925_mcu_uni_rx_ba(dev, vif, params, false);
+		mt7925_mcu_uni_rx_ba(dev, params, false);
 		break;
 	case IEEE80211_AMPDU_TX_OPERATIONAL:
 		mtxq->aggr = true;
 		mtxq->send_bar = false;
-		mt7925_mcu_uni_tx_ba(dev, vif, params, true);
+		mt7925_mcu_uni_tx_ba(dev, params, true);
 		break;
 	case IEEE80211_AMPDU_TX_STOP_FLUSH:
 	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
 		mtxq->aggr = false;
 		clear_bit(tid, &msta->deflink.wcid.ampdu_state);
-		mt7925_mcu_uni_tx_ba(dev, vif, params, false);
+		mt7925_mcu_uni_tx_ba(dev, params, false);
 		break;
 	case IEEE80211_AMPDU_TX_START:
 		set_bit(tid, &msta->deflink.wcid.ampdu_state);
@@ -1313,7 +1328,7 @@ mt7925_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	case IEEE80211_AMPDU_TX_STOP_CONT:
 		mtxq->aggr = false;
 		clear_bit(tid, &msta->deflink.wcid.ampdu_state);
-		mt7925_mcu_uni_tx_ba(dev, vif, params, false);
+		mt7925_mcu_uni_tx_ba(dev, params, false);
 		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		break;
 	}
@@ -1322,6 +1337,38 @@ mt7925_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	return ret;
 }
 
+static void
+mt7925_mlo_pm_iter(void *priv, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct mt792x_dev *dev = priv;
+	struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv;
+	unsigned long valid = ieee80211_vif_is_mld(vif) ?
+				    mvif->valid_links : BIT(0);
+	struct ieee80211_bss_conf *bss_conf;
+	int i;
+
+	if (mvif->mlo_pm_state != MT792x_MLO_CHANGED_PS)
+		return;
+
+	mt792x_mutex_acquire(dev);
+	for_each_set_bit(i, &valid, IEEE80211_MLD_MAX_NUM_LINKS) {
+		bss_conf = mt792x_vif_to_bss_conf(vif, i);
+		mt7925_mcu_uni_bss_ps(dev, bss_conf);
+	}
+	mt792x_mutex_release(dev);
+}
+
+void mt7925_mlo_pm_work(struct work_struct *work)
+{
+	struct mt792x_dev *dev = container_of(work, struct mt792x_dev,
+					      mlo_pm_work.work);
+	struct ieee80211_hw *hw = mt76_hw(dev);
+
+	ieee80211_iterate_active_interfaces(hw,
+					    IEEE80211_IFACE_ITER_RESUME_ALL,
+					    mt7925_mlo_pm_iter, dev);
+}
+
 static bool is_valid_alpha2(const char *alpha2)
 {
 	if (!alpha2)
@@ -1378,6 +1425,8 @@ void mt7925_scan_work(struct work_struct *work)
 				if (!is_valid_alpha2(evt->alpha2))
 					break;
 
+				mt7925_regd_be_ctrl(phy->dev, evt->alpha2);
+
 				if (mdev->alpha2[0] != '0' && mdev->alpha2[1] != '0')
 					break;
 
@@ -1436,7 +1485,7 @@ mt7925_start_sched_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 
 	mt792x_mutex_acquire(dev);
 
-	err = mt7925_mcu_sched_scan_req(mphy, vif, req);
+	err = mt7925_mcu_sched_scan_req(mphy, vif, req, ies);
 	if (err < 0)
 		goto out;
 
@@ -1462,7 +1511,8 @@ mt7925_stop_sched_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 }
 
 static int
-mt7925_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
+mt7925_set_antenna(struct ieee80211_hw *hw, int radio_idx,
+		   u32 tx_ant, u32 rx_ant)
 {
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 	struct mt792x_phy *phy = mt792x_hw_phy(hw);
@@ -1558,6 +1608,9 @@ static void mt7925_sta_set_decap_offload(struct ieee80211_hw *hw,
 	unsigned long valid = mvif->valid_links;
 	u8 i;
 
+	if (!msta->vif)
+		return;
+
 	mt792x_mutex_acquire(dev);
 
 	valid = ieee80211_vif_is_mld(vif) ? mvif->valid_links : BIT(0);
@@ -1572,6 +1625,9 @@ static void mt7925_sta_set_decap_offload(struct ieee80211_hw *hw,
 		else
 			clear_bit(MT_WCID_FLAG_HDR_TRANS, &mlink->wcid.flags);
 
+		if (!mlink->wcid.sta)
+			continue;
+
 		mt7925_mcu_wtbl_update_hdr_trans(dev, vif, sta, i);
 	}
 
@@ -1823,6 +1879,10 @@ mt7925_change_chanctx(struct ieee80211_hw *hw,
 			link_conf = mt792x_vif_to_bss_conf(vif, mconf->link_id);
 			mt7925_mcu_set_chctx(mvif->phy->mt76, &mconf->mt76,
 					     link_conf, ctx);
+
+			if (changed & IEEE80211_CHANCTX_CHANGE_PUNCTURING)
+				mt7925_mcu_set_eht_pp(mvif->phy->mt76, &mconf->mt76,
+						      link_conf, ctx);
 		}
 	}
 
@@ -1871,6 +1931,9 @@ static void mt7925_vif_cfg_changed(struct ieee80211_hw *hw,
 		mt7925_mcu_sta_update(dev, NULL, vif, true,
 				      MT76_STA_INFO_STATE_ASSOC);
 		mt7925_mcu_set_beacon_filter(dev, vif, vif->cfg.assoc);
+
+		if (ieee80211_vif_is_mld(vif))
+			mvif->mlo_pm_state = MT792x_MLO_LINK_ASSOC;
 	}
 
 	if (changed & BSS_CHANGED_ARP_FILTER) {
@@ -1881,9 +1944,19 @@ static void mt7925_vif_cfg_changed(struct ieee80211_hw *hw,
 	}
 
 	if (changed & BSS_CHANGED_PS) {
-		for_each_set_bit(i, &valid, IEEE80211_MLD_MAX_NUM_LINKS) {
-			bss_conf = mt792x_vif_to_bss_conf(vif, i);
+		if (hweight16(mvif->valid_links) < 2) {
+			/* legacy */
+			bss_conf = &vif->bss_conf;
 			mt7925_mcu_uni_bss_ps(dev, bss_conf);
+		} else {
+			if (mvif->mlo_pm_state == MT792x_MLO_LINK_ASSOC) {
+				mvif->mlo_pm_state = MT792x_MLO_CHANGED_PS_PENDING;
+			} else if (mvif->mlo_pm_state == MT792x_MLO_CHANGED_PS) {
+				for_each_set_bit(i, &valid, IEEE80211_MLD_MAX_NUM_LINKS) {
+					bss_conf = mt792x_vif_to_bss_conf(vif, i);
+					mt7925_mcu_uni_bss_ps(dev, bss_conf);
+				}
+			}
 		}
 	}
 
@@ -1899,8 +1972,10 @@ static void mt7925_link_info_changed(struct ieee80211_hw *hw,
 	struct mt792x_phy *phy = mt792x_hw_phy(hw);
 	struct mt792x_dev *dev = mt792x_hw_dev(hw);
 	struct mt792x_bss_conf *mconf;
+	struct ieee80211_bss_conf *link_conf;
 
 	mconf = mt792x_vif_to_link(mvif, info->link_id);
+	link_conf = mt792x_vif_to_bss_conf(vif, mconf->link_id);
 
 	mt792x_mutex_acquire(dev);
 
@@ -1934,13 +2009,18 @@ static void mt7925_link_info_changed(struct ieee80211_hw *hw,
 	if (changed & (BSS_CHANGED_QOS | BSS_CHANGED_BEACON_ENABLED))
 		mt7925_mcu_set_tx(dev, info);
 
-	if (changed & BSS_CHANGED_BSSID) {
-		if (ieee80211_vif_is_mld(vif) &&
-		    hweight16(mvif->valid_links) == 2)
-			/* Indicate the secondary setup done */
-			mt7925_mcu_uni_bss_bcnft(dev, info, true);
+	if (mvif->mlo_pm_state == MT792x_MLO_CHANGED_PS_PENDING) {
+		/* Indicate the secondary setup done */
+		mt7925_mcu_uni_bss_bcnft(dev, info, true);
+
+		ieee80211_queue_delayed_work(hw, &dev->mlo_pm_work, 5 * HZ);
+		mvif->mlo_pm_state = MT792x_MLO_CHANGED_PS;
 	}
 
+	if (changed & IEEE80211_CHANCTX_CHANGE_PUNCTURING)
+		mt7925_mcu_set_eht_pp(mvif->phy->mt76, &mconf->mt76,
+				      link_conf, NULL);
+
 	mt792x_mutex_release(dev);
 }
 
@@ -1992,8 +2072,10 @@ mt7925_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 					     GFP_KERNEL);
 			mlink = devm_kzalloc(dev->mt76.dev, sizeof(*mlink),
 					     GFP_KERNEL);
-			if (!mconf || !mlink)
+			if (!mconf || !mlink) {
+				mt792x_mutex_release(dev);
 				return -ENOMEM;
+			}
 		}
 
 		mconfs[link_id] = mconf;
@@ -2022,8 +2104,6 @@ mt7925_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			goto free;
 
 		if (mconf != &mvif->bss_conf) {
-			mt7925_mcu_set_bss_pm(dev, link_conf, true);
-
 			err = mt7925_set_mlo_roc(phy, &mvif->bss_conf,
 						 vif->active_links);
 			if (err < 0)
@@ -2141,6 +2221,18 @@ static void mt7925_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	mutex_unlock(&dev->mt76.mutex);
 }
 
+static void mt7925_rfkill_poll(struct ieee80211_hw *hw)
+{
+	struct mt792x_phy *phy = mt792x_hw_phy(hw);
+	int ret;
+
+	mt792x_mutex_acquire(phy->dev);
+	ret = mt7925_mcu_wf_rf_pin_ctrl(phy);
+	mt792x_mutex_release(phy->dev);
+
+	wiphy_rfkill_set_hw_state(hw->wiphy, ret == 0);
+}
+
 const struct ieee80211_ops mt7925_ops = {
 	.tx = mt792x_tx,
 	.start = mt7925_start,
@@ -2180,6 +2272,8 @@ const struct ieee80211_ops mt7925_ops = {
 	.sta_statistics = mt792x_sta_statistics,
 	.sched_scan_start = mt7925_start_sched_scan,
 	.sched_scan_stop = mt7925_stop_sched_scan,
+	CFG80211_TESTMODE_CMD(mt7925_testmode_cmd)
+	CFG80211_TESTMODE_DUMP(mt7925_testmode_dump)
 #ifdef CONFIG_PM
 	.suspend = mt7925_suspend,
 	.resume = mt7925_resume,
@@ -2201,6 +2295,7 @@ const struct ieee80211_ops mt7925_ops = {
 	.link_info_changed = mt7925_link_info_changed,
 	.change_vif_links = mt7925_change_vif_links,
 	.change_sta_links = mt7925_change_sta_links,
+	.rfkill_poll = mt7925_rfkill_poll,
 };
 EXPORT_SYMBOL_GPL(mt7925_ops);
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c
index f1be88f3b00b..d62d461db699 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.c
@@ -164,6 +164,7 @@ mt7925_connac_mcu_set_wow_ctrl(struct mt76_phy *phy, struct ieee80211_vif *vif,
 			       bool suspend, struct cfg80211_wowlan *wowlan)
 {
 	struct mt76_vif_link *mvif = (struct mt76_vif_link *)vif->drv_priv;
+	struct ieee80211_scan_ies ies = {};
 	struct mt76_dev *dev = phy->dev;
 	struct {
 		struct {
@@ -194,7 +195,7 @@ mt7925_connac_mcu_set_wow_ctrl(struct mt76_phy *phy, struct ieee80211_vif *vif,
 		req.wow_ctrl_tlv.trigger |= (UNI_WOW_DETECT_TYPE_DISCONNECT |
 					     UNI_WOW_DETECT_TYPE_BCN_LOST);
 	if (wowlan->nd_config) {
-		mt7925_mcu_sched_scan_req(phy, vif, wowlan->nd_config);
+		mt7925_mcu_sched_scan_req(phy, vif, wowlan->nd_config, &ies);
 		req.wow_ctrl_tlv.trigger |= UNI_WOW_DETECT_TYPE_SCH_SCAN_HIT;
 		mt7925_mcu_sched_scan_enable(phy, vif, suspend);
 	}
@@ -348,14 +349,10 @@ mt7925_mcu_handle_hif_ctrl_basic(struct mt792x_dev *dev, struct tlv *tlv)
 	basic = (struct mt7925_mcu_hif_ctrl_basic_tlv *)tlv;
 
 	if (basic->hifsuspend) {
-		if (basic->hif_tx_traffic_status == HIF_TRAFFIC_IDLE &&
-		    basic->hif_rx_traffic_status == HIF_TRAFFIC_IDLE)
-			/* success */
-			dev->hif_idle = true;
-		else
-			/* busy */
-			/* invalid */
-			dev->hif_idle = false;
+		dev->hif_idle = true;
+		if (!(basic->hif_tx_traffic_status == HIF_TRAFFIC_IDLE &&
+		      basic->hif_rx_traffic_status == HIF_TRAFFIC_IDLE))
+			dev_info(dev->mt76.dev, "Hif traffic not idle.\n");
 	} else {
 		dev->hif_resumed = true;
 	}
@@ -576,10 +573,10 @@ void mt7925_mcu_rx_event(struct mt792x_dev *dev, struct sk_buff *skb)
 
 static int
 mt7925_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif_link *mvif,
-		  struct mt76_wcid *wcid,
 		  struct ieee80211_ampdu_params *params,
 		  bool enable, bool tx)
 {
+	struct mt76_wcid *wcid = (struct mt76_wcid *)params->sta->drv_priv;
 	struct sta_rec_ba_uni *ba;
 	struct sk_buff *skb;
 	struct tlv *tlv;
@@ -607,60 +604,76 @@ mt7925_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif_link *mvif,
 
 /** starec & wtbl **/
 int mt7925_mcu_uni_tx_ba(struct mt792x_dev *dev,
-			 struct ieee80211_vif *vif,
 			 struct ieee80211_ampdu_params *params,
 			 bool enable)
 {
 	struct mt792x_sta *msta = (struct mt792x_sta *)params->sta->drv_priv;
-	struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv;
-	struct mt792x_link_sta *mlink;
-	struct mt792x_bss_conf *mconf;
-	unsigned long usable_links = ieee80211_vif_usable_links(vif);
-	struct mt76_wcid *wcid;
-	u8 link_id, ret;
-
-	for_each_set_bit(link_id, &usable_links, IEEE80211_MLD_MAX_NUM_LINKS) {
-		mconf = mt792x_vif_to_link(mvif, link_id);
-		mlink = mt792x_sta_to_link(msta, link_id);
-		wcid = &mlink->wcid;
-
-		if (enable && !params->amsdu)
-			mlink->wcid.amsdu = false;
+	struct mt792x_vif *mvif = msta->vif;
 
-		ret = mt7925_mcu_sta_ba(&dev->mt76, &mconf->mt76, wcid, params,
-					enable, true);
-		if (ret < 0)
-			break;
-	}
+	if (enable && !params->amsdu)
+		msta->deflink.wcid.amsdu = false;
 
-	return ret;
+	return mt7925_mcu_sta_ba(&dev->mt76, &mvif->bss_conf.mt76, params,
+				 enable, true);
 }
 
 int mt7925_mcu_uni_rx_ba(struct mt792x_dev *dev,
-			 struct ieee80211_vif *vif,
 			 struct ieee80211_ampdu_params *params,
 			 bool enable)
 {
 	struct mt792x_sta *msta = (struct mt792x_sta *)params->sta->drv_priv;
-	struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv;
-	struct mt792x_link_sta *mlink;
-	struct mt792x_bss_conf *mconf;
-	unsigned long usable_links = ieee80211_vif_usable_links(vif);
-	struct mt76_wcid *wcid;
-	u8 link_id, ret;
+	struct mt792x_vif *mvif = msta->vif;
 
-	for_each_set_bit(link_id, &usable_links, IEEE80211_MLD_MAX_NUM_LINKS) {
-		mconf = mt792x_vif_to_link(mvif, link_id);
-		mlink = mt792x_sta_to_link(msta, link_id);
-		wcid = &mlink->wcid;
+	return mt7925_mcu_sta_ba(&dev->mt76, &mvif->bss_conf.mt76, params,
+				 enable, false);
+}
 
-		ret = mt7925_mcu_sta_ba(&dev->mt76, &mconf->mt76, wcid, params,
-					enable, false);
-		if (ret < 0)
-			break;
-	}
+static int mt7925_mcu_read_eeprom(struct mt792x_dev *dev, u32 offset, u8 *val)
+{
+	struct {
+		u8 rsv[4];
 
-	return ret;
+		__le16 tag;
+		__le16 len;
+
+		__le32 addr;
+		__le32 valid;
+		u8 data[MT7925_EEPROM_BLOCK_SIZE];
+	} __packed req = {
+		.tag = cpu_to_le16(1),
+		.len = cpu_to_le16(sizeof(req) - 4),
+		.addr = cpu_to_le32(round_down(offset,
+				    MT7925_EEPROM_BLOCK_SIZE)),
+	};
+	struct evt {
+		u8 rsv[4];
+
+		__le16 tag;
+		__le16 len;
+
+		__le32 ver;
+		__le32 addr;
+		__le32 valid;
+		__le32 size;
+		__le32 magic_num;
+		__le32 type;
+		__le32 rsv1[4];
+		u8 data[32];
+	} __packed *res;
+	struct sk_buff *skb;
+	int ret;
+
+	ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_WM_UNI_CMD_QUERY(EFUSE_CTRL),
+					&req, sizeof(req), true, &skb);
+	if (ret)
+		return ret;
+
+	res = (struct evt *)skb->data;
+	*val = res->data[offset % MT7925_EEPROM_BLOCK_SIZE];
+
+	dev_kfree_skb(skb);
+
+	return 0;
 }
 
 static int mt7925_load_clc(struct mt792x_dev *dev, const char *fw_name)
@@ -671,17 +684,26 @@ static int mt7925_load_clc(struct mt792x_dev *dev, const char *fw_name)
 	struct mt76_dev *mdev = &dev->mt76;
 	struct mt792x_phy *phy = &dev->phy;
 	const struct firmware *fw;
-	int ret, i, len, offset = 0;
 #if defined(__linux__)
-	u8 *clc_base = NULL;
+	u8 *clc_base = NULL, hw_encap = 0;
 #elif defined(__FreeBSD__)
 	const u8 *clc_base = NULL;
+	u8 hw_encap = 0;
 #endif
+	int ret, i, len, offset = 0;
 
+	dev->phy.clc_chan_conf = 0xff;
 	if (mt7925_disable_clc ||
 	    mt76_is_usb(&dev->mt76))
 		return 0;
 
+	if (mt76_is_mmio(&dev->mt76)) {
+		ret = mt7925_mcu_read_eeprom(dev, MT_EE_HW_TYPE, &hw_encap);
+		if (ret)
+			return ret;
+		hw_encap = u8_get_bits(hw_encap, MT_EE_HW_TYPE_ENCAP);
+	}
+
 	ret = request_firmware(&fw, fw_name, mdev->dev);
 	if (ret)
 		return ret;
@@ -730,6 +752,12 @@ static int mt7925_load_clc(struct mt792x_dev *dev, const char *fw_name)
 		if (phy->clc[clc->idx])
 			continue;
 
+		/* header content sanity */
+		if ((clc->idx == MT792x_CLC_BE_CTRL &&
+		     u8_get_bits(clc->t2.type, MT_EE_HW_TYPE_ENCAP) != hw_encap) ||
+		    u8_get_bits(clc->t0.type, MT_EE_HW_TYPE_ENCAP) != hw_encap)
+			continue;
+
 		phy->clc[clc->idx] = devm_kmemdup(mdev->dev, clc,
 						  le32_to_cpu(clc->len),
 						  GFP_KERNEL);
@@ -765,7 +793,7 @@ int mt7925_mcu_fw_log_2_host(struct mt792x_dev *dev, u8 ctrl)
 	int ret;
 
 	ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_UNI_CMD(WSYS_CONFIG),
-					&req, sizeof(req), false, NULL);
+					&req, sizeof(req), true, NULL);
 	return ret;
 }
 
@@ -836,7 +864,6 @@ mt7925_mcu_parse_phy_cap(struct mt792x_dev *dev, char *data)
 	mdev->phy.chainmask = mdev->phy.antenna_mask;
 	mdev->phy.cap.has_2ghz = cap->hw_path & BIT(WF0_24G);
 	mdev->phy.cap.has_5ghz = cap->hw_path & BIT(WF0_5G);
-	dev->has_eht = cap->eht;
 }
 
 static void
@@ -957,6 +984,23 @@ int mt7925_mcu_set_deep_sleep(struct mt792x_dev *dev, bool enable)
 }
 EXPORT_SYMBOL_GPL(mt7925_mcu_set_deep_sleep);
 
+int mt7925_mcu_set_thermal_protect(struct mt792x_dev *dev)
+{
+	char cmd[64];
+	int ret = 0;
+
+	snprintf(cmd, sizeof(cmd), "ThermalProtGband %d %d %d %d %d %d %d %d %d %d",
+		 0, 100, 90, 80, 30, 1, 1, 115, 105, 5);
+	ret = mt7925_mcu_chip_config(dev, cmd);
+
+	snprintf(cmd, sizeof(cmd), "ThermalProtAband %d %d %d %d %d %d %d %d %d %d",
+		 1, 100, 90, 80, 30, 1, 1, 115, 105, 5);
+	ret |= mt7925_mcu_chip_config(dev, cmd);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(mt7925_mcu_set_thermal_protect);
+
 int mt7925_run_firmware(struct mt792x_dev *dev)
 {
 	int err;
@@ -1407,7 +1451,7 @@ int mt7925_mcu_set_eeprom(struct mt792x_dev *dev)
 	};
 
 	return mt76_mcu_send_and_get_msg(&dev->mt76, MCU_UNI_CMD(EFUSE_CTRL),
-					 &req, sizeof(req), false, NULL);
+					 &req, sizeof(req), true, NULL);
 }
 EXPORT_SYMBOL_GPL(mt7925_mcu_set_eeprom);
 
@@ -1799,13 +1843,13 @@ mt7925_mcu_sta_eht_mld_tlv(struct sk_buff *skb,
 	struct tlv *tlv;
 	u16 eml_cap;
 
+	if (!ieee80211_vif_is_mld(vif))
+		return;
+
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_EHT_MLD, sizeof(*eht_mld));
 	eht_mld = (struct sta_rec_eht_mld *)tlv;
 	eht_mld->mld_type = 0xff;
 
-	if (!ieee80211_vif_is_mld(vif))
-		return;
-
 	ext_capa = cfg80211_get_iftype_ext_capa(wiphy,
 						ieee80211_vif_type_p2p(vif));
 	if (!ext_capa)
@@ -1858,49 +1902,6 @@ mt7925_mcu_sta_mld_tlv(struct sk_buff *skb,
 	}
 }
 
-static int
-mt7925_mcu_sta_cmd(struct mt76_phy *phy,
-		   struct mt76_sta_cmd_info *info)
-{
-	struct mt76_vif_link *mvif = (struct mt76_vif_link *)info->vif->drv_priv;
-	struct mt76_dev *dev = phy->dev;
-	struct sk_buff *skb;
-	int conn_state;
-
-	skb = __mt76_connac_mcu_alloc_sta_req(dev, mvif, info->wcid,
-					      MT7925_STA_UPDATE_MAX_SIZE);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	conn_state = info->enable ? CONN_STATE_PORT_SECURE :
-				    CONN_STATE_DISCONNECT;
-	if (info->link_sta)
-		mt76_connac_mcu_sta_basic_tlv(dev, skb, info->link_conf,
-					      info->link_sta,
-					      conn_state, info->newly);
-	if (info->link_sta && info->enable) {
-		mt7925_mcu_sta_phy_tlv(skb, info->vif, info->link_sta);
-		mt7925_mcu_sta_ht_tlv(skb, info->link_sta);
-		mt7925_mcu_sta_vht_tlv(skb, info->link_sta);
-		mt76_connac_mcu_sta_uapsd(skb, info->vif, info->link_sta->sta);
-		mt7925_mcu_sta_amsdu_tlv(skb, info->vif, info->link_sta);
-		mt7925_mcu_sta_he_tlv(skb, info->link_sta);
-		mt7925_mcu_sta_he_6g_tlv(skb, info->link_sta);
-		mt7925_mcu_sta_eht_tlv(skb, info->link_sta);
-		mt7925_mcu_sta_rate_ctrl_tlv(skb, info->vif,
-					     info->link_sta);
-		mt7925_mcu_sta_state_v2_tlv(phy, skb, info->link_sta,
-					    info->vif, info->rcpi,
-					    info->state);
-		mt7925_mcu_sta_mld_tlv(skb, info->vif, info->link_sta->sta);
-	}
-
-	if (info->enable)
-		mt7925_mcu_sta_hdr_trans_tlv(skb, info->vif, info->link_sta);
-
-	return mt76_mcu_skb_send_msg(dev, skb, info->cmd, true);
-}
-
 static void
 mt7925_mcu_sta_remove_tlv(struct sk_buff *skb)
 {
@@ -1913,13 +1914,14 @@ mt7925_mcu_sta_remove_tlv(struct sk_buff *skb)
 }
 
 static int
-mt7925_mcu_mlo_sta_cmd(struct mt76_phy *phy,
-		       struct mt76_sta_cmd_info *info)
+mt7925_mcu_sta_cmd(struct mt76_phy *phy,
+		   struct mt76_sta_cmd_info *info)
 {
 	struct mt792x_vif *mvif = (struct mt792x_vif *)info->vif->drv_priv;
 	struct mt76_dev *dev = phy->dev;
 	struct mt792x_bss_conf *mconf;
 	struct sk_buff *skb;
+	int conn_state;
 
 	mconf = mt792x_vif_to_link(mvif, info->wcid->link_id);
 
@@ -1928,12 +1930,13 @@ mt7925_mcu_mlo_sta_cmd(struct mt76_phy *phy,
 	if (IS_ERR(skb))
 		return PTR_ERR(skb);
 
-	if (info->enable)
-		mt76_connac_mcu_sta_basic_tlv(dev, skb, info->link_conf,
-					      info->link_sta,
-					      info->enable, info->newly);
+	conn_state = info->enable ? CONN_STATE_PORT_SECURE :
+				    CONN_STATE_DISCONNECT;
 
 	if (info->enable && info->link_sta) {
+		mt76_connac_mcu_sta_basic_tlv(dev, skb, info->link_conf,
+					      info->link_sta,
+					      conn_state, info->newly);
 		mt7925_mcu_sta_phy_tlv(skb, info->vif, info->link_sta);
 		mt7925_mcu_sta_ht_tlv(skb, info->link_sta);
 		mt7925_mcu_sta_vht_tlv(skb, info->link_sta);
@@ -1952,14 +1955,14 @@ mt7925_mcu_mlo_sta_cmd(struct mt76_phy *phy,
 			mt7925_mcu_sta_mld_tlv(skb, info->vif, info->link_sta->sta);
 			mt7925_mcu_sta_eht_mld_tlv(skb, info->vif, info->link_sta->sta);
 		}
-
-		mt7925_mcu_sta_hdr_trans_tlv(skb, info->vif, info->link_sta);
 	}
 
 	if (!info->enable) {
 		mt7925_mcu_sta_remove_tlv(skb);
 		mt76_connac_mcu_add_tlv(skb, STA_REC_MLD_OFF,
 					sizeof(struct tlv));
+	} else {
+		mt7925_mcu_sta_hdr_trans_tlv(skb, info->vif, info->link_sta);
 	}
 
 	return mt76_mcu_skb_send_msg(dev, skb, info->cmd, true);
@@ -1984,25 +1987,15 @@ int mt7925_mcu_sta_update(struct mt792x_dev *dev,
 	};
 	struct mt792x_sta *msta;
 	struct mt792x_link_sta *mlink;
-	int err;
 
 	if (link_sta) {
 		msta = (struct mt792x_sta *)link_sta->sta->drv_priv;
 		mlink = mt792x_sta_to_link(msta, link_sta->link_id);
 	}
 	info.wcid = link_sta ? &mlink->wcid : &mvif->sta.deflink.wcid;
+	info.newly = state != MT76_STA_INFO_STATE_ASSOC;
 
-	if (link_sta)
-		info.newly = state != MT76_STA_INFO_STATE_ASSOC;
-	else
-		info.newly = state == MT76_STA_INFO_STATE_ASSOC ? false : true;
-
-	if (ieee80211_vif_is_mld(vif))
-		err = mt7925_mcu_mlo_sta_cmd(&dev->mphy, &info);
-	else
-		err = mt7925_mcu_sta_cmd(&dev->mphy, &info);
-
-	return err;
+	return mt7925_mcu_sta_cmd(&dev->mphy, &info);
 }
 
 int mt7925_mcu_set_beacon_filter(struct mt792x_dev *dev,
@@ -2084,8 +2077,6 @@ int mt7925_mcu_set_sniffer(struct mt792x_dev *dev, struct ieee80211_vif *vif,
 		},
 	};
 
-	mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(SNIFFER), &req, sizeof(req), true);
-
 	return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(SNIFFER), &req, sizeof(req),
 				 true);
 }
@@ -2336,6 +2327,40 @@ __mt7925_mcu_alloc_bss_req(struct mt76_dev *dev, struct mt76_vif_link *mvif, int
 	return skb;
 }
 
+static
+void mt7925_mcu_bss_eht_tlv(struct sk_buff *skb, struct mt76_phy *phy,
+			    struct ieee80211_bss_conf *link_conf,
+			    struct ieee80211_chanctx_conf *ctx)
+{
+	struct cfg80211_chan_def *chandef = ctx ? &ctx->def :
+						  &link_conf->chanreq.oper;
+
+	struct bss_eht_tlv *req;
+	struct tlv *tlv;
+
+	tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_EHT, sizeof(*req));
+	req = (struct bss_eht_tlv *)tlv;
+	req->is_eth_dscb_present = chandef->punctured ? 1 : 0;
+	req->eht_dis_sub_chan_bitmap = cpu_to_le16(chandef->punctured);
+}
+
+int mt7925_mcu_set_eht_pp(struct mt76_phy *phy, struct mt76_vif_link *mvif,
+			  struct ieee80211_bss_conf *link_conf,
+			  struct ieee80211_chanctx_conf *ctx)
+{
+	struct sk_buff *skb;
+
+	skb = __mt7925_mcu_alloc_bss_req(phy->dev, mvif,
+					 MT7925_BSS_UPDATE_MAX_SIZE);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	mt7925_mcu_bss_eht_tlv(skb, phy, link_conf, ctx);
+
+	return mt76_mcu_skb_send_msg(phy->dev, skb,
+				     MCU_UNI_CMD(BSS_INFO_UPDATE), true);
+}
+
 int mt7925_mcu_set_chctx(struct mt76_phy *phy, struct mt76_vif_link *mvif,
 			 struct ieee80211_bss_conf *link_conf,
 			 struct ieee80211_chanctx_conf *ctx)
@@ -2567,6 +2592,7 @@ mt7925_mcu_bss_mld_tlv(struct sk_buff *skb,
 	struct ieee80211_vif *vif = link_conf->vif;
 	struct mt792x_bss_conf *mconf = mt792x_link_conf_to_mconf(link_conf);
 	struct mt792x_vif *mvif = (struct mt792x_vif *)link_conf->vif->drv_priv;
+	struct mt792x_phy *phy = mvif->phy;
 	struct bss_mld_tlv *mld;
 	struct tlv *tlv;
 	bool is_mld;
@@ -2582,8 +2608,13 @@ mt7925_mcu_bss_mld_tlv(struct sk_buff *skb,
 	mld->group_mld_id = is_mld ? mvif->bss_conf.mt76.idx : 0xff;
 	mld->own_mld_id = mconf->mt76.idx + 32;
 	mld->remap_idx = 0xff;
-	mld->eml_enable = !!(link_conf->vif->cfg.eml_cap &
-			     IEEE80211_EML_CAP_EMLSR_SUPP);
+
+	if (phy->chip_cap & MT792x_CHIP_CAP_MLO_EML_EN) {
+		mld->eml_enable = !!(link_conf->vif->cfg.eml_cap &
+				     IEEE80211_EML_CAP_EMLSR_SUPP);
+	} else {
+		mld->eml_enable = 0;
+	}
 
 	memcpy(mld->mac_addr, vif->addr, ETH_ALEN);
 }
@@ -2676,6 +2707,62 @@ int mt7925_mcu_set_timing(struct mt792x_phy *phy,
 				     MCU_UNI_CMD(BSS_INFO_UPDATE), true);
 }
 
+void mt7925_mcu_del_dev(struct mt76_dev *mdev,
+			struct ieee80211_vif *vif)
+{
+	struct mt76_vif_link *mvif = (struct mt76_vif_link *)vif->drv_priv;
+	struct {
+		struct {
+			u8 omac_idx;
+			u8 band_idx;
+			__le16 pad;
+		} __packed hdr;
+		struct req_tlv {
+			__le16 tag;
+			__le16 len;
+			u8 active;
+			u8 link_idx; /* hw link idx */
+			u8 omac_addr[ETH_ALEN];
+		} __packed tlv;
+	} dev_req = {
+		.tlv = {
+			.tag = cpu_to_le16(DEV_INFO_ACTIVE),
+			.len = cpu_to_le16(sizeof(struct req_tlv)),
+			.active = true,
+		},
+	};
+	struct {
+		struct {
+			u8 bss_idx;
+			u8 pad[3];
+		} __packed hdr;
+		struct mt76_connac_bss_basic_tlv basic;
+	} basic_req = {
+		.basic = {
+			.tag = cpu_to_le16(UNI_BSS_INFO_BASIC),
+			.len = cpu_to_le16(sizeof(struct mt76_connac_bss_basic_tlv)),
+			.active = true,
+			.conn_state = 1,
+		},
+	};
+
+	dev_req.hdr.omac_idx = mvif->omac_idx;
+	dev_req.hdr.band_idx = mvif->band_idx;
+
+	basic_req.hdr.bss_idx = mvif->idx;
+	basic_req.basic.omac_idx = mvif->omac_idx;
+	basic_req.basic.band_idx = mvif->band_idx;
+	basic_req.basic.link_idx = mvif->link_idx;
+
+	mt76_mcu_send_msg(mdev, MCU_UNI_CMD(BSS_INFO_UPDATE),
+			  &basic_req, sizeof(basic_req), true);
+
+	/* recovery omac address for the legacy interface */
+	memcpy(dev_req.tlv.omac_addr, vif->addr, ETH_ALEN);
+	mt76_mcu_send_msg(mdev, MCU_UNI_CMD(DEV_INFO_UPDATE),
+			  &dev_req, sizeof(dev_req), true);
+}
+
 int mt7925_mcu_add_bss_info(struct mt792x_phy *phy,
 			    struct ieee80211_chanctx_conf *ctx,
 			    struct ieee80211_bss_conf *link_conf,
@@ -2740,11 +2827,59 @@ int mt7925_mcu_set_dbdc(struct mt76_phy *phy, bool enable)
 	conf->band = 0; /* unused */
 
 	err = mt76_mcu_skb_send_msg(mdev, skb, MCU_UNI_CMD(SET_DBDC_PARMS),
-				    false);
+				    true);
 
 	return err;
 }
 
+static void
+mt7925_mcu_build_scan_ie_tlv(struct mt76_dev *mdev,
+			     struct sk_buff *skb,
+			     struct ieee80211_scan_ies *scan_ies)
+{
+	u32 max_len = sizeof(struct scan_ie_tlv) + MT76_CONNAC_SCAN_IE_LEN;
+	struct scan_ie_tlv *ie;
+	enum nl80211_band i;
+	struct tlv *tlv;
+	const u8 *ies;
+	u16 ies_len;
+
+	for (i = 0; i <= NL80211_BAND_6GHZ; i++) {
+		if (i == NL80211_BAND_60GHZ)
+			continue;
+
+		ies = scan_ies->ies[i];
+		ies_len = scan_ies->len[i];
+
+		if (!ies || !ies_len)
+			continue;
+
+		if (ies_len > max_len)
+			return;
+
+		tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_IE,
+					      sizeof(*ie) + ies_len);
+		ie = (struct scan_ie_tlv *)tlv;
+
+		memcpy(ie->ies, ies, ies_len);
+		ie->ies_len = cpu_to_le16(ies_len);
+
+		switch (i) {
+		case NL80211_BAND_2GHZ:
+			ie->band = 1;
+			break;
+		case NL80211_BAND_6GHZ:
+			ie->band = 3;
+			break;
+		default:
+			ie->band = 2;
+			break;
+		}
+
+		max_len -= (sizeof(*ie) + ies_len);
+	}
+}
+
 int mt7925_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 		       struct ieee80211_scan_request *scan_req)
 {
@@ -2755,7 +2890,6 @@ int mt7925_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 	struct mt76_dev *mdev = phy->dev;
 	struct mt76_connac_mcu_scan_channel *chan;
 	struct sk_buff *skb;
-
 	struct scan_hdr_tlv *hdr;
 	struct scan_req_tlv *req;
 	struct scan_ssid_tlv *ssid;
@@ -2766,9 +2900,13 @@ int mt7925_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 	struct tlv *tlv;
 	int max_len;
 
+	if (test_bit(MT76_HW_SCANNING, &phy->state))
+		return -EBUSY;
+
 	max_len = sizeof(*hdr) + sizeof(*req) + sizeof(*ssid) +
-				sizeof(*bssid) + sizeof(*chan_info) +
-				sizeof(*misc) + sizeof(*ie);
+		  sizeof(*bssid) * MT7925_RNR_SCAN_MAX_BSSIDS +
+		  sizeof(*chan_info) + sizeof(*misc) + sizeof(*ie) +
+		  MT76_CONNAC_SCAN_IE_LEN;
 
 	skb = mt76_mcu_msg_alloc(mdev, NULL, max_len);
 	if (!skb)
@@ -2791,19 +2929,42 @@ int mt7925_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 	for (i = 0; i < sreq->n_ssids; i++) {
 		if (!sreq->ssids[i].ssid_len)
 			continue;
+		if (i >= MT7925_RNR_SCAN_MAX_BSSIDS)
+			break;
 
-		ssid->ssids[i].ssid_len = cpu_to_le32(sreq->ssids[i].ssid_len);
-		memcpy(ssid->ssids[i].ssid, sreq->ssids[i].ssid,
+		ssid->ssids[n_ssids].ssid_len = cpu_to_le32(sreq->ssids[i].ssid_len);
+		memcpy(ssid->ssids[n_ssids].ssid, sreq->ssids[i].ssid,
 		       sreq->ssids[i].ssid_len);
 		n_ssids++;
 	}
 	ssid->ssid_type = n_ssids ? BIT(2) : BIT(0);
 	ssid->ssids_num = n_ssids;
 
-	tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_BSSID, sizeof(*bssid));
-	bssid = (struct scan_bssid_tlv *)tlv;
+	if (sreq->n_6ghz_params) {
+		u8 j;
+
+		mt76_connac_mcu_build_rnr_scan_param(mdev, sreq);
+
+		for (j = 0; j < mdev->rnr.bssid_num; j++) {
+			if (j >= MT7925_RNR_SCAN_MAX_BSSIDS)
+				break;
+
+			tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_BSSID,
+						      sizeof(*bssid));
+			bssid = (struct scan_bssid_tlv *)tlv;
+
+			ether_addr_copy(bssid->bssid, mdev->rnr.bssid[j]);
+			bssid->match_ch = mdev->rnr.channel[j];
+			bssid->match_ssid_ind = MT7925_RNR_SCAN_MAX_BSSIDS;
+			bssid->match_short_ssid_ind = MT7925_RNR_SCAN_MAX_BSSIDS;
+		}
+		req->scan_func |= SCAN_FUNC_RNR_SCAN;
+	} else {
+		tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_BSSID, sizeof(*bssid));
+		bssid = (struct scan_bssid_tlv *)tlv;
 
-	memcpy(bssid->bssid, sreq->bssid, ETH_ALEN);
+		ether_addr_copy(bssid->bssid, sreq->bssid);
+	}
 
 	tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_CHANNEL, sizeof(*chan_info));
 	chan_info = (struct scan_chan_info_tlv *)tlv;
@@ -2827,13 +2988,6 @@ int mt7925_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 	}
 	chan_info->channel_type = sreq->n_channels ? 4 : 0;
 
-	tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_IE, sizeof(*ie));
-	ie = (struct scan_ie_tlv *)tlv;
-	if (sreq->ie_len > 0) {
-		memcpy(ie->ies, sreq->ie, sreq->ie_len);
-		ie->ies_len = cpu_to_le16(sreq->ie_len);
-	}
-
 	req->scan_func |= SCAN_FUNC_SPLIT_SCAN;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_MISC, sizeof(*misc));
@@ -2844,8 +2998,11 @@ int mt7925_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 		req->scan_func |= SCAN_FUNC_RANDOM_MAC;
 	}
 
+	/* Append scan probe IEs as the last tlv */
+	mt7925_mcu_build_scan_ie_tlv(mdev, skb, &scan_req->ies);
+
 	err = mt76_mcu_skb_send_msg(mdev, skb, MCU_UNI_CMD(SCAN_REQ),
-				    false);
+				    true);
 	if (err < 0)
 		clear_bit(MT76_HW_SCANNING, &phy->state);
 
@@ -2855,7 +3012,8 @@ EXPORT_SYMBOL_GPL(mt7925_mcu_hw_scan);
 
 int mt7925_mcu_sched_scan_req(struct mt76_phy *phy,
 			      struct ieee80211_vif *vif,
-			      struct cfg80211_sched_scan_request *sreq)
+			      struct cfg80211_sched_scan_request *sreq,
+			      struct ieee80211_scan_ies *ies)
 {
 	struct mt76_vif_link *mvif = (struct mt76_vif_link *)vif->drv_priv;
 	struct ieee80211_channel **scan_list = sreq->channels;
@@ -2943,15 +3101,11 @@ int mt7925_mcu_sched_scan_req(struct mt76_phy *phy,
 	}
 	chan_info->channel_type = sreq->n_channels ? 4 : 0;
 
-	tlv = mt76_connac_mcu_add_tlv(skb, UNI_SCAN_IE, sizeof(*ie));
-	ie = (struct scan_ie_tlv *)tlv;
-	if (sreq->ie_len > 0) {
-		memcpy(ie->ies, sreq->ie, sreq->ie_len);
-		ie->ies_len = cpu_to_le16(sreq->ie_len);
-	}
+	/* Append scan probe IEs as the last tlv */
+	mt7925_mcu_build_scan_ie_tlv(mdev, skb, ies);
 
 	return mt76_mcu_skb_send_msg(mdev, skb, MCU_UNI_CMD(SCAN_REQ),
-				     false);
+				     true);
 }
 EXPORT_SYMBOL_GPL(mt7925_mcu_sched_scan_req);
 
@@ -2987,7 +3141,7 @@ mt7925_mcu_sched_scan_enable(struct mt76_phy *phy,
 		clear_bit(MT76_HW_SCHED_SCANNING, &phy->state);
 
 	return mt76_mcu_skb_send_msg(mdev, skb, MCU_UNI_CMD(SCAN_REQ),
-				     false);
+				     true);
 }
 
 int mt7925_mcu_cancel_hw_scan(struct mt76_phy *phy,
@@ -3026,7 +3180,7 @@ int mt7925_mcu_cancel_hw_scan(struct mt76_phy *phy,
 	}
 
 	return mt76_mcu_send_msg(phy->dev, MCU_UNI_CMD(SCAN_REQ),
-				 &req, sizeof(req), false);
+				 &req, sizeof(req), true);
 }
 EXPORT_SYMBOL_GPL(mt7925_mcu_cancel_hw_scan);
 
@@ -3131,7 +3285,7 @@ int mt7925_mcu_set_channel_domain(struct mt76_phy *phy)
 	memcpy(__skb_push(skb, sizeof(req)), &req, sizeof(req));
 
 	return mt76_mcu_skb_send_msg(dev, skb, MCU_UNI_CMD(SET_DOMAIN_INFO),
-				     false);
+				     true);
 }
 EXPORT_SYMBOL_GPL(mt7925_mcu_set_channel_domain);
 
@@ -3163,16 +3317,17 @@ __mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2,
 
 		.idx = idx,
 		.env = env_cap,
-		.acpi_conf = mt792x_acpi_get_flags(&dev->phy),
 	};
 	int ret, valid_cnt = 0;
-	u8 i, *pos;
+	u8 *pos, *last_pos;
 
 	if (!clc)
 		return 0;
 
-	pos = clc->data + sizeof(*seg) * clc->nr_seg;
-	for (i = 0; i < clc->nr_country; i++) {
+	req.ver = clc->ver;
+	pos = clc->data + sizeof(*seg) * clc->t0.nr_seg;
+	last_pos = clc->data + le32_to_cpu(*(__le32 *)(clc->data + 4));
+	while (pos < last_pos) {
 		struct mt7925_clc_rule *rule = (struct mt7925_clc_rule *)pos;
 
 		pos += sizeof(*rule);
@@ -3187,6 +3342,7 @@ __mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2,
 		memcpy(req.type, rule->type, 2);
 
 		req.size = cpu_to_le16(seg->len);
+		dev->phy.clc_chan_conf = clc->ver == 1 ? 0xff : rule->flag;
 		skb = __mt76_mcu_msg_alloc(&dev->mt76, &req,
 					   le16_to_cpu(req.size) + sizeof(req),
 					   sizeof(req), GFP_KERNEL);
@@ -3202,8 +3358,10 @@ __mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2,
 		valid_cnt++;
 	}
 
-	if (!valid_cnt)
+	if (!valid_cnt) {
+		dev->phy.clc_chan_conf = 0xff;
 		return -ENOENT;
+	}
 
 	return 0;
 }
@@ -3216,6 +3374,9 @@ int mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2,
 
 	/* submit all clc config */
 	for (i = 0; i < ARRAY_SIZE(phy->clc); i++) {
+		if (i == MT792x_CLC_BE_CTRL)
+			continue;
+
 		ret = __mt7925_mcu_set_clc(dev, alpha2, env_cap,
 					   phy->clc[i], i);
 
@@ -3274,6 +3435,18 @@ int mt7925_mcu_fill_message(struct mt76_dev *mdev, struct sk_buff *skb,
 		else
 			uni_txd->option = MCU_CMD_UNI_EXT_ACK;
 
+		if (cmd == MCU_UNI_CMD(HIF_CTRL) ||
+		    cmd == MCU_UNI_CMD(CHIP_CONFIG))
+			uni_txd->option &= ~MCU_CMD_ACK;
+
+		if (mcu_cmd == MCU_UNI_CMD_TESTMODE_CTRL ||
+		    mcu_cmd == MCU_UNI_CMD_TESTMODE_RX_STAT) {
+			if (cmd & __MCU_CMD_FIELD_QUERY)
+				uni_txd->option = 0x2;
+			else
+				uni_txd->option = 0x6;
+		}
+
 		goto exit;
 	}
 
@@ -3565,6 +3738,43 @@ int mt7925_mcu_set_rate_txpower(struct mt76_phy *phy)
 	return 0;
 }
 
+int mt7925_mcu_wf_rf_pin_ctrl(struct mt792x_phy *phy)
+{
+#define UNI_CMD_RADIO_STATUS_GET	0
+	struct mt792x_dev *dev = phy->dev;
+	struct sk_buff *skb;
+	int ret;
+	struct {
+		__le16 tag;
+		__le16 len;
+		u8 rsv[4];
+	} __packed req = {
+		.tag = UNI_CMD_RADIO_STATUS_GET,
+		.len = cpu_to_le16(sizeof(req)),
+	};
+	struct mt7925_radio_status_event {
+		__le16 tag;
+		__le16 len;
+
+		u8 data;
+		u8 rsv[3];
+	} __packed *status;
+
+	ret = mt76_mcu_send_and_get_msg(&dev->mt76,
+					MCU_UNI_CMD(RADIO_STATUS),
+					&req, sizeof(req), true, &skb);
+	if (ret)
+		return ret;
+
+	skb_pull(skb, sizeof(struct tlv));
+	status = (struct mt7925_radio_status_event *)skb->data;
+	ret = status->data;
+
+	dev_kfree_skb(skb);
+
+	return ret;
+}
+
 int mt7925_mcu_set_rxfilter(struct mt792x_dev *dev, u32 fif,
 			    u8 bit_op, u32 bit_map)
 {
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h
index 1e47d2c61b54..a40764d89a1f 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/mcu.h
@@ -104,13 +104,6 @@ enum {
 	MT7925_TM_WIFISPECTRUM,
 };
 
-struct mt7925_rftest_cmd {
-	u8 action;
-	u8 rsv[3];
-	__le32 param0;
-	__le32 param1;
-} __packed;
-
 struct mt7925_rftest_evt {
 	__le32 param0;
 	__le32 param1;
@@ -196,6 +189,7 @@ enum {
 	UNI_SNIFFER_CONFIG,
 };
 
+#define MT7925_RNR_SCAN_MAX_BSSIDS	10
 struct scan_hdr_tlv {
 	/* fixed field */
 	u8 seq_num;
@@ -223,7 +217,7 @@ struct scan_req_tlv {
 	__le16 timeout_value;
 	__le16 probe_delay_time;
 	__le32 func_mask_ext;
-};
+} __packed;
 
 struct scan_ssid_tlv {
 	__le16 tag;
@@ -235,9 +229,10 @@ struct scan_ssid_tlv {
 		       * BIT(2) + ssid_type_ext BIT(0) specified SSID only
 		       */
 	u8 ssids_num;
-	u8 pad[2];
-	struct mt76_connac_mcu_scan_ssid ssids[4];
-};
+	u8 is_short_ssid;
+	u8 pad;
+	struct mt76_connac_mcu_scan_ssid ssids[MT7925_RNR_SCAN_MAX_BSSIDS];
+} __packed;
 
 struct scan_bssid_tlv {
 	__le16 tag;
@@ -247,8 +242,9 @@ struct scan_bssid_tlv {
 	u8 match_ch;
 	u8 match_ssid_ind;
 	u8 rcpi;
-	u8 pad[3];
-};
+	u8 match_short_ssid_ind;
+	u8 pad[2];
+} __packed;
 
 struct scan_chan_info_tlv {
 	__le16 tag;
@@ -264,7 +260,7 @@ struct scan_chan_info_tlv {
 	u8 channels_num; /* valid when channel_type is 4 */
 	u8 pad[2];
 	struct mt76_connac_mcu_scan_channel channels[64];
-};
+} __packed;
 
 struct scan_ie_tlv {
 	__le16 tag;
@@ -273,7 +269,7 @@ struct scan_ie_tlv {
 	__le16 ies_len;
 	u8 band;
 	u8 pad;
-	u8 ies[MT76_CONNAC_SCAN_IE_LEN];
+	u8 ies[];
 };
 
 struct scan_misc_tlv {
@@ -372,6 +368,19 @@ struct bss_mld_tlv {
 	u8 __rsv[3];
 } __packed;
 
+struct bss_eht_tlv {
+	__le16 tag;
+	__le16 len;
+	u8  is_eht_op_present;
+	u8  is_eth_dscb_present;
+	u8  eht_ctrl;
+	u8  eht_ccfs0;
+	u8  eht_ccfs1;
+	u8  pad1;
+	__le16 eht_dis_sub_chan_bitmap;
+	u8  pad2[4];
+} __packed;
+
 struct sta_rec_ba_uni {
 	__le16 tag;
 	__le16 len;
@@ -566,8 +575,8 @@ struct mt7925_wow_pattern_tlv {
 	u8 offset;
 	u8 mask[MT76_CONNAC_WOW_MASK_MAX_LEN];
 	u8 pattern[MT76_CONNAC_WOW_PATTEN_MAX_LEN];
-	u8 rsv[7];
-} __packed;
+	u8 rsv[4];
+};
 
 struct roc_acquire_tlv {
 	__le16 tag;
@@ -589,6 +598,47 @@ struct roc_acquire_tlv {
 	u8 rsv[3];
 } __packed;
 
+enum ENUM_CMD_TEST_CTRL_ACT {
+	CMD_TEST_CTRL_ACT_SWITCH_MODE = 0,
+	CMD_TEST_CTRL_ACT_SET_AT = 1,
+	CMD_TEST_CTRL_ACT_GET_AT = 2,
+	CMD_TEST_CTRL_ACT_SET_AT_ENG = 3,
+	CMD_TEST_CTRL_ACT_GET_AT_ENG = 4,
+	CMD_TEST_CTRL_ACT_NUM
+};
+
+enum ENUM_CMD_TEST_CTRL_ACT_SWITCH_MODE_OP {
+	CMD_TEST_CTRL_ACT_SWITCH_MODE_NORMAL = 0,
+	CMD_TEST_CTRL_ACT_SWITCH_MODE_RF_TEST = 1,
+	CMD_TEST_CTRL_ACT_SWITCH_MODE_ICAP = 2,
+	CMD_TEST_CTRL_ACT_SWITCH_MODE_NUM
+};
+
+union testmode_data {
+	__le32 op_mode;
+	__le32 channel_freq;
+	u8 rf_at_info[84];
+};
+
+union testmode_evt {
+	__le32 op_mode;
+	__le32 channel_freq;
+	u8 rf_at_info[1024];
+};
+
+struct uni_cmd_testmode_ctrl {
+	u16 tag;
+	u16 length;
+	u8 action;
+	u8 reserved[3];
+	union testmode_data data;
+} __packed;
+
+struct mt7925_rftest_cmd {
+	u8 padding[4];
+	struct uni_cmd_testmode_ctrl ctrl;
+} __packed;
+
 static inline enum connac3_mcu_cipher_type
 mt7925_mcu_get_cipher(int cipher)
 {
@@ -623,10 +673,13 @@ int mt7925_mcu_cancel_hw_scan(struct mt76_phy *phy,
 			      struct ieee80211_vif *vif);
 int mt7925_mcu_sched_scan_req(struct mt76_phy *phy,
 			      struct ieee80211_vif *vif,
-			      struct cfg80211_sched_scan_request *sreq);
+			      struct cfg80211_sched_scan_request *sreq,
+			      struct ieee80211_scan_ies *ies);
 int mt7925_mcu_sched_scan_enable(struct mt76_phy *phy,
 				 struct ieee80211_vif *vif,
 				 bool enable);
+void mt7925_mcu_del_dev(struct mt76_dev *mdev,
+			struct ieee80211_vif *vif);
 int mt7925_mcu_add_bss_info(struct mt792x_phy *phy,
 			    struct ieee80211_chanctx_conf *ctx,
 			    struct ieee80211_bss_conf *link_conf,
@@ -635,11 +688,15 @@ int mt7925_mcu_add_bss_info(struct mt792x_phy *phy,
 int mt7925_mcu_set_timing(struct mt792x_phy *phy,
 			  struct ieee80211_bss_conf *link_conf);
 int mt7925_mcu_set_deep_sleep(struct mt792x_dev *dev, bool enable);
+int mt7925_mcu_set_thermal_protect(struct mt792x_dev *dev);
 int mt7925_mcu_set_channel_domain(struct mt76_phy *phy);
 int mt7925_mcu_set_radio_en(struct mt792x_phy *phy, bool enable);
 int mt7925_mcu_set_chctx(struct mt76_phy *phy, struct mt76_vif_link *mvif,
 			 struct ieee80211_bss_conf *link_conf,
 			 struct ieee80211_chanctx_conf *ctx);
+int mt7925_mcu_set_eht_pp(struct mt76_phy *phy, struct mt76_vif_link *mvif,
+			  struct ieee80211_bss_conf *link_conf,
+			  struct ieee80211_chanctx_conf *ctx);
 int mt7925_mcu_set_rate_txpower(struct mt76_phy *phy);
 int mt7925_mcu_update_arp_filter(struct mt76_dev *dev,
 				 struct ieee80211_bss_conf *link_conf);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h b/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h
index 8707b5d04743..1b165d0d8bd3 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/mt7925.h
@@ -137,11 +137,18 @@ enum {
 	MT7925_CLC_MAX_NUM,
 };
 
+struct mt7925_clc_rule_v2 {
+	u32 flag;
+	u8 alpha2[2];
+	u8 rsv[10];
+} __packed;
+
 struct mt7925_clc_rule {
 	u8 alpha2[2];
 	u8 type[2];
 	u8 seg_idx;
-	u8 rsv[3];
+	u8 flag; /* UNII4~8 ctrl flag */
+	u8 rsv[2];
 } __packed;
 
 struct mt7925_clc_segment {
@@ -152,14 +159,26 @@ struct mt7925_clc_segment {
 	u8 rsv2[4];
 } __packed;
 
-struct mt7925_clc {
-	__le32 len;
-	u8 idx;
-	u8 ver;
+struct mt7925_clc_type0 {
 	u8 nr_country;
 	u8 type;
 	u8 nr_seg;
 	u8 rsv[7];
+} __packed;
+
+struct mt7925_clc_type2 {
+	u8 type;
+	u8 rsv[9];
+} __packed;
+
+struct mt7925_clc {
+	__le32 len;
+	u8 idx;
+	u8 ver;
+	union {
+		struct mt7925_clc_type0 t0;
+		struct mt7925_clc_type2 t2;
+	};
 	u8 data[];
 } __packed;
 
@@ -167,9 +186,12 @@ enum mt7925_eeprom_field {
 	MT_EE_CHIP_ID =		0x000,
 	MT_EE_VERSION =		0x002,
 	MT_EE_MAC_ADDR =	0x004,
+	MT_EE_HW_TYPE =		0xa71,
 	__MT_EE_MAX =		0x9ff
 };
 
+#define MT_EE_HW_TYPE_ENCAP     GENMASK(1, 0)
+
 enum {
 	TXPWR_USER,
 	TXPWR_EEPROM,
@@ -235,6 +257,7 @@ int mt7925_mcu_chip_config(struct mt792x_dev *dev, const char *cmd);
 int mt7925_mcu_set_rxfilter(struct mt792x_dev *dev, u32 fif,
 			    u8 bit_op, u32 bit_map);
 
+void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2);
 void mt7925_regd_update(struct mt792x_dev *dev);
 int mt7925_mac_init(struct mt792x_dev *dev);
 int mt7925_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif,
@@ -263,13 +286,12 @@ int mt7925_mcu_set_beacon_filter(struct mt792x_dev *dev,
 				 struct ieee80211_vif *vif,
 				 bool enable);
 int mt7925_mcu_uni_tx_ba(struct mt792x_dev *dev,
-			 struct ieee80211_vif *vif,
 			 struct ieee80211_ampdu_params *params,
 			 bool enable);
 int mt7925_mcu_uni_rx_ba(struct mt792x_dev *dev,
-			 struct ieee80211_vif *vif,
 			 struct ieee80211_ampdu_params *params,
 			 bool enable);
+void mt7925_mlo_pm_work(struct work_struct *work);
 void mt7925_scan_work(struct work_struct *work);
 void mt7925_roc_work(struct work_struct *work);
 int mt7925_mcu_uni_bss_ps(struct mt792x_dev *dev,
@@ -343,5 +365,11 @@ int mt7925_mcu_wtbl_update_hdr_trans(struct mt792x_dev *dev,
 				     struct ieee80211_vif *vif,
 				     struct ieee80211_sta *sta,
 				     int link_id);
+int mt7925_mcu_wf_rf_pin_ctrl(struct mt792x_phy *phy);
+
+int mt7925_testmode_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			void *data, int len);
+int mt7925_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *msg,
+			 struct netlink_callback *cb, void *data, int len);
 
 #endif
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/pci.c b/sys/contrib/dev/mediatek/mt76/mt7925/pci.c
index e2fc3dee0698..3417fc5db6a9 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/pci.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/pci.c
@@ -31,6 +31,10 @@ static void mt7925e_unregister_device(struct mt792x_dev *dev)
 {
 	int i;
 	struct mt76_connac_pm *pm = &dev->pm;
+	struct ieee80211_hw *hw = mt76_hw(dev);
+
+	if (dev->phy.chip_cap & MT792x_CHIP_CAP_WF_RF_PIN_CTRL_EVT_EN)
+		wiphy_rfkill_stop_polling(hw->wiphy);
 
 	cancel_work_sync(&dev->init_work);
 	mt76_unregister_device(&dev->mt76);
@@ -491,9 +495,6 @@ static int mt7925_pci_suspend(struct device *device)
 
 	/* disable interrupt */
 	mt76_wr(dev, dev->irq_map->host_irq_enable, 0);
-	mt76_wr(dev, MT_WFDMA0_HOST_INT_DIS,
-		dev->irq_map->tx.all_complete_mask |
-		MT_INT_RX_DONE_ALL | MT_INT_MCU_CMD);
 
 	mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0x0);
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/regs.h b/sys/contrib/dev/mediatek/mt76/mt7925/regs.h
index 985794a40c1a..341987e47f67 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7925/regs.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/regs.h
@@ -28,7 +28,7 @@
 #define MT_MDP_TO_HIF			0
 #define MT_MDP_TO_WM			1
 
-#define MT_WFDMA0_HOST_INT_ENA		MT_WFDMA0(0x228)
+#define MT_WFDMA0_HOST_INT_ENA		MT_WFDMA0(0x204)
 #define MT_WFDMA0_HOST_INT_DIS		MT_WFDMA0(0x22c)
 #define HOST_RX_DONE_INT_ENA4		BIT(12)
 #define HOST_RX_DONE_INT_ENA5		BIT(13)
@@ -58,7 +58,7 @@
 
 #define MT_INT_TX_DONE_MCU		(MT_INT_TX_DONE_MCU_WM |	\
 					 MT_INT_TX_DONE_FWDL)
-#define MT_INT_TX_DONE_ALL		(MT_INT_TX_DONE_MCU_WM |	\
+#define MT_INT_TX_DONE_ALL		(MT_INT_TX_DONE_MCU |	\
 					 MT_INT_TX_DONE_BAND0 |	\
 					GENMASK(18, 4))
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7925/testmode.c b/sys/contrib/dev/mediatek/mt76/mt7925/testmode.c
new file mode 100644
index 000000000000..a3c97164ba21
--- /dev/null
+++ b/sys/contrib/dev/mediatek/mt76/mt7925/testmode.c
@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: ISC
+
+#include "mt7925.h"
+#include "mcu.h"
+
+#define MT7925_EVT_RSP_LEN 512
+
+enum mt7925_testmode_attr {
+	MT7925_TM_ATTR_UNSPEC,
+	MT7925_TM_ATTR_SET,
+	MT7925_TM_ATTR_QUERY,
+	MT7925_TM_ATTR_RSP,
+
+	/* keep last */
+	NUM_MT7925_TM_ATTRS,
+	MT7925_TM_ATTR_MAX = NUM_MT7925_TM_ATTRS - 1,
+};
+
+struct mt7925_tm_cmd {
+	u8 padding[4];
+	struct uni_cmd_testmode_ctrl c;
+} __packed;
+
+struct mt7925_tm_evt {
+	u32 param0;
+	u32 param1;
+} __packed;
+
+static const struct nla_policy mt7925_tm_policy[NUM_MT7925_TM_ATTRS] = {
+	[MT7925_TM_ATTR_SET] = NLA_POLICY_EXACT_LEN(sizeof(struct mt7925_tm_cmd)),
+	[MT7925_TM_ATTR_QUERY] = NLA_POLICY_EXACT_LEN(sizeof(struct mt7925_tm_cmd)),
+};
+
+static int
+mt7925_tm_set(struct mt792x_dev *dev, struct mt7925_tm_cmd *req)
+{
+	struct mt7925_rftest_cmd cmd;
+	struct mt7925_rftest_cmd *pcmd = &cmd;
+	bool testmode = false, normal = false;
+	struct mt76_connac_pm *pm = &dev->pm;
+	struct mt76_phy *phy = &dev->mphy;
+	int ret = -ENOTCONN;
+
+	memset(pcmd, 0, sizeof(*pcmd));
+	memcpy(pcmd, req, sizeof(struct mt7925_tm_cmd));
+
+	mutex_lock(&dev->mt76.mutex);
+
+	if (pcmd->ctrl.action == CMD_TEST_CTRL_ACT_SWITCH_MODE) {
+		if (pcmd->ctrl.data.op_mode == CMD_TEST_CTRL_ACT_SWITCH_MODE_NORMAL)
+			normal = true;
+		else
+			testmode = true;
+	}
+
+	if (testmode) {
+		/* Make sure testmode running on full power mode */
+		pm->enable = false;
+		cancel_delayed_work_sync(&pm->ps_work);
+		cancel_work_sync(&pm->wake_work);
+		__mt792x_mcu_drv_pmctrl(dev);
+
+		phy->test.state = MT76_TM_STATE_ON;
+	}
+
+	if (!mt76_testmode_enabled(phy))
+		goto out;
+
+	ret = mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(TESTMODE_CTRL), &cmd,
+				sizeof(cmd), false);
+
+	if (ret)
+		goto out;
+
+	if (normal) {
+		/* Switch back to the normal world */
+		phy->test.state = MT76_TM_STATE_OFF;
+		pm->enable = true;
+	}
+out:
+	mutex_unlock(&dev->mt76.mutex);
+
+	return ret;
+}
+
+static int
+mt7925_tm_query(struct mt792x_dev *dev, struct mt7925_tm_cmd *req,
+		char *evt_resp)
+{
+	struct mt7925_rftest_cmd cmd;
+	char *pcmd = (char *)&cmd;
+	struct sk_buff *skb = NULL;
+	int ret = 1;
+
+	memset(pcmd, 0, sizeof(*pcmd));
+	memcpy(pcmd + 4, (char *)&req->c, sizeof(struct uni_cmd_testmode_ctrl));
+
+	if (*((uint16_t *)req->padding) == MCU_UNI_CMD_TESTMODE_CTRL)
+		ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_UNI_QUERY(TESTMODE_CTRL),
+						&cmd, sizeof(cmd), true, &skb);
+	else if (*((uint16_t *)req->padding) == MCU_UNI_CMD_TESTMODE_RX_STAT)
+		ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_UNI_QUERY(TESTMODE_RX_STAT),
+						&cmd, sizeof(cmd), true, &skb);
+
+	if (ret)
+		goto out;
+
+	memcpy((char *)evt_resp, (char *)skb->data + 8, MT7925_EVT_RSP_LEN);
+
+out:
+	dev_kfree_skb(skb);
+
+	return ret;
+}
+
+int mt7925_testmode_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			void *data, int len)
+{
+	struct nlattr *tb[NUM_MT76_TM_ATTRS];
+	struct mt76_phy *mphy = hw->priv;
+	struct mt792x_phy *phy = mphy->priv;
+	int err;
+
+	if (!test_bit(MT76_STATE_RUNNING, &mphy->state) ||
+	    !(hw->conf.flags & IEEE80211_CONF_MONITOR))
+		return -ENOTCONN;
+
+	err = nla_parse_deprecated(tb, MT76_TM_ATTR_MAX, data, len,
+				   mt76_tm_policy, NULL);
+	if (err)
+		return err;
+
+	if (tb[MT76_TM_ATTR_DRV_DATA]) {
+		struct nlattr *drv_tb[NUM_MT7925_TM_ATTRS], *data;
+		int ret;
+
+		data = tb[MT76_TM_ATTR_DRV_DATA];
+		ret = nla_parse_nested_deprecated(drv_tb,
+						  MT7925_TM_ATTR_MAX,
+						  data, mt7925_tm_policy,
+						  NULL);
+		if (ret)
+			return ret;
+
+		data = drv_tb[MT7925_TM_ATTR_SET];
+		if (data)
+			return mt7925_tm_set(phy->dev, nla_data(data));
+	}
+
+	return -EINVAL;
+}
+
+int mt7925_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *msg,
+			 struct netlink_callback *cb, void *data, int len)
+{
+	struct nlattr *tb[NUM_MT76_TM_ATTRS];
+	struct mt76_phy *mphy = hw->priv;
+	struct mt792x_phy *phy = mphy->priv;
+	int err;
+
+	if (!test_bit(MT76_STATE_RUNNING, &mphy->state) ||
+	    !(hw->conf.flags & IEEE80211_CONF_MONITOR) ||
+	    !mt76_testmode_enabled(mphy))
+		return -ENOTCONN;
+
+	if (cb->args[2]++ > 0)
+		return -ENOENT;
+
+	err = nla_parse_deprecated(tb, MT76_TM_ATTR_MAX, data, len,
+				   mt76_tm_policy, NULL);
+	if (err)
+		return err;
+
+	if (tb[MT76_TM_ATTR_DRV_DATA]) {
+		struct nlattr *drv_tb[NUM_MT7925_TM_ATTRS], *data;
+		int ret;
+
+		data = tb[MT76_TM_ATTR_DRV_DATA];
+		ret = nla_parse_nested_deprecated(drv_tb,
+						  MT7925_TM_ATTR_MAX,
+						  data, mt7925_tm_policy,
+						  NULL);
+		if (ret)
+			return ret;
+
+		data = drv_tb[MT7925_TM_ATTR_QUERY];
+		if (data) {
+			char evt_resp[MT7925_EVT_RSP_LEN];
+
+			err = mt7925_tm_query(phy->dev, nla_data(data),
+					      evt_resp);
+			if (err)
+				return err;
+
+			return nla_put(msg, MT7925_TM_ATTR_RSP,
+				       sizeof(evt_resp), evt_resp);
+		}
+	}
+
+	return -EINVAL;
+}
diff --git a/sys/contrib/dev/mediatek/mt76/mt792x.h b/sys/contrib/dev/mediatek/mt76/mt792x.h
index 32ed01a96bf7..443d397d9961 100644
--- a/sys/contrib/dev/mediatek/mt76/mt792x.h
+++ b/sys/contrib/dev/mediatek/mt76/mt792x.h
@@ -27,8 +27,9 @@
 
 #define MT792x_CHIP_CAP_CLC_EVT_EN BIT(0)
 #define MT792x_CHIP_CAP_RSSI_NOTIFY_EVT_EN BIT(1)
-#define MT792x_CHIP_CAP_MLO_EVT_EN BIT(2)
 #define MT792x_CHIP_CAP_WF_RF_PIN_CTRL_EVT_EN BIT(3)
+#define MT792x_CHIP_CAP_MLO_EN BIT(8)
+#define MT792x_CHIP_CAP_MLO_EML_EN BIT(9)
 
 /* NOTE: used to map mt76_rates. idx may change if firmware expands table */
 #define MT792x_BASIC_RATES_TBL	11
@@ -70,6 +71,7 @@ struct mt792x_fw_features {
 enum {
 	MT792x_CLC_POWER,
 	MT792x_CLC_POWER_EXT,
+	MT792x_CLC_BE_CTRL,
 	MT792x_CLC_MAX_NUM,
 };
 
@@ -81,6 +83,13 @@ enum mt792x_reg_power_type {
 	MT_AP_VLP,
 };
 
+enum mt792x_mlo_pm_state {
+	MT792x_MLO_LINK_DISASSOC,
+	MT792x_MLO_LINK_ASSOC,
+	MT792x_MLO_CHANGED_PS_PENDING,
+	MT792x_MLO_CHANGED_PS,
+};
+
 DECLARE_EWMA(avg_signal, 10, 8)
 
 struct mt792x_link_sta {
@@ -134,6 +143,7 @@ struct mt792x_vif {
 	struct mt792x_phy *phy;
 	u16 valid_links;
 	u8 deflink_id;
+	enum mt792x_mlo_pm_state mlo_pm_state;
 
 	struct work_struct csa_work;
 	struct timer_list csa_timer;
@@ -239,6 +249,7 @@ struct mt792x_dev {
 	const struct mt792x_irq_map *irq_map;
 
 	struct work_struct ipv6_ns_work;
+	struct delayed_work mlo_pm_work;
 	/* IPv6 addresses for WoWLAN */
 	struct sk_buff_head ipv6_ns_list;
 
@@ -401,7 +412,8 @@ void mt792x_sta_statistics(struct ieee80211_hw *hw,
 			   struct ieee80211_vif *vif,
 			   struct ieee80211_sta *sta,
 			   struct station_info *sinfo);
-void mt792x_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class);
+void mt792x_set_coverage_class(struct ieee80211_hw *hw, int radio_idx,
+			       s16 coverage_class);
 void mt792x_dma_cleanup(struct mt792x_dev *dev);
 int mt792x_dma_enable(struct mt792x_dev *dev);
 int mt792x_wpdma_reset(struct mt792x_dev *dev, bool force);
@@ -497,7 +509,7 @@ int mt792xe_mcu_fw_pmctrl(struct mt792x_dev *dev);
 int mt792x_init_acpi_sar(struct mt792x_dev *dev);
 int mt792x_init_acpi_sar_power(struct mt792x_phy *phy, bool set_default);
 u8 mt792x_acpi_get_flags(struct mt792x_phy *phy);
-u8 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2);
+u32 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2);
 #else
 static inline int mt792x_init_acpi_sar(struct mt792x_dev *dev)
 {
@@ -515,9 +527,9 @@ static inline u8 mt792x_acpi_get_flags(struct mt792x_phy *phy)
 	return 0;
 }
 
-static inline u8 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2)
+static inline u32 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2)
 {
-	return 0xf;
+	return MT792X_ACPI_MTCL_INVALID;
 }
 #endif
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c
index 9317f8ff2070..d1aebadd50aa 100644
--- a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c
+++ b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.c
@@ -4,6 +4,28 @@
 #include <linux/acpi.h>
 #include "mt792x.h"
 
+static const char * const cc_list_all[] = {
+	"00", "EU", "AR", "AU", "AZ", "BY", "BO", "BR",
+	"CA", "CL", "CN", "ID", "JP", "MY", "MX", "ME",
+	"MA", "NZ", "NG", "PH", "RU", "RS", "SG", "KR",
+	"TW", "TH", "UA", "GB", "US", "VN", "KH", "PY",
+};
+
+static const char * const cc_list_eu[] = {
+	"AD", "AT", "BE", "BG", "CY", "CZ", "HR", "DK",
+	"EE", "FI", "FR", "DE", "GR", "HU", "IS", "IE",
+	"IT", "LV", "LI", "LT", "LU", "MC", "MT", "NL",
+	"NO", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
+	"CH",
+};
+
+static const char * const cc_list_be[] = {
+	"AR", "BR", "BY", "CL", "IQ", "MX", "OM", "RU",
+	"RW", "VN", "KR", "UA", "", "", "", "",
+	"EU", "AT", "CN", "CA", "TW", "NZ", "PH", "UK",
+	"US",
+};
+
 static int
 mt792x_acpi_read(struct mt792x_dev *dev, u8 *method, u8 **tbl, u32 *len)
 {
@@ -66,13 +88,22 @@ free:
 }
 
 /* MTCL : Country List Table for 6G band */
+/* MTCL : Country List Table for 6G band and 11BE */
 static int
 mt792x_asar_acpi_read_mtcl(struct mt792x_dev *dev, u8 **table, u8 *version)
 {
-	int ret;
+	int len, ret;
 
-	*version = ((ret = mt792x_acpi_read(dev, MT792x_ACPI_MTCL, table, NULL)) < 0)
-		   ? 1 : 2;
+	ret = mt792x_acpi_read(dev, MT792x_ACPI_MTCL, table, &len);
+	if (ret)
+		return ret;
+
+	if (len == sizeof(struct mt792x_asar_cl))
+		*version = ((struct mt792x_asar_cl *)*table)->version;
+	else if (len == sizeof(struct mt792x_asar_cl_v3))
+		*version = ((struct mt792x_asar_cl_v3 *)*table)->version;
+	else
+		return -EINVAL;
 
 	return ret;
 }
@@ -351,10 +382,24 @@ u8 mt792x_acpi_get_flags(struct mt792x_phy *phy)
 }
 EXPORT_SYMBOL_GPL(mt792x_acpi_get_flags);
 
-static u8
+static u32
+mt792x_acpi_get_mtcl_map_v3(int row, int column, struct mt792x_asar_cl_v3 *cl)
+{
+	u32 config = 0;
+	u8 mode_be = 0;
+
+	mode_be = (cl->mode_be > 0x02) ? 0 : cl->mode_be;
+
+	if (cl->version > 2 && cl->clbe[row] & BIT(column))
+		config |= (mode_be & 0x3) << 4;
+
+	return config;
+}
+
+static u32
 mt792x_acpi_get_mtcl_map(int row, int column, struct mt792x_asar_cl *cl)
 {
-	u8 config = 0;
+	u32 config = 0;
 	u8 mode_6g, mode_5g9;
 
 	mode_6g = (cl->mode_6g > 0x02) ? 0 : cl->mode_6g;
@@ -368,30 +413,44 @@ mt792x_acpi_get_mtcl_map(int row, int column, struct mt792x_asar_cl *cl)
 	return config;
 }
 
-u8 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2)
+static u32
+mt792x_acpi_parse_mtcl_tbl_v3(struct mt792x_phy *phy, char *alpha2)
 {
-	static const char * const cc_list_all[] = {
-		"00", "EU", "AR", "AU", "AZ", "BY", "BO", "BR",
-		"CA", "CL", "CN", "ID", "JP", "MY", "MX", "ME",
-		"MA", "NZ", "NG", "PH", "RU", "RS", "SG", "KR",
-		"TW", "TH", "UA", "GB", "US", "VN", "KH", "PY",
-	};
-	static const char * const cc_list_eu[] = {
-		"AT", "BE", "BG", "CY", "CZ", "HR", "DK", "EE",
-		"FI", "FR", "DE", "GR", "HU", "IS", "IE", "IT",
-		"LV", "LI", "LT", "LU", "MT", "NL", "NO", "PL",
-		"PT", "RO", "SK", "SI", "ES", "SE", "CH",
-	};
 	struct mt792x_acpi_sar *sar = phy->acpisar;
-	struct mt792x_asar_cl *cl;
+	struct mt792x_asar_cl_v3 *cl = sar->countrylist_v3;
 	int col, row, i;
 
-	if (!sar)
-		return 0xf;
+	if (sar->ver != 3)
+		goto out;
 
-	cl = sar->countrylist;
 	if (!cl)
-		return 0xc;
+		return MT792X_ACPI_MTCL_INVALID;
+
+	for (i = 0; i < ARRAY_SIZE(cc_list_be); i++) {
+		col = 7 - i % 8;
+		row = i / 8;
+		if (!memcmp(cc_list_be[i], alpha2, 2))
+			return mt792x_acpi_get_mtcl_map_v3(row, col, cl);
+	}
+	for (i = 0; i < ARRAY_SIZE(cc_list_eu); i++) {
+		if (!memcmp(cc_list_eu[i], alpha2, 2))
+			return mt792x_acpi_get_mtcl_map_v3(3, 7, cl);
+	}
+
+out:
+	/* Depends on driver */
+	return 0x20;
+}
+
+static u32
+mt792x_acpi_parse_mtcl_tbl(struct mt792x_phy *phy, char *alpha2)
+{
+	struct mt792x_acpi_sar *sar = phy->acpisar;
+	struct mt792x_asar_cl *cl = sar->countrylist;
+	int col, row, i;
+
+	if (!cl)
+		return MT792X_ACPI_MTCL_INVALID;
 
 	for (i = 0; i < ARRAY_SIZE(cc_list_all); i++) {
 		col = 7 - i % 8;
@@ -406,4 +465,22 @@ u8 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2)
 
 	return mt792x_acpi_get_mtcl_map(0, 7, cl);
 }
+
+u32 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2)
+{
+	struct mt792x_acpi_sar *sar = phy->acpisar;
+	u32 config = 0;
+
+	if (!sar)
+		return MT792X_ACPI_MTCL_INVALID;
+
+	config = mt792x_acpi_parse_mtcl_tbl_v3(phy, alpha2);
+
+	if (config == MT792X_ACPI_MTCL_INVALID)
+		return MT792X_ACPI_MTCL_INVALID;
+
+	config |= mt792x_acpi_parse_mtcl_tbl(phy, alpha2);
+
+	return config;
+}
 EXPORT_SYMBOL_GPL(mt792x_acpi_get_mtcl_conf);
diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h
index 2298983b6342..e45dcd7fbdb1 100644
--- a/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h
+++ b/sys/contrib/dev/mediatek/mt76/mt792x_acpi_sar.h
@@ -15,6 +15,8 @@
 #define MT792x_ACPI_MTGS		"MTGS"
 #define MT792x_ACPI_MTFG		"MTFG"
 
+#define MT792X_ACPI_MTCL_INVALID	0xffffffff
+
 struct mt792x_asar_dyn_limit {
 	u8 idx;
 	u8 frp[5];
@@ -72,6 +74,17 @@ struct mt792x_asar_geo_v2 {
 	DECLARE_FLEX_ARRAY(struct mt792x_asar_geo_limit_v2, tbl);
 } __packed;
 
+struct mt792x_asar_cl_v3 {
+	u8 names[4];
+	u8 version;
+	u8 mode_6g;
+	u8 cl6g[6];
+	u8 mode_5g9;
+	u8 cl5g9[6];
+	u8 mode_be;
+	u8 clbe[6];
+} __packed;
+
 struct mt792x_asar_cl {
 	u8 names[4];
 	u8 version;
@@ -100,7 +113,10 @@ struct mt792x_acpi_sar {
 		struct mt792x_asar_geo *geo;
 		struct mt792x_asar_geo_v2 *geo_v2;
 	};
-	struct mt792x_asar_cl *countrylist;
+	union {
+		struct mt792x_asar_cl *countrylist;
+		struct mt792x_asar_cl_v3 *countrylist_v3;
+	};
 	struct mt792x_asar_fg *fg;
 };
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_core.c b/sys/contrib/dev/mediatek/mt76/mt792x_core.c
index c42d1105e5a1..6ce282d7bd50 100644
--- a/sys/contrib/dev/mediatek/mt76/mt792x_core.c
+++ b/sys/contrib/dev/mediatek/mt76/mt792x_core.c
@@ -31,7 +31,7 @@ static const struct ieee80211_iface_combination if_comb[] = {
 	},
 };
 
-static const struct ieee80211_iface_limit if_limits_chanctx[] = {
+static const struct ieee80211_iface_limit if_limits_chanctx_mcc[] = {
 	{
 		.max = 2,
 		.types = BIT(NL80211_IFTYPE_STATION) |
@@ -39,8 +39,23 @@ static const struct ieee80211_iface_limit if_limits_chanctx[] = {
 	},
 	{
 		.max = 1,
-		.types = BIT(NL80211_IFTYPE_AP) |
-			 BIT(NL80211_IFTYPE_P2P_GO)
+		.types = BIT(NL80211_IFTYPE_P2P_GO)
+	},
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_P2P_DEVICE)
+	}
+};
+
+static const struct ieee80211_iface_limit if_limits_chanctx_scc[] = {
+	{
+		.max = 2,
+		.types = BIT(NL80211_IFTYPE_STATION) |
+			 BIT(NL80211_IFTYPE_P2P_CLIENT)
+	},
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_AP)
 	},
 	{
 		.max = 1,
@@ -50,11 +65,18 @@ static const struct ieee80211_iface_limit if_limits_chanctx[] = {
 
 static const struct ieee80211_iface_combination if_comb_chanctx[] = {
 	{
-		.limits = if_limits_chanctx,
-		.n_limits = ARRAY_SIZE(if_limits_chanctx),
+		.limits = if_limits_chanctx_mcc,
+		.n_limits = ARRAY_SIZE(if_limits_chanctx_mcc),
 		.max_interfaces = 3,
 		.num_different_channels = 2,
 		.beacon_int_infra_match = false,
+	},
+	{
+		.limits = if_limits_chanctx_scc,
+		.n_limits = ARRAY_SIZE(if_limits_chanctx_scc),
+		.max_interfaces = 3,
+		.num_different_channels = 1,
+		.beacon_int_infra_match = false,
 	}
 };
 
@@ -286,7 +308,7 @@ EXPORT_SYMBOL_GPL(mt792x_tx_worker);
 
 void mt792x_roc_timer(struct timer_list *timer)
 {
-	struct mt792x_phy *phy = from_timer(phy, timer, roc_timer);
+	struct mt792x_phy *phy = timer_container_of(phy, timer, roc_timer);
 
 	ieee80211_queue_work(phy->mt76->hw, &phy->roc_work);
 }
@@ -294,7 +316,7 @@ EXPORT_SYMBOL_GPL(mt792x_roc_timer);
 
 void mt792x_csa_timer(struct timer_list *timer)
 {
-	struct mt792x_vif *mvif = from_timer(mvif, timer, csa_timer);
+	struct mt792x_vif *mvif = timer_container_of(mvif, timer, csa_timer);
 
 	ieee80211_queue_work(mvif->phy->mt76->hw, &mvif->csa_work);
 }
@@ -343,7 +365,7 @@ void mt792x_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	mutex_unlock(&dev->mt76.mutex);
 
 	if (vif->bss_conf.csa_active) {
-		del_timer_sync(&mvif->csa_timer);
+		timer_delete_sync(&mvif->csa_timer);
 		cancel_work_sync(&mvif->csa_work);
 	}
 }
@@ -582,7 +604,8 @@ void mt792x_sta_statistics(struct ieee80211_hw *hw,
 }
 EXPORT_SYMBOL_GPL(mt792x_sta_statistics);
 
-void mt792x_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
+void mt792x_set_coverage_class(struct ieee80211_hw *hw, int radio_idx,
+			       s16 coverage_class)
 {
 	struct mt792x_phy *phy = mt792x_hw_phy(hw);
 	struct mt792x_dev *dev = phy->dev;
@@ -668,7 +691,9 @@ int mt792x_init_wiphy(struct ieee80211_hw *hw)
 	ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
 	ieee80211_hw_set(hw, SUPPORTS_VHT_EXT_NSS_BW);
 	ieee80211_hw_set(hw, CONNECTION_MONITOR);
-	ieee80211_hw_set(hw, CHANCTX_STA_CSA);
+	ieee80211_hw_set(hw, NO_VIRTUAL_MONITOR);
+	if (is_mt7921(&dev->mt76))
+		ieee80211_hw_set(hw, CHANCTX_STA_CSA);
 
 	if (dev->pm.enable)
 		ieee80211_hw_set(hw, CONNECTION_MONITOR);
diff --git a/sys/contrib/dev/mediatek/mt76/mt792x_mac.c b/sys/contrib/dev/mediatek/mt76/mt792x_mac.c
index d60b4124d71a..d72cdb0215e6 100644
--- a/sys/contrib/dev/mediatek/mt76/mt792x_mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt792x_mac.c
@@ -145,10 +145,7 @@ struct mt76_wcid *mt792x_rx_get_wcid(struct mt792x_dev *dev, u16 idx,
 	struct mt792x_sta *sta;
 	struct mt76_wcid *wcid;
 
-	if (idx >= ARRAY_SIZE(dev->mt76.wcid))
-		return NULL;
-
-	wcid = rcu_dereference(dev->mt76.wcid[idx]);
+	wcid = mt76_wcid_ptr(dev, idx);
 	if (unicast || !wcid)
 		return wcid;
 
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c b/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c
index ccab0d7b9be4..303d6e80a666 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/coredump.c
@@ -48,8 +48,8 @@ const struct mt7996_mem_region*
 mt7996_coredump_get_mem_layout(struct mt7996_dev *dev, u32 *num)
 {
 	switch (mt76_chip(&dev->mt76)) {
-	case 0x7990:
-	case 0x7991:
+	case MT7996_DEVICE_ID:
+	case MT7996_DEVICE_ID_2:
 		*num = ARRAY_SIZE(mt7996_mem_regions);
 		return &mt7996_mem_regions[0];
 	default:
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c b/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c
index 7b2bb72b407d..0ab827f52fd7 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/debugfs.c
@@ -222,18 +222,27 @@ static const struct file_operations mt7996_sys_recovery_ops = {
 static int
 mt7996_radar_trigger(void *data, u64 val)
 {
+#define RADAR_MAIN_CHAIN	1
+#define RADAR_BACKGROUND	2
 	struct mt7996_dev *dev = data;
+	struct mt7996_phy *phy = mt7996_band_phy(dev, NL80211_BAND_5GHZ);
+	int rdd_idx;
 
-	if (val > MT_RX_SEL2)
+	if (!phy || !val || val > RADAR_BACKGROUND)
 		return -EINVAL;
 
-	if (val == MT_RX_SEL2 && !dev->rdd2_phy) {
+	if (val == RADAR_BACKGROUND && !dev->rdd2_phy) {
 		dev_err(dev->mt76.dev, "Background radar is not enabled\n");
 		return -EINVAL;
 	}
 
-	return mt7996_mcu_rdd_cmd(dev, RDD_RADAR_EMULATE,
-				  val, 0, 0);
+	rdd_idx = mt7996_get_rdd_idx(phy, val == RADAR_BACKGROUND);
+	if (rdd_idx < 0) {
+		dev_err(dev->mt76.dev, "No RDD found\n");
+		return -EINVAL;
+	}
+
+	return mt7996_mcu_rdd_cmd(dev, RDD_RADAR_EMULATE, rdd_idx, 0);
 }
 
 DEFINE_DEBUGFS_ATTRIBUTE(fops_radar_trigger, NULL,
@@ -616,28 +625,51 @@ static void
 mt7996_sta_hw_queue_read(void *data, struct ieee80211_sta *sta)
 {
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
-	struct mt7996_dev *dev = msta->vif->deflink.phy->dev;
+	struct mt7996_vif *mvif = msta->vif;
+	struct mt7996_dev *dev = mvif->deflink.phy->dev;
+	struct ieee80211_link_sta *link_sta;
 	struct seq_file *s = data;
-	u8 ac;
+	struct ieee80211_vif *vif;
+	unsigned int link_id;
 
-	for (ac = 0; ac < 4; ac++) {
-		u32 qlen, ctrl, val;
-		u32 idx = msta->wcid.idx >> 5;
-		u8 offs = msta->wcid.idx & GENMASK(4, 0);
+	vif = container_of((void *)mvif, struct ieee80211_vif, drv_priv);
 
-		ctrl = BIT(31) | BIT(11) | (ac << 24);
-		val = mt76_rr(dev, MT_PLE_AC_QEMPTY(ac, idx));
+	rcu_read_lock();
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct mt7996_sta_link *msta_link;
+		struct mt76_vif_link *mlink;
+		u8 ac;
 
-		if (val & BIT(offs))
+		mlink = rcu_dereference(mvif->mt76.link[link_id]);
+		if (!mlink)
 			continue;
 
-		mt76_wr(dev, MT_FL_Q0_CTRL, ctrl | msta->wcid.idx);
-		qlen = mt76_get_field(dev, MT_FL_Q3_CTRL,
-				      GENMASK(11, 0));
-		seq_printf(s, "\tSTA %pM wcid %d: AC%d%d queued:%d\n",
-			   sta->addr, msta->wcid.idx,
-			   msta->vif->deflink.mt76.wmm_idx, ac, qlen);
+		msta_link = rcu_dereference(msta->link[link_id]);
+		if (!msta_link)
+			continue;
+
+		for (ac = 0; ac < 4; ac++) {
+			u32 idx = msta_link->wcid.idx >> 5, qlen, ctrl, val;
+			u8 offs = msta_link->wcid.idx & GENMASK(4, 0);
+
+			ctrl = BIT(31) | BIT(11) | (ac << 24);
+			val = mt76_rr(dev, MT_PLE_AC_QEMPTY(ac, idx));
+
+			if (val & BIT(offs))
+				continue;
+
+			mt76_wr(dev,
+				MT_FL_Q0_CTRL, ctrl | msta_link->wcid.idx);
+			qlen = mt76_get_field(dev, MT_FL_Q3_CTRL,
+					      GENMASK(11, 0));
+			seq_printf(s, "\tSTA %pM wcid %d: AC%d%d queued:%d\n",
+				   sta->addr, msta_link->wcid.idx,
+				   mlink->wmm_idx, ac, qlen);
+		}
 	}
+
+	rcu_read_unlock();
 }
 
 static int
@@ -930,6 +962,7 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file,
 	struct ieee80211_sta *sta = file->private_data;
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
 	struct mt7996_dev *dev = msta->vif->deflink.phy->dev;
+	struct mt7996_sta_link *msta_link = &msta->deflink;
 	struct ra_rate phy = {};
 	char buf[100];
 	int ret;
@@ -964,7 +997,7 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file,
 		goto out;
 	}
 
-	phy.wlan_idx = cpu_to_le16(msta->wcid.idx);
+	phy.wlan_idx = cpu_to_le16(msta_link->wcid.idx);
 	phy.gi = cpu_to_le16(gi);
 	phy.ltf = cpu_to_le16(ltf);
 	phy.ldpc = phy.ldpc ? 7 : 0;
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/dma.c b/sys/contrib/dev/mediatek/mt76/mt7996/dma.c
index af317e04ae9e..304bb5a2318c 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/dma.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/dma.c
@@ -58,20 +58,32 @@ static void mt7996_dma_config(struct mt7996_dev *dev)
 
 	/* rx queue */
 	RXQ_CONFIG(MT_RXQ_MCU, WFDMA0, MT_INT_RX_DONE_WM, MT7996_RXQ_MCU_WM);
+	/* for mt7990, RX ring 1 is for SDO instead */
 	RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_RX_DONE_WA, MT7996_RXQ_MCU_WA);
-
-	/* mt7996: band0 and band1, mt7992: band0 */
 	RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0, MT7996_RXQ_BAND0);
-	RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_RX_DONE_WA_MAIN, MT7996_RXQ_MCU_WA_MAIN);
+	if (mt7996_has_wa(dev))
+		RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_RX_DONE_WA_MAIN,
+			   MT7996_RXQ_MCU_WA_MAIN);
 
-	if (is_mt7996(&dev->mt76)) {
+	switch (mt76_chip(&dev->mt76)) {
+	case MT7992_DEVICE_ID:
+		RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA0, MT_INT_RX_DONE_WA_EXT, MT7996_RXQ_MCU_WA_EXT);
+		RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1, MT7996_RXQ_BAND1);
+		break;
+	case MT7990_DEVICE_ID:
+		RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1, MT7996_RXQ_BAND1);
+		RXQ_CONFIG(MT_RXQ_TXFREE_BAND0, WFDMA0,
+			   MT_INT_RX_TXFREE_BAND0_MT7990, MT7990_RXQ_TXFREE0);
+		if (dev->hif2)
+			RXQ_CONFIG(MT_RXQ_TXFREE_BAND1, WFDMA0,
+				   MT_INT_RX_TXFREE_BAND1_MT7990, MT7990_RXQ_TXFREE1);
+		break;
+	case MT7996_DEVICE_ID:
+	default:
 		/* mt7996 band2 */
-		RXQ_CONFIG(MT_RXQ_BAND2, WFDMA0, MT_INT_RX_DONE_BAND2, MT7996_RXQ_BAND2);
 		RXQ_CONFIG(MT_RXQ_BAND2_WA, WFDMA0, MT_INT_RX_DONE_WA_TRI, MT7996_RXQ_MCU_WA_TRI);
-	} else {
-		/* mt7992 band1 */
-		RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1, MT7996_RXQ_BAND1);
-		RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA0, MT_INT_RX_DONE_WA_EXT, MT7996_RXQ_MCU_WA_EXT);
+		RXQ_CONFIG(MT_RXQ_BAND2, WFDMA0, MT_INT_RX_DONE_BAND2, MT7996_RXQ_BAND2);
+		break;
 	}
 
 	if (dev->has_rro) {
@@ -107,9 +119,11 @@ static void mt7996_dma_config(struct mt7996_dev *dev)
 	}
 
 	/* mcu tx queue */
-	MCUQ_CONFIG(MT_MCUQ_WM, WFDMA0, MT_INT_TX_DONE_MCU_WM, MT7996_TXQ_MCU_WM);
-	MCUQ_CONFIG(MT_MCUQ_WA, WFDMA0, MT_INT_TX_DONE_MCU_WA, MT7996_TXQ_MCU_WA);
 	MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA0, MT_INT_TX_DONE_FWDL, MT7996_TXQ_FWDL);
+	MCUQ_CONFIG(MT_MCUQ_WM, WFDMA0, MT_INT_TX_DONE_MCU_WM, MT7996_TXQ_MCU_WM);
+	if (mt7996_has_wa(dev))
+		MCUQ_CONFIG(MT_MCUQ_WA, WFDMA0, MT_INT_TX_DONE_MCU_WA,
+			    MT7996_TXQ_MCU_WA);
 }
 
 static u32 __mt7996_dma_prefetch_base(u16 *base, u8 depth)
@@ -124,43 +138,62 @@ static u32 __mt7996_dma_prefetch_base(u16 *base, u8 depth)
 static void __mt7996_dma_prefetch(struct mt7996_dev *dev, u32 ofs)
 {
 	u16 base = 0;
-	u8 queue;
+	u8 queue, val;
 
 #define PREFETCH(_depth)	(__mt7996_dma_prefetch_base(&base, (_depth)))
 	/* prefetch SRAM wrapping boundary for tx/rx ring. */
-	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_FWDL) + ofs, PREFETCH(0x2));
-	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WM) + ofs, PREFETCH(0x2));
+	/* Tx Command Rings */
+	val = is_mt7996(&dev->mt76) ? 2 : 4;
+	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_FWDL) + ofs, PREFETCH(val));
+	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WM) + ofs, PREFETCH(val));
+	if (mt7996_has_wa(dev))
+		mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WA) + ofs, PREFETCH(val));
+
+	/* Tx Data Rings */
 	mt76_wr(dev, MT_TXQ_EXT_CTRL(0) + ofs, PREFETCH(0x8));
-	mt76_wr(dev, MT_TXQ_EXT_CTRL(1) + ofs, PREFETCH(0x8));
-	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WA) + ofs, PREFETCH(0x2));
-	mt76_wr(dev, MT_TXQ_EXT_CTRL(2) + ofs, PREFETCH(0x8));
-	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU) + ofs, PREFETCH(0x2));
-	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU_WA) + ofs, PREFETCH(0x2));
-	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN_WA) + ofs, PREFETCH(0x2));
-
-	queue = is_mt7996(&dev->mt76) ? MT_RXQ_BAND2_WA : MT_RXQ_BAND1_WA;
-	mt76_wr(dev, MT_RXQ_BAND1_CTRL(queue) + ofs, PREFETCH(0x2));
-
-	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN) + ofs, PREFETCH(0x10));
+	if (!is_mt7996(&dev->mt76) || dev->hif2)
+		mt76_wr(dev, MT_TXQ_EXT_CTRL(1) + ofs, PREFETCH(0x8));
+	if (is_mt7996(&dev->mt76))
+		mt76_wr(dev, MT_TXQ_EXT_CTRL(2) + ofs, PREFETCH(0x8));
+
+	/* Rx Event Rings */
+	mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_MCU) + ofs, PREFETCH(val));
+	mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_MCU_WA) + ofs, PREFETCH(val));
+
+	/* Rx TxFreeDone From WA Rings */
+	if (mt7996_has_wa(dev)) {
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_MAIN_WA) + ofs, PREFETCH(val));
+		queue = is_mt7996(&dev->mt76) ? MT_RXQ_BAND2_WA : MT_RXQ_BAND1_WA;
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(queue) + ofs, PREFETCH(val));
+	}
 
+	/* Rx TxFreeDone From MAC Rings */
+	val = is_mt7996(&dev->mt76) ? 4 : 8;
+	if (is_mt7990(&dev->mt76) || (is_mt7996(&dev->mt76) && dev->has_rro))
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_TXFREE_BAND0) + ofs, PREFETCH(val));
+	if (is_mt7990(&dev->mt76) && dev->hif2)
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_TXFREE_BAND1) + ofs, PREFETCH(val));
+	else if (is_mt7996(&dev->mt76) && dev->has_rro)
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_TXFREE_BAND2) + ofs, PREFETCH(val));
+
+	/* Rx Data Rings */
+	mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_MAIN) + ofs, PREFETCH(0x10));
 	queue = is_mt7996(&dev->mt76) ? MT_RXQ_BAND2 : MT_RXQ_BAND1;
-	mt76_wr(dev, MT_RXQ_BAND1_CTRL(queue) + ofs, PREFETCH(0x10));
+	mt76_wr(dev, MT_RXQ_EXT_CTRL(queue) + ofs, PREFETCH(0x10));
 
+	/* Rx RRO Rings */
 	if (dev->has_rro) {
-		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_RRO_BAND0) + ofs,
-			PREFETCH(0x10));
-		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_RRO_BAND2) + ofs,
-			PREFETCH(0x10));
-		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MSDU_PAGE_BAND0) + ofs,
-			PREFETCH(0x4));
-		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MSDU_PAGE_BAND1) + ofs,
-			PREFETCH(0x4));
-		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MSDU_PAGE_BAND2) + ofs,
-			PREFETCH(0x4));
-		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_TXFREE_BAND0) + ofs,
-			PREFETCH(0x4));
-		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_TXFREE_BAND2) + ofs,
-			PREFETCH(0x4));
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_RRO_BAND0) + ofs, PREFETCH(0x10));
+		queue = is_mt7996(&dev->mt76) ? MT_RXQ_RRO_BAND2 : MT_RXQ_RRO_BAND1;
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(queue) + ofs, PREFETCH(0x10));
+
+		mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_MSDU_PAGE_BAND0) + ofs, PREFETCH(val));
+		if (is_mt7996(&dev->mt76)) {
+			mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_MSDU_PAGE_BAND1) + ofs,
+				PREFETCH(val));
+			mt76_wr(dev, MT_RXQ_EXT_CTRL(MT_RXQ_MSDU_PAGE_BAND2) + ofs,
+				PREFETCH(val));
+		}
 	}
 #undef PREFETCH
 
@@ -272,6 +305,9 @@ void mt7996_dma_start(struct mt7996_dev *dev, bool reset, bool wed_reset)
 		mtk_wed_device_start(wed, wed_irq_mask);
 	}
 
+	if (!mt7996_has_wa(dev))
+		irq_mask &= ~(MT_INT_RX(MT_RXQ_MAIN_WA) |
+			      MT_INT_RX(MT_RXQ_BAND1_WA));
 	irq_mask = reset ? MT_INT_MCU_CMD : irq_mask;
 
 	mt7996_irq_enable(dev, irq_mask);
@@ -477,12 +513,14 @@ int mt7996_dma_init(struct mt7996_dev *dev)
 		return ret;
 
 	/* command to WA */
-	ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_WA,
-				  MT_MCUQ_ID(MT_MCUQ_WA),
-				  MT7996_TX_MCU_RING_SIZE,
-				  MT_MCUQ_RING_BASE(MT_MCUQ_WA));
-	if (ret)
-		return ret;
+	if (mt7996_has_wa(dev)) {
+		ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_WA,
+					  MT_MCUQ_ID(MT_MCUQ_WA),
+					  MT7996_TX_MCU_RING_SIZE,
+					  MT_MCUQ_RING_BASE(MT_MCUQ_WA));
+		if (ret)
+			return ret;
+	}
 
 	/* firmware download */
 	ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_FWDL,
@@ -496,16 +534,16 @@ int mt7996_dma_init(struct mt7996_dev *dev)
 	ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU],
 			       MT_RXQ_ID(MT_RXQ_MCU),
 			       MT7996_RX_MCU_RING_SIZE,
-			       MT_RX_BUF_SIZE,
+			       MT7996_RX_MCU_BUF_SIZE,
 			       MT_RXQ_RING_BASE(MT_RXQ_MCU));
 	if (ret)
 		return ret;
 
-	/* event from WA */
+	/* event from WA, or SDO event for mt7990 */
 	ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU_WA],
 			       MT_RXQ_ID(MT_RXQ_MCU_WA),
 			       MT7996_RX_MCU_RING_SIZE_WA,
-			       MT_RX_BUF_SIZE,
+			       MT7996_RX_MCU_BUF_SIZE,
 			       MT_RXQ_RING_BASE(MT_RXQ_MCU_WA));
 	if (ret)
 		return ret;
@@ -530,13 +568,41 @@ int mt7996_dma_init(struct mt7996_dev *dev)
 		dev->mt76.q_rx[MT_RXQ_MAIN_WA].wed = wed;
 	}
 
-	ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN_WA],
-			       MT_RXQ_ID(MT_RXQ_MAIN_WA),
-			       MT7996_RX_MCU_RING_SIZE,
-			       MT_RX_BUF_SIZE,
-			       MT_RXQ_RING_BASE(MT_RXQ_MAIN_WA));
-	if (ret)
-		return ret;
+	if (mt7996_has_wa(dev)) {
+		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN_WA],
+				       MT_RXQ_ID(MT_RXQ_MAIN_WA),
+				       MT7996_RX_MCU_RING_SIZE,
+				       MT_RX_BUF_SIZE,
+				       MT_RXQ_RING_BASE(MT_RXQ_MAIN_WA));
+		if (ret)
+			return ret;
+	} else {
+		if (mtk_wed_device_active(wed)) {
+			dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].flags = MT_WED_Q_TXFREE;
+			dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].wed = wed;
+		}
+		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0],
+				       MT_RXQ_ID(MT_RXQ_TXFREE_BAND0),
+				       MT7996_RX_MCU_RING_SIZE,
+				       MT7996_RX_BUF_SIZE,
+				       MT_RXQ_RING_BASE(MT_RXQ_TXFREE_BAND0));
+		if (ret)
+			return ret;
+	}
+
+	if (!mt7996_has_wa(dev) && dev->hif2) {
+		if (mtk_wed_device_active(wed)) {
+			dev->mt76.q_rx[MT_RXQ_TXFREE_BAND1].flags = MT_WED_Q_TXFREE;
+			dev->mt76.q_rx[MT_RXQ_TXFREE_BAND1].wed = wed;
+		}
+		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_TXFREE_BAND1],
+				       MT_RXQ_ID(MT_RXQ_TXFREE_BAND1),
+				       MT7996_RX_MCU_RING_SIZE,
+				       MT7996_RX_BUF_SIZE,
+				       MT_RXQ_RING_BASE(MT_RXQ_TXFREE_BAND1));
+		if (ret)
+			return ret;
+	}
 
 	if (mt7996_band_valid(dev, MT_BAND2)) {
 		/* rx data queue for mt7996 band2 */
@@ -576,14 +642,16 @@ int mt7996_dma_init(struct mt7996_dev *dev)
 			return ret;
 
 		/* tx free notify event from WA for mt7992 band1 */
-		rx_base = MT_RXQ_RING_BASE(MT_RXQ_BAND1_WA) + hif1_ofs;
-		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1_WA],
-				       MT_RXQ_ID(MT_RXQ_BAND1_WA),
-				       MT7996_RX_MCU_RING_SIZE,
-				       MT_RX_BUF_SIZE,
-				       rx_base);
-		if (ret)
-			return ret;
+		if (mt7996_has_wa(dev)) {
+			rx_base = MT_RXQ_RING_BASE(MT_RXQ_BAND1_WA) + hif1_ofs;
+			ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1_WA],
+					       MT_RXQ_ID(MT_RXQ_BAND1_WA),
+					       MT7996_RX_MCU_RING_SIZE,
+					       MT_RX_BUF_SIZE,
+					       rx_base);
+			if (ret)
+				return ret;
+		}
 	}
 
 	if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed) &&
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c b/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c
index e854ab32acba..6354173427a5 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/eeprom.c
@@ -13,10 +13,12 @@ static int mt7996_check_eeprom(struct mt7996_dev *dev)
 	u16 val = get_unaligned_le16(eeprom);
 
 	switch (val) {
-	case 0x7990:
+	case MT7996_DEVICE_ID:
 		return is_mt7996(&dev->mt76) ? 0 : -EINVAL;
-	case 0x7992:
+	case MT7992_DEVICE_ID:
 		return is_mt7992(&dev->mt76) ? 0 : -EINVAL;
+	case MT7990_DEVICE_ID:
+		return is_mt7990(&dev->mt76) ? 0 : -EINVAL;
 	default:
 		return -EINVAL;
 	}
@@ -25,7 +27,7 @@ static int mt7996_check_eeprom(struct mt7996_dev *dev)
 static char *mt7996_eeprom_name(struct mt7996_dev *dev)
 {
 	switch (mt76_chip(&dev->mt76)) {
-	case 0x7992:
+	case MT7992_DEVICE_ID:
 		switch (dev->var.type) {
 		case MT7992_VAR_TYPE_23:
 			if (dev->var.fem == MT7996_FEM_INT)
@@ -39,7 +41,11 @@ static char *mt7996_eeprom_name(struct mt7996_dev *dev)
 				return MT7992_EEPROM_DEFAULT_MIX;
 			return MT7992_EEPROM_DEFAULT;
 		}
-	case 0x7990:
+	case MT7990_DEVICE_ID:
+		if (dev->var.fem == MT7996_FEM_INT)
+			return MT7990_EEPROM_DEFAULT_INT;
+		return MT7990_EEPROM_DEFAULT;
+	case MT7996_DEVICE_ID:
 	default:
 		switch (dev->var.type) {
 		case MT7996_VAR_TYPE_233:
@@ -304,6 +310,7 @@ int mt7996_eeprom_parse_hw_cap(struct mt7996_dev *dev, struct mt7996_phy *phy)
 		phy->has_aux_rx = true;
 
 	mphy->antenna_mask = BIT(nss) - 1;
+	phy->orig_antenna_mask = mphy->antenna_mask;
 	mphy->chainmask = (BIT(path) - 1) << dev->chainshift[band_idx];
 	phy->orig_chainmask = mphy->chainmask;
 	dev->chainmask |= mphy->chainmask;
@@ -374,3 +381,30 @@ s8 mt7996_eeprom_get_power_delta(struct mt7996_dev *dev, int band)
 
 	return val & MT_EE_RATE_DELTA_SIGN ? delta : -delta;
 }
+
+bool mt7996_eeprom_has_background_radar(struct mt7996_dev *dev)
+{
+	switch (mt76_chip(&dev->mt76)) {
+	case MT7996_DEVICE_ID:
+		if (dev->var.type == MT7996_VAR_TYPE_233)
+			return false;
+		break;
+	case MT7992_DEVICE_ID:
+		if (dev->var.type == MT7992_VAR_TYPE_23)
+			return false;
+		break;
+	case MT7990_DEVICE_ID: {
+		u8 path, rx_path, nss, *eeprom = dev->mt76.eeprom.data;
+
+		mt7996_eeprom_parse_stream(eeprom, MT_BAND1, &path, &rx_path, &nss);
+		/* Disable background radar capability in 3T3R */
+		if (path == 3 || rx_path == 3)
+			return false;
+		break;
+		}
+	default:
+		return false;
+	}
+
+	return true;
+}
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/init.c b/sys/contrib/dev/mediatek/mt76/mt7996/init.c
index 1f25bd681fc9..c0c827e1a80b 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/init.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/init.c
@@ -221,6 +221,9 @@ static int mt7996_thermal_init(struct mt7996_phy *phy)
 
 	name = devm_kasprintf(&wiphy->dev, GFP_KERNEL, "mt7996_%s.%d",
 			      wiphy_name(wiphy), phy->mt76->band_idx);
+	if (!name)
+		return -ENOMEM;
+
 	snprintf(cname, sizeof(cname), "cooling_device%d", phy->mt76->band_idx);
 
 	cdev = thermal_cooling_device_register(name, phy, &mt7996_thermal_ops);
@@ -319,8 +322,8 @@ static void __mt7996_init_txpower(struct mt7996_phy *phy,
 				  struct ieee80211_supported_band *sband)
 {
 	struct mt7996_dev *dev = phy->dev;
-	int i, nss = hweight16(phy->mt76->chainmask);
-	int nss_delta = mt76_tx_power_nss_delta(nss);
+	int i, n_chains = hweight16(phy->mt76->chainmask);
+	int path_delta = mt76_tx_power_path_delta(n_chains);
 	int pwr_delta = mt7996_eeprom_get_power_delta(dev, sband->band);
 	struct mt76_power_limits limits;
 
@@ -332,7 +335,7 @@ static void __mt7996_init_txpower(struct mt7996_phy *phy,
 		target_power = mt76_get_rate_power_limits(phy->mt76, chan,
 							  &limits,
 							  target_power);
-		target_power += nss_delta;
+		target_power += path_delta;
 		target_power = DIV_ROUND_UP(target_power, 2);
 		chan->max_power = min_t(int, chan->max_reg_power,
 					target_power);
@@ -437,9 +440,7 @@ static void
 mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-#if defined(CONFIG_OF)
 	struct mt76_dev *mdev = &dev->mt76;
-#endif
 	struct wiphy *wiphy = hw->wiphy;
 	u16 max_subframes = dev->has_eht ? IEEE80211_MAX_AMPDU_BUF_EHT :
 					   IEEE80211_MAX_AMPDU_BUF_HE;
@@ -447,6 +448,9 @@ mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed)
 	hw->queues = 4;
 	hw->max_rx_aggregation_subframes = max_subframes;
 	hw->max_tx_aggregation_subframes = max_subframes;
+	if (is_mt7990(mdev) && dev->has_eht)
+		hw->max_tx_aggregation_subframes = 512;
+
 	hw->netdev_features = NETIF_F_RXCSUM;
 	if (mtk_wed_device_active(wed))
 		hw->netdev_features |= NETIF_F_HW_TC;
@@ -479,13 +483,13 @@ mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed)
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_MU_MIMO_AIR_SNIFFER);
 
-	if (mt7996_has_background_radar(dev) &&
+	if (mt7996_eeprom_has_background_radar(dev) &&
 #if defined(CONFIG_OF)
 	    (!mdev->dev->of_node ||
 	     !of_property_read_bool(mdev->dev->of_node,
 				    "mediatek,disable-radar-background")))
 #else
-		1)
+	    1)
 #endif
 		wiphy_ext_feature_set(wiphy,
 				      NL80211_EXT_FEATURE_RADAR_BACKGROUND);
@@ -950,13 +954,13 @@ static int mt7996_variant_type_init(struct mt7996_dev *dev)
 	u8 var_type;
 
 	switch (mt76_chip(&dev->mt76)) {
-	case 0x7990:
+	case MT7996_DEVICE_ID:
 		if (val & MT_PAD_GPIO_2ADIE_TBTC)
 			var_type = MT7996_VAR_TYPE_233;
 		else
 			var_type = MT7996_VAR_TYPE_444;
 		break;
-	case 0x7992:
+	case MT7992_DEVICE_ID:
 		if (val & MT_PAD_GPIO_ADIE_SINGLE)
 			var_type = MT7992_VAR_TYPE_23;
 		else if (u32_get_bits(val, MT_PAD_GPIO_ADIE_COMB_7992))
@@ -964,6 +968,9 @@ static int mt7996_variant_type_init(struct mt7996_dev *dev)
 		else
 			return -EINVAL;
 		break;
+	case MT7990_DEVICE_ID:
+		var_type = MT7990_VAR_TYPE_23;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -1082,10 +1089,10 @@ void mt7996_set_stream_vht_txbf_caps(struct mt7996_phy *phy)
 	*cap |= IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE |
 		IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE;
 
-	if (is_mt7996(phy->mt76->dev))
-		*cap |= FIELD_PREP(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK, 3);
-	else
+	if (is_mt7992(phy->mt76->dev))
 		*cap |= FIELD_PREP(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK, 4);
+	else
+		*cap |= FIELD_PREP(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK, 3);
 
 	*cap &= ~(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK |
 		  IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE |
@@ -1128,18 +1135,17 @@ mt7996_set_stream_he_txbf_caps(struct mt7996_phy *phy,
 	elem->phy_cap_info[7] &= ~IEEE80211_HE_PHY_CAP7_MAX_NC_MASK;
 
 	c = IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
-	    IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-	    IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO;
+	    IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO;
 	elem->phy_cap_info[2] |= c;
 
 	c = IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE;
 
-	if (is_mt7996(phy->mt76->dev))
-		c |= IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4 |
-		     (IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4 * non_2g);
-	else
+	if (is_mt7992(phy->mt76->dev))
 		c |= IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_5 |
 		     (IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_5 * non_2g);
+	else
+		c |= IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4 |
+		     (IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4 * non_2g);
 
 	elem->phy_cap_info[4] |= c;
 
@@ -1339,6 +1345,9 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band,
 		u8_encode_bits(IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_11454,
 			       IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK);
 
+	eht_cap_elem->mac_cap_info[1] |=
+		IEEE80211_EHT_MAC_CAP1_MAX_AMPDU_LEN_MASK;
+
 	eht_cap_elem->phy_cap_info[0] =
 		IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
 		IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mac.c b/sys/contrib/dev/mediatek/mt76/mt7996/mac.c
index 3f1536f2bbe7..13ea5b55d619 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/mac.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/mac.c
@@ -56,26 +56,45 @@ static const struct mt7996_dfs_radar_spec jp_radar_specs = {
 };
 
 static struct mt76_wcid *mt7996_rx_get_wcid(struct mt7996_dev *dev,
-					    u16 idx, bool unicast)
+					    u16 idx, u8 band_idx)
 {
-	struct mt7996_sta *sta;
+	struct mt7996_sta_link *msta_link;
+	struct mt7996_sta *msta;
+	struct mt7996_vif *mvif;
 	struct mt76_wcid *wcid;
+	int i;
+
+	wcid = mt76_wcid_ptr(dev, idx);
+	if (!wcid || !wcid->sta)
+		return NULL;
 
-	if (idx >= ARRAY_SIZE(dev->mt76.wcid))
+	if (!mt7996_band_valid(dev, band_idx))
 		return NULL;
 
-	wcid = rcu_dereference(dev->mt76.wcid[idx]);
-	if (unicast || !wcid)
+	if (wcid->phy_idx == band_idx)
 		return wcid;
 
-	if (!wcid->sta)
+	msta_link = container_of(wcid, struct mt7996_sta_link, wcid);
+	msta = msta_link->sta;
+	if (!msta || !msta->vif)
 		return NULL;
 
-	sta = container_of(wcid, struct mt7996_sta, wcid);
-	if (!sta->vif)
-		return NULL;
+	mvif = msta->vif;
+	for (i = 0; i < ARRAY_SIZE(mvif->mt76.link); i++) {
+		struct mt76_vif_link *mlink;
+
+		mlink = rcu_dereference(mvif->mt76.link[i]);
+		if (!mlink)
+			continue;
+
+		if (mlink->band_idx != band_idx)
+			continue;
+
+		msta_link = rcu_dereference(msta->link[i]);
+		break;
+	}
 
-	return &sta->vif->deflink.sta.wcid;
+	return &msta_link->wcid;
 }
 
 bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask)
@@ -103,10 +122,13 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev)
 		[IEEE80211_AC_VI] = 4,
 		[IEEE80211_AC_VO] = 6
 	};
+	struct mt7996_sta_link *msta_link;
+	struct mt76_vif_link *mlink;
 	struct ieee80211_sta *sta;
 	struct mt7996_sta *msta;
 	u32 tx_time[IEEE80211_NUM_ACS], rx_time[IEEE80211_NUM_ACS];
 	LIST_HEAD(sta_poll_list);
+	struct mt76_wcid *wcid;
 	int i;
 
 	spin_lock_bh(&dev->mt76.sta_poll_lock);
@@ -126,25 +148,28 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev)
 			spin_unlock_bh(&dev->mt76.sta_poll_lock);
 			break;
 		}
-		msta = list_first_entry(&sta_poll_list,
-					struct mt7996_sta, wcid.poll_list);
-		list_del_init(&msta->wcid.poll_list);
+		msta_link = list_first_entry(&sta_poll_list,
+					     struct mt7996_sta_link,
+					     wcid.poll_list);
+		msta = msta_link->sta;
+		wcid = &msta_link->wcid;
+		list_del_init(&wcid->poll_list);
 		spin_unlock_bh(&dev->mt76.sta_poll_lock);
 
-		idx = msta->wcid.idx;
+		idx = wcid->idx;
 
 		/* refresh peer's airtime reporting */
 		addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 20);
 
 		for (i = 0; i < IEEE80211_NUM_ACS; i++) {
-			u32 tx_last = msta->airtime_ac[i];
-			u32 rx_last = msta->airtime_ac[i + 4];
+			u32 tx_last = msta_link->airtime_ac[i];
+			u32 rx_last = msta_link->airtime_ac[i + 4];
 
-			msta->airtime_ac[i] = mt76_rr(dev, addr);
-			msta->airtime_ac[i + 4] = mt76_rr(dev, addr + 4);
+			msta_link->airtime_ac[i] = mt76_rr(dev, addr);
+			msta_link->airtime_ac[i + 4] = mt76_rr(dev, addr + 4);
 
-			tx_time[i] = msta->airtime_ac[i] - tx_last;
-			rx_time[i] = msta->airtime_ac[i + 4] - rx_last;
+			tx_time[i] = msta_link->airtime_ac[i] - tx_last;
+			rx_time[i] = msta_link->airtime_ac[i + 4] - rx_last;
 
 			if ((tx_last | rx_last) & BIT(30))
 				clear = true;
@@ -155,10 +180,11 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev)
 		if (clear) {
 			mt7996_mac_wtbl_update(dev, idx,
 					       MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
-			memset(msta->airtime_ac, 0, sizeof(msta->airtime_ac));
+			memset(msta_link->airtime_ac, 0,
+			       sizeof(msta_link->airtime_ac));
 		}
 
-		if (!msta->wcid.sta)
+		if (!wcid->sta)
 			continue;
 
 		sta = container_of((void *)msta, struct ieee80211_sta,
@@ -184,28 +210,23 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev)
 		rssi[2] = to_rssi(GENMASK(23, 16), val);
 		rssi[3] = to_rssi(GENMASK(31, 14), val);
 
-		msta->ack_signal =
-			mt76_rx_signal(msta->vif->deflink.phy->mt76->antenna_mask, rssi);
+		mlink = rcu_dereference(msta->vif->mt76.link[wcid->link_id]);
+		if (mlink) {
+			struct mt76_phy *mphy = mt76_vif_link_phy(mlink);
+
+			if (mphy)
+				msta_link->ack_signal =
+					mt76_rx_signal(mphy->antenna_mask,
+						       rssi);
+		}
 
-		ewma_avg_signal_add(&msta->avg_ack_signal, -msta->ack_signal);
+		ewma_avg_signal_add(&msta_link->avg_ack_signal,
+				    -msta_link->ack_signal);
 	}
 
 	rcu_read_unlock();
 }
 
-void mt7996_mac_enable_rtscts(struct mt7996_dev *dev,
-			      struct ieee80211_vif *vif, bool enable)
-{
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	u32 addr;
-
-	addr = mt7996_mac_wtbl_lmac_addr(dev, mvif->deflink.sta.wcid.idx, 5);
-	if (enable)
-		mt76_set(dev, addr, BIT(5));
-	else
-		mt76_clear(dev, addr, BIT(5));
-}
-
 /* The HW does not translate the mac header to 802.3 for mesh point */
 static int mt7996_reverse_frag0_hdr_trans(struct sk_buff *skb, u16 hdr_gap)
 {
@@ -477,11 +498,15 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q,
 
 	unicast = FIELD_GET(MT_RXD3_NORMAL_ADDR_TYPE, rxd3) == MT_RXD3_NORMAL_U2M;
 	idx = FIELD_GET(MT_RXD1_NORMAL_WLAN_IDX, rxd1);
-	status->wcid = mt7996_rx_get_wcid(dev, idx, unicast);
+	status->wcid = mt7996_rx_get_wcid(dev, idx, band_idx);
 
 	if (status->wcid) {
-		msta = container_of(status->wcid, struct mt7996_sta, wcid);
-		mt76_wcid_add_poll(&dev->mt76, &msta->wcid);
+		struct mt7996_sta_link *msta_link;
+
+		msta_link = container_of(status->wcid, struct mt7996_sta_link,
+					 wcid);
+		msta = msta_link->sta;
+		mt76_wcid_add_poll(&dev->mt76, &msta_link->wcid);
 	}
 
 	status->freq = mphy->chandef.chan->center_freq;
@@ -622,6 +647,14 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q,
 		status->last_amsdu = amsdu_info == MT_RXD4_LAST_AMSDU_FRAME;
 	}
 
+	/* IEEE 802.11 fragmentation can only be applied to unicast frames.
+	 * Hence, drop fragments with multicast/broadcast RA.
+	 * This check fixes vulnerabilities, like CVE-2020-26145.
+	 */
+	if ((ieee80211_has_morefrags(fc) || seq_ctrl & IEEE80211_SCTL_FRAG) &&
+	    FIELD_GET(MT_RXD3_NORMAL_ADDR_TYPE, rxd3) != MT_RXD3_NORMAL_U2M)
+		return -EINVAL;
+
 	hdr_gap = (u8 *)rxd - skb->data + 2 * remove_pad;
 	if (hdr_trans && ieee80211_has_morefrags(fc)) {
 		if (mt7996_reverse_frag0_hdr_trans(skb, hdr_gap))
@@ -720,9 +753,8 @@ mt7996_mac_write_txwi_8023(struct mt7996_dev *dev, __le32 *txwi,
 	u32 val;
 
 	if (wcid->sta) {
-		struct ieee80211_sta *sta;
+		struct ieee80211_sta *sta = wcid_to_sta(wcid);
 
-		sta = container_of((void *)wcid, struct ieee80211_sta, drv_priv);
 		wmm = sta->wme;
 	}
 
@@ -749,7 +781,9 @@ mt7996_mac_write_txwi_8023(struct mt7996_dev *dev, __le32 *txwi,
 
 static void
 mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi,
-			    struct sk_buff *skb, struct ieee80211_key_conf *key)
+			    struct sk_buff *skb,
+			    struct ieee80211_key_conf *key,
+			    struct mt76_wcid *wcid)
 {
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
@@ -757,15 +791,19 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi,
 	bool multicast = is_multicast_ether_addr(hdr->addr1);
 	u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK;
 	__le16 fc = hdr->frame_control, sc = hdr->seq_ctrl;
+	u16 seqno = le16_to_cpu(sc);
 	u8 fc_type, fc_stype;
 	u32 val;
 
 	if (ieee80211_is_action(fc) &&
 	    mgmt->u.action.category == WLAN_CATEGORY_BACK &&
-	    mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ)
+	    mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) {
+		if (is_mt7990(&dev->mt76))
+			txwi[6] |= cpu_to_le32(FIELD_PREP(MT_TXD6_TID_ADDBA, tid));
 		tid = MT_TX_ADDBA;
-	else if (ieee80211_is_mgmt(hdr->frame_control))
+	} else if (ieee80211_is_mgmt(hdr->frame_control)) {
 		tid = MT_TX_NORMAL;
+	}
 
 	val = FIELD_PREP(MT_TXD1_HDR_FORMAT, MT_HDR_FORMAT_802_11) |
 	      FIELD_PREP(MT_TXD1_HDR_INFO,
@@ -776,8 +814,7 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi,
 	    info->flags & IEEE80211_TX_CTL_USE_MINRATE)
 		val |= MT_TXD1_FIXED_RATE;
 
-	if (key && multicast && ieee80211_is_robust_mgmt_frame(skb) &&
-	    key->cipher == WLAN_CIPHER_SUITE_AES_CMAC) {
+	if (key && multicast && ieee80211_is_robust_mgmt_frame(skb)) {
 		val |= MT_TXD1_BIP;
 		txwi[3] &= ~cpu_to_le32(MT_TXD3_PROTECT_FRAME);
 	}
@@ -807,9 +844,13 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi,
 		txwi[3] |= cpu_to_le32(MT_TXD3_REM_TX_COUNT);
 	}
 
-	if (info->flags & IEEE80211_TX_CTL_INJECTED) {
-		u16 seqno = le16_to_cpu(sc);
+	if (multicast && ieee80211_vif_is_mld(info->control.vif)) {
+		val = MT_TXD3_SN_VALID |
+		      FIELD_PREP(MT_TXD3_SEQ, IEEE80211_SEQ_TO_SN(seqno));
+		txwi[3] |= cpu_to_le32(val);
+	}
 
+	if (info->flags & IEEE80211_TX_CTL_INJECTED) {
 		if (ieee80211_is_back_req(hdr->frame_control)) {
 			struct ieee80211_bar *bar;
 
@@ -822,6 +863,19 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi,
 		txwi[3] |= cpu_to_le32(val);
 		txwi[3] &= ~cpu_to_le32(MT_TXD3_HW_AMSDU);
 	}
+
+	if (ieee80211_vif_is_mld(info->control.vif) &&
+	    (multicast || unlikely(skb->protocol == cpu_to_be16(ETH_P_PAE))))
+		txwi[5] |= cpu_to_le32(MT_TXD5_FL);
+
+	if (ieee80211_is_nullfunc(fc) && ieee80211_has_a4(fc) &&
+	    ieee80211_vif_is_mld(info->control.vif)) {
+		txwi[5] |= cpu_to_le32(MT_TXD5_FL);
+		txwi[6] |= cpu_to_le32(MT_TXD6_DIS_MAT);
+	}
+
+	if (!wcid->sta && ieee80211_is_mgmt(fc))
+		txwi[6] |= cpu_to_le32(MT_TXD6_DIS_MAT);
 }
 
 void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
@@ -835,7 +889,9 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
 	u8 band_idx = (info->hw_queue & MT_TX_HW_QUEUE_PHY) >> 2;
 	u8 p_fmt, q_idx, omac_idx = 0, wmm_idx = 0;
 	bool is_8023 = info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP;
-	struct mt76_vif_link *mvif;
+	struct mt76_vif_link *mlink = NULL;
+	struct mt7996_vif *mvif;
+	unsigned int link_id;
 	u16 tx_count = 15;
 	u32 val;
 	bool inband_disc = !!(changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP |
@@ -843,11 +899,24 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
 	bool beacon = !!(changed & (BSS_CHANGED_BEACON |
 				    BSS_CHANGED_BEACON_ENABLED)) && (!inband_disc);
 
-	mvif = vif ? (struct mt76_vif_link *)vif->drv_priv : NULL;
+	if (wcid != &dev->mt76.global_wcid)
+		link_id = wcid->link_id;
+	else
+		link_id = u32_get_bits(info->control.flags,
+				       IEEE80211_TX_CTRL_MLO_LINK);
+
+	mvif = vif ? (struct mt7996_vif *)vif->drv_priv : NULL;
 	if (mvif) {
-		omac_idx = mvif->omac_idx;
-		wmm_idx = mvif->wmm_idx;
-		band_idx = mvif->band_idx;
+		if (wcid->offchannel)
+			mlink = rcu_dereference(mvif->mt76.offchannel_link);
+		if (!mlink)
+			mlink = rcu_dereference(mvif->mt76.link[link_id]);
+	}
+
+	if (mlink) {
+		omac_idx = mlink->omac_idx;
+		wmm_idx = mlink->wmm_idx;
+		band_idx = mlink->band_idx;
 	}
 
 	if (inband_disc) {
@@ -894,7 +963,10 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
 		val |= MT_TXD5_TX_STATUS_HOST;
 	txwi[5] = cpu_to_le32(val);
 
-	val = MT_TXD6_DIS_MAT | MT_TXD6_DAS;
+	val = MT_TXD6_DAS;
+	if (q_idx >= MT_LMAC_ALTX0 && q_idx <= MT_LMAC_BCN0)
+		val |= MT_TXD6_DIS_MAT;
+
 	if (is_mt7996(&dev->mt76))
 		val |= FIELD_PREP(MT_TXD6_MSDU_CNT, 1);
 	else if (is_8023 || !ieee80211_is_mgmt(hdr->frame_control))
@@ -906,34 +978,56 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
 	if (is_8023)
 		mt7996_mac_write_txwi_8023(dev, txwi, skb, wcid);
 	else
-		mt7996_mac_write_txwi_80211(dev, txwi, skb, key);
+		mt7996_mac_write_txwi_80211(dev, txwi, skb, key, wcid);
 
 	if (txwi[1] & cpu_to_le32(MT_TXD1_FIXED_RATE)) {
 		bool mcast = ieee80211_is_data(hdr->frame_control) &&
 			     is_multicast_ether_addr(hdr->addr1);
 		u8 idx = MT7996_BASIC_RATES_TBL;
 
-		if (mvif) {
-			if (mcast && mvif->mcast_rates_idx)
-				idx = mvif->mcast_rates_idx;
-			else if (beacon && mvif->beacon_rates_idx)
-				idx = mvif->beacon_rates_idx;
+		if (mlink) {
+			if (mcast && mlink->mcast_rates_idx)
+				idx = mlink->mcast_rates_idx;
+			else if (beacon && mlink->beacon_rates_idx)
+				idx = mlink->beacon_rates_idx;
 			else
-				idx = mvif->basic_rates_idx;
+				idx = mlink->basic_rates_idx;
 		}
 
 		val = FIELD_PREP(MT_TXD6_TX_RATE, idx) | MT_TXD6_FIXED_BW;
+		if (mcast)
+			val |= MT_TXD6_DIS_MAT;
 		txwi[6] |= cpu_to_le32(val);
 		txwi[3] |= cpu_to_le32(MT_TXD3_BA_DISABLE);
 	}
 }
 
+static bool
+mt7996_tx_use_mgmt(struct mt7996_dev *dev, struct sk_buff *skb)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+
+	if (ieee80211_is_mgmt(hdr->frame_control))
+		return true;
+
+	/* for SDO to bypass specific data frame */
+	if (!mt7996_has_wa(dev)) {
+		if (unlikely(skb->protocol == cpu_to_be16(ETH_P_PAE)))
+			return true;
+
+		if (ieee80211_has_a4(hdr->frame_control) &&
+		    !ieee80211_is_data_present(hdr->frame_control))
+			return true;
+	}
+
+	return false;
+}
+
 int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 			  enum mt76_txq_id qid, struct mt76_wcid *wcid,
 			  struct ieee80211_sta *sta,
 			  struct mt76_tx_info *tx_info)
 {
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx_info->skb->data;
 	struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76);
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_info->skb);
 	struct ieee80211_key_conf *key = info->control.hw_key;
@@ -958,8 +1052,11 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 		return id;
 
 	pid = mt76_tx_status_skb_add(mdev, wcid, tx_info->skb);
-	mt7996_mac_write_txwi(dev, txwi_ptr, tx_info->skb, wcid, key,
-			      pid, qid, 0);
+	memset(txwi_ptr, 0, MT_TXD_SIZE);
+	/* Transmit non qos data by 802.11 header and need to fill txd by host*/
+	if (!is_8023 || pid >= MT_PACKET_ID_FIRST)
+		mt7996_mac_write_txwi(dev, txwi_ptr, tx_info->skb, wcid, key,
+				      pid, qid, 0);
 
 	txp = (struct mt76_connac_txp_common *)(txwi + MT_TXD_SIZE);
 	for (i = 0; i < nbuf; i++) {
@@ -976,19 +1073,27 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 	}
 	txp->fw.nbuf = nbuf;
 
-	txp->fw.flags =
-		cpu_to_le16(MT_CT_INFO_FROM_HOST | MT_CT_INFO_APPLY_TXD);
+	txp->fw.flags = cpu_to_le16(MT_CT_INFO_FROM_HOST);
+
+	if (!is_8023 || pid >= MT_PACKET_ID_FIRST)
+		txp->fw.flags |= cpu_to_le16(MT_CT_INFO_APPLY_TXD);
 
 	if (!key)
 		txp->fw.flags |= cpu_to_le16(MT_CT_INFO_NONE_CIPHER_FRAME);
 
-	if (!is_8023 && ieee80211_is_mgmt(hdr->frame_control))
+	if (!is_8023 && mt7996_tx_use_mgmt(dev, tx_info->skb))
 		txp->fw.flags |= cpu_to_le16(MT_CT_INFO_MGMT_FRAME);
 
 	if (vif) {
 		struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
+		struct mt76_vif_link *mlink = NULL;
+
+		if (wcid->offchannel)
+			mlink = rcu_dereference(mvif->mt76.offchannel_link);
+		if (!mlink)
+			mlink = rcu_dereference(mvif->mt76.link[wcid->link_id]);
 
-		txp->fw.bss_idx = mvif->deflink.mt76.idx;
+		txp->fw.bss_idx = mlink ? mlink->idx : mvif->deflink.mt76.idx;
 	}
 
 	txp->fw.token = cpu_to_le16(id);
@@ -1032,14 +1137,14 @@ u32 mt7996_wed_init_buf(void *ptr, dma_addr_t phys, int token_id)
 }
 
 static void
-mt7996_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb)
+mt7996_tx_check_aggr(struct ieee80211_link_sta *link_sta,
+		     struct mt76_wcid *wcid, struct sk_buff *skb)
 {
-	struct mt7996_sta *msta;
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	bool is_8023 = info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP;
 	u16 fc, tid;
 
-	if (!sta || !(sta->deflink.ht_cap.ht_supported || sta->deflink.he_cap.has_he))
+	if (!(link_sta->ht_cap.ht_supported || link_sta->he_cap.has_he))
 		return;
 
 	tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK;
@@ -1048,7 +1153,8 @@ mt7996_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb)
 
 	if (is_8023) {
 		fc = IEEE80211_FTYPE_DATA |
-		     (sta->wme ? IEEE80211_STYPE_QOS_DATA : IEEE80211_STYPE_DATA);
+		     (link_sta->sta->wme ? IEEE80211_STYPE_QOS_DATA
+					 : IEEE80211_STYPE_DATA);
 	} else {
 		/* No need to get precise TID for Action/Management Frame,
 		 * since it will not meet the following Frame Control
@@ -1064,17 +1170,16 @@ mt7996_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb)
 	if (unlikely(fc != (IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_DATA)))
 		return;
 
-	msta = (struct mt7996_sta *)sta->drv_priv;
-	if (!test_and_set_bit(tid, &msta->wcid.ampdu_state))
-		ieee80211_start_tx_ba_session(sta, tid, 0);
+	if (!test_and_set_bit(tid, &wcid->ampdu_state))
+		ieee80211_start_tx_ba_session(link_sta->sta, tid, 0);
 }
 
 static void
 mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t,
-		 struct ieee80211_sta *sta, struct list_head *free_list)
+		 struct ieee80211_link_sta *link_sta,
+		 struct mt76_wcid *wcid, struct list_head *free_list)
 {
 	struct mt76_dev *mdev = &dev->mt76;
-	struct mt76_wcid *wcid;
 	__le32 *txwi;
 	u16 wcid_idx;
 
@@ -1083,12 +1188,10 @@ mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t,
 		goto out;
 
 	txwi = (__le32 *)mt76_get_txwi_ptr(mdev, t);
-	if (sta) {
-		wcid = (struct mt76_wcid *)sta->drv_priv;
+	if (link_sta) {
 		wcid_idx = wcid->idx;
-
 		if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE)))
-			mt7996_tx_check_aggr(sta, t->skb);
+			mt7996_tx_check_aggr(link_sta, wcid, t->skb);
 	} else {
 		wcid_idx = le32_get_bits(txwi[9], MT_TXD9_WLAN_IDX);
 	}
@@ -1107,8 +1210,8 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len)
 	struct mt76_dev *mdev = &dev->mt76;
 	struct mt76_phy *phy2 = mdev->phys[MT_BAND1];
 	struct mt76_phy *phy3 = mdev->phys[MT_BAND2];
+	struct ieee80211_link_sta *link_sta = NULL;
 	struct mt76_txwi_cache *txwi;
-	struct ieee80211_sta *sta = NULL;
 	struct mt76_wcid *wcid = NULL;
 	LIST_HEAD(free_list);
 	struct sk_buff *skb, *tmp;
@@ -1119,6 +1222,7 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len)
 #endif
 	bool wake = false;
 	u16 total, count = 0;
+	u8 ver;
 
 	/* clean DMA queues and unmap buffers first */
 	mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_PSD], false);
@@ -1132,7 +1236,8 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len)
 		mt76_queue_tx_cleanup(dev, phy3->q_tx[MT_TXQ_BE], false);
 	}
 
-	if (WARN_ON_ONCE(le32_get_bits(tx_free[1], MT_TXFREE1_VER) < 5))
+	ver = le32_get_bits(tx_free[1], MT_TXFREE1_VER);
+	if (WARN_ON_ONCE(ver < 5))
 		return;
 
 	total = le32_get_bits(tx_free[0], MT_TXFREE0_MSDU_CNT);
@@ -1147,17 +1252,27 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len)
 		 */
 		info = le32_to_cpu(*cur_info);
 		if (info & MT_TXFREE_INFO_PAIR) {
-			struct mt7996_sta *msta;
+			struct ieee80211_sta *sta;
 			u16 idx;
 
 			idx = FIELD_GET(MT_TXFREE_INFO_WLAN_ID, info);
-			wcid = rcu_dereference(dev->mt76.wcid[idx]);
+			wcid = mt76_wcid_ptr(dev, idx);
 			sta = wcid_to_sta(wcid);
-			if (!sta)
-				continue;
+			if (!sta) {
+				link_sta = NULL;
+				goto next;
+			}
 
-			msta = container_of(wcid, struct mt7996_sta, wcid);
-			mt76_wcid_add_poll(&dev->mt76, &msta->wcid);
+			link_sta = rcu_dereference(sta->link[wcid->link_id]);
+			if (!link_sta)
+				goto next;
+
+			mt76_wcid_add_poll(&dev->mt76, wcid);
+next:
+			/* ver 7 has a new DW with pair = 1, skip it */
+			if (ver == 7 && ((void *)(cur_info + 1) < end) &&
+			    (le32_to_cpu(*(cur_info + 1)) & MT_TXFREE_INFO_PAIR))
+				cur_info++;
 			continue;
 		} else if (info & MT_TXFREE_INFO_HEADER) {
 			u32 tx_retries = 0, tx_failed = 0;
@@ -1185,7 +1300,8 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len)
 			if (!txwi)
 				continue;
 
-			mt7996_txwi_free(dev, txwi, sta, &free_list);
+			mt7996_txwi_free(dev, txwi, link_sta, wcid,
+					 &free_list);
 		}
 	}
 
@@ -1241,7 +1357,7 @@ mt7996_mac_add_txs_skb(struct mt7996_dev *dev, struct mt76_wcid *wcid,
 		struct ieee80211_sta *sta;
 		u8 tid;
 
-		sta = container_of((void *)wcid, struct ieee80211_sta, drv_priv);
+		sta = wcid_to_sta(wcid);
 		tid = FIELD_GET(MT_TXS0_TID, txs);
 		ieee80211_refresh_tx_agg_session_timer(sta, tid);
 	}
@@ -1355,7 +1471,7 @@ out:
 
 static void mt7996_mac_add_txs(struct mt7996_dev *dev, void *data)
 {
-	struct mt7996_sta *msta = NULL;
+	struct mt7996_sta_link *msta_link;
 	struct mt76_wcid *wcid;
 	__le32 *txs_data = data;
 	u16 wcidx;
@@ -1367,23 +1483,19 @@ static void mt7996_mac_add_txs(struct mt7996_dev *dev, void *data)
 	if (pid < MT_PACKET_ID_NO_SKB)
 		return;
 
-	if (wcidx >= mt7996_wtbl_size(dev))
-		return;
-
 	rcu_read_lock();
 
-	wcid = rcu_dereference(dev->mt76.wcid[wcidx]);
+	wcid = mt76_wcid_ptr(dev, wcidx);
 	if (!wcid)
 		goto out;
 
-	msta = container_of(wcid, struct mt7996_sta, wcid);
-
 	mt7996_mac_add_txs_skb(dev, wcid, pid, txs_data);
 
 	if (!wcid->sta)
 		goto out;
 
-	mt76_wcid_add_poll(&dev->mt76, &msta->wcid);
+	msta_link = container_of(wcid, struct mt7996_sta_link, wcid);
+	mt76_wcid_add_poll(&dev->mt76, &msta_link->wcid);
 
 out:
 	rcu_read_unlock();
@@ -1410,7 +1522,7 @@ bool mt7996_rx_check(struct mt76_dev *mdev, void *data, int len)
 		mt7996_mac_tx_free(dev, data, len);
 		return false;
 	case PKT_TYPE_TXS:
-		for (rxd += 4; rxd + 8 <= end; rxd += 8)
+		for (rxd += MT_TXS_HDR_SIZE; rxd + MT_TXS_SIZE <= end; rxd += MT_TXS_SIZE)
 			mt7996_mac_add_txs(dev, rxd);
 		return false;
 	case PKT_TYPE_RX_FW_MONITOR:
@@ -1455,7 +1567,7 @@ void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 		mt7996_mcu_rx_event(dev, skb);
 		break;
 	case PKT_TYPE_TXS:
-		for (rxd += 4; rxd + 8 <= end; rxd += 8)
+		for (rxd += MT_TXS_HDR_SIZE; rxd + MT_TXS_SIZE <= end; rxd += MT_TXS_SIZE)
 			mt7996_mac_add_txs(dev, rxd);
 		dev_kfree_skb(skb);
 		break;
@@ -1603,43 +1715,53 @@ mt7996_wait_reset_state(struct mt7996_dev *dev, u32 state)
 static void
 mt7996_update_vif_beacon(void *priv, u8 *mac, struct ieee80211_vif *vif)
 {
-	struct ieee80211_hw *hw = priv;
+	struct ieee80211_bss_conf *link_conf;
+	struct mt7996_phy *phy = priv;
+	struct mt7996_dev *dev = phy->dev;
+	unsigned int link_id;
+
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_MESH_POINT:
 	case NL80211_IFTYPE_ADHOC:
 	case NL80211_IFTYPE_AP:
-		mt7996_mcu_add_beacon(hw, vif, &vif->bss_conf);
 		break;
 	default:
-		break;
+		return;
+	}
+
+	for_each_vif_active_link(vif, link_conf, link_id) {
+		struct mt7996_vif_link *link;
+
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link || link->phy != phy)
+			continue;
+
+		mt7996_mcu_add_beacon(dev->mt76.hw, vif, link_conf);
 	}
 }
 
+void mt7996_mac_update_beacons(struct mt7996_phy *phy)
+{
+	ieee80211_iterate_active_interfaces(phy->mt76->hw,
+					    IEEE80211_IFACE_ITER_RESUME_ALL,
+					    mt7996_update_vif_beacon, phy);
+}
+
 static void
 mt7996_update_beacons(struct mt7996_dev *dev)
 {
 	struct mt76_phy *phy2, *phy3;
 
-	ieee80211_iterate_active_interfaces(dev->mt76.hw,
-					    IEEE80211_IFACE_ITER_RESUME_ALL,
-					    mt7996_update_vif_beacon, dev->mt76.hw);
+	mt7996_mac_update_beacons(&dev->phy);
 
 	phy2 = dev->mt76.phys[MT_BAND1];
-	if (!phy2)
-		return;
-
-	ieee80211_iterate_active_interfaces(phy2->hw,
-					    IEEE80211_IFACE_ITER_RESUME_ALL,
-					    mt7996_update_vif_beacon, phy2->hw);
+	if (phy2)
+		mt7996_mac_update_beacons(phy2->priv);
 
 	phy3 = dev->mt76.phys[MT_BAND2];
-	if (!phy3)
-		return;
-
-	ieee80211_iterate_active_interfaces(phy3->hw,
-					    IEEE80211_IFACE_ITER_RESUME_ALL,
-					    mt7996_update_vif_beacon, phy3->hw);
+	if (phy3)
+		mt7996_mac_update_beacons(phy3->priv);
 }
 
 void mt7996_tx_token_put(struct mt7996_dev *dev)
@@ -1649,7 +1771,7 @@ void mt7996_tx_token_put(struct mt7996_dev *dev)
 
 	spin_lock_bh(&dev->mt76.token_lock);
 	idr_for_each_entry(&dev->mt76.token, txwi, id) {
-		mt7996_txwi_free(dev, txwi, NULL, NULL);
+		mt7996_txwi_free(dev, txwi, NULL, NULL, NULL);
 		dev->mt76.token_count--;
 	}
 	spin_unlock_bh(&dev->mt76.token_lock);
@@ -2254,32 +2376,38 @@ void mt7996_mac_update_stats(struct mt7996_phy *phy)
 void mt7996_mac_sta_rc_work(struct work_struct *work)
 {
 	struct mt7996_dev *dev = container_of(work, struct mt7996_dev, rc_work);
-	struct ieee80211_sta *sta;
+	struct mt7996_sta_link *msta_link;
 	struct ieee80211_vif *vif;
-	struct mt7996_sta *msta;
-	u32 changed;
+	struct mt7996_vif *mvif;
 	LIST_HEAD(list);
+	u32 changed;
 
 	spin_lock_bh(&dev->mt76.sta_poll_lock);
 	list_splice_init(&dev->sta_rc_list, &list);
 
 	while (!list_empty(&list)) {
-		msta = list_first_entry(&list, struct mt7996_sta, rc_list);
-		list_del_init(&msta->rc_list);
-		changed = msta->changed;
-		msta->changed = 0;
-		spin_unlock_bh(&dev->mt76.sta_poll_lock);
+		msta_link = list_first_entry(&list, struct mt7996_sta_link,
+					     rc_list);
+		list_del_init(&msta_link->rc_list);
+
+		changed = msta_link->changed;
+		msta_link->changed = 0;
+		mvif = msta_link->sta->vif;
+		vif = container_of((void *)mvif, struct ieee80211_vif,
+				   drv_priv);
 
-		sta = container_of((void *)msta, struct ieee80211_sta, drv_priv);
-		vif = container_of((void *)msta->vif, struct ieee80211_vif, drv_priv);
+		spin_unlock_bh(&dev->mt76.sta_poll_lock);
 
 		if (changed & (IEEE80211_RC_SUPP_RATES_CHANGED |
 			       IEEE80211_RC_NSS_CHANGED |
 			       IEEE80211_RC_BW_CHANGED))
-			mt7996_mcu_add_rate_ctrl(dev, vif, sta, true);
+			mt7996_mcu_add_rate_ctrl(dev, msta_link->sta, vif,
+						 msta_link->wcid.link_id,
+						 true);
 
 		if (changed & IEEE80211_RC_SMPS_CHANGED)
-			mt7996_mcu_set_fixed_field(dev, vif, sta, NULL,
+			mt7996_mcu_set_fixed_field(dev, msta_link->sta, NULL,
+						   msta_link->wcid.link_id,
 						   RATE_PARAM_MMPS_UPDATE);
 
 		spin_lock_bh(&dev->mt76.sta_poll_lock);
@@ -2323,16 +2451,15 @@ void mt7996_mac_work(struct work_struct *work)
 static void mt7996_dfs_stop_radar_detector(struct mt7996_phy *phy)
 {
 	struct mt7996_dev *dev = phy->dev;
+	int rdd_idx = mt7996_get_rdd_idx(phy, false);
 
-	if (phy->rdd_state & BIT(0))
-		mt7996_mcu_rdd_cmd(dev, RDD_STOP, 0,
-				   MT_RX_SEL0, 0);
-	if (phy->rdd_state & BIT(1))
-		mt7996_mcu_rdd_cmd(dev, RDD_STOP, 1,
-				   MT_RX_SEL0, 0);
+	if (rdd_idx < 0)
+		return;
+
+	mt7996_mcu_rdd_cmd(dev, RDD_STOP, rdd_idx, 0);
 }
 
-static int mt7996_dfs_start_rdd(struct mt7996_dev *dev, int chain)
+static int mt7996_dfs_start_rdd(struct mt7996_dev *dev, int rdd_idx)
 {
 	int err, region;
 
@@ -2349,44 +2476,30 @@ static int mt7996_dfs_start_rdd(struct mt7996_dev *dev, int chain)
 		break;
 	}
 
-	err = mt7996_mcu_rdd_cmd(dev, RDD_START, chain,
-				 MT_RX_SEL0, region);
+	err = mt7996_mcu_rdd_cmd(dev, RDD_START, rdd_idx, region);
 	if (err < 0)
 		return err;
 
-	return mt7996_mcu_rdd_cmd(dev, RDD_DET_MODE, chain,
-				 MT_RX_SEL0, 1);
+	return mt7996_mcu_rdd_cmd(dev, RDD_DET_MODE, rdd_idx, 1);
 }
 
 static int mt7996_dfs_start_radar_detector(struct mt7996_phy *phy)
 {
-	struct cfg80211_chan_def *chandef = &phy->mt76->chandef;
 	struct mt7996_dev *dev = phy->dev;
-	u8 band_idx = phy->mt76->band_idx;
-	int err;
+	int err, rdd_idx;
 
-	/* start CAC */
-	err = mt7996_mcu_rdd_cmd(dev, RDD_CAC_START, band_idx,
-				 MT_RX_SEL0, 0);
-	if (err < 0)
-		return err;
+	rdd_idx = mt7996_get_rdd_idx(phy, false);
+	if (rdd_idx < 0)
+		return -EINVAL;
 
-	err = mt7996_dfs_start_rdd(dev, band_idx);
+	/* start CAC */
+	err = mt7996_mcu_rdd_cmd(dev, RDD_CAC_START, rdd_idx, 0);
 	if (err < 0)
 		return err;
 
-	phy->rdd_state |= BIT(band_idx);
+	err = mt7996_dfs_start_rdd(dev, rdd_idx);
 
-	if (chandef->width == NL80211_CHAN_WIDTH_160 ||
-	    chandef->width == NL80211_CHAN_WIDTH_80P80) {
-		err = mt7996_dfs_start_rdd(dev, 1);
-		if (err < 0)
-			return err;
-
-		phy->rdd_state |= BIT(1);
-	}
-
-	return 0;
+	return err;
 }
 
 static int
@@ -2427,12 +2540,12 @@ int mt7996_dfs_init_radar_detector(struct mt7996_phy *phy)
 {
 	struct mt7996_dev *dev = phy->dev;
 	enum mt76_dfs_state dfs_state, prev_state;
-	int err;
+	int err, rdd_idx = mt7996_get_rdd_idx(phy, false);
 
 	prev_state = phy->mt76->dfs_state;
 	dfs_state = mt76_phy_dfs_state(phy->mt76);
 
-	if (prev_state == dfs_state)
+	if (prev_state == dfs_state || rdd_idx < 0)
 		return 0;
 
 	if (prev_state == MT_DFS_STATE_UNKNOWN)
@@ -2456,8 +2569,7 @@ int mt7996_dfs_init_radar_detector(struct mt7996_phy *phy)
 	if (dfs_state == MT_DFS_STATE_CAC)
 		return 0;
 
-	err = mt7996_mcu_rdd_cmd(dev, RDD_CAC_END,
-				 phy->mt76->band_idx, MT_RX_SEL0, 0);
+	err = mt7996_mcu_rdd_cmd(dev, RDD_CAC_END, rdd_idx, 0);
 	if (err < 0) {
 		phy->mt76->dfs_state = MT_DFS_STATE_UNKNOWN;
 		return err;
@@ -2467,8 +2579,7 @@ int mt7996_dfs_init_radar_detector(struct mt7996_phy *phy)
 	return 0;
 
 stop:
-	err = mt7996_mcu_rdd_cmd(dev, RDD_NORMAL_START,
-				 phy->mt76->band_idx, MT_RX_SEL0, 0);
+	err = mt7996_mcu_rdd_cmd(dev, RDD_NORMAL_START, rdd_idx, 0);
 	if (err < 0)
 		return err;
 
@@ -2553,7 +2664,7 @@ static int mt7996_mac_check_twt_req(struct ieee80211_twt_setup *twt)
 }
 
 static bool
-mt7996_mac_twt_param_equal(struct mt7996_sta *msta,
+mt7996_mac_twt_param_equal(struct mt7996_sta_link *msta_link,
 			   struct ieee80211_twt_params *twt_agrt)
 {
 	u16 type = le16_to_cpu(twt_agrt->req_type);
@@ -2564,10 +2675,10 @@ mt7996_mac_twt_param_equal(struct mt7996_sta *msta,
 	for (i = 0; i < MT7996_MAX_STA_TWT_AGRT; i++) {
 		struct mt7996_twt_flow *f;
 
-		if (!(msta->twt.flowid_mask & BIT(i)))
+		if (!(msta_link->twt.flowid_mask & BIT(i)))
 			continue;
 
-		f = &msta->twt.flow[i];
+		f = &msta_link->twt.flow[i];
 		if (f->duration == twt_agrt->min_twt_dur &&
 		    f->mantissa == twt_agrt->mantissa &&
 		    f->exp == exp &&
@@ -2587,6 +2698,7 @@ void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw,
 	enum ieee80211_twt_setup_cmd setup_cmd = TWT_SETUP_CMD_REJECT;
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
 	struct ieee80211_twt_params *twt_agrt = (void *)twt->params;
+	struct mt7996_sta_link *msta_link = &msta->deflink;
 	u16 req_type = le16_to_cpu(twt_agrt->req_type);
 	enum ieee80211_twt_setup_cmd sta_setup_cmd;
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
@@ -2601,7 +2713,8 @@ void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw,
 	if (dev->twt.n_agrt == MT7996_MAX_TWT_AGRT)
 		goto unlock;
 
-	if (hweight8(msta->twt.flowid_mask) == ARRAY_SIZE(msta->twt.flow))
+	if (hweight8(msta_link->twt.flowid_mask) ==
+	    ARRAY_SIZE(msta_link->twt.flow))
 		goto unlock;
 
 	if (twt_agrt->min_twt_dur < MT7996_MIN_TWT_DUR) {
@@ -2610,10 +2723,10 @@ void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw,
 		goto unlock;
 	}
 
-	if (mt7996_mac_twt_param_equal(msta, twt_agrt))
+	if (mt7996_mac_twt_param_equal(msta_link, twt_agrt))
 		goto unlock;
 
-	flowid = ffs(~msta->twt.flowid_mask) - 1;
+	flowid = ffs(~msta_link->twt.flowid_mask) - 1;
 	twt_agrt->req_type &= ~cpu_to_le16(IEEE80211_TWT_REQTYPE_FLOWID);
 	twt_agrt->req_type |= le16_encode_bits(flowid,
 					       IEEE80211_TWT_REQTYPE_FLOWID);
@@ -2622,10 +2735,10 @@ void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw,
 	exp = FIELD_GET(IEEE80211_TWT_REQTYPE_WAKE_INT_EXP, req_type);
 	sta_setup_cmd = FIELD_GET(IEEE80211_TWT_REQTYPE_SETUP_CMD, req_type);
 
-	flow = &msta->twt.flow[flowid];
+	flow = &msta_link->twt.flow[flowid];
 	memset(flow, 0, sizeof(*flow));
 	INIT_LIST_HEAD(&flow->list);
-	flow->wcid = msta->wcid.idx;
+	flow->wcid = msta_link->wcid.idx;
 	flow->table_id = table_id;
 	flow->id = flowid;
 	flow->duration = twt_agrt->min_twt_dur;
@@ -2643,7 +2756,7 @@ void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw,
 
 		flow->sched = true;
 		flow->start_tsf = mt7996_mac_twt_sched_list_add(dev, flow);
-		curr_tsf = __mt7996_get_tsf(hw, msta->vif);
+		curr_tsf = __mt7996_get_tsf(hw, &msta->vif->deflink);
 		div_u64_rem(curr_tsf - flow->start_tsf, interval, &rem);
 		flow_tsf = curr_tsf + interval - rem;
 		twt_agrt->twt = cpu_to_le64(flow_tsf);
@@ -2652,12 +2765,13 @@ void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw,
 	}
 	flow->tsf = le64_to_cpu(twt_agrt->twt);
 
-	if (mt7996_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD))
+	if (mt7996_mcu_twt_agrt_update(dev, &msta->vif->deflink, flow,
+				       MCU_TWT_AGRT_ADD))
 		goto unlock;
 
 	setup_cmd = TWT_SETUP_CMD_ACCEPT;
 	dev->twt.table_mask |= BIT(table_id);
-	msta->twt.flowid_mask |= BIT(flowid);
+	msta_link->twt.flowid_mask |= BIT(flowid);
 	dev->twt.n_agrt++;
 
 unlock:
@@ -2670,26 +2784,26 @@ out:
 }
 
 void mt7996_mac_twt_teardown_flow(struct mt7996_dev *dev,
-				  struct mt7996_sta *msta,
+				  struct mt7996_vif_link *link,
+				  struct mt7996_sta_link *msta_link,
 				  u8 flowid)
 {
 	struct mt7996_twt_flow *flow;
 
 	lockdep_assert_held(&dev->mt76.mutex);
 
-	if (flowid >= ARRAY_SIZE(msta->twt.flow))
+	if (flowid >= ARRAY_SIZE(msta_link->twt.flow))
 		return;
 
-	if (!(msta->twt.flowid_mask & BIT(flowid)))
+	if (!(msta_link->twt.flowid_mask & BIT(flowid)))
 		return;
 
-	flow = &msta->twt.flow[flowid];
-	if (mt7996_mcu_twt_agrt_update(dev, msta->vif, flow,
-				       MCU_TWT_AGRT_DELETE))
+	flow = &msta_link->twt.flow[flowid];
+	if (mt7996_mcu_twt_agrt_update(dev, link, flow, MCU_TWT_AGRT_DELETE))
 		return;
 
 	list_del_init(&flow->list);
-	msta->twt.flowid_mask &= ~BIT(flowid);
+	msta_link->twt.flowid_mask &= ~BIT(flowid);
 	dev->twt.table_mask &= ~BIT(flow->table_id);
 	dev->twt.n_agrt--;
 }
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/main.c b/sys/contrib/dev/mediatek/mt76/mt7996/main.c
index 69dd565d8319..84f731b387d2 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/main.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/main.c
@@ -68,11 +68,13 @@ static int mt7996_start(struct ieee80211_hw *hw)
 
 static void mt7996_stop_phy(struct mt7996_phy *phy)
 {
-	struct mt7996_dev *dev = phy->dev;
+	struct mt7996_dev *dev;
 
 	if (!phy || !test_bit(MT76_STATE_RUNNING, &phy->mt76->state))
 		return;
 
+	dev = phy->dev;
+
 	cancel_delayed_work_sync(&phy->mt76->mac_work);
 
 	mutex_lock(&dev->mt76.mutex);
@@ -158,58 +160,101 @@ mt7996_init_bitrate_mask(struct ieee80211_vif *vif, struct mt7996_vif_link *mlin
 static int
 mt7996_set_hw_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		  struct ieee80211_vif *vif, struct ieee80211_sta *sta,
-		  struct mt7996_vif_link *mlink, struct ieee80211_key_conf *key)
+		  struct ieee80211_key_conf *key)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	struct mt7996_sta *msta = sta ? (struct mt7996_sta *)sta->drv_priv :
-				  &mlink->sta;
-	struct mt76_wcid *wcid = &msta->wcid;
-	u8 *wcid_keyidx = &wcid->hw_key_idx;
-	struct mt7996_phy *phy;
 	int idx = key->keyidx;
+	unsigned int link_id;
+	unsigned long links;
+
+	if (key->link_id >= 0)
+		links = BIT(key->link_id);
+	else if (sta && sta->valid_links)
+		links = sta->valid_links;
+	else if (vif->valid_links)
+		links = vif->valid_links;
+	else
+		links = BIT(0);
 
-	phy = mt7996_vif_link_phy(mlink);
-	if (!phy)
-		return -EINVAL;
+	for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct mt7996_sta_link *msta_link;
+		struct mt7996_vif_link *link;
+		u8 *wcid_keyidx;
+		int err;
 
-	if (sta && !wcid->sta)
-		return -EOPNOTSUPP;
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link)
+			continue;
 
-	switch (key->cipher) {
-	case WLAN_CIPHER_SUITE_AES_CMAC:
-	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
-	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
-	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
-		if (key->keyidx == 6 || key->keyidx == 7) {
-			wcid_keyidx = &wcid->hw_key_idx2;
-			key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE;
+		if (sta) {
+			struct mt7996_sta *msta;
+
+			msta = (struct mt7996_sta *)sta->drv_priv;
+			msta_link = mt76_dereference(msta->link[link_id],
+						     &dev->mt76);
+			if (!msta_link)
+				continue;
+
+			if (!msta_link->wcid.sta)
+				return -EOPNOTSUPP;
+		} else {
+			msta_link = &link->msta_link;
+		}
+		wcid_keyidx = &msta_link->wcid.hw_key_idx;
+
+		switch (key->cipher) {
+		case WLAN_CIPHER_SUITE_AES_CMAC:
+		case WLAN_CIPHER_SUITE_BIP_CMAC_256:
+		case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+		case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+			if (key->keyidx == 6 || key->keyidx == 7) {
+				wcid_keyidx = &msta_link->wcid.hw_key_idx2;
+				key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE;
+			}
+			break;
+		default:
+			break;
 		}
-		break;
-	default:
-		break;
-	}
 
-	if (cmd == SET_KEY && !sta && !mlink->mt76.cipher) {
-		mlink->mt76.cipher = mt76_connac_mcu_get_cipher(key->cipher);
-		mt7996_mcu_add_bss_info(phy, vif, &vif->bss_conf, &mlink->mt76, true);
-	}
+		if (cmd == SET_KEY && !sta && !link->mt76.cipher) {
+			struct ieee80211_bss_conf *link_conf;
 
-	if (cmd == SET_KEY) {
-		*wcid_keyidx = idx;
-	} else {
-		if (idx == *wcid_keyidx)
-			*wcid_keyidx = -1;
-		return 0;
-	}
+			link_conf = link_conf_dereference_protected(vif,
+								    link_id);
+			if (!link_conf)
+				link_conf = &vif->bss_conf;
 
-	mt76_wcid_key_setup(&dev->mt76, wcid, key);
+			link->mt76.cipher =
+				mt76_connac_mcu_get_cipher(key->cipher);
+			mt7996_mcu_add_bss_info(link->phy, vif, link_conf,
+						&link->mt76, msta_link, true);
+		}
 
-	if (key->keyidx == 6 || key->keyidx == 7)
-		return mt7996_mcu_bcn_prot_enable(dev, vif, key);
+		if (cmd == SET_KEY) {
+			*wcid_keyidx = idx;
+		} else {
+			if (idx == *wcid_keyidx)
+				*wcid_keyidx = -1;
+			continue;
+		}
+
+		mt76_wcid_key_setup(&dev->mt76, &msta_link->wcid, key);
+
+		if (key->keyidx == 6 || key->keyidx == 7) {
+			err = mt7996_mcu_bcn_prot_enable(dev, link,
+							 msta_link, key);
+			if (err)
+				return err;
+		}
+
+		err = mt7996_mcu_add_key(&dev->mt76, vif, key,
+					 MCU_WMWA_UNI_CMD(STA_REC_UPDATE),
+					 &msta_link->wcid, cmd);
+		if (err)
+			return err;
+	}
 
-	return mt7996_mcu_add_key(&dev->mt76, vif, key,
-				  MCU_WMWA_UNI_CMD(STA_REC_UPDATE),
-				  &msta->wcid, cmd);
+	return 0;
 }
 
 static void
@@ -217,12 +262,10 @@ mt7996_key_iter(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		struct ieee80211_sta *sta, struct ieee80211_key_conf *key,
 		void *data)
 {
-	struct mt7996_vif_link *mlink = data;
-
 	if (sta)
 		return;
 
-	WARN_ON(mt7996_set_hw_key(hw, SET_KEY, vif, NULL, mlink, key));
+	WARN_ON(mt7996_set_hw_key(hw, SET_KEY, vif, NULL, key));
 }
 
 int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif,
@@ -230,6 +273,8 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif,
 			struct mt76_vif_link *mlink)
 {
 	struct mt7996_vif_link *link = container_of(mlink, struct mt7996_vif_link, mt76);
+	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
+	struct mt7996_sta_link *msta_link = &link->msta_link;
 	struct mt7996_phy *phy = mphy->priv;
 	struct mt7996_dev *dev = phy->dev;
 	u8 band_idx = phy->mt76->band_idx;
@@ -248,7 +293,8 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif,
 	mlink->omac_idx = idx;
 	mlink->band_idx = band_idx;
 	mlink->wmm_idx = vif->type == NL80211_IFTYPE_AP ? 0 : 3;
-	mlink->wcid = &link->sta.wcid;
+	mlink->wcid = &msta_link->wcid;
+	mlink->wcid->offchannel = mlink->offchannel;
 
 	ret = mt7996_mcu_add_dev_info(phy, vif, link_conf, mlink, true);
 	if (ret)
@@ -259,10 +305,11 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif,
 
 	idx = MT7996_WTBL_RESERVED - mlink->idx;
 
-	INIT_LIST_HEAD(&link->sta.rc_list);
-	link->sta.wcid.idx = idx;
-	link->sta.wcid.tx_info |= MT_WCID_TX_INFO_SET;
-	mt76_wcid_init(&link->sta.wcid, band_idx);
+	INIT_LIST_HEAD(&msta_link->rc_list);
+	msta_link->wcid.idx = idx;
+	msta_link->wcid.link_id = link_conf->link_id;
+	msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET;
+	mt76_wcid_init(&msta_link->wcid, band_idx);
 
 	mt7996_mac_wtbl_update(dev, idx,
 			       MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
@@ -283,15 +330,19 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif,
 
 	mt7996_init_bitrate_mask(vif, link);
 
-	mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, true);
+	mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, msta_link, true);
 	/* defer the first STA_REC of BMC entry to BSS_CHANGED_BSSID for STA
 	 * interface, since firmware only records BSSID when the entry is new
 	 */
 	if (vif->type != NL80211_IFTYPE_STATION)
-		mt7996_mcu_add_sta(dev, vif, mlink, NULL, CONN_STATE_PORT_SECURE, true);
-	rcu_assign_pointer(dev->mt76.wcid[idx], &link->sta.wcid);
+		mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL,
+				   CONN_STATE_PORT_SECURE, true);
+	rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid);
 
-	ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, link);
+	ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, NULL);
+
+	if (mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED)
+		mvif->mt76.deflink_id = link_conf->link_id;
 
 	return 0;
 }
@@ -301,29 +352,42 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif,
 			    struct mt76_vif_link *mlink)
 {
 	struct mt7996_vif_link *link = container_of(mlink, struct mt7996_vif_link, mt76);
+	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
+	struct mt7996_sta_link *msta_link = &link->msta_link;
 	struct mt7996_phy *phy = mphy->priv;
 	struct mt7996_dev *dev = phy->dev;
-	struct mt7996_sta *msta;
-	int idx;
+	int idx = msta_link->wcid.idx;
 
-	msta = &link->sta;
-	idx = msta->wcid.idx;
-	mt7996_mcu_add_sta(dev, vif, mlink, NULL, CONN_STATE_DISCONNECT, false);
-	mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, false);
+	mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL,
+			   CONN_STATE_DISCONNECT, false);
+	mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, msta_link, false);
 
 	mt7996_mcu_add_dev_info(phy, vif, link_conf, mlink, false);
 
 	rcu_assign_pointer(dev->mt76.wcid[idx], NULL);
 
+	if (mvif->mt76.deflink_id == link_conf->link_id) {
+		struct ieee80211_bss_conf *iter;
+		unsigned int link_id;
+
+		mvif->mt76.deflink_id = IEEE80211_LINK_UNSPECIFIED;
+		for_each_vif_active_link(vif, iter, link_id) {
+			if (link_id != IEEE80211_LINK_UNSPECIFIED) {
+				mvif->mt76.deflink_id = link_id;
+				break;
+			}
+		}
+	}
+
 	dev->mt76.vif_mask &= ~BIT_ULL(mlink->idx);
 	phy->omac_mask &= ~BIT_ULL(mlink->omac_idx);
 
 	spin_lock_bh(&dev->mt76.sta_poll_lock);
-	if (!list_empty(&msta->wcid.poll_list))
-		list_del_init(&msta->wcid.poll_list);
+	if (!list_empty(&msta_link->wcid.poll_list))
+		list_del_init(&msta_link->wcid.poll_list);
 	spin_unlock_bh(&dev->mt76.sta_poll_lock);
 
-	mt76_wcid_cleanup(&dev->mt76, &msta->wcid);
+	mt76_wcid_cleanup(&dev->mt76, &msta_link->wcid);
 }
 
 static void mt7996_phy_set_rxfilter(struct mt7996_phy *phy)
@@ -352,11 +416,13 @@ static void mt7996_phy_set_rxfilter(struct mt7996_phy *phy)
 
 static void mt7996_set_monitor(struct mt7996_phy *phy, bool enabled)
 {
-	struct mt7996_dev *dev = phy->dev;
+	struct mt7996_dev *dev;
 
 	if (!phy)
 		return;
 
+	dev = phy->dev;
+
 	if (enabled == !(phy->rxfilter & MT_WF_RFCR_DROP_OTHER_UC))
 		return;
 
@@ -399,6 +465,7 @@ static int mt7996_add_interface(struct ieee80211_hw *hw,
 	mt76_vif_init(vif, &mvif->mt76);
 
 	vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR;
+	mvif->mt76.deflink_id = IEEE80211_LINK_UNSPECIFIED;
 
 out:
 	mutex_unlock(&dev->mt76.mutex);
@@ -449,6 +516,9 @@ int mt7996_set_channel(struct mt76_phy *mphy)
 	struct mt7996_phy *phy = mphy->priv;
 	int ret;
 
+	if (mphy->offchannel)
+		mt7996_mac_update_beacons(phy);
+
 	ret = mt7996_mcu_set_chan_info(phy, UNI_CHANNEL_SWITCH);
 	if (ret)
 		goto out;
@@ -466,6 +536,8 @@ int mt7996_set_channel(struct mt76_phy *mphy)
 
 	mt7996_mac_reset_counters(phy);
 	phy->noise = 0;
+	if (!mphy->offchannel)
+		mt7996_mac_update_beacons(phy);
 
 out:
 	ieee80211_queue_delayed_work(mphy->hw, &mphy->mac_work,
@@ -480,7 +552,6 @@ static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_vif_link *mlink = &mvif->deflink;
 	int err;
 
 	/* The hardware does not support per-STA RX GTK, fallback
@@ -515,17 +586,17 @@ static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		return -EOPNOTSUPP;
 	}
 
-	if (!mt7996_vif_link_phy(mlink))
-	    return 0; /* defer until after link add */
+	if (!mt7996_vif_link_phy(&mvif->deflink))
+		return 0; /* defer until after link add */
 
 	mutex_lock(&dev->mt76.mutex);
-	err = mt7996_set_hw_key(hw, cmd, vif, sta, mlink, key);
+	err = mt7996_set_hw_key(hw, cmd, vif, sta, key);
 	mutex_unlock(&dev->mt76.mutex);
 
 	return err;
 }
 
-static int mt7996_config(struct ieee80211_hw *hw, u32 changed)
+static int mt7996_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	return 0;
 }
@@ -601,6 +672,33 @@ static void mt7996_configure_filter(struct ieee80211_hw *hw,
 	mutex_unlock(&dev->mt76.mutex);
 }
 
+static int
+mt7996_get_txpower(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		   unsigned int link_id, int *dbm)
+{
+	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
+	struct mt7996_phy *phy = mt7996_vif_link_phy(&mvif->deflink);
+	struct mt7996_dev *dev = mt7996_hw_dev(hw);
+	struct wireless_dev *wdev;
+	int n_chains, delta, i;
+
+	if (!phy) {
+		wdev = ieee80211_vif_to_wdev(vif);
+		for (i = 0; i < hw->wiphy->n_radio; i++)
+			if (wdev->radio_mask & BIT(i))
+				phy = dev->radio_phy[i];
+
+		if (!phy)
+			return -EINVAL;
+	}
+
+	n_chains = hweight16(phy->mt76->chainmask);
+	delta = mt76_tx_power_path_delta(n_chains);
+	*dbm = DIV_ROUND_UP(phy->mt76->txpower_cur + delta, 2);
+
+	return 0;
+}
+
 static u8
 mt7996_get_rates_table(struct mt7996_phy *phy, struct ieee80211_bss_conf *conf,
 		       bool beacon, bool mcast)
@@ -630,12 +728,11 @@ mt7996_get_rates_table(struct mt7996_phy *phy, struct ieee80211_bss_conf *conf,
 }
 
 static void
-mt7996_update_mu_group(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+mt7996_update_mu_group(struct ieee80211_hw *hw, struct mt7996_vif_link *link,
 		       struct ieee80211_bss_conf *info)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	u8 band = mvif->deflink.mt76.band_idx;
+	u8 band = link->mt76.band_idx;
 	u32 *mu;
 
 	mu = (u32 *)info->mu_group.membership;
@@ -649,23 +746,56 @@ mt7996_update_mu_group(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	mt76_wr(dev, MT_WF_PHYRX_BAND_GID_TAB_POS3(band), mu[3]);
 }
 
-static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
-				    struct ieee80211_vif *vif,
-				    struct ieee80211_bss_conf *info,
-				    u64 changed)
+static void
+mt7996_vif_cfg_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       u64 changed)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	struct mt76_vif_link *mvif;
+
+	mutex_lock(&dev->mt76.mutex);
+
+	if ((changed & BSS_CHANGED_ASSOC) && vif->cfg.assoc) {
+		struct ieee80211_bss_conf *link_conf;
+		unsigned long link_id;
+
+		for_each_vif_active_link(vif, link_conf, link_id) {
+			struct mt7996_vif_link *link;
+
+			link = mt7996_vif_link(dev, vif, link_id);
+			if (!link)
+				continue;
+
+			if (!link->phy)
+				continue;
+
+			mt7996_mcu_add_bss_info(link->phy, vif, link_conf,
+						&link->mt76, &link->msta_link,
+						true);
+			mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL,
+					   CONN_STATE_PORT_SECURE,
+					   !!(changed & BSS_CHANGED_BSSID));
+		}
+	}
+
+	mutex_unlock(&dev->mt76.mutex);
+}
+
+static void
+mt7996_link_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			 struct ieee80211_bss_conf *info, u64 changed)
+{
+	struct mt7996_dev *dev = mt7996_hw_dev(hw);
+	struct mt7996_vif_link *link;
 	struct mt7996_phy *phy;
 	struct mt76_phy *mphy;
 
 	mutex_lock(&dev->mt76.mutex);
 
-	mvif = mt76_vif_conf_link(&dev->mt76, vif, info);
-	if (!mvif)
+	link = mt7996_vif_conf_link(dev, vif, info);
+	if (!link)
 		goto out;
 
-	mphy = mt76_vif_link_phy(mvif);
+	mphy = mt76_vif_link_phy(&link->mt76);
 	if (!mphy)
 		goto out;
 
@@ -675,16 +805,14 @@ static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
 	 * and then peer references bss_info_rfch to set bandwidth cap.
 	 */
 	if ((changed & BSS_CHANGED_BSSID && !is_zero_ether_addr(info->bssid)) ||
-	    (changed & BSS_CHANGED_ASSOC && vif->cfg.assoc) ||
 	    (changed & BSS_CHANGED_BEACON_ENABLED && info->enable_beacon)) {
-		mt7996_mcu_add_bss_info(phy, vif, info, mvif, true);
-		mt7996_mcu_add_sta(dev, vif, mvif, NULL, CONN_STATE_PORT_SECURE,
+		mt7996_mcu_add_bss_info(phy, vif, info, &link->mt76,
+					&link->msta_link, true);
+		mt7996_mcu_add_sta(dev, info, NULL, link, NULL,
+				   CONN_STATE_PORT_SECURE,
 				   !!(changed & BSS_CHANGED_BSSID));
 	}
 
-	if (changed & BSS_CHANGED_ERP_CTS_PROT)
-		mt7996_mac_enable_rtscts(dev, vif, info->use_cts_prot);
-
 	if (changed & BSS_CHANGED_ERP_SLOT) {
 		int slottime = info->use_short_slot ? 9 : 20;
 
@@ -695,11 +823,11 @@ static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
 	}
 
 	if (changed & BSS_CHANGED_MCAST_RATE)
-		mvif->mcast_rates_idx =
+		link->mt76.mcast_rates_idx =
 			mt7996_get_rates_table(phy, info, false, true);
 
 	if (changed & BSS_CHANGED_BASIC_RATES)
-		mvif->basic_rates_idx =
+		link->mt76.basic_rates_idx =
 			mt7996_get_rates_table(phy, info, false, false);
 
 	/* ensure that enable txcmd_mode after bss_info */
@@ -707,19 +835,19 @@ static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
 		mt7996_mcu_set_tx(dev, vif, info);
 
 	if (changed & BSS_CHANGED_HE_OBSS_PD)
-		mt7996_mcu_add_obss_spr(phy, vif, &info->he_obss_pd);
+		mt7996_mcu_add_obss_spr(phy, link, &info->he_obss_pd);
 
 	if (changed & BSS_CHANGED_HE_BSS_COLOR) {
 		if ((vif->type == NL80211_IFTYPE_AP &&
-		    mvif->omac_idx <= HW_BSSID_MAX) ||
+		    link->mt76.omac_idx <= HW_BSSID_MAX) ||
 		   vif->type == NL80211_IFTYPE_STATION)
-			mt7996_mcu_update_bss_color(dev, mvif,
+			mt7996_mcu_update_bss_color(dev, &link->mt76,
 						    &info->he_bss_color);
 	}
 
 	if (changed & (BSS_CHANGED_BEACON |
 		       BSS_CHANGED_BEACON_ENABLED)) {
-		mvif->beacon_rates_idx =
+		link->mt76.beacon_rates_idx =
 			mt7996_get_rates_table(phy, info, true, false);
 
 		mt7996_mcu_add_beacon(hw, vif, info);
@@ -727,10 +855,10 @@ static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
 
 	if (changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP |
 		       BSS_CHANGED_FILS_DISCOVERY))
-		mt7996_mcu_beacon_inband_discov(dev, vif, changed);
+		mt7996_mcu_beacon_inband_discov(dev, info, link, changed);
 
 	if (changed & BSS_CHANGED_MU_GROUPS)
-		mt7996_update_mu_group(hw, vif, info);
+		mt7996_update_mu_group(hw, link, info);
 
 	if (changed & BSS_CHANGED_TXPOWER &&
 	    info->txpower != phy->txpower) {
@@ -754,96 +882,332 @@ mt7996_channel_switch_beacon(struct ieee80211_hw *hw,
 	mutex_unlock(&dev->mt76.mutex);
 }
 
-int mt7996_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif,
-		       struct ieee80211_sta *sta)
+static int
+mt7996_mac_sta_init_link(struct mt7996_dev *dev,
+			 struct ieee80211_bss_conf *link_conf,
+			 struct ieee80211_link_sta *link_sta,
+			 struct mt7996_vif_link *link, unsigned int link_id)
 {
-	struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76);
+	struct ieee80211_sta *sta = link_sta->sta;
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_vif_link *link = &mvif->deflink;
-	u8 band_idx = link->phy->mt76->band_idx;
+	struct mt7996_phy *phy = link->phy;
+	struct mt7996_sta_link *msta_link;
 	int idx;
 
 	idx = mt76_wcid_alloc(dev->mt76.wcid_mask, MT7996_WTBL_STA);
 	if (idx < 0)
 		return -ENOSPC;
 
-	INIT_LIST_HEAD(&msta->rc_list);
-	INIT_LIST_HEAD(&msta->wcid.poll_list);
-	msta->vif = mvif;
-	msta->wcid.sta = 1;
-	msta->wcid.idx = idx;
-	msta->wcid.phy_idx = band_idx;
+	if (msta->deflink_id == IEEE80211_LINK_UNSPECIFIED) {
+		int i;
 
-	ewma_avg_signal_init(&msta->avg_ack_signal);
+		msta_link = &msta->deflink;
+		msta->deflink_id = link_id;
 
-	mt7996_mac_wtbl_update(dev, idx,
+		for (i = 0; i < ARRAY_SIZE(sta->txq); i++) {
+			struct mt76_txq *mtxq;
+
+			if (!sta->txq[i])
+				continue;
+
+			mtxq = (struct mt76_txq *)sta->txq[i]->drv_priv;
+			mtxq->wcid = idx;
+		}
+	} else {
+		msta_link = kzalloc(sizeof(*msta_link), GFP_KERNEL);
+		if (!msta_link)
+			return -ENOMEM;
+	}
+
+	INIT_LIST_HEAD(&msta_link->rc_list);
+	INIT_LIST_HEAD(&msta_link->wcid.poll_list);
+	msta_link->sta = msta;
+	msta_link->wcid.sta = 1;
+	msta_link->wcid.idx = idx;
+	msta_link->wcid.link_id = link_id;
+
+	ewma_avg_signal_init(&msta_link->avg_ack_signal);
+	ewma_signal_init(&msta_link->wcid.rssi);
+
+	rcu_assign_pointer(msta->link[link_id], msta_link);
+
+	mt7996_mac_wtbl_update(dev, idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
+	mt7996_mcu_add_sta(dev, link_conf, link_sta, link, msta_link,
+			   CONN_STATE_DISCONNECT, true);
+
+	rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid);
+	mt76_wcid_init(&msta_link->wcid, phy->mt76->band_idx);
+
+	return 0;
+}
+
+static void
+mt7996_mac_sta_deinit_link(struct mt7996_dev *dev,
+			   struct mt7996_sta_link *msta_link)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(msta_link->wcid.aggr); i++)
+		mt76_rx_aggr_stop(&dev->mt76, &msta_link->wcid, i);
+
+	mt7996_mac_wtbl_update(dev, msta_link->wcid.idx,
 			       MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
-	mt7996_mcu_add_sta(dev, vif, &link->mt76, sta, CONN_STATE_DISCONNECT,
-			   true);
+
+	spin_lock_bh(&dev->mt76.sta_poll_lock);
+	if (!list_empty(&msta_link->wcid.poll_list))
+		list_del_init(&msta_link->wcid.poll_list);
+	if (!list_empty(&msta_link->rc_list))
+		list_del_init(&msta_link->rc_list);
+	spin_unlock_bh(&dev->mt76.sta_poll_lock);
+
+	mt76_wcid_cleanup(&dev->mt76, &msta_link->wcid);
+	mt76_wcid_mask_clear(dev->mt76.wcid_mask, msta_link->wcid.idx);
+}
+
+static void
+mt7996_mac_sta_remove_links(struct mt7996_dev *dev, struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta, unsigned long links)
+{
+	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct mt76_dev *mdev = &dev->mt76;
+	unsigned int link_id;
+
+	for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct mt7996_sta_link *msta_link = NULL;
+		struct mt7996_vif_link *link;
+		struct mt76_phy *mphy;
+
+		msta_link = rcu_replace_pointer(msta->link[link_id], msta_link,
+						lockdep_is_held(&mdev->mutex));
+		if (!msta_link)
+			continue;
+
+		mt7996_mac_sta_deinit_link(dev, msta_link);
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link)
+			continue;
+
+		mphy = mt76_vif_link_phy(&link->mt76);
+		if (!mphy)
+			continue;
+
+		mphy->num_sta--;
+		if (msta->deflink_id == link_id) {
+			msta->deflink_id = IEEE80211_LINK_UNSPECIFIED;
+			continue;
+		}
+
+		kfree_rcu(msta_link, rcu_head);
+	}
+}
+
+static int
+mt7996_mac_sta_add_links(struct mt7996_dev *dev, struct ieee80211_vif *vif,
+			 struct ieee80211_sta *sta, unsigned long new_links)
+{
+	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	unsigned int link_id;
+	int err = 0;
+
+	for_each_set_bit(link_id, &new_links, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link_conf;
+		struct ieee80211_link_sta *link_sta;
+		struct mt7996_vif_link *link;
+		struct mt76_phy *mphy;
+
+		if (rcu_access_pointer(msta->link[link_id]))
+			continue;
+
+		link_conf = link_conf_dereference_protected(vif, link_id);
+		if (!link_conf) {
+			err = -EINVAL;
+			goto error_unlink;
+		}
+
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link) {
+			err = -EINVAL;
+			goto error_unlink;
+		}
+
+		link_sta = link_sta_dereference_protected(sta, link_id);
+		if (!link_sta) {
+			err = -EINVAL;
+			goto error_unlink;
+		}
+
+		err = mt7996_mac_sta_init_link(dev, link_conf, link_sta, link,
+					       link_id);
+		if (err)
+			goto error_unlink;
+
+		mphy = mt76_vif_link_phy(&link->mt76);
+		if (!mphy) {
+			err = -EINVAL;
+			goto error_unlink;
+		}
+		mphy->num_sta++;
+	}
 
 	return 0;
+
+error_unlink:
+	mt7996_mac_sta_remove_links(dev, vif, sta, new_links);
+
+	return err;
 }
 
-int mt7996_mac_sta_event(struct mt76_dev *mdev, struct ieee80211_vif *vif,
-			 struct ieee80211_sta *sta, enum mt76_sta_event ev)
+static int
+mt7996_mac_sta_change_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta, u16 old_links,
+			    u16 new_links)
+{
+	struct mt7996_dev *dev = mt7996_hw_dev(hw);
+	unsigned long add = new_links & ~old_links;
+	unsigned long rem = old_links & ~new_links;
+	int ret;
+
+	mutex_lock(&dev->mt76.mutex);
+
+	mt7996_mac_sta_remove_links(dev, vif, sta, rem);
+	ret = mt7996_mac_sta_add_links(dev, vif, sta, add);
+
+	mutex_unlock(&dev->mt76.mutex);
+
+	return ret;
+}
+
+static int
+mt7996_mac_sta_add(struct mt7996_dev *dev, struct ieee80211_vif *vif,
+		   struct ieee80211_sta *sta)
 {
-	struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76);
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_vif_link *link = &mvif->deflink;
-	int i, ret;
+	unsigned long links = sta->valid_links ? sta->valid_links : BIT(0);
+	int err;
 
-	switch (ev) {
-	case MT76_STA_EVENT_ASSOC:
-		ret = mt7996_mcu_add_sta(dev, vif, &link->mt76, sta,
-					 CONN_STATE_CONNECT, true);
-		if (ret)
-			return ret;
+	mutex_lock(&dev->mt76.mutex);
 
-		ret = mt7996_mcu_add_rate_ctrl(dev, vif, sta, false);
-		if (ret)
-			return ret;
+	msta->deflink_id = IEEE80211_LINK_UNSPECIFIED;
+	msta->vif = mvif;
+	err = mt7996_mac_sta_add_links(dev, vif, sta, links);
 
-		msta->wcid.tx_info |= MT_WCID_TX_INFO_SET;
-		msta->wcid.sta = 1;
+	mutex_unlock(&dev->mt76.mutex);
 
-		return 0;
+	return err;
+}
 
-	case MT76_STA_EVENT_AUTHORIZE:
-		return mt7996_mcu_add_sta(dev, vif, &link->mt76, sta,
-					  CONN_STATE_PORT_SECURE, false);
+static int
+mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta, enum mt76_sta_event ev)
+{
+	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	unsigned long links = sta->valid_links;
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct ieee80211_bss_conf *link_conf;
+		struct mt7996_sta_link *msta_link;
+		struct mt7996_vif_link *link;
+		int i, err;
+
+		link_conf = link_conf_dereference_protected(vif, link_id);
+		if (!link_conf)
+			continue;
 
-	case MT76_STA_EVENT_DISASSOC:
-		for (i = 0; i < ARRAY_SIZE(msta->twt.flow); i++)
-			mt7996_mac_twt_teardown_flow(dev, msta, i);
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link)
+			continue;
 
-		mt7996_mcu_add_sta(dev, vif, &link->mt76, sta,
-				   CONN_STATE_DISCONNECT, false);
-		msta->wcid.sta_disabled = 1;
-		msta->wcid.sta = 0;
+		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
+		if (!msta_link)
+			continue;
 
-		return 0;
+		switch (ev) {
+		case MT76_STA_EVENT_ASSOC:
+			err = mt7996_mcu_add_sta(dev, link_conf, link_sta,
+						 link, msta_link,
+						 CONN_STATE_CONNECT, true);
+			if (err)
+				return err;
+
+			err = mt7996_mcu_add_rate_ctrl(dev, msta_link->sta, vif,
+						       link_id, false);
+			if (err)
+				return err;
+
+			msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET;
+			break;
+		case MT76_STA_EVENT_AUTHORIZE:
+			err = mt7996_mcu_add_sta(dev, link_conf, link_sta,
+						 link, msta_link,
+						 CONN_STATE_PORT_SECURE, false);
+			if (err)
+				return err;
+			break;
+		case MT76_STA_EVENT_DISASSOC:
+			for (i = 0; i < ARRAY_SIZE(msta_link->twt.flow); i++)
+				mt7996_mac_twt_teardown_flow(dev, link,
+							     msta_link, i);
+
+			if (sta->mlo && links == BIT(link_id)) /* last link */
+				mt7996_mcu_teardown_mld_sta(dev, link,
+							    msta_link);
+			else
+				mt7996_mcu_add_sta(dev, link_conf, link_sta,
+						   link, msta_link,
+						   CONN_STATE_DISCONNECT, false);
+			msta_link->wcid.sta_disabled = 1;
+			msta_link->wcid.sta = 0;
+			links = links & ~BIT(link_id);
+			break;
+		}
 	}
 
 	return 0;
 }
 
-void mt7996_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif,
-			   struct ieee80211_sta *sta)
+static void
+mt7996_mac_sta_remove(struct mt7996_dev *dev, struct ieee80211_vif *vif,
+		      struct ieee80211_sta *sta)
 {
-	struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76);
-	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	unsigned long links = sta->valid_links ? sta->valid_links : BIT(0);
 
-	mt7996_mac_wtbl_update(dev, msta->wcid.idx,
-			       MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
+	mutex_lock(&dev->mt76.mutex);
+	mt7996_mac_sta_remove_links(dev, vif, sta, links);
+	mutex_unlock(&dev->mt76.mutex);
+}
+
+static int
+mt7996_sta_state(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		 struct ieee80211_sta *sta, enum ieee80211_sta_state old_state,
+		 enum ieee80211_sta_state new_state)
+{
+	struct mt7996_dev *dev = mt7996_hw_dev(hw);
+	enum mt76_sta_event ev;
+
+	if (old_state == IEEE80211_STA_NOTEXIST &&
+	    new_state == IEEE80211_STA_NONE)
+		return mt7996_mac_sta_add(dev, vif, sta);
+
+	if (old_state == IEEE80211_STA_NONE &&
+	    new_state == IEEE80211_STA_NOTEXIST)
+		mt7996_mac_sta_remove(dev, vif, sta);
+
+	if (old_state == IEEE80211_STA_AUTH &&
+	    new_state == IEEE80211_STA_ASSOC)
+		ev = MT76_STA_EVENT_ASSOC;
+	else if (old_state == IEEE80211_STA_ASSOC &&
+		 new_state == IEEE80211_STA_AUTHORIZED)
+		ev = MT76_STA_EVENT_AUTHORIZE;
+	else if (old_state == IEEE80211_STA_ASSOC &&
+		 new_state == IEEE80211_STA_AUTH)
+		ev = MT76_STA_EVENT_DISASSOC;
+	else
+		return 0;
 
-	spin_lock_bh(&mdev->sta_poll_lock);
-	if (!list_empty(&msta->wcid.poll_list))
-		list_del_init(&msta->wcid.poll_list);
-	if (!list_empty(&msta->rc_list))
-		list_del_init(&msta->rc_list);
-	spin_unlock_bh(&mdev->sta_poll_lock);
+	return mt7996_mac_sta_event(dev, vif, sta, ev);
 }
 
 static void mt7996_tx(struct ieee80211_hw *hw,
@@ -855,12 +1219,25 @@ static void mt7996_tx(struct ieee80211_hw *hw,
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_vif *vif = info->control.vif;
 	struct mt76_wcid *wcid = &dev->mt76.global_wcid;
+	u8 link_id = u32_get_bits(info->control.flags,
+				  IEEE80211_TX_CTRL_MLO_LINK);
+
+	rcu_read_lock();
 
 	if (vif) {
-		struct mt7996_vif *mvif;
+		struct mt7996_vif *mvif = (void *)vif->drv_priv;
+		struct mt76_vif_link *mlink = &mvif->deflink.mt76;
 
-		mvif = (struct mt7996_vif *)vif->drv_priv;
-		wcid = &mvif->deflink.sta.wcid;
+		if (link_id < IEEE80211_LINK_UNSPECIFIED)
+			mlink = rcu_dereference(mvif->mt76.link[link_id]);
+
+		if (!mlink) {
+			ieee80211_free_txskb(hw, skb);
+			goto unlock;
+		}
+
+		if (mlink->wcid)
+			wcid = mlink->wcid;
 
 		if (mvif->mt76.roc_phy &&
 		    (info->flags & IEEE80211_TX_CTL_TX_OFFCHAN)) {
@@ -868,29 +1245,33 @@ static void mt7996_tx(struct ieee80211_hw *hw,
 			if (mphy->roc_link)
 				wcid = mphy->roc_link->wcid;
 		} else {
-			mphy = mt76_vif_link_phy(&mvif->deflink.mt76);
+			mphy = mt76_vif_link_phy(mlink);
 		}
 	}
 
-	if (control->sta) {
-		struct mt7996_sta *sta;
-
-		sta = (struct mt7996_sta *)control->sta->drv_priv;
-		wcid = &sta->wcid;
-	}
-
 	if (!mphy) {
 		ieee80211_free_txskb(hw, skb);
-		return;
+		goto unlock;
 	}
 
+	if (control->sta && link_id < IEEE80211_LINK_UNSPECIFIED) {
+		struct mt7996_sta *msta = (void *)control->sta->drv_priv;
+		struct mt7996_sta_link *msta_link;
+
+		msta_link = rcu_dereference(msta->link[link_id]);
+		if (msta_link)
+			wcid = &msta_link->wcid;
+	}
 	mt76_tx(mphy, control->sta, wcid, skb);
+unlock:
+	rcu_read_unlock();
 }
 
-static int mt7996_set_rts_threshold(struct ieee80211_hw *hw, u32 val)
+static int mt7996_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				    u32 val)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	int i, ret;
+	int i, ret = 0;
 
 	mutex_lock(&dev->mt76.mutex);
 
@@ -914,11 +1295,13 @@ mt7996_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	enum ieee80211_ampdu_mlme_action action = params->action;
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct ieee80211_sta *sta = params->sta;
-	struct ieee80211_txq *txq = sta->txq[params->tid];
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct ieee80211_txq *txq = sta->txq[params->tid];
+	struct ieee80211_link_sta *link_sta;
 	u16 tid = params->tid;
 	u16 ssn = params->ssn;
 	struct mt76_txq *mtxq;
+	unsigned int link_id;
 	int ret = 0;
 
 	if (!txq)
@@ -927,38 +1310,61 @@ mt7996_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	mtxq = (struct mt76_txq *)txq->drv_priv;
 
 	mutex_lock(&dev->mt76.mutex);
-	switch (action) {
-	case IEEE80211_AMPDU_RX_START:
-		mt76_rx_aggr_start(&dev->mt76, &msta->wcid, tid, ssn,
-				   params->buf_size);
-		ret = mt7996_mcu_add_rx_ba(dev, params, true);
-		break;
-	case IEEE80211_AMPDU_RX_STOP:
-		mt76_rx_aggr_stop(&dev->mt76, &msta->wcid, tid);
-		ret = mt7996_mcu_add_rx_ba(dev, params, false);
-		break;
-	case IEEE80211_AMPDU_TX_OPERATIONAL:
-		mtxq->aggr = true;
-		mtxq->send_bar = false;
-		ret = mt7996_mcu_add_tx_ba(dev, params, true);
-		break;
-	case IEEE80211_AMPDU_TX_STOP_FLUSH:
-	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
-		mtxq->aggr = false;
-		clear_bit(tid, &msta->wcid.ampdu_state);
-		ret = mt7996_mcu_add_tx_ba(dev, params, false);
-		break;
-	case IEEE80211_AMPDU_TX_START:
-		set_bit(tid, &msta->wcid.ampdu_state);
-		ret = IEEE80211_AMPDU_TX_START_IMMEDIATE;
-		break;
-	case IEEE80211_AMPDU_TX_STOP_CONT:
-		mtxq->aggr = false;
-		clear_bit(tid, &msta->wcid.ampdu_state);
-		ret = mt7996_mcu_add_tx_ba(dev, params, false);
-		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
-		break;
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct mt7996_sta_link *msta_link;
+		struct mt7996_vif_link *link;
+
+		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
+		if (!msta_link)
+			continue;
+
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link)
+			continue;
+
+		switch (action) {
+		case IEEE80211_AMPDU_RX_START:
+			mt76_rx_aggr_start(&dev->mt76, &msta_link->wcid, tid,
+					   ssn, params->buf_size);
+			ret = mt7996_mcu_add_rx_ba(dev, params, link, true);
+			break;
+		case IEEE80211_AMPDU_RX_STOP:
+			mt76_rx_aggr_stop(&dev->mt76, &msta_link->wcid, tid);
+			ret = mt7996_mcu_add_rx_ba(dev, params, link, false);
+			break;
+		case IEEE80211_AMPDU_TX_OPERATIONAL:
+			mtxq->aggr = true;
+			mtxq->send_bar = false;
+			ret = mt7996_mcu_add_tx_ba(dev, params, link,
+						   msta_link, true);
+			break;
+		case IEEE80211_AMPDU_TX_STOP_FLUSH:
+		case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
+			mtxq->aggr = false;
+			clear_bit(tid, &msta_link->wcid.ampdu_state);
+			ret = mt7996_mcu_add_tx_ba(dev, params, link,
+						   msta_link, false);
+			break;
+		case IEEE80211_AMPDU_TX_START:
+			set_bit(tid, &msta_link->wcid.ampdu_state);
+			ret = IEEE80211_AMPDU_TX_START_IMMEDIATE;
+			break;
+		case IEEE80211_AMPDU_TX_STOP_CONT:
+			mtxq->aggr = false;
+			clear_bit(tid, &msta_link->wcid.ampdu_state);
+			ret = mt7996_mcu_add_tx_ba(dev, params, link,
+						   msta_link, false);
+			break;
+		}
+
+		if (ret)
+			break;
 	}
+
+	if (action == IEEE80211_AMPDU_TX_STOP_CONT)
+		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
+
 	mutex_unlock(&dev->mt76.mutex);
 
 	return ret;
@@ -989,10 +1395,10 @@ mt7996_get_stats(struct ieee80211_hw *hw,
 	return 0;
 }
 
-u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif *mvif)
+u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif_link *link)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	struct mt7996_phy *phy = mt7996_vif_link_phy(&mvif->deflink);
+	struct mt7996_phy *phy = link->phy;
 	union {
 		u64 t64;
 		u32 t32[2];
@@ -1004,8 +1410,8 @@ u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif *mvif)
 
 	lockdep_assert_held(&dev->mt76.mutex);
 
-	n = mvif->deflink.mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
-					       : mvif->deflink.mt76.omac_idx;
+	n = link->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
+					       : link->mt76.omac_idx;
 	/* TSF software read */
 	mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE,
 		 MT_LPON_TCR_SW_READ);
@@ -1023,7 +1429,7 @@ mt7996_get_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 	u64 ret;
 
 	mutex_lock(&dev->mt76.mutex);
-	ret = __mt7996_get_tsf(hw, mvif);
+	ret = __mt7996_get_tsf(hw, &mvif->deflink);
 	mutex_unlock(&dev->mt76.mutex);
 
 	return ret;
@@ -1035,26 +1441,33 @@ mt7996_set_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 {
 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	struct mt7996_phy *phy = mt7996_vif_link_phy(&mvif->deflink);
+	struct mt7996_vif_link *link;
+	struct mt7996_phy *phy;
 	union {
 		u64 t64;
 		u32 t32[2];
 	} tsf = { .t64 = timestamp, };
 	u16 n;
 
-	if (!phy)
-		return;
-
 	mutex_lock(&dev->mt76.mutex);
 
-	n = mvif->deflink.mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
-					       : mvif->deflink.mt76.omac_idx;
+	link = mt7996_vif_link(dev, vif, mvif->mt76.deflink_id);
+	if (!link)
+		goto unlock;
+
+	n = link->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
+					       : link->mt76.omac_idx;
+	phy = link->phy;
+	if (!phy)
+		goto unlock;
+
 	mt76_wr(dev, MT_LPON_UTTR0(phy->mt76->band_idx), tsf.t32[0]);
 	mt76_wr(dev, MT_LPON_UTTR1(phy->mt76->band_idx), tsf.t32[1]);
 	/* TSF software overwrite */
 	mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE,
 		 MT_LPON_TCR_SW_WRITE);
 
+unlock:
 	mutex_unlock(&dev->mt76.mutex);
 }
 
@@ -1064,31 +1477,39 @@ mt7996_offset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 {
 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	struct mt7996_phy *phy = mt7996_vif_link_phy(&mvif->deflink);
+	struct mt7996_vif_link *link;
+	struct mt7996_phy *phy;
 	union {
 		u64 t64;
 		u32 t32[2];
 	} tsf = { .t64 = timestamp, };
 	u16 n;
 
-	if (!phy)
-		return;
-
 	mutex_lock(&dev->mt76.mutex);
 
-	n = mvif->deflink.mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
-					       : mvif->deflink.mt76.omac_idx;
+	link = mt7996_vif_link(dev, vif, mvif->mt76.deflink_id);
+	if (!link)
+		goto unlock;
+
+	phy = link->phy;
+	if (!phy)
+		goto unlock;
+
+	n = link->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
+					       : link->mt76.omac_idx;
 	mt76_wr(dev, MT_LPON_UTTR0(phy->mt76->band_idx), tsf.t32[0]);
 	mt76_wr(dev, MT_LPON_UTTR1(phy->mt76->band_idx), tsf.t32[1]);
 	/* TSF software adjust*/
 	mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE,
 		 MT_LPON_TCR_SW_ADJUST);
 
+unlock:
 	mutex_unlock(&dev->mt76.mutex);
 }
 
 static void
-mt7996_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
+mt7996_set_coverage_class(struct ieee80211_hw *hw, int radio_idx,
+			  s16 coverage_class)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct mt7996_phy *phy;
@@ -1102,7 +1523,8 @@ mt7996_set_coverage_class(struct ieee80211_hw *hw, s16 coverage_class)
 }
 
 static int
-mt7996_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
+mt7996_set_antenna(struct ieee80211_hw *hw, int radio_idx,
+		   u32 tx_ant, u32 rx_ant)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	int i;
@@ -1125,7 +1547,8 @@ mt7996_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
 		u8 shift = dev->chainshift[band_idx];
 
 		phy->mt76->chainmask = tx_ant & phy->orig_chainmask;
-		phy->mt76->antenna_mask = phy->mt76->chainmask >> shift;
+		phy->mt76->antenna_mask = (phy->mt76->chainmask >> shift) &
+					  phy->orig_antenna_mask;
 
 		mt76_set_stream_caps(phy->mt76, true);
 		mt7996_set_stream_vht_txbf_caps(phy);
@@ -1145,7 +1568,8 @@ static void mt7996_sta_statistics(struct ieee80211_hw *hw,
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
-	struct rate_info *txrate = &msta->wcid.rate;
+	struct mt7996_sta_link *msta_link = &msta->deflink;
+	struct rate_info *txrate = &msta_link->wcid.rate;
 
 	if (txrate->legacy || txrate->flags) {
 		if (txrate->legacy) {
@@ -1165,55 +1589,67 @@ static void mt7996_sta_statistics(struct ieee80211_hw *hw,
 	sinfo->txrate.flags = txrate->flags;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
 
-	sinfo->tx_failed = msta->wcid.stats.tx_failed;
+	sinfo->tx_failed = msta_link->wcid.stats.tx_failed;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
 
-	sinfo->tx_retries = msta->wcid.stats.tx_retries;
+	sinfo->tx_retries = msta_link->wcid.stats.tx_retries;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
 
-	sinfo->ack_signal = (s8)msta->ack_signal;
+	sinfo->ack_signal = (s8)msta_link->ack_signal;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL);
 
-	sinfo->avg_ack_signal = -(s8)ewma_avg_signal_read(&msta->avg_ack_signal);
+	sinfo->avg_ack_signal =
+		-(s8)ewma_avg_signal_read(&msta_link->avg_ack_signal);
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG);
 
 	if (mtk_wed_device_active(&dev->mt76.mmio.wed)) {
-		sinfo->tx_bytes = msta->wcid.stats.tx_bytes;
+		sinfo->tx_bytes = msta_link->wcid.stats.tx_bytes;
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES64);
 
-		sinfo->rx_bytes = msta->wcid.stats.rx_bytes;
+		sinfo->rx_bytes = msta_link->wcid.stats.rx_bytes;
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES64);
 
-		sinfo->tx_packets = msta->wcid.stats.tx_packets;
+		sinfo->tx_packets = msta_link->wcid.stats.tx_packets;
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS);
 
-		sinfo->rx_packets = msta->wcid.stats.rx_packets;
+		sinfo->rx_packets = msta_link->wcid.stats.rx_packets;
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS);
 	}
 }
 
-static void mt7996_sta_rc_work(void *data, struct ieee80211_sta *sta)
+static void mt7996_link_rate_ctrl_update(void *data, struct ieee80211_sta *sta)
 {
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
 	struct mt7996_dev *dev = msta->vif->deflink.phy->dev;
+	struct mt7996_sta_link *msta_link;
 	u32 *changed = data;
 
+	rcu_read_lock();
+
+	msta_link = rcu_dereference(msta->link[msta->deflink_id]);
+	if (!msta_link)
+		goto out;
+
 	spin_lock_bh(&dev->mt76.sta_poll_lock);
-	msta->changed |= *changed;
-	if (list_empty(&msta->rc_list))
-		list_add_tail(&msta->rc_list, &dev->sta_rc_list);
+
+	msta_link->changed |= *changed;
+	if (list_empty(&msta_link->rc_list))
+		list_add_tail(&msta_link->rc_list, &dev->sta_rc_list);
+
 	spin_unlock_bh(&dev->mt76.sta_poll_lock);
+out:
+	rcu_read_unlock();
 }
 
-static void mt7996_sta_rc_update(struct ieee80211_hw *hw,
-				 struct ieee80211_vif *vif,
-				 struct ieee80211_link_sta *link_sta,
-				 u32 changed)
+static void mt7996_link_sta_rc_update(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_link_sta *link_sta,
+				      u32 changed)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct ieee80211_sta *sta = link_sta->sta;
 
-	mt7996_sta_rc_work(&changed, sta);
+	mt7996_link_rate_ctrl_update(&changed, sta);
 	ieee80211_queue_work(hw, &dev->rc_work);
 }
 
@@ -1235,7 +1671,8 @@ mt7996_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	 * - multiple rates: if it's not in range format i.e 0-{7,8,9} for VHT
 	 * then multiple MCS setting (MCS 4,5,6) is not supported.
 	 */
-	ieee80211_iterate_stations_atomic(hw, mt7996_sta_rc_work, &changed);
+	ieee80211_iterate_stations_atomic(hw, mt7996_link_rate_ctrl_update,
+					  &changed);
 	ieee80211_queue_work(hw, &dev->rc_work);
 
 	return 0;
@@ -1246,18 +1683,37 @@ static void mt7996_sta_set_4addr(struct ieee80211_hw *hw,
 				 struct ieee80211_sta *sta,
 				 bool enabled)
 {
-	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct mt7996_dev *dev = mt7996_hw_dev(hw);
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
 
-	if (enabled)
-		set_bit(MT_WCID_FLAG_4ADDR, &msta->wcid.flags);
-	else
-		clear_bit(MT_WCID_FLAG_4ADDR, &msta->wcid.flags);
+	mutex_lock(&dev->mt76.mutex);
 
-	if (!msta->wcid.sta)
-		return;
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct mt7996_sta_link *msta_link;
+		struct mt7996_vif_link *link;
 
-	mt7996_mcu_wtbl_update_hdr_trans(dev, vif, sta);
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link)
+			continue;
+
+		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
+		if (!msta_link)
+			continue;
+
+		if (enabled)
+			set_bit(MT_WCID_FLAG_4ADDR, &msta_link->wcid.flags);
+		else
+			clear_bit(MT_WCID_FLAG_4ADDR, &msta_link->wcid.flags);
+
+		if (!msta_link->wcid.sta)
+			continue;
+
+		mt7996_mcu_wtbl_update_hdr_trans(dev, vif, link, msta_link);
+	}
+
+	mutex_unlock(&dev->mt76.mutex);
 }
 
 static void mt7996_sta_set_decap_offload(struct ieee80211_hw *hw,
@@ -1265,18 +1721,39 @@ static void mt7996_sta_set_decap_offload(struct ieee80211_hw *hw,
 					 struct ieee80211_sta *sta,
 					 bool enabled)
 {
-	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct mt7996_dev *dev = mt7996_hw_dev(hw);
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
 
-	if (enabled)
-		set_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags);
-	else
-		clear_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags);
+	mutex_lock(&dev->mt76.mutex);
 
-	if (!msta->wcid.sta)
-		return;
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct mt7996_sta_link *msta_link;
+		struct mt7996_vif_link *link;
+
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link)
+			continue;
+
+		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
+		if (!msta_link)
+			continue;
 
-	mt7996_mcu_wtbl_update_hdr_trans(dev, vif, sta);
+		if (enabled)
+			set_bit(MT_WCID_FLAG_HDR_TRANS,
+				&msta_link->wcid.flags);
+		else
+			clear_bit(MT_WCID_FLAG_HDR_TRANS,
+				  &msta_link->wcid.flags);
+
+		if (!msta_link->wcid.sta)
+			continue;
+
+		mt7996_mcu_wtbl_update_hdr_trans(dev, vif, link, msta_link);
+	}
+
+	mutex_unlock(&dev->mt76.mutex);
 }
 
 static const char mt7996_gstrings_stats[][ETH_GSTRING_LEN] = {
@@ -1408,11 +1885,12 @@ static void mt7996_ethtool_worker(void *wi_data, struct ieee80211_sta *sta)
 {
 	struct mt76_ethtool_worker_info *wi = wi_data;
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct mt7996_sta_link *msta_link = &msta->deflink;
 
 	if (msta->vif->deflink.mt76.idx != wi->idx)
 		return;
 
-	mt76_ethtool_worker(wi, &msta->wcid.stats, true);
+	mt76_ethtool_worker(wi, &msta_link->wcid.stats, true);
 }
 
 static
@@ -1516,10 +1994,12 @@ mt7996_twt_teardown_request(struct ieee80211_hw *hw,
 			    u8 flowid)
 {
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct mt7996_sta_link *msta_link = &msta->deflink;
+	struct mt7996_vif_link *link = &msta->vif->deflink;
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 
 	mutex_lock(&dev->mt76.mutex);
-	mt7996_mac_twt_teardown_flow(dev, msta, flowid);
+	mt7996_mac_twt_teardown_flow(dev, link, msta_link, flowid);
 	mutex_unlock(&dev->mt76.mutex);
 }
 
@@ -1589,12 +2069,26 @@ mt7996_net_fill_forward_path(struct ieee80211_hw *hw,
 {
 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
-	struct mt7996_vif_link *mlink = &mvif->deflink;
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct mtk_wed_device *wed = &dev->mt76.mmio.wed;
+	struct mt7996_sta_link *msta_link;
+	struct mt7996_vif_link *link;
+	struct mt76_vif_link *mlink;
 	struct mt7996_phy *phy;
 
-	phy = mt7996_vif_link_phy(mlink);
+	mlink = rcu_dereference(mvif->mt76.link[msta->deflink_id]);
+	if (!mlink)
+		return -EIO;
+
+	msta_link = rcu_dereference(msta->link[msta->deflink_id]);
+	if (!msta_link)
+		return -EIO;
+
+	if (!msta_link->wcid.sta || msta_link->wcid.idx > MT7996_WTBL_STA)
+		return -EIO;
+
+	link = (struct mt7996_vif_link *)mlink;
+	phy = mt7996_vif_link_phy(link);
 	if (!phy)
 		return -ENODEV;
 
@@ -1604,15 +2098,12 @@ mt7996_net_fill_forward_path(struct ieee80211_hw *hw,
 	if (!mtk_wed_device_active(wed))
 		return -ENODEV;
 
-	if (!msta->wcid.sta || msta->wcid.idx > MT7996_WTBL_STA)
-		return -EIO;
-
 	path->type = DEV_PATH_MTK_WDMA;
 	path->dev = ctx->dev;
 	path->mtk_wdma.wdma_idx = wed->wdma_idx;
-	path->mtk_wdma.bss = mvif->deflink.mt76.idx;
+	path->mtk_wdma.bss = mlink->idx;
 	path->mtk_wdma.queue = 0;
-	path->mtk_wdma.wcid = msta->wcid.idx;
+	path->mtk_wdma.wcid = msta_link->wcid.idx;
 
 	path->mtk_wdma.amsdu = mtk_wed_is_amsdu_supported(wed);
 	ctx->dev = NULL;
@@ -1622,6 +2113,14 @@ mt7996_net_fill_forward_path(struct ieee80211_hw *hw,
 
 #endif
 
+static int
+mt7996_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			u16 old_links, u16 new_links,
+			struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
+{
+	return 0;
+}
+
 const struct ieee80211_ops mt7996_ops = {
 	.add_chanctx = mt76_add_chanctx,
 	.remove_chanctx = mt76_remove_chanctx,
@@ -1637,10 +2136,11 @@ const struct ieee80211_ops mt7996_ops = {
 	.config = mt7996_config,
 	.conf_tx = mt7996_conf_tx,
 	.configure_filter = mt7996_configure_filter,
-	.bss_info_changed = mt7996_bss_info_changed,
-	.sta_state = mt76_sta_state,
+	.vif_cfg_changed = mt7996_vif_cfg_changed,
+	.link_info_changed = mt7996_link_info_changed,
+	.sta_state = mt7996_sta_state,
 	.sta_pre_rcu_remove = mt76_sta_pre_rcu_remove,
-	.link_sta_rc_update = mt7996_sta_rc_update,
+	.link_sta_rc_update = mt7996_link_sta_rc_update,
 	.set_key = mt7996_set_key,
 	.ampdu_action = mt7996_ampdu_action,
 	.set_rts_threshold = mt7996_set_rts_threshold,
@@ -1650,7 +2150,7 @@ const struct ieee80211_ops mt7996_ops = {
 	.remain_on_channel = mt76_remain_on_channel,
 	.cancel_remain_on_channel = mt76_cancel_remain_on_channel,
 	.release_buffered_frames = mt76_release_buffered_frames,
-	.get_txpower = mt76_get_txpower,
+	.get_txpower = mt7996_get_txpower,
 	.channel_switch_beacon = mt7996_channel_switch_beacon,
 	.get_stats = mt7996_get_stats,
 	.get_et_sset_count = mt7996_get_et_sset_count,
@@ -1677,4 +2177,6 @@ const struct ieee80211_ops mt7996_ops = {
 	.net_fill_forward_path = mt7996_net_fill_forward_path,
 	.net_setup_tc = mt76_wed_net_setup_tc,
 #endif
+	.change_vif_links = mt7996_change_vif_links,
+	.change_sta_links = mt7996_mac_sta_change_links,
 };
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c
index 381f9ff41d9a..5099aa2004b7 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.c
@@ -13,7 +13,7 @@
 #define fw_name(_dev, name, ...)	({			\
 	char *_fw;						\
 	switch (mt76_chip(&(_dev)->mt76)) {			\
-	case 0x7992:						\
+	case MT7992_DEVICE_ID:						\
 		switch ((_dev)->var.type) {			\
 		case MT7992_VAR_TYPE_23:			\
 			_fw = MT7992_##name##_23;		\
@@ -22,7 +22,10 @@
 			_fw = MT7992_##name;			\
 		}						\
 		break;						\
-	case 0x7990:						\
+	case MT7990_DEVICE_ID:					\
+		_fw = MT7990_##name;				\
+		break;						\
+	case MT7996_DEVICE_ID:						\
 	default:						\
 		switch ((_dev)->var.type) {			\
 		case MT7996_VAR_TYPE_233:			\
@@ -118,13 +121,13 @@ mt7996_mcu_get_sta_nss(u16 mcs_map)
 }
 
 static void
-mt7996_mcu_set_sta_he_mcs(struct ieee80211_sta *sta, __le16 *he_mcs,
-			  u16 mcs_map)
+mt7996_mcu_set_sta_he_mcs(struct ieee80211_link_sta *link_sta,
+			  struct mt7996_vif_link *link,
+			  __le16 *he_mcs, u16 mcs_map)
 {
-	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
-	enum nl80211_band band = msta->vif->deflink.phy->mt76->chandef.chan->band;
-	const u16 *mask = msta->vif->deflink.bitrate_mask.control[band].he_mcs;
-	int nss, max_nss = sta->deflink.rx_nss > 3 ? 4 : sta->deflink.rx_nss;
+	int nss, max_nss = link_sta->rx_nss > 3 ? 4 : link_sta->rx_nss;
+	enum nl80211_band band = link->phy->mt76->chandef.chan->band;
+	const u16 *mask = link->bitrate_mask.control[band].he_mcs;
 
 	for (nss = 0; nss < max_nss; nss++) {
 		int mcs;
@@ -167,11 +170,11 @@ mt7996_mcu_set_sta_he_mcs(struct ieee80211_sta *sta, __le16 *he_mcs,
 }
 
 static void
-mt7996_mcu_set_sta_vht_mcs(struct ieee80211_sta *sta, __le16 *vht_mcs,
-			   const u16 *mask)
+mt7996_mcu_set_sta_vht_mcs(struct ieee80211_link_sta *link_sta,
+			   __le16 *vht_mcs, const u16 *mask)
 {
-	u16 mcs, mcs_map = le16_to_cpu(sta->deflink.vht_cap.vht_mcs.rx_mcs_map);
-	int nss, max_nss = sta->deflink.rx_nss > 3 ? 4 : sta->deflink.rx_nss;
+	u16 mcs, mcs_map = le16_to_cpu(link_sta->vht_cap.vht_mcs.rx_mcs_map);
+	int nss, max_nss = link_sta->rx_nss > 3 ? 4 : link_sta->rx_nss;
 
 	for (nss = 0; nss < max_nss; nss++, mcs_map >>= 2) {
 		switch (mcs_map & 0x3) {
@@ -193,13 +196,13 @@ mt7996_mcu_set_sta_vht_mcs(struct ieee80211_sta *sta, __le16 *vht_mcs,
 }
 
 static void
-mt7996_mcu_set_sta_ht_mcs(struct ieee80211_sta *sta, u8 *ht_mcs,
-			  const u8 *mask)
+mt7996_mcu_set_sta_ht_mcs(struct ieee80211_link_sta *link_sta,
+			  u8 *ht_mcs, const u8 *mask)
 {
-	int nss, max_nss = sta->deflink.rx_nss > 3 ? 4 : sta->deflink.rx_nss;
+	int nss, max_nss = link_sta->rx_nss > 3 ? 4 : link_sta->rx_nss;
 
 	for (nss = 0; nss < max_nss; nss++)
-		ht_mcs[nss] = sta->deflink.ht_cap.mcs.rx_mask[nss] & mask[nss];
+		ht_mcs[nss] = link_sta->ht_cap.mcs.rx_mask[nss] & mask[nss];
 }
 
 static int
@@ -265,7 +268,7 @@ mt7996_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb,
 
 	txd_len = cmd & __MCU_CMD_FIELD_UNI ? sizeof(*uni_txd) : sizeof(*mcu_txd);
 	txd = (__le32 *)skb_push(skb, txd_len);
-	if (test_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state))
+	if (test_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state) && mt7996_has_wa(dev))
 		qid = MT_MCUQ_WA;
 	else
 		qid = MT_MCUQ_WM;
@@ -335,8 +338,12 @@ exit:
 int mt7996_mcu_wa_cmd(struct mt7996_dev *dev, int cmd, u32 a1, u32 a2, u32 a3)
 {
 	struct {
+		u8 _rsv[4];
+
+		__le16 tag;
+		__le16 len;
 		__le32 args[3];
-	} req = {
+	} __packed req = {
 		.args = {
 			cpu_to_le32(a1),
 			cpu_to_le32(a2),
@@ -344,7 +351,16 @@ int mt7996_mcu_wa_cmd(struct mt7996_dev *dev, int cmd, u32 a1, u32 a2, u32 a3)
 		},
 	};
 
-	return mt76_mcu_send_msg(&dev->mt76, cmd, &req, sizeof(req), false);
+	if (mt7996_has_wa(dev))
+		return mt76_mcu_send_msg(&dev->mt76, cmd, &req.args,
+					 sizeof(req.args), false);
+
+	req.tag = cpu_to_le16(cmd == MCU_WA_PARAM_CMD(QUERY) ? UNI_CMD_SDO_QUERY :
+							       UNI_CMD_SDO_SET);
+	req.len = cpu_to_le16(sizeof(req) - 4);
+
+	return mt76_mcu_send_msg(&dev->mt76, MCU_WA_UNI_CMD(SDO), &req,
+				 sizeof(req), false);
 }
 
 static void
@@ -364,21 +380,27 @@ mt7996_mcu_rx_radar_detected(struct mt7996_dev *dev, struct sk_buff *skb)
 
 	r = (struct mt7996_mcu_rdd_report *)skb->data;
 
-	if (r->band_idx >= ARRAY_SIZE(dev->mt76.phys))
-		return;
-
-	if (r->band_idx == MT_RX_SEL2 && !dev->rdd2_phy)
-		return;
-
-	if (r->band_idx == MT_RX_SEL2)
+	switch (r->rdd_idx) {
+	case MT_RDD_IDX_BAND2:
+		mphy = dev->mt76.phys[MT_BAND2];
+		break;
+	case MT_RDD_IDX_BAND1:
+		mphy = dev->mt76.phys[MT_BAND1];
+		break;
+	case MT_RDD_IDX_BACKGROUND:
+		if (!dev->rdd2_phy)
+			return;
 		mphy = dev->rdd2_phy->mt76;
-	else
-		mphy = dev->mt76.phys[r->band_idx];
+		break;
+	default:
+		dev_err(dev->mt76.dev, "Unknown RDD idx %d\n", r->rdd_idx);
+		return;
+	}
 
 	if (!mphy)
 		return;
 
-	if (r->band_idx == MT_RX_SEL2)
+	if (r->rdd_idx == MT_RDD_IDX_BACKGROUND)
 		cfg80211_background_radar_event(mphy->hw->wiphy,
 						&dev->rdd2_chandef,
 						GFP_ATOMIC);
@@ -548,7 +570,7 @@ mt7996_mcu_rx_all_sta_info_event(struct mt7996_dev *dev, struct sk_buff *skb)
 		switch (le16_to_cpu(res->tag)) {
 		case UNI_ALL_STA_TXRX_RATE:
 			wlan_idx = le16_to_cpu(res->rate[i].wlan_idx);
-			wcid = rcu_dereference(dev->mt76.wcid[wlan_idx]);
+			wcid = mt76_wcid_ptr(dev, wlan_idx);
 
 			if (!wcid)
 				break;
@@ -558,7 +580,7 @@ mt7996_mcu_rx_all_sta_info_event(struct mt7996_dev *dev, struct sk_buff *skb)
 			break;
 		case UNI_ALL_STA_TXRX_ADM_STAT:
 			wlan_idx = le16_to_cpu(res->adm_stat[i].wlan_idx);
-			wcid = rcu_dereference(dev->mt76.wcid[wlan_idx]);
+			wcid = mt76_wcid_ptr(dev, wlan_idx);
 
 			if (!wcid)
 				break;
@@ -572,7 +594,7 @@ mt7996_mcu_rx_all_sta_info_event(struct mt7996_dev *dev, struct sk_buff *skb)
 			break;
 		case UNI_ALL_STA_TXRX_MSDU_COUNT:
 			wlan_idx = le16_to_cpu(res->msdu_cnt[i].wlan_idx);
-			wcid = rcu_dereference(dev->mt76.wcid[wlan_idx]);
+			wcid = mt76_wcid_ptr(dev, wlan_idx);
 
 			if (!wcid)
 				break;
@@ -669,10 +691,7 @@ mt7996_mcu_wed_rro_event(struct mt7996_dev *dev, struct sk_buff *skb)
 
 			e = (void *)skb->data;
 			idx = le16_to_cpu(e->wlan_id);
-			if (idx >= ARRAY_SIZE(dev->mt76.wcid))
-				break;
-
-			wcid = rcu_dereference(dev->mt76.wcid[idx]);
+			wcid = mt76_wcid_ptr(dev, idx);
 			if (!wcid || !wcid->sta)
 				break;
 
@@ -1079,7 +1098,8 @@ __mt7996_mcu_alloc_bss_req(struct mt76_dev *dev, struct mt76_vif_link *mvif, int
 
 int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 			    struct ieee80211_bss_conf *link_conf,
-			    struct mt76_vif_link *mlink, int enable)
+			    struct mt76_vif_link *mlink,
+			    struct mt7996_sta_link *msta_link, int enable)
 {
 	struct mt7996_dev *dev = phy->dev;
 	struct sk_buff *skb;
@@ -1096,7 +1116,7 @@ int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 
 	/* bss_basic must be first */
 	mt7996_mcu_bss_basic_tlv(skb, vif, link_conf, mlink, phy->mt76,
-				 mlink->wcid->idx, enable);
+				 msta_link->wcid.idx, enable);
 	mt7996_mcu_bss_sec_tlv(skb, mlink);
 
 	if (vif->type == NL80211_IFTYPE_MONITOR)
@@ -1174,37 +1194,34 @@ mt7996_mcu_sta_ba(struct mt7996_dev *dev, struct mt76_vif_link *mvif,
 /** starec & wtbl **/
 int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev,
 			 struct ieee80211_ampdu_params *params,
-			 bool enable)
+			 struct mt7996_vif_link *link,
+			 struct mt7996_sta_link *msta_link, bool enable)
 {
-	struct mt7996_sta *msta = (struct mt7996_sta *)params->sta->drv_priv;
-	struct mt7996_vif *mvif = msta->vif;
-
 	if (enable && !params->amsdu)
-		msta->wcid.amsdu = false;
+		msta_link->wcid.amsdu = false;
 
-	return mt7996_mcu_sta_ba(dev, &mvif->deflink.mt76, params, enable, true);
+	return mt7996_mcu_sta_ba(dev, &link->mt76, params, enable, true);
 }
 
 int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev,
 			 struct ieee80211_ampdu_params *params,
-			 bool enable)
+			 struct mt7996_vif_link *link, bool enable)
 {
-	struct mt7996_sta *msta = (struct mt7996_sta *)params->sta->drv_priv;
-	struct mt7996_vif *mvif = msta->vif;
-
-	return mt7996_mcu_sta_ba(dev, &mvif->deflink.mt76, params, enable, false);
+	return mt7996_mcu_sta_ba(dev, &link->mt76, params, enable, false);
 }
 
 static void
-mt7996_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
+mt7996_mcu_sta_he_tlv(struct sk_buff *skb,
+		      struct ieee80211_link_sta *link_sta,
+		      struct mt7996_vif_link *link)
 {
-	struct ieee80211_he_cap_elem *elem = &sta->deflink.he_cap.he_cap_elem;
+	struct ieee80211_he_cap_elem *elem = &link_sta->he_cap.he_cap_elem;
 	struct ieee80211_he_mcs_nss_supp mcs_map;
 	struct sta_rec_he_v2 *he;
 	struct tlv *tlv;
 	int i = 0;
 
-	if (!sta->deflink.he_cap.has_he)
+	if (!link_sta->he_cap.has_he)
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HE_V2, sizeof(*he));
@@ -1216,21 +1233,21 @@ mt7996_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
 		he->he_phy_cap[i] = elem->phy_cap_info[i];
 	}
 
-	mcs_map = sta->deflink.he_cap.he_mcs_nss_supp;
-	switch (sta->deflink.bandwidth) {
+	mcs_map = link_sta->he_cap.he_mcs_nss_supp;
+	switch (link_sta->bandwidth) {
 	case IEEE80211_STA_RX_BW_160:
 		if (elem->phy_cap_info[0] &
 		    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G)
-			mt7996_mcu_set_sta_he_mcs(sta,
+			mt7996_mcu_set_sta_he_mcs(link_sta, link,
 						  &he->max_nss_mcs[CMD_HE_MCS_BW8080],
 						  le16_to_cpu(mcs_map.rx_mcs_80p80));
 
-		mt7996_mcu_set_sta_he_mcs(sta,
+		mt7996_mcu_set_sta_he_mcs(link_sta, link,
 					  &he->max_nss_mcs[CMD_HE_MCS_BW160],
 					  le16_to_cpu(mcs_map.rx_mcs_160));
 		fallthrough;
 	default:
-		mt7996_mcu_set_sta_he_mcs(sta,
+		mt7996_mcu_set_sta_he_mcs(link_sta, link,
 					  &he->max_nss_mcs[CMD_HE_MCS_BW80],
 					  le16_to_cpu(mcs_map.rx_mcs_80));
 		break;
@@ -1240,24 +1257,26 @@ mt7996_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
 }
 
 static void
-mt7996_mcu_sta_he_6g_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
+mt7996_mcu_sta_he_6g_tlv(struct sk_buff *skb,
+			 struct ieee80211_link_sta *link_sta)
 {
 	struct sta_rec_he_6g_capa *he_6g;
 	struct tlv *tlv;
 
-	if (!sta->deflink.he_6ghz_capa.capa)
+	if (!link_sta->he_6ghz_capa.capa)
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HE_6G, sizeof(*he_6g));
 
 	he_6g = (struct sta_rec_he_6g_capa *)tlv;
-	he_6g->capa = sta->deflink.he_6ghz_capa.capa;
+	he_6g->capa = link_sta->he_6ghz_capa.capa;
 }
 
 static void
-mt7996_mcu_sta_eht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
+mt7996_mcu_sta_eht_tlv(struct sk_buff *skb,
+		       struct ieee80211_link_sta *link_sta)
 {
-	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct mt7996_sta *msta = (struct mt7996_sta *)link_sta->sta->drv_priv;
 	struct ieee80211_vif *vif = container_of((void *)msta->vif,
 						 struct ieee80211_vif, drv_priv);
 	struct ieee80211_eht_mcs_nss_supp *mcs_map;
@@ -1265,11 +1284,11 @@ mt7996_mcu_sta_eht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
 	struct sta_rec_eht *eht;
 	struct tlv *tlv;
 
-	if (!sta->deflink.eht_cap.has_eht)
+	if (!link_sta->eht_cap.has_eht)
 		return;
 
-	mcs_map = &sta->deflink.eht_cap.eht_mcs_nss_supp;
-	elem = &sta->deflink.eht_cap.eht_cap_elem;
+	mcs_map = &link_sta->eht_cap.eht_mcs_nss_supp;
+	elem = &link_sta->eht_cap.eht_cap_elem;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_EHT, sizeof(*eht));
 
@@ -1280,7 +1299,7 @@ mt7996_mcu_sta_eht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
 	eht->phy_cap_ext = cpu_to_le64(elem->phy_cap_info[8]);
 
 	if (vif->type != NL80211_IFTYPE_STATION &&
-	    (sta->deflink.he_cap.he_cap_elem.phy_cap_info[0] &
+	    (link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
 	     (IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G |
 	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
 	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
@@ -1296,47 +1315,48 @@ mt7996_mcu_sta_eht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
 }
 
 static void
-mt7996_mcu_sta_ht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
+mt7996_mcu_sta_ht_tlv(struct sk_buff *skb, struct ieee80211_link_sta *link_sta)
 {
 	struct sta_rec_ht_uni *ht;
 	struct tlv *tlv;
 
-	if (!sta->deflink.ht_cap.ht_supported)
+	if (!link_sta->ht_cap.ht_supported)
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HT, sizeof(*ht));
 
 	ht = (struct sta_rec_ht_uni *)tlv;
-	ht->ht_cap = cpu_to_le16(sta->deflink.ht_cap.cap);
-	ht->ampdu_param = u8_encode_bits(sta->deflink.ht_cap.ampdu_factor,
+	ht->ht_cap = cpu_to_le16(link_sta->ht_cap.cap);
+	ht->ampdu_param = u8_encode_bits(link_sta->ht_cap.ampdu_factor,
 					 IEEE80211_HT_AMPDU_PARM_FACTOR) |
-			  u8_encode_bits(sta->deflink.ht_cap.ampdu_density,
+			  u8_encode_bits(link_sta->ht_cap.ampdu_density,
 					 IEEE80211_HT_AMPDU_PARM_DENSITY);
 }
 
 static void
-mt7996_mcu_sta_vht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta)
+mt7996_mcu_sta_vht_tlv(struct sk_buff *skb, struct ieee80211_link_sta *link_sta)
 {
 	struct sta_rec_vht *vht;
 	struct tlv *tlv;
 
 	/* For 6G band, this tlv is necessary to let hw work normally */
-	if (!sta->deflink.he_6ghz_capa.capa && !sta->deflink.vht_cap.vht_supported)
+	if (!link_sta->he_6ghz_capa.capa && !link_sta->vht_cap.vht_supported)
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_VHT, sizeof(*vht));
 
 	vht = (struct sta_rec_vht *)tlv;
-	vht->vht_cap = cpu_to_le32(sta->deflink.vht_cap.cap);
-	vht->vht_rx_mcs_map = sta->deflink.vht_cap.vht_mcs.rx_mcs_map;
-	vht->vht_tx_mcs_map = sta->deflink.vht_cap.vht_mcs.tx_mcs_map;
+	vht->vht_cap = cpu_to_le32(link_sta->vht_cap.cap);
+	vht->vht_rx_mcs_map = link_sta->vht_cap.vht_mcs.rx_mcs_map;
+	vht->vht_tx_mcs_map = link_sta->vht_cap.vht_mcs.tx_mcs_map;
 }
 
 static void
 mt7996_mcu_sta_amsdu_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
-			 struct ieee80211_vif *vif, struct ieee80211_sta *sta)
+			 struct ieee80211_vif *vif,
+			 struct ieee80211_link_sta *link_sta,
+			 struct mt7996_sta_link *msta_link)
 {
-	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
 	struct sta_rec_amsdu *amsdu;
 	struct tlv *tlv;
 
@@ -1345,16 +1365,16 @@ mt7996_mcu_sta_amsdu_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 	    vif->type != NL80211_IFTYPE_AP)
 		return;
 
-	if (!sta->deflink.agg.max_amsdu_len)
+	if (!link_sta->agg.max_amsdu_len)
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HW_AMSDU, sizeof(*amsdu));
 	amsdu = (struct sta_rec_amsdu *)tlv;
 	amsdu->max_amsdu_num = 8;
 	amsdu->amsdu_en = true;
-	msta->wcid.amsdu = true;
+	msta_link->wcid.amsdu = true;
 
-	switch (sta->deflink.agg.max_amsdu_len) {
+	switch (link_sta->agg.max_amsdu_len) {
 	case IEEE80211_MAX_MPDU_LEN_VHT_11454:
 		amsdu->max_mpdu_size =
 			IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454;
@@ -1371,30 +1391,31 @@ mt7996_mcu_sta_amsdu_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 
 static void
 mt7996_mcu_sta_muru_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
-			struct ieee80211_vif *vif, struct ieee80211_sta *sta)
+			struct ieee80211_bss_conf *link_conf,
+			struct ieee80211_link_sta *link_sta)
 {
-	struct ieee80211_he_cap_elem *elem = &sta->deflink.he_cap.he_cap_elem;
+	struct ieee80211_he_cap_elem *elem = &link_sta->he_cap.he_cap_elem;
 	struct sta_rec_muru *muru;
 	struct tlv *tlv;
 
-	if (vif->type != NL80211_IFTYPE_STATION &&
-	    vif->type != NL80211_IFTYPE_AP)
+	if (link_conf->vif->type != NL80211_IFTYPE_STATION &&
+	    link_conf->vif->type != NL80211_IFTYPE_AP)
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_MURU, sizeof(*muru));
 
 	muru = (struct sta_rec_muru *)tlv;
-	muru->cfg.mimo_dl_en = vif->bss_conf.eht_mu_beamformer ||
-			       vif->bss_conf.he_mu_beamformer ||
-			       vif->bss_conf.vht_mu_beamformer ||
-			       vif->bss_conf.vht_mu_beamformee;
+	muru->cfg.mimo_dl_en = link_conf->eht_mu_beamformer ||
+			       link_conf->he_mu_beamformer ||
+			       link_conf->vht_mu_beamformer ||
+			       link_conf->vht_mu_beamformee;
 	muru->cfg.ofdma_dl_en = true;
 
-	if (sta->deflink.vht_cap.vht_supported)
+	if (link_sta->vht_cap.vht_supported)
 		muru->mimo_dl.vht_mu_bfee =
-			!!(sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE);
+			!!(link_sta->vht_cap.cap & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE);
 
-	if (!sta->deflink.he_cap.has_he)
+	if (!link_sta->he_cap.has_he)
 		return;
 
 	muru->mimo_dl.partial_bw_dl_mimo =
@@ -1425,49 +1446,50 @@ mt7996_mcu_sta_muru_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 }
 
 static inline bool
-mt7996_is_ebf_supported(struct mt7996_phy *phy, struct ieee80211_vif *vif,
-			struct ieee80211_sta *sta, bool bfee)
+mt7996_is_ebf_supported(struct mt7996_phy *phy,
+			struct ieee80211_bss_conf *link_conf,
+			struct ieee80211_link_sta *link_sta, bool bfee)
 {
 	int sts = hweight16(phy->mt76->chainmask);
 
-	if (vif->type != NL80211_IFTYPE_STATION &&
-	    vif->type != NL80211_IFTYPE_AP)
+	if (link_conf->vif->type != NL80211_IFTYPE_STATION &&
+	    link_conf->vif->type != NL80211_IFTYPE_AP)
 		return false;
 
 	if (!bfee && sts < 2)
 		return false;
 
-	if (sta->deflink.eht_cap.has_eht) {
-		struct ieee80211_sta_eht_cap *pc = &sta->deflink.eht_cap;
+	if (link_sta->eht_cap.has_eht) {
+		struct ieee80211_sta_eht_cap *pc = &link_sta->eht_cap;
 		struct ieee80211_eht_cap_elem_fixed *pe = &pc->eht_cap_elem;
 
 		if (bfee)
-			return vif->bss_conf.eht_su_beamformee &&
+			return link_conf->eht_su_beamformee &&
 			       EHT_PHY(CAP0_SU_BEAMFORMER, pe->phy_cap_info[0]);
 		else
-			return vif->bss_conf.eht_su_beamformer &&
+			return link_conf->eht_su_beamformer &&
 			       EHT_PHY(CAP0_SU_BEAMFORMEE, pe->phy_cap_info[0]);
 	}
 
-	if (sta->deflink.he_cap.has_he) {
-		struct ieee80211_he_cap_elem *pe = &sta->deflink.he_cap.he_cap_elem;
+	if (link_sta->he_cap.has_he) {
+		struct ieee80211_he_cap_elem *pe = &link_sta->he_cap.he_cap_elem;
 
 		if (bfee)
-			return vif->bss_conf.he_su_beamformee &&
+			return link_conf->he_su_beamformee &&
 			       HE_PHY(CAP3_SU_BEAMFORMER, pe->phy_cap_info[3]);
 		else
-			return vif->bss_conf.he_su_beamformer &&
+			return link_conf->he_su_beamformer &&
 			       HE_PHY(CAP4_SU_BEAMFORMEE, pe->phy_cap_info[4]);
 	}
 
-	if (sta->deflink.vht_cap.vht_supported) {
-		u32 cap = sta->deflink.vht_cap.cap;
+	if (link_sta->vht_cap.vht_supported) {
+		u32 cap = link_sta->vht_cap.cap;
 
 		if (bfee)
-			return vif->bss_conf.vht_su_beamformee &&
+			return link_conf->vht_su_beamformee &&
 			       (cap & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE);
 		else
-			return vif->bss_conf.vht_su_beamformer &&
+			return link_conf->vht_su_beamformer &&
 			       (cap & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE);
 	}
 
@@ -1488,10 +1510,11 @@ mt7996_mcu_sta_sounding_rate(struct sta_rec_bf *bf, struct mt7996_phy *phy)
 }
 
 static void
-mt7996_mcu_sta_bfer_ht(struct ieee80211_sta *sta, struct mt7996_phy *phy,
-		       struct sta_rec_bf *bf, bool explicit)
+mt7996_mcu_sta_bfer_ht(struct ieee80211_link_sta *link_sta,
+		       struct mt7996_phy *phy, struct sta_rec_bf *bf,
+		       bool explicit)
 {
-	struct ieee80211_mcs_info *mcs = &sta->deflink.ht_cap.mcs;
+	struct ieee80211_mcs_info *mcs = &link_sta->ht_cap.mcs;
 	u8 n = 0;
 
 	bf->tx_mode = MT_PHY_TYPE_HT;
@@ -1514,10 +1537,11 @@ mt7996_mcu_sta_bfer_ht(struct ieee80211_sta *sta, struct mt7996_phy *phy,
 }
 
 static void
-mt7996_mcu_sta_bfer_vht(struct ieee80211_sta *sta, struct mt7996_phy *phy,
-			struct sta_rec_bf *bf, bool explicit)
+mt7996_mcu_sta_bfer_vht(struct ieee80211_link_sta *link_sta,
+			struct mt7996_phy *phy, struct sta_rec_bf *bf,
+			bool explicit)
 {
-	struct ieee80211_sta_vht_cap *pc = &sta->deflink.vht_cap;
+	struct ieee80211_sta_vht_cap *pc = &link_sta->vht_cap;
 	struct ieee80211_sta_vht_cap *vc = &phy->mt76->sband_5g.sband.vht_cap;
 	u16 mcs_map = le16_to_cpu(pc->vht_mcs.rx_mcs_map);
 	u8 nss_mcs = mt7996_mcu_get_sta_nss(mcs_map);
@@ -1538,24 +1562,24 @@ mt7996_mcu_sta_bfer_vht(struct ieee80211_sta *sta, struct mt7996_phy *phy,
 		bf->ncol = min_t(u8, nss_mcs, bf->nrow);
 		bf->ibf_ncol = min_t(u8, MT7996_IBF_MAX_NC, bf->ncol);
 
-		if (sta->deflink.bandwidth == IEEE80211_STA_RX_BW_160)
+		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_160)
 			bf->nrow = 1;
 	} else {
 		bf->nrow = tx_ant;
 		bf->ncol = min_t(u8, nss_mcs, bf->nrow);
 		bf->ibf_ncol = min_t(u8, MT7996_IBF_MAX_NC, nss_mcs);
 
-		if (sta->deflink.bandwidth == IEEE80211_STA_RX_BW_160)
+		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_160)
 			bf->ibf_nrow = 1;
 	}
 }
 
 static void
-mt7996_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
-		       struct mt7996_phy *phy, struct sta_rec_bf *bf,
-		       bool explicit)
+mt7996_mcu_sta_bfer_he(struct ieee80211_link_sta *link_sta,
+		       struct ieee80211_vif *vif, struct mt7996_phy *phy,
+		       struct sta_rec_bf *bf, bool explicit)
 {
-	struct ieee80211_sta_he_cap *pc = &sta->deflink.he_cap;
+	struct ieee80211_sta_he_cap *pc = &link_sta->he_cap;
 	struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem;
 	const struct ieee80211_sta_he_cap *vc =
 		mt76_connac_get_he_phy_cap(phy->mt76, vif);
@@ -1584,7 +1608,7 @@ mt7996_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
 	bf->ibf_ncol = explicit ? min_t(u8, MT7996_IBF_MAX_NC, bf->ncol) :
 				  min_t(u8, MT7996_IBF_MAX_NC, nss_mcs);
 
-	if (sta->deflink.bandwidth != IEEE80211_STA_RX_BW_160)
+	if (link_sta->bandwidth != IEEE80211_STA_RX_BW_160)
 		return;
 
 	/* go over for 160MHz and 80p80 */
@@ -1616,11 +1640,11 @@ mt7996_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
 }
 
 static void
-mt7996_mcu_sta_bfer_eht(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
-			struct mt7996_phy *phy, struct sta_rec_bf *bf,
-			bool explicit)
+mt7996_mcu_sta_bfer_eht(struct ieee80211_link_sta *link_sta,
+			struct ieee80211_vif *vif, struct mt7996_phy *phy,
+			struct sta_rec_bf *bf, bool explicit)
 {
-	struct ieee80211_sta_eht_cap *pc = &sta->deflink.eht_cap;
+	struct ieee80211_sta_eht_cap *pc = &link_sta->eht_cap;
 	struct ieee80211_eht_cap_elem_fixed *pe = &pc->eht_cap_elem;
 	struct ieee80211_eht_mcs_nss_supp *eht_nss = &pc->eht_mcs_nss_supp;
 	const struct ieee80211_sta_eht_cap *vc =
@@ -1644,10 +1668,10 @@ mt7996_mcu_sta_bfer_eht(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
 	bf->ibf_ncol = explicit ? min_t(u8, MT7996_IBF_MAX_NC, bf->ncol) :
 				  min_t(u8, MT7996_IBF_MAX_NC, nss_mcs);
 
-	if (sta->deflink.bandwidth < IEEE80211_STA_RX_BW_160)
+	if (link_sta->bandwidth < IEEE80211_STA_RX_BW_160)
 		return;
 
-	switch (sta->deflink.bandwidth) {
+	switch (link_sta->bandwidth) {
 	case IEEE80211_STA_RX_BW_160:
 		snd_dim = EHT_PHY(CAP2_SOUNDING_DIM_160MHZ_MASK, ve->phy_cap_info[2]);
 		sts = EHT_PHY(CAP1_BEAMFORMEE_SS_160MHZ_MASK, pe->phy_cap_info[1]);
@@ -1675,13 +1699,15 @@ mt7996_mcu_sta_bfer_eht(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
 
 static void
 mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
-			struct ieee80211_vif *vif, struct ieee80211_sta *sta)
+			struct ieee80211_bss_conf *link_conf,
+			struct ieee80211_link_sta *link_sta,
+			struct mt7996_vif_link *link)
 {
 #define EBF_MODE	BIT(0)
 #define IBF_MODE	BIT(1)
 #define BF_MAT_ORDER	4
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_phy *phy = mvif->deflink.phy;
+	struct ieee80211_vif *vif = link_conf->vif;
+	struct mt7996_phy *phy = link->phy;
 	int tx_ant = hweight16(phy->mt76->chainmask) - 1;
 	struct sta_rec_bf *bf;
 	struct tlv *tlv;
@@ -1693,10 +1719,10 @@ mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 	};
 	bool ebf;
 
-	if (!(sta->deflink.ht_cap.ht_supported || sta->deflink.he_cap.has_he))
+	if (!(link_sta->ht_cap.ht_supported || link_sta->he_cap.has_he))
 		return;
 
-	ebf = mt7996_is_ebf_supported(phy, vif, sta, false);
+	ebf = mt7996_is_ebf_supported(phy, link_conf, link_sta, false);
 	if (!ebf && !dev->ibf)
 		return;
 
@@ -1707,28 +1733,29 @@ mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 	 * vht: support eBF and iBF
 	 * ht: iBF only, since mac80211 lacks of eBF support
 	 */
-	if (sta->deflink.eht_cap.has_eht)
-		mt7996_mcu_sta_bfer_eht(sta, vif, phy, bf, ebf);
-	else if (sta->deflink.he_cap.has_he)
-		mt7996_mcu_sta_bfer_he(sta, vif, phy, bf, ebf);
-	else if (sta->deflink.vht_cap.vht_supported)
-		mt7996_mcu_sta_bfer_vht(sta, phy, bf, ebf);
-	else if (sta->deflink.ht_cap.ht_supported)
-		mt7996_mcu_sta_bfer_ht(sta, phy, bf, ebf);
+	if (link_sta->eht_cap.has_eht)
+		mt7996_mcu_sta_bfer_eht(link_sta, vif, link->phy, bf, ebf);
+	else if (link_sta->he_cap.has_he)
+		mt7996_mcu_sta_bfer_he(link_sta, vif, link->phy, bf, ebf);
+	else if (link_sta->vht_cap.vht_supported)
+		mt7996_mcu_sta_bfer_vht(link_sta, link->phy, bf, ebf);
+	else if (link_sta->ht_cap.ht_supported)
+		mt7996_mcu_sta_bfer_ht(link_sta, link->phy, bf, ebf);
 	else
 		return;
 
 	bf->bf_cap = ebf ? EBF_MODE : (dev->ibf ? IBF_MODE : 0);
 	if (is_mt7992(&dev->mt76) && tx_ant == 4)
 		bf->bf_cap |= IBF_MODE;
-	bf->bw = sta->deflink.bandwidth;
-	bf->ibf_dbw = sta->deflink.bandwidth;
+
+	bf->bw = link_sta->bandwidth;
+	bf->ibf_dbw = link_sta->bandwidth;
 	bf->ibf_nrow = tx_ant;
 
-	if (sta->deflink.eht_cap.has_eht || sta->deflink.he_cap.has_he)
+	if (link_sta->eht_cap.has_eht || link_sta->he_cap.has_he)
 		bf->ibf_timeout = is_mt7996(&dev->mt76) ? MT7996_IBF_TIMEOUT :
 							  MT7992_IBF_TIMEOUT;
-	else if (!ebf && sta->deflink.bandwidth <= IEEE80211_STA_RX_BW_40 && !bf->ncol)
+	else if (!ebf && link_sta->bandwidth <= IEEE80211_STA_RX_BW_40 && !bf->ncol)
 		bf->ibf_timeout = MT7996_IBF_TIMEOUT_LEGACY;
 	else
 		bf->ibf_timeout = MT7996_IBF_TIMEOUT;
@@ -1742,7 +1769,7 @@ mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 				      matrix[bf->nrow][bf->ncol] : 0;
 	}
 
-	switch (sta->deflink.bandwidth) {
+	switch (link_sta->bandwidth) {
 	case IEEE80211_STA_RX_BW_160:
 	case IEEE80211_STA_RX_BW_80:
 		bf->mem_total = bf->mem_20m * 2;
@@ -1758,31 +1785,32 @@ mt7996_mcu_sta_bfer_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 
 static void
 mt7996_mcu_sta_bfee_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
-			struct ieee80211_vif *vif, struct ieee80211_sta *sta)
+			struct ieee80211_bss_conf *link_conf,
+			struct ieee80211_link_sta *link_sta,
+			struct mt7996_vif_link *link)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_phy *phy = mvif->deflink.phy;
+	struct mt7996_phy *phy = link->phy;
 	int tx_ant = hweight8(phy->mt76->antenna_mask) - 1;
 	struct sta_rec_bfee *bfee;
 	struct tlv *tlv;
 	u8 nrow = 0;
 
-	if (!(sta->deflink.vht_cap.vht_supported || sta->deflink.he_cap.has_he))
+	if (!(link_sta->vht_cap.vht_supported || link_sta->he_cap.has_he))
 		return;
 
-	if (!mt7996_is_ebf_supported(phy, vif, sta, true))
+	if (!mt7996_is_ebf_supported(phy, link_conf, link_sta, true))
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_BFEE, sizeof(*bfee));
 	bfee = (struct sta_rec_bfee *)tlv;
 
-	if (sta->deflink.he_cap.has_he) {
-		struct ieee80211_he_cap_elem *pe = &sta->deflink.he_cap.he_cap_elem;
+	if (link_sta->he_cap.has_he) {
+		struct ieee80211_he_cap_elem *pe = &link_sta->he_cap.he_cap_elem;
 
 		nrow = HE_PHY(CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_MASK,
 			      pe->phy_cap_info[5]);
-	} else if (sta->deflink.vht_cap.vht_supported) {
-		struct ieee80211_sta_vht_cap *pc = &sta->deflink.vht_cap;
+	} else if (link_sta->vht_cap.vht_supported) {
+		struct ieee80211_sta_vht_cap *pc = &link_sta->vht_cap;
 
 		nrow = FIELD_GET(IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK,
 				 pc->cap);
@@ -1866,8 +1894,8 @@ mt7996_mcu_get_mmps_mode(enum ieee80211_smps_mode smps)
 int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev,
 				   void *data, u16 version)
 {
+	struct uni_header hdr = {};
 	struct ra_fixed_rate *req;
-	struct uni_header hdr;
 	struct sk_buff *skb;
 	struct tlv *tlv;
 	int len;
@@ -1889,21 +1917,35 @@ int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev,
 				     MCU_WM_UNI_CMD(RA), true);
 }
 
-int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-			       struct ieee80211_sta *sta, void *data, u32 field)
+int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct mt7996_sta *msta,
+			       void *data, u8 link_id, u32 field)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
-	struct sta_phy_uni *phy = data;
+	struct mt7996_vif *mvif = msta->vif;
+	struct mt7996_sta_link *msta_link;
 	struct sta_rec_ra_fixed_uni *ra;
+	struct sta_phy_uni *phy = data;
+	struct mt76_vif_link *mlink;
 	struct sk_buff *skb;
+	int err = -ENODEV;
 	struct tlv *tlv;
 
-	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->deflink.mt76,
-					      &msta->wcid,
+	rcu_read_lock();
+
+	mlink = rcu_dereference(mvif->mt76.link[link_id]);
+	if (!mlink)
+		goto error_unlock;
+
+	msta_link = rcu_dereference(msta->link[link_id]);
+	if (!msta_link)
+		goto error_unlock;
+
+	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, mlink,
+					      &msta_link->wcid,
 					      MT7996_STA_UPDATE_MAX_SIZE);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
+	if (IS_ERR(skb)) {
+		err = PTR_ERR(skb);
+		goto error_unlock;
+	}
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_RA_UPDATE, sizeof(*ra));
 	ra = (struct sta_rec_ra_fixed_uni *)tlv;
@@ -1918,131 +1960,179 @@ int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif
 		if (phy)
 			ra->phy = *phy;
 		break;
-	case RATE_PARAM_MMPS_UPDATE:
-		ra->mmps_mode = mt7996_mcu_get_mmps_mode(sta->deflink.smps_mode);
+	case RATE_PARAM_MMPS_UPDATE: {
+		struct ieee80211_sta *sta = wcid_to_sta(&msta_link->wcid);
+		struct ieee80211_link_sta *link_sta;
+
+		link_sta = rcu_dereference(sta->link[link_id]);
+		if (!link_sta) {
+			dev_kfree_skb(skb);
+			goto error_unlock;
+		}
+
+		ra->mmps_mode = mt7996_mcu_get_mmps_mode(link_sta->smps_mode);
 		break;
+	}
 	default:
 		break;
 	}
 	ra->field = cpu_to_le32(field);
 
+	rcu_read_unlock();
+
 	return mt76_mcu_skb_send_msg(&dev->mt76, skb,
 				     MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true);
+error_unlock:
+	rcu_read_unlock();
+
+	return err;
 }
 
 static int
-mt7996_mcu_add_rate_ctrl_fixed(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-			       struct ieee80211_sta *sta)
+mt7996_mcu_add_rate_ctrl_fixed(struct mt7996_dev *dev, struct mt7996_sta *msta,
+			       struct ieee80211_vif *vif, u8 link_id)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct cfg80211_chan_def *chandef = &mvif->deflink.phy->mt76->chandef;
-	struct cfg80211_bitrate_mask *mask = &mvif->deflink.bitrate_mask;
-	enum nl80211_band band = chandef->chan->band;
+	struct ieee80211_link_sta *link_sta;
+	struct cfg80211_bitrate_mask mask;
+	struct mt7996_sta_link *msta_link;
+	struct mt7996_vif_link *link;
 	struct sta_phy_uni phy = {};
-	int ret, nrates = 0;
+	struct ieee80211_sta *sta;
+	int ret, nrates = 0, idx;
+	enum nl80211_band band;
+	bool has_he;
 
 #define __sta_phy_bitrate_mask_check(_mcs, _gi, _ht, _he)			\
 	do {									\
-		u8 i, gi = mask->control[band]._gi;				\
+		u8 i, gi = mask.control[band]._gi;				\
 		gi = (_he) ? gi : gi == NL80211_TXRATE_FORCE_SGI;		\
 		phy.sgi = gi;							\
-		phy.he_ltf = mask->control[band].he_ltf;			\
-		for (i = 0; i < ARRAY_SIZE(mask->control[band]._mcs); i++) {	\
-			if (!mask->control[band]._mcs[i])			\
+		phy.he_ltf = mask.control[band].he_ltf;				\
+		for (i = 0; i < ARRAY_SIZE(mask.control[band]._mcs); i++) {	\
+			if (!mask.control[band]._mcs[i])			\
 				continue;					\
-			nrates += hweight16(mask->control[band]._mcs[i]);	\
-			phy.mcs = ffs(mask->control[band]._mcs[i]) - 1;		\
+			nrates += hweight16(mask.control[band]._mcs[i]);	\
+			phy.mcs = ffs(mask.control[band]._mcs[i]) - 1;		\
 			if (_ht)						\
 				phy.mcs += 8 * i;				\
 		}								\
 	} while (0)
 
-	if (sta->deflink.he_cap.has_he) {
+	rcu_read_lock();
+
+	link = mt7996_vif_link(dev, vif, link_id);
+	if (!link)
+		goto error_unlock;
+
+	msta_link = rcu_dereference(msta->link[link_id]);
+	if (!msta_link)
+		goto error_unlock;
+
+	sta = wcid_to_sta(&msta_link->wcid);
+	link_sta = rcu_dereference(sta->link[link_id]);
+	if (!link_sta)
+		goto error_unlock;
+
+	band = link->phy->mt76->chandef.chan->band;
+	has_he = link_sta->he_cap.has_he;
+	mask = link->bitrate_mask;
+	idx = msta_link->wcid.idx;
+
+	if (has_he) {
 		__sta_phy_bitrate_mask_check(he_mcs, he_gi, 0, 1);
-	} else if (sta->deflink.vht_cap.vht_supported) {
+	} else if (link_sta->vht_cap.vht_supported) {
 		__sta_phy_bitrate_mask_check(vht_mcs, gi, 0, 0);
-	} else if (sta->deflink.ht_cap.ht_supported) {
+	} else if (link_sta->ht_cap.ht_supported) {
 		__sta_phy_bitrate_mask_check(ht_mcs, gi, 1, 0);
 	} else {
-		nrates = hweight32(mask->control[band].legacy);
-		phy.mcs = ffs(mask->control[band].legacy) - 1;
+		nrates = hweight32(mask.control[band].legacy);
+		phy.mcs = ffs(mask.control[band].legacy) - 1;
 	}
+
+	rcu_read_unlock();
+
 #undef __sta_phy_bitrate_mask_check
 
 	/* fall back to auto rate control */
-	if (mask->control[band].gi == NL80211_TXRATE_DEFAULT_GI &&
-	    mask->control[band].he_gi == GENMASK(7, 0) &&
-	    mask->control[band].he_ltf == GENMASK(7, 0) &&
+	if (mask.control[band].gi == NL80211_TXRATE_DEFAULT_GI &&
+	    mask.control[band].he_gi == GENMASK(7, 0) &&
+	    mask.control[band].he_ltf == GENMASK(7, 0) &&
 	    nrates != 1)
 		return 0;
 
 	/* fixed single rate */
 	if (nrates == 1) {
-		ret = mt7996_mcu_set_fixed_field(dev, vif, sta, &phy,
+		ret = mt7996_mcu_set_fixed_field(dev, msta, &phy, link_id,
 						 RATE_PARAM_FIXED_MCS);
 		if (ret)
 			return ret;
 	}
 
 	/* fixed GI */
-	if (mask->control[band].gi != NL80211_TXRATE_DEFAULT_GI ||
-	    mask->control[band].he_gi != GENMASK(7, 0)) {
-		struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	if (mask.control[band].gi != NL80211_TXRATE_DEFAULT_GI ||
+	    mask.control[band].he_gi != GENMASK(7, 0)) {
 		u32 addr;
 
 		/* firmware updates only TXCMD but doesn't take WTBL into
 		 * account, so driver should update here to reflect the
 		 * actual txrate hardware sends out.
 		 */
-		addr = mt7996_mac_wtbl_lmac_addr(dev, msta->wcid.idx, 7);
-		if (sta->deflink.he_cap.has_he)
+		addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 7);
+		if (has_he)
 			mt76_rmw_field(dev, addr, GENMASK(31, 24), phy.sgi);
 		else
 			mt76_rmw_field(dev, addr, GENMASK(15, 12), phy.sgi);
 
-		ret = mt7996_mcu_set_fixed_field(dev, vif, sta, &phy,
+		ret = mt7996_mcu_set_fixed_field(dev, msta, &phy, link_id,
 						 RATE_PARAM_FIXED_GI);
 		if (ret)
 			return ret;
 	}
 
 	/* fixed HE_LTF */
-	if (mask->control[band].he_ltf != GENMASK(7, 0)) {
-		ret = mt7996_mcu_set_fixed_field(dev, vif, sta, &phy,
+	if (mask.control[band].he_ltf != GENMASK(7, 0)) {
+		ret = mt7996_mcu_set_fixed_field(dev, msta, &phy, link_id,
 						 RATE_PARAM_FIXED_HE_LTF);
 		if (ret)
 			return ret;
 	}
 
 	return 0;
+
+error_unlock:
+	rcu_read_unlock();
+
+	return -ENODEV;
 }
 
 static void
 mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev,
-			     struct ieee80211_vif *vif, struct ieee80211_sta *sta)
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf,
+			     struct ieee80211_link_sta *link_sta,
+			     struct mt7996_vif_link *link)
 {
 #define INIT_RCPI 180
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt76_phy *mphy = mvif->deflink.phy->mt76;
+	struct mt76_phy *mphy = link->phy->mt76;
 	struct cfg80211_chan_def *chandef = &mphy->chandef;
-	struct cfg80211_bitrate_mask *mask = &mvif->deflink.bitrate_mask;
+	struct cfg80211_bitrate_mask *mask = &link->bitrate_mask;
+	u32 cap = link_sta->sta->wme ? STA_CAP_WMM : 0;
 	enum nl80211_band band = chandef->chan->band;
 	struct sta_rec_ra_uni *ra;
 	struct tlv *tlv;
-	u32 supp_rate = sta->deflink.supp_rates[band];
-	u32 cap = sta->wme ? STA_CAP_WMM : 0;
+	u32 supp_rate = link_sta->supp_rates[band];
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_RA, sizeof(*ra));
 	ra = (struct sta_rec_ra_uni *)tlv;
 
 	ra->valid = true;
 	ra->auto_rate = true;
-	ra->phy_mode = mt76_connac_get_phy_mode(mphy, vif, band, &sta->deflink);
+	ra->phy_mode = mt76_connac_get_phy_mode(mphy, vif, band, link_sta);
 	ra->channel = chandef->chan->hw_value;
-	ra->bw = (sta->deflink.bandwidth == IEEE80211_STA_RX_BW_320) ?
-		 CMD_CBW_320MHZ : sta->deflink.bandwidth;
+	ra->bw = (link_sta->bandwidth == IEEE80211_STA_RX_BW_320) ?
+		 CMD_CBW_320MHZ : link_sta->bandwidth;
 	ra->phy.bw = ra->bw;
-	ra->mmps_mode = mt7996_mcu_get_mmps_mode(sta->deflink.smps_mode);
+	ra->mmps_mode = mt7996_mcu_get_mmps_mode(link_sta->smps_mode);
 
 	if (supp_rate) {
 		supp_rate &= mask->control[band].legacy;
@@ -2062,60 +2152,60 @@ mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev,
 		}
 	}
 
-	if (sta->deflink.ht_cap.ht_supported) {
+	if (link_sta->ht_cap.ht_supported) {
 		ra->supp_mode |= MODE_HT;
-		ra->af = sta->deflink.ht_cap.ampdu_factor;
-		ra->ht_gf = !!(sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_GRN_FLD);
+		ra->af = link_sta->ht_cap.ampdu_factor;
+		ra->ht_gf = !!(link_sta->ht_cap.cap & IEEE80211_HT_CAP_GRN_FLD);
 
 		cap |= STA_CAP_HT;
-		if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_SGI_20)
+		if (link_sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20)
 			cap |= STA_CAP_SGI_20;
-		if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_SGI_40)
+		if (link_sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_40)
 			cap |= STA_CAP_SGI_40;
-		if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_TX_STBC)
+		if (link_sta->ht_cap.cap & IEEE80211_HT_CAP_TX_STBC)
 			cap |= STA_CAP_TX_STBC;
-		if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_RX_STBC)
+		if (link_sta->ht_cap.cap & IEEE80211_HT_CAP_RX_STBC)
 			cap |= STA_CAP_RX_STBC;
-		if (vif->bss_conf.ht_ldpc &&
-		    (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_LDPC_CODING))
+		if (link_conf->ht_ldpc &&
+		    (link_sta->ht_cap.cap & IEEE80211_HT_CAP_LDPC_CODING))
 			cap |= STA_CAP_LDPC;
 
-		mt7996_mcu_set_sta_ht_mcs(sta, ra->ht_mcs,
+		mt7996_mcu_set_sta_ht_mcs(link_sta, ra->ht_mcs,
 					  mask->control[band].ht_mcs);
 		ra->supp_ht_mcs = *(__le32 *)ra->ht_mcs;
 	}
 
-	if (sta->deflink.vht_cap.vht_supported) {
+	if (link_sta->vht_cap.vht_supported) {
 		u8 af;
 
 		ra->supp_mode |= MODE_VHT;
 		af = FIELD_GET(IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK,
-			       sta->deflink.vht_cap.cap);
+			       link_sta->vht_cap.cap);
 		ra->af = max_t(u8, ra->af, af);
 
 		cap |= STA_CAP_VHT;
-		if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_80)
+		if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_80)
 			cap |= STA_CAP_VHT_SGI_80;
-		if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_160)
+		if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_160)
 			cap |= STA_CAP_VHT_SGI_160;
-		if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_TXSTBC)
+		if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_TXSTBC)
 			cap |= STA_CAP_VHT_TX_STBC;
-		if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1)
+		if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1)
 			cap |= STA_CAP_VHT_RX_STBC;
-		if ((vif->type != NL80211_IFTYPE_AP || vif->bss_conf.vht_ldpc) &&
-		    (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC))
+		if ((vif->type != NL80211_IFTYPE_AP || link_conf->vht_ldpc) &&
+		    (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC))
 			cap |= STA_CAP_VHT_LDPC;
 
-		mt7996_mcu_set_sta_vht_mcs(sta, ra->supp_vht_mcs,
+		mt7996_mcu_set_sta_vht_mcs(link_sta, ra->supp_vht_mcs,
 					   mask->control[band].vht_mcs);
 	}
 
-	if (sta->deflink.he_cap.has_he) {
+	if (link_sta->he_cap.has_he) {
 		ra->supp_mode |= MODE_HE;
 		cap |= STA_CAP_HE;
 
-		if (sta->deflink.he_6ghz_capa.capa)
-			ra->af = le16_get_bits(sta->deflink.he_6ghz_capa.capa,
+		if (link_sta->he_6ghz_capa.capa)
+			ra->af = le16_get_bits(link_sta->he_6ghz_capa.capa,
 					       IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP);
 	}
 	ra->sta_cap = cpu_to_le32(cap);
@@ -2123,38 +2213,70 @@ mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev,
 	memset(ra->rx_rcpi, INIT_RCPI, sizeof(ra->rx_rcpi));
 }
 
-int mt7996_mcu_add_rate_ctrl(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-			     struct ieee80211_sta *sta, bool changed)
+int mt7996_mcu_add_rate_ctrl(struct mt7996_dev *dev, struct mt7996_sta *msta,
+			     struct ieee80211_vif *vif, u8 link_id,
+			     bool changed)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	struct ieee80211_bss_conf *link_conf;
+	struct ieee80211_link_sta *link_sta;
+	struct mt7996_sta_link *msta_link;
+	struct mt7996_vif_link *link;
+	struct ieee80211_sta *sta;
 	struct sk_buff *skb;
-	int ret;
+	int ret = -ENODEV;
+
+	rcu_read_lock();
+
+	link = mt7996_vif_link(dev, vif, link_id);
+	if (!link)
+		goto error_unlock;
+
+	msta_link = rcu_dereference(msta->link[link_id]);
+	if (!msta_link)
+		goto error_unlock;
+
+	sta = wcid_to_sta(&msta_link->wcid);
+	link_sta = rcu_dereference(sta->link[link_id]);
+	if (!link_sta)
+		goto error_unlock;
 
-	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->deflink.mt76,
-					      &msta->wcid,
+	link_conf = rcu_dereference(vif->link_conf[link_id]);
+	if (!link_conf)
+		goto error_unlock;
+
+	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &link->mt76,
+					      &msta_link->wcid,
 					      MT7996_STA_UPDATE_MAX_SIZE);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
+	if (IS_ERR(skb)) {
+		ret = PTR_ERR(skb);
+		goto error_unlock;
+	}
 
 	/* firmware rc algorithm refers to sta_rec_he for HE control.
 	 * once dev->rc_work changes the settings driver should also
 	 * update sta_rec_he here.
 	 */
 	if (changed)
-		mt7996_mcu_sta_he_tlv(skb, sta);
+		mt7996_mcu_sta_he_tlv(skb, link_sta, link);
 
 	/* sta_rec_ra accommodates BW, NSS and only MCS range format
 	 * i.e 0-{7,8,9} for VHT.
 	 */
-	mt7996_mcu_sta_rate_ctrl_tlv(skb, dev, vif, sta);
+	mt7996_mcu_sta_rate_ctrl_tlv(skb, dev, vif, link_conf, link_sta, link);
+
+	rcu_read_unlock();
 
 	ret = mt76_mcu_skb_send_msg(&dev->mt76, skb,
 				    MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true);
 	if (ret)
 		return ret;
 
-	return mt7996_mcu_add_rate_ctrl_fixed(dev, vif, sta);
+	return mt7996_mcu_add_rate_ctrl_fixed(dev, msta, vif, link_id);
+
+error_unlock:
+	rcu_read_unlock();
+
+	return ret;
 }
 
 static int
@@ -2163,6 +2285,7 @@ mt7996_mcu_add_group(struct mt7996_dev *dev, struct ieee80211_vif *vif,
 {
 #define MT_STA_BSS_GROUP		1
 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
+	struct mt7996_sta_link *msta_link;
 	struct mt7996_sta *msta;
 	struct {
 		u8 __rsv1[4];
@@ -2181,73 +2304,150 @@ mt7996_mcu_add_group(struct mt7996_dev *dev, struct ieee80211_vif *vif,
 		.val = cpu_to_le32(mvif->deflink.mt76.idx % 16),
 	};
 
-	msta = sta ? (struct mt7996_sta *)sta->drv_priv : &mvif->deflink.sta;
-	req.wlan_idx = cpu_to_le16(msta->wcid.idx);
+	msta = sta ? (struct mt7996_sta *)sta->drv_priv : NULL;
+	msta_link = msta ? &msta->deflink : &mvif->deflink.msta_link;
+	req.wlan_idx = cpu_to_le16(msta_link->wcid.idx);
 
 	return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(VOW), &req,
 				 sizeof(req), true);
 }
 
-int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-		       struct mt76_vif_link *mlink,
-		       struct ieee80211_sta *sta, int conn_state, bool newly)
+static void
+mt7996_mcu_sta_mld_setup_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta)
 {
-	struct ieee80211_link_sta *link_sta = NULL;
-	struct mt76_wcid *wcid = mlink->wcid;
-	struct sk_buff *skb;
-	int ret;
+	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	unsigned int nlinks = hweight16(sta->valid_links);
+	struct mld_setup_link *mld_setup_link;
+	struct ieee80211_link_sta *link_sta;
+	struct sta_rec_mld_setup *mld_setup;
+	struct mt7996_sta_link *msta_link;
+	unsigned int link_id;
+	struct tlv *tlv;
+
+	msta_link = mt76_dereference(msta->link[msta->deflink_id], &dev->mt76);
+	if (!msta_link)
+		return;
 
-	if (sta) {
-		struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
+	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_MLD,
+				      sizeof(struct sta_rec_mld_setup) +
+				      sizeof(struct mld_setup_link) * nlinks);
 
-		wcid = &msta->wcid;
-		link_sta = &sta->deflink;
+	mld_setup = (struct sta_rec_mld_setup *)tlv;
+	memcpy(mld_setup->mld_addr, sta->addr, ETH_ALEN);
+	mld_setup->setup_wcid = cpu_to_le16(msta_link->wcid.idx);
+	mld_setup->primary_id = cpu_to_le16(msta_link->wcid.idx);
+
+	if (nlinks > 1) {
+		link_id = __ffs(sta->valid_links & ~BIT(msta->deflink_id));
+		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
+		if (!msta_link)
+			return;
 	}
+	mld_setup->seconed_id = cpu_to_le16(msta_link->wcid.idx);
+	mld_setup->link_num = nlinks;
 
-	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, mlink, wcid,
+	mld_setup_link = (struct mld_setup_link *)mld_setup->link_info;
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct mt7996_vif_link *link;
+
+		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
+		if (!msta_link)
+			continue;
+
+		link = mt7996_vif_link(dev, vif, link_id);
+		if (!link)
+			continue;
+
+		mld_setup_link->wcid = cpu_to_le16(msta_link->wcid.idx);
+		mld_setup_link->bss_idx = link->mt76.idx;
+		mld_setup_link++;
+	}
+}
+
+static void
+mt7996_mcu_sta_eht_mld_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
+			   struct ieee80211_sta *sta)
+{
+	struct sta_rec_eht_mld *eht_mld;
+	struct tlv *tlv;
+	int i;
+
+	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_EHT_MLD, sizeof(*eht_mld));
+	eht_mld = (struct sta_rec_eht_mld *)tlv;
+
+	for (i = 0; i < ARRAY_SIZE(eht_mld->str_cap); i++)
+		eht_mld->str_cap[i] = 0x7;
+}
+
+int mt7996_mcu_add_sta(struct mt7996_dev *dev,
+		       struct ieee80211_bss_conf *link_conf,
+		       struct ieee80211_link_sta *link_sta,
+		       struct mt7996_vif_link *link,
+		       struct mt7996_sta_link *msta_link,
+		       int conn_state, bool newly)
+{
+	struct mt76_wcid *wcid = msta_link ? &msta_link->wcid : link->mt76.wcid;
+	struct ieee80211_sta *sta = link_sta ? link_sta->sta : NULL;
+	struct sk_buff *skb;
+	int ret;
+
+	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &link->mt76, wcid,
 					      MT7996_STA_UPDATE_MAX_SIZE);
 	if (IS_ERR(skb))
 		return PTR_ERR(skb);
 
 	/* starec basic */
-	mt76_connac_mcu_sta_basic_tlv(&dev->mt76, skb, &vif->bss_conf, link_sta,
+	mt76_connac_mcu_sta_basic_tlv(&dev->mt76, skb, link_conf, link_sta,
 				      conn_state, newly);
 
 	if (conn_state == CONN_STATE_DISCONNECT)
 		goto out;
 
 	/* starec hdr trans */
-	mt7996_mcu_sta_hdr_trans_tlv(dev, skb, vif, wcid);
+	mt7996_mcu_sta_hdr_trans_tlv(dev, skb, link_conf->vif, wcid);
 	/* starec tx proc */
 	mt7996_mcu_sta_tx_proc_tlv(skb);
 
 	/* tag order is in accordance with firmware dependency. */
-	if (sta) {
+	if (link_sta) {
 		/* starec hdrt mode */
 		mt7996_mcu_sta_hdrt_tlv(dev, skb);
-		/* starec bfer */
-		mt7996_mcu_sta_bfer_tlv(dev, skb, vif, sta);
+		if (conn_state == CONN_STATE_CONNECT) {
+			/* starec bfer */
+			mt7996_mcu_sta_bfer_tlv(dev, skb, link_conf, link_sta,
+						link);
+			/* starec bfee */
+			mt7996_mcu_sta_bfee_tlv(dev, skb, link_conf, link_sta,
+						link);
+		}
 		/* starec ht */
-		mt7996_mcu_sta_ht_tlv(skb, sta);
+		mt7996_mcu_sta_ht_tlv(skb, link_sta);
 		/* starec vht */
-		mt7996_mcu_sta_vht_tlv(skb, sta);
+		mt7996_mcu_sta_vht_tlv(skb, link_sta);
 		/* starec uapsd */
-		mt76_connac_mcu_sta_uapsd(skb, vif, sta);
+		mt76_connac_mcu_sta_uapsd(skb, link_conf->vif, sta);
 		/* starec amsdu */
-		mt7996_mcu_sta_amsdu_tlv(dev, skb, vif, sta);
+		mt7996_mcu_sta_amsdu_tlv(dev, skb, link_conf->vif, link_sta,
+					 msta_link);
 		/* starec he */
-		mt7996_mcu_sta_he_tlv(skb, sta);
+		mt7996_mcu_sta_he_tlv(skb, link_sta, link);
 		/* starec he 6g*/
-		mt7996_mcu_sta_he_6g_tlv(skb, sta);
+		mt7996_mcu_sta_he_6g_tlv(skb, link_sta);
 		/* starec eht */
-		mt7996_mcu_sta_eht_tlv(skb, sta);
+		mt7996_mcu_sta_eht_tlv(skb, link_sta);
 		/* starec muru */
-		mt7996_mcu_sta_muru_tlv(dev, skb, vif, sta);
-		/* starec bfee */
-		mt7996_mcu_sta_bfee_tlv(dev, skb, vif, sta);
+		mt7996_mcu_sta_muru_tlv(dev, skb, link_conf, link_sta);
+
+		if (sta->mlo) {
+			mt7996_mcu_sta_mld_setup_tlv(dev, skb, link_conf->vif,
+						     sta);
+			mt7996_mcu_sta_eht_mld_tlv(dev, skb, sta);
+		}
 	}
 
-	ret = mt7996_mcu_add_group(dev, vif, sta);
+	ret = mt7996_mcu_add_group(dev, link_conf->vif, sta);
 	if (ret) {
 		dev_kfree_skb(skb);
 		return ret;
@@ -2257,6 +2457,24 @@ out:
 				     MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true);
 }
 
+int mt7996_mcu_teardown_mld_sta(struct mt7996_dev *dev,
+				struct mt7996_vif_link *link,
+				struct mt7996_sta_link *msta_link)
+{
+	struct sk_buff *skb;
+
+	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &link->mt76,
+					      &msta_link->wcid,
+					      MT7996_STA_UPDATE_MAX_SIZE);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	mt76_connac_mcu_add_tlv(skb, STA_REC_MLD_OFF, sizeof(struct tlv));
+
+	return mt76_mcu_skb_send_msg(&dev->mt76, skb,
+				     MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true);
+}
+
 static int
 mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid,
 		       struct sk_buff *skb,
@@ -2322,17 +2540,18 @@ int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif,
 	return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true);
 }
 
-static int mt7996_mcu_get_pn(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-			     u8 *pn)
+static int mt7996_mcu_get_pn(struct mt7996_dev *dev,
+			     struct mt7996_vif_link *link,
+			     struct mt7996_sta_link *msta_link, u8 *pn)
 {
 #define TSC_TYPE_BIGTK_PN 2
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct sta_rec_pn_info *pn_info;
 	struct sk_buff *skb, *rskb;
 	struct tlv *tlv;
 	int ret;
 
-	skb = mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->deflink.mt76, &mvif->deflink.sta.wcid);
+	skb = mt76_connac_mcu_alloc_sta_req(&dev->mt76, &link->mt76,
+					    &msta_link->wcid);
 	if (IS_ERR(skb))
 		return PTR_ERR(skb);
 
@@ -2356,10 +2575,11 @@ static int mt7996_mcu_get_pn(struct mt7996_dev *dev, struct ieee80211_vif *vif,
 	return 0;
 }
 
-int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, struct ieee80211_vif *vif,
+int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev,
+			       struct mt7996_vif_link *link,
+			       struct mt7996_sta_link *msta_link,
 			       struct ieee80211_key_conf *key)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct mt7996_mcu_bcn_prot_tlv *bcn_prot;
 	struct sk_buff *skb;
 	struct tlv *tlv;
@@ -2368,7 +2588,7 @@ int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, struct ieee80211_vif *vif
 		  sizeof(struct mt7996_mcu_bcn_prot_tlv);
 	int ret;
 
-	skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->deflink.mt76, len);
+	skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &link->mt76, len);
 	if (IS_ERR(skb))
 		return PTR_ERR(skb);
 
@@ -2376,7 +2596,7 @@ int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, struct ieee80211_vif *vif
 
 	bcn_prot = (struct mt7996_mcu_bcn_prot_tlv *)tlv;
 
-	ret = mt7996_mcu_get_pn(dev, vif, pn);
+	ret = mt7996_mcu_get_pn(dev, link, msta_link, pn);
 	if (ret) {
 		dev_kfree_skb(skb);
 		return ret;
@@ -2554,13 +2774,15 @@ int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			  struct ieee80211_bss_conf *link_conf)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	struct mt76_vif_link *mlink = mt76_vif_conf_link(&dev->mt76, vif, link_conf);
+	struct mt7996_vif_link *link = mt7996_vif_conf_link(dev, vif, link_conf);
+	struct mt76_vif_link *mlink = link ? &link->mt76 : NULL;
 	struct ieee80211_mutable_offsets offs;
 	struct ieee80211_tx_info *info;
 	struct sk_buff *skb, *rskb;
 	struct tlv *tlv;
 	struct bss_bcn_content_tlv *bcn;
 	int len, extra_len = 0;
+	bool enabled = link_conf->enable_beacon;
 
 	if (link_conf->nontransmitted)
 		return 0;
@@ -2568,13 +2790,16 @@ int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	if (!mlink)
 		return -EINVAL;
 
+	if (link->phy && link->phy->mt76->offchannel)
+		enabled = false;
+
 	rskb = __mt7996_mcu_alloc_bss_req(&dev->mt76, mlink,
 					  MT7996_MAX_BSS_OFFLOAD_SIZE);
 	if (IS_ERR(rskb))
 		return PTR_ERR(rskb);
 
 	skb = ieee80211_beacon_get_template(hw, vif, &offs, link_conf->link_id);
-	if (link_conf->enable_beacon && !skb) {
+	if (enabled && !skb) {
 		dev_kfree_skb(rskb);
 		return -EINVAL;
 	}
@@ -2593,7 +2818,7 @@ int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	len = ALIGN(sizeof(*bcn) + MT_TXD_SIZE + extra_len, 4);
 	tlv = mt7996_mcu_add_uni_tlv(rskb, UNI_BSS_INFO_BCN_CONTENT, len);
 	bcn = (struct bss_bcn_content_tlv *)tlv;
-	bcn->enable = link_conf->enable_beacon;
+	bcn->enable = enabled;
 	if (!bcn->enable)
 		goto out;
 
@@ -2611,13 +2836,14 @@ out:
 }
 
 int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev,
-				    struct ieee80211_vif *vif, u32 changed)
+				    struct ieee80211_bss_conf *link_conf,
+				    struct mt7996_vif_link *link, u32 changed)
 {
 #define OFFLOAD_TX_MODE_SU	BIT(0)
 #define OFFLOAD_TX_MODE_MU	BIT(1)
+	struct ieee80211_vif *vif = link_conf->vif;
 	struct ieee80211_hw *hw = mt76_hw(dev);
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_phy *phy = mt7996_vif_link_phy(&mvif->deflink);
+	struct mt7996_phy *phy = link->phy;
 	struct mt76_wcid *wcid = &dev->mt76.global_wcid;
 	struct bss_inband_discovery_tlv *discov;
 	struct ieee80211_tx_info *info;
@@ -2634,21 +2860,21 @@ int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev,
 	chandef = &phy->mt76->chandef;
 	band = chandef->chan->band;
 
-	if (vif->bss_conf.nontransmitted)
+	if (link_conf->nontransmitted)
 		return 0;
 
-	rskb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->deflink.mt76,
+	rskb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &link->mt76,
 					  MT7996_MAX_BSS_OFFLOAD_SIZE);
 	if (IS_ERR(rskb))
 		return PTR_ERR(rskb);
 
 	if (changed & BSS_CHANGED_FILS_DISCOVERY &&
-	    vif->bss_conf.fils_discovery.max_interval) {
-		interval = vif->bss_conf.fils_discovery.max_interval;
+	    link_conf->fils_discovery.max_interval) {
+		interval = link_conf->fils_discovery.max_interval;
 		skb = ieee80211_get_fils_discovery_tmpl(hw, vif);
 	} else if (changed & BSS_CHANGED_UNSOL_BCAST_PROBE_RESP &&
-		   vif->bss_conf.unsol_bcast_probe_resp_interval) {
-		interval = vif->bss_conf.unsol_bcast_probe_resp_interval;
+		   link_conf->unsol_bcast_probe_resp_interval) {
+		interval = link_conf->unsol_bcast_probe_resp_interval;
 		skb = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif);
 	}
 
@@ -2918,6 +3144,9 @@ static int mt7996_load_ram(struct mt7996_dev *dev)
 	if (ret)
 		return ret;
 
+	if (!mt7996_has_wa(dev))
+		return 0;
+
 	ret = __mt7996_load_ram(dev, "DSP", fw_name(dev, FIRMWARE_DSP),
 				MT7996_RAM_TYPE_DSP);
 	if (ret)
@@ -2928,10 +3157,9 @@ static int mt7996_load_ram(struct mt7996_dev *dev)
 }
 
 static int
-mt7996_firmware_state(struct mt7996_dev *dev, bool wa)
+mt7996_firmware_state(struct mt7996_dev *dev, u8 fw_state)
 {
-	u32 state = FIELD_PREP(MT_TOP_MISC_FW_STATE,
-			       wa ? FW_STATE_RDY : FW_STATE_FW_DOWNLOAD);
+	u32 state = FIELD_PREP(MT_TOP_MISC_FW_STATE, fw_state);
 
 	if (!mt76_poll_msec(dev, MT_TOP_MISC, MT_TOP_MISC_FW_STATE,
 			    state, 1000)) {
@@ -2963,13 +3191,14 @@ mt7996_mcu_restart(struct mt76_dev *dev)
 
 static int mt7996_load_firmware(struct mt7996_dev *dev)
 {
+	u8 fw_state;
 	int ret;
 
 	/* make sure fw is download state */
-	if (mt7996_firmware_state(dev, false)) {
+	if (mt7996_firmware_state(dev, FW_STATE_FW_DOWNLOAD)) {
 		/* restart firmware once */
 		mt7996_mcu_restart(&dev->mt76);
-		ret = mt7996_firmware_state(dev, false);
+		ret = mt7996_firmware_state(dev, FW_STATE_FW_DOWNLOAD);
 		if (ret) {
 			dev_err(dev->mt76.dev,
 				"Firmware is not ready for download\n");
@@ -2985,7 +3214,8 @@ static int mt7996_load_firmware(struct mt7996_dev *dev)
 	if (ret)
 		return ret;
 
-	ret = mt7996_firmware_state(dev, true);
+	fw_state = mt7996_has_wa(dev) ? FW_STATE_RDY : FW_STATE_NORMAL_TRX;
+	ret = mt7996_firmware_state(dev, fw_state);
 	if (ret)
 		return ret;
 
@@ -3123,13 +3353,15 @@ int mt7996_mcu_init_firmware(struct mt7996_dev *dev)
 	if (ret)
 		return ret;
 
-	ret = mt7996_mcu_fw_log_2_host(dev, MCU_FW_LOG_WA, 0);
-	if (ret)
-		return ret;
+	if (mt7996_has_wa(dev)) {
+		ret = mt7996_mcu_fw_log_2_host(dev, MCU_FW_LOG_WA, 0);
+		if (ret)
+			return ret;
 
-	ret = mt7996_mcu_set_mwds(dev, 1);
-	if (ret)
-		return ret;
+		ret = mt7996_mcu_set_mwds(dev, 1);
+		if (ret)
+			return ret;
+	}
 
 	ret = mt7996_mcu_init_rx_airtime(dev);
 	if (ret)
@@ -3155,7 +3387,7 @@ int mt7996_mcu_init(struct mt7996_dev *dev)
 void mt7996_mcu_exit(struct mt7996_dev *dev)
 {
 	mt7996_mcu_restart(&dev->mt76);
-	if (mt7996_firmware_state(dev, false)) {
+	if (mt7996_firmware_state(dev, FW_STATE_FW_DOWNLOAD)) {
 		dev_err(dev->mt76.dev, "Failed to exit mcu\n");
 		goto out;
 	}
@@ -3172,7 +3404,7 @@ int mt7996_mcu_set_hdr_trans(struct mt7996_dev *dev, bool hdr_trans)
 {
 	struct {
 		u8 __rsv[4];
-	} __packed hdr;
+	} __packed hdr = {};
 	struct hdr_trans_blacklist *req_blacklist;
 	struct hdr_trans_en *req_en;
 	struct sk_buff *skb;
@@ -3442,11 +3674,10 @@ int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy,
 				     struct cfg80211_chan_def *chandef)
 {
 	struct mt7996_dev *dev = phy->dev;
-	int err, region;
+	int err, region, rdd_idx = mt7996_get_rdd_idx(phy, true);
 
 	if (!chandef) { /* disable offchain */
-		err = mt7996_mcu_rdd_cmd(dev, RDD_STOP, MT_RX_SEL2,
-					 0, 0);
+		err = mt7996_mcu_rdd_cmd(dev, RDD_STOP, rdd_idx, 0);
 		if (err)
 			return err;
 
@@ -3472,8 +3703,7 @@ int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy,
 		break;
 	}
 
-	return mt7996_mcu_rdd_cmd(dev, RDD_START, MT_RX_SEL2,
-				  0, region);
+	return mt7996_mcu_rdd_cmd(dev, RDD_START, rdd_idx, region);
 }
 
 int mt7996_mcu_set_chan_info(struct mt7996_phy *phy, u16 tag)
@@ -4095,12 +4325,12 @@ mt7996_mcu_set_obss_spr_pd(struct mt7996_phy *phy,
 }
 
 static int
-mt7996_mcu_set_obss_spr_siga(struct mt7996_phy *phy, struct ieee80211_vif *vif,
+mt7996_mcu_set_obss_spr_siga(struct mt7996_phy *phy,
+			     struct mt7996_vif_link *link,
 			     struct ieee80211_he_obss_pd *he_obss_pd)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct mt7996_dev *dev = phy->dev;
-	u8 omac = mvif->deflink.mt76.omac_idx;
+	u8 omac = link->mt76.omac_idx;
 	struct {
 		u8 band_idx;
 		u8 __rsv[3];
@@ -4172,7 +4402,8 @@ mt7996_mcu_set_obss_spr_bitmap(struct mt7996_phy *phy,
 				 sizeof(req), true);
 }
 
-int mt7996_mcu_add_obss_spr(struct mt7996_phy *phy, struct ieee80211_vif *vif,
+int mt7996_mcu_add_obss_spr(struct mt7996_phy *phy,
+			    struct mt7996_vif_link *link,
 			    struct ieee80211_he_obss_pd *he_obss_pd)
 {
 	int ret;
@@ -4206,7 +4437,7 @@ int mt7996_mcu_add_obss_spr(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 		return ret;
 
 	/* Set SR prohibit */
-	ret = mt7996_mcu_set_obss_spr_siga(phy, vif, he_obss_pd);
+	ret = mt7996_mcu_set_obss_spr_siga(phy, link, he_obss_pd);
 	if (ret)
 		return ret;
 
@@ -4242,7 +4473,7 @@ int mt7996_mcu_update_bss_color(struct mt7996_dev *dev,
 #define TWT_AGRT_PROTECT	BIT(2)
 
 int mt7996_mcu_twt_agrt_update(struct mt7996_dev *dev,
-			       struct mt7996_vif *mvif,
+			       struct mt7996_vif_link *link,
 			       struct mt7996_twt_flow *flow,
 			       int cmd)
 {
@@ -4273,12 +4504,12 @@ int mt7996_mcu_twt_agrt_update(struct mt7996_dev *dev,
 		.len = cpu_to_le16(sizeof(req) - 4),
 		.tbl_idx = flow->table_id,
 		.cmd = cmd,
-		.own_mac_idx = mvif->deflink.mt76.omac_idx,
+		.own_mac_idx = link->mt76.omac_idx,
 		.flowid = flow->id,
 		.peer_id = cpu_to_le16(flow->wcid),
 		.duration = flow->duration,
-		.bss = mvif->deflink.mt76.idx,
-		.bss_idx = mvif->deflink.mt76.idx,
+		.bss = link->mt76.idx,
+		.bss_idx = link->mt76.idx,
 		.start_tsf = cpu_to_le64(flow->tsf),
 		.mantissa = flow->mantissa,
 		.exponent = flow->exp,
@@ -4339,8 +4570,7 @@ int mt7996_mcu_set_radio_en(struct mt7996_phy *phy, bool enable)
 				 &req, sizeof(req), true);
 }
 
-int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index,
-		       u8 rx_sel, u8 val)
+int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 rdd_idx, u8 val)
 {
 	struct {
 		u8 _rsv[4];
@@ -4357,8 +4587,7 @@ int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index,
 		.tag = cpu_to_le16(UNI_RDD_CTRL_PARM),
 		.len = cpu_to_le16(sizeof(req) - 4),
 		.ctrl = cmd,
-		.rdd_idx = index,
-		.rdd_rx_sel = rx_sel,
+		.rdd_idx = rdd_idx,
 		.val = val,
 	};
 
@@ -4368,22 +4597,19 @@ int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index,
 
 int mt7996_mcu_wtbl_update_hdr_trans(struct mt7996_dev *dev,
 				     struct ieee80211_vif *vif,
-				     struct ieee80211_sta *sta)
+				     struct mt7996_vif_link *link,
+				     struct mt7996_sta_link *msta_link)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_sta *msta;
 	struct sk_buff *skb;
 
-	msta = sta ? (struct mt7996_sta *)sta->drv_priv : &mvif->deflink.sta;
-
-	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->deflink.mt76,
-					      &msta->wcid,
+	skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &link->mt76,
+					      &msta_link->wcid,
 					      MT7996_STA_UPDATE_MAX_SIZE);
 	if (IS_ERR(skb))
 		return PTR_ERR(skb);
 
 	/* starec hdr trans */
-	mt7996_mcu_sta_hdr_trans_tlv(dev, skb, vif, &msta->wcid);
+	mt7996_mcu_sta_hdr_trans_tlv(dev, skb, vif, &msta_link->wcid);
 	return mt76_mcu_skb_send_msg(&dev->mt76, skb,
 				     MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true);
 }
@@ -4606,7 +4832,7 @@ int mt7996_mcu_set_txpower_sku(struct mt7996_phy *phy)
 	struct sk_buff *skb;
 	int i, tx_power;
 
-	tx_power = mt7996_get_power_bound(phy, phy->txpower);
+	tx_power = mt76_get_power_bound(mphy, phy->txpower);
 	tx_power = mt76_get_rate_power_limits(mphy, mphy->chandef.chan,
 					      &la, tx_power);
 	mphy->txpower_cur = tx_power;
@@ -4651,7 +4877,26 @@ int mt7996_mcu_cp_support(struct mt7996_dev *dev, u8 mode)
 	    mode > mt76_connac_lmac_mapping(IEEE80211_AC_VO))
 		return -EINVAL;
 
+	if (!mt7996_has_wa(dev)) {
+		struct {
+			u8 _rsv[4];
+
+			__le16 tag;
+			__le16 len;
+			u8 cp_mode;
+			u8 rsv[3];
+		} __packed req = {
+			.tag = cpu_to_le16(UNI_CMD_SDO_CP_MODE),
+			.len = cpu_to_le16(sizeof(req) - 4),
+			.cp_mode = mode,
+		};
+
+		return mt76_mcu_send_msg(&dev->mt76, MCU_WA_UNI_CMD(SDO),
+					 &req, sizeof(req), false);
+	}
+
 	cp_mode = cpu_to_le32(mode);
+
 	return mt76_mcu_send_msg(&dev->mt76, MCU_WA_EXT_CMD(CP_SUPPORT),
 				 &cp_mode, sizeof(cp_mode), true);
 }
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h
index 43468bcaffc6..130ea95626d5 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/mcu.h
@@ -69,7 +69,7 @@ struct mt7996_mcu_rdd_report {
 	__le16 tag;
 	__le16 len;
 
-	u8 band_idx;
+	u8 rdd_idx;
 	u8 long_detected;
 	u8 constant_prf_detected;
 	u8 staggered_prf_detected;
@@ -625,6 +625,35 @@ struct sta_rec_hdr_trans {
 	u8 mesh;
 } __packed;
 
+struct sta_rec_mld_setup {
+	__le16 tag;
+	__le16 len;
+	u8 mld_addr[ETH_ALEN];
+	__le16 primary_id;
+	__le16 seconed_id;
+	__le16 setup_wcid;
+	u8 link_num;
+	u8 info;
+	u8 __rsv[2];
+	u8 link_info[];
+} __packed;
+
+struct sta_rec_eht_mld {
+	__le16 tag;
+	__le16 len;
+	u8 nsep;
+	u8 __rsv1[2];
+	u8 str_cap[__MT_MAX_BAND];
+	__le16 eml_cap;
+	u8 __rsv2[4];
+} __packed;
+
+struct mld_setup_link {
+	__le16 wcid;
+	u8 bss_idx;
+	u8 __rsv;
+} __packed;
+
 struct hdr_trans_en {
 	__le16 tag;
 	__le16 len;
@@ -798,29 +827,20 @@ enum {
 					 sizeof(struct sta_rec_eht) +		\
 					 sizeof(struct sta_rec_hdrt) +		\
 					 sizeof(struct sta_rec_hdr_trans) +	\
+					 sizeof(struct sta_rec_mld_setup) +	\
+					 sizeof(struct mld_setup_link) * 3 +	\
+					 sizeof(struct sta_rec_eht_mld) +	\
 					 sizeof(struct tlv))
 
-#define MT7996_MAX_BEACON_SIZE		1338
 #define MT7996_BEACON_UPDATE_SIZE	(sizeof(struct bss_req_hdr) +		\
 					 sizeof(struct bss_bcn_content_tlv) +	\
 					 4 + MT_TXD_SIZE +			\
 					 sizeof(struct bss_bcn_cntdwn_tlv) +	\
 					 sizeof(struct bss_bcn_mbss_tlv))
-#define MT7996_MAX_BSS_OFFLOAD_SIZE	(MT7996_MAX_BEACON_SIZE +		\
+#define MT7996_MAX_BSS_OFFLOAD_SIZE	2048
+#define MT7996_MAX_BEACON_SIZE		(MT7996_MAX_BSS_OFFLOAD_SIZE - \
 					 MT7996_BEACON_UPDATE_SIZE)
 
-static inline s8
-mt7996_get_power_bound(struct mt7996_phy *phy, s8 txpower)
-{
-	struct mt76_phy *mphy = phy->mt76;
-	int n_chains = hweight16(mphy->chainmask);
-
-	txpower = mt76_get_sar_power(mphy, mphy->chandef.chan, txpower * 2);
-	txpower -= mt76_tx_power_nss_delta(n_chains);
-
-	return txpower;
-}
-
 enum {
 	UNI_BAND_CONFIG_RADIO_ENABLE,
 	UNI_BAND_CONFIG_RTS_THRESHOLD = 0x08,
@@ -908,7 +928,8 @@ enum {
 	UNI_CMD_SER_SET_RECOVER_L3_TX_DISABLE,
 	UNI_CMD_SER_SET_RECOVER_L3_BF,
 	UNI_CMD_SER_SET_RECOVER_L4_MDP,
-	UNI_CMD_SER_SET_RECOVER_FULL,
+	UNI_CMD_SER_SET_RECOVER_FROM_ETH,
+	UNI_CMD_SER_SET_RECOVER_FULL = 8,
 	UNI_CMD_SER_SET_SYSTEM_ASSERT,
 	/* action */
 	UNI_CMD_SER_ENABLE = 1,
@@ -917,6 +938,12 @@ enum {
 };
 
 enum {
+	UNI_CMD_SDO_SET = 1,
+	UNI_CMD_SDO_QUERY,
+	UNI_CMD_SDO_CP_MODE = 6,
+};
+
+enum {
 	MT7996_SEC_MODE_PLAIN,
 	MT7996_SEC_MODE_AES,
 	MT7996_SEC_MODE_SCRAMBLE,
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c b/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c
index bf21780427b2..bb82f5807ff8 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/mmio.c
@@ -57,6 +57,17 @@ static const u32 mt7996_offs[] = {
 	[MIB_BSCR7]		= 0x9e8,
 	[MIB_BSCR17]		= 0xa10,
 	[MIB_TRDR1]		= 0xa28,
+	[HIF_REMAP_L1]		= 0x24,
+	[HIF_REMAP_BASE_L1]	= 0x130000,
+	[HIF_REMAP_L2]		= 0x1b4,
+	[HIF_REMAP_BASE_L2]	= 0x1000,
+	[CBTOP1_PHY_END]	= 0x77ffffff,
+	[INFRA_MCU_END]		= 0x7c3fffff,
+	[WTBLON_WDUCR]		= 0x370,
+	[WTBL_UPDATE]		= 0x380,
+	[WTBL_ITCR]		= 0x3b0,
+	[WTBL_ITCR0]		= 0x3b8,
+	[WTBL_ITCR1]		= 0x3bc,
 };
 
 static const u32 mt7992_offs[] = {
@@ -83,6 +94,54 @@ static const u32 mt7992_offs[] = {
 	[MIB_BSCR7]		= 0xae4,
 	[MIB_BSCR17]		= 0xb0c,
 	[MIB_TRDR1]		= 0xb24,
+	[HIF_REMAP_L1]		= 0x8,
+	[HIF_REMAP_BASE_L1]	= 0x40000,
+	[HIF_REMAP_L2]		= 0x1b4,
+	[HIF_REMAP_BASE_L2]	= 0x1000,
+	[CBTOP1_PHY_END]	= 0x77ffffff,
+	[INFRA_MCU_END]		= 0x7c3fffff,
+	[WTBLON_WDUCR]		= 0x370,
+	[WTBL_UPDATE]		= 0x380,
+	[WTBL_ITCR]		= 0x3b0,
+	[WTBL_ITCR0]		= 0x3b8,
+	[WTBL_ITCR1]		= 0x3bc,
+};
+
+static const u32 mt7990_offs[] = {
+	[MIB_RVSR0]		= 0x800,
+	[MIB_RVSR1]		= 0x804,
+	[MIB_BTSCR5]		= 0x868,
+	[MIB_BTSCR6]		= 0x878,
+	[MIB_RSCR1]		= 0x890,
+	[MIB_RSCR27]		= 0xa38,
+	[MIB_RSCR28]		= 0xa3c,
+	[MIB_RSCR29]		= 0xa40,
+	[MIB_RSCR30]		= 0xa44,
+	[MIB_RSCR31]		= 0xa48,
+	[MIB_RSCR33]		= 0xa50,
+	[MIB_RSCR35]		= 0xa58,
+	[MIB_RSCR36]		= 0xa5c,
+	[MIB_BSCR0]		= 0xbb8,
+	[MIB_BSCR1]		= 0xbbc,
+	[MIB_BSCR2]		= 0xbc0,
+	[MIB_BSCR3]		= 0xbc4,
+	[MIB_BSCR4]		= 0xbc8,
+	[MIB_BSCR5]		= 0xbcc,
+	[MIB_BSCR6]		= 0xbd0,
+	[MIB_BSCR7]		= 0xbd4,
+	[MIB_BSCR17]		= 0xbfc,
+	[MIB_TRDR1]		= 0xc14,
+	[HIF_REMAP_L1]		= 0x8,
+	[HIF_REMAP_BASE_L1]	= 0x40000,
+	[HIF_REMAP_L2]		= 0x1b8,
+	[HIF_REMAP_BASE_L2]	= 0x110000,
+	[CBTOP1_PHY_END]	= 0x7fffffff,
+	[INFRA_MCU_END]		= 0x7cffffff,
+	[WTBLON_WDUCR]		= 0x400,
+	[WTBL_UPDATE]		= 0x410,
+	[WTBL_ITCR]		= 0x440,
+	[WTBL_ITCR0]		= 0x448,
+	[WTBL_ITCR1]		= 0x44c,
 };
 
 static const struct __map mt7996_reg_map[] = {
@@ -138,14 +197,83 @@ static const struct __map mt7996_reg_map[] = {
 	{ 0x0, 0x0, 0x0 }, /* imply end of search */
 };
 
+static const struct __map mt7990_reg_map[] = {
+	{0x54000000, 0x02000, 0x1000}, /* WFDMA_0 (PCIE0 MCU DMA0) */
+	{0x55000000, 0x03000, 0x1000}, /* WFDMA_1 (PCIE0 MCU DMA1) */
+	{0x56000000, 0x04000, 0x1000}, /* WFDMA_2 (Reserved) */
+	{0x57000000, 0x05000, 0x1000}, /* WFDMA_3 (MCU wrap CR) */
+	{0x58000000, 0x06000, 0x1000}, /* WFDMA_4 (PCIE1 MCU DMA0 (MEM_DMA)) */
+	{0x59000000, 0x07000, 0x1000}, /* WFDMA_5 (PCIE1 MCU DMA1) */
+	{0x820c0000, 0x08000, 0x4000}, /* WF_UMAC_TOP (PLE) */
+	{0x820c8000, 0x0c000, 0x2000}, /* WF_UMAC_TOP (PSE) */
+	{0x820cc000, 0x0e000, 0x2000}, /* WF_UMAC_TOP (PP) */
+	{0x820e0000, 0x20000, 0x0400}, /* WF_LMAC_TOP BN0 (WF_CFG) */
+	{0x820e1000, 0x20400, 0x0200}, /* WF_LMAC_TOP BN0 (WF_TRB) */
+	{0x820e2000, 0x20800, 0x0400}, /* WF_LMAC_TOP BN0 (WF_AGG) */
+	{0x820e3000, 0x20c00, 0x0400}, /* WF_LMAC_TOP BN0 (WF_ARB) */
+	{0x820e4000, 0x21000, 0x0400}, /* WF_LMAC_TOP BN0 (WF_TMAC) */
+	{0x820e5000, 0x21400, 0x0800}, /* WF_LMAC_TOP BN0 (WF_RMAC) */
+	{0x820ce000, 0x21c00, 0x0200}, /* WF_LMAC_TOP (WF_SEC) */
+	{0x820e7000, 0x21e00, 0x0200}, /* WF_LMAC_TOP BN0 (WF_DMA) */
+	{0x820cf000, 0x22000, 0x1000}, /* WF_LMAC_TOP (WF_PF) */
+	{0x820e9000, 0x23400, 0x0200}, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
+	{0x820ea000, 0x24000, 0x0200}, /* WF_LMAC_TOP BN0 (WF_ETBF) */
+	{0x820eb000, 0x24200, 0x0400}, /* WF_LMAC_TOP BN0 (WF_LPON) */
+	{0x820ec000, 0x24600, 0x0200}, /* WF_LMAC_TOP BN0 (WF_INT) */
+	{0x820ed000, 0x24800, 0x0800}, /* WF_LMAC_TOP BN0 (WF_MIB) */
+	{0x820ca000, 0x26000, 0x2000}, /* WF_LMAC_TOP BN0 (WF_MUCOP) */
+	{0x820d0000, 0x30000, 0x10000}, /* WF_LMAC_TOP (WF_WTBLON) */
+	{0x00400000, 0x80000, 0x10000}, /* WF_MCU_SYSRAM */
+	{0x820f0000, 0xa0000, 0x0400}, /* WF_LMAC_TOP BN1 (WF_CFG) */
+	{0x820f1000, 0xa0600, 0x0200}, /* WF_LMAC_TOP BN1 (WF_TRB) */
+	{0x820f2000, 0xa0800, 0x0400}, /* WF_LMAC_TOP BN1 (WF_AGG) */
+	{0x820f3000, 0xa0c00, 0x0400}, /* WF_LMAC_TOP BN1 (WF_ARB) */
+	{0x820f4000, 0xa1000, 0x0400}, /* WF_LMAC_TOP BN1 (WF_TMAC) */
+	{0x820f5000, 0xa1400, 0x0800}, /* WF_LMAC_TOP BN1 (WF_RMAC) */
+	{0x820f7000, 0xa1e00, 0x0200}, /* WF_LMAC_TOP BN1 (WF_DMA) */
+	{0x820f9000, 0xa3400, 0x0200}, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
+	{0x820fa000, 0xa4000, 0x0200}, /* WF_LMAC_TOP BN1 (WF_ETBF) */
+	{0x820fb000, 0xa4200, 0x0400}, /* WF_LMAC_TOP BN1 (WF_LPON) */
+	{0x820fc000, 0xa4600, 0x0200}, /* WF_LMAC_TOP BN1 (WF_INT) */
+	{0x820fd000, 0xa4800, 0x0800}, /* WF_LMAC_TOP BN1 (WF_MIB) */
+	{0x820cc000, 0xa5000, 0x2000}, /* WF_LMAC_TOP BN1 (WF_MUCOP) */
+	{0x820c4000, 0xa8000, 0x4000}, /* WF_LMAC_TOP (WF_UWTBL) */
+	{0x81030000, 0xae000, 0x100}, /* WFSYS_AON part 1 */
+	{0x81031000, 0xae100, 0x100}, /* WFSYS_AON part 2 */
+	{0x81032000, 0xae200, 0x100}, /* WFSYS_AON part 3 */
+	{0x81033000, 0xae300, 0x100}, /* WFSYS_AON part 4 */
+	{0x81034000, 0xae400, 0x100}, /* WFSYS_AON part 5 */
+	{0x80020000, 0xb0000, 0x10000}, /* WF_TOP_MISC_OFF */
+	{0x81020000, 0xc0000, 0x10000}, /* WF_TOP_MISC_ON */
+	{0x81040000, 0x120000, 0x1000}, /* WF_MCU_CFG_ON */
+	{0x81050000, 0x121000, 0x1000}, /* WF_MCU_EINT */
+	{0x81060000, 0x122000, 0x1000}, /* WF_MCU_GPT */
+	{0x81070000, 0x123000, 0x1000}, /* WF_MCU_WDT */
+	{0x80010000, 0x124000, 0x1000}, /* WF_AXIDMA */
+	{0x7c020000, 0xd0000, 0x10000}, /* CONN_INFRA, wfdma for from CODA flow use */
+	{0x7c060000, 0xe0000, 0x10000}, /* CONN_INFRA, conn_host_csr_top for from CODA flow use */
+	{0x20020000, 0xd0000, 0x10000}, /* CONN_INFRA, wfdma */
+	{0x20060000, 0xe0000, 0x10000}, /* CONN_INFRA, conn_host_csr_top */
+	{0x7c000000, 0xf0000, 0x10000}, /* CONN_INFRA */
+	{0x70020000, 0x1f0000, 0x9000}, /* PCIE remapping (AP2CONN) */
+	{0x0, 0x0, 0x0}, /* imply end of search */
+};
+
 static u32 mt7996_reg_map_l1(struct mt7996_dev *dev, u32 addr)
 {
 	u32 offset = FIELD_GET(MT_HIF_REMAP_L1_OFFSET, addr);
 	u32 base = FIELD_GET(MT_HIF_REMAP_L1_BASE, addr);
+	u32 l1_mask, val;
 
-	dev->bus_ops->rmw(&dev->mt76, MT_HIF_REMAP_L1,
-			  MT_HIF_REMAP_L1_MASK,
-			  FIELD_PREP(MT_HIF_REMAP_L1_MASK, base));
+	if (is_mt7996(&dev->mt76)) {
+		l1_mask = MT_HIF_REMAP_L1_MASK_7996;
+		val = FIELD_PREP(MT_HIF_REMAP_L1_MASK_7996, base);
+	} else {
+		l1_mask = MT_HIF_REMAP_L1_MASK;
+		val = FIELD_PREP(MT_HIF_REMAP_L1_MASK, base);
+	}
+
+	dev->bus_ops->rmw(&dev->mt76, MT_HIF_REMAP_L1, l1_mask, val);
 	/* use read to push write */
 	dev->bus_ops->rr(&dev->mt76, MT_HIF_REMAP_L1);
 
@@ -154,18 +282,41 @@ static u32 mt7996_reg_map_l1(struct mt7996_dev *dev, u32 addr)
 
 static u32 mt7996_reg_map_l2(struct mt7996_dev *dev, u32 addr)
 {
-	u32 offset = FIELD_GET(MT_HIF_REMAP_L2_OFFSET, addr);
-	u32 base = FIELD_GET(MT_HIF_REMAP_L2_BASE, addr);
+	u32 offset, base, l2_mask, val;
 
-	dev->bus_ops->rmw(&dev->mt76, MT_HIF_REMAP_L2,
-			  MT_HIF_REMAP_L2_MASK,
-			  FIELD_PREP(MT_HIF_REMAP_L2_MASK, base));
+	if (is_mt7990(&dev->mt76)) {
+		offset = FIELD_GET(MT_HIF_REMAP_L2_OFFSET_7990, addr);
+		base = FIELD_GET(MT_HIF_REMAP_L2_BASE_7990, addr);
+		l2_mask = MT_HIF_REMAP_L2_MASK_7990;
+		val = FIELD_PREP(MT_HIF_REMAP_L2_MASK_7990, base);
+	} else {
+		offset = FIELD_GET(MT_HIF_REMAP_L2_OFFSET, addr);
+		base = FIELD_GET(MT_HIF_REMAP_L2_BASE, addr);
+		l2_mask = MT_HIF_REMAP_L2_MASK;
+		val = FIELD_PREP(MT_HIF_REMAP_L2_MASK, base);
+	}
+
+	dev->bus_ops->rmw(&dev->mt76, MT_HIF_REMAP_L2, l2_mask, val);
 	/* use read to push write */
 	dev->bus_ops->rr(&dev->mt76, MT_HIF_REMAP_L2);
 
 	return MT_HIF_REMAP_BASE_L2 + offset;
 }
 
+static u32 mt7996_reg_map_cbtop(struct mt7996_dev *dev, u32 addr)
+{
+	u32 offset = FIELD_GET(MT_HIF_REMAP_CBTOP_OFFSET, addr);
+	u32 base = FIELD_GET(MT_HIF_REMAP_CBTOP_BASE, addr);
+
+	dev->bus_ops->rmw(&dev->mt76, MT_HIF_REMAP_CBTOP,
+			  MT_HIF_REMAP_CBTOP_MASK,
+			  FIELD_PREP(MT_HIF_REMAP_CBTOP_MASK, base));
+	/* use read to push write */
+	dev->bus_ops->rr(&dev->mt76, MT_HIF_REMAP_CBTOP);
+
+	return MT_HIF_REMAP_BASE_CBTOP + offset;
+}
+
 static u32 __mt7996_reg_addr(struct mt7996_dev *dev, u32 addr)
 {
 	int i;
@@ -196,17 +347,20 @@ static u32 __mt7996_reg_remap_addr(struct mt7996_dev *dev, u32 addr)
 	    (addr >= MT_WFSYS1_PHY_START && addr <= MT_WFSYS1_PHY_END))
 		return mt7996_reg_map_l1(dev, addr);
 
-	if (dev_is_pci(dev->mt76.dev) &&
-	    ((addr >= MT_CBTOP1_PHY_START && addr <= MT_CBTOP1_PHY_END) ||
-	    addr >= MT_CBTOP2_PHY_START))
-		return mt7996_reg_map_l1(dev, addr);
-
 	/* CONN_INFRA: covert to phyiscal addr and use layer 1 remap */
 	if (addr >= MT_INFRA_MCU_START && addr <= MT_INFRA_MCU_END) {
 		addr = addr - MT_INFRA_MCU_START + MT_INFRA_BASE;
 		return mt7996_reg_map_l1(dev, addr);
 	}
 
+	if (dev_is_pci(dev->mt76.dev) &&
+	    ((addr >= MT_CBTOP1_PHY_START && addr <= MT_CBTOP1_PHY_END) ||
+	    addr >= MT_CBTOP2_PHY_START)) {
+		if (is_mt7990(&dev->mt76))
+			return mt7996_reg_map_cbtop(dev, addr);
+		return mt7996_reg_map_l1(dev, addr);
+	}
+
 	return mt7996_reg_map_l2(dev, addr);
 }
 
@@ -292,7 +446,7 @@ static int mt7996_mmio_wed_reset(struct mtk_wed_device *wed)
 	if (test_and_set_bit(MT76_STATE_WED_RESET, &mphy->state))
 		return -EBUSY;
 
-	ret = mt7996_mcu_set_ser(dev, UNI_CMD_SER_TRIGGER, UNI_CMD_SER_SET_RECOVER_L1,
+	ret = mt7996_mcu_set_ser(dev, UNI_CMD_SER_TRIGGER, UNI_CMD_SER_SET_RECOVER_FROM_ETH,
 				 mphy->band_idx);
 	if (ret)
 		goto out;
@@ -334,6 +488,9 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr,
 	wed->wlan.base = devm_ioremap(dev->mt76.dev,
 				      pci_resource_start(pci_dev, 0),
 				      pci_resource_len(pci_dev, 0));
+	if (!wed->wlan.base)
+		return -ENOMEM;
+
 	wed->wlan.phy_base = pci_resource_start(pci_dev, 0);
 
 	if (hif2) {
@@ -361,7 +518,7 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr,
 				     MT_RXQ_RING_BASE(MT7996_RXQ_BAND0) +
 				     MT7996_RXQ_BAND0 * MT_RING_SIZE;
 
-		wed->wlan.id = 0x7991;
+		wed->wlan.id = MT7996_DEVICE_ID_2;
 		wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND2) - 1;
 	} else {
 		wed->wlan.hw_rro = dev->has_rro; /* default on */
@@ -454,18 +611,24 @@ static int mt7996_mmio_init(struct mt76_dev *mdev,
 	spin_lock_init(&dev->reg_lock);
 
 	switch (device_id) {
-	case 0x7990:
+	case MT7996_DEVICE_ID:
 		dev->reg.base = mt7996_reg_base;
 		dev->reg.offs_rev = mt7996_offs;
 		dev->reg.map = mt7996_reg_map;
 		dev->reg.map_size = ARRAY_SIZE(mt7996_reg_map);
 		break;
-	case 0x7992:
+	case MT7992_DEVICE_ID:
 		dev->reg.base = mt7996_reg_base;
 		dev->reg.offs_rev = mt7992_offs;
 		dev->reg.map = mt7996_reg_map;
 		dev->reg.map_size = ARRAY_SIZE(mt7996_reg_map);
 		break;
+	case MT7990_DEVICE_ID:
+		dev->reg.base = mt7996_reg_base;
+		dev->reg.offs_rev = mt7990_offs;
+		dev->reg.map = mt7990_reg_map;
+		dev->reg.map_size = ARRAY_SIZE(mt7990_reg_map);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -629,9 +792,6 @@ struct mt7996_dev *mt7996_mmio_probe(struct device *pdev,
 		.rx_skb = mt7996_queue_rx_skb,
 		.rx_check = mt7996_rx_check,
 		.rx_poll_complete = mt7996_rx_poll_complete,
-		.sta_add = mt7996_mac_sta_add,
-		.sta_event = mt7996_mac_sta_event,
-		.sta_remove = mt7996_mac_sta_remove,
 		.update_survey = mt7996_update_channel,
 		.set_channel = mt7996_set_channel,
 		.vif_link_add = mt7996_vif_link_add,
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h b/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h
index f6491cf6005f..81cc1ca22cc6 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/mt7996.h
@@ -17,7 +17,7 @@
 #define MT7996_MAX_RADIOS		3
 #define MT7996_MAX_INTERFACES		19	/* per-band */
 #define MT7996_MAX_WMM_SETS		4
-#define MT7996_WTBL_BMC_SIZE		(is_mt7992(&dev->mt76) ? 32 : 64)
+#define MT7996_WTBL_BMC_SIZE		(is_mt7996(&dev->mt76) ? 64 : 32)
 #define MT7996_WTBL_RESERVED		(mt7996_wtbl_size(dev) - 1)
 #define MT7996_WTBL_STA			(MT7996_WTBL_RESERVED - \
 					 mt7996_max_interface_num(dev))
@@ -32,6 +32,16 @@
 #define MT7996_RX_RING_SIZE		1536
 #define MT7996_RX_MCU_RING_SIZE		512
 #define MT7996_RX_MCU_RING_SIZE_WA	1024
+/* scatter-gather of mcu event is not supported in connac3 */
+#define MT7996_RX_MCU_BUF_SIZE		(2048 + \
+					 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
+
+#define MT7996_DEVICE_ID		0x7990
+#define MT7996_DEVICE_ID_2		0x7991
+#define MT7992_DEVICE_ID		0x7992
+#define MT7992_DEVICE_ID_2		0x799a
+#define MT7990_DEVICE_ID		0x7993
+#define MT7990_DEVICE_ID_2		0x799b
 
 #define MT7996_FIRMWARE_WA		"mediatek/mt7996/mt7996_wa.bin"
 #define MT7996_FIRMWARE_WM		"mediatek/mt7996/mt7996_wm.bin"
@@ -53,6 +63,11 @@
 #define MT7992_FIRMWARE_DSP_23		"mediatek/mt7996/mt7992_dsp_23.bin"
 #define MT7992_ROM_PATCH_23		"mediatek/mt7996/mt7992_rom_patch_23.bin"
 
+#define MT7990_FIRMWARE_WA		""
+#define MT7990_FIRMWARE_WM		"mediatek/mt7996/mt7990_wm.bin"
+#define MT7990_FIRMWARE_DSP		""
+#define MT7990_ROM_PATCH		"mediatek/mt7996/mt7990_rom_patch.bin"
+
 #define MT7996_EEPROM_DEFAULT		"mediatek/mt7996/mt7996_eeprom.bin"
 #define MT7996_EEPROM_DEFAULT_INT	"mediatek/mt7996/mt7996_eeprom_2i5i6i.bin"
 #define MT7996_EEPROM_DEFAULT_233	"mediatek/mt7996/mt7996_eeprom_233.bin"
@@ -64,6 +79,9 @@
 #define MT7992_EEPROM_DEFAULT_23	"mediatek/mt7996/mt7992_eeprom_23.bin"
 #define MT7992_EEPROM_DEFAULT_23_INT	"mediatek/mt7996/mt7992_eeprom_23_2i5i.bin"
 
+#define MT7990_EEPROM_DEFAULT		"mediatek/mt7996/mt7990_eeprom.bin"
+#define MT7990_EEPROM_DEFAULT_INT	"mediatek/mt7996/mt7990_eeprom_2i5i.bin"
+
 #define MT7996_EEPROM_SIZE		7680
 #define MT7996_EEPROM_BLOCK_SIZE	16
 #define MT7996_TOKEN_SIZE		16384
@@ -134,6 +152,10 @@ enum mt7992_var_type {
 	MT7992_VAR_TYPE_23,
 };
 
+enum mt7990_var_type {
+	MT7990_VAR_TYPE_23,
+};
+
 enum mt7996_fem_type {
 	MT7996_FEM_EXT,
 	MT7996_FEM_INT,
@@ -168,6 +190,8 @@ enum mt7996_rxq_id {
 	MT7996_RXQ_TXFREE1 = 9,
 	MT7996_RXQ_TXFREE2 = 7,
 	MT7996_RXQ_RRO_IND = 0,
+	MT7990_RXQ_TXFREE0 = 6,
+	MT7990_RXQ_TXFREE1 = 7,
 };
 
 struct mt7996_twt_flow {
@@ -188,10 +212,10 @@ struct mt7996_twt_flow {
 
 DECLARE_EWMA(avg_signal, 10, 8)
 
-struct mt7996_sta {
+struct mt7996_sta_link {
 	struct mt76_wcid wcid; /* must be first */
 
-	struct mt7996_vif *vif;
+	struct mt7996_sta *sta;
 
 	struct list_head rc_list;
 	u32 airtime_ac[8];
@@ -207,12 +231,22 @@ struct mt7996_sta {
 		u8 flowid_mask;
 		struct mt7996_twt_flow flow[MT7996_MAX_STA_TWT_AGRT];
 	} twt;
+
+	struct rcu_head rcu_head;
+};
+
+struct mt7996_sta {
+	struct mt7996_sta_link deflink; /* must be first */
+	struct mt7996_sta_link __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
+	u8 deflink_id;
+
+	struct mt7996_vif *vif;
 };
 
 struct mt7996_vif_link {
 	struct mt76_vif_link mt76; /* must be first */
 
-	struct mt7996_sta sta;
+	struct mt7996_sta_link msta_link;
 	struct mt7996_phy *phy;
 
 	struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS];
@@ -274,8 +308,6 @@ struct mt7996_phy {
 	s16 coverage_class;
 	u8 slottime;
 
-	u8 rdd_state;
-
 	u16 beacon_rate;
 
 	u32 rx_ampdu_ts;
@@ -286,6 +318,7 @@ struct mt7996_phy {
 	struct mt76_channel_state state_ts;
 
 	u16 orig_chainmask;
+	u16 orig_antenna_mask;
 
 	bool has_aux_rx;
 	bool counter_reset;
@@ -398,10 +431,10 @@ enum {
 	__MT_WFDMA_MAX,
 };
 
-enum {
-	MT_RX_SEL0,
-	MT_RX_SEL1,
-	MT_RX_SEL2, /* monitor chain */
+enum rdd_idx {
+	MT_RDD_IDX_BAND2,	/* RDD idx for band idx 2 */
+	MT_RDD_IDX_BAND1,	/* RDD idx for band idx 1 */
+	MT_RDD_IDX_BACKGROUND,	/* RDD idx for background chain */
 };
 
 enum mt7996_rdd_cmd {
@@ -420,6 +453,21 @@ enum mt7996_rdd_cmd {
 	RDD_IRQ_OFF,
 };
 
+static inline int
+mt7996_get_rdd_idx(struct mt7996_phy *phy, bool is_background)
+{
+	if (!phy->mt76->cap.has_5ghz)
+		return -1;
+
+	if (is_background)
+		return MT_RDD_IDX_BACKGROUND;
+
+	if (phy->mt76->band_idx == MT_BAND2)
+		return MT_RDD_IDX_BAND2;
+
+	return MT_RDD_IDX_BAND1;
+}
+
 static inline struct mt7996_dev *
 mt7996_hw_dev(struct ieee80211_hw *hw)
 {
@@ -454,31 +502,12 @@ mt7996_phy3(struct mt7996_dev *dev)
 static inline bool
 mt7996_band_valid(struct mt7996_dev *dev, u8 band)
 {
-	if (is_mt7992(&dev->mt76))
+	if (!is_mt7996(&dev->mt76))
 		return band <= MT_BAND1;
 
 	return band <= MT_BAND2;
 }
 
-static inline bool
-mt7996_has_background_radar(struct mt7996_dev *dev)
-{
-	switch (mt76_chip(&dev->mt76)) {
-	case 0x7990:
-		if (dev->var.type == MT7996_VAR_TYPE_233)
-			return false;
-		break;
-	case 0x7992:
-		if (dev->var.type == MT7992_VAR_TYPE_23)
-			return false;
-		break;
-	default:
-		return false;
-	}
-
-	return true;
-}
-
 static inline struct mt7996_phy *
 mt7996_band_phy(struct mt7996_dev *dev, enum nl80211_band band)
 {
@@ -528,7 +557,7 @@ struct mt7996_dev *mt7996_mmio_probe(struct device *pdev,
 				     void __iomem *mem_base, u32 device_id);
 void mt7996_wfsys_reset(struct mt7996_dev *dev);
 irqreturn_t mt7996_irq_handler(int irq, void *dev_instance);
-u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif *mvif);
+u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif_link *link);
 int mt7996_register_device(struct mt7996_dev *dev);
 void mt7996_unregister_device(struct mt7996_dev *dev);
 int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif,
@@ -542,6 +571,7 @@ int mt7996_eeprom_parse_hw_cap(struct mt7996_dev *dev, struct mt7996_phy *phy);
 int mt7996_eeprom_get_target_power(struct mt7996_dev *dev,
 				   struct ieee80211_channel *chan);
 s8 mt7996_eeprom_get_power_delta(struct mt7996_dev *dev, int band);
+bool mt7996_eeprom_has_background_radar(struct mt7996_dev *dev);
 int mt7996_dma_init(struct mt7996_dev *dev);
 void mt7996_dma_reset(struct mt7996_dev *dev, bool force);
 void mt7996_dma_prefetch(struct mt7996_dev *dev);
@@ -556,7 +586,7 @@ int mt7996_run(struct mt7996_phy *phy);
 int mt7996_mcu_init(struct mt7996_dev *dev);
 int mt7996_mcu_init_firmware(struct mt7996_dev *dev);
 int mt7996_mcu_twt_agrt_update(struct mt7996_dev *dev,
-			       struct mt7996_vif *mvif,
+			       struct mt7996_vif_link *link,
 			       struct mt7996_twt_flow *flow,
 			       int cmd);
 int mt7996_mcu_add_dev_info(struct mt7996_phy *phy, struct ieee80211_vif *vif,
@@ -564,35 +594,46 @@ int mt7996_mcu_add_dev_info(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 			    struct mt76_vif_link *mlink, bool enable);
 int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 			    struct ieee80211_bss_conf *link_conf,
-			    struct mt76_vif_link *mlink, int enable);
-int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-		       struct mt76_vif_link *mlink,
-		       struct ieee80211_sta *sta, int conn_state, bool newly);
+			    struct mt76_vif_link *mlink,
+			    struct mt7996_sta_link *msta_link, int enable);
+int mt7996_mcu_add_sta(struct mt7996_dev *dev,
+		       struct ieee80211_bss_conf *link_conf,
+		       struct ieee80211_link_sta *link_sta,
+		       struct mt7996_vif_link *link,
+		       struct mt7996_sta_link *msta_link,
+		       int conn_state, bool newly);
+int mt7996_mcu_teardown_mld_sta(struct mt7996_dev *dev,
+				struct mt7996_vif_link *link,
+				struct mt7996_sta_link *msta_link);
 int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev,
 			 struct ieee80211_ampdu_params *params,
-			 bool add);
+			 struct mt7996_vif_link *link,
+			 struct mt7996_sta_link *msta_link, bool enable);
 int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev,
 			 struct ieee80211_ampdu_params *params,
-			 bool add);
+			 struct mt7996_vif_link *link, bool enable);
 int mt7996_mcu_update_bss_color(struct mt7996_dev *dev,
 				struct mt76_vif_link *mlink,
 				struct cfg80211_he_bss_color *he_bss_color);
 int mt7996_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			  struct ieee80211_bss_conf *link_conf);
 int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev,
-				    struct ieee80211_vif *vif, u32 changed);
-int mt7996_mcu_add_obss_spr(struct mt7996_phy *phy, struct ieee80211_vif *vif,
+				    struct ieee80211_bss_conf *link_conf,
+				    struct mt7996_vif_link *link, u32 changed);
+int mt7996_mcu_add_obss_spr(struct mt7996_phy *phy,
+			    struct mt7996_vif_link *link,
 			    struct ieee80211_he_obss_pd *he_obss_pd);
-int mt7996_mcu_add_rate_ctrl(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-			     struct ieee80211_sta *sta, bool changed);
+int mt7996_mcu_add_rate_ctrl(struct mt7996_dev *dev, struct mt7996_sta *msta,
+			     struct ieee80211_vif *vif, u8 link_id,
+			     bool changed);
 int mt7996_set_channel(struct mt76_phy *mphy);
 int mt7996_mcu_set_chan_info(struct mt7996_phy *phy, u16 tag);
 int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif,
 		      struct ieee80211_bss_conf *link_conf);
 int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev,
 				   void *data, u16 version);
-int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif,
-			       struct ieee80211_sta *sta, void *data, u32 field);
+int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct mt7996_sta *msta,
+			       void *data, u8 link_id, u32 field);
 int mt7996_mcu_set_eeprom(struct mt7996_dev *dev);
 int mt7996_mcu_get_eeprom(struct mt7996_dev *dev, u32 offset, u8 *buf, u32 buf_len);
 int mt7996_mcu_get_eeprom_free_block(struct mt7996_dev *dev, u8 *block_num);
@@ -613,8 +654,7 @@ int mt7996_mcu_get_temperature(struct mt7996_phy *phy);
 int mt7996_mcu_set_thermal_throttling(struct mt7996_phy *phy, u8 state);
 int mt7996_mcu_set_thermal_protect(struct mt7996_phy *phy, bool enable);
 int mt7996_mcu_set_txpower_sku(struct mt7996_phy *phy);
-int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index,
-		       u8 rx_sel, u8 val);
+int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 rdd_idx, u8 val);
 int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy,
 				     struct cfg80211_chan_def *chandef);
 int mt7996_mcu_set_fixed_rate_table(struct mt7996_phy *phy, u8 table_idx,
@@ -680,32 +720,31 @@ static inline u16 mt7996_rx_chainmask(struct mt7996_phy *phy)
 	return tx_chainmask | (BIT(fls(tx_chainmask)) * phy->has_aux_rx);
 }
 
+static inline bool mt7996_has_wa(struct mt7996_dev *dev)
+{
+	return !is_mt7990(&dev->mt76);
+}
+
 void mt7996_mac_init(struct mt7996_dev *dev);
 u32 mt7996_mac_wtbl_lmac_addr(struct mt7996_dev *dev, u16 wcid, u8 dw);
 bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask);
 void mt7996_mac_reset_counters(struct mt7996_phy *phy);
 void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy);
 void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band);
-void mt7996_mac_enable_rtscts(struct mt7996_dev *dev,
-			      struct ieee80211_vif *vif, bool enable);
 void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
 			   struct sk_buff *skb, struct mt76_wcid *wcid,
 			   struct ieee80211_key_conf *key, int pid,
 			   enum mt76_txq_id qid, u32 changed);
+void mt7996_mac_update_beacons(struct mt7996_phy *phy);
 void mt7996_mac_set_coverage_class(struct mt7996_phy *phy);
-int mt7996_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif,
-		       struct ieee80211_sta *sta);
-int mt7996_mac_sta_event(struct mt76_dev *mdev, struct ieee80211_vif *vif,
-			 struct ieee80211_sta *sta, enum mt76_sta_event ev);
-void mt7996_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif,
-			   struct ieee80211_sta *sta);
 void mt7996_mac_work(struct work_struct *work);
 void mt7996_mac_reset_work(struct work_struct *work);
 void mt7996_mac_dump_work(struct work_struct *work);
 void mt7996_mac_sta_rc_work(struct work_struct *work);
 void mt7996_mac_update_stats(struct mt7996_phy *phy);
 void mt7996_mac_twt_teardown_flow(struct mt7996_dev *dev,
-				  struct mt7996_sta *msta,
+				  struct mt7996_vif_link *link,
+				  struct mt7996_sta_link *msta_link,
 				  u8 flowid);
 void mt7996_mac_add_twt_setup(struct ieee80211_hw *hw,
 			      struct ieee80211_sta *sta,
@@ -730,11 +769,14 @@ bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len);
 int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif,
 		       struct ieee80211_key_conf *key, int mcu_cmd,
 		       struct mt76_wcid *wcid, enum set_key_cmd cmd);
-int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, struct ieee80211_vif *vif,
+int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev,
+			       struct mt7996_vif_link *link,
+			       struct mt7996_sta_link *msta_link,
 			       struct ieee80211_key_conf *key);
 int mt7996_mcu_wtbl_update_hdr_trans(struct mt7996_dev *dev,
 				     struct ieee80211_vif *vif,
-				     struct ieee80211_sta *sta);
+				     struct mt7996_vif_link *link,
+				     struct mt7996_sta_link *msta_link);
 int mt7996_mcu_cp_support(struct mt7996_dev *dev, u8 mode);
 #ifdef CONFIG_MAC80211_DEBUGFS
 void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/pci.c b/sys/contrib/dev/mediatek/mt76/mt7996/pci.c
index dfb86ba5bedb..ed32ff3089b9 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/pci.c
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/pci.c
@@ -20,14 +20,16 @@ static DEFINE_SPINLOCK(hif_lock);
 static u32 hif_idx;
 
 static const struct pci_device_id mt7996_pci_device_table[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7990) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7992) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, MT7996_DEVICE_ID) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, MT7992_DEVICE_ID) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, MT7990_DEVICE_ID) },
 	{ },
 };
 
 static const struct pci_device_id mt7996_hif_device_table[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7991) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x799a) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, MT7996_DEVICE_ID_2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, MT7992_DEVICE_ID_2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, MT7990_DEVICE_ID_2) },
 	{ },
 };
 
@@ -72,11 +74,13 @@ static struct mt7996_hif *mt7996_pci_init_hif2(struct pci_dev *pdev)
 	hif_idx++;
 
 #if defined(__linux__)
-	if (!pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x7991, NULL) &&
-	    !pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x799a, NULL))
+	if (!pci_get_device(PCI_VENDOR_ID_MEDIATEK, MT7996_DEVICE_ID_2, NULL) &&
+	    !pci_get_device(PCI_VENDOR_ID_MEDIATEK, MT7992_DEVICE_ID_2, NULL) &&
+	    !pci_get_device(PCI_VENDOR_ID_MEDIATEK, MT7990_DEVICE_ID_2, NULL))
 #elif defined(__FreeBSD__)
-	if (!linuxkpi_pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x7991, NULL) &&
-	    !linuxkpi_pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x799a, NULL))
+	if (!linuxkpi_pci_get_device(PCI_VENDOR_ID_MEDIATEK, MT7996_DEVICE_ID_2, NULL) &&
+	    !linuxkpi_pci_get_device(PCI_VENDOR_ID_MEDIATEK, MT7992_DEVICE_ID_2, NULL) &&
+	    !linuxkpi_pci_get_device(PCI_VENDOR_ID_MEDIATEK, MT7990_DEVICE_ID_2, NULL))
 #endif
 		return NULL;
 
@@ -138,7 +142,9 @@ static int mt7996_pci_probe(struct pci_dev *pdev,
 
 	mt76_pci_disable_aspm(pdev);
 
-	if (id->device == 0x7991 || id->device == 0x799a)
+	if (id->device == MT7996_DEVICE_ID_2 ||
+	    id->device == MT7992_DEVICE_ID_2 ||
+	    id->device == MT7990_DEVICE_ID_2)
 		return mt7996_pci_hif2_probe(pdev);
 
 	dev = mt7996_mmio_probe(&pdev->dev, pcim_iomap_table(pdev)[0],
@@ -273,6 +279,8 @@ MODULE_FIRMWARE(MT7992_FIRMWARE_WA);
 MODULE_FIRMWARE(MT7992_FIRMWARE_WM);
 MODULE_FIRMWARE(MT7992_FIRMWARE_DSP);
 MODULE_FIRMWARE(MT7992_ROM_PATCH);
+MODULE_FIRMWARE(MT7990_FIRMWARE_WM);
+MODULE_FIRMWARE(MT7990_ROM_PATCH);
 #if defined(__FreeBSD__)
 MODULE_VERSION(mt7996_pci, 1);
 MODULE_DEPEND(mt7996_pci, linuxkpi, 1, 1, 1);
diff --git a/sys/contrib/dev/mediatek/mt76/mt7996/regs.h b/sys/contrib/dev/mediatek/mt76/mt7996/regs.h
index 1876a968c92d..e942c0058731 100644
--- a/sys/contrib/dev/mediatek/mt76/mt7996/regs.h
+++ b/sys/contrib/dev/mediatek/mt76/mt7996/regs.h
@@ -64,6 +64,17 @@ enum offs_rev {
 	MIB_BSCR7,
 	MIB_BSCR17,
 	MIB_TRDR1,
+	HIF_REMAP_L1,
+	HIF_REMAP_BASE_L1,
+	HIF_REMAP_L2,
+	HIF_REMAP_BASE_L2,
+	CBTOP1_PHY_END,
+	INFRA_MCU_END,
+	WTBLON_WDUCR,
+	WTBL_UPDATE,
+	WTBL_ITCR,
+	WTBL_ITCR0,
+	WTBL_ITCR1,
 	__MT_OFFS_MAX,
 };
 
@@ -291,19 +302,19 @@ enum offs_rev {
 /* WTBLON TOP */
 #define MT_WTBLON_TOP_BASE			0x820d4000
 #define MT_WTBLON_TOP(ofs)			(MT_WTBLON_TOP_BASE + (ofs))
-#define MT_WTBLON_TOP_WDUCR			MT_WTBLON_TOP(0x370)
+#define MT_WTBLON_TOP_WDUCR			MT_WTBLON_TOP(__OFFS(WTBLON_WDUCR))
 #define MT_WTBLON_TOP_WDUCR_GROUP		GENMASK(4, 0)
 
-#define MT_WTBL_UPDATE				MT_WTBLON_TOP(0x380)
+#define MT_WTBL_UPDATE				MT_WTBLON_TOP(__OFFS(WTBL_UPDATE))
 #define MT_WTBL_UPDATE_WLAN_IDX			GENMASK(11, 0)
 #define MT_WTBL_UPDATE_ADM_COUNT_CLEAR		BIT(14)
 #define MT_WTBL_UPDATE_BUSY			BIT(31)
 
-#define MT_WTBL_ITCR				MT_WTBLON_TOP(0x3b0)
+#define MT_WTBL_ITCR				MT_WTBLON_TOP(__OFFS(WTBL_ITCR))
 #define MT_WTBL_ITCR_WR				BIT(16)
 #define MT_WTBL_ITCR_EXEC			BIT(31)
-#define MT_WTBL_ITDR0				MT_WTBLON_TOP(0x3b8)
-#define MT_WTBL_ITDR1				MT_WTBLON_TOP(0x3bc)
+#define MT_WTBL_ITDR0				MT_WTBLON_TOP(__OFFS(WTBL_ITCR0))
+#define MT_WTBL_ITDR1				MT_WTBLON_TOP(__OFFS(WTBL_ITCR1))
 #define MT_WTBL_SPE_IDX_SEL			BIT(6)
 
 /* WTBL */
@@ -483,7 +494,7 @@ enum offs_rev {
 
 #define MT_MCUQ_EXT_CTRL(q)			(MT_Q_BASE(q) +	0x600 +	\
 						 MT_MCUQ_ID(q) * 0x4)
-#define MT_RXQ_BAND1_CTRL(q)			(MT_Q_BASE(__RXQ(q)) + 0x680 +	\
+#define MT_RXQ_EXT_CTRL(q)			(MT_Q_BASE(__RXQ(q)) + 0x680 +	\
 						 MT_RXQ_ID(q) * 0x4)
 #define MT_TXQ_EXT_CTRL(q)			(MT_Q_BASE(__TXQ(q)) + 0x600 +	\
 						 MT_TXQ_ID(q) * 0x4)
@@ -504,6 +515,8 @@ enum offs_rev {
 #define MT_INT_RX_DONE_WA_TRI			BIT(3)
 #define MT_INT_RX_TXFREE_MAIN			BIT(17)
 #define MT_INT_RX_TXFREE_TRI			BIT(15)
+#define MT_INT_RX_TXFREE_BAND0_MT7990		BIT(14)
+#define MT_INT_RX_TXFREE_BAND1_MT7990		BIT(15)
 #define MT_INT_RX_DONE_BAND2_EXT		BIT(23)
 #define MT_INT_RX_TXFREE_EXT			BIT(26)
 #define MT_INT_MCU_CMD				BIT(29)
@@ -576,27 +589,39 @@ enum offs_rev {
 #define MT_MCU_CMD_WDT_MASK			GENMASK(31, 30)
 
 /* l1/l2 remap */
-#define MT_HIF_REMAP_L1				0x155024
-#define MT_HIF_REMAP_L1_MASK			GENMASK(31, 16)
+#define CONN_BUS_CR_VON_BASE			0x155000
+#define MT_HIF_REMAP_L1				(CONN_BUS_CR_VON_BASE + __OFFS(HIF_REMAP_L1))
+#define MT_HIF_REMAP_L1_MASK_7996		GENMASK(31, 16)
+#define MT_HIF_REMAP_L1_MASK			GENMASK(15, 0)
 #define MT_HIF_REMAP_L1_OFFSET			GENMASK(15, 0)
 #define MT_HIF_REMAP_L1_BASE			GENMASK(31, 16)
-#define MT_HIF_REMAP_BASE_L1			0x130000
+#define MT_HIF_REMAP_BASE_L1			__OFFS(HIF_REMAP_BASE_L1)
 
-#define MT_HIF_REMAP_L2				0x1b4
+#define MT_HIF_REMAP_L2				__OFFS(HIF_REMAP_L2)
 #define MT_HIF_REMAP_L2_MASK			GENMASK(19, 0)
 #define MT_HIF_REMAP_L2_OFFSET			GENMASK(11, 0)
 #define MT_HIF_REMAP_L2_BASE			GENMASK(31, 12)
-#define MT_HIF_REMAP_BASE_L2			0x1000
+#define MT_HIF_REMAP_L2_MASK_7990		GENMASK(15, 0)
+#define MT_HIF_REMAP_L2_OFFSET_7990		GENMASK(15, 0)
+#define MT_HIF_REMAP_L2_BASE_7990		GENMASK(31, 16)
+#define MT_HIF_REMAP_BASE_L2			__OFFS(HIF_REMAP_BASE_L2)
+
+/* for mt7990 only */
+#define MT_HIF_REMAP_CBTOP			0x1f6554
+#define MT_HIF_REMAP_CBTOP_MASK			GENMASK(15, 0)
+#define MT_HIF_REMAP_CBTOP_OFFSET		GENMASK(15, 0)
+#define MT_HIF_REMAP_CBTOP_BASE			GENMASK(31, 16)
+#define MT_HIF_REMAP_BASE_CBTOP			0x1c0000
 
 #define MT_INFRA_BASE				0x18000000
 #define MT_WFSYS0_PHY_START			0x18400000
 #define MT_WFSYS1_PHY_START			0x18800000
 #define MT_WFSYS1_PHY_END			0x18bfffff
 #define MT_CBTOP1_PHY_START			0x70000000
-#define MT_CBTOP1_PHY_END			0x77ffffff
+#define MT_CBTOP1_PHY_END			__OFFS(CBTOP1_PHY_END)
 #define MT_CBTOP2_PHY_START			0xf0000000
 #define MT_INFRA_MCU_START			0x7c000000
-#define MT_INFRA_MCU_END			0x7c3fffff
+#define MT_INFRA_MCU_END			__OFFS(INFRA_MCU_END)
 
 /* FW MODE SYNC */
 #define MT_FW_ASSERT_CNT			0x02208274
diff --git a/sys/contrib/dev/mediatek/mt76/scan.c b/sys/contrib/dev/mediatek/mt76/scan.c
index 1c4f9deaaada..9b20ccbeb8cf 100644
--- a/sys/contrib/dev/mediatek/mt76/scan.c
+++ b/sys/contrib/dev/mediatek/mt76/scan.c
@@ -52,11 +52,6 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid)
 		ether_addr_copy(hdr->addr3, req->bssid);
 	}
 
-	info = IEEE80211_SKB_CB(skb);
-	if (req->no_cck)
-		info->flags |= IEEE80211_TX_CTL_NO_CCK_RATE;
-	info->control.flags |= IEEE80211_TX_CTRL_DONT_USE_RATE_MASK;
-
 	if (req->ie_len)
 		skb_put_data(skb, req->ie, req->ie_len);
 
@@ -64,10 +59,20 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid)
 	skb_set_queue_mapping(skb, IEEE80211_AC_VO);
 
 	rcu_read_lock();
-	if (ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL))
-		mt76_tx(phy, NULL, mvif->wcid, skb);
-	else
+
+	if (!ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL)) {
 		ieee80211_free_txskb(phy->hw, skb);
+		goto out;
+	}
+
+	info = IEEE80211_SKB_CB(skb);
+	if (req->no_cck)
+		info->flags |= IEEE80211_TX_CTL_NO_CCK_RATE;
+	info->control.flags |= IEEE80211_TX_CTRL_DONT_USE_RATE_MASK;
+
+	mt76_tx(phy, NULL, mvif->wcid, skb);
+
+out:
 	rcu_read_unlock();
 }
 
diff --git a/sys/contrib/dev/mediatek/mt76/sdio_txrx.c b/sys/contrib/dev/mediatek/mt76/sdio_txrx.c
index 0a927a7313a6..f882d21c9f63 100644
--- a/sys/contrib/dev/mediatek/mt76/sdio_txrx.c
+++ b/sys/contrib/dev/mediatek/mt76/sdio_txrx.c
@@ -112,6 +112,7 @@ mt76s_rx_run_queue(struct mt76_dev *dev, enum mt76_rxq_id qid,
 
 	if (err < 0) {
 		dev_err(dev->dev, "sdio read data failed:%d\n", err);
+		atomic_set(&dev->bus_hung, true);
 		put_page(page);
 		return err;
 	}
@@ -234,9 +235,10 @@ static int __mt76s_xmit_queue(struct mt76_dev *dev, u8 *data, int len)
 	err = sdio_writesb(sdio->func, MCR_WTDR1, data, len);
 	sdio_release_host(sdio->func);
 
-	if (err)
+	if (err) {
 		dev_err(dev->dev, "sdio write failed: %d\n", err);
-
+		atomic_set(&dev->bus_hung, true);
+	}
 	return err;
 }
 
diff --git a/sys/contrib/dev/mediatek/mt76/tx.c b/sys/contrib/dev/mediatek/mt76/tx.c
index af0c50c983ec..8ab5840fee57 100644
--- a/sys/contrib/dev/mediatek/mt76/tx.c
+++ b/sys/contrib/dev/mediatek/mt76/tx.c
@@ -64,7 +64,7 @@ mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list)
 		struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb);
 		struct mt76_wcid *wcid;
 
-		wcid = rcu_dereference(dev->wcid[cb->wcid]);
+		wcid = __mt76_wcid_ptr(dev, cb->wcid);
 		if (wcid) {
 			status.sta = wcid_to_sta(wcid);
 			if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) {
@@ -100,7 +100,8 @@ __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags,
 		return;
 
 	/* Tx status can be unreliable. if it fails, mark the frame as ACKed */
-	if (flags & MT_TX_CB_TXS_FAILED) {
+	if (flags & MT_TX_CB_TXS_FAILED &&
+	    (dev->drv->drv_flags & MT_DRV_IGNORE_TXS_FAILED)) {
 		info->status.rates[0].count = 0;
 		info->status.rates[0].idx = -1;
 		info->flags |= IEEE80211_TX_STAT_ACK;
@@ -250,9 +251,7 @@ void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *
 
 	rcu_read_lock();
 
-	if (wcid_idx < ARRAY_SIZE(dev->wcid))
-		wcid = rcu_dereference(dev->wcid[wcid_idx]);
-
+	wcid = __mt76_wcid_ptr(dev, wcid_idx);
 	mt76_tx_check_non_aql(dev, wcid, skb);
 
 #ifdef CONFIG_NL80211_TESTMODE
@@ -333,6 +332,7 @@ mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta,
 	struct mt76_wcid *wcid, struct sk_buff *skb)
 {
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
 	struct sk_buff_head *head;
 
 	if (mt76_testmode_enabled(phy)) {
@@ -350,7 +350,8 @@ mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta,
 	info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx);
 
 	if ((info->flags & IEEE80211_TX_CTL_TX_OFFCHAN) ||
-	    (info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK))
+	    ((info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK) &&
+	     ieee80211_is_probe_req(hdr->frame_control)))
 		head = &wcid->tx_offchannel;
 	else
 		head = &wcid->tx_pending;
@@ -537,7 +538,7 @@ mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid)
 			break;
 
 		mtxq = (struct mt76_txq *)txq->drv_priv;
-		wcid = rcu_dereference(dev->wcid[mtxq->wcid]);
+		wcid = __mt76_wcid_ptr(dev, mtxq->wcid);
 		if (!wcid || test_bit(MT_WCID_FLAG_PS, &wcid->flags))
 			continue;
 
@@ -616,7 +617,8 @@ mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid,
 		if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) &&
 		    !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) &&
 		    !ieee80211_is_data(hdr->frame_control) &&
-		    !ieee80211_is_bufferable_mmpdu(skb))
+		    (!ieee80211_is_bufferable_mmpdu(skb) ||
+		     ieee80211_is_deauth(hdr->frame_control)))
 			qid = MT_TXQ_PSD;
 
 		q = phy->q_tx[qid];
@@ -644,6 +646,7 @@ mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid,
 static void mt76_txq_schedule_pending(struct mt76_phy *phy)
 {
 	LIST_HEAD(tx_list);
+	int ret = 0;
 
 	if (list_empty(&phy->tx_list))
 		return;
@@ -655,13 +658,13 @@ static void mt76_txq_schedule_pending(struct mt76_phy *phy)
 	list_splice_init(&phy->tx_list, &tx_list);
 	while (!list_empty(&tx_list)) {
 		struct mt76_wcid *wcid;
-		int ret;
 
 		wcid = list_first_entry(&tx_list, struct mt76_wcid, tx_list);
 		list_del_init(&wcid->tx_list);
 
 		spin_unlock(&phy->tx_lock);
-		ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_offchannel);
+		if (ret >= 0)
+			ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_offchannel);
 		if (ret >= 0 && !phy->offchannel)
 			ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending);
 		spin_lock(&phy->tx_lock);
@@ -670,9 +673,6 @@ static void mt76_txq_schedule_pending(struct mt76_phy *phy)
 		    !skb_queue_empty(&wcid->tx_offchannel) &&
 		    list_empty(&wcid->tx_list))
 			list_add_tail(&wcid->tx_list, &phy->tx_list);
-
-		if (ret < 0)
-			break;
 	}
 	spin_unlock(&phy->tx_lock);
 
diff --git a/sys/contrib/dev/mediatek/mt76/util.c b/sys/contrib/dev/mediatek/mt76/util.c
index 9be1cca22ae9..37cfa133010f 100644
--- a/sys/contrib/dev/mediatek/mt76/util.c
+++ b/sys/contrib/dev/mediatek/mt76/util.c
@@ -87,7 +87,7 @@ int mt76_get_min_avg_rssi(struct mt76_dev *dev, u8 phy_idx)
 			if (!(mask & 1))
 				continue;
 
-			wcid = rcu_dereference(dev->wcid[j]);
+			wcid = __mt76_wcid_ptr(dev, j);
 			if (!wcid || wcid->phy_idx != phy_idx)
 				continue;
 
diff --git a/sys/contrib/dev/mediatek/mt76/wed.c b/sys/contrib/dev/mediatek/mt76/wed.c
index f89e4537555c..63f69e152b1c 100644
--- a/sys/contrib/dev/mediatek/mt76/wed.c
+++ b/sys/contrib/dev/mediatek/mt76/wed.c
@@ -34,11 +34,10 @@ u32 mt76_wed_init_rx_buf(struct mtk_wed_device *wed, int size)
 	struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed);
 	struct mtk_wed_bm_desc *desc = wed->rx_buf_ring.desc;
 	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
-	int i, len = SKB_WITH_OVERHEAD(q->buf_size);
 	struct mt76_txwi_cache *t = NULL;
+	int i;
 
 	for (i = 0; i < size; i++) {
-		enum dma_data_direction dir;
 		dma_addr_t addr;
 		u32 offset;
 		int token;
@@ -53,9 +52,6 @@ u32 mt76_wed_init_rx_buf(struct mtk_wed_device *wed, int size)
 			goto unmap;
 
 		addr = page_pool_get_dma_addr(virt_to_head_page(buf)) + offset;
-		dir = page_pool_get_dma_dir(q->page_pool);
-		dma_sync_single_for_device(dev->dma_dev, addr, len, dir);
-
 		desc->buf0 = cpu_to_le32(addr);
 		token = mt76_rx_token_consume(dev, buf, t, addr);
 		if (token < 0) {
diff --git a/sys/contrib/dev/qat/qat_402xx.bin b/sys/contrib/dev/qat/qat_402xx.bin
new file mode 100644
index 000000000000..74151547edce
--- /dev/null
+++ b/sys/contrib/dev/qat/qat_402xx.bin
diff --git a/sys/contrib/dev/qat/qat_402xx_mmp.bin b/sys/contrib/dev/qat/qat_402xx_mmp.bin
new file mode 100644
index 000000000000..6404eb009d2f
--- /dev/null
+++ b/sys/contrib/dev/qat/qat_402xx_mmp.bin
diff --git a/sys/contrib/dev/rtw88/Makefile b/sys/contrib/dev/rtw88/Makefile
index 0cbbb366e393..0b3da05a2938 100644
--- a/sys/contrib/dev/rtw88/Makefile
+++ b/sys/contrib/dev/rtw88/Makefile
@@ -94,6 +94,15 @@ rtw88_8821au-objs		:= rtw8821au.o
 obj-$(CONFIG_RTW88_8812AU)	+= rtw88_8812au.o
 rtw88_8812au-objs		:= rtw8812au.o
 
+obj-$(CONFIG_RTW88_8814A)	+= rtw88_8814a.o
+rtw88_8814a-objs		:= rtw8814a.o rtw8814a_table.o
+
+obj-$(CONFIG_RTW88_8814AE)	+= rtw88_8814ae.o
+rtw88_8814ae-objs		:= rtw8814ae.o
+
+obj-$(CONFIG_RTW88_8814AU)	+= rtw88_8814au.o
+rtw88_8814au-objs		:= rtw8814au.o
+
 obj-$(CONFIG_RTW88_PCI)		+= rtw88_pci.o
 rtw88_pci-objs			:= pci.o
 
diff --git a/sys/contrib/dev/rtw88/coex.c b/sys/contrib/dev/rtw88/coex.c
index c929db1e53ca..b4dc6ff2c175 100644
--- a/sys/contrib/dev/rtw88/coex.c
+++ b/sys/contrib/dev/rtw88/coex.c
@@ -309,7 +309,7 @@ static void rtw_coex_tdma_timer_base(struct rtw_dev *rtwdev, u8 type)
 {
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
-	u8 para[2] = {0};
+	u8 para[6] = {};
 	u8 times;
 	u16 tbtt_interval = coex_stat->wl_beacon_interval;
 
@@ -1501,28 +1501,28 @@ static u8 rtw_coex_algorithm(struct rtw_dev *rtwdev)
 		algorithm = COEX_ALGO_HFP;
 		break;
 	case           BPM_HID:
-	case BPM_HFP + BPM_HID:
+	case BPM_HFP | BPM_HID:
 		algorithm = COEX_ALGO_HID;
 		break;
-	case BPM_HFP           + BPM_A2DP:
-	case           BPM_HID + BPM_A2DP:
-	case BPM_HFP + BPM_HID + BPM_A2DP:
+	case BPM_HFP           | BPM_A2DP:
+	case           BPM_HID | BPM_A2DP:
+	case BPM_HFP | BPM_HID | BPM_A2DP:
 		algorithm = COEX_ALGO_A2DP_HID;
 		break;
-	case BPM_HFP                      + BPM_PAN:
-	case           BPM_HID            + BPM_PAN:
-	case BPM_HFP + BPM_HID            + BPM_PAN:
+	case BPM_HFP                      | BPM_PAN:
+	case           BPM_HID            | BPM_PAN:
+	case BPM_HFP | BPM_HID            | BPM_PAN:
 		algorithm = COEX_ALGO_PAN_HID;
 		break;
-	case BPM_HFP           + BPM_A2DP + BPM_PAN:
-	case           BPM_HID + BPM_A2DP + BPM_PAN:
-	case BPM_HFP + BPM_HID + BPM_A2DP + BPM_PAN:
+	case BPM_HFP           | BPM_A2DP | BPM_PAN:
+	case           BPM_HID | BPM_A2DP | BPM_PAN:
+	case BPM_HFP | BPM_HID | BPM_A2DP | BPM_PAN:
 		algorithm = COEX_ALGO_A2DP_PAN_HID;
 		break;
 	case                                BPM_PAN:
 		algorithm = COEX_ALGO_PAN;
 		break;
-	case                     BPM_A2DP + BPM_PAN:
+	case                     BPM_A2DP | BPM_PAN:
 		algorithm = COEX_ALGO_A2DP_PAN;
 		break;
 	case                     BPM_A2DP:
diff --git a/sys/contrib/dev/rtw88/debug.c b/sys/contrib/dev/rtw88/debug.c
index f0ee8e62da3b..53742a3220f2 100644
--- a/sys/contrib/dev/rtw88/debug.c
+++ b/sys/contrib/dev/rtw88/debug.c
@@ -654,10 +654,10 @@ static void rtw_print_rate(struct seq_file *m, u8 rate)
 	case DESC_RATE6M...DESC_RATE54M:
 		rtw_print_ofdm_rate_txt(m, rate);
 		break;
-	case DESC_RATEMCS0...DESC_RATEMCS15:
+	case DESC_RATEMCS0...DESC_RATEMCS31:
 		rtw_print_ht_rate_txt(m, rate);
 		break;
-	case DESC_RATEVHT1SS_MCS0...DESC_RATEVHT2SS_MCS9:
+	case DESC_RATEVHT1SS_MCS0...DESC_RATEVHT4SS_MCS9:
 		rtw_print_vht_rate_txt(m, rate);
 		break;
 	default:
@@ -692,9 +692,11 @@ static int rtw_debugfs_get_tx_pwr_tbl(struct seq_file *m, void *v)
 {
 	struct rtw_debugfs_priv *debugfs_priv = m->private;
 	struct rtw_dev *rtwdev = debugfs_priv->rtwdev;
+	struct rtw_power_params pwr_param = {0};
 	struct rtw_hal *hal = &rtwdev->hal;
+	u8 nss = rtwdev->efuse.hw_cap.nss;
 	u8 path, rate, bw, ch, regd;
-	struct rtw_power_params pwr_param = {0};
+	u8 max_ht_rate, max_rate;
 
 	mutex_lock(&rtwdev->mutex);
 	bw = hal->current_band_width;
@@ -707,19 +709,23 @@ static int rtw_debugfs_get_tx_pwr_tbl(struct seq_file *m, void *v)
 	seq_printf(m, "%-4s %-10s %-9s %-9s (%-4s %-4s %-4s) %-4s\n",
 		   "path", "rate", "pwr", "base", "byr", "lmt", "sar", "rem");
 
+	max_ht_rate = DESC_RATEMCS0 + nss * 8 - 1;
+
+	if (rtwdev->chip->vht_supported)
+		max_rate = DESC_RATEVHT1SS_MCS0 + nss * 10 - 1;
+	else
+		max_rate = max_ht_rate;
+
 	mutex_lock(&hal->tx_power_mutex);
-	for (path = RF_PATH_A; path <= RF_PATH_B; path++) {
+	for (path = RF_PATH_A; path < hal->rf_path_num; path++) {
 		/* there is no CCK rates used in 5G */
 		if (hal->current_band_type == RTW_BAND_5G)
 			rate = DESC_RATE6M;
 		else
 			rate = DESC_RATE1M;
 
-		/* now, not support vht 3ss and vht 4ss*/
-		for (; rate <= DESC_RATEVHT2SS_MCS9; rate++) {
-			/* now, not support ht 3ss and ht 4ss*/
-			if (rate > DESC_RATEMCS15 &&
-			    rate < DESC_RATEVHT1SS_MCS0)
+		for (; rate <= max_rate; rate++) {
+			if (rate > max_ht_rate && rate <= DESC_RATEMCS31)
 				continue;
 
 			rtw_get_tx_power_params(rtwdev, path, rate, bw,
@@ -849,20 +855,28 @@ static int rtw_debugfs_get_phy_info(struct seq_file *m, void *v)
 			   last_cnt->num_qry_pkt[rate_id + 9]);
 	}
 
-	seq_printf(m, "[RSSI(dBm)] = {%d, %d}\n",
+	seq_printf(m, "[RSSI(dBm)] = {%d, %d, %d, %d}\n",
 		   dm_info->rssi[RF_PATH_A] - 100,
-		   dm_info->rssi[RF_PATH_B] - 100);
-	seq_printf(m, "[Rx EVM(dB)] = {-%d, -%d}\n",
+		   dm_info->rssi[RF_PATH_B] - 100,
+		   dm_info->rssi[RF_PATH_C] - 100,
+		   dm_info->rssi[RF_PATH_D] - 100);
+	seq_printf(m, "[Rx EVM(dB)] = {-%d, -%d, -%d, -%d}\n",
 		   dm_info->rx_evm_dbm[RF_PATH_A],
-		   dm_info->rx_evm_dbm[RF_PATH_B]);
-	seq_printf(m, "[Rx SNR] = {%d, %d}\n",
+		   dm_info->rx_evm_dbm[RF_PATH_B],
+		   dm_info->rx_evm_dbm[RF_PATH_C],
+		   dm_info->rx_evm_dbm[RF_PATH_D]);
+	seq_printf(m, "[Rx SNR] = {%d, %d, %d, %d}\n",
 		   dm_info->rx_snr[RF_PATH_A],
-		   dm_info->rx_snr[RF_PATH_B]);
-	seq_printf(m, "[CFO_tail(KHz)] = {%d, %d}\n",
+		   dm_info->rx_snr[RF_PATH_B],
+		   dm_info->rx_snr[RF_PATH_C],
+		   dm_info->rx_snr[RF_PATH_D]);
+	seq_printf(m, "[CFO_tail(KHz)] = {%d, %d, %d, %d}\n",
 		   dm_info->cfo_tail[RF_PATH_A],
-		   dm_info->cfo_tail[RF_PATH_B]);
+		   dm_info->cfo_tail[RF_PATH_B],
+		   dm_info->cfo_tail[RF_PATH_C],
+		   dm_info->cfo_tail[RF_PATH_D]);
 
-	if (dm_info->curr_rx_rate >= DESC_RATE11M) {
+	if (dm_info->curr_rx_rate >= DESC_RATE6M) {
 		seq_puts(m, "[Rx Average Status]:\n");
 		seq_printf(m, " * OFDM, EVM: {-%d}, SNR: {%d}\n",
 			   (u8)ewma_evm_read(&ewma_evm[RTW_EVM_OFDM]),
@@ -875,6 +889,13 @@ static int rtw_debugfs_get_phy_info(struct seq_file *m, void *v)
 			   (u8)ewma_evm_read(&ewma_evm[RTW_EVM_2SS_B]),
 			   (u8)ewma_snr_read(&ewma_snr[RTW_SNR_2SS_A]),
 			   (u8)ewma_snr_read(&ewma_snr[RTW_SNR_2SS_B]));
+		seq_printf(m, " * 3SS, EVM: {-%d, -%d, -%d}, SNR: {%d, %d, %d}\n",
+			   (u8)ewma_evm_read(&ewma_evm[RTW_EVM_3SS_A]),
+			   (u8)ewma_evm_read(&ewma_evm[RTW_EVM_3SS_B]),
+			   (u8)ewma_evm_read(&ewma_evm[RTW_EVM_3SS_C]),
+			   (u8)ewma_snr_read(&ewma_snr[RTW_SNR_3SS_A]),
+			   (u8)ewma_snr_read(&ewma_snr[RTW_SNR_3SS_B]),
+			   (u8)ewma_snr_read(&ewma_snr[RTW_SNR_3SS_C]));
 	}
 
 	seq_puts(m, "[Rx Counter]:\n");
diff --git a/sys/contrib/dev/rtw88/fw.c b/sys/contrib/dev/rtw88/fw.c
index ae5af6bc3e92..5ce4a6bcffb6 100644
--- a/sys/contrib/dev/rtw88/fw.c
+++ b/sys/contrib/dev/rtw88/fw.c
@@ -521,7 +521,7 @@ rtw_fw_send_general_info(struct rtw_dev *rtwdev)
 	u8 h2c_pkt[H2C_PKT_SIZE] = {0};
 	u16 total_size = H2C_PKT_HDR_SIZE + 4;
 
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		return;
 
 	rtw_h2c_pkt_set_header(h2c_pkt, H2C_PKT_GENERAL_INFO);
@@ -544,7 +544,7 @@ rtw_fw_send_phydm_info(struct rtw_dev *rtwdev)
 	u16 total_size = H2C_PKT_HDR_SIZE + 8;
 	u8 fw_rf_type = 0;
 
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		return;
 
 	if (hal->rf_type == RF_1T1R)
@@ -735,6 +735,7 @@ void rtw_fw_send_ra_info(struct rtw_dev *rtwdev, struct rtw_sta_info *si,
 {
 	u8 h2c_pkt[H2C_PKT_SIZE] = {0};
 	bool disable_pt = true;
+	u32 mask_hi;
 
 	SET_H2C_CMD_ID_CLASS(h2c_pkt, H2C_CMD_RA_INFO);
 
@@ -755,6 +756,20 @@ void rtw_fw_send_ra_info(struct rtw_dev *rtwdev, struct rtw_sta_info *si,
 	si->init_ra_lv = 0;
 
 	rtw_fw_send_h2c_command(rtwdev, h2c_pkt);
+
+	if (rtwdev->chip->id != RTW_CHIP_TYPE_8814A)
+		return;
+
+	SET_H2C_CMD_ID_CLASS(h2c_pkt, H2C_CMD_RA_INFO_HI);
+
+	mask_hi = si->ra_mask >> 32;
+
+	SET_RA_INFO_RA_MASK0(h2c_pkt, (mask_hi & 0xff));
+	SET_RA_INFO_RA_MASK1(h2c_pkt, (mask_hi & 0xff00) >> 8);
+	SET_RA_INFO_RA_MASK2(h2c_pkt, (mask_hi & 0xff0000) >> 16);
+	SET_RA_INFO_RA_MASK3(h2c_pkt, (mask_hi & 0xff000000) >> 24);
+
+	rtw_fw_send_h2c_command(rtwdev, h2c_pkt);
 }
 
 void rtw_fw_media_status_report(struct rtw_dev *rtwdev, u8 mac_id, bool connect)
@@ -1451,7 +1466,7 @@ void rtw_add_rsvd_page_sta(struct rtw_dev *rtwdev,
 int rtw_fw_write_data_rsvd_page(struct rtw_dev *rtwdev, u16 pg_addr,
 				u8 *buf, u32 size)
 {
-	u8 bckp[2];
+	u8 bckp[3];
 	u8 val;
 	u16 rsvd_pg_head;
 	u32 bcn_valid_addr;
@@ -1463,7 +1478,9 @@ int rtw_fw_write_data_rsvd_page(struct rtw_dev *rtwdev, u16 pg_addr,
 	if (!size)
 		return -EINVAL;
 
-	if (rtw_chip_wcpu_11n(rtwdev)) {
+	bckp[2] = rtw_read8(rtwdev, REG_BCN_CTRL);
+
+	if (rtw_chip_wcpu_8051(rtwdev)) {
 		rtw_write32_set(rtwdev, REG_DWBCN0_CTRL, BIT_BCN_VALID);
 	} else {
 		pg_addr &= BIT_MASK_BCN_HEAD_1_V1;
@@ -1476,6 +1493,9 @@ int rtw_fw_write_data_rsvd_page(struct rtw_dev *rtwdev, u16 pg_addr,
 	val |= BIT_ENSWBCN >> 8;
 	rtw_write8(rtwdev, REG_CR + 1, val);
 
+	rtw_write8(rtwdev, REG_BCN_CTRL,
+		   (bckp[2] & ~BIT_EN_BCN_FUNCTION) | BIT_DIS_TSF_UDT);
+
 	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_PCIE) {
 		val = rtw_read8(rtwdev, REG_FWHW_TXQ_CTRL + 2);
 		bckp[1] = val;
@@ -1489,7 +1509,7 @@ int rtw_fw_write_data_rsvd_page(struct rtw_dev *rtwdev, u16 pg_addr,
 		goto restore;
 	}
 
-	if (rtw_chip_wcpu_11n(rtwdev)) {
+	if (rtw_chip_wcpu_8051(rtwdev)) {
 		bcn_valid_addr = REG_DWBCN0_CTRL;
 		bcn_valid_mask = BIT_BCN_VALID;
 	} else {
@@ -1506,6 +1526,7 @@ restore:
 	rsvd_pg_head = rtwdev->fifo.rsvd_boundary;
 	rtw_write16(rtwdev, REG_FIFOPAGE_CTRL_2,
 		    rsvd_pg_head | BIT_BCN_VALID_V1);
+	rtw_write8(rtwdev, REG_BCN_CTRL, bckp[2]);
 	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_PCIE)
 		rtw_write8(rtwdev, REG_FWHW_TXQ_CTRL + 2, bckp[1]);
 	rtw_write8(rtwdev, REG_CR + 1, bckp[0]);
diff --git a/sys/contrib/dev/rtw88/fw.h b/sys/contrib/dev/rtw88/fw.h
index 404de1b0c407..48ad9ceab6ea 100644
--- a/sys/contrib/dev/rtw88/fw.h
+++ b/sys/contrib/dev/rtw88/fw.h
@@ -557,6 +557,7 @@ static inline void rtw_h2c_pkt_set_header(u8 *h2c_pkt, u8 sub_id)
 #define H2C_CMD_DEFAULT_PORT		0x2c
 #define H2C_CMD_RA_INFO			0x40
 #define H2C_CMD_RSSI_MONITOR		0x42
+#define H2C_CMD_RA_INFO_HI		0x46
 #define H2C_CMD_BCN_FILTER_OFFLOAD_P0	0x56
 #define H2C_CMD_BCN_FILTER_OFFLOAD_P1	0x57
 #define H2C_CMD_WL_PHY_INFO		0x58
diff --git a/sys/contrib/dev/rtw88/hci.h b/sys/contrib/dev/rtw88/hci.h
index 96aeda26014e..d4bee9c3ecfe 100644
--- a/sys/contrib/dev/rtw88/hci.h
+++ b/sys/contrib/dev/rtw88/hci.h
@@ -19,6 +19,8 @@ struct rtw_hci_ops {
 	void (*link_ps)(struct rtw_dev *rtwdev, bool enter);
 	void (*interface_cfg)(struct rtw_dev *rtwdev);
 	void (*dynamic_rx_agg)(struct rtw_dev *rtwdev, bool enable);
+	void (*write_firmware_page)(struct rtw_dev *rtwdev, u32 page,
+				    const u8 *data, u32 size);
 
 	int (*write_data_rsvd_page)(struct rtw_dev *rtwdev, u8 *buf, u32 size);
 	int (*write_data_h2c)(struct rtw_dev *rtwdev, u8 *buf, u32 size);
@@ -79,6 +81,12 @@ static inline void rtw_hci_dynamic_rx_agg(struct rtw_dev *rtwdev, bool enable)
 		rtwdev->hci.ops->dynamic_rx_agg(rtwdev, enable);
 }
 
+static inline void rtw_hci_write_firmware_page(struct rtw_dev *rtwdev, u32 page,
+					       const u8 *data, u32 size)
+{
+	rtwdev->hci.ops->write_firmware_page(rtwdev, page, data, size);
+}
+
 static inline int
 rtw_hci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, u32 size)
 {
diff --git a/sys/contrib/dev/rtw88/mac.c b/sys/contrib/dev/rtw88/mac.c
index cae9cca6dca3..eaa928bab240 100644
--- a/sys/contrib/dev/rtw88/mac.c
+++ b/sys/contrib/dev/rtw88/mac.c
@@ -41,7 +41,7 @@ void rtw_set_channel_mac(struct rtw_dev *rtwdev, u8 channel, u8 bw,
 	}
 	rtw_write32(rtwdev, REG_WMAC_TRXPTCL_CTL, value32);
 
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		return;
 
 	value32 = rtw_read32(rtwdev, REG_AFE_CTRL1) & ~(BIT_MAC_CLK_SEL);
@@ -67,7 +67,7 @@ static int rtw_mac_pre_system_cfg(struct rtw_dev *rtwdev)
 
 	rtw_write8(rtwdev, REG_RSV_CTRL, 0);
 
-	if (rtw_chip_wcpu_11n(rtwdev)) {
+	if (rtw_chip_wcpu_8051(rtwdev)) {
 		if (rtw_read32(rtwdev, REG_SYS_CFG1) & BIT_LDO)
 			rtw_write8(rtwdev, REG_LDO_SWR_CTRL, LDO_SEL);
 		else
@@ -278,7 +278,7 @@ static int rtw_mac_power_switch(struct rtw_dev *rtwdev, bool pwr_on)
 	bool cur_pwr;
 	int ret;
 
-	if (rtw_chip_wcpu_11ac(rtwdev)) {
+	if (rtw_chip_wcpu_3081(rtwdev)) {
 		rpwm = rtw_read8(rtwdev, rtwdev->hci.rpwm_addr);
 
 		/* Check FW still exist or not */
@@ -291,6 +291,7 @@ static int rtw_mac_power_switch(struct rtw_dev *rtwdev, bool pwr_on)
 	if (rtw_read8(rtwdev, REG_CR) == 0xea)
 		cur_pwr = false;
 	else if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB &&
+		 chip->id != RTW_CHIP_TYPE_8814A &&
 		 (rtw_read8(rtwdev, REG_SYS_STATUS1 + 1) & BIT(0)))
 		cur_pwr = false;
 	else
@@ -368,7 +369,7 @@ static int __rtw_mac_init_system_cfg_legacy(struct rtw_dev *rtwdev)
 
 static int rtw_mac_init_system_cfg(struct rtw_dev *rtwdev)
 {
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		return __rtw_mac_init_system_cfg_legacy(rtwdev);
 
 	return __rtw_mac_init_system_cfg(rtwdev);
@@ -784,7 +785,8 @@ static int __rtw_download_firmware(struct rtw_dev *rtwdev,
 	if (!check_firmware_size(data, size))
 		return -EINVAL;
 
-	if (!ltecoex_read_reg(rtwdev, 0x38, &ltecoex_bckp))
+	if (rtwdev->chip->ltecoex_addr &&
+	    !ltecoex_read_reg(rtwdev, 0x38, &ltecoex_bckp))
 		return -EBUSY;
 
 	wlan_cpu_enable(rtwdev, false);
@@ -802,7 +804,8 @@ static int __rtw_download_firmware(struct rtw_dev *rtwdev,
 
 	wlan_cpu_enable(rtwdev, true);
 
-	if (!ltecoex_reg_write(rtwdev, 0x38, ltecoex_bckp)) {
+	if (rtwdev->chip->ltecoex_addr &&
+	    !ltecoex_reg_write(rtwdev, 0x38, ltecoex_bckp)) {
 		ret = -EBUSY;
 		goto dlfw_fail;
 	}
@@ -853,8 +856,8 @@ fwdl_ready:
 	}
 }
 
-static void
-write_firmware_page(struct rtw_dev *rtwdev, u32 page, const u8 *data, u32 size)
+void rtw_write_firmware_page(struct rtw_dev *rtwdev, u32 page,
+			     const u8 *data, u32 size)
 {
 	u32 val32;
 	u32 block_nr;
@@ -884,6 +887,7 @@ write_firmware_page(struct rtw_dev *rtwdev, u32 page, const u8 *data, u32 size)
 		rtw_write32(rtwdev, write_addr, le32_to_cpu(remain_data));
 	}
 }
+EXPORT_SYMBOL(rtw_write_firmware_page);
 
 static int
 download_firmware_legacy(struct rtw_dev *rtwdev, const u8 *data, u32 size)
@@ -901,11 +905,13 @@ download_firmware_legacy(struct rtw_dev *rtwdev, const u8 *data, u32 size)
 	rtw_write8_set(rtwdev, REG_MCUFW_CTRL, BIT_FWDL_CHK_RPT);
 
 	for (page = 0; page < total_page; page++) {
-		write_firmware_page(rtwdev, page, data, DLFW_PAGE_SIZE_LEGACY);
+		rtw_hci_write_firmware_page(rtwdev, page, data,
+					    DLFW_PAGE_SIZE_LEGACY);
 		data += DLFW_PAGE_SIZE_LEGACY;
 	}
 	if (last_page_size)
-		write_firmware_page(rtwdev, page, data, last_page_size);
+		rtw_hci_write_firmware_page(rtwdev, page, data,
+					    last_page_size);
 
 	if (!check_hw_ready(rtwdev, REG_MCUFW_CTRL, BIT_FWDL_CHK_RPT, 1)) {
 		rtw_err(rtwdev, "failed to check download firmware report\n");
@@ -975,7 +981,7 @@ out:
 static
 int _rtw_download_firmware(struct rtw_dev *rtwdev, struct rtw_fw_state *fw)
 {
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		return __rtw_download_firmware_legacy(rtwdev, fw);
 
 	return __rtw_download_firmware(rtwdev, fw);
@@ -1116,7 +1122,7 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev)
 
 	rtw_write8(rtwdev, REG_CR, 0);
 	rtw_write8(rtwdev, REG_CR, MAC_TRX_ENABLE);
-	if (rtw_chip_wcpu_11ac(rtwdev))
+	if (rtw_chip_wcpu_3081(rtwdev))
 		rtw_write32(rtwdev, REG_H2CQ_CSR, BIT_H2CQ_FULL);
 
 	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_SDIO) {
@@ -1139,7 +1145,7 @@ int rtw_set_trx_fifo_info(struct rtw_dev *rtwdev)
 	/* config rsvd page num */
 	fifo->rsvd_drv_pg_num = chip->rsvd_drv_pg_num;
 	fifo->txff_pg_num = chip->txff_size / chip->page_size;
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		fifo->rsvd_pg_num = fifo->rsvd_drv_pg_num;
 	else
 		fifo->rsvd_pg_num = fifo->rsvd_drv_pg_num +
@@ -1157,7 +1163,7 @@ int rtw_set_trx_fifo_info(struct rtw_dev *rtwdev)
 	fifo->rsvd_boundary = fifo->txff_pg_num - fifo->rsvd_pg_num;
 
 	cur_pg_addr = fifo->txff_pg_num;
-	if (rtw_chip_wcpu_11ac(rtwdev)) {
+	if (rtw_chip_wcpu_3081(rtwdev)) {
 		cur_pg_addr -= csi_buf_pg_num;
 		fifo->rsvd_csibuf_addr = cur_pg_addr;
 		cur_pg_addr -= RSVD_PG_FW_TXBUF_NUM;
@@ -1286,7 +1292,7 @@ static int priority_queue_cfg(struct rtw_dev *rtwdev)
 
 	pubq_num = fifo->acq_pg_num - pg_tbl->hq_num - pg_tbl->lq_num -
 		   pg_tbl->nq_num - pg_tbl->exq_num - pg_tbl->gapq_num;
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		return __priority_queue_cfg_legacy(rtwdev, pg_tbl, pubq_num);
 	else
 		return __priority_queue_cfg(rtwdev, pg_tbl, pubq_num);
@@ -1302,7 +1308,7 @@ static int init_h2c(struct rtw_dev *rtwdev)
 	u32 h2cq_free;
 	u32 wp, rp;
 
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		return 0;
 
 	h2cq_addr = fifo->rsvd_h2cq_addr << TX_PAGE_SIZE_SHIFT;
@@ -1369,7 +1375,7 @@ static int rtw_drv_info_cfg(struct rtw_dev *rtwdev)
 	u8 value8;
 
 	rtw_write8(rtwdev, REG_RX_DRVINFO_SZ, PHY_STATUS_SIZE);
-	if (rtw_chip_wcpu_11ac(rtwdev)) {
+	if (rtw_chip_wcpu_3081(rtwdev)) {
 		value8 = rtw_read8(rtwdev, REG_TRXFF_BNDY + 1);
 		value8 &= 0xF0;
 		/* For rxdesc len = 0 issue */
@@ -1403,3 +1409,13 @@ int rtw_mac_init(struct rtw_dev *rtwdev)
 
 	return 0;
 }
+
+int rtw_mac_postinit(struct rtw_dev *rtwdev)
+{
+	const struct rtw_chip_info *chip = rtwdev->chip;
+
+	if (!chip->ops->mac_postinit)
+		return 0;
+
+	return chip->ops->mac_postinit(rtwdev);
+}
diff --git a/sys/contrib/dev/rtw88/mac.h b/sys/contrib/dev/rtw88/mac.h
index 6905e2747372..b73af90ee1d7 100644
--- a/sys/contrib/dev/rtw88/mac.h
+++ b/sys/contrib/dev/rtw88/mac.h
@@ -34,8 +34,11 @@ int rtw_pwr_seq_parser(struct rtw_dev *rtwdev,
 		       const struct rtw_pwr_seq_cmd * const *cmd_seq);
 int rtw_mac_power_on(struct rtw_dev *rtwdev);
 void rtw_mac_power_off(struct rtw_dev *rtwdev);
+void rtw_write_firmware_page(struct rtw_dev *rtwdev, u32 page,
+			     const u8 *data, u32 size);
 int rtw_download_firmware(struct rtw_dev *rtwdev, struct rtw_fw_state *fw);
 int rtw_mac_init(struct rtw_dev *rtwdev);
+int rtw_mac_postinit(struct rtw_dev *rtwdev);
 void rtw_mac_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop);
 int rtw_set_trx_fifo_info(struct rtw_dev *rtwdev);
 int rtw_ddma_to_fw_fifo(struct rtw_dev *rtwdev, u32 ocp_src, u32 size);
diff --git a/sys/contrib/dev/rtw88/mac80211.c b/sys/contrib/dev/rtw88/mac80211.c
index 719052b7c6bb..8d84f1a3bc18 100644
--- a/sys/contrib/dev/rtw88/mac80211.c
+++ b/sys/contrib/dev/rtw88/mac80211.c
@@ -71,7 +71,7 @@ static void rtw_ops_stop(struct ieee80211_hw *hw, bool suspend)
 	mutex_unlock(&rtwdev->mutex);
 }
 
-static int rtw_ops_config(struct ieee80211_hw *hw, u32 changed)
+static int rtw_ops_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct rtw_dev *rtwdev = hw->priv;
 	int ret = 0;
@@ -411,6 +411,8 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw,
 			if (rtw_bf_support)
 				rtw_bf_assoc(rtwdev, vif, conf);
 
+			rtw_set_ampdu_factor(rtwdev, vif, conf);
+
 			rtw_fw_beacon_filter_config(rtwdev, true, vif);
 		} else {
 			rtw_leave_lps(rtwdev);
@@ -721,7 +723,8 @@ static void rtw_ops_mgd_prepare_tx(struct ieee80211_hw *hw,
 	mutex_unlock(&rtwdev->mutex);
 }
 
-static int rtw_ops_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
+static int rtw_ops_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				     u32 value)
 {
 	struct rtw_dev *rtwdev = hw->priv;
 
@@ -810,6 +813,7 @@ static int rtw_ops_set_bitrate_mask(struct ieee80211_hw *hw,
 }
 
 static int rtw_ops_set_antenna(struct ieee80211_hw *hw,
+			       int radio_idx,
 			       u32 tx_antenna,
 			       u32 rx_antenna)
 {
@@ -821,13 +825,14 @@ static int rtw_ops_set_antenna(struct ieee80211_hw *hw,
 		return -EOPNOTSUPP;
 
 	mutex_lock(&rtwdev->mutex);
-	ret = chip->ops->set_antenna(rtwdev, tx_antenna, rx_antenna);
+	ret = chip->ops->set_antenna(rtwdev, -1, tx_antenna, rx_antenna);
 	mutex_unlock(&rtwdev->mutex);
 
 	return ret;
 }
 
 static int rtw_ops_get_antenna(struct ieee80211_hw *hw,
+			       int radio_idx,
 			       u32 *tx_antenna,
 			       u32 *rx_antenna)
 {
diff --git a/sys/contrib/dev/rtw88/main.c b/sys/contrib/dev/rtw88/main.c
index 021d076808e0..d9e6e9477dfb 100644
--- a/sys/contrib/dev/rtw88/main.c
+++ b/sys/contrib/dev/rtw88/main.c
@@ -57,6 +57,62 @@ module_param_named(support_vht, rtw_vht_support, bool, 0644);
 MODULE_PARM_DESC(support_vht, "Set to Y to enable VHT support");
 #endif
 
+#if defined(__FreeBSD__)
+/* Macros based on rtw89::core.c. */
+#define	RTW88_DEF_CHAN(_freq, _hw_val, _flags, _band)		\
+	{ .center_freq = _freq, .hw_value = _hw_val, .flags = _flags, .band = _band, }
+#define	RTW88_DEF_CHAN_2G(_freq, _hw_val)			\
+        RTW88_DEF_CHAN(_freq, _hw_val, 0, NL80211_BAND_2GHZ)
+#define	RTW88_DEF_CHAN_5G(_freq, _hw_val)			\
+        RTW88_DEF_CHAN(_freq, _hw_val, 0, NL80211_BAND_5GHZ)
+#define	RTW88_DEF_CHAN_5G_NO_HT40MINUS(_freq, _hw_val)		\
+        RTW88_DEF_CHAN(_freq, _hw_val, IEEE80211_CHAN_NO_HT40MINUS, NL80211_BAND_5GHZ)
+
+static struct ieee80211_channel rtw_channeltable_2g[] = {
+	RTW88_DEF_CHAN_2G(2412, 1),
+	RTW88_DEF_CHAN_2G(2417, 2),
+	RTW88_DEF_CHAN_2G(2422, 3),
+	RTW88_DEF_CHAN_2G(2427, 4),
+	RTW88_DEF_CHAN_2G(2432, 5),
+	RTW88_DEF_CHAN_2G(2437, 6),
+	RTW88_DEF_CHAN_2G(2442, 7),
+	RTW88_DEF_CHAN_2G(2447, 8),
+	RTW88_DEF_CHAN_2G(2452, 9),
+	RTW88_DEF_CHAN_2G(2457, 10),
+	RTW88_DEF_CHAN_2G(2462, 11),
+	RTW88_DEF_CHAN_2G(2467, 12),
+	RTW88_DEF_CHAN_2G(2472, 13),
+	RTW88_DEF_CHAN_2G(2484, 14),
+};
+
+static struct ieee80211_channel rtw_channeltable_5g[] = {
+	RTW88_DEF_CHAN_5G(5180, 36),
+	RTW88_DEF_CHAN_5G(5200, 40),
+	RTW88_DEF_CHAN_5G(5220, 44),
+	RTW88_DEF_CHAN_5G(5240, 48),
+	RTW88_DEF_CHAN_5G(5260, 52),
+	RTW88_DEF_CHAN_5G(5280, 56),
+	RTW88_DEF_CHAN_5G(5300, 60),
+	RTW88_DEF_CHAN_5G(5320, 64),
+	RTW88_DEF_CHAN_5G(5500, 100),
+	RTW88_DEF_CHAN_5G(5520, 104),
+	RTW88_DEF_CHAN_5G(5540, 108),
+	RTW88_DEF_CHAN_5G(5560, 112),
+	RTW88_DEF_CHAN_5G(5580, 116),
+	RTW88_DEF_CHAN_5G(5600, 120),
+	RTW88_DEF_CHAN_5G(5620, 124),
+	RTW88_DEF_CHAN_5G(5640, 128),
+	RTW88_DEF_CHAN_5G(5660, 132),
+	RTW88_DEF_CHAN_5G(5680, 136),
+	RTW88_DEF_CHAN_5G(5700, 140),
+	RTW88_DEF_CHAN_5G(5720, 144),
+	RTW88_DEF_CHAN_5G(5745, 149),
+	RTW88_DEF_CHAN_5G(5765, 153),
+	RTW88_DEF_CHAN_5G(5785, 157),
+	RTW88_DEF_CHAN_5G(5805, 161),
+	RTW88_DEF_CHAN_5G_NO_HT40MINUS(5825, 165),
+};
+#elif deifned(__linux__)
 static struct ieee80211_channel rtw_channeltable_2g[] = {
 	{.center_freq = 2412, .hw_value = 1,},
 	{.center_freq = 2417, .hw_value = 2,},
@@ -102,6 +158,7 @@ static struct ieee80211_channel rtw_channeltable_5g[] = {
 	{.center_freq = 5825, .hw_value = 165,
 	 .flags = IEEE80211_CHAN_NO_HT40MINUS},
 };
+#endif
 
 static struct ieee80211_rate rtw_ratetable[] = {
 	{.bitrate = 10, .hw_value = 0x00,},
@@ -150,7 +207,7 @@ u16 rtw_desc_to_bitrate(u8 desc_rate)
 	return rate.bitrate;
 }
 
-static struct ieee80211_supported_band rtw_band_2ghz = {
+static const struct ieee80211_supported_band rtw_band_2ghz = {
 	.band = NL80211_BAND_2GHZ,
 
 	.channels = rtw_channeltable_2g,
@@ -163,7 +220,7 @@ static struct ieee80211_supported_band rtw_band_2ghz = {
 	.vht_cap = {0},
 };
 
-static struct ieee80211_supported_band rtw_band_5ghz = {
+static const struct ieee80211_supported_band rtw_band_5ghz = {
 	.band = NL80211_BAND_5GHZ,
 
 	.channels = rtw_channeltable_5g,
@@ -363,7 +420,7 @@ int rtw_sta_add(struct rtw_dev *rtwdev, struct ieee80211_sta *sta,
 	struct rtw_vif *rtwvif = (struct rtw_vif *)vif->drv_priv;
 	int i;
 
-	if (vif->type == NL80211_IFTYPE_STATION) {
+	if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) {
 		si->mac_id = rtwvif->mac_id;
 	} else {
 		si->mac_id = rtw_acquire_macid(rtwdev);
@@ -405,7 +462,7 @@ void rtw_sta_remove(struct rtw_dev *rtwdev, struct ieee80211_sta *sta,
 
 	cancel_work_sync(&si->rc_work);
 
-	if (vif->type != NL80211_IFTYPE_STATION)
+	if (vif->type != NL80211_IFTYPE_STATION || sta->tdls)
 		rtw_release_macid(rtwdev, si->mac_id);
 	if (fw_exist)
 		rtw_fw_media_status_report(rtwdev, si->mac_id, false);
@@ -660,6 +717,7 @@ void rtw_fw_recovery(struct rtw_dev *rtwdev)
 	if (!test_bit(RTW_FLAG_RESTARTING, rtwdev->flags))
 		ieee80211_queue_work(rtwdev->hw, &rtwdev->fw_recovery_work);
 }
+EXPORT_SYMBOL(rtw_fw_recovery);
 
 static void __fw_recovery_work(struct rtw_dev *rtwdev)
 {
@@ -1258,7 +1316,9 @@ void rtw_update_sta_info(struct rtw_dev *rtwdev, struct rtw_sta_info *si,
 		if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC)
 			ldpc_en = VHT_LDPC_EN;
 	} else if (sta->deflink.ht_cap.ht_supported) {
-		ra_mask |= (sta->deflink.ht_cap.mcs.rx_mask[1] << 20) |
+		ra_mask |= ((u64)sta->deflink.ht_cap.mcs.rx_mask[3] << 36) |
+			   ((u64)sta->deflink.ht_cap.mcs.rx_mask[2] << 28) |
+			   (sta->deflink.ht_cap.mcs.rx_mask[1] << 20) |
 			   (sta->deflink.ht_cap.mcs.rx_mask[0] << 12);
 		if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_RX_STBC)
 			stbc_en = HT_STBC_EN;
@@ -1268,6 +1328,9 @@ void rtw_update_sta_info(struct rtw_dev *rtwdev, struct rtw_sta_info *si,
 
 	if (efuse->hw_cap.nss == 1 || rtwdev->hal.txrx_1ss)
 		ra_mask &= RA_MASK_VHT_RATES_1SS | RA_MASK_HT_RATES_1SS;
+	else if (efuse->hw_cap.nss == 2)
+		ra_mask &= RA_MASK_VHT_RATES_2SS | RA_MASK_HT_RATES_2SS |
+			   RA_MASK_VHT_RATES_1SS | RA_MASK_HT_RATES_1SS;
 
 	if (hal->current_band_type == RTW_BAND_5G) {
 		ra_mask |= (u64)sta->deflink.supp_rates[NL80211_BAND_5GHZ] << 4;
@@ -1330,10 +1393,9 @@ void rtw_update_sta_info(struct rtw_dev *rtwdev, struct rtw_sta_info *si,
 		break;
 	}
 
-	if (sta->deflink.vht_cap.vht_supported && ra_mask & 0xffc00000)
-		tx_num = 2;
-	else if (sta->deflink.ht_cap.ht_supported && ra_mask & 0xfff00000)
-		tx_num = 2;
+	if (sta->deflink.vht_cap.vht_supported ||
+	    sta->deflink.ht_cap.ht_supported)
+		tx_num = efuse->hw_cap.nss;
 
 	rate_id = get_rate_id(wireless_set, bw_mode, tx_num);
 
@@ -1435,6 +1497,12 @@ int rtw_power_on(struct rtw_dev *rtwdev)
 
 	chip->ops->phy_set_param(rtwdev);
 
+	ret = rtw_mac_postinit(rtwdev);
+	if (ret) {
+		rtw_err(rtwdev, "failed to configure mac in postinit\n");
+		goto err_off;
+	}
+
 	ret = rtw_hci_start(rtwdev);
 	if (ret) {
 		rtw_err(rtwdev, "failed to start hci\n");
@@ -1589,6 +1657,7 @@ static void rtw_init_ht_cap(struct rtw_dev *rtwdev,
 {
 	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
+	int i;
 
 	ht_cap->ht_supported = true;
 	ht_cap->cap = 0;
@@ -1608,25 +1677,20 @@ static void rtw_init_ht_cap(struct rtw_dev *rtwdev,
 	ht_cap->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K;
 	ht_cap->ampdu_density = chip->ampdu_density;
 	ht_cap->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED;
-	if (efuse->hw_cap.nss > 1) {
-		ht_cap->mcs.rx_mask[0] = 0xFF;
-		ht_cap->mcs.rx_mask[1] = 0xFF;
-		ht_cap->mcs.rx_mask[4] = 0x01;
-		ht_cap->mcs.rx_highest = cpu_to_le16(300);
-	} else {
-		ht_cap->mcs.rx_mask[0] = 0xFF;
-		ht_cap->mcs.rx_mask[1] = 0x00;
-		ht_cap->mcs.rx_mask[4] = 0x01;
-		ht_cap->mcs.rx_highest = cpu_to_le16(150);
-	}
+
+	for (i = 0; i < efuse->hw_cap.nss; i++)
+		ht_cap->mcs.rx_mask[i] = 0xFF;
+	ht_cap->mcs.rx_mask[4] = 0x01;
+	ht_cap->mcs.rx_highest = cpu_to_le16(150 * efuse->hw_cap.nss);
 }
 
 static void rtw_init_vht_cap(struct rtw_dev *rtwdev,
 			     struct ieee80211_sta_vht_cap *vht_cap)
 {
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	u16 mcs_map;
+	u16 mcs_map = 0;
 	__le16 highest;
+	int i;
 
 	if (efuse->hw_cap.ptcl != EFUSE_HW_CAP_IGNORE &&
 	    efuse->hw_cap.ptcl != EFUSE_HW_CAP_PTCL_VHT)
@@ -1649,21 +1713,15 @@ static void rtw_init_vht_cap(struct rtw_dev *rtwdev,
 	if (rtw_chip_has_rx_ldpc(rtwdev))
 		vht_cap->cap |= IEEE80211_VHT_CAP_RXLDPC;
 
-	mcs_map = IEEE80211_VHT_MCS_SUPPORT_0_9 << 0 |
-		  IEEE80211_VHT_MCS_NOT_SUPPORTED << 4 |
-		  IEEE80211_VHT_MCS_NOT_SUPPORTED << 6 |
-		  IEEE80211_VHT_MCS_NOT_SUPPORTED << 8 |
-		  IEEE80211_VHT_MCS_NOT_SUPPORTED << 10 |
-		  IEEE80211_VHT_MCS_NOT_SUPPORTED << 12 |
-		  IEEE80211_VHT_MCS_NOT_SUPPORTED << 14;
-	if (efuse->hw_cap.nss > 1) {
-		highest = cpu_to_le16(780);
-		mcs_map |= IEEE80211_VHT_MCS_SUPPORT_0_9 << 2;
-	} else {
-		highest = cpu_to_le16(390);
-		mcs_map |= IEEE80211_VHT_MCS_NOT_SUPPORTED << 2;
+	for (i = 0; i < 8; i++) {
+		if (i < efuse->hw_cap.nss)
+			mcs_map |= IEEE80211_VHT_MCS_SUPPORT_0_9 << (i * 2);
+		else
+			mcs_map |= IEEE80211_VHT_MCS_NOT_SUPPORTED << (i * 2);
 	}
 
+	highest = cpu_to_le16(390 * efuse->hw_cap.nss);
+
 	vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(mcs_map);
 	vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(mcs_map);
 	vht_cap->vht_mcs.rx_highest = highest;
@@ -1815,7 +1873,7 @@ static void __update_firmware_info_legacy(struct rtw_dev *rtwdev,
 static void update_firmware_info(struct rtw_dev *rtwdev,
 				 struct rtw_fw_state *fw)
 {
-	if (rtw_chip_wcpu_11n(rtwdev))
+	if (rtw_chip_wcpu_8051(rtwdev))
 		__update_firmware_info_legacy(rtwdev, fw);
 	else
 		__update_firmware_info(rtwdev, fw);
@@ -2272,7 +2330,6 @@ EXPORT_SYMBOL(rtw_core_deinit);
 
 int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 {
-	bool sta_mode_only = rtwdev->hci.type == RTW_HCI_TYPE_SDIO;
 	struct rtw_hal *hal = &rtwdev->hal;
 	int max_tx_headroom = 0;
 	int ret;
@@ -2296,17 +2353,15 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 	ieee80211_hw_set(hw, SUPPORTS_PS);
 	ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
 	ieee80211_hw_set(hw, SUPPORT_FAST_XMIT);
-	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
+	if (rtwdev->chip->amsdu_in_ampdu)
+		ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
 	ieee80211_hw_set(hw, HAS_RATE_CONTROL);
 	ieee80211_hw_set(hw, TX_AMSDU);
 	ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS);
 
-	if (sta_mode_only)
-		hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION);
-	else
-		hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
-					     BIT(NL80211_IFTYPE_AP) |
-					     BIT(NL80211_IFTYPE_ADHOC);
+	hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
+				     BIT(NL80211_IFTYPE_AP) |
+				     BIT(NL80211_IFTYPE_ADHOC);
 	hw->wiphy->available_antennas_tx = hal->antenna_tx;
 	hw->wiphy->available_antennas_rx = hal->antenna_rx;
 
@@ -2317,7 +2372,7 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 	hw->wiphy->max_scan_ssids = RTW_SCAN_MAX_SSIDS;
 	hw->wiphy->max_scan_ie_len = rtw_get_max_scan_ie_len(rtwdev);
 
-	if (!sta_mode_only && rtwdev->chip->id == RTW_CHIP_TYPE_8822C) {
+	if (rtwdev->chip->id == RTW_CHIP_TYPE_8822C) {
 		hw->wiphy->iface_combinations = rtw_iface_combs;
 		hw->wiphy->n_iface_combinations = ARRAY_SIZE(rtw_iface_combs);
 	}
@@ -2501,6 +2556,38 @@ void rtw_core_enable_beacon(struct rtw_dev *rtwdev, bool enable)
 	}
 }
 
+void rtw_set_ampdu_factor(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *bss_conf)
+{
+	const struct rtw_chip_ops *ops = rtwdev->chip->ops;
+	struct ieee80211_sta *sta;
+	u8 factor = 0xff;
+
+	if (!ops->set_ampdu_factor)
+		return;
+
+	rcu_read_lock();
+
+	sta = ieee80211_find_sta(vif, bss_conf->bssid);
+	if (!sta) {
+		rcu_read_unlock();
+		rtw_warn(rtwdev, "%s: failed to find station %pM\n",
+			 __func__, bss_conf->bssid);
+		return;
+	}
+
+	if (sta->deflink.vht_cap.vht_supported)
+		factor = u32_get_bits(sta->deflink.vht_cap.cap,
+				      IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK);
+	else if (sta->deflink.ht_cap.ht_supported)
+		factor = sta->deflink.ht_cap.ampdu_factor;
+
+	rcu_read_unlock();
+
+	if (factor != 0xff)
+		ops->set_ampdu_factor(rtwdev, factor);
+}
+
 MODULE_AUTHOR("Realtek Corporation");
 MODULE_DESCRIPTION("Realtek 802.11ac wireless core module");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sys/contrib/dev/rtw88/main.h b/sys/contrib/dev/rtw88/main.h
index 963863618523..d1e4f3e41ba1 100644
--- a/sys/contrib/dev/rtw88/main.h
+++ b/sys/contrib/dev/rtw88/main.h
@@ -70,7 +70,7 @@ enum rtw_hci_type {
 };
 
 struct rtw_hci {
-	struct rtw_hci_ops *ops;
+	const struct rtw_hci_ops *ops;
 	enum rtw_hci_type type;
 
 	u32 rpwm_addr;
@@ -175,9 +175,14 @@ enum rtw_rate_section {
 	RTW_RATE_SECTION_HT_2S,
 	RTW_RATE_SECTION_VHT_1S,
 	RTW_RATE_SECTION_VHT_2S,
+	__RTW_RATE_SECTION_2SS_MAX = RTW_RATE_SECTION_VHT_2S,
+	RTW_RATE_SECTION_HT_3S,
+	RTW_RATE_SECTION_HT_4S,
+	RTW_RATE_SECTION_VHT_3S,
+	RTW_RATE_SECTION_VHT_4S,
 
 	/* keep last */
-	RTW_RATE_SECTION_MAX,
+	RTW_RATE_SECTION_NUM,
 };
 
 enum rtw_wireless_set {
@@ -200,6 +205,7 @@ enum rtw_chip_type {
 	RTW_CHIP_TYPE_8703B,
 	RTW_CHIP_TYPE_8821A,
 	RTW_CHIP_TYPE_8812A,
+	RTW_CHIP_TYPE_8814A,
 };
 
 enum rtw_tx_queue_type {
@@ -389,6 +395,9 @@ enum rtw_evm {
 	RTW_EVM_1SS,
 	RTW_EVM_2SS_A,
 	RTW_EVM_2SS_B,
+	RTW_EVM_3SS_A,
+	RTW_EVM_3SS_B,
+	RTW_EVM_3SS_C,
 	/* keep it last */
 	RTW_EVM_NUM
 };
@@ -406,6 +415,10 @@ enum rtw_snr {
 	RTW_SNR_2SS_B,
 	RTW_SNR_2SS_C,
 	RTW_SNR_2SS_D,
+	RTW_SNR_3SS_A,
+	RTW_SNR_3SS_B,
+	RTW_SNR_3SS_C,
+	RTW_SNR_3SS_D,
 	/* keep it last */
 	RTW_SNR_NUM
 };
@@ -831,7 +844,7 @@ struct rtw_vif {
 };
 
 struct rtw_regulatory {
-	char alpha2[2];
+	char alpha2[2] __nonstring;
 	u8 txpwr_regd_2g;
 	u8 txpwr_regd_5g;
 };
@@ -854,6 +867,7 @@ struct rtw_chip_ops {
 	int (*power_on)(struct rtw_dev *rtwdev);
 	void (*power_off)(struct rtw_dev *rtwdev);
 	int (*mac_init)(struct rtw_dev *rtwdev);
+	int (*mac_postinit)(struct rtw_dev *rtwdev);
 	int (*dump_fw_crash)(struct rtw_dev *rtwdev);
 	void (*shutdown)(struct rtw_dev *rtwdev);
 	int (*read_efuse)(struct rtw_dev *rtwdev, u8 *map);
@@ -869,11 +883,12 @@ struct rtw_chip_ops {
 	void (*set_tx_power_index)(struct rtw_dev *rtwdev);
 	int (*rsvd_page_dump)(struct rtw_dev *rtwdev, u8 *buf, u32 offset,
 			      u32 size);
-	int (*set_antenna)(struct rtw_dev *rtwdev,
+	int (*set_antenna)(struct rtw_dev *rtwdev, int radio_idx,
 			   u32 antenna_tx,
 			   u32 antenna_rx);
 	void (*cfg_ldo25)(struct rtw_dev *rtwdev, bool enable);
 	void (*efuse_grant)(struct rtw_dev *rtwdev, bool enable);
+	void (*set_ampdu_factor)(struct rtw_dev *rtwdev, u8 factor);
 	void (*false_alarm_statistics)(struct rtw_dev *rtwdev);
 	void (*phy_calibration)(struct rtw_dev *rtwdev);
 	void (*dpk_track)(struct rtw_dev *rtwdev);
@@ -1139,14 +1154,26 @@ struct rtw_rfe_def {
  * For 2G there are cck rate and ofdm rate with different settings.
  */
 struct rtw_pwr_track_tbl {
+	const u8 *pwrtrk_5gd_n[RTW_PWR_TRK_5G_NUM];
+	const u8 *pwrtrk_5gd_p[RTW_PWR_TRK_5G_NUM];
+	const u8 *pwrtrk_5gc_n[RTW_PWR_TRK_5G_NUM];
+	const u8 *pwrtrk_5gc_p[RTW_PWR_TRK_5G_NUM];
 	const u8 *pwrtrk_5gb_n[RTW_PWR_TRK_5G_NUM];
 	const u8 *pwrtrk_5gb_p[RTW_PWR_TRK_5G_NUM];
 	const u8 *pwrtrk_5ga_n[RTW_PWR_TRK_5G_NUM];
 	const u8 *pwrtrk_5ga_p[RTW_PWR_TRK_5G_NUM];
+	const u8 *pwrtrk_2gd_n;
+	const u8 *pwrtrk_2gd_p;
+	const u8 *pwrtrk_2gc_n;
+	const u8 *pwrtrk_2gc_p;
 	const u8 *pwrtrk_2gb_n;
 	const u8 *pwrtrk_2gb_p;
 	const u8 *pwrtrk_2ga_n;
 	const u8 *pwrtrk_2ga_p;
+	const u8 *pwrtrk_2g_cckd_n;
+	const u8 *pwrtrk_2g_cckd_p;
+	const u8 *pwrtrk_2g_cckc_n;
+	const u8 *pwrtrk_2g_cckc_p;
 	const u8 *pwrtrk_2g_cckb_n;
 	const u8 *pwrtrk_2g_cckb_p;
 	const u8 *pwrtrk_2g_ccka_n;
@@ -1156,8 +1183,8 @@ struct rtw_pwr_track_tbl {
 };
 
 enum rtw_wlan_cpu {
-	RTW_WCPU_11AC,
-	RTW_WCPU_11N,
+	RTW_WCPU_3081,
+	RTW_WCPU_8051,
 };
 
 enum rtw_fw_fifo_sel {
@@ -1213,6 +1240,7 @@ struct rtw_chip_info {
 	u16 fw_fifo_addr[RTW_FW_FIFO_MAX];
 	const struct rtw_fwcd_segs *fwcd_segs;
 
+	bool amsdu_in_ampdu;
 	u8 usb_tx_agg_desc_num;
 	bool hw_feature_report;
 	u8 c2h_ra_report_size;
@@ -1236,8 +1264,8 @@ struct rtw_chip_info {
 
 	const struct rtw_hw_reg *dig;
 	const struct rtw_hw_reg *dig_cck;
-	u32 rf_base_addr[2];
-	u32 rf_sipi_addr[2];
+	u32 rf_base_addr[RTW_RF_PATH_MAX];
+	u32 rf_sipi_addr[RTW_RF_PATH_MAX];
 	const struct rtw_rf_sipi_addr *rf_sipi_read_addr;
 	u8 fix_rf_phy_num;
 	const struct rtw_ltecoex_addr *ltecoex_addr;
@@ -1933,7 +1961,7 @@ union rtw_sar_cfg {
 
 struct rtw_sar {
 	enum rtw_sar_sources src;
-	union rtw_sar_cfg cfg[RTW_RF_PATH_MAX][RTW_RATE_SECTION_MAX];
+	union rtw_sar_cfg cfg[RTW_RF_PATH_MAX][RTW_RATE_SECTION_NUM];
 };
 
 struct rtw_hal {
@@ -1977,16 +2005,16 @@ struct rtw_hal {
 	s8 tx_pwr_by_rate_offset_5g[RTW_RF_PATH_MAX]
 				   [DESC_RATE_MAX];
 	s8 tx_pwr_by_rate_base_2g[RTW_RF_PATH_MAX]
-				 [RTW_RATE_SECTION_MAX];
+				 [RTW_RATE_SECTION_NUM];
 	s8 tx_pwr_by_rate_base_5g[RTW_RF_PATH_MAX]
-				 [RTW_RATE_SECTION_MAX];
+				 [RTW_RATE_SECTION_NUM];
 	s8 tx_pwr_limit_2g[RTW_REGD_MAX]
 			  [RTW_CHANNEL_WIDTH_MAX]
-			  [RTW_RATE_SECTION_MAX]
+			  [RTW_RATE_SECTION_NUM]
 			  [RTW_MAX_CHANNEL_NUM_2G];
 	s8 tx_pwr_limit_5g[RTW_REGD_MAX]
 			  [RTW_CHANNEL_WIDTH_MAX]
-			  [RTW_RATE_SECTION_MAX]
+			  [RTW_RATE_SECTION_NUM]
 			  [RTW_MAX_CHANNEL_NUM_5G];
 	s8 tx_pwr_tbl[RTW_RF_PATH_MAX]
 		     [DESC_RATE_MAX];
@@ -2148,14 +2176,14 @@ static inline void rtw_chip_efuse_grant_off(struct rtw_dev *rtwdev)
 		rtwdev->chip->ops->efuse_grant(rtwdev, false);
 }
 
-static inline bool rtw_chip_wcpu_11n(struct rtw_dev *rtwdev)
+static inline bool rtw_chip_wcpu_8051(struct rtw_dev *rtwdev)
 {
-	return rtwdev->chip->wlan_cpu == RTW_WCPU_11N;
+	return rtwdev->chip->wlan_cpu == RTW_WCPU_8051;
 }
 
-static inline bool rtw_chip_wcpu_11ac(struct rtw_dev *rtwdev)
+static inline bool rtw_chip_wcpu_3081(struct rtw_dev *rtwdev)
 {
-	return rtwdev->chip->wlan_cpu == RTW_WCPU_11AC;
+	return rtwdev->chip->wlan_cpu == RTW_WCPU_3081;
 }
 
 static inline bool rtw_chip_has_rx_ldpc(struct rtw_dev *rtwdev)
@@ -2256,6 +2284,9 @@ void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel,
 void rtw_core_port_switch(struct rtw_dev *rtwdev, struct ieee80211_vif *vif);
 bool rtw_core_check_sta_active(struct rtw_dev *rtwdev);
 void rtw_core_enable_beacon(struct rtw_dev *rtwdev, bool enable);
+void rtw_set_ampdu_factor(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *bss_conf);
+
 #if defined(__linux__)
 #define	rtw88_static_assert(_x)		static_assert(_x)
 #elif defined(__FreeBSD__)
diff --git a/sys/contrib/dev/rtw88/pci.c b/sys/contrib/dev/rtw88/pci.c
index 6f2fedea116c..e4cbaae4646f 100644
--- a/sys/contrib/dev/rtw88/pci.c
+++ b/sys/contrib/dev/rtw88/pci.c
@@ -16,6 +16,7 @@
 #include "fw.h"
 #include "ps.h"
 #include "debug.h"
+#include "mac.h"
 #if defined(__FreeBSD__)
 #include <sys/rman.h>
 #include <linux/pm.h>
@@ -28,7 +29,7 @@ module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644);
 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support");
 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support");
 
-static u32 rtw_pci_tx_queue_idx_addr[] = {
+static const u32 rtw_pci_tx_queue_idx_addr[] = {
 	[RTW_TX_QUEUE_BK]	= RTK_PCI_TXBD_IDX_BKQ,
 	[RTW_TX_QUEUE_BE]	= RTK_PCI_TXBD_IDX_BEQ,
 	[RTW_TX_QUEUE_VI]	= RTK_PCI_TXBD_IDX_VIQ,
@@ -451,7 +452,7 @@ static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev)
 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;
 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma);
 
-	if (!rtw_chip_wcpu_11n(rtwdev)) {
+	if (!rtw_chip_wcpu_8051(rtwdev)) {
 		len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;
 		dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;
 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;
@@ -513,7 +514,7 @@ static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev)
 	rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff);
 
 	/* reset H2C Queue index in a single write */
-	if (rtw_chip_wcpu_11ac(rtwdev))
+	if (rtw_chip_wcpu_3081(rtwdev))
 		rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR,
 				BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX);
 }
@@ -533,7 +534,7 @@ static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev,
 
 	rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);
 	rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);
-	if (rtw_chip_wcpu_11ac(rtwdev))
+	if (rtw_chip_wcpu_3081(rtwdev))
 		rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);
 
 	rtwpci->irq_enabled = true;
@@ -553,7 +554,7 @@ static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev,
 
 	rtw_write32(rtwdev, RTK_PCI_HIMR0, 0);
 	rtw_write32(rtwdev, RTK_PCI_HIMR1, 0);
-	if (rtw_chip_wcpu_11ac(rtwdev))
+	if (rtw_chip_wcpu_3081(rtwdev))
 		rtw_write32(rtwdev, RTK_PCI_HIMR3, 0);
 
 	rtwpci->irq_enabled = false;
@@ -1181,7 +1182,7 @@ static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev,
 
 	irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0);
 	irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1);
-	if (rtw_chip_wcpu_11ac(rtwdev))
+	if (rtw_chip_wcpu_3081(rtwdev))
 		irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3);
 	else
 		irq_status[3] = 0;
@@ -1190,7 +1191,7 @@ static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev,
 	irq_status[3] &= rtwpci->irq_mask[3];
 	rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]);
 	rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]);
-	if (rtw_chip_wcpu_11ac(rtwdev))
+	if (rtw_chip_wcpu_3081(rtwdev))
 		rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]);
 
 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
@@ -1670,7 +1671,7 @@ static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev)
 	rtw_pci_io_unmapping(rtwdev, pdev);
 }
 
-static struct rtw_hci_ops rtw_pci_ops = {
+static const struct rtw_hci_ops rtw_pci_ops = {
 	.tx_write = rtw_pci_tx_write,
 	.tx_kick_off = rtw_pci_tx_kick_off,
 	.flush_queues = rtw_pci_flush_queues,
@@ -1681,6 +1682,7 @@ static struct rtw_hci_ops rtw_pci_ops = {
 	.link_ps = rtw_pci_link_ps,
 	.interface_cfg = rtw_pci_interface_cfg,
 	.dynamic_rx_agg = NULL,
+	.write_firmware_page = rtw_write_firmware_page,
 
 	.read8 = rtw_pci_read8,
 	.read16 = rtw_pci_read16,
@@ -1784,6 +1786,43 @@ static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)
 	free_netdev(rtwpci->netdev);
 }
 
+static pci_ers_result_t rtw_pci_io_err_detected(struct pci_dev *pdev,
+						pci_channel_state_t state)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+
+	netif_device_detach(netdev);
+
+	return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static pci_ers_result_t rtw_pci_io_slot_reset(struct pci_dev *pdev)
+{
+	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
+	struct rtw_dev *rtwdev = hw->priv;
+
+	rtw_fw_recovery(rtwdev);
+
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+static void rtw_pci_io_resume(struct pci_dev *pdev)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+
+	/* ack any pending wake events, disable PME */
+	pci_enable_wake(pdev, PCI_D0, 0);
+
+	netif_device_attach(netdev);
+}
+
+const struct pci_error_handlers rtw_pci_err_handler = {
+	.error_detected = rtw_pci_io_err_detected,
+	.slot_reset = rtw_pci_io_slot_reset,
+	.resume = rtw_pci_io_resume,
+};
+EXPORT_SYMBOL(rtw_pci_err_handler);
+
 int rtw_pci_probe(struct pci_dev *pdev,
 		  const struct pci_device_id *id)
 {
diff --git a/sys/contrib/dev/rtw88/pci.h b/sys/contrib/dev/rtw88/pci.h
index 13988db1cb4c..8ffdea11378f 100644
--- a/sys/contrib/dev/rtw88/pci.h
+++ b/sys/contrib/dev/rtw88/pci.h
@@ -231,6 +231,7 @@ struct rtw_pci {
 };
 
 extern const struct dev_pm_ops rtw_pm_ops;
+extern const struct pci_error_handlers rtw_pci_err_handler;
 
 int rtw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
 void rtw_pci_remove(struct pci_dev *pdev);
diff --git a/sys/contrib/dev/rtw88/phy.c b/sys/contrib/dev/rtw88/phy.c
index 8ed20c89d216..55be0d8e0c28 100644
--- a/sys/contrib/dev/rtw88/phy.c
+++ b/sys/contrib/dev/rtw88/phy.c
@@ -52,60 +52,93 @@ static const u32 db_invert_table[12][8] = {
 	 1995262315,	2511886432U,	3162277660U,	3981071706U}
 };
 
-u8 rtw_cck_rates[] = { DESC_RATE1M, DESC_RATE2M, DESC_RATE5_5M, DESC_RATE11M };
-u8 rtw_ofdm_rates[] = {
+const u8 rtw_cck_rates[] = { DESC_RATE1M, DESC_RATE2M, DESC_RATE5_5M, DESC_RATE11M };
+
+const u8 rtw_ofdm_rates[] = {
 	DESC_RATE6M,  DESC_RATE9M,  DESC_RATE12M,
 	DESC_RATE18M, DESC_RATE24M, DESC_RATE36M,
 	DESC_RATE48M, DESC_RATE54M
 };
-u8 rtw_ht_1s_rates[] = {
+
+const u8 rtw_ht_1s_rates[] = {
 	DESC_RATEMCS0, DESC_RATEMCS1, DESC_RATEMCS2,
 	DESC_RATEMCS3, DESC_RATEMCS4, DESC_RATEMCS5,
 	DESC_RATEMCS6, DESC_RATEMCS7
 };
-u8 rtw_ht_2s_rates[] = {
+
+const u8 rtw_ht_2s_rates[] = {
 	DESC_RATEMCS8,  DESC_RATEMCS9,  DESC_RATEMCS10,
 	DESC_RATEMCS11, DESC_RATEMCS12, DESC_RATEMCS13,
 	DESC_RATEMCS14, DESC_RATEMCS15
 };
-u8 rtw_vht_1s_rates[] = {
+
+const u8 rtw_vht_1s_rates[] = {
 	DESC_RATEVHT1SS_MCS0, DESC_RATEVHT1SS_MCS1,
 	DESC_RATEVHT1SS_MCS2, DESC_RATEVHT1SS_MCS3,
 	DESC_RATEVHT1SS_MCS4, DESC_RATEVHT1SS_MCS5,
 	DESC_RATEVHT1SS_MCS6, DESC_RATEVHT1SS_MCS7,
 	DESC_RATEVHT1SS_MCS8, DESC_RATEVHT1SS_MCS9
 };
-u8 rtw_vht_2s_rates[] = {
+
+const u8 rtw_vht_2s_rates[] = {
 	DESC_RATEVHT2SS_MCS0, DESC_RATEVHT2SS_MCS1,
 	DESC_RATEVHT2SS_MCS2, DESC_RATEVHT2SS_MCS3,
 	DESC_RATEVHT2SS_MCS4, DESC_RATEVHT2SS_MCS5,
 	DESC_RATEVHT2SS_MCS6, DESC_RATEVHT2SS_MCS7,
 	DESC_RATEVHT2SS_MCS8, DESC_RATEVHT2SS_MCS9
 };
-u8 *rtw_rate_section[RTW_RATE_SECTION_MAX] = {
+
+const u8 rtw_ht_3s_rates[] = {
+	DESC_RATEMCS16, DESC_RATEMCS17, DESC_RATEMCS18,
+	DESC_RATEMCS19, DESC_RATEMCS20, DESC_RATEMCS21,
+	DESC_RATEMCS22, DESC_RATEMCS23
+};
+
+const u8 rtw_ht_4s_rates[] = {
+	DESC_RATEMCS24, DESC_RATEMCS25, DESC_RATEMCS26,
+	DESC_RATEMCS27, DESC_RATEMCS28, DESC_RATEMCS29,
+	DESC_RATEMCS30, DESC_RATEMCS31
+};
+
+const u8 rtw_vht_3s_rates[] = {
+	DESC_RATEVHT3SS_MCS0, DESC_RATEVHT3SS_MCS1,
+	DESC_RATEVHT3SS_MCS2, DESC_RATEVHT3SS_MCS3,
+	DESC_RATEVHT3SS_MCS4, DESC_RATEVHT3SS_MCS5,
+	DESC_RATEVHT3SS_MCS6, DESC_RATEVHT3SS_MCS7,
+	DESC_RATEVHT3SS_MCS8, DESC_RATEVHT3SS_MCS9
+};
+
+const u8 rtw_vht_4s_rates[] = {
+	DESC_RATEVHT4SS_MCS0, DESC_RATEVHT4SS_MCS1,
+	DESC_RATEVHT4SS_MCS2, DESC_RATEVHT4SS_MCS3,
+	DESC_RATEVHT4SS_MCS4, DESC_RATEVHT4SS_MCS5,
+	DESC_RATEVHT4SS_MCS6, DESC_RATEVHT4SS_MCS7,
+	DESC_RATEVHT4SS_MCS8, DESC_RATEVHT4SS_MCS9
+};
+
+const u8 * const rtw_rate_section[RTW_RATE_SECTION_NUM] = {
 	rtw_cck_rates, rtw_ofdm_rates,
 	rtw_ht_1s_rates, rtw_ht_2s_rates,
-	rtw_vht_1s_rates, rtw_vht_2s_rates
+	rtw_vht_1s_rates, rtw_vht_2s_rates,
+	rtw_ht_3s_rates, rtw_ht_4s_rates,
+	rtw_vht_3s_rates, rtw_vht_4s_rates
 };
 EXPORT_SYMBOL(rtw_rate_section);
 
-u8 rtw_rate_size[RTW_RATE_SECTION_MAX] = {
+const u8 rtw_rate_size[RTW_RATE_SECTION_NUM] = {
 	ARRAY_SIZE(rtw_cck_rates),
 	ARRAY_SIZE(rtw_ofdm_rates),
 	ARRAY_SIZE(rtw_ht_1s_rates),
 	ARRAY_SIZE(rtw_ht_2s_rates),
 	ARRAY_SIZE(rtw_vht_1s_rates),
-	ARRAY_SIZE(rtw_vht_2s_rates)
+	ARRAY_SIZE(rtw_vht_2s_rates),
+	ARRAY_SIZE(rtw_ht_3s_rates),
+	ARRAY_SIZE(rtw_ht_4s_rates),
+	ARRAY_SIZE(rtw_vht_3s_rates),
+	ARRAY_SIZE(rtw_vht_4s_rates)
 };
 EXPORT_SYMBOL(rtw_rate_size);
 
-static const u8 rtw_cck_size = ARRAY_SIZE(rtw_cck_rates);
-static const u8 rtw_ofdm_size = ARRAY_SIZE(rtw_ofdm_rates);
-static const u8 rtw_ht_1s_size = ARRAY_SIZE(rtw_ht_1s_rates);
-static const u8 rtw_ht_2s_size = ARRAY_SIZE(rtw_ht_2s_rates);
-static const u8 rtw_vht_1s_size = ARRAY_SIZE(rtw_vht_1s_rates);
-static const u8 rtw_vht_2s_size = ARRAY_SIZE(rtw_vht_2s_rates);
-
 enum rtw_phy_band_type {
 	PHY_BAND_2G	= 0,
 	PHY_BAND_5G	= 1,
@@ -1590,7 +1623,7 @@ static void rtw_phy_set_tx_power_limit(struct rtw_dev *rtwdev, u8 regd, u8 band,
 	ch_idx = rtw_channel_to_idx(band, ch);
 
 	if (regd >= RTW_REGD_MAX || bw >= RTW_CHANNEL_WIDTH_MAX ||
-	    rs >= RTW_RATE_SECTION_MAX || ch_idx < 0) {
+	    rs >= RTW_RATE_SECTION_NUM || ch_idx < 0) {
 		WARN(1,
 		     "wrong txpwr_lmt regd=%u, band=%u bw=%u, rs=%u, ch_idx=%u, pwr_limit=%d\n",
 		     regd, band, bw, rs, ch_idx, pwr_limit);
@@ -1634,11 +1667,15 @@ rtw_xref_5g_txpwr_lmt(struct rtw_dev *rtwdev, u8 regd,
 static void
 rtw_xref_txpwr_lmt_by_rs(struct rtw_dev *rtwdev, u8 regd, u8 bw, u8 ch_idx)
 {
+	static const u8 rs_cmp[4][2] = {
+		{RTW_RATE_SECTION_HT_1S, RTW_RATE_SECTION_VHT_1S},
+		{RTW_RATE_SECTION_HT_2S, RTW_RATE_SECTION_VHT_2S},
+		{RTW_RATE_SECTION_HT_3S, RTW_RATE_SECTION_VHT_3S},
+		{RTW_RATE_SECTION_HT_4S, RTW_RATE_SECTION_VHT_4S}
+	};
 	u8 rs_idx, rs_ht, rs_vht;
-	u8 rs_cmp[2][2] = {{RTW_RATE_SECTION_HT_1S, RTW_RATE_SECTION_VHT_1S},
-			   {RTW_RATE_SECTION_HT_2S, RTW_RATE_SECTION_VHT_2S} };
 
-	for (rs_idx = 0; rs_idx < 2; rs_idx++) {
+	for (rs_idx = 0; rs_idx < 4; rs_idx++) {
 		rs_ht = rs_cmp[rs_idx][0];
 		rs_vht = rs_cmp[rs_idx][1];
 
@@ -1695,7 +1732,7 @@ rtw_cfg_txpwr_lmt_by_alt(struct rtw_dev *rtwdev, u8 regd, u8 regd_alt)
 	u8 bw, rs;
 
 	for (bw = 0; bw < RTW_CHANNEL_WIDTH_MAX; bw++)
-		for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++)
+		for (rs = 0; rs < RTW_RATE_SECTION_NUM; rs++)
 			__cfg_txpwr_lmt_by_alt(&rtwdev->hal, regd, regd_alt,
 					       bw, rs);
 }
@@ -1959,10 +1996,10 @@ static u8 rtw_phy_get_2g_tx_power_index(struct rtw_dev *rtwdev,
 					u8 rate, u8 group)
 {
 	const struct rtw_chip_info *chip = rtwdev->chip;
-	u8 tx_power;
-	bool mcs_rate;
-	bool above_2ss;
+	bool above_2ss, above_3ss, above_4ss;
 	u8 factor = chip->txgi_factor;
+	bool mcs_rate;
+	u8 tx_power;
 
 	if (rate <= DESC_RATE11M)
 		tx_power = pwr_idx_2g->cck_base[group];
@@ -1972,11 +2009,15 @@ static u8 rtw_phy_get_2g_tx_power_index(struct rtw_dev *rtwdev,
 	if (rate >= DESC_RATE6M && rate <= DESC_RATE54M)
 		tx_power += pwr_idx_2g->ht_1s_diff.ofdm * factor;
 
-	mcs_rate = (rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+	mcs_rate = (rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS31) ||
 		   (rate >= DESC_RATEVHT1SS_MCS0 &&
-		    rate <= DESC_RATEVHT2SS_MCS9);
-	above_2ss = (rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+		    rate <= DESC_RATEVHT4SS_MCS9);
+	above_2ss = (rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS31) ||
 		    (rate >= DESC_RATEVHT2SS_MCS0);
+	above_3ss = (rate >= DESC_RATEMCS16 && rate <= DESC_RATEMCS31) ||
+		    (rate >= DESC_RATEVHT3SS_MCS0);
+	above_4ss = (rate >= DESC_RATEMCS24 && rate <= DESC_RATEMCS31) ||
+		    (rate >= DESC_RATEVHT4SS_MCS0);
 
 	if (!mcs_rate)
 		return tx_power;
@@ -1989,11 +2030,19 @@ static u8 rtw_phy_get_2g_tx_power_index(struct rtw_dev *rtwdev,
 		tx_power += pwr_idx_2g->ht_1s_diff.bw20 * factor;
 		if (above_2ss)
 			tx_power += pwr_idx_2g->ht_2s_diff.bw20 * factor;
+		if (above_3ss)
+			tx_power += pwr_idx_2g->ht_3s_diff.bw20 * factor;
+		if (above_4ss)
+			tx_power += pwr_idx_2g->ht_4s_diff.bw20 * factor;
 		break;
 	case RTW_CHANNEL_WIDTH_40:
 		/* bw40 is the base power */
 		if (above_2ss)
 			tx_power += pwr_idx_2g->ht_2s_diff.bw40 * factor;
+		if (above_3ss)
+			tx_power += pwr_idx_2g->ht_3s_diff.bw40 * factor;
+		if (above_4ss)
+			tx_power += pwr_idx_2g->ht_4s_diff.bw40 * factor;
 		break;
 	}
 
@@ -2006,19 +2055,23 @@ static u8 rtw_phy_get_5g_tx_power_index(struct rtw_dev *rtwdev,
 					u8 rate, u8 group)
 {
 	const struct rtw_chip_info *chip = rtwdev->chip;
-	u8 tx_power;
+	bool above_2ss, above_3ss, above_4ss;
+	u8 factor = chip->txgi_factor;
 	u8 upper, lower;
 	bool mcs_rate;
-	bool above_2ss;
-	u8 factor = chip->txgi_factor;
+	u8 tx_power;
 
 	tx_power = pwr_idx_5g->bw40_base[group];
 
-	mcs_rate = (rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+	mcs_rate = (rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS31) ||
 		   (rate >= DESC_RATEVHT1SS_MCS0 &&
-		    rate <= DESC_RATEVHT2SS_MCS9);
-	above_2ss = (rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+		    rate <= DESC_RATEVHT4SS_MCS9);
+	above_2ss = (rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS31) ||
 		    (rate >= DESC_RATEVHT2SS_MCS0);
+	above_3ss = (rate >= DESC_RATEMCS16 && rate <= DESC_RATEMCS31) ||
+		    (rate >= DESC_RATEVHT3SS_MCS0);
+	above_4ss = (rate >= DESC_RATEMCS24 && rate <= DESC_RATEMCS31) ||
+		    (rate >= DESC_RATEVHT4SS_MCS0);
 
 	if (!mcs_rate) {
 		tx_power += pwr_idx_5g->ht_1s_diff.ofdm * factor;
@@ -2033,11 +2086,19 @@ static u8 rtw_phy_get_5g_tx_power_index(struct rtw_dev *rtwdev,
 		tx_power += pwr_idx_5g->ht_1s_diff.bw20 * factor;
 		if (above_2ss)
 			tx_power += pwr_idx_5g->ht_2s_diff.bw20 * factor;
+		if (above_3ss)
+			tx_power += pwr_idx_5g->ht_3s_diff.bw20 * factor;
+		if (above_4ss)
+			tx_power += pwr_idx_5g->ht_4s_diff.bw20 * factor;
 		break;
 	case RTW_CHANNEL_WIDTH_40:
 		/* bw40 is the base power */
 		if (above_2ss)
 			tx_power += pwr_idx_5g->ht_2s_diff.bw40 * factor;
+		if (above_3ss)
+			tx_power += pwr_idx_5g->ht_3s_diff.bw40 * factor;
+		if (above_4ss)
+			tx_power += pwr_idx_5g->ht_4s_diff.bw40 * factor;
 		break;
 	case RTW_CHANNEL_WIDTH_80:
 		/* the base idx of bw80 is the average of bw40+/bw40- */
@@ -2048,13 +2109,17 @@ static u8 rtw_phy_get_5g_tx_power_index(struct rtw_dev *rtwdev,
 		tx_power += pwr_idx_5g->vht_1s_diff.bw80 * factor;
 		if (above_2ss)
 			tx_power += pwr_idx_5g->vht_2s_diff.bw80 * factor;
+		if (above_3ss)
+			tx_power += pwr_idx_5g->vht_3s_diff.bw80 * factor;
+		if (above_4ss)
+			tx_power += pwr_idx_5g->vht_4s_diff.bw80 * factor;
 		break;
 	}
 
 	return tx_power;
 }
 
-/* return RTW_RATE_SECTION_MAX to indicate rate is invalid */
+/* return RTW_RATE_SECTION_NUM to indicate rate is invalid */
 static u8 rtw_phy_rate_to_rate_section(u8 rate)
 {
 	if (rate >= DESC_RATE1M && rate <= DESC_RATE11M)
@@ -2065,12 +2130,20 @@ static u8 rtw_phy_rate_to_rate_section(u8 rate)
 		return RTW_RATE_SECTION_HT_1S;
 	else if (rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15)
 		return RTW_RATE_SECTION_HT_2S;
+	else if (rate >= DESC_RATEMCS16 && rate <= DESC_RATEMCS23)
+		return RTW_RATE_SECTION_HT_3S;
+	else if (rate >= DESC_RATEMCS24 && rate <= DESC_RATEMCS31)
+		return RTW_RATE_SECTION_HT_4S;
 	else if (rate >= DESC_RATEVHT1SS_MCS0 && rate <= DESC_RATEVHT1SS_MCS9)
 		return RTW_RATE_SECTION_VHT_1S;
 	else if (rate >= DESC_RATEVHT2SS_MCS0 && rate <= DESC_RATEVHT2SS_MCS9)
 		return RTW_RATE_SECTION_VHT_2S;
+	else if (rate >= DESC_RATEVHT3SS_MCS0 && rate <= DESC_RATEVHT3SS_MCS9)
+		return RTW_RATE_SECTION_VHT_3S;
+	else if (rate >= DESC_RATEVHT4SS_MCS0 && rate <= DESC_RATEVHT4SS_MCS9)
+		return RTW_RATE_SECTION_VHT_4S;
 	else
-		return RTW_RATE_SECTION_MAX;
+		return RTW_RATE_SECTION_NUM;
 }
 
 static s8 rtw_phy_get_tx_power_limit(struct rtw_dev *rtwdev, u8 band,
@@ -2088,7 +2161,7 @@ static s8 rtw_phy_get_tx_power_limit(struct rtw_dev *rtwdev, u8 band,
 	if (regd > RTW_REGD_WW)
 		return power_limit;
 
-	if (rs == RTW_RATE_SECTION_MAX)
+	if (rs == RTW_RATE_SECTION_NUM)
 		goto err;
 
 	/* only 20M BW with cck and ofdm */
@@ -2096,7 +2169,7 @@ static s8 rtw_phy_get_tx_power_limit(struct rtw_dev *rtwdev, u8 band,
 		bw = RTW_CHANNEL_WIDTH_20;
 
 	/* only 20/40M BW with ht */
-	if (rs == RTW_RATE_SECTION_HT_1S || rs == RTW_RATE_SECTION_HT_2S)
+	if (rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS31)
 		bw = min_t(u8, bw, RTW_CHANNEL_WIDTH_40);
 
 	/* select min power limit among [20M BW ~ current BW] */
@@ -2132,7 +2205,7 @@ static s8 rtw_phy_get_tx_power_sar(struct rtw_dev *rtwdev, u8 sar_band,
 		.rs = rs,
 	};
 
-	if (rs == RTW_RATE_SECTION_MAX)
+	if (rs == RTW_RATE_SECTION_NUM)
 		goto err;
 
 	return rtw_query_sar(rtwdev, &arg);
@@ -2214,14 +2287,14 @@ static void rtw_phy_set_tx_power_index_by_rs(struct rtw_dev *rtwdev,
 {
 	struct rtw_hal *hal = &rtwdev->hal;
 	u8 regd = rtw_regd_get(rtwdev);
-	u8 *rates;
+	const u8 *rates;
 	u8 size;
 	u8 rate;
 	u8 pwr_idx;
 	u8 bw;
 	int i;
 
-	if (rs >= RTW_RATE_SECTION_MAX)
+	if (rs >= RTW_RATE_SECTION_NUM)
 		return;
 
 	rates = rtw_rate_section[rs];
@@ -2252,7 +2325,7 @@ static void rtw_phy_set_tx_power_level_by_path(struct rtw_dev *rtwdev,
 	else
 		rs = RTW_RATE_SECTION_OFDM;
 
-	for (; rs < RTW_RATE_SECTION_MAX; rs++)
+	for (; rs < RTW_RATE_SECTION_NUM; rs++)
 		rtw_phy_set_tx_power_index_by_rs(rtwdev, ch, path, rs);
 }
 
@@ -2274,13 +2347,13 @@ EXPORT_SYMBOL(rtw_phy_set_tx_power_level);
 
 static void
 rtw_phy_tx_power_by_rate_config_by_path(struct rtw_hal *hal, u8 path,
-					u8 rs, u8 size, u8 *rates)
+					u8 rs, u8 size, const u8 *rates)
 {
 	u8 rate;
 	u8 base_idx, rate_idx;
 	s8 base_2g, base_5g;
 
-	if (rs >= RTW_RATE_SECTION_VHT_1S)
+	if (size == 10) /* VHT rates */
 		base_idx = rates[size - 3];
 	else
 		base_idx = rates[size - 1];
@@ -2297,28 +2370,12 @@ rtw_phy_tx_power_by_rate_config_by_path(struct rtw_hal *hal, u8 path,
 
 void rtw_phy_tx_power_by_rate_config(struct rtw_hal *hal)
 {
-	u8 path;
+	u8 path, rs;
 
-	for (path = 0; path < RTW_RF_PATH_MAX; path++) {
-		rtw_phy_tx_power_by_rate_config_by_path(hal, path,
-				RTW_RATE_SECTION_CCK,
-				rtw_cck_size, rtw_cck_rates);
-		rtw_phy_tx_power_by_rate_config_by_path(hal, path,
-				RTW_RATE_SECTION_OFDM,
-				rtw_ofdm_size, rtw_ofdm_rates);
-		rtw_phy_tx_power_by_rate_config_by_path(hal, path,
-				RTW_RATE_SECTION_HT_1S,
-				rtw_ht_1s_size, rtw_ht_1s_rates);
-		rtw_phy_tx_power_by_rate_config_by_path(hal, path,
-				RTW_RATE_SECTION_HT_2S,
-				rtw_ht_2s_size, rtw_ht_2s_rates);
-		rtw_phy_tx_power_by_rate_config_by_path(hal, path,
-				RTW_RATE_SECTION_VHT_1S,
-				rtw_vht_1s_size, rtw_vht_1s_rates);
-		rtw_phy_tx_power_by_rate_config_by_path(hal, path,
-				RTW_RATE_SECTION_VHT_2S,
-				rtw_vht_2s_size, rtw_vht_2s_rates);
-	}
+	for (path = 0; path < RTW_RF_PATH_MAX; path++)
+		for (rs = 0; rs < RTW_RATE_SECTION_NUM; rs++)
+			rtw_phy_tx_power_by_rate_config_by_path(hal, path, rs,
+				rtw_rate_size[rs], rtw_rate_section[rs]);
 }
 
 static void
@@ -2347,7 +2404,7 @@ void rtw_phy_tx_power_limit_config(struct rtw_hal *hal)
 
 	for (regd = 0; regd < RTW_REGD_MAX; regd++)
 		for (bw = 0; bw < RTW_CHANNEL_WIDTH_MAX; bw++)
-			for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++)
+			for (rs = 0; rs < RTW_RATE_SECTION_NUM; rs++)
 				__rtw_phy_tx_power_limit_config(hal, regd, bw, rs);
 }
 
@@ -2383,7 +2440,7 @@ void rtw_phy_init_tx_power(struct rtw_dev *rtwdev)
 	/* init tx power limit */
 	for (regd = 0; regd < RTW_REGD_MAX; regd++)
 		for (bw = 0; bw < RTW_CHANNEL_WIDTH_MAX; bw++)
-			for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++)
+			for (rs = 0; rs < RTW_RATE_SECTION_NUM; rs++)
 				rtw_phy_init_tx_power_limit(rtwdev, regd, bw,
 							    rs);
 }
@@ -2401,32 +2458,56 @@ void rtw_phy_config_swing_table(struct rtw_dev *rtwdev,
 			swing_table->n[RF_PATH_A] = tbl->pwrtrk_2g_ccka_n;
 			swing_table->p[RF_PATH_B] = tbl->pwrtrk_2g_cckb_p;
 			swing_table->n[RF_PATH_B] = tbl->pwrtrk_2g_cckb_n;
+			swing_table->p[RF_PATH_C] = tbl->pwrtrk_2g_cckc_p;
+			swing_table->n[RF_PATH_C] = tbl->pwrtrk_2g_cckc_n;
+			swing_table->p[RF_PATH_D] = tbl->pwrtrk_2g_cckd_p;
+			swing_table->n[RF_PATH_D] = tbl->pwrtrk_2g_cckd_n;
 		} else {
 			swing_table->p[RF_PATH_A] = tbl->pwrtrk_2ga_p;
 			swing_table->n[RF_PATH_A] = tbl->pwrtrk_2ga_n;
 			swing_table->p[RF_PATH_B] = tbl->pwrtrk_2gb_p;
 			swing_table->n[RF_PATH_B] = tbl->pwrtrk_2gb_n;
+			swing_table->p[RF_PATH_C] = tbl->pwrtrk_2gc_p;
+			swing_table->n[RF_PATH_C] = tbl->pwrtrk_2gc_n;
+			swing_table->p[RF_PATH_D] = tbl->pwrtrk_2gd_p;
+			swing_table->n[RF_PATH_D] = tbl->pwrtrk_2gd_n;
 		}
 	} else if (IS_CH_5G_BAND_1(channel) || IS_CH_5G_BAND_2(channel)) {
 		swing_table->p[RF_PATH_A] = tbl->pwrtrk_5ga_p[RTW_PWR_TRK_5G_1];
 		swing_table->n[RF_PATH_A] = tbl->pwrtrk_5ga_n[RTW_PWR_TRK_5G_1];
 		swing_table->p[RF_PATH_B] = tbl->pwrtrk_5gb_p[RTW_PWR_TRK_5G_1];
 		swing_table->n[RF_PATH_B] = tbl->pwrtrk_5gb_n[RTW_PWR_TRK_5G_1];
+		swing_table->p[RF_PATH_C] = tbl->pwrtrk_5gc_p[RTW_PWR_TRK_5G_1];
+		swing_table->n[RF_PATH_C] = tbl->pwrtrk_5gc_n[RTW_PWR_TRK_5G_1];
+		swing_table->p[RF_PATH_D] = tbl->pwrtrk_5gd_p[RTW_PWR_TRK_5G_1];
+		swing_table->n[RF_PATH_D] = tbl->pwrtrk_5gd_n[RTW_PWR_TRK_5G_1];
 	} else if (IS_CH_5G_BAND_3(channel)) {
 		swing_table->p[RF_PATH_A] = tbl->pwrtrk_5ga_p[RTW_PWR_TRK_5G_2];
 		swing_table->n[RF_PATH_A] = tbl->pwrtrk_5ga_n[RTW_PWR_TRK_5G_2];
 		swing_table->p[RF_PATH_B] = tbl->pwrtrk_5gb_p[RTW_PWR_TRK_5G_2];
 		swing_table->n[RF_PATH_B] = tbl->pwrtrk_5gb_n[RTW_PWR_TRK_5G_2];
+		swing_table->p[RF_PATH_C] = tbl->pwrtrk_5gc_p[RTW_PWR_TRK_5G_2];
+		swing_table->n[RF_PATH_C] = tbl->pwrtrk_5gc_n[RTW_PWR_TRK_5G_2];
+		swing_table->p[RF_PATH_D] = tbl->pwrtrk_5gd_p[RTW_PWR_TRK_5G_2];
+		swing_table->n[RF_PATH_D] = tbl->pwrtrk_5gd_n[RTW_PWR_TRK_5G_2];
 	} else if (IS_CH_5G_BAND_4(channel)) {
 		swing_table->p[RF_PATH_A] = tbl->pwrtrk_5ga_p[RTW_PWR_TRK_5G_3];
 		swing_table->n[RF_PATH_A] = tbl->pwrtrk_5ga_n[RTW_PWR_TRK_5G_3];
 		swing_table->p[RF_PATH_B] = tbl->pwrtrk_5gb_p[RTW_PWR_TRK_5G_3];
 		swing_table->n[RF_PATH_B] = tbl->pwrtrk_5gb_n[RTW_PWR_TRK_5G_3];
+		swing_table->p[RF_PATH_C] = tbl->pwrtrk_5gc_p[RTW_PWR_TRK_5G_3];
+		swing_table->n[RF_PATH_C] = tbl->pwrtrk_5gc_n[RTW_PWR_TRK_5G_3];
+		swing_table->p[RF_PATH_D] = tbl->pwrtrk_5gd_p[RTW_PWR_TRK_5G_3];
+		swing_table->n[RF_PATH_D] = tbl->pwrtrk_5gd_n[RTW_PWR_TRK_5G_3];
 	} else {
 		swing_table->p[RF_PATH_A] = tbl->pwrtrk_2ga_p;
 		swing_table->n[RF_PATH_A] = tbl->pwrtrk_2ga_n;
 		swing_table->p[RF_PATH_B] = tbl->pwrtrk_2gb_p;
 		swing_table->n[RF_PATH_B] = tbl->pwrtrk_2gb_n;
+		swing_table->p[RF_PATH_C] = tbl->pwrtrk_2gc_p;
+		swing_table->n[RF_PATH_C] = tbl->pwrtrk_2gc_n;
+		swing_table->p[RF_PATH_D] = tbl->pwrtrk_2gd_p;
+		swing_table->n[RF_PATH_D] = tbl->pwrtrk_2gd_n;
 	}
 }
 EXPORT_SYMBOL(rtw_phy_config_swing_table);
diff --git a/sys/contrib/dev/rtw88/phy.h b/sys/contrib/dev/rtw88/phy.h
index ccfcbd3ced03..c9e6b869661d 100644
--- a/sys/contrib/dev/rtw88/phy.h
+++ b/sys/contrib/dev/rtw88/phy.h
@@ -7,14 +7,18 @@
 
 #include "debug.h"
 
-extern u8 rtw_cck_rates[];
-extern u8 rtw_ofdm_rates[];
-extern u8 rtw_ht_1s_rates[];
-extern u8 rtw_ht_2s_rates[];
-extern u8 rtw_vht_1s_rates[];
-extern u8 rtw_vht_2s_rates[];
-extern u8 *rtw_rate_section[];
-extern u8 rtw_rate_size[];
+extern const u8 rtw_cck_rates[];
+extern const u8 rtw_ofdm_rates[];
+extern const u8 rtw_ht_1s_rates[];
+extern const u8 rtw_ht_2s_rates[];
+extern const u8 rtw_vht_1s_rates[];
+extern const u8 rtw_vht_2s_rates[];
+extern const u8 rtw_ht_3s_rates[];
+extern const u8 rtw_ht_4s_rates[];
+extern const u8 rtw_vht_3s_rates[];
+extern const u8 rtw_vht_4s_rates[];
+extern const u8 * const rtw_rate_section[];
+extern const u8 rtw_rate_size[];
 
 void rtw_phy_init(struct rtw_dev *rtwdev);
 void rtw_phy_dynamic_mechanism(struct rtw_dev *rtwdev);
diff --git a/sys/contrib/dev/rtw88/reg.h b/sys/contrib/dev/rtw88/reg.h
index e438405fba56..08e9494977e0 100644
--- a/sys/contrib/dev/rtw88/reg.h
+++ b/sys/contrib/dev/rtw88/reg.h
@@ -8,6 +8,7 @@
 #define REG_SYS_FUNC_EN		0x0002
 #define BIT_FEN_EN_25_1		BIT(13)
 #define BIT_FEN_ELDR		BIT(12)
+#define BIT_FEN_PCIEA		BIT(6)
 #define BIT_FEN_CPUEN		BIT(2)
 #define BIT_FEN_USBA		BIT(2)
 #define BIT_FEN_BB_GLB_RST	BIT(1)
@@ -39,6 +40,9 @@
 #define BIT_RF_RSTB		BIT(1)
 #define BIT_RF_EN		BIT(0)
 
+#define REG_RF_CTRL1		0x0020
+#define REG_RF_CTRL2		0x0021
+
 #define REG_AFE_CTRL1		0x0024
 #define BIT_MAC_CLK_SEL		(BIT(20) | BIT(21))
 #define REG_EFUSE_CTRL		0x0030
@@ -73,6 +77,8 @@
 #define BIT_BT_PTA_EN		BIT(5)
 #define BIT_WLRFE_4_5_EN	BIT(2)
 
+#define REG_GPIO_PIN_CTRL	0x0044
+
 #define REG_LED_CFG		0x004C
 #define BIT_LNAON_SEL_EN	BIT(26)
 #define BIT_PAPE_SEL_EN		BIT(25)
@@ -110,6 +116,7 @@
 #define BIT_SDIO_PAD_E5		BIT(18)
 
 #define REG_RF_B_CTRL		0x76
+#define REG_RF_CTRL3		0x0076
 
 #define REG_AFE_CTRL_4		0x0078
 #define BIT_CK320M_AFE_EN	BIT(4)
@@ -130,6 +137,7 @@
 #define BIT_SHIFT_ROM_PGE	16
 #define BIT_FW_INIT_RDY		BIT(15)
 #define BIT_FW_DW_RDY		BIT(14)
+#define BIT_CPU_CLK_SEL		(BIT(12) | BIT(13))
 #define BIT_RPWM_TOGGLE		BIT(7)
 #define BIT_RAM_DL_SEL		BIT(7)	/* legacy only */
 #define BIT_DMEM_CHKSUM_OK	BIT(6)
@@ -147,7 +155,7 @@
 				 BIT_CHECK_SUM_OK)
 #define FW_READY_LEGACY		(BIT_MCUFWDL_RDY | BIT_FWDL_CHK_RPT |	       \
 				 BIT_WINTINI_RDY | BIT_RAM_DL_SEL)
-#define FW_READY_MASK		0xffff
+#define FW_READY_MASK		(0xffff & ~BIT_CPU_CLK_SEL)
 
 #define REG_MCU_TST_CFG		0x84
 #define VAL_FW_TRIGGER		0x1
@@ -602,15 +610,25 @@
 #define REG_CCA2ND		0x0838
 #define REG_L1PKTH		0x0848
 #define REG_CLKTRK		0x0860
+#define REG_CSI_MASK_SETTING1	0x0874
+#define REG_NBI_SETTING		0x087c
+#define BIT_NBI_ENABLE		BIT(13)
+#define REG_CSI_FIX_MASK0	0x0880
+#define REG_CSI_FIX_MASK1	0x0884
+#define REG_CSI_FIX_MASK6	0x0898
+#define REG_CSI_FIX_MASK7	0x089c
 #define REG_ADCCLK		0x08AC
 #define REG_HSSI_READ		0x08B0
 #define REG_FPGA0_XCD_RF_PARA	0x08B4
 #define REG_RX_MCS_LIMIT	0x08BC
 #define REG_ADC160		0x08C4
+#define REG_DBGSEL		0x08fc
 #define REG_ANTSEL_SW		0x0900
 #define REG_DAC_RSTB		0x090c
+#define REG_PSD			0x0910
+#define BIT_PSD_INI		GENMASK(23, 22)
 #define REG_SINGLE_TONE_CONT_TX	0x0914
-
+#define REG_AGC_TABLE		0x0958
 #define REG_RFE_CTRL_E		0x0974
 #define REG_2ND_CCA_CTRL	0x0976
 #define REG_IQK_COM00		0x0978
@@ -620,10 +638,18 @@
 
 #define REG_FAS			0x09a4
 #define REG_RXSB		0x0a00
+#define BIT_RXSB_ANA_DIV	BIT(15)
 #define REG_CCK_RX		0x0a04
 #define REG_CCK_PD_TH		0x0a0a
-
-#define REG_CCK0_FAREPORT	0xa2c
+#define REG_PRECTRL		0x0a14
+#define BIT_DIS_CO_PATHSEL	BIT(7)
+#define BIT_IQ_WGT		GENMASK(9, 8)
+#define REG_CCA_MF		0x0a20
+#define BIT_MBC_WIN		GENMASK(5, 4)
+#define REG_CCK0_TX_FILTER1	0x0a20
+#define REG_CCK0_TX_FILTER2	0x0a24
+#define REG_CCK0_DEBUG_PORT	0x0a28
+#define REG_CCK0_FAREPORT	0x0a2c
 #define BIT_CCK0_2RX		BIT(18)
 #define BIT_CCK0_MRC		BIT(22)
 #define REG_FA_CCK		0x0a5c
@@ -642,10 +668,18 @@
 #define DIS_DPD_RATEVHT2SS_MCS1	BIT(9)
 #define DIS_DPD_RATEALL		GENMASK(9, 0)
 
+#define REG_CCA			0x0a70
+#define BIT_CCA_CO		BIT(7)
+#define REG_ANTSEL		0x0a74
+#define BIT_ANT_BYCO		BIT(8)
+#define REG_CCKTX		0x0a84
+#define BIT_CMB_CCA_2R		BIT(28)
+
 #define REG_CNTRST		0x0b58
 
 #define REG_3WIRE_SWA		0x0c00
 #define REG_RX_IQC_AB_A		0x0c10
+#define REG_RX_IQC_CD_A		0x0c14
 #define REG_TXSCALE_A		0x0c1c
 #define BB_SWING_MASK		GENMASK(31, 21)
 #define REG_TX_AGC_A_CCK_11_CCK_1		0xc20
@@ -673,7 +707,7 @@
 #define REG_LSSI_WRITE_A	0x0c90
 #define REG_PREDISTA		0x0c90
 #define REG_TXAGCIDX		0x0c94
-
+#define REG_TX_AGC_A		0x0c94
 #define REG_RFE_PINMUX_A	0x0cb0
 #define REG_RFE_INV_A		0x0cb4
 #define REG_RFE_CTRL8		0x0cb4
@@ -682,6 +716,7 @@
 #define DPDT_CTRL_PIN		0x77
 #define RFE_INV_MASK		0x3ff00000
 #define REG_RFECTL_A		0x0cb8
+#define REG_RFE_INV0		0x0cbc
 #define REG_RFE_INV8		0x0cbd
 #define BIT_MASK_RFE_INV89	GENMASK(1, 0)
 #define REG_RFE_INV16		0x0cbe
@@ -702,6 +737,7 @@
 
 #define REG_3WIRE_SWB		0x0e00
 #define REG_RX_IQC_AB_B		0x0e10
+#define REG_RX_IQC_CD_B		0x0e14
 #define REG_TXSCALE_B		0x0e1c
 #define REG_TX_AGC_B_CCK_11_CCK_1		0xe20
 #define REG_TX_AGC_B_OFDM18_OFDM6		0xe24
@@ -728,6 +764,7 @@
 #define REG_LSSI_WRITE_B	0x0e90
 #define REG_PREDISTB		0x0e90
 #define REG_INIDLYB		0x0e94
+#define REG_TX_AGC_B		0x0e94
 #define REG_RFE_PINMUX_B	0x0eb0
 #define REG_RFE_INV_B		0x0eb4
 #define REG_RFECTL_B		0x0eb8
@@ -743,8 +780,11 @@
 #define REG_CRC_HT		0x0f10
 #define REG_CRC_OFDM		0x0f14
 #define REG_FA_OFDM		0x0f48
+#define REG_DBGRPT		0x0fa0
 #define REG_CCA_CCK		0x0fcc
 
+#define REG_SYS_CFG3_8814A	0x1000
+
 #define REG_ANAPARSW_MAC_0	0x1010
 #define BIT_CF_L_V2		GENMASK(29, 28)
 
@@ -862,9 +902,27 @@
 #define LTECOEX_WRITE_DATA REG_WL2LTECOEX_INDIRECT_ACCESS_WRITE_DATA_V1
 #define LTECOEX_READ_DATA REG_WL2LTECOEX_INDIRECT_ACCESS_READ_DATA_V1
 
+#define REG_RX_IQC_AB_C		0x1810
+#define REG_RX_IQC_CD_C		0x1814
+#define REG_TXSCALE_C		0x181c
+#define REG_CK_MONHC		0x185c
+#define REG_AFE_PWR1_C		0x1860
 #define REG_IGN_GNT_BT1	0x1860
+#define REG_TX_AGC_C		0x1894
+#define REG_RFE_PINMUX_C	0x18b4
 
 #define REG_RFESEL_CTRL	0x1990
+#define REG_AGC_TBL		0x1998
+
+#define REG_RX_IQC_AB_D		0x1a10
+#define REG_RX_IQC_CD_D		0x1a14
+#define REG_TXSCALE_D		0x1a1c
+#define REG_CK_MONHD		0x1a5c
+#define REG_AFE_PWR1_D		0x1a60
+#define REG_TX_AGC_D		0x1a94
+#define REG_RFE_PINMUX_D	0x1ab4
+#define REG_RFE_INVSEL_D	0x1abc
+#define BIT_RFE_SELSW0_D	GENMASK(27, 20)
 
 #define REG_NOMASK_TXBT	0x1ca7
 #define REG_ANAPAR	0x1c30
@@ -905,6 +963,7 @@
 #define RF18_BAND_MASK	(BIT(16) | BIT(9) | BIT(8))
 #define RF18_CHANNEL_MASK	(MASKBYTE0)
 #define RF18_RFSI_MASK	(BIT(18) | BIT(17))
+#define RF_RCK1_V1	0x1c
 #define RF_RCK		0x1d
 #define RF_MODE_TABLE_ADDR	0x30
 #define RF_MODE_TABLE_DATA0	0x31
diff --git a/sys/contrib/dev/rtw88/rtw8703b.c b/sys/contrib/dev/rtw88/rtw8703b.c
index 4cf3a08b785e..18ca16f521f2 100644
--- a/sys/contrib/dev/rtw88/rtw8703b.c
+++ b/sys/contrib/dev/rtw88/rtw8703b.c
@@ -519,15 +519,6 @@ static const struct rtw_rqpn rqpn_table_8703b[] = {
 	 RTW_DMA_MAPPING_EXTRA, RTW_DMA_MAPPING_HIGH},
 };
 
-/* Default power index table for RTL8703B, used if EFUSE does not
- * contain valid data. Replaces EFUSE data from offset 0x10 (start of
- * txpwr_idx_table).
- */
-static const u8 rtw8703b_txpwr_idx_table[] = {
-	0x2D, 0x2D, 0x2D, 0x2D, 0x2D, 0x2D,
-	0x2D, 0x2D, 0x2D, 0x2D, 0x2D, 0x02
-};
-
 static void try_mac_from_devicetree(struct rtw_dev *rtwdev)
 {
 #if defined(CONFIG_OF)
@@ -546,15 +537,9 @@ static void try_mac_from_devicetree(struct rtw_dev *rtwdev)
 #endif
 }
 
-#define DBG_EFUSE_FIX(rtwdev, name)					\
-	rtw_dbg(rtwdev, RTW_DBG_EFUSE, "Fixed invalid EFUSE value: "	\
-		# name "=0x%x\n", rtwdev->efuse.name)
-
 static int rtw8703b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 {
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	u8 *pwr = (u8 *)efuse->txpwr_idx_table;
-	bool valid = false;
 	int ret;
 
 	ret = rtw8723x_read_efuse(rtwdev, log_map);
@@ -564,51 +549,6 @@ static int rtw8703b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 	if (!is_valid_ether_addr(efuse->addr))
 		try_mac_from_devicetree(rtwdev);
 
-	/* If TX power index table in EFUSE is invalid, fall back to
-	 * built-in table.
-	 */
-	for (int i = 0; i < ARRAY_SIZE(rtw8703b_txpwr_idx_table); i++)
-		if (pwr[i] != 0xff) {
-			valid = true;
-			break;
-		}
-	if (!valid) {
-		for (int i = 0; i < ARRAY_SIZE(rtw8703b_txpwr_idx_table); i++)
-			pwr[i] = rtw8703b_txpwr_idx_table[i];
-		rtw_dbg(rtwdev, RTW_DBG_EFUSE,
-			"Replaced invalid EFUSE TX power index table.");
-		rtw8723x_debug_txpwr_limit(rtwdev,
-					   efuse->txpwr_idx_table, 2);
-	}
-
-	/* Override invalid antenna settings. */
-	if (efuse->bt_setting == 0xff) {
-		/* shared antenna */
-		efuse->bt_setting |= BIT(0);
-		/* RF path A */
-		efuse->bt_setting &= ~BIT(6);
-		DBG_EFUSE_FIX(rtwdev, bt_setting);
-	}
-
-	/* Override invalid board options: The coex code incorrectly
-	 * assumes that if bits 6 & 7 are set the board doesn't
-	 * support coex. Regd is also derived from rf_board_option and
-	 * should be 0 if there's no valid data.
-	 */
-	if (efuse->rf_board_option == 0xff) {
-		efuse->regd = 0;
-		efuse->rf_board_option &= GENMASK(5, 0);
-		DBG_EFUSE_FIX(rtwdev, rf_board_option);
-	}
-
-	/* Override invalid crystal cap setting, default comes from
-	 * vendor driver. Chip specific.
-	 */
-	if (efuse->crystal_cap == 0xff) {
-		efuse->crystal_cap = 0x20;
-		DBG_EFUSE_FIX(rtwdev, crystal_cap);
-	}
-
 	return 0;
 }
 
@@ -1904,6 +1844,7 @@ static const struct rtw_chip_ops rtw8703b_ops = {
 	.power_on		= rtw_power_on,
 	.power_off		= rtw_power_off,
 	.mac_init		= rtw8723x_mac_init,
+	.mac_postinit		= rtw8723x_mac_postinit,
 	.dump_fw_crash		= NULL,
 	.shutdown		= NULL,
 	.read_efuse		= rtw8703b_read_efuse,
@@ -1916,6 +1857,7 @@ static const struct rtw_chip_ops rtw8703b_ops = {
 	.set_antenna		= NULL,
 	.cfg_ldo25		= rtw8723x_cfg_ldo25,
 	.efuse_grant		= rtw8723x_efuse_grant,
+	.set_ampdu_factor	= NULL,
 	.false_alarm_statistics	= rtw8723x_false_alarm_statistics,
 	.phy_calibration	= rtw8703b_phy_calibration,
 	.dpk_track		= NULL,
@@ -1953,7 +1895,7 @@ const struct rtw_chip_info rtw8703b_hw_spec = {
 	.id = RTW_CHIP_TYPE_8703B,
 
 	.fw_name = "rtw88/rtw8703b_fw.bin",
-	.wlan_cpu = RTW_WCPU_11N,
+	.wlan_cpu = RTW_WCPU_8051,
 	.tx_pkt_desc_sz = 40,
 	.tx_buf_desc_sz = 16,
 	.rx_pkt_desc_sz = 24,
diff --git a/sys/contrib/dev/rtw88/rtw8723cs.c b/sys/contrib/dev/rtw88/rtw8723cs.c
index 8d38d36be8c0..1f98d35a8dd1 100644
--- a/sys/contrib/dev/rtw88/rtw8723cs.c
+++ b/sys/contrib/dev/rtw88/rtw8723cs.c
@@ -19,7 +19,7 @@ static const struct sdio_device_id rtw_8723cs_id_table[] = {
 MODULE_DEVICE_TABLE(sdio, rtw_8723cs_id_table);
 
 static struct sdio_driver rtw_8723cs_driver = {
-	.name = "rtw8723cs",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8723cs_id_table,
 	.probe = rtw_sdio_probe,
 	.remove = rtw_sdio_remove,
diff --git a/sys/contrib/dev/rtw88/rtw8723d.c b/sys/contrib/dev/rtw88/rtw8723d.c
index eeca31bf71f1..8715e0435f17 100644
--- a/sys/contrib/dev/rtw88/rtw8723d.c
+++ b/sys/contrib/dev/rtw88/rtw8723d.c
@@ -444,7 +444,7 @@ static u8 rtw8723d_iqk_check_tx_failed(struct rtw_dev *rtwdev,
 		rtw_read32(rtwdev, REG_IQK_RES_TX),
 		rtw_read32(rtwdev, REG_IQK_RES_TY));
 	rtw_dbg(rtwdev, RTW_DBG_RFK,
-		"[IQK] 0xe90(before IQK)= 0x%x, 0xe98(afer IQK) = 0x%x\n",
+		"[IQK] 0xe90(before IQK)= 0x%x, 0xe98(after IQK) = 0x%x\n",
 		rtw_read32(rtwdev, 0xe90),
 		rtw_read32(rtwdev, 0xe98));
 
@@ -472,7 +472,7 @@ static u8 rtw8723d_iqk_check_rx_failed(struct rtw_dev *rtwdev,
 		rtw_read32(rtwdev, REG_IQK_RES_RY));
 
 	rtw_dbg(rtwdev, RTW_DBG_RFK,
-		"[IQK] 0xea0(before IQK)= 0x%x, 0xea8(afer IQK) = 0x%x\n",
+		"[IQK] 0xea0(before IQK)= 0x%x, 0xea8(after IQK) = 0x%x\n",
 		rtw_read32(rtwdev, 0xea0),
 		rtw_read32(rtwdev, 0xea8));
 
@@ -1397,6 +1397,7 @@ static const struct rtw_chip_ops rtw8723d_ops = {
 	.query_phy_status	= query_phy_status,
 	.set_channel		= rtw8723d_set_channel,
 	.mac_init		= rtw8723x_mac_init,
+	.mac_postinit		= rtw8723x_mac_postinit,
 	.shutdown		= rtw8723d_shutdown,
 	.read_rf		= rtw_phy_read_rf_sipi,
 	.write_rf		= rtw_phy_write_rf_reg_sipi,
@@ -1404,6 +1405,7 @@ static const struct rtw_chip_ops rtw8723d_ops = {
 	.set_antenna		= NULL,
 	.cfg_ldo25		= rtw8723x_cfg_ldo25,
 	.efuse_grant		= rtw8723x_efuse_grant,
+	.set_ampdu_factor	= NULL,
 	.false_alarm_statistics	= rtw8723x_false_alarm_statistics,
 	.phy_calibration	= rtw8723d_phy_calibration,
 	.cck_pd_set		= rtw8723d_phy_cck_pd_set,
@@ -2115,7 +2117,7 @@ const struct rtw_chip_info rtw8723d_hw_spec = {
 	.ops = &rtw8723d_ops,
 	.id = RTW_CHIP_TYPE_8723D,
 	.fw_name = "rtw88/rtw8723d_fw.bin",
-	.wlan_cpu = RTW_WCPU_11N,
+	.wlan_cpu = RTW_WCPU_8051,
 	.tx_pkt_desc_sz = 40,
 	.tx_buf_desc_sz = 16,
 	.rx_pkt_desc_sz = 24,
diff --git a/sys/contrib/dev/rtw88/rtw8723de.c b/sys/contrib/dev/rtw88/rtw8723de.c
index 75cceb0172ce..f92a460a49c2 100644
--- a/sys/contrib/dev/rtw88/rtw8723de.c
+++ b/sys/contrib/dev/rtw88/rtw8723de.c
@@ -17,12 +17,13 @@ static const struct pci_device_id rtw_8723de_id_table[] = {
 MODULE_DEVICE_TABLE(pci, rtw_8723de_id_table);
 
 static struct pci_driver rtw_8723de_driver = {
-	.name = "rtw_8723de",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8723de_id_table,
 	.probe = rtw_pci_probe,
 	.remove = rtw_pci_remove,
 	.driver.pm = &rtw_pm_ops,
 	.shutdown = rtw_pci_shutdown,
+	.err_handler  = &rtw_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name = KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw88/rtw8723ds.c b/sys/contrib/dev/rtw88/rtw8723ds.c
index e5b6960ba0a0..206b77e5b98e 100644
--- a/sys/contrib/dev/rtw88/rtw8723ds.c
+++ b/sys/contrib/dev/rtw88/rtw8723ds.c
@@ -25,7 +25,7 @@ static const struct sdio_device_id rtw_8723ds_id_table[] =  {
 MODULE_DEVICE_TABLE(sdio, rtw_8723ds_id_table);
 
 static struct sdio_driver rtw_8723ds_driver = {
-	.name = "rtw_8723ds",
+	.name = KBUILD_MODNAME,
 	.probe = rtw_sdio_probe,
 	.remove = rtw_sdio_remove,
 	.id_table = rtw_8723ds_id_table,
diff --git a/sys/contrib/dev/rtw88/rtw8723du.c b/sys/contrib/dev/rtw88/rtw8723du.c
index eebe8026c6c3..064cbb4d9760 100644
--- a/sys/contrib/dev/rtw88/rtw8723du.c
+++ b/sys/contrib/dev/rtw88/rtw8723du.c
@@ -24,7 +24,7 @@ static int rtw8723du_probe(struct usb_interface *intf,
 }
 
 static struct usb_driver rtw_8723du_driver = {
-	.name = "rtw_8723du",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8723du_id_table,
 	.probe = rtw8723du_probe,
 	.disconnect = rtw_usb_disconnect,
diff --git a/sys/contrib/dev/rtw88/rtw8723x.c b/sys/contrib/dev/rtw88/rtw8723x.c
index eb56f469742d..0ccdc306209f 100644
--- a/sys/contrib/dev/rtw88/rtw8723x.c
+++ b/sys/contrib/dev/rtw88/rtw8723x.c
@@ -72,6 +72,9 @@ static void __rtw8723x_lck(struct rtw_dev *rtwdev)
 #define DBG_EFUSE_2BYTE(rtwdev, map, name)			\
 	rtw_dbg(rtwdev, RTW_DBG_EFUSE, # name "=0x%02x%02x\n",	\
 		(map)->name[0], (map)->name[1])
+#define DBG_EFUSE_FIX(rtwdev, name)					\
+	rtw_dbg(rtwdev, RTW_DBG_EFUSE, "Fixed invalid EFUSE value: "	\
+		# name "=0x%x\n", rtwdev->efuse.name)
 
 static void rtw8723xe_efuse_debug(struct rtw_dev *rtwdev,
 				  struct rtw8723x_efuse *map)
@@ -241,10 +244,21 @@ static void rtw8723xs_efuse_parsing(struct rtw_efuse *efuse,
 	ether_addr_copy(efuse->addr, map->s.mac_addr);
 }
 
+/* Default power index table for RTL8703B/RTL8723D, used if EFUSE does
+ * not contain valid data. Replaces EFUSE data from offset 0x10 (start
+ * of txpwr_idx_table).
+ */
+static const u8 rtw8723x_txpwr_idx_table[] = {
+	0x2D, 0x2D, 0x2D, 0x2D, 0x2D, 0x2D,
+	0x2D, 0x2D, 0x2D, 0x2D, 0x2D, 0x02
+};
+
 static int __rtw8723x_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 {
 	struct rtw_efuse *efuse = &rtwdev->efuse;
+	u8 *pwr = (u8 *)efuse->txpwr_idx_table;
 	struct rtw8723x_efuse *map;
+	bool valid = false;
 	int i;
 
 	map = (struct rtw8723x_efuse *)log_map;
@@ -282,6 +296,51 @@ static int __rtw8723x_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 		return -EOPNOTSUPP;
 	}
 
+	/* If TX power index table in EFUSE is invalid, fall back to
+	 * built-in table.
+	 */
+	for (i = 0; i < ARRAY_SIZE(rtw8723x_txpwr_idx_table); i++)
+		if (pwr[i] != 0xff) {
+			valid = true;
+			break;
+		}
+	if (!valid) {
+		for (i = 0; i < ARRAY_SIZE(rtw8723x_txpwr_idx_table); i++)
+			pwr[i] = rtw8723x_txpwr_idx_table[i];
+		rtw_dbg(rtwdev, RTW_DBG_EFUSE,
+			"Replaced invalid EFUSE TX power index table.");
+		rtw8723x_debug_txpwr_limit(rtwdev,
+					   efuse->txpwr_idx_table, 2);
+	}
+
+	/* Override invalid antenna settings. */
+	if (efuse->bt_setting == 0xff) {
+		/* shared antenna */
+		efuse->bt_setting |= BIT(0);
+		/* RF path A */
+		efuse->bt_setting &= ~BIT(6);
+		DBG_EFUSE_FIX(rtwdev, bt_setting);
+	}
+
+	/* Override invalid board options: The coex code incorrectly
+	 * assumes that if bits 6 & 7 are set the board doesn't
+	 * support coex. Regd is also derived from rf_board_option and
+	 * should be 0 if there's no valid data.
+	 */
+	if (efuse->rf_board_option == 0xff) {
+		efuse->regd = 0;
+		efuse->rf_board_option &= GENMASK(5, 0);
+		DBG_EFUSE_FIX(rtwdev, rf_board_option);
+	}
+
+	/* Override invalid crystal cap setting, default comes from
+	 * vendor driver. Chip specific.
+	 */
+	if (efuse->crystal_cap == 0xff) {
+		efuse->crystal_cap = 0x20;
+		DBG_EFUSE_FIX(rtwdev, crystal_cap);
+	}
+
 	return 0;
 }
 
@@ -297,7 +356,6 @@ static int __rtw8723x_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 
 static int __rtw8723x_mac_init(struct rtw_dev *rtwdev)
 {
-	rtw_write8(rtwdev, REG_FWHW_TXQ_CTRL + 1, WLAN_TXQ_RPT_EN);
 	rtw_write32(rtwdev, REG_TCR, BIT_TCR_CFG);
 
 	rtw_write16(rtwdev, REG_RXFLTMAP0, WLAN_RX_FILTER0);
@@ -314,6 +372,13 @@ static int __rtw8723x_mac_init(struct rtw_dev *rtwdev)
 	return 0;
 }
 
+static int __rtw8723x_mac_postinit(struct rtw_dev *rtwdev)
+{
+	rtw_write8(rtwdev, REG_FWHW_TXQ_CTRL + 1, WLAN_TXQ_RPT_EN);
+
+	return 0;
+}
+
 static void __rtw8723x_cfg_ldo25(struct rtw_dev *rtwdev, bool enable)
 {
 	u8 ldo_pwr;
@@ -704,6 +769,7 @@ const struct rtw8723x_common rtw8723x_common = {
 	.lck = __rtw8723x_lck,
 	.read_efuse = __rtw8723x_read_efuse,
 	.mac_init = __rtw8723x_mac_init,
+	.mac_postinit = __rtw8723x_mac_postinit,
 	.cfg_ldo25 = __rtw8723x_cfg_ldo25,
 	.set_tx_power_index = __rtw8723x_set_tx_power_index,
 	.efuse_grant = __rtw8723x_efuse_grant,
diff --git a/sys/contrib/dev/rtw88/rtw8723x.h b/sys/contrib/dev/rtw88/rtw8723x.h
index a99af527c92c..0fc70dfdfc8b 100644
--- a/sys/contrib/dev/rtw88/rtw8723x.h
+++ b/sys/contrib/dev/rtw88/rtw8723x.h
@@ -137,6 +137,7 @@ struct rtw8723x_common {
 	void (*lck)(struct rtw_dev *rtwdev);
 	int (*read_efuse)(struct rtw_dev *rtwdev, u8 *log_map);
 	int (*mac_init)(struct rtw_dev *rtwdev);
+	int (*mac_postinit)(struct rtw_dev *rtwdev);
 	void (*cfg_ldo25)(struct rtw_dev *rtwdev, bool enable);
 	void (*set_tx_power_index)(struct rtw_dev *rtwdev);
 	void (*efuse_grant)(struct rtw_dev *rtwdev, bool on);
@@ -383,6 +384,11 @@ static inline int rtw8723x_mac_init(struct rtw_dev *rtwdev)
 	return rtw8723x_common.mac_init(rtwdev);
 }
 
+static inline int rtw8723x_mac_postinit(struct rtw_dev *rtwdev)
+{
+	return rtw8723x_common.mac_postinit(rtwdev);
+}
+
 static inline void rtw8723x_cfg_ldo25(struct rtw_dev *rtwdev, bool enable)
 {
 	rtw8723x_common.cfg_ldo25(rtwdev, enable);
diff --git a/sys/contrib/dev/rtw88/rtw8812a.c b/sys/contrib/dev/rtw88/rtw8812a.c
index f9ba2aa2928a..2078eb6e3628 100644
--- a/sys/contrib/dev/rtw88/rtw8812a.c
+++ b/sys/contrib/dev/rtw88/rtw8812a.c
@@ -919,12 +919,14 @@ static const struct rtw_chip_ops rtw8812a_ops = {
 	.query_phy_status	= rtw8812a_query_phy_status,
 	.set_channel		= rtw88xxa_set_channel,
 	.mac_init		= NULL,
+	.mac_postinit		= NULL,
 	.read_rf		= rtw88xxa_phy_read_rf,
 	.write_rf		= rtw_phy_write_rf_reg_sipi,
 	.set_antenna		= NULL,
 	.set_tx_power_index	= rtw88xxa_set_tx_power_index,
 	.cfg_ldo25		= rtw8812a_cfg_ldo25,
 	.efuse_grant		= rtw88xxa_efuse_grant,
+	.set_ampdu_factor	= NULL,
 	.false_alarm_statistics	= rtw88xxa_false_alarm_statistics,
 	.phy_calibration	= rtw8812a_phy_calibration,
 	.cck_pd_set		= rtw88xxa_phy_cck_pd_set,
@@ -1037,7 +1039,7 @@ const struct rtw_chip_info rtw8812a_hw_spec = {
 	.ops = &rtw8812a_ops,
 	.id = RTW_CHIP_TYPE_8812A,
 	.fw_name = "rtw88/rtw8812a_fw.bin",
-	.wlan_cpu = RTW_WCPU_11N,
+	.wlan_cpu = RTW_WCPU_8051,
 	.tx_pkt_desc_sz = 40,
 	.tx_buf_desc_sz = 16,
 	.rx_pkt_desc_sz = 24,
@@ -1075,6 +1077,7 @@ const struct rtw_chip_info rtw8812a_hw_spec = {
 	.rfe_defs = rtw8812a_rfe_defs,
 	.rfe_defs_size = ARRAY_SIZE(rtw8812a_rfe_defs),
 	.rx_ldpc = false,
+	.amsdu_in_ampdu = true,
 	.hw_feature_report = false,
 	.c2h_ra_report_size = 4,
 	.old_datarate_fb_limit = true,
diff --git a/sys/contrib/dev/rtw88/rtw8812au.c b/sys/contrib/dev/rtw88/rtw8812au.c
index e18995f4cc78..dfea89670372 100644
--- a/sys/contrib/dev/rtw88/rtw8812au.c
+++ b/sys/contrib/dev/rtw88/rtw8812au.c
@@ -82,7 +82,7 @@ static const struct usb_device_id rtw_8812au_id_table[] = {
 MODULE_DEVICE_TABLE(usb, rtw_8812au_id_table);
 
 static struct usb_driver rtw_8812au_driver = {
-	.name = "rtw_8812au",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8812au_id_table,
 	.probe = rtw_usb_probe,
 	.disconnect = rtw_usb_disconnect,
diff --git a/sys/contrib/dev/rtw88/rtw8814a.c b/sys/contrib/dev/rtw88/rtw8814a.c
new file mode 100644
index 000000000000..6cf70814d78f
--- /dev/null
+++ b/sys/contrib/dev/rtw88/rtw8814a.c
@@ -0,0 +1,2281 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#include <linux/usb.h>
+#include "main.h"
+#include "coex.h"
+#include "tx.h"
+#include "phy.h"
+#include "rtw8814a.h"
+#include "rtw8814a_table.h"
+#include "rtw88xxa.h"
+#include "reg.h"
+#include "debug.h"
+#include "efuse.h"
+#include "regd.h"
+#include "usb.h"
+
+static void rtw8814a_efuse_grant(struct rtw_dev *rtwdev, bool on)
+{
+	if (on) {
+		rtw_write8(rtwdev, REG_EFUSE_ACCESS, EFUSE_ACCESS_ON);
+
+		rtw_write16_set(rtwdev, REG_SYS_FUNC_EN, BIT_FEN_ELDR);
+		rtw_write16_set(rtwdev, REG_SYS_CLKR,
+				BIT_LOADER_CLK_EN | BIT_ANA8M);
+	} else {
+		rtw_write8(rtwdev, REG_EFUSE_ACCESS, EFUSE_ACCESS_OFF);
+	}
+}
+
+static void rtw8814a_read_rfe_type(struct rtw_dev *rtwdev)
+{
+	struct rtw_efuse *efuse = &rtwdev->efuse;
+
+	if (!(efuse->rfe_option & BIT(7)))
+		return;
+
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_PCIE)
+		efuse->rfe_option = 0;
+	else if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB)
+		efuse->rfe_option = 1;
+}
+
+static void rtw8814a_read_amplifier_type(struct rtw_dev *rtwdev)
+{
+	struct rtw_efuse *efuse = &rtwdev->efuse;
+
+	switch (efuse->rfe_option) {
+	case 1:
+		/* Internal 2G */
+		efuse->pa_type_2g = 0;
+		efuse->lna_type_2g = 0;
+		/* External 5G */
+		efuse->pa_type_5g = BIT(0);
+		efuse->lna_type_5g = BIT(3);
+		break;
+	case 2 ... 5:
+		/* External everything */
+		efuse->pa_type_2g = BIT(4);
+		efuse->lna_type_2g = BIT(3);
+		efuse->pa_type_5g = BIT(0);
+		efuse->lna_type_5g = BIT(3);
+		break;
+	case 6:
+		efuse->lna_type_5g = BIT(3);
+		break;
+	default:
+		break;
+	}
+}
+
+static void rtw8814a_read_rf_type(struct rtw_dev *rtwdev,
+				  struct rtw8814a_efuse *map)
+{
+#if IS_ENABLED(CONFIG_RTW88_USB)
+	struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev);
+#endif
+	struct rtw_hal *hal = &rtwdev->hal;
+
+	switch (map->trx_antenna_option) {
+	case 0xff: /* 4T4R */
+	case 0xee: /* 3T3R */
+#if IS_ENABLED(CONFIG_RTW88_USB)
+		if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB &&
+		    rtwusb->udev->speed != USB_SPEED_SUPER)
+			hal->rf_type = RF_2T2R;
+		else
+#endif
+			hal->rf_type = RF_3T3R;
+
+		break;
+	case 0x66: /* 2T2R */
+	case 0x6f: /* 2T4R */
+	default:
+		hal->rf_type = RF_2T2R;
+		break;
+	}
+
+	hal->rf_path_num = 4;
+	hal->rf_phy_num = 4;
+
+	if (hal->rf_type == RF_3T3R) {
+		hal->antenna_rx = BB_PATH_ABC;
+		hal->antenna_tx = BB_PATH_ABC;
+	} else {
+		hal->antenna_rx = BB_PATH_AB;
+		hal->antenna_tx = BB_PATH_AB;
+	}
+}
+
+static void rtw8814a_init_hwcap(struct rtw_dev *rtwdev)
+{
+	struct rtw_efuse *efuse = &rtwdev->efuse;
+	struct rtw_hal *hal = &rtwdev->hal;
+
+	efuse->hw_cap.bw = BIT(RTW_CHANNEL_WIDTH_20) |
+			   BIT(RTW_CHANNEL_WIDTH_40) |
+			   BIT(RTW_CHANNEL_WIDTH_80);
+	efuse->hw_cap.ptcl = EFUSE_HW_CAP_PTCL_VHT;
+
+	if (hal->rf_type == RF_3T3R)
+		efuse->hw_cap.nss = 3;
+	else
+		efuse->hw_cap.nss = 2;
+
+	rtw_dbg(rtwdev, RTW_DBG_EFUSE,
+		"hw cap: hci=0x%02x, bw=0x%02x, ptcl=0x%02x, ant_num=%d, nss=%d\n",
+		efuse->hw_cap.hci, efuse->hw_cap.bw, efuse->hw_cap.ptcl,
+		efuse->hw_cap.ant_num, efuse->hw_cap.nss);
+}
+
+static int rtw8814a_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
+{
+	struct rtw_efuse *efuse = &rtwdev->efuse;
+	struct rtw8814a_efuse *map;
+	int i;
+
+	if (rtw_dbg_is_enabled(rtwdev, RTW_DBG_EFUSE))
+		print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
+			       log_map, rtwdev->chip->log_efuse_size, true);
+
+	map = (struct rtw8814a_efuse *)log_map;
+
+	efuse->usb_mode_switch = u8_get_bits(map->usb_mode, BIT(4));
+	efuse->rfe_option = map->rfe_option;
+	efuse->rf_board_option = map->rf_board_option;
+	efuse->crystal_cap = map->xtal_k;
+	efuse->channel_plan = map->channel_plan;
+	efuse->country_code[0] = map->country_code[0];
+	efuse->country_code[1] = map->country_code[1];
+	efuse->bt_setting = map->rf_bt_setting;
+	efuse->regd = map->rf_board_option & 0x7;
+	efuse->thermal_meter[RF_PATH_A] = map->thermal_meter;
+	efuse->thermal_meter_k = map->thermal_meter;
+	efuse->tx_bb_swing_setting_2g = map->tx_bb_swing_setting_2g;
+	efuse->tx_bb_swing_setting_5g = map->tx_bb_swing_setting_5g;
+
+	rtw8814a_read_rfe_type(rtwdev);
+	rtw8814a_read_amplifier_type(rtwdev);
+
+	/* Override rtw_chip_parameter_setup() */
+	rtw8814a_read_rf_type(rtwdev, map);
+
+	rtw8814a_init_hwcap(rtwdev);
+
+	for (i = 0; i < 4; i++)
+		efuse->txpwr_idx_table[i] = map->txpwr_idx_table[i];
+
+	switch (rtw_hci_type(rtwdev)) {
+	case RTW_HCI_TYPE_USB:
+		ether_addr_copy(efuse->addr, map->u.mac_addr);
+		break;
+	case RTW_HCI_TYPE_PCIE:
+		ether_addr_copy(efuse->addr, map->e.mac_addr);
+		break;
+	case RTW_HCI_TYPE_SDIO:
+	default:
+		/* unsupported now */
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static void rtw8814a_init_rfe_reg(struct rtw_dev *rtwdev)
+{
+	u8 rfe_option = rtwdev->efuse.rfe_option;
+
+	if (rfe_option == 2 || rfe_option == 1) {
+		rtw_write32_mask(rtwdev, 0x1994, 0xf, 0xf);
+		rtw_write8_set(rtwdev, REG_GPIO_MUXCFG + 2, 0xf0);
+	} else if (rfe_option == 0) {
+		rtw_write32_mask(rtwdev, 0x1994, 0xf, 0xf);
+		rtw_write8_set(rtwdev, REG_GPIO_MUXCFG + 2, 0xc0);
+	}
+}
+
+#define RTW_TXSCALE_SIZE 37
+static const u32 rtw8814a_txscale_tbl[RTW_TXSCALE_SIZE] = {
+	0x081, 0x088, 0x090, 0x099, 0x0a2, 0x0ac, 0x0b6, 0x0c0, 0x0cc, 0x0d8,
+	0x0e5, 0x0f2, 0x101, 0x110, 0x120, 0x131, 0x143, 0x156, 0x16a, 0x180,
+	0x197, 0x1af, 0x1c8, 0x1e3, 0x200, 0x21e, 0x23e, 0x261, 0x285, 0x2ab,
+	0x2d3, 0x2fe, 0x32b, 0x35c, 0x38e, 0x3c4, 0x3fe
+};
+
+static u32 rtw8814a_get_bb_swing(struct rtw_dev *rtwdev, u8 band, u8 rf_path)
+{
+	static const u32 swing2setting[4] = {0x200, 0x16a, 0x101, 0x0b6};
+	struct rtw_efuse *efuse = &rtwdev->efuse;
+	u8 tx_bb_swing;
+
+	if (band == RTW_BAND_2G)
+		tx_bb_swing = efuse->tx_bb_swing_setting_2g;
+	else
+		tx_bb_swing = efuse->tx_bb_swing_setting_5g;
+
+	tx_bb_swing >>= 2 * rf_path;
+	tx_bb_swing &= 0x3;
+
+	return swing2setting[tx_bb_swing];
+}
+
+static u8 rtw8814a_get_swing_index(struct rtw_dev *rtwdev)
+{
+	u32 swing, table_value;
+	u8 i;
+
+	swing = rtw8814a_get_bb_swing(rtwdev, rtwdev->hal.current_band_type,
+				      RF_PATH_A);
+
+	for (i = 0; i < ARRAY_SIZE(rtw8814a_txscale_tbl); i++) {
+		table_value = rtw8814a_txscale_tbl[i];
+		if (swing == table_value)
+			return i;
+	}
+
+	return 24;
+}
+
+static void rtw8814a_pwrtrack_init(struct rtw_dev *rtwdev)
+{
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+	u8 path;
+
+	dm_info->default_ofdm_index = rtw8814a_get_swing_index(rtwdev);
+
+	for (path = RF_PATH_A; path < rtwdev->hal.rf_path_num; path++) {
+		ewma_thermal_init(&dm_info->avg_thermal[path]);
+		dm_info->delta_power_index[path] = 0;
+		dm_info->delta_power_index_last[path] = 0;
+	}
+	dm_info->pwr_trk_triggered = false;
+	dm_info->pwr_trk_init_trigger = true;
+	dm_info->thermal_meter_k = rtwdev->efuse.thermal_meter_k;
+}
+
+static void rtw8814a_config_trx_path(struct rtw_dev *rtwdev)
+{
+	/* RX CCK disable 2R CCA */
+	rtw_write32_clr(rtwdev, REG_CCK0_FAREPORT,
+			BIT_CCK0_2RX | BIT_CCK0_MRC);
+	/* pathB tx on, path A/C/D tx off */
+	rtw_write32_mask(rtwdev, REG_CCK_RX, 0xf0000000, 0x4);
+	/* pathB rx */
+	rtw_write32_mask(rtwdev, REG_CCK_RX, 0x0f000000, 0x5);
+}
+
+static void rtw8814a_config_cck_rx_antenna_init(struct rtw_dev *rtwdev)
+{
+	/* CCK 2R CCA parameters */
+
+	/* Disable Ant diversity */
+	rtw_write32_mask(rtwdev, REG_RXSB, BIT_RXSB_ANA_DIV, 0x0);
+	/* Concurrent CCA at LSB & USB */
+	rtw_write32_mask(rtwdev, REG_CCA, BIT_CCA_CO, 0);
+	/* RX path diversity enable */
+	rtw_write32_mask(rtwdev, REG_ANTSEL, BIT_ANT_BYCO, 0);
+	/* r_en_mrc_antsel */
+	rtw_write32_mask(rtwdev, REG_PRECTRL, BIT_DIS_CO_PATHSEL, 0);
+	/* MBC weighting */
+	rtw_write32_mask(rtwdev, REG_CCA_MF, BIT_MBC_WIN, 1);
+	/* 2R CCA only */
+	rtw_write32_mask(rtwdev, REG_CCKTX, BIT_CMB_CCA_2R, 1);
+}
+
+static void rtw8814a_phy_set_param(struct rtw_dev *rtwdev)
+{
+	u32 crystal_cap, val32;
+	u8 val8, rf_path;
+
+	/* power on BB/RF domain */
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB)
+		rtw_write8_set(rtwdev, REG_SYS_FUNC_EN, BIT_FEN_USBA);
+	else if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_PCIE)
+		rtw_write8_set(rtwdev, REG_SYS_FUNC_EN, BIT_FEN_PCIEA);
+
+	rtw_write8_set(rtwdev, REG_SYS_CFG3_8814A + 2,
+		       BIT_FEN_BB_GLB_RST | BIT_FEN_BB_RSTB);
+
+	/* Power on RF paths A..D */
+	val8 = BIT_RF_EN | BIT_RF_RSTB | BIT_RF_SDM_RSTB;
+	rtw_write8(rtwdev, REG_RF_CTRL, val8);
+	rtw_write8(rtwdev, REG_RF_CTRL1, val8);
+	rtw_write8(rtwdev, REG_RF_CTRL2, val8);
+	rtw_write8(rtwdev, REG_RF_CTRL3, val8);
+
+	rtw_load_table(rtwdev, rtwdev->chip->bb_tbl);
+	rtw_load_table(rtwdev, rtwdev->chip->agc_tbl);
+
+	crystal_cap = rtwdev->efuse.crystal_cap & 0x3F;
+	crystal_cap |= crystal_cap << 6;
+	rtw_write32_mask(rtwdev, REG_AFE_CTRL3, 0x07ff8000, crystal_cap);
+
+	rtw8814a_config_trx_path(rtwdev);
+
+	for (rf_path = 0; rf_path < rtwdev->hal.rf_path_num; rf_path++)
+		rtw_load_table(rtwdev, rtwdev->chip->rf_tbl[rf_path]);
+
+	val32 = rtw_read_rf(rtwdev, RF_PATH_A, RF_RCK1_V1, RFREG_MASK);
+	rtw_write_rf(rtwdev, RF_PATH_B, RF_RCK1_V1, RFREG_MASK, val32);
+	rtw_write_rf(rtwdev, RF_PATH_C, RF_RCK1_V1, RFREG_MASK, val32);
+	rtw_write_rf(rtwdev, RF_PATH_D, RF_RCK1_V1, RFREG_MASK, val32);
+
+	rtw_write32_set(rtwdev, REG_RXPSEL, BIT_RX_PSEL_RST);
+
+	rtw_write8(rtwdev, REG_HWSEQ_CTRL, 0xFF);
+
+	rtw_write32(rtwdev, REG_BAR_MODE_CTRL, 0x0201ffff);
+
+	rtw_write8(rtwdev, REG_MISC_CTRL, BIT_DIS_SECOND_CCA);
+
+	rtw_write8(rtwdev, REG_NAV_CTRL + 2, 0);
+
+	rtw_write8_clr(rtwdev, REG_GPIO_MUXCFG, BIT(5));
+
+	rtw8814a_config_cck_rx_antenna_init(rtwdev);
+
+	rtw_phy_init(rtwdev);
+	rtw8814a_pwrtrack_init(rtwdev);
+
+	rtw8814a_init_rfe_reg(rtwdev);
+
+	rtw_write8_clr(rtwdev, REG_QUEUE_CTRL, BIT(3));
+
+	rtw_write8(rtwdev, REG_NAV_CTRL + 2, 235);
+
+	/* enable Tx report. */
+	rtw_write8(rtwdev,  REG_FWHW_TXQ_CTRL + 1, 0x1F);
+
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB) {
+		/* Reset USB mode switch setting */
+		rtw_write8(rtwdev, REG_SYS_SDIO_CTRL, 0x0);
+		rtw_write8(rtwdev, REG_ACLK_MON, 0x0);
+	}
+}
+
+static void rtw8814ae_enable_rf_1_2v(struct rtw_dev *rtwdev)
+{
+	/* This is for fullsize card, because GPIO7 there is floating.
+	 * We should pull GPIO7 high to enable RF 1.2V Switch Power Supply
+	 */
+
+	/* 1. set 0x40[1:0] to 0, BIT_GPIOSEL=0, select pin as GPIO */
+	rtw_write8_clr(rtwdev, REG_GPIO_MUXCFG, BIT(1) | BIT(0));
+
+	/* 2. set 0x44[31] to 0
+	 * mode=0: data port;
+	 * mode=1 and BIT_GPIO_IO_SEL=0: interrupt mode;
+	 */
+	rtw_write8_clr(rtwdev, REG_GPIO_PIN_CTRL + 3, BIT(7));
+
+	/* 3. data mode
+	 * 3.1 set 0x44[23] to 1
+	 * sel=0: input;
+	 * sel=1: output;
+	 */
+	rtw_write8_set(rtwdev, REG_GPIO_PIN_CTRL + 2, BIT(7));
+
+	/* 3.2 set 0x44[15] to 1
+	 * output high value;
+	 */
+	rtw_write8_set(rtwdev, REG_GPIO_PIN_CTRL + 1, BIT(7));
+}
+
+static int rtw8814a_mac_init(struct rtw_dev *rtwdev)
+{
+#if IS_ENABLED(CONFIG_RTW88_USB)
+	struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev);
+#endif
+
+	rtw_write16(rtwdev, REG_CR,
+		    MAC_TRX_ENABLE | BIT_MAC_SEC_EN | BIT_32K_CAL_TMR_EN);
+
+	rtw_load_table(rtwdev, rtwdev->chip->mac_tbl);
+
+#if IS_ENABLED(CONFIG_RTW88_USB)
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB)
+		rtw_write8(rtwdev, REG_AUTO_LLT_V1 + 3,
+			   rtwdev->chip->usb_tx_agg_desc_num << 1);
+#endif
+
+	rtw_write32(rtwdev, REG_HIMR0, 0);
+	rtw_write32(rtwdev, REG_HIMR1, 0);
+
+	rtw_write32_mask(rtwdev, REG_RRSR, 0xfffff, 0xfffff);
+
+	rtw_write16(rtwdev, REG_RETRY_LIMIT, 0x3030);
+
+	rtw_write16(rtwdev, REG_RXFLTMAP0, 0xffff);
+	rtw_write16(rtwdev, REG_RXFLTMAP1, 0x0400);
+	rtw_write16(rtwdev, REG_RXFLTMAP2, 0xffff);
+
+	rtw_write8(rtwdev, REG_MAX_AGGR_NUM, 0x36);
+	rtw_write8(rtwdev, REG_MAX_AGGR_NUM + 1, 0x36);
+
+	/* Set Spec SIFS (used in NAV) */
+	rtw_write16(rtwdev, REG_SPEC_SIFS, 0x100a);
+	rtw_write16(rtwdev, REG_MAC_SPEC_SIFS, 0x100a);
+
+	/* Set SIFS for CCK */
+	rtw_write16(rtwdev, REG_SIFS, 0x100a);
+
+	/* Set SIFS for OFDM */
+	rtw_write16(rtwdev, REG_SIFS + 2, 0x100a);
+
+	/* TXOP */
+	rtw_write32(rtwdev, REG_EDCA_BE_PARAM, 0x005EA42B);
+	rtw_write32(rtwdev, REG_EDCA_BK_PARAM, 0x0000A44F);
+	rtw_write32(rtwdev, REG_EDCA_VI_PARAM, 0x005EA324);
+	rtw_write32(rtwdev, REG_EDCA_VO_PARAM, 0x002FA226);
+
+	rtw_write8_set(rtwdev, REG_FWHW_TXQ_CTRL, BIT(7));
+
+	rtw_write8(rtwdev, REG_ACKTO, 0x80);
+
+	rtw_write16(rtwdev, REG_BCN_CTRL,
+		    BIT_DIS_TSF_UDT | (BIT_DIS_TSF_UDT << 8));
+	rtw_write32_mask(rtwdev, REG_TBTT_PROHIBIT, 0xfffff, WLAN_TBTT_TIME);
+	rtw_write8(rtwdev, REG_DRVERLYINT, 0x05);
+	rtw_write8(rtwdev, REG_BCNDMATIM, WLAN_BCN_DMA_TIME);
+	rtw_write16(rtwdev, REG_BCNTCFG, 0x4413);
+	rtw_write8(rtwdev, REG_BCN_MAX_ERR, 0xFF);
+
+	rtw_write32(rtwdev, REG_FAST_EDCA_VOVI_SETTING, 0x08070807);
+	rtw_write32(rtwdev, REG_FAST_EDCA_BEBK_SETTING, 0x08070807);
+
+#if IS_ENABLED(CONFIG_RTW88_USB)
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB &&
+	    rtwusb->udev->speed == USB_SPEED_SUPER) {
+		/* Disable U1/U2 Mode to avoid 2.5G spur in USB3.0. */
+		rtw_write8_clr(rtwdev, REG_USB_MOD, BIT(4) | BIT(3));
+		/* To avoid usb 3.0 H2C fail. */
+		rtw_write16(rtwdev, 0xf002, 0);
+
+		rtw_write8_clr(rtwdev, REG_SW_AMPDU_BURST_MODE_CTRL,
+			       BIT_PRE_TX_CMD);
+	} else
+#endif
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_PCIE) {
+		rtw8814ae_enable_rf_1_2v(rtwdev);
+
+		/* Force the antenna b to wifi. */
+		rtw_write8_set(rtwdev, REG_PAD_CTRL1, BIT(2));
+		rtw_write8_set(rtwdev, REG_PAD_CTRL1 + 1, BIT(0));
+		rtw_write8_set(rtwdev, REG_LED_CFG + 3,
+			       (BIT(27) | BIT_DPDT_WL_SEL) >> 24);
+	}
+
+	return 0;
+}
+
+static void rtw8814a_set_rfe_reg_24g(struct rtw_dev *rtwdev)
+{
+	switch (rtwdev->efuse.rfe_option) {
+	case 2:
+		rtw_write32(rtwdev, REG_RFE_PINMUX_A, 0x72707270);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_B, 0x72707270);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_C, 0x72707270);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_D, 0x77707770);
+
+		rtw_write32_mask(rtwdev, REG_RFE_INVSEL_D,
+				 BIT_RFE_SELSW0_D, 0x72);
+
+		break;
+	case 1:
+		rtw_write32(rtwdev, REG_RFE_PINMUX_A, 0x77777777);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_B, 0x77777777);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_C, 0x77777777);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_D, 0x77777777);
+
+		rtw_write32_mask(rtwdev, REG_RFE_INVSEL_D,
+				 BIT_RFE_SELSW0_D, 0x77);
+
+		break;
+	case 0:
+	default:
+		rtw_write32(rtwdev, REG_RFE_PINMUX_A, 0x77777777);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_B, 0x77777777);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_C, 0x77777777);
+		/* Is it not necessary to set REG_RFE_PINMUX_D ? */
+
+		rtw_write32_mask(rtwdev, REG_RFE_INVSEL_D,
+				 BIT_RFE_SELSW0_D, 0x77);
+
+		break;
+	}
+}
+
+static void rtw8814a_set_rfe_reg_5g(struct rtw_dev *rtwdev)
+{
+	switch (rtwdev->efuse.rfe_option) {
+	case 2:
+		rtw_write32(rtwdev, REG_RFE_PINMUX_A, 0x37173717);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_B, 0x37173717);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_C, 0x37173717);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_D, 0x77177717);
+
+		rtw_write32_mask(rtwdev, REG_RFE_INVSEL_D,
+				 BIT_RFE_SELSW0_D, 0x37);
+
+		break;
+	case 1:
+		rtw_write32(rtwdev, REG_RFE_PINMUX_A, 0x33173317);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_B, 0x33173317);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_C, 0x33173317);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_D, 0x77177717);
+
+		rtw_write32_mask(rtwdev, REG_RFE_INVSEL_D,
+				 BIT_RFE_SELSW0_D, 0x33);
+
+		break;
+	case 0:
+	default:
+		rtw_write32(rtwdev, REG_RFE_PINMUX_A, 0x54775477);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_B, 0x54775477);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_C, 0x54775477);
+		rtw_write32(rtwdev, REG_RFE_PINMUX_D, 0x54775477);
+
+		rtw_write32_mask(rtwdev, REG_RFE_INVSEL_D,
+				 BIT_RFE_SELSW0_D, 0x54);
+
+		break;
+	}
+}
+
+static void rtw8814a_set_channel_bb_swing(struct rtw_dev *rtwdev, u8 band)
+{
+	rtw_write32_mask(rtwdev, REG_TXSCALE_A, BB_SWING_MASK,
+			 rtw8814a_get_bb_swing(rtwdev, band, RF_PATH_A));
+	rtw_write32_mask(rtwdev, REG_TXSCALE_B, BB_SWING_MASK,
+			 rtw8814a_get_bb_swing(rtwdev, band, RF_PATH_B));
+	rtw_write32_mask(rtwdev, REG_TXSCALE_C, BB_SWING_MASK,
+			 rtw8814a_get_bb_swing(rtwdev, band, RF_PATH_C));
+	rtw_write32_mask(rtwdev, REG_TXSCALE_D, BB_SWING_MASK,
+			 rtw8814a_get_bb_swing(rtwdev, band, RF_PATH_D));
+	rtw8814a_pwrtrack_init(rtwdev);
+}
+
+static void rtw8814a_set_bw_reg_adc(struct rtw_dev *rtwdev, u8 bw)
+{
+	u32 adc = 0;
+
+	if (bw == RTW_CHANNEL_WIDTH_20)
+		adc = 0;
+	else if (bw == RTW_CHANNEL_WIDTH_40)
+		adc = 1;
+	else if (bw == RTW_CHANNEL_WIDTH_80)
+		adc = 2;
+
+	rtw_write32_mask(rtwdev, REG_ADCCLK, BIT(1) | BIT(0), adc);
+}
+
+static void rtw8814a_set_bw_reg_agc(struct rtw_dev *rtwdev, u8 new_band, u8 bw)
+{
+	u32 agc = 7;
+
+	if (bw == RTW_CHANNEL_WIDTH_20) {
+		agc = 6;
+	} else if (bw == RTW_CHANNEL_WIDTH_40) {
+		if (new_band == RTW_BAND_5G)
+			agc = 8;
+		else
+			agc = 7;
+	} else if (bw == RTW_CHANNEL_WIDTH_80) {
+		agc = 3;
+	}
+
+	rtw_write32_mask(rtwdev, REG_CCASEL, 0xf000, agc);
+}
+
+static void rtw8814a_switch_band(struct rtw_dev *rtwdev, u8 new_band, u8 bw)
+{
+	/* Clear 0x1000[16], When this bit is set to 0, CCK and OFDM
+	 * are disabled, and clock are gated. Otherwise, CCK and OFDM
+	 * are enabled.
+	 */
+	rtw_write8_clr(rtwdev, REG_SYS_CFG3_8814A + 2, BIT_FEN_BB_RSTB);
+
+	if (new_band == RTW_BAND_2G) {
+		rtw_write32_mask(rtwdev, REG_AGC_TABLE, 0x1f, 0);
+
+		rtw8814a_set_rfe_reg_24g(rtwdev);
+
+		rtw_write32_mask(rtwdev, REG_TXPSEL, 0xf0, 0x2);
+		rtw_write32_mask(rtwdev, REG_CCK_RX, 0x0f000000, 0x5);
+
+		rtw_write32_mask(rtwdev, REG_RXPSEL, BIT_RX_PSEL_RST, 0x3);
+
+		rtw_write8(rtwdev, REG_CCK_CHECK, 0);
+
+		rtw_write32_mask(rtwdev, 0xa80, BIT(18), 0);
+	} else {
+		rtw_write8(rtwdev, REG_CCK_CHECK, BIT_CHECK_CCK_EN);
+
+		/* Enable CCK Tx function, even when CCK is off */
+		rtw_write32_mask(rtwdev, 0xa80, BIT(18), 1);
+
+		rtw8814a_set_rfe_reg_5g(rtwdev);
+
+		rtw_write32_mask(rtwdev, REG_TXPSEL, 0xf0, 0x0);
+		rtw_write32_mask(rtwdev, REG_CCK_RX, 0x0f000000, 0xf);
+
+		rtw_write32_mask(rtwdev, REG_RXPSEL, BIT_RX_PSEL_RST, 0x2);
+	}
+
+	rtw8814a_set_channel_bb_swing(rtwdev, new_band);
+
+	rtw8814a_set_bw_reg_adc(rtwdev, bw);
+	rtw8814a_set_bw_reg_agc(rtwdev, new_band, bw);
+
+	rtw_write8_set(rtwdev, REG_SYS_CFG3_8814A + 2, BIT_FEN_BB_RSTB);
+}
+
+static void rtw8814a_switch_channel(struct rtw_dev *rtwdev, u8 channel)
+{
+	struct rtw_hal *hal = &rtwdev->hal;
+	u32 fc_area, rf_mod_ag, cfgch;
+	u8 path;
+
+	switch (channel) {
+	case 36 ... 48:
+		fc_area = 0x494;
+		break;
+	case 50 ... 64:
+		fc_area = 0x453;
+		break;
+	case 100 ... 116:
+		fc_area = 0x452;
+		break;
+	default:
+		if (channel >= 118)
+			fc_area = 0x412;
+		else
+			fc_area = 0x96a;
+		break;
+	}
+
+	rtw_write32_mask(rtwdev, REG_CLKTRK, 0x1ffe0000, fc_area);
+
+	for (path = 0; path < hal->rf_path_num; path++) {
+		switch (channel) {
+		case 36 ... 64:
+			rf_mod_ag = 0x101;
+			break;
+		case 100 ... 140:
+			rf_mod_ag = 0x301;
+			break;
+		default:
+			if (channel > 140)
+				rf_mod_ag = 0x501;
+			else
+				rf_mod_ag = 0x000;
+			break;
+		}
+
+		cfgch = (rf_mod_ag << 8) | channel;
+
+		rtw_write_rf(rtwdev, path, RF_CFGCH,
+			     RF18_RFSI_MASK | RF18_BAND_MASK | RF18_CHANNEL_MASK, cfgch);
+	}
+
+	switch (channel) {
+	case 36 ... 64:
+		rtw_write32_mask(rtwdev, REG_AGC_TABLE, 0x1f, 1);
+		break;
+	case 100 ... 144:
+		rtw_write32_mask(rtwdev, REG_AGC_TABLE, 0x1f, 2);
+		break;
+	default:
+		if (channel >= 149)
+			rtw_write32_mask(rtwdev, REG_AGC_TABLE, 0x1f, 3);
+
+		break;
+	}
+}
+
+static void rtw8814a_24g_cck_tx_dfir(struct rtw_dev *rtwdev, u8 channel)
+{
+	if (channel >= 1 && channel <= 11) {
+		rtw_write32(rtwdev, REG_CCK0_TX_FILTER1, 0x1a1b0030);
+		rtw_write32(rtwdev, REG_CCK0_TX_FILTER2, 0x090e1317);
+		rtw_write32(rtwdev, REG_CCK0_DEBUG_PORT, 0x00000204);
+	} else if (channel >= 12 && channel <= 13) {
+		rtw_write32(rtwdev, REG_CCK0_TX_FILTER1, 0x1a1b0030);
+		rtw_write32(rtwdev, REG_CCK0_TX_FILTER2, 0x090e1217);
+		rtw_write32(rtwdev, REG_CCK0_DEBUG_PORT, 0x00000305);
+	} else if (channel == 14) {
+		rtw_write32(rtwdev, REG_CCK0_TX_FILTER1, 0x1a1b0030);
+		rtw_write32(rtwdev, REG_CCK0_TX_FILTER2, 0x00000E17);
+		rtw_write32(rtwdev, REG_CCK0_DEBUG_PORT, 0x00000000);
+	}
+}
+
+static void rtw8814a_set_bw_reg_mac(struct rtw_dev *rtwdev, u8 bw)
+{
+	u16 val16 = rtw_read16(rtwdev, REG_WMAC_TRXPTCL_CTL);
+
+	val16 &= ~BIT_RFMOD;
+	if (bw == RTW_CHANNEL_WIDTH_80)
+		val16 |= BIT_RFMOD_80M;
+	else if (bw == RTW_CHANNEL_WIDTH_40)
+		val16 |= BIT_RFMOD_40M;
+
+	rtw_write16(rtwdev, REG_WMAC_TRXPTCL_CTL, val16);
+}
+
+static void rtw8814a_set_bw_rf(struct rtw_dev *rtwdev, u8 bw)
+{
+	u8 path;
+
+	for (path = RF_PATH_A; path < rtwdev->hal.rf_path_num; path++) {
+		switch (bw) {
+		case RTW_CHANNEL_WIDTH_5:
+		case RTW_CHANNEL_WIDTH_10:
+		case RTW_CHANNEL_WIDTH_20:
+		default:
+			rtw_write_rf(rtwdev, path, RF_CFGCH, RF18_BW_MASK, 3);
+			break;
+		case RTW_CHANNEL_WIDTH_40:
+			rtw_write_rf(rtwdev, path, RF_CFGCH, RF18_BW_MASK, 1);
+			break;
+		case RTW_CHANNEL_WIDTH_80:
+			rtw_write_rf(rtwdev, path, RF_CFGCH, RF18_BW_MASK, 0);
+			break;
+		}
+	}
+}
+
+static void rtw8814a_adc_clk(struct rtw_dev *rtwdev)
+{
+	static const u32 rxiqc_reg[2][4] = {
+		{ REG_RX_IQC_AB_A, REG_RX_IQC_AB_B,
+		  REG_RX_IQC_AB_C, REG_RX_IQC_AB_D },
+		{ REG_RX_IQC_CD_A, REG_RX_IQC_CD_B,
+		  REG_RX_IQC_CD_C, REG_RX_IQC_CD_D }
+	};
+	u32 bb_reg_8fc, bb_reg_808, rxiqc[4];
+	u32 i = 0, mac_active = 1;
+	u8 mac_reg_522;
+
+	if (rtwdev->hal.cut_version != RTW_CHIP_VER_CUT_A)
+		return;
+
+	/* 1 Step1. MAC TX pause */
+	mac_reg_522 = rtw_read8(rtwdev, REG_TXPAUSE);
+	bb_reg_8fc = rtw_read32(rtwdev, REG_DBGSEL);
+	bb_reg_808 = rtw_read32(rtwdev, REG_RXPSEL);
+	rtw_write8(rtwdev, REG_TXPAUSE, 0x3f);
+
+	/* 1 Step 2. Backup rxiqc & rxiqc = 0 */
+	for (i = 0; i < 4; i++) {
+		rxiqc[i] = rtw_read32(rtwdev, rxiqc_reg[0][i]);
+		rtw_write32(rtwdev, rxiqc_reg[0][i], 0x0);
+		rtw_write32(rtwdev, rxiqc_reg[1][i], 0x0);
+	}
+	rtw_write32_mask(rtwdev, REG_PRECTRL, BIT_IQ_WGT, 0x3);
+	i = 0;
+
+	/* 1 Step 3. Monitor MAC IDLE */
+	rtw_write32(rtwdev, REG_DBGSEL, 0x0);
+	while (mac_active) {
+		mac_active = rtw_read32(rtwdev, REG_DBGRPT) & 0x803e0008;
+		i++;
+		if (i > 1000)
+			break;
+	}
+
+	/* 1 Step 4. ADC clk flow */
+	rtw_write8(rtwdev, REG_RXPSEL, 0x11);
+	rtw_write32_mask(rtwdev, REG_DAC_RSTB, BIT(13), 0x1);
+	rtw_write8_mask(rtwdev, REG_GNT_BT, BIT(2) | BIT(1), 0x3);
+	rtw_write32_mask(rtwdev, REG_CCK_RPT_FORMAT, BIT(2), 0x1);
+
+	/* 0xc1c/0xe1c/0x181c/0x1a1c[4] must=1 to ensure table can be
+	 * written when bbrstb=0
+	 * 0xc60/0xe60/0x1860/0x1a60[15] always = 1 after this line
+	 * 0xc60/0xe60/0x1860/0x1a60[14] always = 0 bcz its error in A-cut
+	 */
+
+	/* power_off/clk_off @ anapar_state=idle mode */
+	rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x15800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x01808003);
+	rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x15800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x01808003);
+	rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x15800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x01808003);
+	rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x15800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x01808003);
+
+	rtw_write8_mask(rtwdev, REG_GNT_BT, BIT(2), 0x0);
+	rtw_write32_mask(rtwdev, REG_CCK_RPT_FORMAT, BIT(2), 0x0);
+	/* [19] = 1 to turn off ADC */
+	rtw_write32(rtwdev, REG_CK_MONHA, 0x0D080058);
+	rtw_write32(rtwdev, REG_CK_MONHB, 0x0D080058);
+	rtw_write32(rtwdev, REG_CK_MONHC, 0x0D080058);
+	rtw_write32(rtwdev, REG_CK_MONHD, 0x0D080058);
+
+	/* power_on/clk_off */
+	/* [19] = 0 to turn on ADC */
+	rtw_write32(rtwdev, REG_CK_MONHA, 0x0D000058);
+	rtw_write32(rtwdev, REG_CK_MONHB, 0x0D000058);
+	rtw_write32(rtwdev, REG_CK_MONHC, 0x0D000058);
+	rtw_write32(rtwdev, REG_CK_MONHD, 0x0D000058);
+
+	/* power_on/clk_on @ anapar_state=BT mode */
+	rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x05808032);
+	rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x05808032);
+	rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x05808032);
+	rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x05808032);
+	rtw_write8_mask(rtwdev, REG_GNT_BT, BIT(2), 0x1);
+	rtw_write32_mask(rtwdev, REG_CCK_RPT_FORMAT, BIT(2), 0x1);
+
+	/* recover original setting @ anapar_state=BT mode */
+	rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x05808032);
+	rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x05808032);
+	rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x05808032);
+	rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x05808032);
+
+	rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x05800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x07808003);
+	rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x05800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x07808003);
+	rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x05800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x07808003);
+	rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x05800002);
+	rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x07808003);
+
+	rtw_write8_mask(rtwdev, REG_GNT_BT, BIT(2) | BIT(1), 0x0);
+	rtw_write32_mask(rtwdev, REG_CCK_RPT_FORMAT, BIT(2), 0x0);
+	rtw_write32_mask(rtwdev, REG_DAC_RSTB, BIT(13), 0x0);
+
+	/* 1 Step 5. Recover MAC TX & IQC */
+	rtw_write8(rtwdev, REG_TXPAUSE, mac_reg_522);
+	rtw_write32(rtwdev, REG_DBGSEL, bb_reg_8fc);
+	rtw_write32(rtwdev, REG_RXPSEL, bb_reg_808);
+	for (i = 0; i < 4; i++) {
+		rtw_write32(rtwdev, rxiqc_reg[0][i], rxiqc[i]);
+		rtw_write32(rtwdev, rxiqc_reg[1][i], 0x01000000);
+	}
+	rtw_write32_mask(rtwdev, REG_PRECTRL, BIT_IQ_WGT, 0x0);
+}
+
+static void rtw8814a_spur_calibration_ch140(struct rtw_dev *rtwdev, u8 channel)
+{
+	struct rtw_hal *hal = &rtwdev->hal;
+
+	/* Add for 8814AE module ch140 MP Rx */
+	if (channel == 140) {
+		if (hal->ch_param[0] == 0)
+			hal->ch_param[0] = rtw_read32(rtwdev, REG_CCASEL);
+		if (hal->ch_param[1] == 0)
+			hal->ch_param[1] = rtw_read32(rtwdev, REG_PDMFTH);
+
+		rtw_write32(rtwdev, REG_CCASEL, 0x75438170);
+		rtw_write32(rtwdev, REG_PDMFTH, 0x79a18a0a);
+	} else {
+		if (rtw_read32(rtwdev, REG_CCASEL) == 0x75438170 &&
+		    hal->ch_param[0] != 0)
+			rtw_write32(rtwdev, REG_CCASEL, hal->ch_param[0]);
+
+		if (rtw_read32(rtwdev, REG_PDMFTH) == 0x79a18a0a &&
+		    hal->ch_param[1] != 0)
+			rtw_write32(rtwdev, REG_PDMFTH, hal->ch_param[1]);
+
+		hal->ch_param[0] = rtw_read32(rtwdev, REG_CCASEL);
+		hal->ch_param[1] = rtw_read32(rtwdev, REG_PDMFTH);
+	}
+}
+
+static void rtw8814a_set_nbi_reg(struct rtw_dev *rtwdev, u32 tone_idx)
+{
+	/* tone_idx X 10 */
+	static const u32 nbi_128[] = {
+		25, 55, 85, 115, 135,
+		155, 185, 205, 225, 245,
+		265, 285, 305, 335, 355,
+		375, 395, 415, 435, 455,
+		485, 505, 525, 555, 585, 615, 635
+	};
+	u32 reg_idx = 0;
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(nbi_128); i++) {
+		if (tone_idx < nbi_128[i]) {
+			reg_idx = i + 1;
+			break;
+		}
+	}
+
+	rtw_write32_mask(rtwdev, REG_NBI_SETTING, 0xfc000, reg_idx);
+}
+
+static void rtw8814a_nbi_setting(struct rtw_dev *rtwdev, u32 ch, u32 f_intf)
+{
+	u32 fc, int_distance, tone_idx;
+
+	fc = 2412 + (ch - 1) * 5;
+	int_distance = abs_diff(fc, f_intf);
+
+	/* 10 * (int_distance / 0.3125) */
+	tone_idx = int_distance << 5;
+
+	rtw8814a_set_nbi_reg(rtwdev, tone_idx);
+
+	rtw_write32_mask(rtwdev, REG_NBI_SETTING, BIT_NBI_ENABLE, 1);
+}
+
+static void rtw8814a_spur_nbi_setting(struct rtw_dev *rtwdev)
+{
+	u8 primary_channel = rtwdev->hal.primary_channel;
+	u8 rfe_type = rtwdev->efuse.rfe_option;
+
+	if (rfe_type != 0 && rfe_type != 1 && rfe_type != 6 && rfe_type != 7)
+		return;
+
+	if (primary_channel == 14)
+		rtw8814a_nbi_setting(rtwdev, primary_channel, 2480);
+	else if (primary_channel >= 4 && primary_channel <= 8)
+		rtw8814a_nbi_setting(rtwdev, primary_channel, 2440);
+	else
+		rtw_write32_mask(rtwdev, REG_NBI_SETTING, BIT_NBI_ENABLE, 0);
+}
+
+/* A workaround to eliminate the 5280 MHz & 5600 MHz & 5760 MHz spur of 8814A */
+static void rtw8814a_spur_calibration(struct rtw_dev *rtwdev, u8 channel, u8 bw)
+{
+	u8 rfe_type = rtwdev->efuse.rfe_option;
+	bool reset_nbi_csi = true;
+
+	if (rfe_type == 0) {
+		switch (bw) {
+		case RTW_CHANNEL_WIDTH_40:
+			if (channel == 54 || channel == 118) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x3e >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK0, 0);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK1,
+						 BIT(0), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK6, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK7, 0);
+
+				reset_nbi_csi = false;
+			} else if (channel == 151) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x1e >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK0,
+						 BIT(16), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK6, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK7, 0);
+
+				reset_nbi_csi = false;
+			}
+			break;
+		case RTW_CHANNEL_WIDTH_80:
+			if (channel == 58 || channel == 122) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x3a >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK0, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK6, 0);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK7,
+						 BIT(0), 1);
+
+				reset_nbi_csi = false;
+			} else if (channel == 155) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x5a >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK0, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK6,
+						 BIT(16), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK7, 0);
+
+				reset_nbi_csi = false;
+			}
+			break;
+		case RTW_CHANNEL_WIDTH_20:
+			if (channel == 153) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x1e >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK0, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK6, 0);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK7,
+						 BIT(16), 1);
+
+				reset_nbi_csi = false;
+			}
+
+			rtw8814a_spur_calibration_ch140(rtwdev, channel);
+			break;
+		default:
+			break;
+		}
+	} else if (rfe_type == 1 || rfe_type == 2) {
+		switch (bw) {
+		case RTW_CHANNEL_WIDTH_20:
+			if (channel == 153) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x1E >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK0, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK6, 0);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK7,
+						 BIT(16), 1);
+
+				reset_nbi_csi = false;
+			}
+			break;
+		case RTW_CHANNEL_WIDTH_40:
+			if (channel == 151) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x1e >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK0,
+						 BIT(16), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK6, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK7, 0);
+
+				reset_nbi_csi = false;
+			}
+			break;
+		case RTW_CHANNEL_WIDTH_80:
+			if (channel == 155) {
+				rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+						 0x000fe000, 0x5a >> 1);
+				rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1,
+						 BIT(0), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK0, 0);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+				rtw_write32_mask(rtwdev, REG_CSI_FIX_MASK6,
+						 BIT(16), 1);
+				rtw_write32(rtwdev, REG_CSI_FIX_MASK7, 0);
+
+				reset_nbi_csi = false;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (reset_nbi_csi) {
+		rtw_write32_mask(rtwdev, REG_NBI_SETTING,
+				 0x000fe000, 0xfc >> 1);
+		rtw_write32_mask(rtwdev, REG_CSI_MASK_SETTING1, BIT(0), 0);
+		rtw_write32(rtwdev, REG_CSI_FIX_MASK0, 0);
+		rtw_write32(rtwdev, REG_CSI_FIX_MASK1, 0);
+		rtw_write32(rtwdev, REG_CSI_FIX_MASK6, 0);
+		rtw_write32(rtwdev, REG_CSI_FIX_MASK7, 0);
+	}
+
+	rtw8814a_spur_nbi_setting(rtwdev);
+}
+
+static void rtw8814a_set_bw_mode(struct rtw_dev *rtwdev, u8 new_band,
+				 u8 channel, u8 bw, u8 primary_chan_idx)
+{
+	u8 txsc40 = 0, txsc20, txsc;
+
+	rtw8814a_set_bw_reg_mac(rtwdev, bw);
+
+	txsc20 = primary_chan_idx;
+	if (bw == RTW_CHANNEL_WIDTH_80) {
+		if (txsc20 == RTW_SC_20_UPPER || txsc20 == RTW_SC_20_UPMOST)
+			txsc40 = RTW_SC_40_UPPER;
+		else
+			txsc40 = RTW_SC_40_LOWER;
+	}
+
+	txsc = BIT_TXSC_20M(txsc20) | BIT_TXSC_40M(txsc40);
+	rtw_write8(rtwdev, REG_DATA_SC, txsc);
+
+	rtw8814a_set_bw_reg_adc(rtwdev, bw);
+	rtw8814a_set_bw_reg_agc(rtwdev, new_band, bw);
+
+	if (bw == RTW_CHANNEL_WIDTH_80) {
+		rtw_write32_mask(rtwdev, REG_ADCCLK, 0x3c, txsc);
+	} else if (bw == RTW_CHANNEL_WIDTH_40) {
+		rtw_write32_mask(rtwdev, REG_ADCCLK, 0x3c, txsc);
+
+		if (txsc == RTW_SC_20_UPPER)
+			rtw_write32_set(rtwdev, REG_RXSB, BIT(4));
+		else
+			rtw_write32_clr(rtwdev, REG_RXSB, BIT(4));
+	}
+
+	rtw8814a_set_bw_rf(rtwdev, bw);
+
+	rtw8814a_adc_clk(rtwdev);
+
+	rtw8814a_spur_calibration(rtwdev, channel, bw);
+}
+
+static void rtw8814a_set_channel(struct rtw_dev *rtwdev, u8 channel, u8 bw,
+				 u8 primary_chan_idx)
+{
+	u8 old_band, new_band;
+
+	if (rtw_read8(rtwdev, REG_CCK_CHECK) & BIT_CHECK_CCK_EN)
+		old_band = RTW_BAND_5G;
+	else
+		old_band = RTW_BAND_2G;
+
+	if (channel > 14)
+		new_band = RTW_BAND_5G;
+	else
+		new_band = RTW_BAND_2G;
+
+	if (new_band != old_band)
+		rtw8814a_switch_band(rtwdev, new_band, bw);
+
+	rtw8814a_switch_channel(rtwdev, channel);
+
+	rtw8814a_24g_cck_tx_dfir(rtwdev, channel);
+
+	rtw8814a_set_bw_mode(rtwdev, new_band, channel, bw, primary_chan_idx);
+}
+
+static s8 rtw8814a_cck_rx_pwr(u8 lna_idx, u8 vga_idx)
+{
+	s8 rx_pwr_all = 0;
+
+	switch (lna_idx) {
+	case 7:
+		rx_pwr_all = -38 - 2 * vga_idx;
+		break;
+	case 5:
+		rx_pwr_all = -28 - 2 * vga_idx;
+		break;
+	case 3:
+		rx_pwr_all = -8 - 2 * vga_idx;
+		break;
+	case 2:
+		rx_pwr_all = -1 - 2 * vga_idx;
+		break;
+	default:
+		break;
+	}
+
+	return rx_pwr_all;
+}
+
+static void rtw8814a_query_phy_status(struct rtw_dev *rtwdev, u8 *phy_status,
+				      struct rtw_rx_pkt_stat *pkt_stat)
+{
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+	struct rtw_jaguar_phy_status_rpt *rpt;
+	u8 gain[RTW_RF_PATH_MAX], rssi, i;
+	s8 rx_pwr_db, middle1, middle2;
+	s8 snr[RTW_RF_PATH_MAX];
+	s8 evm[RTW_RF_PATH_MAX];
+	u8 rfmode, subchannel;
+	u8 lna, vga;
+	s8 cfo[2];
+
+	rpt = (struct rtw_jaguar_phy_status_rpt *)phy_status;
+
+	pkt_stat->bw = RTW_CHANNEL_WIDTH_20;
+
+	if (pkt_stat->rate <= DESC_RATE11M) {
+		lna = le32_get_bits(rpt->w1, RTW_JGRPHY_W1_AGC_RPT_LNA_IDX);
+		vga = le32_get_bits(rpt->w1, RTW_JGRPHY_W1_AGC_RPT_VGA_IDX);
+
+		rx_pwr_db = rtw8814a_cck_rx_pwr(lna, vga);
+
+		pkt_stat->rx_power[RF_PATH_A] = rx_pwr_db;
+		pkt_stat->rssi = rtw_phy_rf_power_2_rssi(pkt_stat->rx_power, 1);
+		dm_info->rssi[RF_PATH_A] = pkt_stat->rssi;
+		pkt_stat->signal_power = rx_pwr_db;
+	} else { /* OFDM rate */
+		gain[RF_PATH_A] = le32_get_bits(rpt->w0, RTW_JGRPHY_W0_GAIN_A);
+		gain[RF_PATH_B] = le32_get_bits(rpt->w0, RTW_JGRPHY_W0_GAIN_B);
+		gain[RF_PATH_C] = le32_get_bits(rpt->w5, RTW_JGRPHY_W5_GAIN_C);
+		gain[RF_PATH_D] = le32_get_bits(rpt->w6, RTW_JGRPHY_W6_GAIN_D);
+
+		snr[RF_PATH_A] = le32_get_bits(rpt->w3, RTW_JGRPHY_W3_RXSNR_A);
+		snr[RF_PATH_B] = le32_get_bits(rpt->w4, RTW_JGRPHY_W4_RXSNR_B);
+		snr[RF_PATH_C] = le32_get_bits(rpt->w5, RTW_JGRPHY_W5_RXSNR_C);
+		snr[RF_PATH_D] = le32_get_bits(rpt->w5, RTW_JGRPHY_W5_RXSNR_D);
+
+		evm[RF_PATH_A] = le32_get_bits(rpt->w3, RTW_JGRPHY_W3_RXEVM_1);
+		evm[RF_PATH_B] = le32_get_bits(rpt->w3, RTW_JGRPHY_W3_RXEVM_2);
+		evm[RF_PATH_C] = le32_get_bits(rpt->w4, RTW_JGRPHY_W4_RXEVM_3);
+		evm[RF_PATH_D] = le32_get_bits(rpt->w5, RTW_JGRPHY_W5_RXEVM_4);
+
+		if (pkt_stat->rate <= DESC_RATE54M)
+			evm[RF_PATH_A] = le32_get_bits(rpt->w6,
+						       RTW_JGRPHY_W6_SIGEVM);
+
+		for (i = RF_PATH_A; i < RTW_RF_PATH_MAX; i++) {
+			pkt_stat->rx_power[i] = gain[i] - 110;
+
+			rssi = rtw_phy_rf_power_2_rssi(&pkt_stat->rx_power[i], 1);
+			dm_info->rssi[i] = rssi;
+
+			pkt_stat->rx_snr[i] = snr[i];
+			dm_info->rx_snr[i] = snr[i] >> 1;
+
+			pkt_stat->rx_evm[i] = evm[i];
+			evm[i] = max_t(s8, -127, evm[i]);
+			dm_info->rx_evm_dbm[i] = abs(evm[i]) >> 1;
+		}
+
+		rssi = rtw_phy_rf_power_2_rssi(pkt_stat->rx_power,
+					       RTW_RF_PATH_MAX);
+		pkt_stat->rssi = rssi;
+
+		/* When power saving is enabled the hardware sometimes
+		 * reports unbelievably high gain for paths A and C
+		 * (e.g. one frame 64 68 68 72, the next frame 106 66 88 72,
+		 * the next 66 66 68 72), so use the second lowest gain
+		 * instead of the highest.
+		 */
+		middle1 = max(min(gain[RF_PATH_A], gain[RF_PATH_B]),
+			      min(gain[RF_PATH_C], gain[RF_PATH_D]));
+		middle2 = min(max(gain[RF_PATH_A], gain[RF_PATH_B]),
+			      max(gain[RF_PATH_C], gain[RF_PATH_D]));
+		rx_pwr_db = min(middle1, middle2);
+		rx_pwr_db -= 110;
+		pkt_stat->signal_power = rx_pwr_db;
+
+		rfmode = le32_get_bits(rpt->w0, RTW_JGRPHY_W0_R_RFMOD);
+		subchannel = le32_get_bits(rpt->w0, RTW_JGRPHY_W0_SUB_CHNL);
+
+		if (rfmode == 1 && subchannel == 0) {
+			pkt_stat->bw = RTW_CHANNEL_WIDTH_40;
+		} else if (rfmode == 2) {
+			if (subchannel == 0)
+				pkt_stat->bw = RTW_CHANNEL_WIDTH_80;
+			else if (subchannel == 9 || subchannel == 10)
+				pkt_stat->bw = RTW_CHANNEL_WIDTH_40;
+		}
+
+		cfo[RF_PATH_A] = le32_get_bits(rpt->w2, RTW_JGRPHY_W2_CFO_TAIL_A);
+		cfo[RF_PATH_B] = le32_get_bits(rpt->w2, RTW_JGRPHY_W2_CFO_TAIL_B);
+
+		for (i = RF_PATH_A; i < 2; i++) {
+			pkt_stat->cfo_tail[i] = cfo[i];
+			dm_info->cfo_tail[i] = (cfo[i] * 5) >> 1;
+		}
+	}
+}
+
+static void
+rtw8814a_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs)
+{
+	struct rtw_hal *hal = &rtwdev->hal;
+	u32 txagc_table_wd;
+	u8 rate, pwr_index;
+	int j;
+
+	for (j = 0; j < rtw_rate_size[rs]; j++) {
+		rate = rtw_rate_section[rs][j];
+
+		pwr_index = hal->tx_pwr_tbl[path][rate] + 2;
+		if (pwr_index > rtwdev->chip->max_power_index)
+			pwr_index = rtwdev->chip->max_power_index;
+
+		txagc_table_wd = 0x00801000;
+		txagc_table_wd |= (pwr_index << 24) | (path << 8) | rate;
+
+		rtw_write32(rtwdev, REG_AGC_TBL, txagc_table_wd);
+
+		/* first time to turn on the txagc table
+		 * second to write the addr0
+		 */
+		if (rate == DESC_RATE1M)
+			rtw_write32(rtwdev, REG_AGC_TBL, txagc_table_wd);
+	}
+}
+
+static void rtw8814a_set_tx_power_index(struct rtw_dev *rtwdev)
+{
+	struct rtw_hal *hal = &rtwdev->hal;
+	int path;
+
+	for (path = 0; path < hal->rf_path_num; path++) {
+		if (hal->current_band_type == RTW_BAND_2G)
+			rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+							    RTW_RATE_SECTION_CCK);
+
+		rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+						    RTW_RATE_SECTION_OFDM);
+
+		if (test_bit(RTW_FLAG_SCANNING, rtwdev->flags))
+			continue;
+
+		rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+						    RTW_RATE_SECTION_HT_1S);
+		rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+						    RTW_RATE_SECTION_VHT_1S);
+
+		rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+						    RTW_RATE_SECTION_HT_2S);
+		rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+						    RTW_RATE_SECTION_VHT_2S);
+
+		rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+						    RTW_RATE_SECTION_HT_3S);
+		rtw8814a_set_tx_power_index_by_rate(rtwdev, path,
+						    RTW_RATE_SECTION_VHT_3S);
+	}
+}
+
+static void rtw8814a_cfg_ldo25(struct rtw_dev *rtwdev, bool enable)
+{
+}
+
+/* Without this RTL8814A sends too many frames and (some?) 11n AP
+ * can't handle it, resulting in low TX speed. Other chips seem fine.
+ */
+static void rtw8814a_set_ampdu_factor(struct rtw_dev *rtwdev, u8 factor)
+{
+	factor = min_t(u8, factor, IEEE80211_VHT_MAX_AMPDU_256K);
+
+	rtw_write32(rtwdev, REG_AMPDU_MAX_LENGTH, (8192 << factor) - 1);
+}
+
+static void rtw8814a_false_alarm_statistics(struct rtw_dev *rtwdev)
+{
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+	u32 cck_fa_cnt, ofdm_fa_cnt;
+	u32 crc32_cnt, cca32_cnt;
+	u32 cck_enable;
+
+	cck_enable = rtw_read32(rtwdev, REG_RXPSEL) & BIT(28);
+	cck_fa_cnt = rtw_read16(rtwdev, REG_FA_CCK);
+	ofdm_fa_cnt = rtw_read16(rtwdev, REG_FA_OFDM);
+
+	dm_info->cck_fa_cnt = cck_fa_cnt;
+	dm_info->ofdm_fa_cnt = ofdm_fa_cnt;
+	dm_info->total_fa_cnt = ofdm_fa_cnt;
+	if (cck_enable)
+		dm_info->total_fa_cnt += cck_fa_cnt;
+
+	crc32_cnt = rtw_read32(rtwdev, REG_CRC_CCK);
+	dm_info->cck_ok_cnt = u32_get_bits(crc32_cnt, MASKLWORD);
+	dm_info->cck_err_cnt = u32_get_bits(crc32_cnt, MASKHWORD);
+
+	crc32_cnt = rtw_read32(rtwdev, REG_CRC_OFDM);
+	dm_info->ofdm_ok_cnt = u32_get_bits(crc32_cnt, MASKLWORD);
+	dm_info->ofdm_err_cnt = u32_get_bits(crc32_cnt, MASKHWORD);
+
+	crc32_cnt = rtw_read32(rtwdev, REG_CRC_HT);
+	dm_info->ht_ok_cnt = u32_get_bits(crc32_cnt, MASKLWORD);
+	dm_info->ht_err_cnt = u32_get_bits(crc32_cnt, MASKHWORD);
+
+	crc32_cnt = rtw_read32(rtwdev, REG_CRC_VHT);
+	dm_info->vht_ok_cnt = u32_get_bits(crc32_cnt, MASKLWORD);
+	dm_info->vht_err_cnt = u32_get_bits(crc32_cnt, MASKHWORD);
+
+	cca32_cnt = rtw_read32(rtwdev, REG_CCA_OFDM);
+	dm_info->ofdm_cca_cnt = u32_get_bits(cca32_cnt, MASKHWORD);
+	dm_info->total_cca_cnt = dm_info->ofdm_cca_cnt;
+	if (cck_enable) {
+		cca32_cnt = rtw_read32(rtwdev, REG_CCA_CCK);
+		dm_info->cck_cca_cnt = u32_get_bits(cca32_cnt, MASKLWORD);
+		dm_info->total_cca_cnt += dm_info->cck_cca_cnt;
+	}
+
+	rtw_write32_set(rtwdev, REG_FAS, BIT(17));
+	rtw_write32_clr(rtwdev, REG_FAS, BIT(17));
+	rtw_write32_clr(rtwdev, REG_CCK0_FAREPORT, BIT(15));
+	rtw_write32_set(rtwdev, REG_CCK0_FAREPORT, BIT(15));
+	rtw_write32_set(rtwdev, REG_CNTRST, BIT(0));
+	rtw_write32_clr(rtwdev, REG_CNTRST, BIT(0));
+}
+
+#define MAC_REG_NUM_8814 2
+#define BB_REG_NUM_8814 14
+#define RF_REG_NUM_8814 1
+
+static void rtw8814a_iqk_backup_mac_bb(struct rtw_dev *rtwdev,
+				       u32 *mac_backup, u32 *bb_backup,
+				       const u32 *mac_regs,
+				       const u32 *bb_regs)
+{
+	u32 i;
+
+	/* save MACBB default value */
+	for (i = 0; i < MAC_REG_NUM_8814; i++)
+		mac_backup[i] = rtw_read32(rtwdev, mac_regs[i]);
+
+	for (i = 0; i < BB_REG_NUM_8814; i++)
+		bb_backup[i] = rtw_read32(rtwdev, bb_regs[i]);
+}
+
+static void rtw8814a_iqk_backup_rf(struct rtw_dev *rtwdev,
+				   u32 rf_backup[][4], const u32 *rf_regs)
+{
+	u32 i;
+
+	/* Save RF Parameters */
+	for (i = 0; i < RF_REG_NUM_8814; i++) {
+		rf_backup[i][RF_PATH_A] = rtw_read_rf(rtwdev, RF_PATH_A,
+						      rf_regs[i], RFREG_MASK);
+		rf_backup[i][RF_PATH_B] = rtw_read_rf(rtwdev, RF_PATH_B,
+						      rf_regs[i], RFREG_MASK);
+		rf_backup[i][RF_PATH_C] = rtw_read_rf(rtwdev, RF_PATH_C,
+						      rf_regs[i], RFREG_MASK);
+		rf_backup[i][RF_PATH_D] = rtw_read_rf(rtwdev, RF_PATH_D,
+						      rf_regs[i], RFREG_MASK);
+	}
+}
+
+static void rtw8814a_iqk_afe_setting(struct rtw_dev *rtwdev, bool do_iqk)
+{
+	if (do_iqk) {
+		/* IQK AFE setting RX_WAIT_CCA mode */
+		rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x0e808003);
+		rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x0e808003);
+		rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x0e808003);
+		rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x0e808003);
+	} else {
+		rtw_write32(rtwdev, REG_AFE_PWR1_A, 0x07808003);
+		rtw_write32(rtwdev, REG_AFE_PWR1_B, 0x07808003);
+		rtw_write32(rtwdev, REG_AFE_PWR1_C, 0x07808003);
+		rtw_write32(rtwdev, REG_AFE_PWR1_D, 0x07808003);
+	}
+
+	rtw_write32_mask(rtwdev, REG_DAC_RSTB, BIT(13), 0x1);
+
+	rtw_write8_set(rtwdev, REG_GNT_BT, BIT(2) | BIT(1));
+	rtw_write8_clr(rtwdev, REG_GNT_BT, BIT(2) | BIT(1));
+
+	rtw_write32_set(rtwdev, REG_CCK_RPT_FORMAT, BIT(2));
+	rtw_write32_clr(rtwdev, REG_CCK_RPT_FORMAT, BIT(2));
+}
+
+static void rtw8814a_iqk_restore_mac_bb(struct rtw_dev *rtwdev,
+					u32 *mac_backup, u32 *bb_backup,
+					const u32 *mac_regs,
+					const u32 *bb_regs)
+{
+	u32 i;
+
+	/* Reload MacBB Parameters */
+	for (i = 0; i < MAC_REG_NUM_8814; i++)
+		rtw_write32(rtwdev, mac_regs[i], mac_backup[i]);
+
+	for (i = 0; i < BB_REG_NUM_8814; i++)
+		rtw_write32(rtwdev, bb_regs[i], bb_backup[i]);
+}
+
+static void rtw8814a_iqk_restore_rf(struct rtw_dev *rtwdev,
+				    const u32 rf_backup[][4],
+				    const u32 *rf_regs)
+{
+	u32 i;
+
+	rtw_write_rf(rtwdev, RF_PATH_A, RF_LUTWE, RFREG_MASK, 0x0);
+	rtw_write_rf(rtwdev, RF_PATH_B, RF_LUTWE, RFREG_MASK, 0x0);
+	rtw_write_rf(rtwdev, RF_PATH_C, RF_LUTWE, RFREG_MASK, 0x0);
+	rtw_write_rf(rtwdev, RF_PATH_D, RF_LUTWE, RFREG_MASK, 0x0);
+
+	rtw_write_rf(rtwdev, RF_PATH_A, RF_RXBB2, RFREG_MASK, 0x88001);
+	rtw_write_rf(rtwdev, RF_PATH_B, RF_RXBB2, RFREG_MASK, 0x88001);
+	rtw_write_rf(rtwdev, RF_PATH_C, RF_RXBB2, RFREG_MASK, 0x88001);
+	rtw_write_rf(rtwdev, RF_PATH_D, RF_RXBB2, RFREG_MASK, 0x88001);
+
+	for (i = 0; i < RF_REG_NUM_8814; i++) {
+		rtw_write_rf(rtwdev, RF_PATH_A, rf_regs[i],
+			     RFREG_MASK, rf_backup[i][RF_PATH_A]);
+		rtw_write_rf(rtwdev, RF_PATH_B, rf_regs[i],
+			     RFREG_MASK, rf_backup[i][RF_PATH_B]);
+		rtw_write_rf(rtwdev, RF_PATH_C, rf_regs[i],
+			     RFREG_MASK, rf_backup[i][RF_PATH_C]);
+		rtw_write_rf(rtwdev, RF_PATH_D, rf_regs[i],
+			     RFREG_MASK, rf_backup[i][RF_PATH_D]);
+	}
+}
+
+static void rtw8814a_iqk_reset_nctl(struct rtw_dev *rtwdev)
+{
+	rtw_write32(rtwdev, 0x1b00, 0xf8000000);
+	rtw_write32(rtwdev, 0x1b80, 0x00000006);
+
+	rtw_write32(rtwdev, 0x1b00, 0xf8000000);
+	rtw_write32(rtwdev, 0x1b80, 0x00000002);
+}
+
+static void rtw8814a_iqk_configure_mac(struct rtw_dev *rtwdev)
+{
+	rtw_write8(rtwdev, REG_TXPAUSE, 0x3f);
+	rtw_write32_clr(rtwdev, REG_BCN_CTRL,
+			(BIT_EN_BCN_FUNCTION << 8) | BIT_EN_BCN_FUNCTION);
+
+	/* RX ante off */
+	rtw_write8(rtwdev, REG_RXPSEL, 0x00);
+	/* CCA off */
+	rtw_write32_mask(rtwdev, REG_CCA2ND, 0xf, 0xe);
+	/* CCK RX path off */
+	rtw_write32_set(rtwdev, REG_PRECTRL, BIT_IQ_WGT);
+	rtw_write32(rtwdev, REG_RFE_PINMUX_A, 0x77777777);
+	rtw_write32(rtwdev, REG_RFE_PINMUX_B, 0x77777777);
+	rtw_write32(rtwdev, REG_RFE_PINMUX_C, 0x77777777);
+	rtw_write32(rtwdev, REG_RFE_PINMUX_D, 0x77777777);
+	rtw_write32_mask(rtwdev, REG_RFE_INVSEL_D, BIT_RFE_SELSW0_D, 0x77);
+	rtw_write32_mask(rtwdev, REG_PSD, BIT_PSD_INI, 0x0);
+
+	rtw_write32_mask(rtwdev, REG_RFE_INV0, 0xf, 0x0);
+}
+
+static void rtw8814a_lok_one_shot(struct rtw_dev *rtwdev, u8 path)
+{
+	u32 lok_temp1, lok_temp2;
+	bool lok_ready;
+	u8 ii;
+
+	/* ADC Clock source */
+	rtw_write32_mask(rtwdev, REG_FAS, BIT(21) | BIT(20), path);
+	/* LOK: CMD ID = 0
+	 * {0xf8000011, 0xf8000021, 0xf8000041, 0xf8000081}
+	 */
+	rtw_write32(rtwdev, 0x1b00, 0xf8000001 | (BIT(path) << 4));
+
+	usleep_range(1000, 1100);
+
+	if (read_poll_timeout(!rtw_read32_mask, lok_ready, lok_ready,
+			      1000, 10000, false,
+			      rtwdev, 0x1b00, BIT(0))) {
+		rtw_dbg(rtwdev, RTW_DBG_RFK, "==>S%d LOK timed out\n", path);
+
+		rtw8814a_iqk_reset_nctl(rtwdev);
+
+		rtw_write_rf(rtwdev, path, RF_DTXLOK, RFREG_MASK, 0x08400);
+
+		return;
+	}
+
+	rtw_write32(rtwdev, 0x1b00, 0xf8000000 | (path << 1));
+	rtw_write32(rtwdev, 0x1bd4, 0x003f0001);
+
+	lok_temp2 = rtw_read32_mask(rtwdev, 0x1bfc, 0x003e0000);
+	lok_temp2 = (lok_temp2 + 0x10) & 0x1f;
+
+	lok_temp1 = rtw_read32_mask(rtwdev, 0x1bfc, 0x0000003e);
+	lok_temp1 = (lok_temp1 + 0x10) & 0x1f;
+
+	for (ii = 1; ii < 5; ii++) {
+		lok_temp1 += (lok_temp1 & BIT(4 - ii)) << (ii * 2);
+		lok_temp2 += (lok_temp2 & BIT(4 - ii)) << (ii * 2);
+	}
+
+	rtw_dbg(rtwdev, RTW_DBG_RFK,
+		"path %d lok_temp1 = %#x, lok_temp2 = %#x\n",
+		path, lok_temp1 >> 4, lok_temp2 >> 4);
+
+	rtw_write_rf(rtwdev, path, RF_DTXLOK, 0x07c00, lok_temp1 >> 4);
+	rtw_write_rf(rtwdev, path, RF_DTXLOK, 0xf8000, lok_temp2 >> 4);
+}
+
+static void rtw8814a_iqk_tx_one_shot(struct rtw_dev *rtwdev, u8 path,
+				     u32 *tx_matrix, bool *tx_ok)
+{
+	u8 bw = rtwdev->hal.current_band_width;
+	u8 cal_retry;
+	u32 iqk_cmd;
+
+	for (cal_retry = 0; cal_retry < 4; cal_retry++) {
+		rtw_write32_mask(rtwdev, REG_FAS, BIT(21) | BIT(20), path);
+
+		iqk_cmd = 0xf8000001 | ((bw + 3) << 8) | (BIT(path) << 4);
+
+		rtw_dbg(rtwdev, RTW_DBG_RFK, "TXK_Trigger = %#x\n", iqk_cmd);
+
+		rtw_write32(rtwdev, 0x1b00, iqk_cmd);
+
+		usleep_range(10000, 11000);
+
+		if (read_poll_timeout(!rtw_read32_mask, *tx_ok, *tx_ok,
+				      1000, 20000, false,
+				      rtwdev, 0x1b00, BIT(0))) {
+			rtw_dbg(rtwdev, RTW_DBG_RFK,
+				"tx iqk S%d timed out\n", path);
+
+			rtw8814a_iqk_reset_nctl(rtwdev);
+		} else {
+			*tx_ok = !rtw_read32_mask(rtwdev, 0x1b08, BIT(26));
+
+			if (*tx_ok)
+				break;
+		}
+	}
+
+	rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d tx ==> 0x1b00 = 0x%x\n",
+		path, rtw_read32(rtwdev, 0x1b00));
+	rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d tx ==> 0x1b08 = 0x%x\n",
+		path, rtw_read32(rtwdev, 0x1b08));
+	rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d tx ==> cal_retry = %x\n",
+		path, cal_retry);
+
+	rtw_write32(rtwdev, 0x1b00, 0xf8000000 | (path << 1));
+
+	if (*tx_ok) {
+		*tx_matrix = rtw_read32(rtwdev, 0x1b38);
+
+		rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d_IQC = 0x%x\n",
+			path, *tx_matrix);
+	}
+}
+
+static void rtw8814a_iqk_rx_one_shot(struct rtw_dev *rtwdev, u8 path,
+				     u32 *tx_matrix, bool *tx_ok)
+{
+	static const u16 iqk_apply[RTW_RF_PATH_MAX] = {
+		REG_TXAGCIDX, REG_TX_AGC_B, REG_TX_AGC_C, REG_TX_AGC_D
+	};
+	u8 band = rtwdev->hal.current_band_type;
+	u8 bw = rtwdev->hal.current_band_width;
+	u32 rx_matrix;
+	u8 cal_retry;
+	u32 iqk_cmd;
+	bool rx_ok;
+
+	for (cal_retry = 0; cal_retry < 4; cal_retry++) {
+		rtw_write32_mask(rtwdev, REG_FAS, BIT(21) | BIT(20), path);
+
+		if (band == RTW_BAND_2G) {
+			rtw_write_rf(rtwdev, path, RF_LUTDBG, BIT(11), 0x1);
+			rtw_write_rf(rtwdev, path, RF_GAINTX, 0xfffff, 0x51ce1);
+
+			switch (path) {
+			case 0:
+			case 1:
+				rtw_write32(rtwdev, REG_RFE_PINMUX_B,
+					    0x54775477);
+				break;
+			case 2:
+				rtw_write32(rtwdev, REG_RFE_PINMUX_C,
+					    0x54775477);
+				break;
+			case 3:
+				rtw_write32(rtwdev, REG_RFE_INVSEL_D, 0x75400000);
+				rtw_write32(rtwdev, REG_RFE_PINMUX_D,
+					    0x77777777);
+				break;
+			}
+		}
+
+		iqk_cmd = 0xf8000001 | ((9 - bw) << 8) | (BIT(path) << 4);
+
+		rtw_dbg(rtwdev, RTW_DBG_RFK, "RXK_Trigger = 0x%x\n", iqk_cmd);
+
+		rtw_write32(rtwdev, 0x1b00, iqk_cmd);
+
+		usleep_range(10000, 11000);
+
+		if (read_poll_timeout(!rtw_read32_mask, rx_ok, rx_ok,
+				      1000, 20000, false,
+				      rtwdev, 0x1b00, BIT(0))) {
+			rtw_dbg(rtwdev, RTW_DBG_RFK,
+				"rx iqk S%d timed out\n", path);
+
+			rtw8814a_iqk_reset_nctl(rtwdev);
+		} else {
+			rx_ok = !rtw_read32_mask(rtwdev, 0x1b08, BIT(26));
+
+			if (rx_ok)
+				break;
+		}
+	}
+
+	rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d rx ==> 0x1b00 = 0x%x\n",
+		path, rtw_read32(rtwdev, 0x1b00));
+	rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d rx ==> 0x1b08 = 0x%x\n",
+		path, rtw_read32(rtwdev, 0x1b08));
+	rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d rx ==> cal_retry = %x\n",
+		path, cal_retry);
+
+	rtw_write32(rtwdev, 0x1b00, 0xf8000000 | (path << 1));
+
+	if (rx_ok) {
+		rtw_write32(rtwdev, 0x1b3c, 0x20000000);
+		rx_matrix = rtw_read32(rtwdev, 0x1b3c);
+
+		rtw_dbg(rtwdev, RTW_DBG_RFK, "S%d_IQC = 0x%x\n",
+			path, rx_matrix);
+	}
+
+	if (*tx_ok)
+		rtw_write32(rtwdev, 0x1b38, *tx_matrix);
+	else
+		rtw_write32_mask(rtwdev, iqk_apply[path], BIT(0), 0x0);
+
+	if (!rx_ok)
+		rtw_write32_mask(rtwdev, iqk_apply[path],
+				 BIT(11) | BIT(10), 0x0);
+
+	if (band == RTW_BAND_2G)
+		rtw_write_rf(rtwdev, path, RF_LUTDBG, BIT(11), 0x0);
+}
+
+static void rtw8814a_iqk(struct rtw_dev *rtwdev)
+{
+	u8 band = rtwdev->hal.current_band_type;
+	u8 bw = rtwdev->hal.current_band_width;
+	u32 tx_matrix[RTW_RF_PATH_MAX];
+	bool tx_ok[RTW_RF_PATH_MAX];
+	u8 path;
+
+	rtw_dbg(rtwdev, RTW_DBG_RFK, "IQK band = %d GHz bw = %d MHz\n",
+		band == RTW_BAND_2G ? 2 : 5, (1 << (bw + 1)) * 10);
+
+	rtw_write_rf(rtwdev, RF_PATH_A, RF_TXMOD, BIT(19), 0x1);
+	rtw_write_rf(rtwdev, RF_PATH_B, RF_TXMOD, BIT(19), 0x1);
+	rtw_write_rf(rtwdev, RF_PATH_C, RF_TXMOD, BIT(19), 0x1);
+	rtw_write_rf(rtwdev, RF_PATH_D, RF_TXMOD, BIT(19), 0x1);
+
+	rtw_write32_mask(rtwdev, REG_TXAGCIDX,
+			 (BIT(11) | BIT(10) | BIT(0)), 0x401);
+	rtw_write32_mask(rtwdev, REG_TX_AGC_B,
+			 (BIT(11) | BIT(10) | BIT(0)), 0x401);
+	rtw_write32_mask(rtwdev, REG_TX_AGC_C,
+			 (BIT(11) | BIT(10) | BIT(0)), 0x401);
+	rtw_write32_mask(rtwdev, REG_TX_AGC_D,
+			 (BIT(11) | BIT(10) | BIT(0)), 0x401);
+
+	if (band == RTW_BAND_5G)
+		rtw_write32(rtwdev, 0x1b00, 0xf8000ff1);
+	else
+		rtw_write32(rtwdev, 0x1b00, 0xf8000ef1);
+
+	usleep_range(1000, 1100);
+
+	rtw_write32(rtwdev, 0x810, 0x20101063);
+	rtw_write32(rtwdev, REG_DAC_RSTB, 0x0B00C000);
+
+	for (path = RF_PATH_A; path < RTW_RF_PATH_MAX; path++)
+		rtw8814a_lok_one_shot(rtwdev, path);
+
+	for (path = RF_PATH_A; path < RTW_RF_PATH_MAX; path++)
+		rtw8814a_iqk_tx_one_shot(rtwdev, path,
+					 &tx_matrix[path], &tx_ok[path]);
+
+	for (path = RF_PATH_A; path < RTW_RF_PATH_MAX; path++)
+		rtw8814a_iqk_rx_one_shot(rtwdev, path,
+					 &tx_matrix[path], &tx_ok[path]);
+}
+
+static void rtw8814a_do_iqk(struct rtw_dev *rtwdev)
+{
+	static const u32 backup_mac_reg[MAC_REG_NUM_8814] = {0x520, 0x550};
+	static const u32 backup_bb_reg[BB_REG_NUM_8814] = {
+		0xa14, 0x808, 0x838, 0x90c, 0x810, 0xcb0, 0xeb0,
+		0x18b4, 0x1ab4, 0x1abc, 0x9a4, 0x764, 0xcbc, 0x910
+	};
+	static const u32 backup_rf_reg[RF_REG_NUM_8814] = {0x0};
+	u32 rf_backup[RF_REG_NUM_8814][RTW_RF_PATH_MAX];
+	u32 mac_backup[MAC_REG_NUM_8814];
+	u32 bb_backup[BB_REG_NUM_8814];
+
+	rtw8814a_iqk_backup_mac_bb(rtwdev, mac_backup, bb_backup,
+				   backup_mac_reg, backup_bb_reg);
+	rtw8814a_iqk_afe_setting(rtwdev, true);
+	rtw8814a_iqk_backup_rf(rtwdev, rf_backup, backup_rf_reg);
+	rtw8814a_iqk_configure_mac(rtwdev);
+	rtw8814a_iqk(rtwdev);
+	rtw8814a_iqk_reset_nctl(rtwdev); /* for 3-wire to BB use */
+	rtw8814a_iqk_afe_setting(rtwdev, false);
+	rtw8814a_iqk_restore_mac_bb(rtwdev, mac_backup, bb_backup,
+				    backup_mac_reg, backup_bb_reg);
+	rtw8814a_iqk_restore_rf(rtwdev, rf_backup, backup_rf_reg);
+}
+
+static void rtw8814a_phy_calibration(struct rtw_dev *rtwdev)
+{
+	rtw8814a_do_iqk(rtwdev);
+}
+
+static void rtw8814a_coex_cfg_init(struct rtw_dev *rtwdev)
+{
+}
+
+static void rtw8814a_coex_cfg_ant_switch(struct rtw_dev *rtwdev, u8 ctrl_type,
+					 u8 pos_type)
+{
+	/* Override rtw_coex_coex_ctrl_owner(). RF path C does not
+	 * function when BIT_LTE_MUX_CTRL_PATH is set.
+	 */
+	rtw_write8_clr(rtwdev, REG_SYS_SDIO_CTRL + 3,
+		       BIT_LTE_MUX_CTRL_PATH >> 24);
+}
+
+static void rtw8814a_coex_cfg_gnt_fix(struct rtw_dev *rtwdev)
+{
+}
+
+static void rtw8814a_coex_cfg_gnt_debug(struct rtw_dev *rtwdev)
+{
+}
+
+static void rtw8814a_coex_cfg_rfe_type(struct rtw_dev *rtwdev)
+{
+	struct rtw_coex *coex = &rtwdev->coex;
+	struct rtw_coex_rfe *coex_rfe = &coex->rfe;
+
+	/* Only needed to make rtw8814a_coex_cfg_ant_switch() run. */
+	coex_rfe->ant_switch_exist = true;
+}
+
+static void rtw8814a_coex_cfg_wl_tx_power(struct rtw_dev *rtwdev, u8 wl_pwr)
+{
+}
+
+static void rtw8814a_coex_cfg_wl_rx_gain(struct rtw_dev *rtwdev, bool low_gain)
+{
+}
+
+static void rtw8814a_txagc_swing_offset(struct rtw_dev *rtwdev, u8 path,
+					u8 tx_pwr_idx_offset,
+					s8 *txagc_idx, u8 *swing_idx)
+{
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+	u8 swing_upper_bound = dm_info->default_ofdm_index + 10;
+	s8 delta_pwr_idx = dm_info->delta_power_index[path];
+	u8 swing_index = dm_info->default_ofdm_index;
+	u8 max_tx_pwr_idx_offset = 0xf;
+	u8 swing_lower_bound = 0;
+	s8 agc_index = 0;
+
+	tx_pwr_idx_offset = min_t(u8, tx_pwr_idx_offset, max_tx_pwr_idx_offset);
+
+	if (delta_pwr_idx >= 0) {
+		if (delta_pwr_idx <= tx_pwr_idx_offset) {
+			agc_index = delta_pwr_idx;
+			swing_index = dm_info->default_ofdm_index;
+		} else if (delta_pwr_idx > tx_pwr_idx_offset) {
+			agc_index = tx_pwr_idx_offset;
+			swing_index = dm_info->default_ofdm_index +
+					delta_pwr_idx - tx_pwr_idx_offset;
+			swing_index = min_t(u8, swing_index, swing_upper_bound);
+		}
+	} else {
+		if (dm_info->default_ofdm_index > abs(delta_pwr_idx))
+			swing_index =
+				dm_info->default_ofdm_index + delta_pwr_idx;
+		else
+			swing_index = swing_lower_bound;
+		swing_index = max_t(u8, swing_index, swing_lower_bound);
+
+		agc_index = 0;
+	}
+
+	if (swing_index >= RTW_TXSCALE_SIZE) {
+		rtw_warn(rtwdev, "swing index overflow\n");
+		swing_index = RTW_TXSCALE_SIZE - 1;
+	}
+	*txagc_idx = agc_index;
+	*swing_idx = swing_index;
+}
+
+static void rtw8814a_pwrtrack_set_pwr(struct rtw_dev *rtwdev, u8 path,
+				      u8 pwr_idx_offset)
+{
+	static const u32 txagc_reg[RTW_RF_PATH_MAX] = {
+		REG_TX_AGC_A, REG_TX_AGC_B, REG_TX_AGC_C, REG_TX_AGC_D
+	};
+	static const u32 txscale_reg[RTW_RF_PATH_MAX] = {
+		REG_TXSCALE_A, REG_TXSCALE_B, REG_TXSCALE_C, REG_TXSCALE_D
+	};
+	s8 txagc_idx;
+	u8 swing_idx;
+
+	rtw8814a_txagc_swing_offset(rtwdev, path, pwr_idx_offset,
+				    &txagc_idx, &swing_idx);
+	rtw_write32_mask(rtwdev, txagc_reg[path], GENMASK(29, 25),
+			 txagc_idx);
+	rtw_write32_mask(rtwdev, txscale_reg[path], BB_SWING_MASK,
+			 rtw8814a_txscale_tbl[swing_idx]);
+}
+
+static void rtw8814a_pwrtrack_set(struct rtw_dev *rtwdev, u8 path)
+{
+	u8 max_pwr_idx = rtwdev->chip->max_power_index;
+	u8 band_width = rtwdev->hal.current_band_width;
+	u8 channel = rtwdev->hal.current_channel;
+	u8 tx_rate = rtwdev->dm_info.tx_rate;
+	u8 regd = rtw_regd_get(rtwdev);
+	u8 pwr_idx_offset, tx_pwr_idx;
+
+	tx_pwr_idx = rtw_phy_get_tx_power_index(rtwdev, path, tx_rate,
+						band_width, channel, regd);
+
+	tx_pwr_idx = min_t(u8, tx_pwr_idx, max_pwr_idx);
+
+	pwr_idx_offset = max_pwr_idx - tx_pwr_idx;
+
+	rtw8814a_pwrtrack_set_pwr(rtwdev, path, pwr_idx_offset);
+}
+
+static void rtw8814a_phy_pwrtrack_path(struct rtw_dev *rtwdev,
+				       struct rtw_swing_table *swing_table,
+				       u8 path)
+{
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+	u8 power_idx_cur, power_idx_last;
+	u8 delta;
+
+	/* 8814A only has one thermal meter at PATH A */
+	delta = rtw_phy_pwrtrack_get_delta(rtwdev, RF_PATH_A);
+
+	power_idx_last = dm_info->delta_power_index[path];
+	power_idx_cur = rtw_phy_pwrtrack_get_pwridx(rtwdev, swing_table,
+						    path, RF_PATH_A, delta);
+
+	/* if delta of power indexes are the same, just skip */
+	if (power_idx_cur == power_idx_last)
+		return;
+
+	dm_info->delta_power_index[path] = power_idx_cur;
+	rtw8814a_pwrtrack_set(rtwdev, path);
+}
+
+static void rtw8814a_phy_pwrtrack(struct rtw_dev *rtwdev)
+{
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+	struct rtw_swing_table swing_table;
+	u8 thermal_value, path;
+
+	rtw_phy_config_swing_table(rtwdev, &swing_table);
+
+	if (rtwdev->efuse.thermal_meter[RF_PATH_A] == 0xff)
+		return;
+
+	thermal_value = rtw_read_rf(rtwdev, RF_PATH_A, RF_T_METER, 0xfc00);
+
+	rtw_phy_pwrtrack_avg(rtwdev, thermal_value, RF_PATH_A);
+
+	if (dm_info->pwr_trk_init_trigger)
+		dm_info->pwr_trk_init_trigger = false;
+	else if (!rtw_phy_pwrtrack_thermal_changed(rtwdev, thermal_value,
+						   RF_PATH_A))
+		goto iqk;
+
+	for (path = RF_PATH_A; path < rtwdev->hal.rf_path_num; path++)
+		rtw8814a_phy_pwrtrack_path(rtwdev, &swing_table, path);
+
+iqk:
+	if (rtw_phy_pwrtrack_need_iqk(rtwdev))
+		rtw8814a_do_iqk(rtwdev);
+}
+
+static void rtw8814a_pwr_track(struct rtw_dev *rtwdev)
+{
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+
+	if (!dm_info->pwr_trk_triggered) {
+		rtw_write_rf(rtwdev, RF_PATH_A, RF_T_METER,
+			     GENMASK(17, 16), 0x03);
+		dm_info->pwr_trk_triggered = true;
+		return;
+	}
+
+	rtw8814a_phy_pwrtrack(rtwdev);
+	dm_info->pwr_trk_triggered = false;
+}
+
+static void rtw8814a_phy_cck_pd_set(struct rtw_dev *rtwdev, u8 new_lvl)
+{
+	static const u8 pd[CCK_PD_LV_MAX] = {0x40, 0x83, 0xcd, 0xdd, 0xed};
+	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
+
+	/* Override rtw_phy_cck_pd_lv_link(). It implements something
+	 * like type 2/3/4. We need type 1 here.
+	 */
+	if (rtw_is_assoc(rtwdev)) {
+		if (dm_info->min_rssi > 60) {
+			new_lvl = CCK_PD_LV3;
+		} else if (dm_info->min_rssi > 35) {
+			new_lvl = CCK_PD_LV2;
+		} else if (dm_info->min_rssi > 20) {
+			if (dm_info->cck_fa_avg > 500)
+				new_lvl = CCK_PD_LV2;
+			else if (dm_info->cck_fa_avg < 250)
+				new_lvl = CCK_PD_LV1;
+			else
+				return;
+		} else {
+			new_lvl = CCK_PD_LV1;
+		}
+	}
+
+	rtw_dbg(rtwdev, RTW_DBG_PHY, "lv: (%d) -> (%d)\n",
+		dm_info->cck_pd_lv[RTW_CHANNEL_WIDTH_20][RF_PATH_A], new_lvl);
+
+	if (dm_info->cck_pd_lv[RTW_CHANNEL_WIDTH_20][RF_PATH_A] == new_lvl)
+		return;
+
+	dm_info->cck_fa_avg = CCK_FA_AVG_RESET;
+	dm_info->cck_pd_lv[RTW_CHANNEL_WIDTH_20][RF_PATH_A] = new_lvl;
+
+	rtw_write8(rtwdev, REG_CCK_PD_TH, pd[new_lvl]);
+}
+
+static void rtw8814a_led_set(struct led_classdev *led,
+			     enum led_brightness brightness)
+{
+	struct rtw_dev *rtwdev = container_of(led, struct rtw_dev, led_cdev);
+	u32 led_gpio_cfg;
+
+	led_gpio_cfg = rtw_read32(rtwdev, REG_GPIO_PIN_CTRL_2);
+	led_gpio_cfg |= BIT(16) | BIT(17) | BIT(21) | BIT(22);
+
+	if (brightness == LED_OFF) {
+		led_gpio_cfg |= BIT(8) | BIT(9) | BIT(13) | BIT(14);
+	} else {
+		led_gpio_cfg &= ~(BIT(8) | BIT(9) | BIT(13) | BIT(14));
+		led_gpio_cfg &= ~(BIT(0) | BIT(1) | BIT(5) | BIT(6));
+	}
+
+	rtw_write32(rtwdev, REG_GPIO_PIN_CTRL_2, led_gpio_cfg);
+}
+
+static void rtw8814a_fill_txdesc_checksum(struct rtw_dev *rtwdev,
+					  struct rtw_tx_pkt_info *pkt_info,
+					  u8 *txdesc)
+{
+	size_t words = 32 / 2; /* calculate the first 32 bytes (16 words) */
+
+	fill_txdesc_checksum_common(txdesc, words);
+}
+
+static const struct rtw_chip_ops rtw8814a_ops = {
+	.power_on		= rtw_power_on,
+	.power_off		= rtw_power_off,
+	.phy_set_param		= rtw8814a_phy_set_param,
+	.read_efuse		= rtw8814a_read_efuse,
+	.query_phy_status	= rtw8814a_query_phy_status,
+	.set_channel		= rtw8814a_set_channel,
+	.mac_init		= rtw8814a_mac_init,
+	.mac_postinit		= NULL,
+	.read_rf		= rtw_phy_read_rf,
+	.write_rf		= rtw_phy_write_rf_reg_sipi,
+	.set_tx_power_index	= rtw8814a_set_tx_power_index,
+	.set_antenna		= NULL,
+	.cfg_ldo25		= rtw8814a_cfg_ldo25,
+	.efuse_grant		= rtw8814a_efuse_grant,
+	.set_ampdu_factor	= rtw8814a_set_ampdu_factor,
+	.false_alarm_statistics	= rtw8814a_false_alarm_statistics,
+	.phy_calibration	= rtw8814a_phy_calibration,
+	.cck_pd_set		= rtw8814a_phy_cck_pd_set,
+	.pwr_track		= rtw8814a_pwr_track,
+	.config_bfee		= NULL,
+	.set_gid_table		= NULL,
+	.cfg_csi_rate		= NULL,
+	.led_set		= rtw8814a_led_set,
+	.fill_txdesc_checksum	= rtw8814a_fill_txdesc_checksum,
+
+	.coex_set_init		= rtw8814a_coex_cfg_init,
+	.coex_set_ant_switch	= rtw8814a_coex_cfg_ant_switch,
+	.coex_set_gnt_fix	= rtw8814a_coex_cfg_gnt_fix,
+	.coex_set_gnt_debug	= rtw8814a_coex_cfg_gnt_debug,
+	.coex_set_rfe_type	= rtw8814a_coex_cfg_rfe_type,
+	.coex_set_wl_tx_power	= rtw8814a_coex_cfg_wl_tx_power,
+	.coex_set_wl_rx_gain	= rtw8814a_coex_cfg_wl_rx_gain,
+};
+
+static const struct rtw_rqpn rqpn_table_8814a[] = {
+	/* SDIO */
+	{RTW_DMA_MAPPING_NORMAL, RTW_DMA_MAPPING_NORMAL, /* vo vi */
+	 RTW_DMA_MAPPING_LOW, RTW_DMA_MAPPING_LOW,	 /* be bk */
+	 RTW_DMA_MAPPING_EXTRA, RTW_DMA_MAPPING_HIGH},	 /* mg hi */
+	/* PCIE */
+	{RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_NORMAL,
+	 RTW_DMA_MAPPING_LOW, RTW_DMA_MAPPING_LOW,
+	 RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_HIGH},
+	/* USB, 2 bulk out */
+	{RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_HIGH,
+	 RTW_DMA_MAPPING_NORMAL, RTW_DMA_MAPPING_NORMAL,
+	 RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_HIGH},
+	/* USB, 3 bulk out */
+	{RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_NORMAL,
+	 RTW_DMA_MAPPING_LOW, RTW_DMA_MAPPING_LOW,
+	 RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_HIGH},
+	/* USB, 4 bulk out */
+	{RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_NORMAL,
+	 RTW_DMA_MAPPING_LOW, RTW_DMA_MAPPING_LOW,
+	 RTW_DMA_MAPPING_HIGH, RTW_DMA_MAPPING_HIGH},
+};
+
+static const struct rtw_prioq_addrs prioq_addrs_8814a = {
+	.prio[RTW_DMA_MAPPING_EXTRA] = {
+		.rsvd = REG_FIFOPAGE_INFO_4, .avail = REG_FIFOPAGE_INFO_4 + 2,
+	},
+	.prio[RTW_DMA_MAPPING_LOW] = {
+		.rsvd = REG_FIFOPAGE_INFO_2, .avail = REG_FIFOPAGE_INFO_2 + 2,
+	},
+	.prio[RTW_DMA_MAPPING_NORMAL] = {
+		.rsvd = REG_FIFOPAGE_INFO_3, .avail = REG_FIFOPAGE_INFO_3 + 2,
+	},
+	.prio[RTW_DMA_MAPPING_HIGH] = {
+		.rsvd = REG_FIFOPAGE_INFO_1, .avail = REG_FIFOPAGE_INFO_1 + 2,
+	},
+	.wsize = true,
+};
+
+static const struct rtw_page_table page_table_8814a[] = {
+	/* SDIO */
+	{0, 0, 0, 0, 0},	/* hq nq lq exq gapq */
+	/* PCIE */
+	{32, 32, 32, 32, 0},
+	/* USB, 2 bulk out */
+	{32, 32, 32, 32, 0},
+	/* USB, 3 bulk out */
+	{32, 32, 32, 32, 0},
+	/* USB, 4 bulk out */
+	{32, 32, 32, 32, 0},
+};
+
+static const struct rtw_intf_phy_para_table phy_para_table_8814a = {};
+
+static const struct rtw_hw_reg rtw8814a_dig[] = {
+	[0] = { .addr = 0xc50, .mask = 0x7f },
+	[1] = { .addr = 0xe50, .mask = 0x7f },
+	[2] = { .addr = 0x1850, .mask = 0x7f },
+	[3] = { .addr = 0x1a50, .mask = 0x7f },
+};
+
+static const struct rtw_rfe_def rtw8814a_rfe_defs[] = {
+	[0] = { .phy_pg_tbl	= &rtw8814a_bb_pg_type0_tbl,
+		.txpwr_lmt_tbl	= &rtw8814a_txpwr_lmt_type0_tbl,
+		.pwr_track_tbl	= &rtw8814a_rtw_pwrtrk_type0_tbl },
+	[1] = { .phy_pg_tbl	= &rtw8814a_bb_pg_tbl,
+		.txpwr_lmt_tbl	= &rtw8814a_txpwr_lmt_type1_tbl,
+		.pwr_track_tbl	= &rtw8814a_rtw_pwrtrk_tbl },
+};
+
+/* rssi in percentage % (dbm = % - 100) */
+static const u8 wl_rssi_step_8814a[] = {60, 50, 44, 30};
+static const u8 bt_rssi_step_8814a[] = {30, 30, 30, 30};
+
+/* wl_tx_dec_power, bt_tx_dec_power, wl_rx_gain, bt_rx_lna_constrain */
+static const struct coex_rf_para rf_para_tx_8814a[] = {
+	{0, 0, false, 7},  /* for normal */
+	{0, 16, false, 7}, /* for WL-CPT */
+	{4, 0, true, 1},
+	{3, 6, true, 1},
+	{2, 9, true, 1},
+	{1, 13, true, 1}
+};
+
+static const struct coex_rf_para rf_para_rx_8814a[] = {
+	{0, 0, false, 7},  /* for normal */
+	{0, 16, false, 7}, /* for WL-CPT */
+	{4, 0, true, 1},
+	{3, 6, true, 1},
+	{2, 9, true, 1},
+	{1, 13, true, 1}
+};
+
+static_assert(ARRAY_SIZE(rf_para_tx_8814a) == ARRAY_SIZE(rf_para_rx_8814a));
+
+const struct rtw_chip_info rtw8814a_hw_spec = {
+	.ops = &rtw8814a_ops,
+	.id = RTW_CHIP_TYPE_8814A,
+	.fw_name = "rtw88/rtw8814a_fw.bin",
+	.wlan_cpu = RTW_WCPU_3081,
+	.tx_pkt_desc_sz = 40,
+	.tx_buf_desc_sz = 16,
+	.rx_pkt_desc_sz = 24,
+	.rx_buf_desc_sz = 8,
+	.phy_efuse_size = 1024,
+	.log_efuse_size = 512,
+	.ptct_efuse_size = 0,
+	.txff_size = (2048 - 10) * TX_PAGE_SIZE,
+	.rxff_size = 23552,
+	.rsvd_drv_pg_num = 8,
+	.band = RTW_BAND_2G | RTW_BAND_5G,
+	.page_size = TX_PAGE_SIZE,
+	.csi_buf_pg_num = 0,
+	.dig_min = 0x1c,
+	.txgi_factor = 1,
+	.is_pwr_by_rate_dec = true,
+	.rx_ldpc = true,
+	.max_power_index = 0x3f,
+	.ampdu_density = IEEE80211_HT_MPDU_DENSITY_2,
+	.amsdu_in_ampdu = false, /* RX speed is better without AMSDU */
+	.usb_tx_agg_desc_num = 3,
+	.hw_feature_report = false,
+	.c2h_ra_report_size = 6,
+	.old_datarate_fb_limit = false,
+	.ht_supported = true,
+	.vht_supported = true,
+	.lps_deep_mode_supported = BIT(LPS_DEEP_MODE_LCLK),
+	.sys_func_en = 0xDC,
+	.pwr_on_seq = card_enable_flow_8814a,
+	.pwr_off_seq = card_disable_flow_8814a,
+	.rqpn_table = rqpn_table_8814a,
+	.prioq_addrs = &prioq_addrs_8814a,
+	.page_table = page_table_8814a,
+	.intf_table = &phy_para_table_8814a,
+	.dig = rtw8814a_dig,
+	.dig_cck = NULL,
+	.rf_base_addr = {0x2800, 0x2c00, 0x3800, 0x3c00},
+	.rf_sipi_addr = {0xc90, 0xe90, 0x1890, 0x1a90},
+	.ltecoex_addr = NULL,
+	.mac_tbl = &rtw8814a_mac_tbl,
+	.agc_tbl = &rtw8814a_agc_tbl,
+	.bb_tbl = &rtw8814a_bb_tbl,
+	.rf_tbl = {&rtw8814a_rf_a_tbl, &rtw8814a_rf_b_tbl,
+		   &rtw8814a_rf_c_tbl, &rtw8814a_rf_d_tbl},
+	.rfe_defs = rtw8814a_rfe_defs,
+	.rfe_defs_size = ARRAY_SIZE(rtw8814a_rfe_defs),
+	.iqk_threshold = 8,
+	.max_scan_ie_len = IEEE80211_MAX_DATA_LEN,
+
+	.coex_para_ver = 0,
+	.bt_desired_ver = 0,
+	.scbd_support = false,
+	.new_scbd10_def = false,
+	.ble_hid_profile_support = false,
+	.wl_mimo_ps_support = false,
+	.pstdma_type = COEX_PSTDMA_FORCE_LPSOFF,
+	.bt_rssi_type = COEX_BTRSSI_RATIO,
+	.ant_isolation = 15,
+	.rssi_tolerance = 2,
+	.wl_rssi_step = wl_rssi_step_8814a,
+	.bt_rssi_step = bt_rssi_step_8814a,
+	.table_sant_num = 0,
+	.table_sant = NULL,
+	.table_nsant_num = 0,
+	.table_nsant = NULL,
+	.tdma_sant_num = 0,
+	.tdma_sant = NULL,
+	.tdma_nsant_num = 0,
+	.tdma_nsant = NULL,
+	.wl_rf_para_num = ARRAY_SIZE(rf_para_tx_8814a),
+	.wl_rf_para_tx = rf_para_tx_8814a,
+	.wl_rf_para_rx = rf_para_rx_8814a,
+	.bt_afh_span_bw20 = 0x24,
+	.bt_afh_span_bw40 = 0x36,
+	.afh_5g_num = 0,
+	.afh_5g = NULL,
+	.coex_info_hw_regs_num = 0,
+	.coex_info_hw_regs = NULL,
+};
+EXPORT_SYMBOL(rtw8814a_hw_spec);
+
+MODULE_FIRMWARE("rtw88/rtw8814a_fw.bin");
+
+MODULE_AUTHOR("Bitterblue Smith <rtl8821cerfe2@gmail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8814a driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sys/contrib/dev/rtw88/rtw8814a.h b/sys/contrib/dev/rtw88/rtw8814a.h
new file mode 100644
index 000000000000..c57c7c8f915e
--- /dev/null
+++ b/sys/contrib/dev/rtw88/rtw8814a.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#ifndef __RTW8814A_H__
+#define __RTW8814A_H__
+
+struct rtw8814au_efuse {
+	u8 vid[2];			/* 0xd0 */
+	u8 pid[2];			/* 0xd2 */
+	u8 res[4];			/* 0xd4 */
+	u8 mac_addr[ETH_ALEN];		/* 0xd8 */
+} __packed;
+
+struct rtw8814ae_efuse {
+	u8 mac_addr[ETH_ALEN];		/* 0xd0 */
+	u8 vid[2];			/* 0xd6 */
+	u8 did[2];			/* 0xd8 */
+	u8 svid[2];			/* 0xda */
+	u8 smid[2];			/* 0xdc */
+} __packed;
+
+struct rtw8814a_efuse {
+	__le16 rtl_id;
+	u8 res0[0x0c];
+	u8 usb_mode;			/* 0x0e */
+	u8 res1;
+
+	/* power index for four RF paths */
+	struct rtw_txpwr_idx txpwr_idx_table[4];
+
+	u8 channel_plan;		/* 0xb8 */
+	u8 xtal_k;			/* 0xb9 */
+	u8 thermal_meter;		/* 0xba */
+	u8 iqk_lck;			/* 0xbb */
+	u8 pa_type;			/* 0xbc */
+	u8 lna_type_2g[2];		/* 0xbd */
+	u8 lna_type_5g[2];		/* 0xbf */
+	u8 rf_board_option;		/* 0xc1 */
+	u8 res2;
+	u8 rf_bt_setting;		/* 0xc3 */
+	u8 eeprom_version;		/* 0xc4 */
+	u8 eeprom_customer_id;		/* 0xc5 */
+	u8 tx_bb_swing_setting_2g;	/* 0xc6 */
+	u8 tx_bb_swing_setting_5g;	/* 0xc7 */
+	u8 res3;
+	u8 trx_antenna_option;		/* 0xc9 */
+	u8 rfe_option;			/* 0xca */
+	u8 country_code[2];		/* 0xcb */
+	u8 res4[3];
+	union {
+		struct rtw8814au_efuse u;
+		struct rtw8814ae_efuse e;
+	};
+	u8 res5[0x122];			/* 0xde */
+} __packed;
+
+static_assert(sizeof(struct rtw8814a_efuse) == 512);
+
+extern const struct rtw_chip_info rtw8814a_hw_spec;
+
+#endif
diff --git a/sys/contrib/dev/rtw88/rtw8814a_table.c b/sys/contrib/dev/rtw88/rtw8814a_table.c
new file mode 100644
index 000000000000..b9ce51a09fc8
--- /dev/null
+++ b/sys/contrib/dev/rtw88/rtw8814a_table.c
@@ -0,0 +1,23930 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#include "main.h"
+#include "phy.h"
+#include "rtw8814a_table.h"
+
+static const u32 rtw8814a_mac[] = {
+	0x010, 0x0000007C,
+	0x014, 0x000000DB,
+	0x016, 0x00000002,
+	0x073, 0x00000010,
+	0x420, 0x00000080,
+	0x421, 0x0000000F,
+	0x428, 0x0000000A,
+	0x429, 0x00000010,
+	0x430, 0x00000000,
+	0x431, 0x00000000,
+	0x432, 0x00000000,
+	0x433, 0x00000001,
+	0x434, 0x00000004,
+	0x435, 0x00000005,
+	0x436, 0x00000007,
+	0x437, 0x00000008,
+	0x43C, 0x00000004,
+	0x43D, 0x00000005,
+	0x43E, 0x00000007,
+	0x43F, 0x00000008,
+	0x440, 0x0000005D,
+	0x441, 0x00000001,
+	0x442, 0x00000000,
+	0x444, 0x00000010,
+	0x445, 0x000000F0,
+	0x446, 0x00000001,
+	0x447, 0x000000FE,
+	0x448, 0x00000000,
+	0x449, 0x00000000,
+	0x44A, 0x00000000,
+	0x44B, 0x00000040,
+	0x44C, 0x00000010,
+	0x44D, 0x000000F0,
+	0x44E, 0x0000003F,
+	0x44F, 0x00000000,
+	0x450, 0x00000000,
+	0x451, 0x00000000,
+	0x452, 0x00000000,
+	0x453, 0x00000040,
+	0x45E, 0x00000004,
+	0x49C, 0x00000010,
+	0x49D, 0x000000F0,
+	0x49E, 0x00000000,
+	0x49F, 0x00000006,
+	0x4A0, 0x000000E0,
+	0x4A1, 0x00000003,
+	0x4A2, 0x00000000,
+	0x4A3, 0x00000040,
+	0x4A4, 0x00000015,
+	0x4A5, 0x000000F0,
+	0x4A6, 0x00000000,
+	0x4A7, 0x00000006,
+	0x4A8, 0x000000E0,
+	0x4A9, 0x00000000,
+	0x4AA, 0x00000000,
+	0x4AB, 0x00000000,
+	0x7DA, 0x00000008,
+	0x1448, 0x00000006,
+	0x144A, 0x00000006,
+	0x144C, 0x00000006,
+	0x144E, 0x00000006,
+	0x4C8, 0x000000FF,
+	0x4C9, 0x00000008,
+	0x4CA, 0x0000003C,
+	0x4CB, 0x0000003C,
+	0x4CC, 0x000000FF,
+	0x4CD, 0x000000FF,
+	0x4CE, 0x00000001,
+	0x4CF, 0x00000008,
+	0x500, 0x00000026,
+	0x501, 0x000000A2,
+	0x502, 0x0000002F,
+	0x503, 0x00000000,
+	0x504, 0x00000028,
+	0x505, 0x000000A3,
+	0x506, 0x0000005E,
+	0x507, 0x00000000,
+	0x508, 0x0000002B,
+	0x509, 0x000000A4,
+	0x50A, 0x0000005E,
+	0x50B, 0x00000000,
+	0x50C, 0x0000004F,
+	0x50D, 0x000000A4,
+	0x50E, 0x00000000,
+	0x50F, 0x00000000,
+	0x512, 0x0000001C,
+	0x514, 0x0000000A,
+	0x516, 0x0000000A,
+	0x521, 0x0000002F,
+	0x525, 0x00000047,
+	0x550, 0x00000010,
+	0x551, 0x00000010,
+	0x559, 0x00000002,
+	0x55C, 0x00000064,
+	0x55D, 0x000000FF,
+	0x577, 0x00000003,
+	0x5BE, 0x00000064,
+	0x604, 0x00000001,
+	0x605, 0x00000030,
+	0x607, 0x00000001,
+	0x608, 0x0000000E,
+	0x609, 0x0000002A,
+	0x60A, 0x00000000,
+	0x60C, 0x00000018,
+	0x60D, 0x00000050,
+	0x6A0, 0x000000FF,
+	0x6A1, 0x000000FF,
+	0x6A2, 0x000000FF,
+	0x6A3, 0x000000FF,
+	0x6A4, 0x000000FF,
+	0x6A5, 0x000000FF,
+	0x6DE, 0x00000084,
+	0x620, 0x000000FF,
+	0x621, 0x000000FF,
+	0x622, 0x000000FF,
+	0x623, 0x000000FF,
+	0x624, 0x000000FF,
+	0x625, 0x000000FF,
+	0x626, 0x000000FF,
+	0x627, 0x000000FF,
+	0x638, 0x00000064,
+	0x63C, 0x0000000A,
+	0x63D, 0x0000000A,
+	0x63E, 0x0000000E,
+	0x63F, 0x0000000E,
+	0x640, 0x00000040,
+	0x642, 0x00000040,
+	0x643, 0x00000000,
+	0x652, 0x000000C8,
+	0x66E, 0x00000005,
+	0x700, 0x00000021,
+	0x701, 0x00000043,
+	0x702, 0x00000065,
+	0x703, 0x00000087,
+	0x708, 0x00000021,
+	0x709, 0x00000043,
+	0x70A, 0x00000065,
+	0x70B, 0x00000087,
+	0x718, 0x00000040,
+	0x7D5, 0x000000BC,
+	0x7D8, 0x00000028,
+	0x7D9, 0x00000000,
+	0x7DA, 0x0000000B,
+};
+
+RTW_DECL_TABLE_PHY_COND(rtw8814a_mac, rtw_phy_cfg_mac);
+
+static const u32 rtw8814a_agc[] = {
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000003,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000003,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0xA0000000,	0x00000000,
+	0x81C, 0xFF000003,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x81C, 0xFD020003,
+	0x81C, 0xFC040003,
+	0x81C, 0xFB060003,
+	0x81C, 0xFA080003,
+	0x81C, 0xF90A0003,
+	0x81C, 0xF80C0003,
+	0x81C, 0xF70E0003,
+	0x81C, 0xF6100003,
+	0x81C, 0xF5120003,
+	0x81C, 0xF4140003,
+	0x81C, 0xF3160003,
+	0x81C, 0xF2180003,
+	0x81C, 0xF11A0003,
+	0x81C, 0xF01C0003,
+	0x81C, 0xEF1E0003,
+	0x81C, 0xEE200003,
+	0x81C, 0xED220003,
+	0x81C, 0xCF240003,
+	0x81C, 0xCE260003,
+	0x81C, 0xCD280003,
+	0x81C, 0xCC2A0003,
+	0x81C, 0xCB2C0003,
+	0x81C, 0xCA2E0003,
+	0x81C, 0xC9300003,
+	0x81C, 0xC8320003,
+	0x81C, 0xC7340003,
+	0x81C, 0xC6360003,
+	0x81C, 0xC5380003,
+	0x81C, 0xC43A0003,
+	0x81C, 0xA63C0003,
+	0x81C, 0xA53E0003,
+	0x81C, 0xA4400003,
+	0x81C, 0xA3420003,
+	0x81C, 0xA2440003,
+	0x81C, 0xA1460003,
+	0x81C, 0x86480003,
+	0x81C, 0x854A0003,
+	0x81C, 0x844C0003,
+	0x81C, 0x834E0003,
+	0x81C, 0x66500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x435C0003,
+	0x81C, 0x425E0003,
+	0x81C, 0x41600003,
+	0x81C, 0x27620003,
+	0x81C, 0x26640003,
+	0x81C, 0x25660003,
+	0x81C, 0x24680003,
+	0x81C, 0x236A0003,
+	0x81C, 0x226C0003,
+	0x81C, 0x216E0003,
+	0x81C, 0x21700003,
+	0x81C, 0x21720003,
+	0x81C, 0x21740003,
+	0x81C, 0x21760003,
+	0x81C, 0x21780003,
+	0x81C, 0x217A0003,
+	0x81C, 0x217C0003,
+	0x81C, 0x217E0003,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000003,
+	0x81C, 0xFE020003,
+	0x81C, 0xFD040003,
+	0x81C, 0xFC060003,
+	0x81C, 0xFB080003,
+	0x81C, 0xFA0A0003,
+	0x81C, 0xF90C0003,
+	0x81C, 0xF80E0003,
+	0x81C, 0xF7100003,
+	0x81C, 0xF6120003,
+	0x81C, 0xF5140003,
+	0x81C, 0xF4160003,
+	0x81C, 0xF3180003,
+	0x81C, 0xF21A0003,
+	0x81C, 0xF11C0003,
+	0x81C, 0xF01E0003,
+	0x81C, 0xEF200003,
+	0x81C, 0xEE220003,
+	0x81C, 0xED240003,
+	0x81C, 0xEC260003,
+	0x81C, 0xEB280003,
+	0x81C, 0xEA2A0003,
+	0x81C, 0xE92C0003,
+	0x81C, 0xE82E0003,
+	0x81C, 0xE7300003,
+	0x81C, 0xE6320003,
+	0x81C, 0xE5340003,
+	0x81C, 0xE4360003,
+	0x81C, 0xE3380003,
+	0x81C, 0xC53A0003,
+	0x81C, 0xC43C0003,
+	0x81C, 0xC33E0003,
+	0x81C, 0xC2400003,
+	0x81C, 0xC1420003,
+	0x81C, 0xA8440003,
+	0x81C, 0xA7460003,
+	0x81C, 0xA6480003,
+	0x81C, 0xA54A0003,
+	0x81C, 0xA44C0003,
+	0x81C, 0xA34E0003,
+	0x81C, 0xA2500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x475C0003,
+	0x81C, 0x465E0003,
+	0x81C, 0x45600003,
+	0x81C, 0x44620003,
+	0x81C, 0x43640003,
+	0x81C, 0x42660003,
+	0x81C, 0x41680003,
+	0x81C, 0x416A0003,
+	0x81C, 0x416C0003,
+	0x81C, 0x416E0003,
+	0x81C, 0x41700003,
+	0x81C, 0x41720003,
+	0x81C, 0x41740003,
+	0x81C, 0x41760003,
+	0x81C, 0x41780003,
+	0x81C, 0x417A0003,
+	0x81C, 0x417C0003,
+	0x81C, 0x417E0003,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x81C, 0xFD020003,
+	0x81C, 0xFC040003,
+	0x81C, 0xFB060003,
+	0x81C, 0xFA080003,
+	0x81C, 0xF90A0003,
+	0x81C, 0xF80C0003,
+	0x81C, 0xF70E0003,
+	0x81C, 0xF6100003,
+	0x81C, 0xF5120003,
+	0x81C, 0xF4140003,
+	0x81C, 0xF3160003,
+	0x81C, 0xF2180003,
+	0x81C, 0xF11A0003,
+	0x81C, 0xF01C0003,
+	0x81C, 0xEF1E0003,
+	0x81C, 0xEE200003,
+	0x81C, 0xED220003,
+	0x81C, 0xEC240003,
+	0x81C, 0xEB260003,
+	0x81C, 0xEA280003,
+	0x81C, 0xE92A0003,
+	0x81C, 0xE82C0003,
+	0x81C, 0xE72E0003,
+	0x81C, 0xE6300003,
+	0x81C, 0xE5320003,
+	0x81C, 0xE4340003,
+	0x81C, 0xE3360003,
+	0x81C, 0xC6380003,
+	0x81C, 0xC53A0003,
+	0x81C, 0xC43C0003,
+	0x81C, 0xC33E0003,
+	0x81C, 0xC2400003,
+	0x81C, 0xA9420003,
+	0x81C, 0xA8440003,
+	0x81C, 0xA7460003,
+	0x81C, 0xA6480003,
+	0x81C, 0xA54A0003,
+	0x81C, 0xA44C0003,
+	0x81C, 0xA34E0003,
+	0x81C, 0x66500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x49580003,
+	0x81C, 0x485A0003,
+	0x81C, 0x475C0003,
+	0x81C, 0x465E0003,
+	0x81C, 0x45600003,
+	0x81C, 0x44620003,
+	0x81C, 0x43640003,
+	0x81C, 0x42660003,
+	0x81C, 0x41680003,
+	0x81C, 0x416A0003,
+	0x81C, 0x416C0003,
+	0x81C, 0x416E0003,
+	0x81C, 0x41700003,
+	0x81C, 0x41720003,
+	0x81C, 0x41740003,
+	0x81C, 0x41760003,
+	0x81C, 0x41780003,
+	0x81C, 0x417A0003,
+	0x81C, 0x417C0003,
+	0x81C, 0x417E0003,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000003,
+	0x81C, 0xFE020003,
+	0x81C, 0xFD040003,
+	0x81C, 0xFC060003,
+	0x81C, 0xFB080003,
+	0x81C, 0xFA0A0003,
+	0x81C, 0xF90C0003,
+	0x81C, 0xF80E0003,
+	0x81C, 0xF7100003,
+	0x81C, 0xF6120003,
+	0x81C, 0xF5140003,
+	0x81C, 0xF4160003,
+	0x81C, 0xF3180003,
+	0x81C, 0xF21A0003,
+	0x81C, 0xF11C0003,
+	0x81C, 0xF01E0003,
+	0x81C, 0xEF200003,
+	0x81C, 0xEE220003,
+	0x81C, 0xED240003,
+	0x81C, 0xEC260003,
+	0x81C, 0xEB280003,
+	0x81C, 0xEA2A0003,
+	0x81C, 0xE92C0003,
+	0x81C, 0xE82E0003,
+	0x81C, 0xE7300003,
+	0x81C, 0xE6320003,
+	0x81C, 0xE5340003,
+	0x81C, 0xE4360003,
+	0x81C, 0xE3380003,
+	0x81C, 0xC53A0003,
+	0x81C, 0xC43C0003,
+	0x81C, 0xC33E0003,
+	0x81C, 0xC2400003,
+	0x81C, 0xC1420003,
+	0x81C, 0xA8440003,
+	0x81C, 0xA7460003,
+	0x81C, 0xA6480003,
+	0x81C, 0xA54A0003,
+	0x81C, 0xA44C0003,
+	0x81C, 0xA34E0003,
+	0x81C, 0xA2500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x475C0003,
+	0x81C, 0x465E0003,
+	0x81C, 0x45600003,
+	0x81C, 0x44620003,
+	0x81C, 0x43640003,
+	0x81C, 0x42660003,
+	0x81C, 0x41680003,
+	0x81C, 0x416A0003,
+	0x81C, 0x416C0003,
+	0x81C, 0x416E0003,
+	0x81C, 0x41700003,
+	0x81C, 0x41720003,
+	0x81C, 0x41740003,
+	0x81C, 0x41760003,
+	0x81C, 0x41780003,
+	0x81C, 0x417A0003,
+	0x81C, 0x417C0003,
+	0x81C, 0x417E0003,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x81C, 0xFD020003,
+	0x81C, 0xFC040003,
+	0x81C, 0xFB060003,
+	0x81C, 0xFA080003,
+	0x81C, 0xF90A0003,
+	0x81C, 0xF80C0003,
+	0x81C, 0xF70E0003,
+	0x81C, 0xF6100003,
+	0x81C, 0xF5120003,
+	0x81C, 0xF4140003,
+	0x81C, 0xF3160003,
+	0x81C, 0xF2180003,
+	0x81C, 0xF11A0003,
+	0x81C, 0xF01C0003,
+	0x81C, 0xEF1E0003,
+	0x81C, 0xEE200003,
+	0x81C, 0xED220003,
+	0x81C, 0xEC240003,
+	0x81C, 0xEB260003,
+	0x81C, 0xEA280003,
+	0x81C, 0xE92A0003,
+	0x81C, 0xE82C0003,
+	0x81C, 0xE72E0003,
+	0x81C, 0xE6300003,
+	0x81C, 0xE5320003,
+	0x81C, 0xE4340003,
+	0x81C, 0xE3360003,
+	0x81C, 0xC6380003,
+	0x81C, 0xC53A0003,
+	0x81C, 0xC43C0003,
+	0x81C, 0xC33E0003,
+	0x81C, 0xC2400003,
+	0x81C, 0xA9420003,
+	0x81C, 0xA8440003,
+	0x81C, 0xA7460003,
+	0x81C, 0xA6480003,
+	0x81C, 0xA54A0003,
+	0x81C, 0xA44C0003,
+	0x81C, 0xA34E0003,
+	0x81C, 0x66500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x49580003,
+	0x81C, 0x485A0003,
+	0x81C, 0x475C0003,
+	0x81C, 0x465E0003,
+	0x81C, 0x45600003,
+	0x81C, 0x44620003,
+	0x81C, 0x43640003,
+	0x81C, 0x42660003,
+	0x81C, 0x41680003,
+	0x81C, 0x416A0003,
+	0x81C, 0x416C0003,
+	0x81C, 0x416E0003,
+	0x81C, 0x41700003,
+	0x81C, 0x41720003,
+	0x81C, 0x41740003,
+	0x81C, 0x41760003,
+	0x81C, 0x41780003,
+	0x81C, 0x417A0003,
+	0x81C, 0x417C0003,
+	0x81C, 0x417E0003,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xDF000003,
+	0x81C, 0xDF020003,
+	0x81C, 0xDF040003,
+	0x81C, 0xDE060003,
+	0x81C, 0xDD080003,
+	0x81C, 0xDC0A0003,
+	0x81C, 0xDB0C0003,
+	0x81C, 0xDA0E0003,
+	0x81C, 0xD9100003,
+	0x81C, 0xD8120003,
+	0x81C, 0xD7140003,
+	0x81C, 0xD6160003,
+	0x81C, 0xD5180003,
+	0x81C, 0xD41A0003,
+	0x81C, 0xD31C0003,
+	0x81C, 0xD21E0003,
+	0x81C, 0xD1200003,
+	0x81C, 0xD0220003,
+	0x81C, 0xCF240003,
+	0x81C, 0xCE260003,
+	0x81C, 0xCD280003,
+	0x81C, 0xCC2A0003,
+	0x81C, 0xCB2C0003,
+	0x81C, 0xCA2E0003,
+	0x81C, 0xC9300003,
+	0x81C, 0xC8320003,
+	0x81C, 0xC7340003,
+	0x81C, 0xC6360003,
+	0x81C, 0xC5380003,
+	0x81C, 0xA73A0003,
+	0x81C, 0xA63C0003,
+	0x81C, 0xA53E0003,
+	0x81C, 0xA4400003,
+	0x81C, 0xA3420003,
+	0x81C, 0xA2440003,
+	0x81C, 0x87460003,
+	0x81C, 0x86480003,
+	0x81C, 0x854A0003,
+	0x81C, 0x844C0003,
+	0x81C, 0x834E0003,
+	0x81C, 0x82500003,
+	0x81C, 0x81520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x445C0003,
+	0x81C, 0x435E0003,
+	0x81C, 0x42600003,
+	0x81C, 0x41620003,
+	0x81C, 0x27640003,
+	0x81C, 0x26660003,
+	0x81C, 0x25680003,
+	0x81C, 0x246A0003,
+	0x81C, 0x236C0003,
+	0x81C, 0x226E0003,
+	0x81C, 0x21700003,
+	0x81C, 0x21720003,
+	0x81C, 0x21740003,
+	0x81C, 0x21760003,
+	0x81C, 0x21780003,
+	0x81C, 0x217A0003,
+	0x81C, 0x217C0003,
+	0x81C, 0x217E0003,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000003,
+	0x81C, 0xFE020003,
+	0x81C, 0xFD040003,
+	0x81C, 0xFC060003,
+	0x81C, 0xFB080003,
+	0x81C, 0xFA0A0003,
+	0x81C, 0xF90C0003,
+	0x81C, 0xF80E0003,
+	0x81C, 0xF7100003,
+	0x81C, 0xF6120003,
+	0x81C, 0xF5140003,
+	0x81C, 0xF4160003,
+	0x81C, 0xF3180003,
+	0x81C, 0xF21A0003,
+	0x81C, 0xF11C0003,
+	0x81C, 0xF01E0003,
+	0x81C, 0xEF200003,
+	0x81C, 0xEE220003,
+	0x81C, 0xED240003,
+	0x81C, 0xEC260003,
+	0x81C, 0xEB280003,
+	0x81C, 0xEA2A0003,
+	0x81C, 0xE92C0003,
+	0x81C, 0xE82E0003,
+	0x81C, 0xE7300003,
+	0x81C, 0xE6320003,
+	0x81C, 0xE5340003,
+	0x81C, 0xE4360003,
+	0x81C, 0xE3380003,
+	0x81C, 0xC53A0003,
+	0x81C, 0xC43C0003,
+	0x81C, 0xC33E0003,
+	0x81C, 0xC2400003,
+	0x81C, 0xC1420003,
+	0x81C, 0xA8440003,
+	0x81C, 0xA7460003,
+	0x81C, 0xA6480003,
+	0x81C, 0xA54A0003,
+	0x81C, 0xA44C0003,
+	0x81C, 0xA34E0003,
+	0x81C, 0xA2500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x475C0003,
+	0x81C, 0x465E0003,
+	0x81C, 0x45600003,
+	0x81C, 0x44620003,
+	0x81C, 0x43640003,
+	0x81C, 0x42660003,
+	0x81C, 0x41680003,
+	0x81C, 0x416A0003,
+	0x81C, 0x416C0003,
+	0x81C, 0x416E0003,
+	0x81C, 0x41700003,
+	0x81C, 0x41720003,
+	0x81C, 0x41740003,
+	0x81C, 0x41760003,
+	0x81C, 0x41780003,
+	0x81C, 0x417A0003,
+	0x81C, 0x417C0003,
+	0x81C, 0x417E0003,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xDF000003,
+	0x81C, 0xDF020003,
+	0x81C, 0xDF040003,
+	0x81C, 0xDE060003,
+	0x81C, 0xDD080003,
+	0x81C, 0xDC0A0003,
+	0x81C, 0xDB0C0003,
+	0x81C, 0xDA0E0003,
+	0x81C, 0xD9100003,
+	0x81C, 0xD8120003,
+	0x81C, 0xD7140003,
+	0x81C, 0xD6160003,
+	0x81C, 0xD5180003,
+	0x81C, 0xD41A0003,
+	0x81C, 0xD31C0003,
+	0x81C, 0xD21E0003,
+	0x81C, 0xD1200003,
+	0x81C, 0xD0220003,
+	0x81C, 0xCF240003,
+	0x81C, 0xCE260003,
+	0x81C, 0xCD280003,
+	0x81C, 0xCC2A0003,
+	0x81C, 0xCB2C0003,
+	0x81C, 0xCA2E0003,
+	0x81C, 0xC9300003,
+	0x81C, 0xC8320003,
+	0x81C, 0xC7340003,
+	0x81C, 0xC6360003,
+	0x81C, 0xC5380003,
+	0x81C, 0xA73A0003,
+	0x81C, 0xA63C0003,
+	0x81C, 0xA53E0003,
+	0x81C, 0xA4400003,
+	0x81C, 0xA3420003,
+	0x81C, 0xA2440003,
+	0x81C, 0x87460003,
+	0x81C, 0x86480003,
+	0x81C, 0x854A0003,
+	0x81C, 0x844C0003,
+	0x81C, 0x834E0003,
+	0x81C, 0x82500003,
+	0x81C, 0x81520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x445C0003,
+	0x81C, 0x435E0003,
+	0x81C, 0x42600003,
+	0x81C, 0x41620003,
+	0x81C, 0x27640003,
+	0x81C, 0x26660003,
+	0x81C, 0x25680003,
+	0x81C, 0x246A0003,
+	0x81C, 0x236C0003,
+	0x81C, 0x226E0003,
+	0x81C, 0x21700003,
+	0x81C, 0x21720003,
+	0x81C, 0x21740003,
+	0x81C, 0x21760003,
+	0x81C, 0x21780003,
+	0x81C, 0x217A0003,
+	0x81C, 0x217C0003,
+	0x81C, 0x217E0003,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x81C, 0xFE020003,
+	0x81C, 0xFD040003,
+	0x81C, 0xFC060003,
+	0x81C, 0xFB080003,
+	0x81C, 0xFA0A0003,
+	0x81C, 0xF90C0003,
+	0x81C, 0xF80E0003,
+	0x81C, 0xF7100003,
+	0x81C, 0xF6120003,
+	0x81C, 0xF5140003,
+	0x81C, 0xF4160003,
+	0x81C, 0xF3180003,
+	0x81C, 0xF21A0003,
+	0x81C, 0xF11C0003,
+	0x81C, 0xF01E0003,
+	0x81C, 0xEF200003,
+	0x81C, 0xEE220003,
+	0x81C, 0xED240003,
+	0x81C, 0x0F260003,
+	0x81C, 0x0E280003,
+	0x81C, 0x0D2A0003,
+	0x81C, 0x0C2C0003,
+	0x81C, 0x0B2E0003,
+	0x81C, 0x0A300003,
+	0x81C, 0x09320003,
+	0x81C, 0x08340003,
+	0x81C, 0x07360003,
+	0x81C, 0x06380003,
+	0x81C, 0x053A0003,
+	0x81C, 0x043C0003,
+	0x81C, 0x033E0003,
+	0x81C, 0x23400003,
+	0x81C, 0x22420003,
+	0x81C, 0xA8440003,
+	0x81C, 0xA7460003,
+	0x81C, 0xA6480003,
+	0x81C, 0xA54A0003,
+	0x81C, 0xA44C0003,
+	0x81C, 0x684E0003,
+	0x81C, 0x67500003,
+	0x81C, 0x66520003,
+	0x81C, 0x65540003,
+	0x81C, 0x64560003,
+	0x81C, 0x63580003,
+	0x81C, 0x625A0003,
+	0x81C, 0x615C0003,
+	0x81C, 0x475E0003,
+	0x81C, 0x46600003,
+	0x81C, 0x45620003,
+	0x81C, 0x44640003,
+	0x81C, 0x43660003,
+	0x81C, 0x42680003,
+	0x81C, 0x416A0003,
+	0x81C, 0x416C0003,
+	0x81C, 0x416E0003,
+	0x81C, 0x41700003,
+	0x81C, 0x41720003,
+	0x81C, 0x41740003,
+	0x81C, 0x41760003,
+	0x81C, 0x41780003,
+	0x81C, 0x417A0003,
+	0x81C, 0x417C0003,
+	0x81C, 0x417E0003,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x81C, 0xFE020003,
+	0x81C, 0xFD040003,
+	0x81C, 0xFC060003,
+	0x81C, 0xFB080003,
+	0x81C, 0xFA0A0003,
+	0x81C, 0xF90C0003,
+	0x81C, 0xF80E0003,
+	0x81C, 0xF7100003,
+	0x81C, 0xF6120003,
+	0x81C, 0xF5140003,
+	0x81C, 0xF4160003,
+	0x81C, 0xF3180003,
+	0x81C, 0xF21A0003,
+	0x81C, 0xF11C0003,
+	0x81C, 0xF01E0003,
+	0x81C, 0xEF200003,
+	0x81C, 0xEE220003,
+	0x81C, 0xED240003,
+	0x81C, 0xEC260003,
+	0x81C, 0xEB280003,
+	0x81C, 0xEA2A0003,
+	0x81C, 0xE92C0003,
+	0x81C, 0xE72E0003,
+	0x81C, 0xE6300003,
+	0x81C, 0xE5320003,
+	0x81C, 0x08340003,
+	0x81C, 0x07360003,
+	0x81C, 0x06380003,
+	0x81C, 0x053A0003,
+	0x81C, 0x043C0003,
+	0x81C, 0x033E0003,
+	0x81C, 0x02400003,
+	0x81C, 0xA9420003,
+	0x81C, 0xA8440003,
+	0x81C, 0xA7460003,
+	0x81C, 0xA6480003,
+	0x81C, 0xA54A0003,
+	0x81C, 0x684C0003,
+	0x81C, 0x674E0003,
+	0x81C, 0x66500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x475C0003,
+	0x81C, 0x465E0003,
+	0x81C, 0x45600003,
+	0x81C, 0x44620003,
+	0x81C, 0x43640003,
+	0x81C, 0x42660003,
+	0x81C, 0x41680003,
+	0x81C, 0x416A0003,
+	0x81C, 0x416C0003,
+	0x81C, 0x416E0003,
+	0x81C, 0x41700003,
+	0x81C, 0x41720003,
+	0x81C, 0x41740003,
+	0x81C, 0x41760003,
+	0x81C, 0x41780003,
+	0x81C, 0x417A0003,
+	0x81C, 0x417C0003,
+	0x81C, 0x417E0003,
+	0xA0000000,	0x00000000,
+	0x81C, 0xFE000003,
+	0x81C, 0xFD020003,
+	0x81C, 0xFC040003,
+	0x81C, 0xFB060003,
+	0x81C, 0xFA080003,
+	0x81C, 0xF90A0003,
+	0x81C, 0xF80C0003,
+	0x81C, 0xF70E0003,
+	0x81C, 0xF6100003,
+	0x81C, 0xF5120003,
+	0x81C, 0xF4140003,
+	0x81C, 0xF3160003,
+	0x81C, 0xF2180003,
+	0x81C, 0xF11A0003,
+	0x81C, 0xF01C0003,
+	0x81C, 0xEF1E0003,
+	0x81C, 0xEE200003,
+	0x81C, 0xED220003,
+	0x81C, 0xCF240003,
+	0x81C, 0xCE260003,
+	0x81C, 0xCD280003,
+	0x81C, 0xCC2A0003,
+	0x81C, 0xCB2C0003,
+	0x81C, 0xCA2E0003,
+	0x81C, 0xC9300003,
+	0x81C, 0xC8320003,
+	0x81C, 0xC7340003,
+	0x81C, 0xC6360003,
+	0x81C, 0xC5380003,
+	0x81C, 0xC43A0003,
+	0x81C, 0xA63C0003,
+	0x81C, 0xA53E0003,
+	0x81C, 0xA4400003,
+	0x81C, 0xA3420003,
+	0x81C, 0xA2440003,
+	0x81C, 0xA1460003,
+	0x81C, 0x86480003,
+	0x81C, 0x854A0003,
+	0x81C, 0x844C0003,
+	0x81C, 0x834E0003,
+	0x81C, 0x66500003,
+	0x81C, 0x65520003,
+	0x81C, 0x64540003,
+	0x81C, 0x63560003,
+	0x81C, 0x62580003,
+	0x81C, 0x615A0003,
+	0x81C, 0x435C0003,
+	0x81C, 0x425E0003,
+	0x81C, 0x41600003,
+	0x81C, 0x27620003,
+	0x81C, 0x26640003,
+	0x81C, 0x25660003,
+	0x81C, 0x24680003,
+	0x81C, 0x236A0003,
+	0x81C, 0x226C0003,
+	0x81C, 0x216E0003,
+	0x81C, 0x21700003,
+	0x81C, 0x21720003,
+	0x81C, 0x21740003,
+	0x81C, 0x21760003,
+	0x81C, 0x21780003,
+	0x81C, 0x217A0003,
+	0x81C, 0x217C0003,
+	0x81C, 0x217E0003,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF9000103,
+	0x81C, 0xF8020103,
+	0x81C, 0xF7040103,
+	0x81C, 0xF6060103,
+	0x81C, 0xF5080103,
+	0x81C, 0xF40A0103,
+	0x81C, 0xF30C0103,
+	0x81C, 0xF20E0103,
+	0x81C, 0xF1100103,
+	0x81C, 0xF0120103,
+	0x81C, 0xEF140103,
+	0x81C, 0xEE160103,
+	0x81C, 0xED180103,
+	0x81C, 0xEC1A0103,
+	0x81C, 0xEB1C0103,
+	0x81C, 0xEA1E0103,
+	0x81C, 0xE9200103,
+	0x81C, 0xE8220103,
+	0x81C, 0xE7240103,
+	0x81C, 0xE6260103,
+	0x81C, 0xE5280103,
+	0x81C, 0xE42A0103,
+	0x81C, 0xE32C0103,
+	0x81C, 0xC32E0103,
+	0x81C, 0xC2300103,
+	0x81C, 0xC1320103,
+	0x81C, 0xA5340103,
+	0x81C, 0xA4360103,
+	0x81C, 0xA3380103,
+	0x81C, 0xA23A0103,
+	0x81C, 0xA13C0103,
+	0x81C, 0x843E0103,
+	0x81C, 0x83400103,
+	0x81C, 0x82420103,
+	0x81C, 0x81440103,
+	0x81C, 0x64460103,
+	0x81C, 0x63480103,
+	0x81C, 0x624A0103,
+	0x81C, 0x614C0103,
+	0x81C, 0x444E0103,
+	0x81C, 0x43500103,
+	0x81C, 0x42520103,
+	0x81C, 0x41540103,
+	0x81C, 0x25560103,
+	0x81C, 0x24580103,
+	0x81C, 0x235A0103,
+	0x81C, 0x065C0103,
+	0x81C, 0x055E0103,
+	0x81C, 0x04600103,
+	0x81C, 0x03620103,
+	0x81C, 0x02640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF8000103,
+	0x81C, 0xF7020103,
+	0x81C, 0xF6040103,
+	0x81C, 0xF5060103,
+	0x81C, 0xF4080103,
+	0x81C, 0xF30A0103,
+	0x81C, 0xF20C0103,
+	0x81C, 0xF10E0103,
+	0x81C, 0xF0100103,
+	0x81C, 0xEF120103,
+	0x81C, 0xEE140103,
+	0x81C, 0xED160103,
+	0x81C, 0xEC180103,
+	0x81C, 0xEB1A0103,
+	0x81C, 0xEA1C0103,
+	0x81C, 0xE91E0103,
+	0x81C, 0xE8200103,
+	0x81C, 0xE7220103,
+	0x81C, 0xE6240103,
+	0x81C, 0xE5260103,
+	0x81C, 0xE4280103,
+	0x81C, 0xE32A0103,
+	0x81C, 0xE22C0103,
+	0x81C, 0xE12E0103,
+	0x81C, 0xA5300103,
+	0x81C, 0xA4320103,
+	0x81C, 0xA3340103,
+	0x81C, 0xA2360103,
+	0x81C, 0xA1380103,
+	0x81C, 0x843A0103,
+	0x81C, 0x833C0103,
+	0x81C, 0x823E0103,
+	0x81C, 0x81400103,
+	0x81C, 0x64420103,
+	0x81C, 0x63440103,
+	0x81C, 0x62460103,
+	0x81C, 0x61480103,
+	0x81C, 0x454A0103,
+	0x81C, 0x444C0103,
+	0x81C, 0x434E0103,
+	0x81C, 0x42500103,
+	0x81C, 0x25520103,
+	0x81C, 0x24540103,
+	0x81C, 0x23560103,
+	0x81C, 0x06580103,
+	0x81C, 0x055A0103,
+	0x81C, 0x045C0103,
+	0x81C, 0x035E0103,
+	0x81C, 0x02600103,
+	0x81C, 0x01620103,
+	0x81C, 0x01640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFC000103,
+	0x81C, 0xFB020103,
+	0x81C, 0xFA040103,
+	0x81C, 0xF9060103,
+	0x81C, 0xF8080103,
+	0x81C, 0xF70A0103,
+	0x81C, 0xF60C0103,
+	0x81C, 0xF50E0103,
+	0x81C, 0xF4100103,
+	0x81C, 0xF3120103,
+	0x81C, 0xF2140103,
+	0x81C, 0xF1160103,
+	0x81C, 0xF0180103,
+	0x81C, 0xEF1A0103,
+	0x81C, 0xEE1C0103,
+	0x81C, 0xED1E0103,
+	0x81C, 0xEC200103,
+	0x81C, 0xEB220103,
+	0x81C, 0xEA240103,
+	0x81C, 0xE9260103,
+	0x81C, 0xE8280103,
+	0x81C, 0xE72A0103,
+	0x81C, 0xE62C0103,
+	0x81C, 0xE52E0103,
+	0x81C, 0xE4300103,
+	0x81C, 0xE3320103,
+	0x81C, 0xE2340103,
+	0x81C, 0xE1360103,
+	0x81C, 0x87380103,
+	0x81C, 0x863A0103,
+	0x81C, 0x853C0103,
+	0x81C, 0x843E0103,
+	0x81C, 0x83400103,
+	0x81C, 0x82420103,
+	0x81C, 0x81440103,
+	0x81C, 0x64460103,
+	0x81C, 0x63480103,
+	0x81C, 0x624A0103,
+	0x81C, 0x464C0103,
+	0x81C, 0x454E0103,
+	0x81C, 0x44500103,
+	0x81C, 0x43520103,
+	0x81C, 0x26540103,
+	0x81C, 0x25560103,
+	0x81C, 0x24580103,
+	0x81C, 0x075A0103,
+	0x81C, 0x065C0103,
+	0x81C, 0x055E0103,
+	0x81C, 0x04600103,
+	0x81C, 0x03620103,
+	0x81C, 0x02640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF9000103,
+	0x81C, 0xF8020103,
+	0x81C, 0xF7040103,
+	0x81C, 0xF6060103,
+	0x81C, 0xF5080103,
+	0x81C, 0xF40A0103,
+	0x81C, 0xF30C0103,
+	0x81C, 0xF20E0103,
+	0x81C, 0xF1100103,
+	0x81C, 0xF0120103,
+	0x81C, 0xEF140103,
+	0x81C, 0xEE160103,
+	0x81C, 0xED180103,
+	0x81C, 0xEC1A0103,
+	0x81C, 0xEB1C0103,
+	0x81C, 0xEA1E0103,
+	0x81C, 0xE9200103,
+	0x81C, 0xE8220103,
+	0x81C, 0xE7240103,
+	0x81C, 0xE6260103,
+	0x81C, 0xE5280103,
+	0x81C, 0xE42A0103,
+	0x81C, 0xE32C0103,
+	0x81C, 0xE22E0103,
+	0x81C, 0xA6300103,
+	0x81C, 0xA5320103,
+	0x81C, 0xA4340103,
+	0x81C, 0xA3360103,
+	0x81C, 0xA2380103,
+	0x81C, 0xA13A0103,
+	0x81C, 0x843C0103,
+	0x81C, 0x833E0103,
+	0x81C, 0x82400103,
+	0x81C, 0x81420103,
+	0x81C, 0x64440103,
+	0x81C, 0x63460103,
+	0x81C, 0x62480103,
+	0x81C, 0x614A0103,
+	0x81C, 0x444C0103,
+	0x81C, 0x434E0103,
+	0x81C, 0x42500103,
+	0x81C, 0x41520103,
+	0x81C, 0x25540103,
+	0x81C, 0x24560103,
+	0x81C, 0x23580103,
+	0x81C, 0x225A0103,
+	0x81C, 0x055C0103,
+	0x81C, 0x045E0103,
+	0x81C, 0x03600103,
+	0x81C, 0x02620103,
+	0x81C, 0x01640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFD000103,
+	0x81C, 0xFC020103,
+	0x81C, 0xFB040103,
+	0x81C, 0xFA060103,
+	0x81C, 0xF9080103,
+	0x81C, 0xF80A0103,
+	0x81C, 0xF70C0103,
+	0x81C, 0xF60E0103,
+	0x81C, 0xF5100103,
+	0x81C, 0xF4120103,
+	0x81C, 0xF3140103,
+	0x81C, 0xF2160103,
+	0x81C, 0xF1180103,
+	0x81C, 0xF01A0103,
+	0x81C, 0xEF1C0103,
+	0x81C, 0xEE1E0103,
+	0x81C, 0xED200103,
+	0x81C, 0xEC220103,
+	0x81C, 0xEB240103,
+	0x81C, 0xEA260103,
+	0x81C, 0xE9280103,
+	0x81C, 0xE82A0103,
+	0x81C, 0xE72C0103,
+	0x81C, 0xE62E0103,
+	0x81C, 0xE5300103,
+	0x81C, 0xE4320103,
+	0x81C, 0xE3340103,
+	0x81C, 0xE2360103,
+	0x81C, 0xE1380103,
+	0x81C, 0xA33A0103,
+	0x81C, 0xA23C0103,
+	0x81C, 0xA13E0103,
+	0x81C, 0x84400103,
+	0x81C, 0x83420103,
+	0x81C, 0x82440103,
+	0x81C, 0x81460103,
+	0x81C, 0x64480103,
+	0x81C, 0x634A0103,
+	0x81C, 0x624C0103,
+	0x81C, 0x614E0103,
+	0x81C, 0x45500103,
+	0x81C, 0x44520103,
+	0x81C, 0x43540103,
+	0x81C, 0x42560103,
+	0x81C, 0x25580103,
+	0x81C, 0x245A0103,
+	0x81C, 0x235C0103,
+	0x81C, 0x065E0103,
+	0x81C, 0x05600103,
+	0x81C, 0x04620103,
+	0x81C, 0x03640103,
+	0x81C, 0x02660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFA000103,
+	0x81C, 0xF9020103,
+	0x81C, 0xF8040103,
+	0x81C, 0xF7060103,
+	0x81C, 0xF6080103,
+	0x81C, 0xF50A0103,
+	0x81C, 0xF40C0103,
+	0x81C, 0xF30E0103,
+	0x81C, 0xF2100103,
+	0x81C, 0xF1120103,
+	0x81C, 0xF0140103,
+	0x81C, 0xEF160103,
+	0x81C, 0xEE180103,
+	0x81C, 0xED1A0103,
+	0x81C, 0xEC1C0103,
+	0x81C, 0xEB1E0103,
+	0x81C, 0xEA200103,
+	0x81C, 0xE9220103,
+	0x81C, 0xE8240103,
+	0x81C, 0xE7260103,
+	0x81C, 0xE6280103,
+	0x81C, 0xE52A0103,
+	0x81C, 0xE42C0103,
+	0x81C, 0xE32E0103,
+	0x81C, 0xE2300103,
+	0x81C, 0xE1320103,
+	0x81C, 0xA5340103,
+	0x81C, 0xA4360103,
+	0x81C, 0xA3380103,
+	0x81C, 0xA23A0103,
+	0x81C, 0xA13C0103,
+	0x81C, 0x843E0103,
+	0x81C, 0x83400103,
+	0x81C, 0x82420103,
+	0x81C, 0x81440103,
+	0x81C, 0x64460103,
+	0x81C, 0x63480103,
+	0x81C, 0x624A0103,
+	0x81C, 0x614C0103,
+	0x81C, 0x454E0103,
+	0x81C, 0x44500103,
+	0x81C, 0x43520103,
+	0x81C, 0x42540103,
+	0x81C, 0x41560103,
+	0x81C, 0x24580103,
+	0x81C, 0x235A0103,
+	0x81C, 0x225C0103,
+	0x81C, 0x055E0103,
+	0x81C, 0x04600103,
+	0x81C, 0x03620103,
+	0x81C, 0x02640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000103,
+	0x81C, 0xFF020103,
+	0x81C, 0xFE040103,
+	0x81C, 0xFD060103,
+	0x81C, 0xFC080103,
+	0x81C, 0xFB0A0103,
+	0x81C, 0xFA0C0103,
+	0x81C, 0xF90E0103,
+	0x81C, 0xF8100103,
+	0x81C, 0xF7120103,
+	0x81C, 0xF6140103,
+	0x81C, 0xF5160103,
+	0x81C, 0xF4180103,
+	0x81C, 0xF31A0103,
+	0x81C, 0xF21C0103,
+	0x81C, 0xF11E0103,
+	0x81C, 0xF0200103,
+	0x81C, 0xEF220103,
+	0x81C, 0xEE240103,
+	0x81C, 0xED260103,
+	0x81C, 0xEC280103,
+	0x81C, 0xEB2A0103,
+	0x81C, 0xEA2C0103,
+	0x81C, 0xE92E0103,
+	0x81C, 0xE8300103,
+	0x81C, 0xE7320103,
+	0x81C, 0xE6340103,
+	0x81C, 0xE5360103,
+	0x81C, 0xE4380103,
+	0x81C, 0xE33A0103,
+	0x81C, 0xA53C0103,
+	0x81C, 0xA43E0103,
+	0x81C, 0xA3400103,
+	0x81C, 0xA2420103,
+	0x81C, 0xA1440103,
+	0x81C, 0x85460103,
+	0x81C, 0x84480103,
+	0x81C, 0x834A0103,
+	0x81C, 0x824C0103,
+	0x81C, 0x814E0103,
+	0x81C, 0x64500103,
+	0x81C, 0x63520103,
+	0x81C, 0x62540103,
+	0x81C, 0x44560103,
+	0x81C, 0x43580103,
+	0x81C, 0x425A0103,
+	0x81C, 0x265C0103,
+	0x81C, 0x255E0103,
+	0x81C, 0x24600103,
+	0x81C, 0x07620103,
+	0x81C, 0x06640103,
+	0x81C, 0x05660103,
+	0x81C, 0x04680103,
+	0x81C, 0x036A0103,
+	0x81C, 0x026C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF8000103,
+	0x81C, 0xF7020103,
+	0x81C, 0xF6040103,
+	0x81C, 0xF5060103,
+	0x81C, 0xF4080103,
+	0x81C, 0xF30A0103,
+	0x81C, 0xF20C0103,
+	0x81C, 0xF10E0103,
+	0x81C, 0xF0100103,
+	0x81C, 0xEF120103,
+	0x81C, 0xEE140103,
+	0x81C, 0xED160103,
+	0x81C, 0xEC180103,
+	0x81C, 0xEB1A0103,
+	0x81C, 0xEA1C0103,
+	0x81C, 0xE91E0103,
+	0x81C, 0xE8200103,
+	0x81C, 0xE7220103,
+	0x81C, 0xE6240103,
+	0x81C, 0xE5260103,
+	0x81C, 0xE4280103,
+	0x81C, 0xE32A0103,
+	0x81C, 0xE22C0103,
+	0x81C, 0xE12E0103,
+	0x81C, 0xA4300103,
+	0x81C, 0xA3320103,
+	0x81C, 0xA2340103,
+	0x81C, 0xA1360103,
+	0x81C, 0x85380103,
+	0x81C, 0x843A0103,
+	0x81C, 0x833C0103,
+	0x81C, 0x823E0103,
+	0x81C, 0x65400103,
+	0x81C, 0x64420103,
+	0x81C, 0x63440103,
+	0x81C, 0x62460103,
+	0x81C, 0x45480103,
+	0x81C, 0x444A0103,
+	0x81C, 0x434C0103,
+	0x81C, 0x264E0103,
+	0x81C, 0x25500103,
+	0x81C, 0x24520103,
+	0x81C, 0x08540103,
+	0x81C, 0x07560103,
+	0x81C, 0x06580103,
+	0x81C, 0x055A0103,
+	0x81C, 0x045C0103,
+	0x81C, 0x035E0103,
+	0x81C, 0x02600103,
+	0x81C, 0x01620103,
+	0x81C, 0x01640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000103,
+	0x81C, 0xFF020103,
+	0x81C, 0xFE040103,
+	0x81C, 0xFD060103,
+	0x81C, 0xFC080103,
+	0x81C, 0xFB0A0103,
+	0x81C, 0xFA0C0103,
+	0x81C, 0xF90E0103,
+	0x81C, 0xF8100103,
+	0x81C, 0xF7120103,
+	0x81C, 0xF6140103,
+	0x81C, 0xF5160103,
+	0x81C, 0xF4180103,
+	0x81C, 0xF31A0103,
+	0x81C, 0xF21C0103,
+	0x81C, 0xF11E0103,
+	0x81C, 0xF0200103,
+	0x81C, 0xEF220103,
+	0x81C, 0xEE240103,
+	0x81C, 0xED260103,
+	0x81C, 0xEC280103,
+	0x81C, 0xEB2A0103,
+	0x81C, 0xEA2C0103,
+	0x81C, 0xE92E0103,
+	0x81C, 0xE8300103,
+	0x81C, 0xE7320103,
+	0x81C, 0xE6340103,
+	0x81C, 0xE5360103,
+	0x81C, 0xE4380103,
+	0x81C, 0xE33A0103,
+	0x81C, 0xA53C0103,
+	0x81C, 0xA43E0103,
+	0x81C, 0xA3400103,
+	0x81C, 0xA2420103,
+	0x81C, 0xA1440103,
+	0x81C, 0x85460103,
+	0x81C, 0x84480103,
+	0x81C, 0x834A0103,
+	0x81C, 0x824C0103,
+	0x81C, 0x814E0103,
+	0x81C, 0x64500103,
+	0x81C, 0x63520103,
+	0x81C, 0x62540103,
+	0x81C, 0x44560103,
+	0x81C, 0x43580103,
+	0x81C, 0x425A0103,
+	0x81C, 0x265C0103,
+	0x81C, 0x255E0103,
+	0x81C, 0x24600103,
+	0x81C, 0x07620103,
+	0x81C, 0x06640103,
+	0x81C, 0x05660103,
+	0x81C, 0x04680103,
+	0x81C, 0x036A0103,
+	0x81C, 0x026C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF9000103,
+	0x81C, 0xF8020103,
+	0x81C, 0xF7040103,
+	0x81C, 0xF6060103,
+	0x81C, 0xF5080103,
+	0x81C, 0xF40A0103,
+	0x81C, 0xF30C0103,
+	0x81C, 0xF20E0103,
+	0x81C, 0xF1100103,
+	0x81C, 0xF0120103,
+	0x81C, 0xEF140103,
+	0x81C, 0xEE160103,
+	0x81C, 0xED180103,
+	0x81C, 0xEC1A0103,
+	0x81C, 0xEB1C0103,
+	0x81C, 0xEA1E0103,
+	0x81C, 0xE9200103,
+	0x81C, 0xE8220103,
+	0x81C, 0xE7240103,
+	0x81C, 0xE6260103,
+	0x81C, 0xE5280103,
+	0x81C, 0xE42A0103,
+	0x81C, 0xE32C0103,
+	0x81C, 0xE22E0103,
+	0x81C, 0xA6300103,
+	0x81C, 0xA5320103,
+	0x81C, 0xA4340103,
+	0x81C, 0xA3360103,
+	0x81C, 0xA2380103,
+	0x81C, 0xA13A0103,
+	0x81C, 0x843C0103,
+	0x81C, 0x833E0103,
+	0x81C, 0x82400103,
+	0x81C, 0x81420103,
+	0x81C, 0x64440103,
+	0x81C, 0x63460103,
+	0x81C, 0x62480103,
+	0x81C, 0x614A0103,
+	0x81C, 0x444C0103,
+	0x81C, 0x434E0103,
+	0x81C, 0x42500103,
+	0x81C, 0x41520103,
+	0x81C, 0x25540103,
+	0x81C, 0x24560103,
+	0x81C, 0x23580103,
+	0x81C, 0x225A0103,
+	0x81C, 0x055C0103,
+	0x81C, 0x045E0103,
+	0x81C, 0x03600103,
+	0x81C, 0x02620103,
+	0x81C, 0x01640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFA000103,
+	0x81C, 0xF9020103,
+	0x81C, 0xF8040103,
+	0x81C, 0xF7060103,
+	0x81C, 0xF6080103,
+	0x81C, 0xF50A0103,
+	0x81C, 0xF40C0103,
+	0x81C, 0xF30E0103,
+	0x81C, 0xF2100103,
+	0x81C, 0xF1120103,
+	0x81C, 0xF0140103,
+	0x81C, 0xEF160103,
+	0x81C, 0xEE180103,
+	0x81C, 0xED1A0103,
+	0x81C, 0xCC1C0103,
+	0x81C, 0xCB1E0103,
+	0x81C, 0xCA200103,
+	0x81C, 0xE9220103,
+	0x81C, 0xE8240103,
+	0x81C, 0xE7260103,
+	0x81C, 0xE6280103,
+	0x81C, 0xE42A0103,
+	0x81C, 0xE32C0103,
+	0x81C, 0xE22E0103,
+	0x81C, 0xA7300103,
+	0x81C, 0xA6320103,
+	0x81C, 0xA5340103,
+	0x81C, 0xA4360103,
+	0x81C, 0xA3380103,
+	0x81C, 0xA23A0103,
+	0x81C, 0xA13C0103,
+	0x81C, 0x843E0103,
+	0x81C, 0x83400103,
+	0x81C, 0x82420103,
+	0x81C, 0x65440103,
+	0x81C, 0x64460103,
+	0x81C, 0x63480103,
+	0x81C, 0x624A0103,
+	0x81C, 0x614C0103,
+	0x81C, 0x444E0103,
+	0x81C, 0x43500103,
+	0x81C, 0x42520103,
+	0x81C, 0x41540103,
+	0x81C, 0x24560103,
+	0x81C, 0x23580103,
+	0x81C, 0x055A0103,
+	0x81C, 0x045C0103,
+	0x81C, 0x035E0103,
+	0x81C, 0x02600103,
+	0x81C, 0x01620103,
+	0x81C, 0x01640103,
+	0x81C, 0x01660103,
+	0x81C, 0x01680103,
+	0x81C, 0x016A0103,
+	0x81C, 0x016C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0xA0000000,	0x00000000,
+	0x81C, 0xFF000103,
+	0x81C, 0xFE020103,
+	0x81C, 0xFD040103,
+	0x81C, 0xFC060103,
+	0x81C, 0xFB080103,
+	0x81C, 0xFA0A0103,
+	0x81C, 0xF90C0103,
+	0x81C, 0xF80E0103,
+	0x81C, 0xF7100103,
+	0x81C, 0xF6120103,
+	0x81C, 0xF5140103,
+	0x81C, 0xF4160103,
+	0x81C, 0xF3180103,
+	0x81C, 0xF21A0103,
+	0x81C, 0xF11C0103,
+	0x81C, 0xF01E0103,
+	0x81C, 0xEF200103,
+	0x81C, 0xEE220103,
+	0x81C, 0xED240103,
+	0x81C, 0xEC260103,
+	0x81C, 0xEB280103,
+	0x81C, 0xEA2A0103,
+	0x81C, 0xE92C0103,
+	0x81C, 0xE82E0103,
+	0x81C, 0xE7300103,
+	0x81C, 0xE6320103,
+	0x81C, 0xE5340103,
+	0x81C, 0xE4360103,
+	0x81C, 0xE3380103,
+	0x81C, 0xE23A0103,
+	0x81C, 0xE13C0103,
+	0x81C, 0xA43E0103,
+	0x81C, 0xA3400103,
+	0x81C, 0xA2420103,
+	0x81C, 0xA1440103,
+	0x81C, 0x86460103,
+	0x81C, 0x85480103,
+	0x81C, 0x844A0103,
+	0x81C, 0x834C0103,
+	0x81C, 0x824E0103,
+	0x81C, 0x81500103,
+	0x81C, 0x64520103,
+	0x81C, 0x63540103,
+	0x81C, 0x62560103,
+	0x81C, 0x61580103,
+	0x81C, 0x435A0103,
+	0x81C, 0x425C0103,
+	0x81C, 0x415E0103,
+	0x81C, 0x25600103,
+	0x81C, 0x24620103,
+	0x81C, 0x06640103,
+	0x81C, 0x05660103,
+	0x81C, 0x04680103,
+	0x81C, 0x036A0103,
+	0x81C, 0x026C0103,
+	0x81C, 0x016E0103,
+	0x81C, 0x01700103,
+	0x81C, 0x01720103,
+	0x81C, 0x01740103,
+	0x81C, 0x01760103,
+	0x81C, 0x01780103,
+	0x81C, 0x017A0103,
+	0x81C, 0x017C0103,
+	0x81C, 0x017E0103,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFA000203,
+	0x81C, 0xF9020203,
+	0x81C, 0xF8040203,
+	0x81C, 0xF7060203,
+	0x81C, 0xF6080203,
+	0x81C, 0xF50A0203,
+	0x81C, 0xF40C0203,
+	0x81C, 0xF30E0203,
+	0x81C, 0xF2100203,
+	0x81C, 0xF1120203,
+	0x81C, 0xF0140203,
+	0x81C, 0xEF160203,
+	0x81C, 0xEE180203,
+	0x81C, 0xED1A0203,
+	0x81C, 0xEC1C0203,
+	0x81C, 0xEB1E0203,
+	0x81C, 0xEA200203,
+	0x81C, 0xE9220203,
+	0x81C, 0xE8240203,
+	0x81C, 0xE7260203,
+	0x81C, 0xE6280203,
+	0x81C, 0xE52A0203,
+	0x81C, 0xE42C0203,
+	0x81C, 0xE32E0203,
+	0x81C, 0xE2300203,
+	0x81C, 0xE1320203,
+	0x81C, 0xA5340203,
+	0x81C, 0xA4360203,
+	0x81C, 0xA3380203,
+	0x81C, 0xA23A0203,
+	0x81C, 0xA13C0203,
+	0x81C, 0x843E0203,
+	0x81C, 0x83400203,
+	0x81C, 0x82420203,
+	0x81C, 0x81440203,
+	0x81C, 0x63460203,
+	0x81C, 0x62480203,
+	0x81C, 0x614A0203,
+	0x81C, 0x464C0203,
+	0x81C, 0x454E0203,
+	0x81C, 0x44500203,
+	0x81C, 0x43520203,
+	0x81C, 0x42540203,
+	0x81C, 0x41560203,
+	0x81C, 0x24580203,
+	0x81C, 0x235A0203,
+	0x81C, 0x065C0203,
+	0x81C, 0x055E0203,
+	0x81C, 0x04600203,
+	0x81C, 0x03620203,
+	0x81C, 0x02640203,
+	0x81C, 0x01660203,
+	0x81C, 0x01680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF8000203,
+	0x81C, 0xF7020203,
+	0x81C, 0xF6040203,
+	0x81C, 0xF5060203,
+	0x81C, 0xF4080203,
+	0x81C, 0xF30A0203,
+	0x81C, 0xF20C0203,
+	0x81C, 0xF10E0203,
+	0x81C, 0xF0100203,
+	0x81C, 0xEF120203,
+	0x81C, 0xEE140203,
+	0x81C, 0xED160203,
+	0x81C, 0xEC180203,
+	0x81C, 0xEB1A0203,
+	0x81C, 0xEA1C0203,
+	0x81C, 0xE91E0203,
+	0x81C, 0xE8200203,
+	0x81C, 0xE7220203,
+	0x81C, 0xE6240203,
+	0x81C, 0xE5260203,
+	0x81C, 0xE4280203,
+	0x81C, 0xE32A0203,
+	0x81C, 0xE22C0203,
+	0x81C, 0xE12E0203,
+	0x81C, 0xA6300203,
+	0x81C, 0xA5320203,
+	0x81C, 0xA4340203,
+	0x81C, 0xA3360203,
+	0x81C, 0xA2380203,
+	0x81C, 0x853A0203,
+	0x81C, 0x843C0203,
+	0x81C, 0x833E0203,
+	0x81C, 0x82400203,
+	0x81C, 0x81420203,
+	0x81C, 0x64440203,
+	0x81C, 0x63460203,
+	0x81C, 0x62480203,
+	0x81C, 0x614A0203,
+	0x81C, 0x444C0203,
+	0x81C, 0x434E0203,
+	0x81C, 0x42500203,
+	0x81C, 0x25520203,
+	0x81C, 0x24540203,
+	0x81C, 0x23560203,
+	0x81C, 0x06580203,
+	0x81C, 0x055A0203,
+	0x81C, 0x045C0203,
+	0x81C, 0x035E0203,
+	0x81C, 0x02600203,
+	0x81C, 0x01620203,
+	0x81C, 0x01640203,
+	0x81C, 0x01660203,
+	0x81C, 0x01680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFC000203,
+	0x81C, 0xFB020203,
+	0x81C, 0xFA040203,
+	0x81C, 0xF9060203,
+	0x81C, 0xF8080203,
+	0x81C, 0xF70A0203,
+	0x81C, 0xF60C0203,
+	0x81C, 0xF50E0203,
+	0x81C, 0xF4100203,
+	0x81C, 0xF3120203,
+	0x81C, 0xF2140203,
+	0x81C, 0xF1160203,
+	0x81C, 0xF0180203,
+	0x81C, 0xEF1A0203,
+	0x81C, 0xEE1C0203,
+	0x81C, 0xED1E0203,
+	0x81C, 0xEC200203,
+	0x81C, 0xEB220203,
+	0x81C, 0xEA240203,
+	0x81C, 0xE9260203,
+	0x81C, 0xE8280203,
+	0x81C, 0xE72A0203,
+	0x81C, 0xE62C0203,
+	0x81C, 0xE52E0203,
+	0x81C, 0xE4300203,
+	0x81C, 0xE3320203,
+	0x81C, 0xE2340203,
+	0x81C, 0xE1360203,
+	0x81C, 0x87380203,
+	0x81C, 0x863A0203,
+	0x81C, 0x853C0203,
+	0x81C, 0x843E0203,
+	0x81C, 0x83400203,
+	0x81C, 0x82420203,
+	0x81C, 0x81440203,
+	0x81C, 0x64460203,
+	0x81C, 0x63480203,
+	0x81C, 0x624A0203,
+	0x81C, 0x474C0203,
+	0x81C, 0x464E0203,
+	0x81C, 0x45500203,
+	0x81C, 0x44520203,
+	0x81C, 0x43540203,
+	0x81C, 0x42560203,
+	0x81C, 0x24580203,
+	0x81C, 0x235A0203,
+	0x81C, 0x075C0203,
+	0x81C, 0x065E0203,
+	0x81C, 0x05600203,
+	0x81C, 0x04620203,
+	0x81C, 0x03640203,
+	0x81C, 0x02660203,
+	0x81C, 0x01680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF8000203,
+	0x81C, 0xF7020203,
+	0x81C, 0xF6040203,
+	0x81C, 0xF5060203,
+	0x81C, 0xF4080203,
+	0x81C, 0xF30A0203,
+	0x81C, 0xF20C0203,
+	0x81C, 0xF10E0203,
+	0x81C, 0xF0100203,
+	0x81C, 0xEF120203,
+	0x81C, 0xEE140203,
+	0x81C, 0xED160203,
+	0x81C, 0xEC180203,
+	0x81C, 0xEB1A0203,
+	0x81C, 0xEA1C0203,
+	0x81C, 0xE91E0203,
+	0x81C, 0xE8200203,
+	0x81C, 0xE7220203,
+	0x81C, 0xE6240203,
+	0x81C, 0xE5260203,
+	0x81C, 0xE4280203,
+	0x81C, 0xE32A0203,
+	0x81C, 0xE22C0203,
+	0x81C, 0xE12E0203,
+	0x81C, 0xA6300203,
+	0x81C, 0xA5320203,
+	0x81C, 0xA4340203,
+	0x81C, 0xA3360203,
+	0x81C, 0xA2380203,
+	0x81C, 0xA13A0203,
+	0x81C, 0x843C0203,
+	0x81C, 0x833E0203,
+	0x81C, 0x82400203,
+	0x81C, 0x81420203,
+	0x81C, 0x64440203,
+	0x81C, 0x63460203,
+	0x81C, 0x62480203,
+	0x81C, 0x614A0203,
+	0x81C, 0x444C0203,
+	0x81C, 0x434E0203,
+	0x81C, 0x42500203,
+	0x81C, 0x41520203,
+	0x81C, 0x25540203,
+	0x81C, 0x24560203,
+	0x81C, 0x23580203,
+	0x81C, 0x065A0203,
+	0x81C, 0x055C0203,
+	0x81C, 0x045E0203,
+	0x81C, 0x03600203,
+	0x81C, 0x02620203,
+	0x81C, 0x01640203,
+	0x81C, 0x01660203,
+	0x81C, 0x01680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFB000203,
+	0x81C, 0xFA020203,
+	0x81C, 0xF9040203,
+	0x81C, 0xF8060203,
+	0x81C, 0xF7080203,
+	0x81C, 0xF60A0203,
+	0x81C, 0xF50C0203,
+	0x81C, 0xF40E0203,
+	0x81C, 0xF3100203,
+	0x81C, 0xF2120203,
+	0x81C, 0xF1140203,
+	0x81C, 0xF0160203,
+	0x81C, 0xEF180203,
+	0x81C, 0xEE1A0203,
+	0x81C, 0xED1C0203,
+	0x81C, 0xEC1E0203,
+	0x81C, 0xEB200203,
+	0x81C, 0xEA220203,
+	0x81C, 0xE9240203,
+	0x81C, 0xE8260203,
+	0x81C, 0xE7280203,
+	0x81C, 0xE62A0203,
+	0x81C, 0xE52C0203,
+	0x81C, 0xE42E0203,
+	0x81C, 0xE3300203,
+	0x81C, 0xE2320203,
+	0x81C, 0xE1340203,
+	0x81C, 0xA5360203,
+	0x81C, 0xA4380203,
+	0x81C, 0xA33A0203,
+	0x81C, 0xA23C0203,
+	0x81C, 0x843E0203,
+	0x81C, 0x83400203,
+	0x81C, 0x82420203,
+	0x81C, 0x81440203,
+	0x81C, 0x64460203,
+	0x81C, 0x63480203,
+	0x81C, 0x624A0203,
+	0x81C, 0x614C0203,
+	0x81C, 0x474E0203,
+	0x81C, 0x46500203,
+	0x81C, 0x45520203,
+	0x81C, 0x44540203,
+	0x81C, 0x43560203,
+	0x81C, 0x25580203,
+	0x81C, 0x245A0203,
+	0x81C, 0x235C0203,
+	0x81C, 0x075E0203,
+	0x81C, 0x06600203,
+	0x81C, 0x05620203,
+	0x81C, 0x04640203,
+	0x81C, 0x03660203,
+	0x81C, 0x02680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFC000203,
+	0x81C, 0xFB020203,
+	0x81C, 0xFA040203,
+	0x81C, 0xF9060203,
+	0x81C, 0xF8080203,
+	0x81C, 0xF70A0203,
+	0x81C, 0xF60C0203,
+	0x81C, 0xF50E0203,
+	0x81C, 0xF4100203,
+	0x81C, 0xF3120203,
+	0x81C, 0xF2140203,
+	0x81C, 0xF1160203,
+	0x81C, 0xF0180203,
+	0x81C, 0xEF1A0203,
+	0x81C, 0xEE1C0203,
+	0x81C, 0xED1E0203,
+	0x81C, 0xEC200203,
+	0x81C, 0xEB220203,
+	0x81C, 0xEA240203,
+	0x81C, 0xE9260203,
+	0x81C, 0xE8280203,
+	0x81C, 0xE72A0203,
+	0x81C, 0xE62C0203,
+	0x81C, 0xE52E0203,
+	0x81C, 0xE4300203,
+	0x81C, 0xE3320203,
+	0x81C, 0xE2340203,
+	0x81C, 0xE1360203,
+	0x81C, 0xA5380203,
+	0x81C, 0xA43A0203,
+	0x81C, 0xA33C0203,
+	0x81C, 0x853E0203,
+	0x81C, 0x84400203,
+	0x81C, 0x83420203,
+	0x81C, 0x82440203,
+	0x81C, 0x81460203,
+	0x81C, 0x64480203,
+	0x81C, 0x634A0203,
+	0x81C, 0x624C0203,
+	0x81C, 0x614E0203,
+	0x81C, 0x46500203,
+	0x81C, 0x45520203,
+	0x81C, 0x44540203,
+	0x81C, 0x43560203,
+	0x81C, 0x25580203,
+	0x81C, 0x245A0203,
+	0x81C, 0x235C0203,
+	0x81C, 0x075E0203,
+	0x81C, 0x06600203,
+	0x81C, 0x05620203,
+	0x81C, 0x04640203,
+	0x81C, 0x03660203,
+	0x81C, 0x02680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000203,
+	0x81C, 0xFF020203,
+	0x81C, 0xFE040203,
+	0x81C, 0xFD060203,
+	0x81C, 0xFC080203,
+	0x81C, 0xFB0A0203,
+	0x81C, 0xFA0C0203,
+	0x81C, 0xF90E0203,
+	0x81C, 0xF8100203,
+	0x81C, 0xF7120203,
+	0x81C, 0xF6140203,
+	0x81C, 0xF5160203,
+	0x81C, 0xF4180203,
+	0x81C, 0xF31A0203,
+	0x81C, 0xF21C0203,
+	0x81C, 0xF11E0203,
+	0x81C, 0xF0200203,
+	0x81C, 0xEF220203,
+	0x81C, 0xEE240203,
+	0x81C, 0xED260203,
+	0x81C, 0xEC280203,
+	0x81C, 0xEB2A0203,
+	0x81C, 0xEA2C0203,
+	0x81C, 0xE92E0203,
+	0x81C, 0xE8300203,
+	0x81C, 0xE7320203,
+	0x81C, 0xE6340203,
+	0x81C, 0xE5360203,
+	0x81C, 0xE4380203,
+	0x81C, 0xE33A0203,
+	0x81C, 0xE23C0203,
+	0x81C, 0xE13E0203,
+	0x81C, 0xA4400203,
+	0x81C, 0xA3420203,
+	0x81C, 0xA2440203,
+	0x81C, 0xA1460203,
+	0x81C, 0x84480203,
+	0x81C, 0x834A0203,
+	0x81C, 0x824C0203,
+	0x81C, 0x814E0203,
+	0x81C, 0x64500203,
+	0x81C, 0x63520203,
+	0x81C, 0x62540203,
+	0x81C, 0x61560203,
+	0x81C, 0x45580203,
+	0x81C, 0x445A0203,
+	0x81C, 0x435C0203,
+	0x81C, 0x425E0203,
+	0x81C, 0x24600203,
+	0x81C, 0x23620203,
+	0x81C, 0x07640203,
+	0x81C, 0x06660203,
+	0x81C, 0x05680203,
+	0x81C, 0x046A0203,
+	0x81C, 0x036C0203,
+	0x81C, 0x026E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF7000203,
+	0x81C, 0xF6020203,
+	0x81C, 0xF5040203,
+	0x81C, 0xF4060203,
+	0x81C, 0xF3080203,
+	0x81C, 0xF20A0203,
+	0x81C, 0xF10C0203,
+	0x81C, 0xF00E0203,
+	0x81C, 0xEF100203,
+	0x81C, 0xEE120203,
+	0x81C, 0xED140203,
+	0x81C, 0xEC160203,
+	0x81C, 0xEB180203,
+	0x81C, 0xEA1A0203,
+	0x81C, 0xE91C0203,
+	0x81C, 0xE81E0203,
+	0x81C, 0xE7200203,
+	0x81C, 0xE6220203,
+	0x81C, 0xE5240203,
+	0x81C, 0xE4260203,
+	0x81C, 0xE3280203,
+	0x81C, 0xE22A0203,
+	0x81C, 0xA62C0203,
+	0x81C, 0xA52E0203,
+	0x81C, 0xA4300203,
+	0x81C, 0xA3320203,
+	0x81C, 0xA2340203,
+	0x81C, 0xA1360203,
+	0x81C, 0x86380203,
+	0x81C, 0x853A0203,
+	0x81C, 0x843C0203,
+	0x81C, 0x833E0203,
+	0x81C, 0x65400203,
+	0x81C, 0x64420203,
+	0x81C, 0x63440203,
+	0x81C, 0x46460203,
+	0x81C, 0x45480203,
+	0x81C, 0x444A0203,
+	0x81C, 0x434C0203,
+	0x81C, 0x264E0203,
+	0x81C, 0x25500203,
+	0x81C, 0x24520203,
+	0x81C, 0x08540203,
+	0x81C, 0x07560203,
+	0x81C, 0x06580203,
+	0x81C, 0x055A0203,
+	0x81C, 0x045C0203,
+	0x81C, 0x035E0203,
+	0x81C, 0x02600203,
+	0x81C, 0x01620203,
+	0x81C, 0x01640203,
+	0x81C, 0x01660203,
+	0x81C, 0x01680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFF000203,
+	0x81C, 0xFF020203,
+	0x81C, 0xFE040203,
+	0x81C, 0xFD060203,
+	0x81C, 0xFC080203,
+	0x81C, 0xFB0A0203,
+	0x81C, 0xFA0C0203,
+	0x81C, 0xF90E0203,
+	0x81C, 0xF8100203,
+	0x81C, 0xF7120203,
+	0x81C, 0xF6140203,
+	0x81C, 0xF5160203,
+	0x81C, 0xF4180203,
+	0x81C, 0xF31A0203,
+	0x81C, 0xF21C0203,
+	0x81C, 0xF11E0203,
+	0x81C, 0xF0200203,
+	0x81C, 0xEF220203,
+	0x81C, 0xEE240203,
+	0x81C, 0xED260203,
+	0x81C, 0xEC280203,
+	0x81C, 0xEB2A0203,
+	0x81C, 0xEA2C0203,
+	0x81C, 0xE92E0203,
+	0x81C, 0xE8300203,
+	0x81C, 0xE7320203,
+	0x81C, 0xE6340203,
+	0x81C, 0xE5360203,
+	0x81C, 0xE4380203,
+	0x81C, 0xE33A0203,
+	0x81C, 0xE23C0203,
+	0x81C, 0xE13E0203,
+	0x81C, 0xA4400203,
+	0x81C, 0xA3420203,
+	0x81C, 0xA2440203,
+	0x81C, 0xA1460203,
+	0x81C, 0x84480203,
+	0x81C, 0x834A0203,
+	0x81C, 0x824C0203,
+	0x81C, 0x814E0203,
+	0x81C, 0x64500203,
+	0x81C, 0x63520203,
+	0x81C, 0x62540203,
+	0x81C, 0x61560203,
+	0x81C, 0x45580203,
+	0x81C, 0x445A0203,
+	0x81C, 0x435C0203,
+	0x81C, 0x425E0203,
+	0x81C, 0x24600203,
+	0x81C, 0x23620203,
+	0x81C, 0x07640203,
+	0x81C, 0x06660203,
+	0x81C, 0x05680203,
+	0x81C, 0x046A0203,
+	0x81C, 0x036C0203,
+	0x81C, 0x026E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF8000203,
+	0x81C, 0xF7020203,
+	0x81C, 0xF6040203,
+	0x81C, 0xF5060203,
+	0x81C, 0xF4080203,
+	0x81C, 0xF30A0203,
+	0x81C, 0xF20C0203,
+	0x81C, 0xF10E0203,
+	0x81C, 0xF0100203,
+	0x81C, 0xEF120203,
+	0x81C, 0xEE140203,
+	0x81C, 0xED160203,
+	0x81C, 0xEC180203,
+	0x81C, 0xEB1A0203,
+	0x81C, 0xEA1C0203,
+	0x81C, 0xE91E0203,
+	0x81C, 0xE8200203,
+	0x81C, 0xE7220203,
+	0x81C, 0xE6240203,
+	0x81C, 0xE5260203,
+	0x81C, 0xE4280203,
+	0x81C, 0xE32A0203,
+	0x81C, 0xE22C0203,
+	0x81C, 0xE12E0203,
+	0x81C, 0xA6300203,
+	0x81C, 0xA5320203,
+	0x81C, 0xA4340203,
+	0x81C, 0xA3360203,
+	0x81C, 0xA2380203,
+	0x81C, 0xA13A0203,
+	0x81C, 0x843C0203,
+	0x81C, 0x833E0203,
+	0x81C, 0x82400203,
+	0x81C, 0x81420203,
+	0x81C, 0x64440203,
+	0x81C, 0x63460203,
+	0x81C, 0x62480203,
+	0x81C, 0x614A0203,
+	0x81C, 0x444C0203,
+	0x81C, 0x434E0203,
+	0x81C, 0x42500203,
+	0x81C, 0x41520203,
+	0x81C, 0x25540203,
+	0x81C, 0x24560203,
+	0x81C, 0x23580203,
+	0x81C, 0x065A0203,
+	0x81C, 0x055C0203,
+	0x81C, 0x045E0203,
+	0x81C, 0x03600203,
+	0x81C, 0x02620203,
+	0x81C, 0x01640203,
+	0x81C, 0x01660203,
+	0x81C, 0x01680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF9000203,
+	0x81C, 0xF8020203,
+	0x81C, 0xF7040203,
+	0x81C, 0xF6060203,
+	0x81C, 0xF5080203,
+	0x81C, 0xF40A0203,
+	0x81C, 0xF30C0203,
+	0x81C, 0xF20E0203,
+	0x81C, 0xF1100203,
+	0x81C, 0xF0120203,
+	0x81C, 0xEF140203,
+	0x81C, 0xCE160203,
+	0x81C, 0xCD180203,
+	0x81C, 0xCC1A0203,
+	0x81C, 0xCB1C0203,
+	0x81C, 0xCA1E0203,
+	0x81C, 0xC9200203,
+	0x81C, 0xC8220203,
+	0x81C, 0xC7240203,
+	0x81C, 0xC6260203,
+	0x81C, 0xC5280203,
+	0x81C, 0xC42A0203,
+	0x81C, 0xC32C0203,
+	0x81C, 0xC22E0203,
+	0x81C, 0xC1300203,
+	0x81C, 0xA5320203,
+	0x81C, 0xA4340203,
+	0x81C, 0xA3360203,
+	0x81C, 0xA2380203,
+	0x81C, 0xA13A0203,
+	0x81C, 0x853C0203,
+	0x81C, 0x843E0203,
+	0x81C, 0x83400203,
+	0x81C, 0x82420203,
+	0x81C, 0x81440203,
+	0x81C, 0x64460203,
+	0x81C, 0x63480203,
+	0x81C, 0x624A0203,
+	0x81C, 0x614C0203,
+	0x81C, 0x444E0203,
+	0x81C, 0x43500203,
+	0x81C, 0x42520203,
+	0x81C, 0x41540203,
+	0x81C, 0x24560203,
+	0x81C, 0x23580203,
+	0x81C, 0x075A0203,
+	0x81C, 0x065C0203,
+	0x81C, 0x055E0203,
+	0x81C, 0x04600203,
+	0x81C, 0x03620203,
+	0x81C, 0x02640203,
+	0x81C, 0x01660203,
+	0x81C, 0x01680203,
+	0x81C, 0x016A0203,
+	0x81C, 0x016C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0xA0000000,	0x00000000,
+	0x81C, 0xFF000203,
+	0x81C, 0xFF020203,
+	0x81C, 0xFE040203,
+	0x81C, 0xFD060203,
+	0x81C, 0xFC080203,
+	0x81C, 0xFB0A0203,
+	0x81C, 0xFA0C0203,
+	0x81C, 0xF90E0203,
+	0x81C, 0xF8100203,
+	0x81C, 0xF7120203,
+	0x81C, 0xF6140203,
+	0x81C, 0xF5160203,
+	0x81C, 0xF4180203,
+	0x81C, 0xF31A0203,
+	0x81C, 0xF21C0203,
+	0x81C, 0xF11E0203,
+	0x81C, 0xF0200203,
+	0x81C, 0xEF220203,
+	0x81C, 0xEE240203,
+	0x81C, 0xED260203,
+	0x81C, 0xEC280203,
+	0x81C, 0xEB2A0203,
+	0x81C, 0xEA2C0203,
+	0x81C, 0xE92E0203,
+	0x81C, 0xE8300203,
+	0x81C, 0xE7320203,
+	0x81C, 0xE6340203,
+	0x81C, 0xE5360203,
+	0x81C, 0xE4380203,
+	0x81C, 0xE33A0203,
+	0x81C, 0xE23C0203,
+	0x81C, 0xE13E0203,
+	0x81C, 0xA4400203,
+	0x81C, 0xA3420203,
+	0x81C, 0xA2440203,
+	0x81C, 0xA1460203,
+	0x81C, 0x85480203,
+	0x81C, 0x844A0203,
+	0x81C, 0x834C0203,
+	0x81C, 0x824E0203,
+	0x81C, 0x81500203,
+	0x81C, 0x64520203,
+	0x81C, 0x63540203,
+	0x81C, 0x62560203,
+	0x81C, 0x61580203,
+	0x81C, 0x445A0203,
+	0x81C, 0x435C0203,
+	0x81C, 0x425E0203,
+	0x81C, 0x25600203,
+	0x81C, 0x24620203,
+	0x81C, 0x06640203,
+	0x81C, 0x05660203,
+	0x81C, 0x04680203,
+	0x81C, 0x036A0203,
+	0x81C, 0x026C0203,
+	0x81C, 0x016E0203,
+	0x81C, 0x01700203,
+	0x81C, 0x01720203,
+	0x81C, 0x01740203,
+	0x81C, 0x01760203,
+	0x81C, 0x01780203,
+	0x81C, 0x017A0203,
+	0x81C, 0x017C0203,
+	0x81C, 0x017E0203,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF8000303,
+	0x81C, 0xF7020303,
+	0x81C, 0xF6040303,
+	0x81C, 0xF5060303,
+	0x81C, 0xF4080303,
+	0x81C, 0xF30A0303,
+	0x81C, 0xF20C0303,
+	0x81C, 0xF10E0303,
+	0x81C, 0xF0100303,
+	0x81C, 0xEF120303,
+	0x81C, 0xEE140303,
+	0x81C, 0xED160303,
+	0x81C, 0xEC180303,
+	0x81C, 0xEB1A0303,
+	0x81C, 0xEA1C0303,
+	0x81C, 0xE91E0303,
+	0x81C, 0xE8200303,
+	0x81C, 0xE7220303,
+	0x81C, 0xE6240303,
+	0x81C, 0xE5260303,
+	0x81C, 0xE4280303,
+	0x81C, 0xE32A0303,
+	0x81C, 0xE22C0303,
+	0x81C, 0xE12E0303,
+	0x81C, 0xA6300303,
+	0x81C, 0xA5320303,
+	0x81C, 0xA4340303,
+	0x81C, 0xA3360303,
+	0x81C, 0xA2380303,
+	0x81C, 0xA13A0303,
+	0x81C, 0x843C0303,
+	0x81C, 0x833E0303,
+	0x81C, 0x82400303,
+	0x81C, 0x81420303,
+	0x81C, 0x64440303,
+	0x81C, 0x63460303,
+	0x81C, 0x62480303,
+	0x81C, 0x614A0303,
+	0x81C, 0x454C0303,
+	0x81C, 0x444E0303,
+	0x81C, 0x43500303,
+	0x81C, 0x42520303,
+	0x81C, 0x41540303,
+	0x81C, 0x24560303,
+	0x81C, 0x23580303,
+	0x81C, 0x065A0303,
+	0x81C, 0x055C0303,
+	0x81C, 0x045E0303,
+	0x81C, 0x03600303,
+	0x81C, 0x02620303,
+	0x81C, 0x01640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF7000303,
+	0x81C, 0xF6020303,
+	0x81C, 0xF5040303,
+	0x81C, 0xF4060303,
+	0x81C, 0xF3080303,
+	0x81C, 0xF20A0303,
+	0x81C, 0xF10C0303,
+	0x81C, 0xF00E0303,
+	0x81C, 0xEF100303,
+	0x81C, 0xEE120303,
+	0x81C, 0xED140303,
+	0x81C, 0xEC160303,
+	0x81C, 0xEB180303,
+	0x81C, 0xEA1A0303,
+	0x81C, 0xE91C0303,
+	0x81C, 0xE81E0303,
+	0x81C, 0xE7200303,
+	0x81C, 0xE6220303,
+	0x81C, 0xE5240303,
+	0x81C, 0xE4260303,
+	0x81C, 0xE3280303,
+	0x81C, 0xC32A0303,
+	0x81C, 0xC22C0303,
+	0x81C, 0xC12E0303,
+	0x81C, 0xA5300303,
+	0x81C, 0xA4320303,
+	0x81C, 0xA3340303,
+	0x81C, 0xA2360303,
+	0x81C, 0xA1380303,
+	0x81C, 0x853A0303,
+	0x81C, 0x843C0303,
+	0x81C, 0x833E0303,
+	0x81C, 0x82400303,
+	0x81C, 0x81420303,
+	0x81C, 0x64440303,
+	0x81C, 0x63460303,
+	0x81C, 0x62480303,
+	0x81C, 0x614A0303,
+	0x81C, 0x454C0303,
+	0x81C, 0x444E0303,
+	0x81C, 0x43500303,
+	0x81C, 0x25520303,
+	0x81C, 0x24540303,
+	0x81C, 0x23560303,
+	0x81C, 0x06580303,
+	0x81C, 0x055A0303,
+	0x81C, 0x045C0303,
+	0x81C, 0x035E0303,
+	0x81C, 0x02600303,
+	0x81C, 0x01620303,
+	0x81C, 0x01640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF9000303,
+	0x81C, 0xF8020303,
+	0x81C, 0xF7040303,
+	0x81C, 0xF6060303,
+	0x81C, 0xF5080303,
+	0x81C, 0xF40A0303,
+	0x81C, 0xF30C0303,
+	0x81C, 0xF20E0303,
+	0x81C, 0xF1100303,
+	0x81C, 0xF0120303,
+	0x81C, 0xEF140303,
+	0x81C, 0xEE160303,
+	0x81C, 0xED180303,
+	0x81C, 0xEC1A0303,
+	0x81C, 0xEB1C0303,
+	0x81C, 0xEA1E0303,
+	0x81C, 0xE9200303,
+	0x81C, 0xE8220303,
+	0x81C, 0xE7240303,
+	0x81C, 0xE6260303,
+	0x81C, 0xE5280303,
+	0x81C, 0xE42A0303,
+	0x81C, 0xE32C0303,
+	0x81C, 0xE22E0303,
+	0x81C, 0xE1300303,
+	0x81C, 0xA4320303,
+	0x81C, 0xA3340303,
+	0x81C, 0xA2360303,
+	0x81C, 0xA1380303,
+	0x81C, 0x853A0303,
+	0x81C, 0x843C0303,
+	0x81C, 0x833E0303,
+	0x81C, 0x82400303,
+	0x81C, 0x81420303,
+	0x81C, 0x64440303,
+	0x81C, 0x63460303,
+	0x81C, 0x62480303,
+	0x81C, 0x614A0303,
+	0x81C, 0x444C0303,
+	0x81C, 0x434E0303,
+	0x81C, 0x42500303,
+	0x81C, 0x25520303,
+	0x81C, 0x24540303,
+	0x81C, 0x23560303,
+	0x81C, 0x07580303,
+	0x81C, 0x065A0303,
+	0x81C, 0x055C0303,
+	0x81C, 0x045E0303,
+	0x81C, 0x03600303,
+	0x81C, 0x02620303,
+	0x81C, 0x01640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF7000303,
+	0x81C, 0xF6020303,
+	0x81C, 0xF5040303,
+	0x81C, 0xF4060303,
+	0x81C, 0xF3080303,
+	0x81C, 0xF20A0303,
+	0x81C, 0xF10C0303,
+	0x81C, 0xF00E0303,
+	0x81C, 0xEF100303,
+	0x81C, 0xEE120303,
+	0x81C, 0xED140303,
+	0x81C, 0xEC160303,
+	0x81C, 0xEB180303,
+	0x81C, 0xEA1A0303,
+	0x81C, 0xE91C0303,
+	0x81C, 0xE81E0303,
+	0x81C, 0xE7200303,
+	0x81C, 0xE6220303,
+	0x81C, 0xE5240303,
+	0x81C, 0xE4260303,
+	0x81C, 0xE3280303,
+	0x81C, 0xE22A0303,
+	0x81C, 0xE12C0303,
+	0x81C, 0xA72E0303,
+	0x81C, 0xA6300303,
+	0x81C, 0xA5320303,
+	0x81C, 0xA4340303,
+	0x81C, 0xA3360303,
+	0x81C, 0xA2380303,
+	0x81C, 0xA13A0303,
+	0x81C, 0x843C0303,
+	0x81C, 0x833E0303,
+	0x81C, 0x82400303,
+	0x81C, 0x81420303,
+	0x81C, 0x64440303,
+	0x81C, 0x63460303,
+	0x81C, 0x62480303,
+	0x81C, 0x614A0303,
+	0x81C, 0x454C0303,
+	0x81C, 0x444E0303,
+	0x81C, 0x43500303,
+	0x81C, 0x42520303,
+	0x81C, 0x41540303,
+	0x81C, 0x24560303,
+	0x81C, 0x23580303,
+	0x81C, 0x065A0303,
+	0x81C, 0x055C0303,
+	0x81C, 0x045E0303,
+	0x81C, 0x03600303,
+	0x81C, 0x02620303,
+	0x81C, 0x01640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFB000303,
+	0x81C, 0xFA020303,
+	0x81C, 0xF9040303,
+	0x81C, 0xF8060303,
+	0x81C, 0xF7080303,
+	0x81C, 0xF60A0303,
+	0x81C, 0xF50C0303,
+	0x81C, 0xF40E0303,
+	0x81C, 0xF3100303,
+	0x81C, 0xF2120303,
+	0x81C, 0xF1140303,
+	0x81C, 0xF0160303,
+	0x81C, 0xEF180303,
+	0x81C, 0xEE1A0303,
+	0x81C, 0xED1C0303,
+	0x81C, 0xEC1E0303,
+	0x81C, 0xEB200303,
+	0x81C, 0xEA220303,
+	0x81C, 0xE9240303,
+	0x81C, 0xE8260303,
+	0x81C, 0xE7280303,
+	0x81C, 0xE62A0303,
+	0x81C, 0xE52C0303,
+	0x81C, 0xE42E0303,
+	0x81C, 0xE3300303,
+	0x81C, 0xE2320303,
+	0x81C, 0xE1340303,
+	0x81C, 0xC2360303,
+	0x81C, 0xC1380303,
+	0x81C, 0xA33A0303,
+	0x81C, 0xA23C0303,
+	0x81C, 0x853E0303,
+	0x81C, 0x84400303,
+	0x81C, 0x83420303,
+	0x81C, 0x66440303,
+	0x81C, 0x65460303,
+	0x81C, 0x64480303,
+	0x81C, 0x634A0303,
+	0x81C, 0x624C0303,
+	0x81C, 0x614E0303,
+	0x81C, 0x45500303,
+	0x81C, 0x44520303,
+	0x81C, 0x43540303,
+	0x81C, 0x42560303,
+	0x81C, 0x25580303,
+	0x81C, 0x245A0303,
+	0x81C, 0x235C0303,
+	0x81C, 0x065E0303,
+	0x81C, 0x05600303,
+	0x81C, 0x04620303,
+	0x81C, 0x03640303,
+	0x81C, 0x02660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF9000303,
+	0x81C, 0xF8020303,
+	0x81C, 0xF7040303,
+	0x81C, 0xF6060303,
+	0x81C, 0xF5080303,
+	0x81C, 0xF40A0303,
+	0x81C, 0xF30C0303,
+	0x81C, 0xF20E0303,
+	0x81C, 0xF1100303,
+	0x81C, 0xF0120303,
+	0x81C, 0xEF140303,
+	0x81C, 0xEE160303,
+	0x81C, 0xED180303,
+	0x81C, 0xEC1A0303,
+	0x81C, 0xEB1C0303,
+	0x81C, 0xEA1E0303,
+	0x81C, 0xE9200303,
+	0x81C, 0xE8220303,
+	0x81C, 0xE7240303,
+	0x81C, 0xE6260303,
+	0x81C, 0xE5280303,
+	0x81C, 0xE42A0303,
+	0x81C, 0xE32C0303,
+	0x81C, 0xE22E0303,
+	0x81C, 0xE1300303,
+	0x81C, 0xA6320303,
+	0x81C, 0xA5340303,
+	0x81C, 0xA4360303,
+	0x81C, 0xA3380303,
+	0x81C, 0xA23A0303,
+	0x81C, 0xA13C0303,
+	0x81C, 0x853E0303,
+	0x81C, 0x84400303,
+	0x81C, 0x83420303,
+	0x81C, 0x82440303,
+	0x81C, 0x81460303,
+	0x81C, 0x64480303,
+	0x81C, 0x634A0303,
+	0x81C, 0x624C0303,
+	0x81C, 0x614E0303,
+	0x81C, 0x44500303,
+	0x81C, 0x43520303,
+	0x81C, 0x42540303,
+	0x81C, 0x41560303,
+	0x81C, 0x25580303,
+	0x81C, 0x245A0303,
+	0x81C, 0x235C0303,
+	0x81C, 0x055E0303,
+	0x81C, 0x04600303,
+	0x81C, 0x03620303,
+	0x81C, 0x02640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000303,
+	0x81C, 0xFD020303,
+	0x81C, 0xFC040303,
+	0x81C, 0xFB060303,
+	0x81C, 0xFA080303,
+	0x81C, 0xF90A0303,
+	0x81C, 0xF80C0303,
+	0x81C, 0xF70E0303,
+	0x81C, 0xF6100303,
+	0x81C, 0xF5120303,
+	0x81C, 0xF4140303,
+	0x81C, 0xF3160303,
+	0x81C, 0xF2180303,
+	0x81C, 0xF11A0303,
+	0x81C, 0xF01C0303,
+	0x81C, 0xEF1E0303,
+	0x81C, 0xEE200303,
+	0x81C, 0xED220303,
+	0x81C, 0xEC240303,
+	0x81C, 0xEB260303,
+	0x81C, 0xEA280303,
+	0x81C, 0xE92A0303,
+	0x81C, 0xE82C0303,
+	0x81C, 0xE72E0303,
+	0x81C, 0xE6300303,
+	0x81C, 0xE5320303,
+	0x81C, 0xE4340303,
+	0x81C, 0xE3360303,
+	0x81C, 0xC3380303,
+	0x81C, 0xC23A0303,
+	0x81C, 0xC13C0303,
+	0x81C, 0xA43E0303,
+	0x81C, 0xA3400303,
+	0x81C, 0xA2420303,
+	0x81C, 0xA1440303,
+	0x81C, 0x85460303,
+	0x81C, 0x84480303,
+	0x81C, 0x834A0303,
+	0x81C, 0x824C0303,
+	0x81C, 0x814E0303,
+	0x81C, 0x64500303,
+	0x81C, 0x63520303,
+	0x81C, 0x62540303,
+	0x81C, 0x61560303,
+	0x81C, 0x44580303,
+	0x81C, 0x435A0303,
+	0x81C, 0x425C0303,
+	0x81C, 0x265E0303,
+	0x81C, 0x25600303,
+	0x81C, 0x24620303,
+	0x81C, 0x06640303,
+	0x81C, 0x05660303,
+	0x81C, 0x04680303,
+	0x81C, 0x036A0303,
+	0x81C, 0x026C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF7000303,
+	0x81C, 0xF6020303,
+	0x81C, 0xF5040303,
+	0x81C, 0xF4060303,
+	0x81C, 0xF3080303,
+	0x81C, 0xF20A0303,
+	0x81C, 0xF10C0303,
+	0x81C, 0xF00E0303,
+	0x81C, 0xEF100303,
+	0x81C, 0xEE120303,
+	0x81C, 0xED140303,
+	0x81C, 0xEC160303,
+	0x81C, 0xEB180303,
+	0x81C, 0xEA1A0303,
+	0x81C, 0xE91C0303,
+	0x81C, 0xE81E0303,
+	0x81C, 0xE7200303,
+	0x81C, 0xE6220303,
+	0x81C, 0xE5240303,
+	0x81C, 0xE4260303,
+	0x81C, 0xE3280303,
+	0x81C, 0xE22A0303,
+	0x81C, 0xA62C0303,
+	0x81C, 0xA52E0303,
+	0x81C, 0xA4300303,
+	0x81C, 0xA3320303,
+	0x81C, 0xA2340303,
+	0x81C, 0x87360303,
+	0x81C, 0x86380303,
+	0x81C, 0x853A0303,
+	0x81C, 0x843C0303,
+	0x81C, 0x833E0303,
+	0x81C, 0x66400303,
+	0x81C, 0x65420303,
+	0x81C, 0x64440303,
+	0x81C, 0x45460303,
+	0x81C, 0x44480303,
+	0x81C, 0x434A0303,
+	0x81C, 0x274C0303,
+	0x81C, 0x264E0303,
+	0x81C, 0x25500303,
+	0x81C, 0x24520303,
+	0x81C, 0x23540303,
+	0x81C, 0x08560303,
+	0x81C, 0x07580303,
+	0x81C, 0x065A0303,
+	0x81C, 0x055C0303,
+	0x81C, 0x045E0303,
+	0x81C, 0x03600303,
+	0x81C, 0x02620303,
+	0x81C, 0x01640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xFE000303,
+	0x81C, 0xFD020303,
+	0x81C, 0xFC040303,
+	0x81C, 0xFB060303,
+	0x81C, 0xFA080303,
+	0x81C, 0xF90A0303,
+	0x81C, 0xF80C0303,
+	0x81C, 0xF70E0303,
+	0x81C, 0xF6100303,
+	0x81C, 0xF5120303,
+	0x81C, 0xF4140303,
+	0x81C, 0xF3160303,
+	0x81C, 0xF2180303,
+	0x81C, 0xF11A0303,
+	0x81C, 0xF01C0303,
+	0x81C, 0xEF1E0303,
+	0x81C, 0xEE200303,
+	0x81C, 0xED220303,
+	0x81C, 0xEC240303,
+	0x81C, 0xEB260303,
+	0x81C, 0xEA280303,
+	0x81C, 0xE92A0303,
+	0x81C, 0xE82C0303,
+	0x81C, 0xE72E0303,
+	0x81C, 0xE6300303,
+	0x81C, 0xE5320303,
+	0x81C, 0xE4340303,
+	0x81C, 0xE3360303,
+	0x81C, 0xC3380303,
+	0x81C, 0xC23A0303,
+	0x81C, 0xC13C0303,
+	0x81C, 0xA43E0303,
+	0x81C, 0xA3400303,
+	0x81C, 0xA2420303,
+	0x81C, 0xA1440303,
+	0x81C, 0x85460303,
+	0x81C, 0x84480303,
+	0x81C, 0x834A0303,
+	0x81C, 0x824C0303,
+	0x81C, 0x814E0303,
+	0x81C, 0x64500303,
+	0x81C, 0x63520303,
+	0x81C, 0x62540303,
+	0x81C, 0x61560303,
+	0x81C, 0x44580303,
+	0x81C, 0x435A0303,
+	0x81C, 0x425C0303,
+	0x81C, 0x265E0303,
+	0x81C, 0x25600303,
+	0x81C, 0x24620303,
+	0x81C, 0x06640303,
+	0x81C, 0x05660303,
+	0x81C, 0x04680303,
+	0x81C, 0x036A0303,
+	0x81C, 0x026C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF7000303,
+	0x81C, 0xF6020303,
+	0x81C, 0xF5040303,
+	0x81C, 0xF4060303,
+	0x81C, 0xF3080303,
+	0x81C, 0xF20A0303,
+	0x81C, 0xF10C0303,
+	0x81C, 0xF00E0303,
+	0x81C, 0xEF100303,
+	0x81C, 0xEE120303,
+	0x81C, 0xED140303,
+	0x81C, 0xEC160303,
+	0x81C, 0xEB180303,
+	0x81C, 0xEA1A0303,
+	0x81C, 0xE91C0303,
+	0x81C, 0xE81E0303,
+	0x81C, 0xE7200303,
+	0x81C, 0xE6220303,
+	0x81C, 0xE5240303,
+	0x81C, 0xE4260303,
+	0x81C, 0xE3280303,
+	0x81C, 0xE22A0303,
+	0x81C, 0xE12C0303,
+	0x81C, 0xA72E0303,
+	0x81C, 0xA6300303,
+	0x81C, 0xA5320303,
+	0x81C, 0xA4340303,
+	0x81C, 0xA3360303,
+	0x81C, 0xA2380303,
+	0x81C, 0xA13A0303,
+	0x81C, 0x843C0303,
+	0x81C, 0x833E0303,
+	0x81C, 0x82400303,
+	0x81C, 0x81420303,
+	0x81C, 0x64440303,
+	0x81C, 0x63460303,
+	0x81C, 0x62480303,
+	0x81C, 0x614A0303,
+	0x81C, 0x454C0303,
+	0x81C, 0x444E0303,
+	0x81C, 0x43500303,
+	0x81C, 0x42520303,
+	0x81C, 0x41540303,
+	0x81C, 0x24560303,
+	0x81C, 0x23580303,
+	0x81C, 0x065A0303,
+	0x81C, 0x055C0303,
+	0x81C, 0x045E0303,
+	0x81C, 0x03600303,
+	0x81C, 0x02620303,
+	0x81C, 0x01640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x81C, 0xF7000303,
+	0x81C, 0xF6020303,
+	0x81C, 0xF5040303,
+	0x81C, 0xF4060303,
+	0x81C, 0xF3080303,
+	0x81C, 0xF20A0303,
+	0x81C, 0xF10C0303,
+	0x81C, 0xF00E0303,
+	0x81C, 0xEF100303,
+	0x81C, 0xEE120303,
+	0x81C, 0xED140303,
+	0x81C, 0xEC160303,
+	0x81C, 0xEB180303,
+	0x81C, 0xEA1A0303,
+	0x81C, 0xAF1C0303,
+	0x81C, 0xAE1E0303,
+	0x81C, 0xAD200303,
+	0x81C, 0xAC220303,
+	0x81C, 0xAB240303,
+	0x81C, 0xAA260303,
+	0x81C, 0xC5280303,
+	0x81C, 0xC42A0303,
+	0x81C, 0xC32C0303,
+	0x81C, 0xC22E0303,
+	0x81C, 0xA5300303,
+	0x81C, 0xA4320303,
+	0x81C, 0xA3340303,
+	0x81C, 0xA2360303,
+	0x81C, 0xA1380303,
+	0x81C, 0x853A0303,
+	0x81C, 0x843C0303,
+	0x81C, 0x833E0303,
+	0x81C, 0x82400303,
+	0x81C, 0x81420303,
+	0x81C, 0x64440303,
+	0x81C, 0x63460303,
+	0x81C, 0x62480303,
+	0x81C, 0x614A0303,
+	0x81C, 0x444C0303,
+	0x81C, 0x434E0303,
+	0x81C, 0x42500303,
+	0x81C, 0x41520303,
+	0x81C, 0x25540303,
+	0x81C, 0x24560303,
+	0x81C, 0x06580303,
+	0x81C, 0x055A0303,
+	0x81C, 0x045C0303,
+	0x81C, 0x035E0303,
+	0x81C, 0x02600303,
+	0x81C, 0x01620303,
+	0x81C, 0x01640303,
+	0x81C, 0x01660303,
+	0x81C, 0x01680303,
+	0x81C, 0x016A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0xA0000000,	0x00000000,
+	0x81C, 0xFD000303,
+	0x81C, 0xFC020303,
+	0x81C, 0xFB040303,
+	0x81C, 0xFA060303,
+	0x81C, 0xF9080303,
+	0x81C, 0xF80A0303,
+	0x81C, 0xF70C0303,
+	0x81C, 0xF60E0303,
+	0x81C, 0xF5100303,
+	0x81C, 0xF4120303,
+	0x81C, 0xF3140303,
+	0x81C, 0xF2160303,
+	0x81C, 0xF1180303,
+	0x81C, 0xF01A0303,
+	0x81C, 0xEF1C0303,
+	0x81C, 0xEE1E0303,
+	0x81C, 0xED200303,
+	0x81C, 0xEC220303,
+	0x81C, 0xEB240303,
+	0x81C, 0xEA260303,
+	0x81C, 0xE9280303,
+	0x81C, 0xE82A0303,
+	0x81C, 0xE72C0303,
+	0x81C, 0xE62E0303,
+	0x81C, 0xE5300303,
+	0x81C, 0xE4320303,
+	0x81C, 0xE3340303,
+	0x81C, 0xE2360303,
+	0x81C, 0xE1380303,
+	0x81C, 0xA53A0303,
+	0x81C, 0xA43C0303,
+	0x81C, 0xA33E0303,
+	0x81C, 0xA2400303,
+	0x81C, 0xA1420303,
+	0x81C, 0x87440303,
+	0x81C, 0x86460303,
+	0x81C, 0x85480303,
+	0x81C, 0x844A0303,
+	0x81C, 0x834C0303,
+	0x81C, 0x824E0303,
+	0x81C, 0x81500303,
+	0x81C, 0x64520303,
+	0x81C, 0x63540303,
+	0x81C, 0x62560303,
+	0x81C, 0x61580303,
+	0x81C, 0x435A0303,
+	0x81C, 0x425C0303,
+	0x81C, 0x415E0303,
+	0x81C, 0x07600303,
+	0x81C, 0x06620303,
+	0x81C, 0x05640303,
+	0x81C, 0x04660303,
+	0x81C, 0x03680303,
+	0x81C, 0x026A0303,
+	0x81C, 0x016C0303,
+	0x81C, 0x016E0303,
+	0x81C, 0x01700303,
+	0x81C, 0x01720303,
+	0x81C, 0x01740303,
+	0x81C, 0x01760303,
+	0x81C, 0x01780303,
+	0x81C, 0x017A0303,
+	0x81C, 0x017C0303,
+	0x81C, 0x017E0303,
+	0xB0000000,	0x00000000,
+	0xC50, 0x00000022,
+	0xC50, 0x00000020,
+	0xE50, 0x00000022,
+	0xE50, 0x00000020,
+	0x1850, 0x00000022,
+	0x1850, 0x00000020,
+	0x1A50, 0x00000022,
+	0x1A50, 0x00000020,
+};
+
+RTW_DECL_TABLE_PHY_COND(rtw8814a_agc, rtw_phy_cfg_agc);
+
+static const u32 rtw8814a_bb[] = {
+	0x800, 0x9020D010,
+	0x804, 0x08011280,
+	0x808, 0x0E0282FF,
+	0x80C, 0x1000002F,
+	0x80000003,	0x00000000,	0x40000000,	0x00000000,
+	0x810, 0x33303265,
+	0xA0000000,	0x00000000,
+	0x810, 0x33303265,
+	0xB0000000,	0x00000000,
+	0x814, 0x020C3D10,
+	0x818, 0x04A10385,
+	0x820, 0x00000000,
+	0x824, 0x00033E40,
+	0x828, 0x00000000,
+	0x82C, 0x73985170,
+	0x830, 0x79A0EA08,
+	0x834, 0x042E708A,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0x838, 0x86667640,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x838, 0x86667641,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x838, 0x86667641,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x838, 0x86667641,
+	0xA0000000,	0x00000000,
+	0x838, 0x86667640,
+	0xB0000000,	0x00000000,
+	0x83C, 0x9798B9B9,
+	0x840, 0x17578F60,
+	0x844, 0x4BBDFCDE,
+	0x848, 0x5CD07F8B,
+	0x84C, 0x6CFBF7B5,
+	0x850, 0x28834706,
+	0x854, 0x0001520C,
+	0x858, 0x4060C000,
+	0x85C, 0x74210368,
+	0x860, 0x6929C321,
+	0x864, 0x79727432,
+	0x868, 0x8CA7A314,
+	0x86C, 0x438C2878,
+	0x870, 0x44444444,
+	0x874, 0x21612C2E,
+	0x878, 0x00003152,
+	0x87C, 0x000FC000,
+	0x8A0, 0x00000013,
+	0x8A4, 0x7F7F7F7F,
+	0x8A8, 0xA202033E,
+	0x8AC, 0xF40F550A,
+	0x8B0, 0x00000600,
+	0x8B4, 0x000FC080,
+	0x8B8, 0xEC0057FF,
+	0x8BC, 0x8CA520C3,
+	0x8C0, 0x3FF00020,
+	0x8C4, 0x44C00000,
+	0x80000009,	0x00000000,	0x40000000,	0x00000000,
+	0x8C8, 0x80025969,
+	0xA0000000,	0x00000000,
+	0x8C8, 0x80025167,
+	0xB0000000,	0x00000000,
+	0x8CC, 0x08250492,
+	0x8D0, 0x0000B800,
+	0x8D4, 0x940008A0,
+	0x8D8, 0x290B5612,
+	0x8DC, 0x00000000,
+	0x8E0, 0x32316407,
+	0x8E4, 0x4A092925,
+	0x8E8, 0xFFFFC42C,
+	0x8EC, 0x99999999,
+	0x8F0, 0x00009999,
+	0x8F4, 0x00F80FA1,
+	0x8F8, 0x400082C0,
+	0x8FC, 0x00000000,
+	0x900, 0x00400700,
+	0x90C, 0x09004000,
+	0x910, 0x0000FC00,
+	0x914, 0xD6400404,
+	0x918, 0x1C1028C0,
+	0x91C, 0x64B11A1C,
+	0x920, 0xE0767233,
+	0x924, 0x055AA500,
+	0x928, 0x4AB0E4E4,
+	0x92C, 0xFFFE0000,
+	0x930, 0xFFFFFFFE,
+	0x934, 0x001FFFFF,
+	0x938, 0x00008400,
+	0x93C, 0x932C0642,
+	0x940, 0x093E9360,
+	0x944, 0x08000000,
+	0x948, 0x04000000,
+	0x950, 0x02010080,
+	0x954, 0x86510080,
+	0x960, 0x00000000,
+	0x964, 0x00000000,
+	0x968, 0x00000000,
+	0x96C, 0x00000000,
+	0x970, 0x801FFFFF,
+	0x978, 0x00000000,
+	0x97C, 0x00000000,
+	0x980, 0x00000000,
+	0x984, 0x00000000,
+	0x988, 0x00000000,
+	0x98C, 0x03440000,
+	0x990, 0x27100000,
+	0x994, 0xFFFF0100,
+	0x998, 0xFFFFFF5C,
+	0x99C, 0xFFFFFFFF,
+	0x9A0, 0x000000FF,
+	0x9A4, 0x00080080,
+	0x9A8, 0x0C2F0000,
+	0x9AC, 0x00560000,
+	0x9B0, 0x81081008,
+	0x9B4, 0x00000000,
+	0x9B8, 0x01081008,
+	0x9BC, 0x01081008,
+	0x9D0, 0x00000000,
+	0x9D4, 0x00000000,
+	0x9D8, 0x00000000,
+	0x9DC, 0x00000000,
+	0x9E4, 0x00000002,
+	0x9E8, 0x000022D5,
+	0x9FC, 0xEFFFF7FF,
+	0xB00, 0xE3100000,
+	0xB04, 0x0000B000,
+	0xB0C, 0x31EAA006,
+	0xB5C, 0x41CFFFFF,
+	0xC00, 0x00000007,
+	0xC04, 0x00042020,
+	0xC08, 0x80410231,
+	0xC0C, 0x00000000,
+	0xC10, 0x00000100,
+	0xC14, 0x01000000,
+	0xC1C, 0x40000053,
+	0xC50, 0x00000020,
+	0xC54, 0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0xC58, 0x3C0A0C14,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0xC58, 0x3C0A0C14,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0xC58, 0x3C0A0C14,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xC58, 0x3C0A0C14,
+	0xA0000000,	0x00000000,
+	0xC58, 0x3C020C14,
+	0xB0000000,	0x00000000,
+	0xC5C, 0x0D000058,
+	0xC60, 0x1B800000,
+	0xC60, 0x0B800001,
+	0xC60, 0x05800002,
+	0xC60, 0x07800003,
+	0xC60, 0x1A800004,
+	0xC60, 0x0B800005,
+	0xC60, 0x05800006,
+	0xC60, 0x0E800007,
+	0xC60, 0x1A800008,
+	0xC60, 0x0B800009,
+	0xC60, 0x1580000A,
+	0xC60, 0x0880000B,
+	0xC60, 0x1A80000C,
+	0xC60, 0x0B80000D,
+	0xC60, 0x0580000E,
+	0xC60, 0x0E80000F,
+	0xC60, 0x1A800010,
+	0xC60, 0x0B800011,
+	0xC60, 0x15800012,
+	0xC60, 0x08800013,
+	0xC60, 0x1A800014,
+	0xC60, 0x0B800015,
+	0xC60, 0x05800016,
+	0xC60, 0x07800017,
+	0xC60, 0x1A800018,
+	0xC60, 0x0B800019,
+	0xC60, 0x1580001A,
+	0xC60, 0x0880001B,
+	0xC60, 0x1B80001C,
+	0xC60, 0x0B80001D,
+	0xC60, 0x0580001E,
+	0xC60, 0x0780001F,
+	0xC60, 0x1B800020,
+	0xC60, 0x0B800021,
+	0xC60, 0x05800022,
+	0xC60, 0x07800023,
+	0xC60, 0x1B800024,
+	0xC60, 0x0B800025,
+	0xC60, 0x05800026,
+	0xC60, 0x07800027,
+	0xC60, 0x1B800028,
+	0xC60, 0x0B800029,
+	0xC60, 0x0580002A,
+	0xC60, 0x0780002B,
+	0xC60, 0x1B800030,
+	0xC60, 0x0B800031,
+	0xC60, 0x05800032,
+	0xC60, 0x00800033,
+	0xC60, 0x1B800034,
+	0xC60, 0x0B800035,
+	0xC60, 0x05800036,
+	0xC60, 0x00800037,
+	0xC60, 0x1B800038,
+	0xC60, 0x0B800039,
+	0xC60, 0x0580003A,
+	0xC60, 0x0E80803B,
+	0xC94, 0x01000401,
+	0xC98, 0x00188000,
+	0xCA0, 0x00002929,
+	0xCA4, 0x08040201,
+	0xCA8, 0x80402010,
+	0xCAC, 0x77777000,
+	0xCB0, 0x54775477,
+	0xCB4, 0x54775477,
+	0xCB8, 0x00500000,
+	0xCBC, 0x77700000,
+	0xCC0, 0x00000010,
+	0xCC8, 0x00000010,
+	0xE00, 0x00000007,
+	0xE04, 0x00042020,
+	0xE08, 0x80410231,
+	0xE0C, 0x00000000,
+	0xE10, 0x00000100,
+	0xE14, 0x01000000,
+	0xE1C, 0x40000053,
+	0xE50, 0x00000020,
+	0xE54, 0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0xE58, 0x3C0A0C14,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0xE58, 0x3C0A0C14,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0xE58, 0x3C0A0C14,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xE58, 0x3C0A0C14,
+	0xA0000000,	0x00000000,
+	0xE58, 0x3C020C14,
+	0xB0000000,	0x00000000,
+	0xE5C, 0x0D000058,
+	0xE60, 0x1B800000,
+	0xE60, 0x0B800001,
+	0xE60, 0x05800002,
+	0xE60, 0x07800003,
+	0xE60, 0x1A800004,
+	0xE60, 0x0B800005,
+	0xE60, 0x05800006,
+	0xE60, 0x0E800007,
+	0xE60, 0x1A800008,
+	0xE60, 0x0B800009,
+	0xE60, 0x1580000A,
+	0xE60, 0x0880000B,
+	0xE60, 0x1A80000C,
+	0xE60, 0x0B80000D,
+	0xE60, 0x0580000E,
+	0xE60, 0x0E80000F,
+	0xE60, 0x1A800010,
+	0xE60, 0x0B800011,
+	0xE60, 0x15800012,
+	0xE60, 0x08800013,
+	0xE60, 0x1A800014,
+	0xE60, 0x0B800015,
+	0xE60, 0x05800016,
+	0xE60, 0x07800017,
+	0xE60, 0x1A800018,
+	0xE60, 0x0B800019,
+	0xE60, 0x1580001A,
+	0xE60, 0x0880001B,
+	0xE60, 0x1B80001C,
+	0xE60, 0x0B80001D,
+	0xE60, 0x0580001E,
+	0xE60, 0x0780001F,
+	0xE60, 0x1B800020,
+	0xE60, 0x0B800021,
+	0xE60, 0x05800022,
+	0xE60, 0x07800023,
+	0xE60, 0x1B800024,
+	0xE60, 0x0B800025,
+	0xE60, 0x05800026,
+	0xE60, 0x07800027,
+	0xE60, 0x1B800028,
+	0xE60, 0x0B800029,
+	0xE60, 0x0580002A,
+	0xE60, 0x0780002B,
+	0xE60, 0x1B800030,
+	0xE60, 0x0B800031,
+	0xE60, 0x05800032,
+	0xE60, 0x00800033,
+	0xE60, 0x1B800034,
+	0xE60, 0x0B800035,
+	0xE60, 0x05800036,
+	0xE60, 0x00800037,
+	0xE60, 0x1B800038,
+	0xE60, 0x0B800039,
+	0xE60, 0x0580003A,
+	0xE60, 0x0E80803B,
+	0xE94, 0x01000401,
+	0xE98, 0x00188000,
+	0xEA0, 0x00002929,
+	0xEA4, 0x08040201,
+	0xEA8, 0x80402010,
+	0xEAC, 0x77777000,
+	0xEB0, 0x54775477,
+	0xEB4, 0x54775477,
+	0xEB8, 0x00500000,
+	0xEBC, 0x77700000,
+	0x1800, 0x00000007,
+	0x1804, 0x00042020,
+	0x1808, 0x80410231,
+	0x180C, 0x00000000,
+	0x1810, 0x00000100,
+	0x1814, 0x01000000,
+	0x181C, 0x40000053,
+	0x1850, 0x00000020,
+	0x1854, 0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1858, 0x3C0A0C14,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1858, 0x3C0A0C14,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1858, 0x3C0A0C14,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1858, 0x3C0A0C14,
+	0xA0000000,	0x00000000,
+	0x1858, 0x3C020C14,
+	0xB0000000,	0x00000000,
+	0x185C, 0x0D000058,
+	0x1860, 0x1B800000,
+	0x1860, 0x0B800001,
+	0x1860, 0x05800002,
+	0x1860, 0x07800003,
+	0x1860, 0x1A800004,
+	0x1860, 0x0B800005,
+	0x1860, 0x05800006,
+	0x1860, 0x0E800007,
+	0x1860, 0x1A800008,
+	0x1860, 0x0B800009,
+	0x1860, 0x1580000A,
+	0x1860, 0x0880000B,
+	0x1860, 0x1A80000C,
+	0x1860, 0x0B80000D,
+	0x1860, 0x0580000E,
+	0x1860, 0x0E80000F,
+	0x1860, 0x1A800010,
+	0x1860, 0x0B800011,
+	0x1860, 0x15800012,
+	0x1860, 0x08800013,
+	0x1860, 0x1A800014,
+	0x1860, 0x0B800015,
+	0x1860, 0x05800016,
+	0x1860, 0x07800017,
+	0x1860, 0x1A800018,
+	0x1860, 0x0B800019,
+	0x1860, 0x1580001A,
+	0x1860, 0x0880001B,
+	0x1860, 0x1B80001C,
+	0x1860, 0x0B80001D,
+	0x1860, 0x0580001E,
+	0x1860, 0x0780001F,
+	0x1860, 0x1B800020,
+	0x1860, 0x0B800021,
+	0x1860, 0x05800022,
+	0x1860, 0x07800023,
+	0x1860, 0x1B800024,
+	0x1860, 0x0B800025,
+	0x1860, 0x05800026,
+	0x1860, 0x07800027,
+	0x1860, 0x1B800028,
+	0x1860, 0x0B800029,
+	0x1860, 0x0580002A,
+	0x1860, 0x0780002B,
+	0x1860, 0x1B800030,
+	0x1860, 0x0B800031,
+	0x1860, 0x05800032,
+	0x1860, 0x00800033,
+	0x1860, 0x1B800034,
+	0x1860, 0x0B800035,
+	0x1860, 0x05800036,
+	0x1860, 0x00800037,
+	0x1860, 0x1B800038,
+	0x1860, 0x0B800039,
+	0x1860, 0x0580003A,
+	0x1860, 0x0E80803B,
+	0x1894, 0x01000401,
+	0x1898, 0x00188000,
+	0x18A0, 0x00002929,
+	0x18A4, 0x08040201,
+	0x18A8, 0x80402010,
+	0x18AC, 0x77777000,
+	0x18B0, 0x54775477,
+	0x18B4, 0x54775477,
+	0x18B8, 0x00500000,
+	0x18BC, 0x77700000,
+	0x1A00, 0x00000007,
+	0x1A04, 0x00042020,
+	0x1A08, 0x80410231,
+	0x1A0C, 0x00000000,
+	0x1A10, 0x00000100,
+	0x1A14, 0x01000000,
+	0x1A1C, 0x40000053,
+	0x1A50, 0x00000020,
+	0x1A54, 0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1A58, 0x3C0A0C14,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1A58, 0x3C0A0C14,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1A58, 0x3C0A0C14,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1A58, 0x3C0A0C14,
+	0xA0000000,	0x00000000,
+	0x1A58, 0x3C020C14,
+	0xB0000000,	0x00000000,
+	0x1A5C, 0x0D000058,
+	0x1A60, 0x1B800000,
+	0x1A60, 0x0B800001,
+	0x1A60, 0x05800002,
+	0x1A60, 0x07800003,
+	0x1A60, 0x1A800004,
+	0x1A60, 0x0B800005,
+	0x1A60, 0x05800006,
+	0x1A60, 0x0E800007,
+	0x1A60, 0x1A800008,
+	0x1A60, 0x0B800009,
+	0x1A60, 0x1580000A,
+	0x1A60, 0x0880000B,
+	0x1A60, 0x1A80000C,
+	0x1A60, 0x0B80000D,
+	0x1A60, 0x0580000E,
+	0x1A60, 0x0E80000F,
+	0x1A60, 0x1A800010,
+	0x1A60, 0x0B800011,
+	0x1A60, 0x15800012,
+	0x1A60, 0x08800013,
+	0x1A60, 0x1A800014,
+	0x1A60, 0x0B800015,
+	0x1A60, 0x05800016,
+	0x1A60, 0x07800017,
+	0x1A60, 0x1A800018,
+	0x1A60, 0x0B800019,
+	0x1A60, 0x1580001A,
+	0x1A60, 0x0880001B,
+	0x1A60, 0x1B80001C,
+	0x1A60, 0x0B80001D,
+	0x1A60, 0x0580001E,
+	0x1A60, 0x0780001F,
+	0x1A60, 0x1B800020,
+	0x1A60, 0x0B800021,
+	0x1A60, 0x05800022,
+	0x1A60, 0x07800023,
+	0x1A60, 0x1B800024,
+	0x1A60, 0x0B800025,
+	0x1A60, 0x05800026,
+	0x1A60, 0x07800027,
+	0x1A60, 0x1B800028,
+	0x1A60, 0x0B800029,
+	0x1A60, 0x0580002A,
+	0x1A60, 0x0780002B,
+	0x1A60, 0x1B800030,
+	0x1A60, 0x0B800031,
+	0x1A60, 0x05800032,
+	0x1A60, 0x00800033,
+	0x1A60, 0x1B800034,
+	0x1A60, 0x0B800035,
+	0x1A60, 0x05800036,
+	0x1A60, 0x00800037,
+	0x1A60, 0x1B800038,
+	0x1A60, 0x0B800039,
+	0x1A60, 0x0580003A,
+	0x1A60, 0x0E80803B,
+	0x1A94, 0x01000401,
+	0x1A98, 0x00188000,
+	0x1AA0, 0x00002929,
+	0x1AA4, 0x08040201,
+	0x1AA8, 0x80402010,
+	0x1AAC, 0x77777000,
+	0x1AB0, 0x54775477,
+	0x1AB4, 0x54775477,
+	0x1AB8, 0x00500000,
+	0x1ABC, 0x77700000,
+	0x1904, 0x00030000,
+	0x1914, 0x00030000,
+	0x1984, 0x03000000,
+	0x1988, 0x00000087,
+	0x198C, 0x00000007,
+	0x1990, 0xFFAA5500,
+	0x1994, 0x00000077,
+	0x1998, 0x12801000,
+	0x1998, 0x12801000,
+	0x1998, 0x12801001,
+	0x1998, 0x12801002,
+	0x1998, 0x12801003,
+	0x1998, 0x12801004,
+	0x1998, 0x12801005,
+	0x1998, 0x12801006,
+	0x1998, 0x12801007,
+	0x1998, 0x12801008,
+	0x1998, 0x12801009,
+	0x1998, 0x1280100A,
+	0x1998, 0x1280100B,
+	0x1998, 0x1280100C,
+	0x1998, 0x1280100D,
+	0x1998, 0x1280100E,
+	0x1998, 0x1280100F,
+	0x1998, 0x12801010,
+	0x1998, 0x12801011,
+	0x1998, 0x12801012,
+	0x1998, 0x12801013,
+	0x1998, 0x12801014,
+	0x1998, 0x12801015,
+	0x1998, 0x12801016,
+	0x1998, 0x12801017,
+	0x1998, 0x12801018,
+	0x1998, 0x12801019,
+	0x1998, 0x1280101A,
+	0x1998, 0x1280101B,
+	0x1998, 0x1280101C,
+	0x1998, 0x1280101D,
+	0x1998, 0x1280101E,
+	0x1998, 0x1280101F,
+	0x1998, 0x12801020,
+	0x1998, 0x12801021,
+	0x1998, 0x12801022,
+	0x1998, 0x12801023,
+	0x1998, 0x1280102C,
+	0x1998, 0x1280102D,
+	0x1998, 0x1280102E,
+	0x1998, 0x1280102F,
+	0x1998, 0x12801030,
+	0x1998, 0x12801031,
+	0x1998, 0x12801032,
+	0x1998, 0x12801033,
+	0x1998, 0x12801034,
+	0x1998, 0x12801035,
+	0x1998, 0x12801036,
+	0x1998, 0x12801037,
+	0x1998, 0x12801038,
+	0x1998, 0x12801039,
+	0x1998, 0x1280103A,
+	0x1998, 0x1280103B,
+	0x1998, 0x1280103C,
+	0x1998, 0x1280103D,
+	0x1998, 0x1280103E,
+	0x1998, 0x1280103F,
+	0x1998, 0x12801040,
+	0x1998, 0x12801041,
+	0x1998, 0x12801042,
+	0x1998, 0x12801043,
+	0x1998, 0x12801044,
+	0x1998, 0x12801045,
+	0x1998, 0x12801046,
+	0x1998, 0x12801047,
+	0x1998, 0x12801048,
+	0x1998, 0x12801049,
+	0x1998, 0x12801100,
+	0x1998, 0x12801101,
+	0x1998, 0x12801102,
+	0x1998, 0x12801103,
+	0x1998, 0x12801104,
+	0x1998, 0x12801105,
+	0x1998, 0x12801106,
+	0x1998, 0x12801107,
+	0x1998, 0x12801108,
+	0x1998, 0x12801109,
+	0x1998, 0x1280110A,
+	0x1998, 0x1280110B,
+	0x1998, 0x1280110C,
+	0x1998, 0x1280110D,
+	0x1998, 0x1280110E,
+	0x1998, 0x1280110F,
+	0x1998, 0x12801110,
+	0x1998, 0x12801111,
+	0x1998, 0x12801112,
+	0x1998, 0x12801113,
+	0x1998, 0x12801114,
+	0x1998, 0x12801115,
+	0x1998, 0x12801116,
+	0x1998, 0x12801117,
+	0x1998, 0x12801118,
+	0x1998, 0x12801119,
+	0x1998, 0x1280111A,
+	0x1998, 0x1280111B,
+	0x1998, 0x1280111C,
+	0x1998, 0x1280111D,
+	0x1998, 0x1280111E,
+	0x1998, 0x1280111F,
+	0x1998, 0x12801120,
+	0x1998, 0x12801121,
+	0x1998, 0x12801122,
+	0x1998, 0x12801123,
+	0x1998, 0x1280112C,
+	0x1998, 0x1280112D,
+	0x1998, 0x1280112E,
+	0x1998, 0x1280112F,
+	0x1998, 0x12801130,
+	0x1998, 0x12801131,
+	0x1998, 0x12801132,
+	0x1998, 0x12801133,
+	0x1998, 0x12801134,
+	0x1998, 0x12801135,
+	0x1998, 0x12801136,
+	0x1998, 0x12801137,
+	0x1998, 0x12801138,
+	0x1998, 0x12801139,
+	0x1998, 0x1280113A,
+	0x1998, 0x1280113B,
+	0x1998, 0x1280113C,
+	0x1998, 0x1280113D,
+	0x1998, 0x1280113E,
+	0x1998, 0x1280113F,
+	0x1998, 0x12801140,
+	0x1998, 0x12801141,
+	0x1998, 0x12801142,
+	0x1998, 0x12801143,
+	0x1998, 0x12801144,
+	0x1998, 0x12801145,
+	0x1998, 0x12801146,
+	0x1998, 0x12801147,
+	0x1998, 0x12801148,
+	0x1998, 0x12801149,
+	0x1998, 0x12801200,
+	0x1998, 0x12801201,
+	0x1998, 0x12801202,
+	0x1998, 0x12801203,
+	0x1998, 0x12801204,
+	0x1998, 0x12801205,
+	0x1998, 0x12801206,
+	0x1998, 0x12801207,
+	0x1998, 0x12801208,
+	0x1998, 0x12801209,
+	0x1998, 0x1280120A,
+	0x1998, 0x1280120B,
+	0x1998, 0x1280120C,
+	0x1998, 0x1280120D,
+	0x1998, 0x1280120E,
+	0x1998, 0x1280120F,
+	0x1998, 0x12801210,
+	0x1998, 0x12801211,
+	0x1998, 0x12801212,
+	0x1998, 0x12801213,
+	0x1998, 0x12801214,
+	0x1998, 0x12801215,
+	0x1998, 0x12801216,
+	0x1998, 0x12801217,
+	0x1998, 0x12801218,
+	0x1998, 0x12801219,
+	0x1998, 0x1280121A,
+	0x1998, 0x1280121B,
+	0x1998, 0x1280121C,
+	0x1998, 0x1280121D,
+	0x1998, 0x1280121E,
+	0x1998, 0x1280121F,
+	0x1998, 0x12801220,
+	0x1998, 0x12801221,
+	0x1998, 0x12801222,
+	0x1998, 0x12801223,
+	0x1998, 0x1280122C,
+	0x1998, 0x1280122D,
+	0x1998, 0x1280122E,
+	0x1998, 0x1280122F,
+	0x1998, 0x12801230,
+	0x1998, 0x12801231,
+	0x1998, 0x12801232,
+	0x1998, 0x12801233,
+	0x1998, 0x12801234,
+	0x1998, 0x12801235,
+	0x1998, 0x12801236,
+	0x1998, 0x12801237,
+	0x1998, 0x12801238,
+	0x1998, 0x12801239,
+	0x1998, 0x1280123A,
+	0x1998, 0x1280123B,
+	0x1998, 0x1280123C,
+	0x1998, 0x1280123D,
+	0x1998, 0x1280123E,
+	0x1998, 0x1280123F,
+	0x1998, 0x12801240,
+	0x1998, 0x12801241,
+	0x1998, 0x12801242,
+	0x1998, 0x12801243,
+	0x1998, 0x12801244,
+	0x1998, 0x12801245,
+	0x1998, 0x12801246,
+	0x1998, 0x12801247,
+	0x1998, 0x12801248,
+	0x1998, 0x12801249,
+	0x1998, 0x12801300,
+	0x1998, 0x12801301,
+	0x1998, 0x12801302,
+	0x1998, 0x12801303,
+	0x1998, 0x12801304,
+	0x1998, 0x12801305,
+	0x1998, 0x12801306,
+	0x1998, 0x12801307,
+	0x1998, 0x12801308,
+	0x1998, 0x12801309,
+	0x1998, 0x1280130A,
+	0x1998, 0x1280130B,
+	0x1998, 0x1280130C,
+	0x1998, 0x1280130D,
+	0x1998, 0x1280130E,
+	0x1998, 0x1280130F,
+	0x1998, 0x12801310,
+	0x1998, 0x12801311,
+	0x1998, 0x12801312,
+	0x1998, 0x12801313,
+	0x1998, 0x12801314,
+	0x1998, 0x12801315,
+	0x1998, 0x12801316,
+	0x1998, 0x12801317,
+	0x1998, 0x12801318,
+	0x1998, 0x12801319,
+	0x1998, 0x1280131A,
+	0x1998, 0x1280131B,
+	0x1998, 0x1280131C,
+	0x1998, 0x1280131D,
+	0x1998, 0x1280131E,
+	0x1998, 0x1280131F,
+	0x1998, 0x12801320,
+	0x1998, 0x12801321,
+	0x1998, 0x12801322,
+	0x1998, 0x12801323,
+	0x1998, 0x1280132C,
+	0x1998, 0x1280132D,
+	0x1998, 0x1280132E,
+	0x1998, 0x1280132F,
+	0x1998, 0x12801330,
+	0x1998, 0x12801331,
+	0x1998, 0x12801332,
+	0x1998, 0x12801333,
+	0x1998, 0x12801334,
+	0x1998, 0x12801335,
+	0x1998, 0x12801336,
+	0x1998, 0x12801337,
+	0x1998, 0x12801338,
+	0x1998, 0x12801339,
+	0x1998, 0x1280133A,
+	0x1998, 0x1280133B,
+	0x1998, 0x1280133C,
+	0x1998, 0x1280133D,
+	0x1998, 0x1280133E,
+	0x1998, 0x1280133F,
+	0x1998, 0x12801340,
+	0x1998, 0x12801341,
+	0x1998, 0x12801342,
+	0x1998, 0x12801343,
+	0x1998, 0x12801344,
+	0x1998, 0x12801345,
+	0x1998, 0x12801346,
+	0x1998, 0x12801347,
+	0x1998, 0x12801348,
+	0x1998, 0x12801349,
+	0x19D4, 0x88888888,
+	0x19D8, 0x00000888,
+	0xB00, 0xE3100100,
+	0xB00, 0xE7100100,
+	0xC60, 0x15808002,
+	0xC60, 0x01808003,
+	0xE60, 0x15808002,
+	0xE60, 0x01808003,
+	0x1860, 0x15808002,
+	0x1860, 0x01808003,
+	0x1A60, 0x15808002,
+	0x1A60, 0x01808003,
+	0xB00, 0xE3100100,
+	0xC5C, 0x0D080058,
+	0xE5C, 0x0D080058,
+	0x185C, 0x0D080058,
+	0x1A5C, 0x0D080058,
+	0xC5C, 0x0D000058,
+	0xE5C, 0x0D000058,
+	0x185C, 0x0D000058,
+	0x1A5C, 0x0D000058,
+	0xC60, 0x05808002,
+	0xC60, 0x0E808003,
+	0xE60, 0x05808002,
+	0xE60, 0x0E808003,
+	0x1860, 0x05808002,
+	0x1860, 0x0E808003,
+	0x1A60, 0x05808002,
+	0x1A60, 0x0E808003,
+	0xB00, 0xE7100100,
+	0xB00, 0xE3100100,
+	0xB00, 0xE3100000,
+	0x1C38, 0x00000002,
+	0xA00, 0x00D047C8,
+	0xA04, 0x46FF800C,
+	0xA08, 0x8C838300,
+	0xA0C, 0x2E7E000F,
+	0xA10, 0x9500BB78,
+	0xA14, 0x11144028,
+	0xA18, 0x00881117,
+	0xA1C, 0x89140F00,
+	0xA20, 0x1A1B0030,
+	0xA24, 0x090E1317,
+	0xA28, 0x00000204,
+	0xA2C, 0x00900000,
+	0xA70, 0x101FFF00,
+	0xA74, 0x00000128,
+	0xA78, 0x00000900,
+	0xA7C, 0x225B0606,
+	0xA80, 0x218075B2,
+	0xA84, 0x9C1F8C00,
+	0x1B04, 0xE24628D2,
+	0x1B10, 0x88010D46,
+	0x1B14, 0x00000000,
+	0x1B18, 0x00292903,
+	0x1B00, 0xF8000000,
+	0x1B00, 0xF800D000,
+	0x1B00, 0xF801F000,
+	0x1B1C, 0xA2123DB2,
+	0x1B20, 0x07040001,
+	0x1B24, 0x07060807,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0xA0000000,	0x00000000,
+	0x1B28, 0xC0060348,
+	0xB0000000,	0x00000000,
+	0x1B2C, 0x20000003,
+	0x1B30, 0x20000000,
+	0x1B38, 0x20000000,
+	0x1B3C, 0x20000000,
+	0x1BD4, 0x00000001,
+	0x1B94, 0x80000000,
+	0x1B34, 0x00000000,
+	0x1B34, 0x00000002,
+	0x1B34, 0x00000000,
+	0x1B00, 0xF8000002,
+	0x1B00, 0xF800D002,
+	0x1B00, 0xF801F002,
+	0x1B1C, 0xA2123DB2,
+	0x1B20, 0x07040001,
+	0x1B24, 0x07060807,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0xA0000000,	0x00000000,
+	0x1B28, 0xC0060348,
+	0xB0000000,	0x00000000,
+	0x1B2C, 0x20000003,
+	0x1B30, 0x20000000,
+	0x1B38, 0x20000000,
+	0x1B3C, 0x20000000,
+	0x1BD4, 0x00000001,
+	0x1B94, 0x80000000,
+	0x1B34, 0x00000000,
+	0x1B34, 0x00000002,
+	0x1B34, 0x00000000,
+	0x1B00, 0xF8000004,
+	0x1B00, 0xF800D004,
+	0x1B00, 0xF801F004,
+	0x1B1C, 0xA2123DB2,
+	0x1B20, 0x07040001,
+	0x1B24, 0x07060807,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0xA0000000,	0x00000000,
+	0x1B28, 0xC0060348,
+	0xB0000000,	0x00000000,
+	0x1B2C, 0x20000003,
+	0x1B30, 0x20000000,
+	0x1B38, 0x20000000,
+	0x1B3C, 0x20000000,
+	0x1BD4, 0x00000001,
+	0x1B94, 0x80000000,
+	0x1B34, 0x00000000,
+	0x1B34, 0x00000002,
+	0x1B34, 0x00000000,
+	0x1B00, 0xF8000006,
+	0x1B00, 0xF800D006,
+	0x1B00, 0xF801F006,
+	0x1B1C, 0xA2123DB2,
+	0x1B20, 0x07040001,
+	0x1B24, 0x07060807,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B28, 0xC0060324,
+	0xA0000000,	0x00000000,
+	0x1B28, 0xC0060348,
+	0xB0000000,	0x00000000,
+	0x1B2C, 0x20000003,
+	0x1B30, 0x20000000,
+	0x1B38, 0x20000000,
+	0x1B3C, 0x20000000,
+	0x1BD4, 0x00000001,
+	0x1B94, 0x80000000,
+	0x1B34, 0x00000000,
+	0x1B34, 0x00000002,
+	0x1B34, 0x00000000,
+	0x1B00, 0xF8000000,
+	0x1B80, 0x00000007,
+	0x1B80, 0x09060005,
+	0x1B80, 0x09060007,
+	0x1B80, 0x0FFE0015,
+	0x1B80, 0x0FFE0017,
+	0x1B80, 0x00240025,
+	0x1B80, 0x00240027,
+	0x1B80, 0x00040035,
+	0x1B80, 0x00040037,
+	0x1B80, 0x05C00045,
+	0x1B80, 0x05C00047,
+	0x1B80, 0x00070055,
+	0x1B80, 0x00070057,
+	0x1B80, 0x64000065,
+	0x1B80, 0x64000067,
+	0x1B80, 0x00020075,
+	0x1B80, 0x00020077,
+	0x1B80, 0x00080085,
+	0x1B80, 0x00080087,
+	0x1B80, 0x80000095,
+	0x1B80, 0x80000097,
+	0x1B80, 0x090100A5,
+	0x1B80, 0x090100A7,
+	0x1B80, 0x0F0200B5,
+	0x1B80, 0x0F0200B7,
+	0x1B80, 0x002400C5,
+	0x1B80, 0x002400C7,
+	0x1B80, 0x000400D5,
+	0x1B80, 0x000400D7,
+	0x1B80, 0x05C000E5,
+	0x1B80, 0x05C000E7,
+	0x1B80, 0x000700F5,
+	0x1B80, 0x000700F7,
+	0x1B80, 0x64020105,
+	0x1B80, 0x64020107,
+	0x1B80, 0x00020115,
+	0x1B80, 0x00020117,
+	0x1B80, 0x00040125,
+	0x1B80, 0x00040127,
+	0x1B80, 0x4A000135,
+	0x1B80, 0x4A000137,
+	0x1B80, 0x4B040145,
+	0x1B80, 0x4B040147,
+	0x1B80, 0x85030155,
+	0x1B80, 0x85030157,
+	0x1B80, 0x40010165,
+	0x1B80, 0x40010167,
+	0x1B80, 0xE0290175,
+	0x1B80, 0xE0290177,
+	0x1B80, 0x00040185,
+	0x1B80, 0x00040187,
+	0x1B80, 0x4B050195,
+	0x1B80, 0x4B050197,
+	0x1B80, 0x860301A5,
+	0x1B80, 0x860301A7,
+	0x1B80, 0x400301B5,
+	0x1B80, 0x400301B7,
+	0x1B80, 0xE02901C5,
+	0x1B80, 0xE02901C7,
+	0x1B80, 0x000401D5,
+	0x1B80, 0x000401D7,
+	0x1B80, 0x4B0601E5,
+	0x1B80, 0x4B0601E7,
+	0x1B80, 0x870301F5,
+	0x1B80, 0x870301F7,
+	0x1B80, 0x40050205,
+	0x1B80, 0x40050207,
+	0x1B80, 0xE0290215,
+	0x1B80, 0xE0290217,
+	0x1B80, 0x00040225,
+	0x1B80, 0x00040227,
+	0x1B80, 0x4B070235,
+	0x1B80, 0x4B070237,
+	0x1B80, 0x88030245,
+	0x1B80, 0x88030247,
+	0x1B80, 0x40070255,
+	0x1B80, 0x40070257,
+	0x1B80, 0xE0290265,
+	0x1B80, 0xE0290267,
+	0x1B80, 0x4B000275,
+	0x1B80, 0x4B000277,
+	0x1B80, 0x30000285,
+	0x1B80, 0x30000287,
+	0x1B80, 0xFE100295,
+	0x1B80, 0xFE100297,
+	0x1B80, 0xFF1002A5,
+	0x1B80, 0xFF1002A7,
+	0x1B80, 0xE18602B5,
+	0x1B80, 0xE18602B7,
+	0x1B80, 0xF00A02C5,
+	0x1B80, 0xF00A02C7,
+	0x1B80, 0xF10A02D5,
+	0x1B80, 0xF10A02D7,
+	0x1B80, 0xF20A02E5,
+	0x1B80, 0xF20A02E7,
+	0x1B80, 0xF30802F5,
+	0x1B80, 0xF30802F7,
+	0x1B80, 0xF4070305,
+	0x1B80, 0xF4070307,
+	0x1B80, 0xF5060315,
+	0x1B80, 0xF5060317,
+	0x1B80, 0xF7060325,
+	0x1B80, 0xF7060327,
+	0x1B80, 0xF8050335,
+	0x1B80, 0xF8050337,
+	0x1B80, 0xF9040345,
+	0x1B80, 0xF9040347,
+	0x1B80, 0x00010355,
+	0x1B80, 0x00010357,
+	0x1B80, 0x303B0365,
+	0x1B80, 0x303B0367,
+	0x1B80, 0x30500375,
+	0x1B80, 0x30500377,
+	0x1B80, 0x305C0385,
+	0x1B80, 0x305C0387,
+	0x1B80, 0x31D50395,
+	0x1B80, 0x31D50397,
+	0x1B80, 0x31C503A5,
+	0x1B80, 0x31C503A7,
+	0x1B80, 0x4D0403B5,
+	0x1B80, 0x4D0403B7,
+	0x1B80, 0x2EF003C5,
+	0x1B80, 0x2EF003C7,
+	0x1B80, 0x000203D5,
+	0x1B80, 0x000203D7,
+	0x1B80, 0x208003E5,
+	0x1B80, 0x208003E7,
+	0x1B80, 0x000003F5,
+	0x1B80, 0x000003F7,
+	0x1B80, 0x4D000405,
+	0x1B80, 0x4D000407,
+	0x1B80, 0x55070415,
+	0x1B80, 0x55070417,
+	0x1B80, 0xE1230425,
+	0x1B80, 0xE1230427,
+	0x1B80, 0xE1230435,
+	0x1B80, 0xE1230437,
+	0x1B80, 0x4D040445,
+	0x1B80, 0x4D040447,
+	0x1B80, 0x20800455,
+	0x1B80, 0x20800457,
+	0x1B80, 0x84000465,
+	0x1B80, 0x84000467,
+	0x1B80, 0x4D000475,
+	0x1B80, 0x4D000477,
+	0x1B80, 0x550F0485,
+	0x1B80, 0x550F0487,
+	0x1B80, 0xE1230495,
+	0x1B80, 0xE1230497,
+	0x1B80, 0x4F0204A5,
+	0x1B80, 0x4F0204A7,
+	0x1B80, 0x4E0004B5,
+	0x1B80, 0x4E0004B7,
+	0x1B80, 0x530204C5,
+	0x1B80, 0x530204C7,
+	0x1B80, 0x520104D5,
+	0x1B80, 0x520104D7,
+	0x1B80, 0xE12704E5,
+	0x1B80, 0xE12704E7,
+	0x1B80, 0x000104F5,
+	0x1B80, 0x000104F7,
+	0x1B80, 0x5C720505,
+	0x1B80, 0x5C720507,
+	0x1B80, 0xE1320515,
+	0x1B80, 0xE1320517,
+	0x1B80, 0x54E50525,
+	0x1B80, 0x54E50527,
+	0x1B80, 0x54BF0535,
+	0x1B80, 0x54BF0537,
+	0x1B80, 0x54C50545,
+	0x1B80, 0x54C50547,
+	0x1B80, 0x54BE0555,
+	0x1B80, 0x54BE0557,
+	0x1B80, 0x54DF0565,
+	0x1B80, 0x54DF0567,
+	0x1B80, 0x0BA60575,
+	0x1B80, 0x0BA60577,
+	0x1B80, 0xF3130585,
+	0x1B80, 0xF3130587,
+	0x1B80, 0xF41E0595,
+	0x1B80, 0xF41E0597,
+	0x1B80, 0xF53C05A5,
+	0x1B80, 0xF53C05A7,
+	0x1B80, 0x000105B5,
+	0x1B80, 0x000105B7,
+	0x1B80, 0x620605C5,
+	0x1B80, 0x620605C7,
+	0x1B80, 0x600605D5,
+	0x1B80, 0x600605D7,
+	0x1B80, 0xE1A905E5,
+	0x1B80, 0xE1A905E7,
+	0x1B80, 0x0C0005F5,
+	0x1B80, 0x0C0005F7,
+	0x1B80, 0x5C720605,
+	0x1B80, 0x5C720607,
+	0x1B80, 0xE1320615,
+	0x1B80, 0xE1320617,
+	0x1B80, 0x5CF10625,
+	0x1B80, 0x5CF10627,
+	0x1B80, 0x0C010635,
+	0x1B80, 0x0C010637,
+	0x1B80, 0xF2020645,
+	0x1B80, 0xF2020647,
+	0x1B80, 0x30D60655,
+	0x1B80, 0x30D60657,
+	0x1B80, 0x0AC60665,
+	0x1B80, 0x0AC60667,
+	0x1B80, 0xE1B60675,
+	0x1B80, 0xE1B60677,
+	0x1B80, 0xE1580685,
+	0x1B80, 0xE1580687,
+	0x1B80, 0x54E50695,
+	0x1B80, 0x54E50697,
+	0x1B80, 0x000106A5,
+	0x1B80, 0x000106A7,
+	0x1B80, 0x560106B5,
+	0x1B80, 0x560106B7,
+	0x1B80, 0x5CE206C5,
+	0x1B80, 0x5CE206C7,
+	0x1B80, 0x0AE106D5,
+	0x1B80, 0x0AE106D7,
+	0x1B80, 0x630C06E5,
+	0x1B80, 0x630C06E7,
+	0x1B80, 0xE13F06F5,
+	0x1B80, 0xE13F06F7,
+	0x1B80, 0x00270705,
+	0x1B80, 0x00270707,
+	0x1B80, 0xE16C0715,
+	0x1B80, 0xE16C0717,
+	0x1B80, 0x00020725,
+	0x1B80, 0x00020727,
+	0x1B80, 0x002A0735,
+	0x1B80, 0x002A0737,
+	0x1B80, 0x07140745,
+	0x1B80, 0x07140747,
+	0x1B80, 0x00020755,
+	0x1B80, 0x00020757,
+	0x1B80, 0x30C30765,
+	0x1B80, 0x30C30767,
+	0x1B80, 0x56010775,
+	0x1B80, 0x56010777,
+	0x1B80, 0x5CE20785,
+	0x1B80, 0x5CE20787,
+	0x1B80, 0x0AE10795,
+	0x1B80, 0x0AE10797,
+	0x1B80, 0x631707A5,
+	0x1B80, 0x631707A7,
+	0x1B80, 0xE13F07B5,
+	0x1B80, 0xE13F07B7,
+	0x1B80, 0x002507C5,
+	0x1B80, 0x002507C7,
+	0x1B80, 0xE16C07D5,
+	0x1B80, 0xE16C07D7,
+	0x1B80, 0x000207E5,
+	0x1B80, 0x000207E7,
+	0x1B80, 0x630F07F5,
+	0x1B80, 0x630F07F7,
+	0x1B80, 0xE13F0805,
+	0x1B80, 0xE13F0807,
+	0x1B80, 0x63070815,
+	0x1B80, 0x63070817,
+	0x1B80, 0xE13F0825,
+	0x1B80, 0xE13F0827,
+	0x1B80, 0x07140835,
+	0x1B80, 0x07140837,
+	0x1B80, 0x56000845,
+	0x1B80, 0x56000847,
+	0x1B80, 0x5CF20855,
+	0x1B80, 0x5CF20857,
+	0x1B80, 0x0AF10865,
+	0x1B80, 0x0AF10867,
+	0x1B80, 0x07140875,
+	0x1B80, 0x07140877,
+	0x1B80, 0x07140885,
+	0x1B80, 0x07140887,
+	0x1B80, 0x630F0895,
+	0x1B80, 0x630F0897,
+	0x1B80, 0xE13F08A5,
+	0x1B80, 0xE13F08A7,
+	0x1B80, 0x631708B5,
+	0x1B80, 0x631708B7,
+	0x1B80, 0xE13F08C5,
+	0x1B80, 0xE13F08C7,
+	0x1B80, 0x002508D5,
+	0x1B80, 0x002508D7,
+	0x1B80, 0xE16C08E5,
+	0x1B80, 0xE16C08E7,
+	0x1B80, 0x000208F5,
+	0x1B80, 0x000208F7,
+	0x1B80, 0x30C30905,
+	0x1B80, 0x30C30907,
+	0x1B80, 0xE1A90915,
+	0x1B80, 0xE1A90917,
+	0x1B80, 0x62060925,
+	0x1B80, 0x62060927,
+	0x1B80, 0x60060935,
+	0x1B80, 0x60060937,
+	0x1B80, 0xE1160945,
+	0x1B80, 0xE1160947,
+	0x1B80, 0x54BE0955,
+	0x1B80, 0x54BE0957,
+	0x1B80, 0x56010965,
+	0x1B80, 0x56010967,
+	0x1B80, 0x5CE20975,
+	0x1B80, 0x5CE20977,
+	0x1B80, 0x0AE10985,
+	0x1B80, 0x0AE10987,
+	0x1B80, 0x633A0995,
+	0x1B80, 0x633A0997,
+	0x1B80, 0xE13F09A5,
+	0x1B80, 0xE13F09A7,
+	0x1B80, 0x633709B5,
+	0x1B80, 0x633709B7,
+	0x1B80, 0xE13F09C5,
+	0x1B80, 0xE13F09C7,
+	0x1B80, 0x632F09D5,
+	0x1B80, 0x632F09D7,
+	0x1B80, 0xE13F09E5,
+	0x1B80, 0xE13F09E7,
+	0x1B80, 0x632709F5,
+	0x1B80, 0x632709F7,
+	0x1B80, 0xE13F0A05,
+	0x1B80, 0xE13F0A07,
+	0x1B80, 0x631F0A15,
+	0x1B80, 0x631F0A17,
+	0x1B80, 0xE13F0A25,
+	0x1B80, 0xE13F0A27,
+	0x1B80, 0x63170A35,
+	0x1B80, 0x63170A37,
+	0x1B80, 0xE13F0A45,
+	0x1B80, 0xE13F0A47,
+	0x1B80, 0x630F0A55,
+	0x1B80, 0x630F0A57,
+	0x1B80, 0xE13F0A65,
+	0x1B80, 0xE13F0A67,
+	0x1B80, 0x63070A75,
+	0x1B80, 0x63070A77,
+	0x1B80, 0xE13F0A85,
+	0x1B80, 0xE13F0A87,
+	0x1B80, 0xE16C0A95,
+	0x1B80, 0xE16C0A97,
+	0x1B80, 0x56000AA5,
+	0x1B80, 0x56000AA7,
+	0x1B80, 0x5CF20AB5,
+	0x1B80, 0x5CF20AB7,
+	0x1B80, 0x0AF10AC5,
+	0x1B80, 0x0AF10AC7,
+	0x1B80, 0xF5040AD5,
+	0x1B80, 0xF5040AD7,
+	0x1B80, 0xE13F0AE5,
+	0x1B80, 0xE13F0AE7,
+	0x1B80, 0xE16C0AF5,
+	0x1B80, 0xE16C0AF7,
+	0x1B80, 0x30B30B05,
+	0x1B80, 0x30B30B07,
+	0x1B80, 0x07140B15,
+	0x1B80, 0x07140B17,
+	0x1B80, 0x07140B25,
+	0x1B80, 0x07140B27,
+	0x1B80, 0x630F0B35,
+	0x1B80, 0x630F0B37,
+	0x1B80, 0xE13F0B45,
+	0x1B80, 0xE13F0B47,
+	0x1B80, 0x63170B55,
+	0x1B80, 0x63170B57,
+	0x1B80, 0xE13F0B65,
+	0x1B80, 0xE13F0B67,
+	0x1B80, 0x631F0B75,
+	0x1B80, 0x631F0B77,
+	0x1B80, 0xE13F0B85,
+	0x1B80, 0xE13F0B87,
+	0x1B80, 0x63270B95,
+	0x1B80, 0x63270B97,
+	0x1B80, 0xE13F0BA5,
+	0x1B80, 0xE13F0BA7,
+	0x1B80, 0x632F0BB5,
+	0x1B80, 0x632F0BB7,
+	0x1B80, 0xE13F0BC5,
+	0x1B80, 0xE13F0BC7,
+	0x1B80, 0x63370BD5,
+	0x1B80, 0x63370BD7,
+	0x1B80, 0xE13F0BE5,
+	0x1B80, 0xE13F0BE7,
+	0x1B80, 0x633A0BF5,
+	0x1B80, 0x633A0BF7,
+	0x1B80, 0xE13F0C05,
+	0x1B80, 0xE13F0C07,
+	0x1B80, 0xF60B0C15,
+	0x1B80, 0xF60B0C17,
+	0x1B80, 0xF7170C25,
+	0x1B80, 0xF7170C27,
+	0x1B80, 0x4D300C35,
+	0x1B80, 0x4D300C37,
+	0x1B80, 0x57040C45,
+	0x1B80, 0x57040C47,
+	0x1B80, 0x57000C55,
+	0x1B80, 0x57000C57,
+	0x1B80, 0x96000C65,
+	0x1B80, 0x96000C67,
+	0x1B80, 0x57080C75,
+	0x1B80, 0x57080C77,
+	0x1B80, 0x57000C85,
+	0x1B80, 0x57000C87,
+	0x1B80, 0x95000C95,
+	0x1B80, 0x95000C97,
+	0x1B80, 0x4D000CA5,
+	0x1B80, 0x4D000CA7,
+	0x1B80, 0x6C070CB5,
+	0x1B80, 0x6C070CB7,
+	0x1B80, 0x00010CC5,
+	0x1B80, 0x00010CC7,
+	0x1B80, 0x00220CD5,
+	0x1B80, 0x00220CD7,
+	0x1B80, 0x06140CE5,
+	0x1B80, 0x06140CE7,
+	0x1B80, 0xE16C0CF5,
+	0x1B80, 0xE16C0CF7,
+	0x1B80, 0x00020D05,
+	0x1B80, 0x00020D07,
+	0x1B80, 0x00250D15,
+	0x1B80, 0x00250D17,
+	0x1B80, 0x06140D25,
+	0x1B80, 0x06140D27,
+	0x1B80, 0xE16C0D35,
+	0x1B80, 0xE16C0D37,
+	0x1B80, 0x00020D45,
+	0x1B80, 0x00020D47,
+	0x1B80, 0x00010D55,
+	0x1B80, 0x00010D57,
+	0x1B80, 0x00320D65,
+	0x1B80, 0x00320D67,
+	0x1B80, 0xE16C0D75,
+	0x1B80, 0xE16C0D77,
+	0x1B80, 0x00020D85,
+	0x1B80, 0x00020D87,
+	0x1B80, 0xE1860D95,
+	0x1B80, 0xE1860D97,
+	0x1B80, 0xE1B60DA5,
+	0x1B80, 0xE1B60DA7,
+	0x1B80, 0x5CD10DB5,
+	0x1B80, 0x5CD10DB7,
+	0x1B80, 0x673A0DC5,
+	0x1B80, 0x673A0DC7,
+	0x1B80, 0xE1230DD5,
+	0x1B80, 0xE1230DD7,
+	0x1B80, 0xF80B0DE5,
+	0x1B80, 0xF80B0DE7,
+	0x1B80, 0xF9110DF5,
+	0x1B80, 0xF9110DF7,
+	0x1B80, 0xE1580E05,
+	0x1B80, 0xE1580E07,
+	0x1B80, 0x67370E15,
+	0x1B80, 0x67370E17,
+	0x1B80, 0xE1580E25,
+	0x1B80, 0xE1580E27,
+	0x1B80, 0x672F0E35,
+	0x1B80, 0x672F0E37,
+	0x1B80, 0xE1580E45,
+	0x1B80, 0xE1580E47,
+	0x1B80, 0x67270E55,
+	0x1B80, 0x67270E57,
+	0x1B80, 0xE1580E65,
+	0x1B80, 0xE1580E67,
+	0x1B80, 0x671F0E75,
+	0x1B80, 0x671F0E77,
+	0x1B80, 0xE1580E85,
+	0x1B80, 0xE1580E87,
+	0x1B80, 0x67170E95,
+	0x1B80, 0x67170E97,
+	0x1B80, 0xE1580EA5,
+	0x1B80, 0xE1580EA7,
+	0x1B80, 0xF8020EB5,
+	0x1B80, 0xF8020EB7,
+	0x1B80, 0x30EE0EC5,
+	0x1B80, 0x30EE0EC7,
+	0x1B80, 0xE0D10ED5,
+	0x1B80, 0xE0D10ED7,
+	0x1B80, 0x670F0EE5,
+	0x1B80, 0x670F0EE7,
+	0x1B80, 0xE1580EF5,
+	0x1B80, 0xE1580EF7,
+	0x1B80, 0x67070F05,
+	0x1B80, 0x67070F07,
+	0x1B80, 0xE1580F15,
+	0x1B80, 0xE1580F17,
+	0x1B80, 0xF9020F25,
+	0x1B80, 0xF9020F27,
+	0x1B80, 0x30F50F35,
+	0x1B80, 0x30F50F37,
+	0x1B80, 0xE0CD0F45,
+	0x1B80, 0xE0CD0F47,
+	0x1B80, 0x06140F55,
+	0x1B80, 0x06140F57,
+	0x1B80, 0xE16C0F65,
+	0x1B80, 0xE16C0F67,
+	0x1B80, 0x5CF10F75,
+	0x1B80, 0x5CF10F77,
+	0x1B80, 0xE1580F85,
+	0x1B80, 0xE1580F87,
+	0x1B80, 0x06140F95,
+	0x1B80, 0x06140F97,
+	0x1B80, 0xE16C0FA5,
+	0x1B80, 0xE16C0FA7,
+	0x1B80, 0xF9020FB5,
+	0x1B80, 0xF9020FB7,
+	0x1B80, 0x30FF0FC5,
+	0x1B80, 0x30FF0FC7,
+	0x1B80, 0xE0CD0FD5,
+	0x1B80, 0xE0CD0FD7,
+	0x1B80, 0x31130FE5,
+	0x1B80, 0x31130FE7,
+	0x1B80, 0x670F0FF5,
+	0x1B80, 0x670F0FF7,
+	0x1B80, 0xE1581005,
+	0x1B80, 0xE1581007,
+	0x1B80, 0x67171015,
+	0x1B80, 0x67171017,
+	0x1B80, 0xE1581025,
+	0x1B80, 0xE1581027,
+	0x1B80, 0xF8021035,
+	0x1B80, 0xF8021037,
+	0x1B80, 0x31071045,
+	0x1B80, 0x31071047,
+	0x1B80, 0xE0D11055,
+	0x1B80, 0xE0D11057,
+	0x1B80, 0x31131065,
+	0x1B80, 0x31131067,
+	0x1B80, 0x670F1075,
+	0x1B80, 0x670F1077,
+	0x1B80, 0xE1581085,
+	0x1B80, 0xE1581087,
+	0x1B80, 0x671F1095,
+	0x1B80, 0x671F1097,
+	0x1B80, 0xE15810A5,
+	0x1B80, 0xE15810A7,
+	0x1B80, 0x672710B5,
+	0x1B80, 0x672710B7,
+	0x1B80, 0xE15810C5,
+	0x1B80, 0xE15810C7,
+	0x1B80, 0x672F10D5,
+	0x1B80, 0x672F10D7,
+	0x1B80, 0xE15810E5,
+	0x1B80, 0xE15810E7,
+	0x1B80, 0x673710F5,
+	0x1B80, 0x673710F7,
+	0x1B80, 0xE1581105,
+	0x1B80, 0xE1581107,
+	0x1B80, 0x673A1115,
+	0x1B80, 0x673A1117,
+	0x1B80, 0xE1581125,
+	0x1B80, 0xE1581127,
+	0x1B80, 0x4D101135,
+	0x1B80, 0x4D101137,
+	0x1B80, 0x30C41145,
+	0x1B80, 0x30C41147,
+	0x1B80, 0x00011155,
+	0x1B80, 0x00011157,
+	0x1B80, 0x6F241165,
+	0x1B80, 0x6F241167,
+	0x1B80, 0x6E401175,
+	0x1B80, 0x6E401177,
+	0x1B80, 0x6D001185,
+	0x1B80, 0x6D001187,
+	0x1B80, 0x55031195,
+	0x1B80, 0x55031197,
+	0x1B80, 0x312311A5,
+	0x1B80, 0x312311A7,
+	0x1B80, 0x6F1C11B5,
+	0x1B80, 0x6F1C11B7,
+	0x1B80, 0x6E4011C5,
+	0x1B80, 0x6E4011C7,
+	0x1B80, 0x550B11D5,
+	0x1B80, 0x550B11D7,
+	0x1B80, 0x312311E5,
+	0x1B80, 0x312311E7,
+	0x1B80, 0x061C11F5,
+	0x1B80, 0x061C11F7,
+	0x1B80, 0x54DE1205,
+	0x1B80, 0x54DE1207,
+	0x1B80, 0x06DC1215,
+	0x1B80, 0x06DC1217,
+	0x1B80, 0x55131225,
+	0x1B80, 0x55131227,
+	0x1B80, 0x74011235,
+	0x1B80, 0x74011237,
+	0x1B80, 0x74001245,
+	0x1B80, 0x74001247,
+	0x1B80, 0x8E001255,
+	0x1B80, 0x8E001257,
+	0x1B80, 0x00011265,
+	0x1B80, 0x00011267,
+	0x1B80, 0x57021275,
+	0x1B80, 0x57021277,
+	0x1B80, 0x57001285,
+	0x1B80, 0x57001287,
+	0x1B80, 0x97001295,
+	0x1B80, 0x97001297,
+	0x1B80, 0x000112A5,
+	0x1B80, 0x000112A7,
+	0x1B80, 0x54BF12B5,
+	0x1B80, 0x54BF12B7,
+	0x1B80, 0x54C112C5,
+	0x1B80, 0x54C112C7,
+	0x1B80, 0x54A212D5,
+	0x1B80, 0x54A212D7,
+	0x1B80, 0x54C012E5,
+	0x1B80, 0x54C012E7,
+	0x1B80, 0x54A112F5,
+	0x1B80, 0x54A112F7,
+	0x1B80, 0x54DF1305,
+	0x1B80, 0x54DF1307,
+	0x1B80, 0x00011315,
+	0x1B80, 0x00011317,
+	0x1B80, 0x55001325,
+	0x1B80, 0x55001327,
+	0x1B80, 0xE1231335,
+	0x1B80, 0xE1231337,
+	0x1B80, 0x54811345,
+	0x1B80, 0x54811347,
+	0x1B80, 0xE1231355,
+	0x1B80, 0xE1231357,
+	0x1B80, 0x54801365,
+	0x1B80, 0x54801367,
+	0x1B80, 0x002A1375,
+	0x1B80, 0x002A1377,
+	0x1B80, 0xE12B1385,
+	0x1B80, 0xE12B1387,
+	0x1B80, 0xE1231395,
+	0x1B80, 0xE1231397,
+	0x1B80, 0x548013A5,
+	0x1B80, 0x548013A7,
+	0x1B80, 0xE17213B5,
+	0x1B80, 0xE17213B7,
+	0x1B80, 0xBF3013C5,
+	0x1B80, 0xBF3013C7,
+	0x1B80, 0x000213D5,
+	0x1B80, 0x000213D7,
+	0x1B80, 0x302813E5,
+	0x1B80, 0x302813E7,
+	0x1B80, 0x4F7813F5,
+	0x1B80, 0x4F7813F7,
+	0x1B80, 0x4E001405,
+	0x1B80, 0x4E001407,
+	0x1B80, 0x53871415,
+	0x1B80, 0x53871417,
+	0x1B80, 0x52F11425,
+	0x1B80, 0x52F11427,
+	0x1B80, 0xE1161435,
+	0x1B80, 0xE1161437,
+	0x1B80, 0xE11B1445,
+	0x1B80, 0xE11B1447,
+	0x1B80, 0xE11F1455,
+	0x1B80, 0xE11F1457,
+	0x1B80, 0xE1271465,
+	0x1B80, 0xE1271467,
+	0x1B80, 0x54811475,
+	0x1B80, 0x54811477,
+	0x1B80, 0xE1161485,
+	0x1B80, 0xE1161487,
+	0x1B80, 0xE11B1495,
+	0x1B80, 0xE11B1497,
+	0x1B80, 0xE11F14A5,
+	0x1B80, 0xE11F14A7,
+	0x1B80, 0xE12714B5,
+	0x1B80, 0xE12714B7,
+	0x1B80, 0x548014C5,
+	0x1B80, 0x548014C7,
+	0x1B80, 0x002A14D5,
+	0x1B80, 0x002A14D7,
+	0x1B80, 0xE12B14E5,
+	0x1B80, 0xE12B14E7,
+	0x1B80, 0xE11614F5,
+	0x1B80, 0xE11614F7,
+	0x1B80, 0xE11B1505,
+	0x1B80, 0xE11B1507,
+	0x1B80, 0xE11F1515,
+	0x1B80, 0xE11F1517,
+	0x1B80, 0xE1271525,
+	0x1B80, 0xE1271527,
+	0x1B80, 0x54801535,
+	0x1B80, 0x54801537,
+	0x1B80, 0xE1721545,
+	0x1B80, 0xE1721547,
+	0x1B80, 0xBF171555,
+	0x1B80, 0xBF171557,
+	0x1B80, 0x00021565,
+	0x1B80, 0x00021567,
+	0x1B80, 0x30281575,
+	0x1B80, 0x30281577,
+	0x1B80, 0x06141585,
+	0x1B80, 0x06141587,
+	0x1B80, 0x73201595,
+	0x1B80, 0x73201597,
+	0x1B80, 0x720015A5,
+	0x1B80, 0x720015A7,
+	0x1B80, 0x710015B5,
+	0x1B80, 0x710015B7,
+	0x1B80, 0x550115C5,
+	0x1B80, 0x550115C7,
+	0x1B80, 0xE12315D5,
+	0x1B80, 0xE12315D7,
+	0x1B80, 0xE12715E5,
+	0x1B80, 0xE12715E7,
+	0x1B80, 0x548115F5,
+	0x1B80, 0x548115F7,
+	0x1B80, 0xE1231605,
+	0x1B80, 0xE1231607,
+	0x1B80, 0xE1271615,
+	0x1B80, 0xE1271617,
+	0x1B80, 0x54801625,
+	0x1B80, 0x54801627,
+	0x1B80, 0x002A1635,
+	0x1B80, 0x002A1637,
+	0x1B80, 0xE12B1645,
+	0x1B80, 0xE12B1647,
+	0x1B80, 0xE1231655,
+	0x1B80, 0xE1231657,
+	0x1B80, 0xE1271665,
+	0x1B80, 0xE1271667,
+	0x1B80, 0x54801675,
+	0x1B80, 0x54801677,
+	0x1B80, 0xE1721685,
+	0x1B80, 0xE1721687,
+	0x1B80, 0xBF031695,
+	0x1B80, 0xBF031697,
+	0x1B80, 0x000216A5,
+	0x1B80, 0x000216A7,
+	0x1B80, 0x302816B5,
+	0x1B80, 0x302816B7,
+	0x1B80, 0x54BF16C5,
+	0x1B80, 0x54BF16C7,
+	0x1B80, 0x54C516D5,
+	0x1B80, 0x54C516D7,
+	0x1B80, 0x050A16E5,
+	0x1B80, 0x050A16E7,
+	0x1B80, 0x071416F5,
+	0x1B80, 0x071416F7,
+	0x1B80, 0x54DF1705,
+	0x1B80, 0x54DF1707,
+	0x1B80, 0x00011715,
+	0x1B80, 0x00011717,
+	0x1B80, 0x54BF1725,
+	0x1B80, 0x54BF1727,
+	0x1B80, 0x54C01735,
+	0x1B80, 0x54C01737,
+	0x1B80, 0x54A31745,
+	0x1B80, 0x54A31747,
+	0x1B80, 0x54C11755,
+	0x1B80, 0x54C11757,
+	0x1B80, 0x54A41765,
+	0x1B80, 0x54A41767,
+	0x1B80, 0x4C831775,
+	0x1B80, 0x4C831777,
+	0x1B80, 0x4C031785,
+	0x1B80, 0x4C031787,
+	0x1B80, 0xBF0B1795,
+	0x1B80, 0xBF0B1797,
+	0x1B80, 0x54C217A5,
+	0x1B80, 0x54C217A7,
+	0x1B80, 0x54A417B5,
+	0x1B80, 0x54A417B7,
+	0x1B80, 0x4C8517C5,
+	0x1B80, 0x4C8517C7,
+	0x1B80, 0x4C0517D5,
+	0x1B80, 0x4C0517D7,
+	0x1B80, 0xBF0617E5,
+	0x1B80, 0xBF0617E7,
+	0x1B80, 0x54C117F5,
+	0x1B80, 0x54C117F7,
+	0x1B80, 0x54A31805,
+	0x1B80, 0x54A31807,
+	0x1B80, 0x4C861815,
+	0x1B80, 0x4C861817,
+	0x1B80, 0x4C061825,
+	0x1B80, 0x4C061827,
+	0x1B80, 0xBF011835,
+	0x1B80, 0xBF011837,
+	0x1B80, 0x54DF1845,
+	0x1B80, 0x54DF1847,
+	0x1B80, 0x00011855,
+	0x1B80, 0x00011857,
+	0x1B80, 0x00071865,
+	0x1B80, 0x00071867,
+	0x1B80, 0x54011875,
+	0x1B80, 0x54011877,
+	0x1B80, 0x00041885,
+	0x1B80, 0x00041887,
+	0x1B80, 0x56001895,
+	0x1B80, 0x56001897,
+	0x1B80, 0x5CF218A5,
+	0x1B80, 0x5CF218A7,
+	0x1B80, 0x630718B5,
+	0x1B80, 0x630718B7,
+	0x1B80, 0x620418C5,
+	0x1B80, 0x620418C7,
+	0x1B80, 0x610018D5,
+	0x1B80, 0x610018D7,
+	0x1B80, 0x670718E5,
+	0x1B80, 0x670718E7,
+	0x1B80, 0x660618F5,
+	0x1B80, 0x660618F7,
+	0x1B80, 0x6F201905,
+	0x1B80, 0x6F201907,
+	0x1B80, 0x6E001915,
+	0x1B80, 0x6E001917,
+	0x1B80, 0x6D001925,
+	0x1B80, 0x6D001927,
+	0x1B80, 0x6C031935,
+	0x1B80, 0x6C031937,
+	0x1B80, 0x73201945,
+	0x1B80, 0x73201947,
+	0x1B80, 0x72001955,
+	0x1B80, 0x72001957,
+	0x1B80, 0x71001965,
+	0x1B80, 0x71001967,
+	0x1B80, 0x7B201975,
+	0x1B80, 0x7B201977,
+	0x1B80, 0x7A001985,
+	0x1B80, 0x7A001987,
+	0x1B80, 0x79001995,
+	0x1B80, 0x79001997,
+	0x1B80, 0x7F2019A5,
+	0x1B80, 0x7F2019A7,
+	0x1B80, 0x7E0019B5,
+	0x1B80, 0x7E0019B7,
+	0x1B80, 0x7D0019C5,
+	0x1B80, 0x7D0019C7,
+	0x1B80, 0x090119D5,
+	0x1B80, 0x090119D7,
+	0x1B80, 0x0AC619E5,
+	0x1B80, 0x0AC619E7,
+	0x1B80, 0x0BA619F5,
+	0x1B80, 0x0BA619F7,
+	0x1B80, 0x0C011A05,
+	0x1B80, 0x0C011A07,
+	0x1B80, 0x0D021A15,
+	0x1B80, 0x0D021A17,
+	0x1B80, 0x0E041A25,
+	0x1B80, 0x0E041A27,
+	0x1B80, 0x0FFF1A35,
+	0x1B80, 0x0FFF1A37,
+	0x1B80, 0x4D041A45,
+	0x1B80, 0x4D041A47,
+	0x1B80, 0x28F81A55,
+	0x1B80, 0x28F81A57,
+	0x1B80, 0xE0001A65,
+	0x1B80, 0xE0001A67,
+	0x1B80, 0x4D001A75,
+	0x1B80, 0x4D001A77,
+	0x1B80, 0x00011A85,
+	0x1B80, 0x00011A87,
+	0x1B80, 0x4D041A95,
+	0x1B80, 0x4D041A97,
+	0x1B80, 0x2EF81AA5,
+	0x1B80, 0x2EF81AA7,
+	0x1B80, 0x00021AB5,
+	0x1B80, 0x00021AB7,
+	0x1B80, 0x23031AC5,
+	0x1B80, 0x23031AC7,
+	0x1B80, 0x00001AD5,
+	0x1B80, 0x00001AD7,
+	0x1B80, 0x23131AE5,
+	0x1B80, 0x23131AE7,
+	0x1B80, 0xE77F1AF5,
+	0x1B80, 0xE77F1AF7,
+	0x1B80, 0x232F1B05,
+	0x1B80, 0x232F1B07,
+	0x1B80, 0xEFBF1B15,
+	0x1B80, 0xEFBF1B17,
+	0x1B80, 0x2EF01B25,
+	0x1B80, 0x2EF01B27,
+	0x1B80, 0x00021B35,
+	0x1B80, 0x00021B37,
+	0x1B80, 0x4D001B45,
+	0x1B80, 0x4D001B47,
+	0x1B80, 0x00011B55,
+	0x1B80, 0x00011B57,
+	0x1B80, 0x4D041B65,
+	0x1B80, 0x4D041B67,
+	0x1B80, 0x2EF81B75,
+	0x1B80, 0x2EF81B77,
+	0x1B80, 0x00021B85,
+	0x1B80, 0x00021B87,
+	0x1B80, 0x23031B95,
+	0x1B80, 0x23031B97,
+	0x1B80, 0x00001BA5,
+	0x1B80, 0x00001BA7,
+	0x1B80, 0x23131BB5,
+	0x1B80, 0x23131BB7,
+	0x1B80, 0xE77F1BC5,
+	0x1B80, 0xE77F1BC7,
+	0x1B80, 0x232F1BD5,
+	0x1B80, 0x232F1BD7,
+	0x1B80, 0xE79F1BE5,
+	0x1B80, 0xE79F1BE7,
+	0x1B80, 0x2EF01BF5,
+	0x1B80, 0x2EF01BF7,
+	0x1B80, 0x00021C05,
+	0x1B80, 0x00021C07,
+	0x1B80, 0x28F81C15,
+	0x1B80, 0x28F81C17,
+	0x1B80, 0x80001C25,
+	0x1B80, 0x80001C27,
+	0x1B80, 0x4D001C35,
+	0x1B80, 0x4D001C37,
+	0x1B80, 0x00011C45,
+	0x1B80, 0x00011C47,
+	0x1B80, 0x00041C55,
+	0x1B80, 0x00041C57,
+	0x1B80, 0x6BC01C65,
+	0x1B80, 0x6BC01C67,
+	0x1B80, 0x4D041C75,
+	0x1B80, 0x4D041C77,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241C85,
+	0x1B80, 0x68241C87,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x68481C85,
+	0x1B80, 0x68481C87,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x66061C95,
+	0x1B80, 0x66061C97,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x650C1CA5,
+	0x1B80, 0x650C1CA7,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x65041CA5,
+	0x1B80, 0x65041CA7,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x64471CB5,
+	0x1B80, 0x64471CB7,
+	0x1B80, 0x23411CC5,
+	0x1B80, 0x23411CC7,
+	0x1B80, 0x100E1CD5,
+	0x1B80, 0x100E1CD7,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60101CE5,
+	0x1B80, 0x60101CE7,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x60011CE5,
+	0x1B80, 0x60011CE7,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x23411CF5,
+	0x1B80, 0x23411CF7,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60811D05,
+	0x1B80, 0x60811D07,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x60611D05,
+	0x1B80, 0x60611D07,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x23411D15,
+	0x1B80, 0x23411D17,
+	0x1B80, 0x70E11D25,
+	0x1B80, 0x70E11D27,
+	0x1B80, 0x4D001D35,
+	0x1B80, 0x4D001D37,
+	0x1B80, 0x00011D45,
+	0x1B80, 0x00011D47,
+	0x1B80, 0x00041D55,
+	0x1B80, 0x00041D57,
+	0x1B80, 0x6B401D65,
+	0x1B80, 0x6B401D67,
+	0x1B80, 0x4D041D75,
+	0x1B80, 0x4D041D77,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241D85,
+	0x1B80, 0x68241D87,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241D85,
+	0x1B80, 0x68241D87,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241D85,
+	0x1B80, 0x68241D87,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241D85,
+	0x1B80, 0x68241D87,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241D85,
+	0x1B80, 0x68241D87,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x68241D85,
+	0x1B80, 0x68241D87,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x68481D85,
+	0x1B80, 0x68481D87,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x66061D95,
+	0x1B80, 0x66061D97,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65081DA5,
+	0x1B80, 0x65081DA7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65181DA5,
+	0x1B80, 0x65181DA7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65181DA5,
+	0x1B80, 0x65181DA7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65181DA5,
+	0x1B80, 0x65181DA7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65181DA5,
+	0x1B80, 0x65181DA7,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65081DA5,
+	0x1B80, 0x65081DA7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65181DA5,
+	0x1B80, 0x65181DA7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x65181DA5,
+	0x1B80, 0x65181DA7,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x65081DA5,
+	0x1B80, 0x65081DA7,
+	0xB0000000,	0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x64481DB5,
+	0x1B80, 0x64481DB7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x64481DB5,
+	0x1B80, 0x64481DB7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x64481DB5,
+	0x1B80, 0x64481DB7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x64481DB5,
+	0x1B80, 0x64481DB7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x64481DB5,
+	0x1B80, 0x64481DB7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x64481DB5,
+	0x1B80, 0x64481DB7,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x64471DB5,
+	0x1B80, 0x64471DB7,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x23411DC5,
+	0x1B80, 0x23411DC7,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E41DD5,
+	0x1B80, 0x11E41DD7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E81DD5,
+	0x1B80, 0x11E81DD7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E81DD5,
+	0x1B80, 0x11E81DD7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E81DD5,
+	0x1B80, 0x11E81DD7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E81DD5,
+	0x1B80, 0x11E81DD7,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E41DD5,
+	0x1B80, 0x11E41DD7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E81DD5,
+	0x1B80, 0x11E81DD7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x11E81DD5,
+	0x1B80, 0x11E81DD7,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x11E41DD5,
+	0x1B80, 0x11E41DD7,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x60011DE5,
+	0x1B80, 0x60011DE7,
+	0x1B80, 0x23411DF5,
+	0x1B80, 0x23411DF7,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60E11E05,
+	0x1B80, 0x60E11E07,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x61E11E05,
+	0x1B80, 0x61E11E07,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x61E11E05,
+	0x1B80, 0x61E11E07,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x61E11E05,
+	0x1B80, 0x61E11E07,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x61E11E05,
+	0x1B80, 0x61E11E07,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x60E11E05,
+	0x1B80, 0x60E11E07,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x61E11E05,
+	0x1B80, 0x61E11E07,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0x1B80, 0x61E11E05,
+	0x1B80, 0x61E11E07,
+	0xA0000000,	0x00000000,
+	0x1B80, 0x60E11E05,
+	0x1B80, 0x60E11E07,
+	0xB0000000,	0x00000000,
+	0x1B80, 0x23411E15,
+	0x1B80, 0x23411E17,
+	0x1B80, 0x70611E25,
+	0x1B80, 0x70611E27,
+	0x1B80, 0x4D001E35,
+	0x1B80, 0x4D001E37,
+	0x1B80, 0x00011E45,
+	0x1B80, 0x00011E47,
+	0x1B80, 0x00001E55,
+	0x1B80, 0x00001E57,
+	0x1B80, 0x00001E65,
+	0x1B80, 0x00001E67,
+	0x1B80, 0x00001E75,
+	0x1B80, 0x00001E77,
+	0x1B80, 0x00001E85,
+	0x1B80, 0x00001E87,
+	0x1B80, 0x00001E95,
+	0x1B80, 0x00001E97,
+	0x1B80, 0x00001EA5,
+	0x1B80, 0x00001EA7,
+	0x1B80, 0x00001EB5,
+	0x1B80, 0x00001EB7,
+	0x1B80, 0x00001EC5,
+	0x1B80, 0x00001EC7,
+	0x1B80, 0x00001ED5,
+	0x1B80, 0x00001ED7,
+	0x1B80, 0x00001EE5,
+	0x1B80, 0x00001EE7,
+	0x1B80, 0x00001EF5,
+	0x1B80, 0x00001EF7,
+	0x1B80, 0x00001F05,
+	0x1B80, 0x00001F07,
+	0x1B80, 0x00001F15,
+	0x1B80, 0x00001F17,
+	0x1B80, 0x00001F25,
+	0x1B80, 0x00001F27,
+	0x1B80, 0x00001F35,
+	0x1B80, 0x00001F37,
+	0x1B80, 0x00001F45,
+	0x1B80, 0x00001F47,
+	0x1B80, 0x00001F55,
+	0x1B80, 0x00001F57,
+	0x1B80, 0x00001F65,
+	0x1B80, 0x00001F67,
+	0x1B80, 0x00001F75,
+	0x1B80, 0x00001F77,
+	0x1B80, 0x00001F85,
+	0x1B80, 0x00001F87,
+	0x1B80, 0x00001F95,
+	0x1B80, 0x00001F97,
+	0x1B80, 0x00001FA5,
+	0x1B80, 0x00001FA7,
+	0x1B80, 0x00001FB5,
+	0x1B80, 0x00001FB7,
+	0x1B80, 0x00001FC5,
+	0x1B80, 0x00001FC7,
+	0x1B80, 0x00001FD5,
+	0x1B80, 0x00001FD7,
+	0x1B80, 0x00001FE5,
+	0x1B80, 0x00001FE7,
+	0x1B80, 0x00001FF5,
+	0x1B80, 0x00001FF7,
+	0x1B80, 0x00000006,
+	0x1B80, 0x00000002,
+};
+
+RTW_DECL_TABLE_PHY_COND(rtw8814a_bb, rtw_phy_cfg_bb);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x30323434, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x28303234, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x32323232, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x26283032, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x30303030, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x24262830, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x28303234, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x32322426, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x30323232, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x22242628, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x30303030, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x24262830, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x20222222, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x30323434, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x28303234, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x32323232, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x26283032, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x30303030, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x24262830, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x28303234, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x32322426, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x30323232, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x22242628, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x30303030, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x24262830, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x20222222, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x30323434, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x28303234, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x32323232, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x26283032, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x30303030, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x24262830, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x28303234, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x32322426, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x30323232, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x22242628, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x30303030, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x24262830, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x20222222, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x30323434, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x28303234, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x32323232, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x26283032, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x30303030, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x24262830, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x28303234, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x32322426, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x30323232, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x22242628, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x30303030, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x24262830, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x20222222, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x30323434, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x28303234, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x32323232, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x26283032, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x30303030, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x24262830, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x28303234, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x32322426, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x30323232, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x22242628, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x30303030, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x24262830, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x20222222, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x30323434, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x28303234, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x32323232, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x26283032, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x30303030, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x24262830, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x28303234, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x32322426, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x30323232, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x22242628, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x30303030, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x24262830, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x20222222, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x30323434, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x28303234, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x32323232, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x26283032, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x30303030, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x24262830, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x28303234, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x32322426, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x30323232, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x22242628, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x30303030, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x24262830, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x20222222, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x30323434, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x28303234, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x32323232, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x26283032, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x30303030, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x24262830, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x28303234, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x32322426, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x30323232, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x22242628, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x30303030, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x24262830, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x20222222, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg_type0[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x32323232, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x32323232, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x28303232, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x32323232, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x26283032, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x30303030, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x24262830, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x28282828, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x22242628, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x32323232, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x26283032, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x30302224, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x28303030, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x20222426, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x28282828, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x22242628, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x18202020, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x32323232, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x32323232, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x28303232, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x32323232, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x26283032, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x30303030, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x24262830, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x28282828, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x22242628, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x32323232, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x26283032, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x30302224, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x28303030, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x20222426, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x28282828, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x22242628, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x18202020, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x32323232, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x32323232, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x28303232, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x32323232, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x26283032, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x30303030, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x24262830, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x28282828, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x22242628, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x32323232, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x26283032, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x30302224, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x28303030, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x20222426, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x28282828, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x22242628, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x18202020, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x32323232, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x32323232, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x28303232, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x32323232, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x26283032, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x30303030, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x24262830, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x28282828, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x22242628, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x32323232, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x26283032, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x30302224, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x28303030, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x20222426, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x28282828, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x22242628, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x18202020, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x32323232, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x28303232, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x32323232, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x26283032, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x30303030, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x24262830, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x28282828, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x22242628, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x32323232, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x26283032, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x30302224, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x28303030, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x20222426, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x28282828, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x22242628, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x18202020, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x32323232, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x28303232, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x32323232, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x26283032, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x30303030, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x24262830, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x28282828, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x22242628, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x32323232, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x26283032, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x30302224, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x28303030, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x20222426, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x28282828, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x22242628, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x18202020, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x32323232, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x28303232, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x32323232, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x26283032, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x30303030, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x24262830, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x28282828, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x22242628, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x32323232, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x26283032, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x30302224, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x28303030, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x20222426, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x28282828, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x22242628, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x18202020, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x32323232, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x28303232, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x32323232, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x26283032, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x30303030, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x24262830, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x28282828, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x22242628, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x32323232, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x26283032, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x30302224, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x28303030, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x20222426, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x28282828, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x22242628, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x18202020, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg_type0);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg_type2[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x30323434, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x28303234, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x32323232, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x26283032, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x30303030, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x24262830, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x28303234, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x32322426, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x30323232, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x22242628, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x30303030, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x24262830, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x20222222, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x30323434, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x28303234, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x32323232, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x26283032, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x30303030, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x24262830, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x28303234, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x32322426, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x30323232, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x22242628, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x30303030, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x24262830, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x20222222, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x30323434, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x28303234, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x32323232, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x26283032, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x30303030, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x24262830, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x28303234, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x32322426, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x30323232, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x22242628, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x30303030, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x24262830, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x20222222, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x30323434, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x28303234, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x32323232, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x26283032, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x30303030, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x24262830, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x28303234, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x32322426, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x30323232, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x22242628, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x30303030, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x24262830, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x20222222, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x30323434, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x28303234, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x32323232, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x26283032, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x30303030, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x24262830, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x28303234, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x32322426, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x30323232, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x22242628, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x30303030, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x24262830, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x20222222, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x30323434, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x28303234, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x32323232, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x26283032, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x30303030, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x24262830, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x28303234, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x32322426, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x30323232, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x22242628, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x30303030, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x24262830, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x20222222, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x30323434, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x28303234, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x32323232, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x26283032, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x30303030, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x24262830, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x28303234, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x32322426, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x30323232, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x22242628, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x30303030, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x24262830, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x20222222, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x30323434, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x28303234, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x32323232, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x26283032, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x30303030, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x24262830, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x28303234, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x32322426, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x30323232, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x22242628, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x30303030, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x24262830, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x20222222, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg_type2);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg_type3[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x48484848, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x46464646, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x44464646, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x46464646, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x42444646, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x46464646, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x42444646, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x46464646, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x42444646, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x46464646, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x42444646, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x46463840, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x46464646, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x38404244, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x46464646, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x42444646, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x38383840, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x48484848, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x46464646, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x44464646, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x46464646, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x42444646, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x46464646, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x42444646, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x46464646, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x42444646, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x46464646, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x42444646, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x46463840, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x46464646, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x38404244, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x46464646, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x42444646, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x38383840, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x48484848, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x46464646, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x44464646, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x46464646, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x42444646, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x46464646, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x42444646, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x46464646, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x42444646, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x46464646, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x42444646, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x46463840, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x46464646, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x38404244, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x46464646, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x42444646, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x38383840, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x48484848, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x46464646, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x44464646, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x46464646, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x42444646, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x46464646, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x42444646, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x46464646, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x42444646, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x46464646, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x42444646, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x46463840, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x46464646, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x38404244, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x46464646, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x42444646, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x38383840, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x44464646, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x42444646, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x46464646, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x42444646, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x46464646, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x42444646, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x42444646, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x46463840, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x46464646, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x38404244, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x46464646, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x42444646, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x38383840, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x44464646, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x42444646, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x46464646, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x42444646, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x46464646, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x42444646, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x42444646, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x46463840, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x46464646, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x38404244, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x46464646, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x42444646, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x38383840, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x44464646, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x42444646, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x46464646, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x42444646, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x46464646, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x42444646, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x42444646, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x46463840, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x46464646, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x38404244, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x46464646, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x42444646, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x38383840, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x44464646, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x42444646, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x46464646, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x42444646, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x46464646, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x42444646, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x42444646, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x46463840, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x46464646, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x38404244, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x46464646, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x42444646, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x38383840, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg_type3);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg_type4[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x42424242, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x42424242, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x36384042, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x42424242, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x34363840, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x42424242, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x34363840, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x42424242, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x34363840, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x42424242, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x34363840, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x42423032, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x38404242, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x30323436, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x42424242, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x34363840, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x30303032, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x42424242, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x42424242, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x36384042, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x42424242, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x34363840, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x42424242, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x34363840, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x42424242, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x34363840, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x42424242, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x34363840, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x42423032, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x38404242, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x30323436, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x42424242, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x34363840, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x30303032, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x42424242, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x42424242, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x36384042, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x42424242, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x34363840, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x42424242, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x34363840, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x42424242, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x34363840, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x42424242, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x34363840, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x42423032, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x38404242, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x30323436, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x42424242, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x34363840, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x30303032, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x42424242, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x42424242, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x36384042, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x42424242, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x34363840, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x42424242, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x34363840, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x42424242, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x34363840, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x42424242, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x34363840, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x42423032, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x38404242, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x30323436, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x42424242, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x34363840, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x30303032, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x42424242, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x36384042, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x42424242, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x34363840, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x42424242, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x34363840, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x42424242, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x34363840, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x42424242, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x34363840, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x42423032, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x38404242, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x30323436, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x42424242, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x34363840, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x30303032, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x42424242, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x36384042, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x42424242, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x34363840, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x42424242, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x34363840, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x42424242, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x34363840, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x42424242, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x34363840, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x42423032, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x38404242, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x30323436, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x42424242, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x34363840, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x30303032, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x42424242, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x36384042, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x42424242, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x34363840, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x42424242, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x34363840, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x42424242, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x34363840, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x42424242, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x34363840, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x42423032, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x38404242, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x30323436, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x42424242, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x34363840, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x30303032, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x42424242, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x36384042, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x42424242, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x34363840, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x42424242, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x34363840, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x42424242, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x34363840, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x42424242, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x34363840, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x42423032, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x38404242, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x30323436, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x42424242, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x34363840, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x30303032, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg_type4);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg_type5[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x48484848, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x46464646, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x44464646, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x46464646, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x42444646, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x44444444, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x40424444, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x42424242, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x38404242, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x46464646, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x42444646, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x44444040, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x44444444, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x38384042, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x42424242, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x38404242, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x20203636, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x48484848, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x46464646, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x44464646, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x46464646, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x42444646, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x44444444, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x40424444, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x42424242, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x38404242, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x46464646, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x42444646, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x44444040, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x44444444, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x38384042, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x42424242, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x38404242, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x20203636, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x48484848, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x46464646, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x44464646, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x46464646, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x42444646, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x44444444, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x40424444, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x42424242, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x38404242, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x46464646, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x42444646, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x44444040, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x44444444, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x38384042, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x42424242, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x38404242, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x20203636, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x48484848, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x46464646, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x44464646, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x46464646, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x42444646, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x44444444, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x40424444, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x42424242, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x38404242, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x46464646, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x42444646, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x44444040, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x44444444, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x38384042, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x42424242, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x38404242, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x20203636, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x44464646, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x42444646, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x44444444, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x40424444, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x42424242, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x38404242, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x42444646, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x44443840, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x44444444, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x36384042, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x42424242, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x38404242, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x20203436, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x44464646, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x42444646, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x44444444, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x40424444, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x42424242, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x38404242, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x42444646, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x44443840, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x44444444, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x36384042, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x42424242, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x38404242, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x20203436, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x44464646, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x42444646, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x44444444, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x40424444, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x42424242, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x38404242, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x42444646, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x44443840, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x44444444, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x36384042, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x42424242, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x38404242, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x20203436, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x44464646, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x42444646, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x44444444, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x40424444, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x42424242, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x38404242, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x42444646, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x44443840, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x44444444, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x36384042, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x42424242, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x38404242, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x20203436, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg_type5);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg_type7[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x30323434, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x28303234, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x34343434, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x28303234, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x34343434, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x28303234, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x28303234, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x34342426, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x32343434, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x24262830, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x34343434, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x28303234, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x24263434, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x30323434, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x28303234, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x34343434, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x28303234, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x34343434, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x28303234, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x28303234, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x34342426, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x32343434, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x24262830, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x34343434, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x28303234, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x24263434, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x30323434, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x28303234, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x34343434, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x28303234, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x34343434, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x28303234, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x28303234, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x34342426, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x32343434, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x24262830, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x34343434, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x28303234, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x24263434, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x30323434, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x28303234, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x34343434, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x28303234, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x34343434, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x28303234, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x28303234, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x34342426, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x32343434, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x24262830, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x34343434, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x28303234, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x24263434, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x30323434, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x28303234, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x34343434, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x28303234, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x34343434, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x28303234, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x34343434, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x28303234, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x34342426, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x32343434, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x24262830, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x34343434, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x28303234, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x24263434, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x30323434, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x28303234, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x34343434, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x28303234, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x34343434, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x28303234, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x34343434, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x28303234, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x34342426, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x32343434, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x24262830, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x34343434, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x28303234, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x24263434, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x30323434, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x28303234, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x34343434, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x28303234, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x34343434, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x28303234, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x34343434, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x28303234, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x34342426, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x32343434, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x24262830, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x34343434, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x28303234, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x24263434, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x30323434, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x28303234, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x34343434, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x28303234, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x34343434, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x28303234, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x34343434, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x28303234, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x34342426, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x32343434, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x24262830, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x34343434, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x28303234, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x24263434, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg_type7);
+
+static const struct rtw_phy_pg_cfg_pair rtw8814a_bb_pg_type8[] = {
+	{ 0, 0, 0, 0x00000c20, 0xffffffff, 0x43434343, },
+	{ 0, 0, 0, 0x00000c24, 0xffffffff, 0x43434343, },
+	{ 0, 0, 0, 0x00000c28, 0xffffffff, 0x35373941, },
+	{ 0, 0, 0, 0x00000c2c, 0xffffffff, 0x43434343, },
+	{ 0, 0, 0, 0x00000c30, 0xffffffff, 0x33353739, },
+	{ 0, 0, 1, 0x00000c34, 0xffffffff, 0x43434343, },
+	{ 0, 0, 1, 0x00000c38, 0xffffffff, 0x31333537, },
+	{ 0, 0, 2, 0x00000cd8, 0xffffffff, 0x43434343, },
+	{ 0, 0, 2, 0x00000cdc, 0xffffffff, 0x29313335, },
+	{ 0, 0, 0, 0x00000c3c, 0xffffffff, 0x34343434, },
+	{ 0, 0, 0, 0x00000c40, 0xffffffff, 0x28303234, },
+	{ 0, 0, 0, 0x00000c44, 0xffffffff, 0x32322426, },
+	{ 0, 0, 1, 0x00000c48, 0xffffffff, 0x30323232, },
+	{ 0, 0, 1, 0x00000c4c, 0xffffffff, 0x22242628, },
+	{ 0, 0, 2, 0x00000ce0, 0xffffffff, 0x30303030, },
+	{ 0, 0, 2, 0x00000ce4, 0xffffffff, 0x24262830, },
+	{ 0, 0, 2, 0x00000ce8, 0x0000ffff, 0x20222222, },
+	{ 0, 1, 0, 0x00000e20, 0xffffffff, 0x43434343, },
+	{ 0, 1, 0, 0x00000e24, 0xffffffff, 0x43434343, },
+	{ 0, 1, 0, 0x00000e28, 0xffffffff, 0x35373941, },
+	{ 0, 1, 0, 0x00000e2c, 0xffffffff, 0x41434343, },
+	{ 0, 1, 0, 0x00000e30, 0xffffffff, 0x33353739, },
+	{ 0, 1, 1, 0x00000e34, 0xffffffff, 0x39414141, },
+	{ 0, 1, 1, 0x00000e38, 0xffffffff, 0x31333537, },
+	{ 0, 1, 2, 0x00000ed8, 0xffffffff, 0x37393939, },
+	{ 0, 1, 2, 0x00000edc, 0xffffffff, 0x29313335, },
+	{ 0, 1, 0, 0x00000e3c, 0xffffffff, 0x34343434, },
+	{ 0, 1, 0, 0x00000e40, 0xffffffff, 0x28303234, },
+	{ 0, 1, 0, 0x00000e44, 0xffffffff, 0x32322426, },
+	{ 0, 1, 1, 0x00000e48, 0xffffffff, 0x30323232, },
+	{ 0, 1, 1, 0x00000e4c, 0xffffffff, 0x22242628, },
+	{ 0, 1, 2, 0x00000ee0, 0xffffffff, 0x30303030, },
+	{ 0, 1, 2, 0x00000ee4, 0xffffffff, 0x24262830, },
+	{ 0, 1, 2, 0x00000ee8, 0x0000ffff, 0x20222222, },
+	{ 0, 2, 0, 0x00001820, 0xffffffff, 0x43434343, },
+	{ 0, 2, 0, 0x00001824, 0xffffffff, 0x43434343, },
+	{ 0, 2, 0, 0x00001828, 0xffffffff, 0x35373941, },
+	{ 0, 2, 0, 0x0000182c, 0xffffffff, 0x41434343, },
+	{ 0, 2, 0, 0x00001830, 0xffffffff, 0x33353739, },
+	{ 0, 2, 1, 0x00001834, 0xffffffff, 0x39414141, },
+	{ 0, 2, 1, 0x00001838, 0xffffffff, 0x31333537, },
+	{ 0, 2, 2, 0x000018d8, 0xffffffff, 0x37393939, },
+	{ 0, 2, 2, 0x000018dc, 0xffffffff, 0x29313335, },
+	{ 0, 2, 0, 0x0000183c, 0xffffffff, 0x34343434, },
+	{ 0, 2, 0, 0x00001840, 0xffffffff, 0x28303234, },
+	{ 0, 2, 0, 0x00001844, 0xffffffff, 0x32322426, },
+	{ 0, 2, 1, 0x00001848, 0xffffffff, 0x30323232, },
+	{ 0, 2, 1, 0x0000184c, 0xffffffff, 0x22242628, },
+	{ 0, 2, 2, 0x000018e0, 0xffffffff, 0x30303030, },
+	{ 0, 2, 2, 0x000018e4, 0xffffffff, 0x24262830, },
+	{ 0, 2, 2, 0x000018e8, 0x0000ffff, 0x20222222, },
+	{ 0, 3, 0, 0x00001a20, 0xffffffff, 0x43434343, },
+	{ 0, 3, 0, 0x00001a24, 0xffffffff, 0x43434343, },
+	{ 0, 3, 0, 0x00001a28, 0xffffffff, 0x35373941, },
+	{ 0, 3, 0, 0x00001a2c, 0xffffffff, 0x41434343, },
+	{ 0, 3, 0, 0x00001a30, 0xffffffff, 0x33353739, },
+	{ 0, 3, 1, 0x00001a34, 0xffffffff, 0x39414141, },
+	{ 0, 3, 1, 0x00001a38, 0xffffffff, 0x31333537, },
+	{ 0, 3, 2, 0x00001ad8, 0xffffffff, 0x37393939, },
+	{ 0, 3, 2, 0x00001adc, 0xffffffff, 0x29313335, },
+	{ 0, 3, 0, 0x00001a3c, 0xffffffff, 0x34343434, },
+	{ 0, 3, 0, 0x00001a40, 0xffffffff, 0x28303234, },
+	{ 0, 3, 0, 0x00001a44, 0xffffffff, 0x32322426, },
+	{ 0, 3, 1, 0x00001a48, 0xffffffff, 0x30323232, },
+	{ 0, 3, 1, 0x00001a4c, 0xffffffff, 0x22242628, },
+	{ 0, 3, 2, 0x00001ae0, 0xffffffff, 0x30303030, },
+	{ 0, 3, 2, 0x00001ae4, 0xffffffff, 0x24262830, },
+	{ 0, 3, 2, 0x00001ae8, 0x0000ffff, 0x20222222, },
+	{ 1, 0, 0, 0x00000c24, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c28, 0xffffffff, 0x39414345, },
+	{ 1, 0, 0, 0x00000c2c, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c30, 0xffffffff, 0x38404244, },
+	{ 1, 0, 1, 0x00000c34, 0xffffffff, 0x46464646, },
+	{ 1, 0, 1, 0x00000c38, 0xffffffff, 0x36384042, },
+	{ 1, 0, 2, 0x00000cd8, 0xffffffff, 0x46464646, },
+	{ 1, 0, 2, 0x00000cdc, 0xffffffff, 0x34363840, },
+	{ 1, 0, 0, 0x00000c3c, 0xffffffff, 0x46464646, },
+	{ 1, 0, 0, 0x00000c40, 0xffffffff, 0x38404244, },
+	{ 1, 0, 0, 0x00000c44, 0xffffffff, 0x46463738, },
+	{ 1, 0, 1, 0x00000c48, 0xffffffff, 0x42444646, },
+	{ 1, 0, 1, 0x00000c4c, 0xffffffff, 0x35373840, },
+	{ 1, 0, 2, 0x00000ce0, 0xffffffff, 0x46464646, },
+	{ 1, 0, 2, 0x00000ce4, 0xffffffff, 0x37394143, },
+	{ 1, 0, 2, 0x00000ce8, 0x0000ffff, 0x33333335, },
+	{ 1, 1, 0, 0x00000e24, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e28, 0xffffffff, 0x39414345, },
+	{ 1, 1, 0, 0x00000e2c, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e30, 0xffffffff, 0x38404244, },
+	{ 1, 1, 1, 0x00000e34, 0xffffffff, 0x46464646, },
+	{ 1, 1, 1, 0x00000e38, 0xffffffff, 0x36384042, },
+	{ 1, 1, 2, 0x00000ed8, 0xffffffff, 0x46464646, },
+	{ 1, 1, 2, 0x00000edc, 0xffffffff, 0x34363840, },
+	{ 1, 1, 0, 0x00000e3c, 0xffffffff, 0x46464646, },
+	{ 1, 1, 0, 0x00000e40, 0xffffffff, 0x38404244, },
+	{ 1, 1, 0, 0x00000e44, 0xffffffff, 0x46463738, },
+	{ 1, 1, 1, 0x00000e48, 0xffffffff, 0x42444646, },
+	{ 1, 1, 1, 0x00000e4c, 0xffffffff, 0x35373840, },
+	{ 1, 1, 2, 0x00000ee0, 0xffffffff, 0x46464646, },
+	{ 1, 1, 2, 0x00000ee4, 0xffffffff, 0x37394143, },
+	{ 1, 1, 2, 0x00000ee8, 0x0000ffff, 0x33333335, },
+	{ 1, 2, 0, 0x00001824, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001828, 0xffffffff, 0x39414345, },
+	{ 1, 2, 0, 0x0000182c, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001830, 0xffffffff, 0x38404244, },
+	{ 1, 2, 1, 0x00001834, 0xffffffff, 0x46464646, },
+	{ 1, 2, 1, 0x00001838, 0xffffffff, 0x36384042, },
+	{ 1, 2, 2, 0x000018d8, 0xffffffff, 0x46464646, },
+	{ 1, 2, 2, 0x000018dc, 0xffffffff, 0x34363840, },
+	{ 1, 2, 0, 0x0000183c, 0xffffffff, 0x46464646, },
+	{ 1, 2, 0, 0x00001840, 0xffffffff, 0x38404244, },
+	{ 1, 2, 0, 0x00001844, 0xffffffff, 0x46463738, },
+	{ 1, 2, 1, 0x00001848, 0xffffffff, 0x42444646, },
+	{ 1, 2, 1, 0x0000184c, 0xffffffff, 0x35373840, },
+	{ 1, 2, 2, 0x000018e0, 0xffffffff, 0x46464646, },
+	{ 1, 2, 2, 0x000018e4, 0xffffffff, 0x37394143, },
+	{ 1, 2, 2, 0x000018e8, 0x0000ffff, 0x33333335, },
+	{ 1, 3, 0, 0x00001a24, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a28, 0xffffffff, 0x39414345, },
+	{ 1, 3, 0, 0x00001a2c, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a30, 0xffffffff, 0x38404244, },
+	{ 1, 3, 1, 0x00001a34, 0xffffffff, 0x46464646, },
+	{ 1, 3, 1, 0x00001a38, 0xffffffff, 0x36384042, },
+	{ 1, 3, 2, 0x00001ad8, 0xffffffff, 0x46464646, },
+	{ 1, 3, 2, 0x00001adc, 0xffffffff, 0x34363840, },
+	{ 1, 3, 0, 0x00001a3c, 0xffffffff, 0x46464646, },
+	{ 1, 3, 0, 0x00001a40, 0xffffffff, 0x38404244, },
+	{ 1, 3, 0, 0x00001a44, 0xffffffff, 0x46463738, },
+	{ 1, 3, 1, 0x00001a48, 0xffffffff, 0x42444646, },
+	{ 1, 3, 1, 0x00001a4c, 0xffffffff, 0x35373840, },
+	{ 1, 3, 2, 0x00001ae0, 0xffffffff, 0x46464646, },
+	{ 1, 3, 2, 0x00001ae4, 0xffffffff, 0x37394143, },
+	{ 1, 3, 2, 0x00001ae8, 0x0000ffff, 0x33333335, },
+};
+
+RTW_DECL_TABLE_BB_PG(rtw8814a_bb_pg_type8);
+
+static const u32 rtw8814a_rf_a[] = {
+		0x018, 0x00013124,
+		0x040, 0x00000C00,
+		0x058, 0x00000F98,
+		0x07F, 0x00068004,
+		0x0B0, 0x000FFFFE,
+		0x0B1, 0x0003FF48,
+		0x0B2, 0x0006AA3F,
+		0x0B3, 0x000FFC9A,
+		0x0B4, 0x0000A78F,
+		0x0B5, 0x00000A3F,
+		0x0B6, 0x0000C09C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0B7, 0x00030008,
+	0xA0000000,	0x00000000,
+		0x0B7, 0x0003000C,
+	0xB0000000,	0x00000000,
+		0x0B8, 0x0007400E,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0xA0000000,	0x00000000,
+		0x0B9, 0x000FBF50,
+	0xB0000000,	0x00000000,
+		0x0BA, 0x00050780,
+		0x0BB, 0x00000000,
+		0x0BC, 0x00040009,
+		0x0BD, 0x00000000,
+		0x0BE, 0x00000000,
+		0x0BF, 0x00000000,
+		0x0EF, 0x00020000,
+		0x03E, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00030000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00030000,
+	0xA0000000,	0x00000000,
+		0x03F, 0x00030000,
+	0xB0000000,	0x00000000,
+		0x03E, 0x00020000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00040000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00040000,
+	0xA0000000,	0x00000000,
+		0x03F, 0x00040000,
+	0xB0000000,	0x00000000,
+		0x03E, 0x00040000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00030000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00030000,
+	0xA0000000,	0x00000000,
+		0x03F, 0x00030000,
+	0xB0000000,	0x00000000,
+		0x03E, 0x00060000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00030000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03F, 0x00030000,
+	0xA0000000,	0x00000000,
+		0x03F, 0x00030000,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x00010000,
+		0x03E, 0x00000000,
+		0x03F, 0x00006800,
+		0x03E, 0x00000080,
+		0x03F, 0x00006000,
+		0x03E, 0x00000100,
+		0x03F, 0x00004800,
+		0x03E, 0x00000180,
+		0x03F, 0x00004000,
+		0x03E, 0x00000200,
+		0x03F, 0x00004000,
+		0x03E, 0x00000280,
+		0x03F, 0x00002800,
+		0x03E, 0x00000300,
+		0x03F, 0x00002800,
+		0x03E, 0x00000380,
+		0x03F, 0x00002000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x00040000,
+		0x03E, 0x00000000,
+		0x03F, 0x000000BC,
+		0x03E, 0x00000040,
+		0x03F, 0x00000053,
+		0x03E, 0x00000050,
+		0x03F, 0x00000050,
+		0x03E, 0x00000060,
+		0x03F, 0x00000050,
+		0x0EF, 0x00000000,
+		0x0EF, 0x00000400,
+		0x03E, 0x00000006,
+		0x041, 0x000EE080,
+		0x03E, 0x00000008,
+		0x041, 0x000EE0C0,
+		0x03E, 0x0000000A,
+		0x041, 0x000EE100,
+		0x03E, 0x0000000C,
+		0x041, 0x000EE100,
+		0x0EF, 0x00000000,
+		0x018, 0x00000006,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0xA0000000,	0x00000000,
+		0x086, 0x000E4B58,
+		0x087, 0x00049F80,
+	0xB0000000,	0x00000000,
+		0x0DF, 0x00000008,
+		0x0EF, 0x00002000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x000179C3,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x000179C3,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x000179C3,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x000179C3,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x000179C3,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x000179C3,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x000179C3,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0003F258,
+		0x03B, 0x00030A58,
+		0x03B, 0x0002FA58,
+		0x03B, 0x00022590,
+		0x03B, 0x0001FA50,
+		0x03B, 0x00010248,
+		0x03B, 0x00008240,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000100,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000ADF6,
+		0x034, 0x00009DF3,
+		0x034, 0x00008DF0,
+		0x034, 0x00007DED,
+		0x034, 0x00006DEA,
+		0x034, 0x00005CED,
+		0x034, 0x00004CEA,
+		0x034, 0x000034EA,
+		0x034, 0x000024E7,
+		0x034, 0x0000146A,
+		0x034, 0x0000006B,
+	0xB0000000,	0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008ADF6,
+		0x034, 0x00089DF3,
+		0x034, 0x00088DF0,
+		0x034, 0x00087DED,
+		0x034, 0x00086DEA,
+		0x034, 0x00085CED,
+		0x034, 0x00084CEA,
+		0x034, 0x000834EA,
+		0x034, 0x000824E7,
+		0x034, 0x0008146A,
+		0x034, 0x0008006B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x000020A2,
+		0x0DF, 0x00000080,
+		0x035, 0x00000192,
+		0x035, 0x00008192,
+		0x035, 0x00010192,
+		0x036, 0x00000024,
+		0x036, 0x00008024,
+		0x036, 0x00010024,
+		0x036, 0x00018024,
+		0x0EF, 0x00000000,
+		0x051, 0x00000C21,
+		0x052, 0x000006D9,
+		0x053, 0x000FC649,
+		0x054, 0x0000017E,
+		0x018, 0x0001012A,
+		0x081, 0x0007FC00,
+		0x089, 0x00050110,
+		0x08A, 0x00043E50,
+		0x08B, 0x0002E180,
+		0x08C, 0x00093C3C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xA0000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0xA0000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00001000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00028000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00030023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00028623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00021633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0001C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00010293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00009593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0000118B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0000078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x000AC000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00040000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0004C000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00070023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00068623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00061633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0005C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00050293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00049593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0004138B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0004078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0008C000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00060000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00004000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B0023,
+	0x80000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A8623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A1633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0009C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00090293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00089593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0008118B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0008078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x00000800,
+		0x03B, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000801,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00000803,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00040000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001801,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000003,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000003,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001001,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001000,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00080000,
+	0x80000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001802,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000800,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001002,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x00013124,
+		0x0EF, 0x00000100,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A1AD,
+		0x034, 0x000491AA,
+		0x034, 0x000481A7,
+		0x034, 0x000470AA,
+		0x034, 0x000460A7,
+		0x034, 0x00045049,
+		0x034, 0x00044046,
+		0x034, 0x00043026,
+		0x034, 0x00042009,
+		0x034, 0x00041006,
+		0x034, 0x00040003,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AF,
+		0x034, 0x000483AB,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004406A,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AF,
+		0x034, 0x000483AB,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004406A,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AF,
+		0x034, 0x000483AB,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004406A,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AF,
+		0x034, 0x000483AB,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004406A,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3F5,
+		0x034, 0x000493F2,
+		0x034, 0x000483B0,
+		0x034, 0x00047370,
+		0x034, 0x0004636D,
+		0x034, 0x0004536A,
+		0x034, 0x00044349,
+		0x034, 0x0004316A,
+		0x034, 0x00042167,
+		0x034, 0x00041129,
+		0x034, 0x00040049,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AF,
+		0x034, 0x000483AB,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004406A,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AF,
+		0x034, 0x000483AB,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004406A,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0004AFF1,
+		0x034, 0x00049FEE,
+		0x034, 0x00048FEB,
+		0x034, 0x00047FE8,
+		0x034, 0x00046DEA,
+		0x034, 0x00045DE7,
+		0x034, 0x00044CEA,
+		0x034, 0x00043CE7,
+		0x034, 0x00042C69,
+		0x034, 0x00041C66,
+		0x034, 0x00040C28,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A1AD,
+		0x034, 0x000291AA,
+		0x034, 0x000281A7,
+		0x034, 0x000270AA,
+		0x034, 0x000260A7,
+		0x034, 0x00025049,
+		0x034, 0x00024046,
+		0x034, 0x00023026,
+		0x034, 0x00022009,
+		0x034, 0x00021006,
+		0x034, 0x00020003,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3F5,
+		0x034, 0x000293F2,
+		0x034, 0x000282F1,
+		0x034, 0x000272B0,
+		0x034, 0x000262AD,
+		0x034, 0x000252AA,
+		0x034, 0x000242A7,
+		0x034, 0x000230EC,
+		0x034, 0x000220E9,
+		0x034, 0x0002106A,
+		0x034, 0x00020067,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0002AFF1,
+		0x034, 0x00029FEE,
+		0x034, 0x00028FEB,
+		0x034, 0x00027FE8,
+		0x034, 0x00026DEA,
+		0x034, 0x00025DE7,
+		0x034, 0x00024CEA,
+		0x034, 0x00023CE7,
+		0x034, 0x00022C69,
+		0x034, 0x00021C66,
+		0x034, 0x00020C28,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EC,
+		0x034, 0x0000938C,
+		0x034, 0x000081AD,
+		0x034, 0x000071AA,
+		0x034, 0x000061A7,
+		0x034, 0x000050AA,
+		0x034, 0x000040A7,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x0000100C,
+		0x034, 0x00000009,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3F4,
+		0x034, 0x000093F1,
+		0x034, 0x000082B1,
+		0x034, 0x000071D1,
+		0x034, 0x000061CE,
+		0x034, 0x000051CB,
+		0x034, 0x000041C8,
+		0x034, 0x000030CB,
+		0x034, 0x000020C8,
+		0x034, 0x00001087,
+		0x034, 0x00000084,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000AFF1,
+		0x034, 0x00009FEE,
+		0x034, 0x00008FEB,
+		0x034, 0x00007FE8,
+		0x034, 0x00006DEA,
+		0x034, 0x00005DE7,
+		0x034, 0x00004CEA,
+		0x034, 0x00003CE7,
+		0x034, 0x00002C69,
+		0x034, 0x00001C66,
+		0x034, 0x00000C28,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA1AD,
+		0x034, 0x000C91AA,
+		0x034, 0x000C81A7,
+		0x034, 0x000C70AA,
+		0x034, 0x000C60A7,
+		0x034, 0x000C5049,
+		0x034, 0x000C4046,
+		0x034, 0x000C3026,
+		0x034, 0x000C2009,
+		0x034, 0x000C1006,
+		0x034, 0x000C0003,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AF,
+		0x034, 0x000C83AB,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C406A,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AF,
+		0x034, 0x000C83AB,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C406A,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AF,
+		0x034, 0x000C83AB,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C406A,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AF,
+		0x034, 0x000C83AB,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C406A,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3F5,
+		0x034, 0x000C93F2,
+		0x034, 0x000C83B0,
+		0x034, 0x000C7370,
+		0x034, 0x000C636D,
+		0x034, 0x000C536A,
+		0x034, 0x000C4349,
+		0x034, 0x000C316A,
+		0x034, 0x000C2167,
+		0x034, 0x000C1129,
+		0x034, 0x000C0049,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AF,
+		0x034, 0x000C83AB,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C406A,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AF,
+		0x034, 0x000C83AB,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C406A,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0xA0000000,	0x00000000,
+		0x034, 0x000CA794,
+		0x034, 0x000C9791,
+		0x034, 0x000C878E,
+		0x034, 0x000C778B,
+		0x034, 0x000C658D,
+		0x034, 0x000C558A,
+		0x034, 0x000C448D,
+		0x034, 0x000C348A,
+		0x034, 0x000C244C,
+		0x034, 0x000C1449,
+		0x034, 0x000C042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA1AD,
+		0x034, 0x000A91AA,
+		0x034, 0x000A81A7,
+		0x034, 0x000A70AA,
+		0x034, 0x000A60A7,
+		0x034, 0x000A5049,
+		0x034, 0x000A4046,
+		0x034, 0x000A3026,
+		0x034, 0x000A2009,
+		0x034, 0x000A1006,
+		0x034, 0x000A0003,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3F5,
+		0x034, 0x000A93F2,
+		0x034, 0x000A82F1,
+		0x034, 0x000A72B0,
+		0x034, 0x000A62AD,
+		0x034, 0x000A52AA,
+		0x034, 0x000A42A7,
+		0x034, 0x000A30EC,
+		0x034, 0x000A20E9,
+		0x034, 0x000A106A,
+		0x034, 0x000A0067,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0xA0000000,	0x00000000,
+		0x034, 0x000AA794,
+		0x034, 0x000A9791,
+		0x034, 0x000A878E,
+		0x034, 0x000A778B,
+		0x034, 0x000A658D,
+		0x034, 0x000A558A,
+		0x034, 0x000A448D,
+		0x034, 0x000A348A,
+		0x034, 0x000A244C,
+		0x034, 0x000A1449,
+		0x034, 0x000A042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EC,
+		0x034, 0x0008938C,
+		0x034, 0x000881AD,
+		0x034, 0x000871AA,
+		0x034, 0x000861A7,
+		0x034, 0x000850AA,
+		0x034, 0x000840A7,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x0008100C,
+		0x034, 0x00080009,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3F4,
+		0x034, 0x000893F1,
+		0x034, 0x000882B1,
+		0x034, 0x000871D1,
+		0x034, 0x000861CE,
+		0x034, 0x000851CB,
+		0x034, 0x000841C8,
+		0x034, 0x000830CB,
+		0x034, 0x000820C8,
+		0x034, 0x00081087,
+		0x034, 0x00080084,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008A794,
+		0x034, 0x00089791,
+		0x034, 0x0008878E,
+		0x034, 0x0008778B,
+		0x034, 0x0008658D,
+		0x034, 0x0008558A,
+		0x034, 0x0008448D,
+		0x034, 0x0008348A,
+		0x034, 0x0008244C,
+		0x034, 0x00081449,
+		0x034, 0x0008042B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x0DF, 0x00000001,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000040,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0xA0000000,	0x00000000,
+		0x035, 0x00000747,
+		0x035, 0x00008747,
+		0x035, 0x00010747,
+		0x035, 0x00020747,
+		0x035, 0x00028747,
+		0x035, 0x00030747,
+		0x035, 0x00040747,
+		0x035, 0x00048747,
+		0x035, 0x00050747,
+		0x035, 0x000805FB,
+		0x035, 0x000885FB,
+		0x035, 0x000905FB,
+		0x035, 0x000A05FB,
+		0x035, 0x000A85FB,
+		0x035, 0x000B05FB,
+		0x035, 0x000C05FB,
+		0x035, 0x000C85FB,
+		0x035, 0x000D05FB,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x0DF, 0x00000001,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000010,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000473,
+		0x036, 0x00008473,
+		0x036, 0x00010473,
+		0x036, 0x00020473,
+		0x036, 0x00028473,
+		0x036, 0x00030473,
+		0x036, 0x00040473,
+		0x036, 0x00048473,
+		0x036, 0x00050473,
+		0x036, 0x00080473,
+		0x036, 0x00088473,
+		0x036, 0x00090473,
+		0x036, 0x000A0473,
+		0x036, 0x000A8473,
+		0x036, 0x000B0473,
+		0x036, 0x000C0473,
+		0x036, 0x000C8473,
+		0x036, 0x000D0473,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0xA0000000,	0x00000000,
+		0x036, 0x00000473,
+		0x036, 0x00008473,
+		0x036, 0x00010473,
+		0x036, 0x00020473,
+		0x036, 0x00028473,
+		0x036, 0x00030473,
+		0x036, 0x00040473,
+		0x036, 0x00048473,
+		0x036, 0x00050473,
+		0x036, 0x00080473,
+		0x036, 0x00088473,
+		0x036, 0x00090473,
+		0x036, 0x000A0473,
+		0x036, 0x000A8473,
+		0x036, 0x000B0473,
+		0x036, 0x000C0473,
+		0x036, 0x000C8473,
+		0x036, 0x000D0473,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x0EF, 0x00000004,
+		0x037, 0x00000000,
+		0x038, 0x00005146,
+		0x037, 0x00004000,
+		0x038, 0x00005146,
+		0x037, 0x00008000,
+		0x038, 0x00005146,
+		0x037, 0x00010000,
+		0x038, 0x00005146,
+		0x037, 0x00014000,
+		0x038, 0x00005146,
+		0x037, 0x00018000,
+		0x038, 0x00004D4E,
+		0x037, 0x0001C000,
+		0x038, 0x00004D4E,
+		0x037, 0x00020000,
+		0x038, 0x00004D4E,
+		0x037, 0x00024000,
+		0x038, 0x000071C6,
+		0x037, 0x00028000,
+		0x038, 0x000071C6,
+		0x037, 0x0002C000,
+		0x038, 0x000071C6,
+		0x037, 0x00030000,
+		0x038, 0x000071CE,
+		0x037, 0x00034000,
+		0x038, 0x000071CE,
+		0x037, 0x00038000,
+		0x038, 0x00005126,
+		0x037, 0x0003C000,
+		0x038, 0x00005126,
+		0x037, 0x00040000,
+		0x038, 0x00005126,
+		0x037, 0x00044000,
+		0x038, 0x00005126,
+		0x037, 0x00048000,
+		0x038, 0x00005126,
+		0x037, 0x00080000,
+		0x038, 0x00005ECE,
+		0x037, 0x00084000,
+		0x038, 0x00005ECE,
+		0x037, 0x00088000,
+		0x038, 0x00005ECE,
+		0x037, 0x00090000,
+		0x038, 0x00005ECE,
+		0x037, 0x00094000,
+		0x038, 0x00005ECE,
+		0x037, 0x00098000,
+		0x038, 0x00005ECE,
+		0x037, 0x0009C000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000AC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000BC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C8000,
+		0x038, 0x00005ECE,
+		0x0EF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000008,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000007D,
+		0x03C, 0x0000047D,
+		0x03C, 0x0000087D,
+		0x03C, 0x0000107D,
+		0x03C, 0x0000147D,
+		0x03C, 0x0000187D,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x0000054A,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x0000154A,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x0000254A,
+		0x03C, 0x00002821,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000037E,
+		0x03C, 0x00000575,
+		0x03C, 0x00000971,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001575,
+		0x03C, 0x00001871,
+		0x03C, 0x0000217E,
+		0x03C, 0x00002575,
+		0x03C, 0x00002871,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x061, 0x000C0D47,
+		0x062, 0x0000133C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0xA0000000,	0x00000000,
+		0x063, 0x0007D0E7,
+	0xB0000000,	0x00000000,
+		0x064, 0x00014FEC,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0xA0000000,	0x00000000,
+		0x065, 0x000933FF,
+	0xB0000000,	0x00000000,
+		0x066, 0x00000040,
+		0x057, 0x00050000,
+		0x056, 0x00051DF0,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x055, 0x00082061,
+	0xA0000000,	0x00000000,
+		0x055, 0x00082060,
+	0xB0000000,	0x00000000,
+		0x01C, 0x000739D2,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x01F, 0x0002255C,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x01F, 0x0002255C,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x01F, 0x0002255C,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x01F, 0x0002255C,
+	0xA0000000,	0x00000000,
+		0x01F, 0x0002255C,
+	0xB0000000,	0x00000000,
+		0x0B1, 0x0007FF48,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0C4, 0x00081700,
+	0xA0000000,	0x00000000,
+		0x0C4, 0x00083F00,
+	0xB0000000,	0x00000000,
+		0x018, 0x0001B126,
+		0xFFE, 0x00000000,
+		0xFFE, 0x00000000,
+		0xFFE, 0x00000000,
+		0x018, 0x00013126,
+		0x018, 0x00013124,
+};
+
+RTW_DECL_TABLE_RF_RADIO(rtw8814a_rf_a, A);
+
+static const u32 rtw8814a_rf_b[] = {
+		0x018, 0x00013124,
+		0x040, 0x00000C00,
+		0x058, 0x00000F98,
+		0x07F, 0x00068004,
+		0x018, 0x00000006,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0xA0000000,	0x00000000,
+		0x086, 0x000E4B58,
+		0x087, 0x00049F80,
+	0xB0000000,	0x00000000,
+		0x0DF, 0x00000008,
+		0x0EF, 0x00002000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017BC3,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F39B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017BC3,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017BC3,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F39B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017BC3,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017BC3,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017BC3,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017BC3,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0003F258,
+		0x03B, 0x00030A58,
+		0x03B, 0x0002FA58,
+		0x03B, 0x00022590,
+		0x03B, 0x0001FA50,
+		0x03B, 0x00010248,
+		0x03B, 0x00008240,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000100,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000ADF6,
+		0x034, 0x00009DF3,
+		0x034, 0x00008DF0,
+		0x034, 0x00007DED,
+		0x034, 0x00006DEA,
+		0x034, 0x00005CED,
+		0x034, 0x00004CEA,
+		0x034, 0x000034EA,
+		0x034, 0x000024E7,
+		0x034, 0x0000146A,
+		0x034, 0x0000006B,
+	0xB0000000,	0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008ADF6,
+		0x034, 0x00089DF3,
+		0x034, 0x00088DF0,
+		0x034, 0x00087DED,
+		0x034, 0x00086DEA,
+		0x034, 0x00085CED,
+		0x034, 0x00084CEA,
+		0x034, 0x000834EA,
+		0x034, 0x000824E7,
+		0x034, 0x0008146A,
+		0x034, 0x0008006B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x000020A2,
+		0x0DF, 0x00000080,
+		0x035, 0x00000192,
+		0x035, 0x00008192,
+		0x035, 0x00010192,
+		0x036, 0x00000024,
+		0x036, 0x00008024,
+		0x036, 0x00010024,
+		0x036, 0x00018024,
+		0x0EF, 0x00000000,
+		0x051, 0x00000C21,
+		0x052, 0x000006D9,
+		0x053, 0x000FC649,
+		0x054, 0x0000017E,
+		0x018, 0x0001012A,
+		0x081, 0x0007FC00,
+		0x089, 0x00050110,
+		0x08A, 0x00043E50,
+		0x08B, 0x0002E180,
+		0x08C, 0x00093C3C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xA0000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xB0000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+		0x0EF, 0x00001000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00040000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00030023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00028623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00021633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0001C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00010293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00009593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x00000F8B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0000078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00060000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00048000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00048000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00048000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00048000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00070023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00068623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00061633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0005C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00050293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00049593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+		0x03B, 0x0004078B,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00048000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00060000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0004C000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0004C000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00048000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00004000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B0023,
+	0x80000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A8623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A1633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0009C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00090293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00089593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0008138B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0008078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x00000800,
+		0x03B, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00000803,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00040000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000800,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000800,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001000,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00080000,
+	0x80000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001802,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001802,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001002,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x00013124,
+		0x0EF, 0x00000100,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38C,
+		0x034, 0x000491AD,
+		0x034, 0x000481AA,
+		0x034, 0x000471A7,
+		0x034, 0x000460AA,
+		0x034, 0x000450A7,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x0004200C,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38C,
+		0x034, 0x00049389,
+		0x034, 0x0004816D,
+		0x034, 0x0004716A,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38B,
+		0x034, 0x00049388,
+		0x034, 0x0004818B,
+		0x034, 0x00047188,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38C,
+		0x034, 0x00049389,
+		0x034, 0x0004816D,
+		0x034, 0x0004716A,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38B,
+		0x034, 0x00049388,
+		0x034, 0x0004818B,
+		0x034, 0x00047188,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3F5,
+		0x034, 0x000493F3,
+		0x034, 0x000483B2,
+		0x034, 0x00047390,
+		0x034, 0x0004638D,
+		0x034, 0x0004538A,
+		0x034, 0x00044387,
+		0x034, 0x0004324A,
+		0x034, 0x00042247,
+		0x034, 0x0004104D,
+		0x034, 0x0004004A,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004AFF7,
+		0x034, 0x00049FF6,
+		0x034, 0x00048FF3,
+		0x034, 0x00047FF0,
+		0x034, 0x00046FED,
+		0x034, 0x00045FEA,
+		0x034, 0x00044FE7,
+		0x034, 0x00043DEA,
+		0x034, 0x00042DE7,
+		0x034, 0x00041DE4,
+		0x034, 0x00040CE7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38C,
+		0x034, 0x00049389,
+		0x034, 0x0004816D,
+		0x034, 0x0004716A,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38C,
+		0x034, 0x00049389,
+		0x034, 0x0004816D,
+		0x034, 0x0004716A,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0xA0000000,	0x00000000,
+		0x034, 0x0004AFF4,
+		0x034, 0x00049FF1,
+		0x034, 0x00048FEE,
+		0x034, 0x00047FEB,
+		0x034, 0x00046FE8,
+		0x034, 0x00045DEA,
+		0x034, 0x00044CED,
+		0x034, 0x00043CEA,
+		0x034, 0x00042C6C,
+		0x034, 0x00041C69,
+		0x034, 0x00040C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A38C,
+		0x034, 0x000291AD,
+		0x034, 0x000281AA,
+		0x034, 0x000271A7,
+		0x034, 0x000260AA,
+		0x034, 0x000250A7,
+		0x034, 0x0002402C,
+		0x034, 0x00023029,
+		0x034, 0x0002200C,
+		0x034, 0x00021009,
+		0x034, 0x00020006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EE,
+		0x034, 0x000293AC,
+		0x034, 0x00028389,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AD,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EE,
+		0x034, 0x000293AC,
+		0x034, 0x00028389,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EF,
+		0x034, 0x000293AD,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3F5,
+		0x034, 0x000293F3,
+		0x034, 0x000283D0,
+		0x034, 0x00027371,
+		0x034, 0x0002636E,
+		0x034, 0x0002536B,
+		0x034, 0x00024368,
+		0x034, 0x0002332A,
+		0x034, 0x00022327,
+		0x034, 0x0002104C,
+		0x034, 0x00020049,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002AFF7,
+		0x034, 0x00029FF6,
+		0x034, 0x00028FF3,
+		0x034, 0x00027FF0,
+		0x034, 0x00026FED,
+		0x034, 0x00025FEA,
+		0x034, 0x00024FE7,
+		0x034, 0x00023DEA,
+		0x034, 0x00022DE7,
+		0x034, 0x00021DE4,
+		0x034, 0x00020F25,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EE,
+		0x034, 0x000293AC,
+		0x034, 0x00028389,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EE,
+		0x034, 0x000293AC,
+		0x034, 0x00028389,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0002AFF4,
+		0x034, 0x00029FF1,
+		0x034, 0x00028FEE,
+		0x034, 0x00027FEB,
+		0x034, 0x00026FE8,
+		0x034, 0x00025DEA,
+		0x034, 0x00024CED,
+		0x034, 0x00023CEA,
+		0x034, 0x00022C6C,
+		0x034, 0x00021C69,
+		0x034, 0x00020C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A38C,
+		0x034, 0x000091AD,
+		0x034, 0x000081AA,
+		0x034, 0x000071A7,
+		0x034, 0x000060AA,
+		0x034, 0x000050A7,
+		0x034, 0x0000402C,
+		0x034, 0x00003029,
+		0x034, 0x00002026,
+		0x034, 0x00001009,
+		0x034, 0x00000006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EC,
+		0x034, 0x000093AC,
+		0x034, 0x000081EC,
+		0x034, 0x0000716D,
+		0x034, 0x0000616A,
+		0x034, 0x0000506D,
+		0x034, 0x0000404C,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EF,
+		0x034, 0x000093AD,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EC,
+		0x034, 0x000093AC,
+		0x034, 0x000081EC,
+		0x034, 0x0000716D,
+		0x034, 0x0000616A,
+		0x034, 0x0000506D,
+		0x034, 0x0000404C,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EF,
+		0x034, 0x000093AD,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3F4,
+		0x034, 0x000093F0,
+		0x034, 0x000083AE,
+		0x034, 0x00007350,
+		0x034, 0x0000634D,
+		0x034, 0x0000534A,
+		0x034, 0x00004347,
+		0x034, 0x0000312D,
+		0x034, 0x0000212A,
+		0x034, 0x00001127,
+		0x034, 0x0000002A,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000AFF7,
+		0x034, 0x00009FF4,
+		0x034, 0x00008FF1,
+		0x034, 0x00007FEE,
+		0x034, 0x00006FEB,
+		0x034, 0x00005FE8,
+		0x034, 0x00004DEB,
+		0x034, 0x00003DE8,
+		0x034, 0x00002DE5,
+		0x034, 0x00001C8B,
+		0x034, 0x00000C88,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EC,
+		0x034, 0x000093AC,
+		0x034, 0x000081EC,
+		0x034, 0x0000716D,
+		0x034, 0x0000616A,
+		0x034, 0x0000506D,
+		0x034, 0x0000404C,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EC,
+		0x034, 0x000093AC,
+		0x034, 0x000081EC,
+		0x034, 0x0000716D,
+		0x034, 0x0000616A,
+		0x034, 0x0000506D,
+		0x034, 0x0000404C,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000AFF4,
+		0x034, 0x00009FF1,
+		0x034, 0x00008FEE,
+		0x034, 0x00007FEB,
+		0x034, 0x00006FE8,
+		0x034, 0x00005DEA,
+		0x034, 0x00004CED,
+		0x034, 0x00003CEA,
+		0x034, 0x00002C6C,
+		0x034, 0x00001C69,
+		0x034, 0x00000C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38C,
+		0x034, 0x000C91AD,
+		0x034, 0x000C81AA,
+		0x034, 0x000C71A7,
+		0x034, 0x000C60AA,
+		0x034, 0x000C50A7,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C200C,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38C,
+		0x034, 0x000C9389,
+		0x034, 0x000C816D,
+		0x034, 0x000C716A,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38B,
+		0x034, 0x000C9388,
+		0x034, 0x000C818B,
+		0x034, 0x000C7188,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38C,
+		0x034, 0x000C9389,
+		0x034, 0x000C816D,
+		0x034, 0x000C716A,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38B,
+		0x034, 0x000C9388,
+		0x034, 0x000C818B,
+		0x034, 0x000C7188,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3F5,
+		0x034, 0x000C93F3,
+		0x034, 0x000C83B2,
+		0x034, 0x000C7390,
+		0x034, 0x000C638D,
+		0x034, 0x000C538A,
+		0x034, 0x000C4387,
+		0x034, 0x000C324A,
+		0x034, 0x000C2247,
+		0x034, 0x000C104D,
+		0x034, 0x000C004A,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CAFF7,
+		0x034, 0x000C9FF6,
+		0x034, 0x000C8FF3,
+		0x034, 0x000C7FF0,
+		0x034, 0x000C6FED,
+		0x034, 0x000C5FEA,
+		0x034, 0x000C4FE7,
+		0x034, 0x000C3DEA,
+		0x034, 0x000C2DE7,
+		0x034, 0x000C1DE4,
+		0x034, 0x000C0CE7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38C,
+		0x034, 0x000C9389,
+		0x034, 0x000C816D,
+		0x034, 0x000C716A,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38C,
+		0x034, 0x000C9389,
+		0x034, 0x000C816D,
+		0x034, 0x000C716A,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0xA0000000,	0x00000000,
+		0x034, 0x000CA794,
+		0x034, 0x000C9791,
+		0x034, 0x000C878E,
+		0x034, 0x000C778B,
+		0x034, 0x000C658D,
+		0x034, 0x000C558A,
+		0x034, 0x000C448D,
+		0x034, 0x000C348A,
+		0x034, 0x000C244C,
+		0x034, 0x000C1449,
+		0x034, 0x000C042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA38C,
+		0x034, 0x000A91AD,
+		0x034, 0x000A81AA,
+		0x034, 0x000A71A7,
+		0x034, 0x000A60AA,
+		0x034, 0x000A50A7,
+		0x034, 0x000A402C,
+		0x034, 0x000A3029,
+		0x034, 0x000A200C,
+		0x034, 0x000A1009,
+		0x034, 0x000A0006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EE,
+		0x034, 0x000A93AC,
+		0x034, 0x000A8389,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AD,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EE,
+		0x034, 0x000A93AC,
+		0x034, 0x000A8389,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EF,
+		0x034, 0x000A93AD,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3F5,
+		0x034, 0x000A93F3,
+		0x034, 0x000A83D0,
+		0x034, 0x000A7371,
+		0x034, 0x000A636E,
+		0x034, 0x000A536B,
+		0x034, 0x000A4368,
+		0x034, 0x000A332A,
+		0x034, 0x000A2327,
+		0x034, 0x000A104C,
+		0x034, 0x000A0049,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AAFF7,
+		0x034, 0x000A9FF6,
+		0x034, 0x000A8FF3,
+		0x034, 0x000A7FF0,
+		0x034, 0x000A6FED,
+		0x034, 0x000A5FEA,
+		0x034, 0x000A4FE7,
+		0x034, 0x000A3DEA,
+		0x034, 0x000A2DE7,
+		0x034, 0x000A1DE4,
+		0x034, 0x000A0F25,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EE,
+		0x034, 0x000A93AC,
+		0x034, 0x000A8389,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EE,
+		0x034, 0x000A93AC,
+		0x034, 0x000A8389,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0xA0000000,	0x00000000,
+		0x034, 0x000AA794,
+		0x034, 0x000A9791,
+		0x034, 0x000A878E,
+		0x034, 0x000A778B,
+		0x034, 0x000A658D,
+		0x034, 0x000A558A,
+		0x034, 0x000A448D,
+		0x034, 0x000A348A,
+		0x034, 0x000A244C,
+		0x034, 0x000A1449,
+		0x034, 0x000A042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A38C,
+		0x034, 0x000891AD,
+		0x034, 0x000881AA,
+		0x034, 0x000871A7,
+		0x034, 0x000860AA,
+		0x034, 0x000850A7,
+		0x034, 0x0008402C,
+		0x034, 0x00083029,
+		0x034, 0x00082026,
+		0x034, 0x00081009,
+		0x034, 0x00080006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EC,
+		0x034, 0x000893AC,
+		0x034, 0x000881EC,
+		0x034, 0x0008716D,
+		0x034, 0x0008616A,
+		0x034, 0x0008506D,
+		0x034, 0x0008404C,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EF,
+		0x034, 0x000893AD,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EC,
+		0x034, 0x000893AC,
+		0x034, 0x000881EC,
+		0x034, 0x0008716D,
+		0x034, 0x0008616A,
+		0x034, 0x0008506D,
+		0x034, 0x0008404C,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EF,
+		0x034, 0x000893AD,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3F4,
+		0x034, 0x000893F0,
+		0x034, 0x000883AE,
+		0x034, 0x00087350,
+		0x034, 0x0008634D,
+		0x034, 0x0008534A,
+		0x034, 0x00084347,
+		0x034, 0x0008312D,
+		0x034, 0x0008212A,
+		0x034, 0x00081127,
+		0x034, 0x0008002A,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008AFF7,
+		0x034, 0x00089FF4,
+		0x034, 0x00088FF1,
+		0x034, 0x00087FEE,
+		0x034, 0x00086FEB,
+		0x034, 0x00085FE8,
+		0x034, 0x00084DEB,
+		0x034, 0x00083DE8,
+		0x034, 0x00082DE5,
+		0x034, 0x00081C8B,
+		0x034, 0x00080C88,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EC,
+		0x034, 0x000893AC,
+		0x034, 0x000881EC,
+		0x034, 0x0008716D,
+		0x034, 0x0008616A,
+		0x034, 0x0008506D,
+		0x034, 0x0008404C,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EC,
+		0x034, 0x000893AC,
+		0x034, 0x000881EC,
+		0x034, 0x0008716D,
+		0x034, 0x0008616A,
+		0x034, 0x0008506D,
+		0x034, 0x0008404C,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008A794,
+		0x034, 0x00089791,
+		0x034, 0x0008878E,
+		0x034, 0x0008778B,
+		0x034, 0x0008658D,
+		0x034, 0x0008558A,
+		0x034, 0x0008448D,
+		0x034, 0x0008348A,
+		0x034, 0x0008244C,
+		0x034, 0x00081449,
+		0x034, 0x0008042B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x0DF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000040,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0xA0000000,	0x00000000,
+		0x035, 0x00000484,
+		0x035, 0x00008484,
+		0x035, 0x00010484,
+		0x035, 0x00020584,
+		0x035, 0x00028584,
+		0x035, 0x00030584,
+		0x035, 0x00040584,
+		0x035, 0x00048584,
+		0x035, 0x00050584,
+		0x035, 0x000805FB,
+		0x035, 0x000885FB,
+		0x035, 0x000905FB,
+		0x035, 0x000A05FB,
+		0x035, 0x000A85FB,
+		0x035, 0x000B05FB,
+		0x035, 0x000C05FB,
+		0x035, 0x000C85FB,
+		0x035, 0x000D05FB,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x0DF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000010,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000473,
+		0x036, 0x00008473,
+		0x036, 0x00010473,
+		0x036, 0x00020473,
+		0x036, 0x00028473,
+		0x036, 0x00030473,
+		0x036, 0x00040473,
+		0x036, 0x00048473,
+		0x036, 0x00050473,
+		0x036, 0x00080473,
+		0x036, 0x00088473,
+		0x036, 0x00090473,
+		0x036, 0x000A0473,
+		0x036, 0x000A8473,
+		0x036, 0x000B0473,
+		0x036, 0x000C0473,
+		0x036, 0x000C8473,
+		0x036, 0x000D0473,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0xA0000000,	0x00000000,
+		0x036, 0x00000474,
+		0x036, 0x00008474,
+		0x036, 0x00010474,
+		0x036, 0x00020474,
+		0x036, 0x00028474,
+		0x036, 0x00030474,
+		0x036, 0x00040474,
+		0x036, 0x00048474,
+		0x036, 0x00050474,
+		0x036, 0x00080474,
+		0x036, 0x00088474,
+		0x036, 0x00090474,
+		0x036, 0x000A0474,
+		0x036, 0x000A8474,
+		0x036, 0x000B0474,
+		0x036, 0x000C0474,
+		0x036, 0x000C8474,
+		0x036, 0x000D0474,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x0EF, 0x00000004,
+		0x037, 0x00000000,
+		0x038, 0x0000514E,
+		0x037, 0x00004000,
+		0x038, 0x0000514E,
+		0x037, 0x00008000,
+		0x038, 0x0000514E,
+		0x037, 0x00010000,
+		0x038, 0x0000514E,
+		0x037, 0x00014000,
+		0x038, 0x0000514E,
+		0x037, 0x00018000,
+		0x038, 0x0000514E,
+		0x037, 0x0001C000,
+		0x038, 0x0000514E,
+		0x037, 0x00020000,
+		0x038, 0x0000514E,
+		0x037, 0x00024000,
+		0x038, 0x0000514E,
+		0x037, 0x00028000,
+		0x038, 0x0000514E,
+		0x037, 0x0002C000,
+		0x038, 0x0000714E,
+		0x037, 0x00030000,
+		0x038, 0x0000514E,
+		0x037, 0x00034000,
+		0x038, 0x0000514E,
+		0x037, 0x00038000,
+		0x038, 0x0000514E,
+		0x037, 0x0003C000,
+		0x038, 0x0000514E,
+		0x037, 0x00040000,
+		0x038, 0x0000514E,
+		0x037, 0x00044000,
+		0x038, 0x0000514E,
+		0x037, 0x00048000,
+		0x038, 0x0000514E,
+		0x037, 0x00080000,
+		0x038, 0x00005ECE,
+		0x037, 0x00084000,
+		0x038, 0x00005ECE,
+		0x037, 0x00088000,
+		0x038, 0x00005ECE,
+		0x037, 0x00090000,
+		0x038, 0x00005ECE,
+		0x037, 0x00094000,
+		0x038, 0x00005ECE,
+		0x037, 0x00098000,
+		0x038, 0x00005ECE,
+		0x037, 0x0009C000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000AC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000BC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C8000,
+		0x038, 0x00005ECE,
+		0x0EF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000008,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000007D,
+		0x03C, 0x0000047D,
+		0x03C, 0x0000087D,
+		0x03C, 0x0000107D,
+		0x03C, 0x0000147D,
+		0x03C, 0x0000187D,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027E,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227E,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000037E,
+		0x03C, 0x00000575,
+		0x03C, 0x00000971,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001575,
+		0x03C, 0x00001871,
+		0x03C, 0x0000217E,
+		0x03C, 0x00002575,
+		0x03C, 0x00002871,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x061, 0x000C0D47,
+		0x062, 0x0000133C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0xA0000000,	0x00000000,
+		0x063, 0x0007D0E7,
+	0xB0000000,	0x00000000,
+		0x064, 0x00014FEC,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0xA0000000,	0x00000000,
+		0x065, 0x000923FF,
+	0xB0000000,	0x00000000,
+		0x066, 0x00000040,
+		0x057, 0x00050000,
+		0x056, 0x00051DF0,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x055, 0x00082060,
+	0xB0000000,	0x00000000,
+};
+
+RTW_DECL_TABLE_RF_RADIO(rtw8814a_rf_b, B);
+
+static const u32 rtw8814a_rf_c[] = {
+		0x018, 0x00013124,
+		0x040, 0x00000C00,
+		0x058, 0x00000F98,
+		0x07F, 0x00068004,
+		0x018, 0x00000006,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0xA0000000,	0x00000000,
+		0x086, 0x000E4B58,
+		0x087, 0x00049F80,
+	0xB0000000,	0x00000000,
+		0x0DF, 0x00000008,
+		0x0EF, 0x00002000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017823,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017823,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017823,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017823,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017823,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017823,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017823,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0003F258,
+		0x03B, 0x00030A58,
+		0x03B, 0x0002FA58,
+		0x03B, 0x00022590,
+		0x03B, 0x0001FA50,
+		0x03B, 0x00010248,
+		0x03B, 0x00008240,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000100,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000ADF6,
+		0x034, 0x00009DF3,
+		0x034, 0x00008DF0,
+		0x034, 0x00007DED,
+		0x034, 0x00006DEA,
+		0x034, 0x00005CED,
+		0x034, 0x00004CEA,
+		0x034, 0x000034EA,
+		0x034, 0x000024E7,
+		0x034, 0x0000146A,
+		0x034, 0x0000006B,
+	0xB0000000,	0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008ADF6,
+		0x034, 0x00089DF3,
+		0x034, 0x00088DF0,
+		0x034, 0x00087DED,
+		0x034, 0x00086DEA,
+		0x034, 0x00085CED,
+		0x034, 0x00084CEA,
+		0x034, 0x000834EA,
+		0x034, 0x000824E7,
+		0x034, 0x0008146A,
+		0x034, 0x0008006B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x000020A2,
+		0x0DF, 0x00000080,
+		0x035, 0x00000192,
+		0x035, 0x00008192,
+		0x035, 0x00010192,
+		0x036, 0x00000024,
+		0x036, 0x00008024,
+		0x036, 0x00010024,
+		0x036, 0x00018024,
+		0x0EF, 0x00000000,
+		0x051, 0x00000C21,
+		0x052, 0x000006D9,
+		0x053, 0x000FC649,
+		0x054, 0x0000017E,
+		0x018, 0x0001012A,
+		0x081, 0x0007FC00,
+		0x089, 0x00050110,
+		0x08A, 0x00043E50,
+		0x08B, 0x0002E180,
+		0x08C, 0x00093C3C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xA0000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0xA0000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00001000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0006C000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x000D4000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00080000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0006C000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0008C000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00004000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x000A0000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00030023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00028623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00021633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0001C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00010293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00009593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0000118B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0000078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0004C000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00084000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00080000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0004C000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x000D0000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00080000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00080000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00028000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00070023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00068623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00061633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0005C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00050293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00049593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+		0x03B, 0x0004078B,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00060000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00080000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B0023,
+	0x80000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A8623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A1633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0009C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00090293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00089593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0008128B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0008078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x00000800,
+		0x03B, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001803,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001803,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00000803,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00040000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000800,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001000,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00080000,
+	0x80000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001802,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001802,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001002,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+	0xB0000000,	0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x018, 0x00013124,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x018, 0x00013124,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x018, 0x00013124,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x018, 0x00013124,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x018, 0x00013124,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x018, 0x00013124,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x018, 0x00013124,
+	0xA0000000,	0x00000000,
+		0x018, 0x00013124,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000100,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A38C,
+		0x034, 0x000491AD,
+		0x034, 0x000481AA,
+		0x034, 0x000471A7,
+		0x034, 0x000460AA,
+		0x034, 0x000450A7,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x0004200C,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AD,
+		0x034, 0x0004838A,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004404C,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AD,
+		0x034, 0x0004838A,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004404C,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AD,
+		0x034, 0x0004838A,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004404C,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AD,
+		0x034, 0x0004838A,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004404C,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3F5,
+		0x034, 0x000493F3,
+		0x034, 0x00048393,
+		0x034, 0x00047390,
+		0x034, 0x0004638D,
+		0x034, 0x0004538A,
+		0x034, 0x00044387,
+		0x034, 0x000430ED,
+		0x034, 0x000420EA,
+		0x034, 0x000410E7,
+		0x034, 0x0004002D,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004AFF7,
+		0x034, 0x00049FF6,
+		0x034, 0x00048FF3,
+		0x034, 0x00047FF0,
+		0x034, 0x00046FED,
+		0x034, 0x00045FEA,
+		0x034, 0x00044FE7,
+		0x034, 0x00043CD0,
+		0x034, 0x00042CCD,
+		0x034, 0x00041CCA,
+		0x034, 0x00040CC7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AD,
+		0x034, 0x0004838A,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004404C,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EF,
+		0x034, 0x000493AD,
+		0x034, 0x0004838A,
+		0x034, 0x0004718C,
+		0x034, 0x00046189,
+		0x034, 0x0004506D,
+		0x034, 0x0004404C,
+		0x034, 0x0004302C,
+		0x034, 0x00042029,
+		0x034, 0x00041026,
+		0x034, 0x00040023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0004AFF4,
+		0x034, 0x00049FF1,
+		0x034, 0x00048FEE,
+		0x034, 0x00047FEB,
+		0x034, 0x00046FE8,
+		0x034, 0x00045DEA,
+		0x034, 0x00044CED,
+		0x034, 0x00043CEA,
+		0x034, 0x00042C6C,
+		0x034, 0x00041C69,
+		0x034, 0x00040C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EC,
+		0x034, 0x0002938C,
+		0x034, 0x000281AD,
+		0x034, 0x000271AA,
+		0x034, 0x000261A7,
+		0x034, 0x000250AA,
+		0x034, 0x000240A7,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x0002100C,
+		0x034, 0x00020009,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EC,
+		0x034, 0x0002936D,
+		0x034, 0x0002836A,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EC,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EC,
+		0x034, 0x0002936D,
+		0x034, 0x0002836A,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EC,
+		0x034, 0x000293AC,
+		0x034, 0x0002838A,
+		0x034, 0x0002718C,
+		0x034, 0x00026189,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3F5,
+		0x034, 0x000293F3,
+		0x034, 0x000282F2,
+		0x034, 0x000272D0,
+		0x034, 0x000262CD,
+		0x034, 0x000252CA,
+		0x034, 0x000242C7,
+		0x034, 0x000230CD,
+		0x034, 0x000220CA,
+		0x034, 0x000210C7,
+		0x034, 0x00020086,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002AFF7,
+		0x034, 0x00029FF6,
+		0x034, 0x00028FF3,
+		0x034, 0x00027FF0,
+		0x034, 0x00026FED,
+		0x034, 0x00025FEA,
+		0x034, 0x00024FE7,
+		0x034, 0x00023DEA,
+		0x034, 0x00022DE7,
+		0x034, 0x00021DE4,
+		0x034, 0x00020E44,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EC,
+		0x034, 0x0002936D,
+		0x034, 0x0002836A,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EC,
+		0x034, 0x0002936D,
+		0x034, 0x0002836A,
+		0x034, 0x0002716D,
+		0x034, 0x0002616A,
+		0x034, 0x0002506D,
+		0x034, 0x0002406A,
+		0x034, 0x0002302C,
+		0x034, 0x00022029,
+		0x034, 0x00021026,
+		0x034, 0x00020023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0002AFF4,
+		0x034, 0x00029FF1,
+		0x034, 0x00028FEE,
+		0x034, 0x00027FEB,
+		0x034, 0x00026FE8,
+		0x034, 0x00025DEA,
+		0x034, 0x00024CED,
+		0x034, 0x00023CEA,
+		0x034, 0x00022C6C,
+		0x034, 0x00021C69,
+		0x034, 0x00020C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A38C,
+		0x034, 0x000091AD,
+		0x034, 0x000081AA,
+		0x034, 0x000071A7,
+		0x034, 0x000060AA,
+		0x034, 0x000050A7,
+		0x034, 0x0000402C,
+		0x034, 0x00003029,
+		0x034, 0x0000200C,
+		0x034, 0x00001009,
+		0x034, 0x00000006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AB,
+		0x034, 0x00008389,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AB,
+		0x034, 0x00008389,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AB,
+		0x034, 0x00008389,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AB,
+		0x034, 0x00008389,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3F5,
+		0x034, 0x000093F1,
+		0x034, 0x000083B0,
+		0x034, 0x00007370,
+		0x034, 0x0000636D,
+		0x034, 0x0000536A,
+		0x034, 0x00004367,
+		0x034, 0x0000308E,
+		0x034, 0x0000208B,
+		0x034, 0x00001088,
+		0x034, 0x00000085,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000AFF7,
+		0x034, 0x00009FF5,
+		0x034, 0x00008FF2,
+		0x034, 0x00007FEF,
+		0x034, 0x00006FEC,
+		0x034, 0x00005FE9,
+		0x034, 0x00004EAA,
+		0x034, 0x00003EA7,
+		0x034, 0x00002C70,
+		0x034, 0x00001C6D,
+		0x034, 0x00000C6A,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AB,
+		0x034, 0x00008389,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AB,
+		0x034, 0x00008389,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000AFF4,
+		0x034, 0x00009FF1,
+		0x034, 0x00008FEE,
+		0x034, 0x00007FEB,
+		0x034, 0x00006FE8,
+		0x034, 0x00005DEA,
+		0x034, 0x00004CED,
+		0x034, 0x00003CEA,
+		0x034, 0x00002C6C,
+		0x034, 0x00001C69,
+		0x034, 0x00000C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA38C,
+		0x034, 0x000C91AD,
+		0x034, 0x000C81AA,
+		0x034, 0x000C71A7,
+		0x034, 0x000C60AA,
+		0x034, 0x000C50A7,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C200C,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AD,
+		0x034, 0x000C838A,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C404C,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AD,
+		0x034, 0x000C838A,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C404C,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AD,
+		0x034, 0x000C838A,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C404C,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AD,
+		0x034, 0x000C838A,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C404C,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3F5,
+		0x034, 0x000C93F3,
+		0x034, 0x000C8393,
+		0x034, 0x000C7390,
+		0x034, 0x000C638D,
+		0x034, 0x000C538A,
+		0x034, 0x000C4387,
+		0x034, 0x000C30ED,
+		0x034, 0x000C20EA,
+		0x034, 0x000C10E7,
+		0x034, 0x000C002D,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CAFF7,
+		0x034, 0x000C9FF6,
+		0x034, 0x000C8FF3,
+		0x034, 0x000C7FF0,
+		0x034, 0x000C6FED,
+		0x034, 0x000C5FEA,
+		0x034, 0x000C4FE7,
+		0x034, 0x000C3CD0,
+		0x034, 0x000C2CCD,
+		0x034, 0x000C1CCA,
+		0x034, 0x000C0CC7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AD,
+		0x034, 0x000C838A,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C404C,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EF,
+		0x034, 0x000C93AD,
+		0x034, 0x000C838A,
+		0x034, 0x000C718C,
+		0x034, 0x000C6189,
+		0x034, 0x000C506D,
+		0x034, 0x000C404C,
+		0x034, 0x000C302C,
+		0x034, 0x000C2029,
+		0x034, 0x000C1026,
+		0x034, 0x000C0023,
+	0xA0000000,	0x00000000,
+		0x034, 0x000CA794,
+		0x034, 0x000C9791,
+		0x034, 0x000C878E,
+		0x034, 0x000C778B,
+		0x034, 0x000C658D,
+		0x034, 0x000C558A,
+		0x034, 0x000C448D,
+		0x034, 0x000C348A,
+		0x034, 0x000C244C,
+		0x034, 0x000C1449,
+		0x034, 0x000C042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EC,
+		0x034, 0x000A938C,
+		0x034, 0x000A81AD,
+		0x034, 0x000A71AA,
+		0x034, 0x000A61A7,
+		0x034, 0x000A50AA,
+		0x034, 0x000A40A7,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A100C,
+		0x034, 0x000A0009,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EC,
+		0x034, 0x000A936D,
+		0x034, 0x000A836A,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EC,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EC,
+		0x034, 0x000A936D,
+		0x034, 0x000A836A,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EC,
+		0x034, 0x000A93AC,
+		0x034, 0x000A838A,
+		0x034, 0x000A718C,
+		0x034, 0x000A6189,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3F5,
+		0x034, 0x000A93F3,
+		0x034, 0x000A82F2,
+		0x034, 0x000A72D0,
+		0x034, 0x000A62CD,
+		0x034, 0x000A52CA,
+		0x034, 0x000A42C7,
+		0x034, 0x000A30CD,
+		0x034, 0x000A20CA,
+		0x034, 0x000A10C7,
+		0x034, 0x000A0086,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AAFF7,
+		0x034, 0x000A9FF6,
+		0x034, 0x000A8FF3,
+		0x034, 0x000A7FF0,
+		0x034, 0x000A6FED,
+		0x034, 0x000A5FEA,
+		0x034, 0x000A4FE7,
+		0x034, 0x000A3DEA,
+		0x034, 0x000A2DE7,
+		0x034, 0x000A1DE4,
+		0x034, 0x000A0E44,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EC,
+		0x034, 0x000A936D,
+		0x034, 0x000A836A,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EC,
+		0x034, 0x000A936D,
+		0x034, 0x000A836A,
+		0x034, 0x000A716D,
+		0x034, 0x000A616A,
+		0x034, 0x000A506D,
+		0x034, 0x000A406A,
+		0x034, 0x000A302C,
+		0x034, 0x000A2029,
+		0x034, 0x000A1026,
+		0x034, 0x000A0023,
+	0xA0000000,	0x00000000,
+		0x034, 0x000AA794,
+		0x034, 0x000A9791,
+		0x034, 0x000A878E,
+		0x034, 0x000A778B,
+		0x034, 0x000A658D,
+		0x034, 0x000A558A,
+		0x034, 0x000A448D,
+		0x034, 0x000A348A,
+		0x034, 0x000A244C,
+		0x034, 0x000A1449,
+		0x034, 0x000A042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A38C,
+		0x034, 0x000891AD,
+		0x034, 0x000881AA,
+		0x034, 0x000871A7,
+		0x034, 0x000860AA,
+		0x034, 0x000850A7,
+		0x034, 0x0008402C,
+		0x034, 0x00083029,
+		0x034, 0x0008200C,
+		0x034, 0x00081009,
+		0x034, 0x00000006,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AB,
+		0x034, 0x00088389,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AB,
+		0x034, 0x00088389,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AB,
+		0x034, 0x00088389,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AB,
+		0x034, 0x00088389,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3F5,
+		0x034, 0x000893F1,
+		0x034, 0x000883B0,
+		0x034, 0x00087370,
+		0x034, 0x0008636D,
+		0x034, 0x0008536A,
+		0x034, 0x00084367,
+		0x034, 0x0008308E,
+		0x034, 0x0008208B,
+		0x034, 0x00081088,
+		0x034, 0x00080085,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008AFF7,
+		0x034, 0x00089FF5,
+		0x034, 0x00088FF2,
+		0x034, 0x00087FEF,
+		0x034, 0x00086FEC,
+		0x034, 0x00085FE9,
+		0x034, 0x00084EAA,
+		0x034, 0x00083EA7,
+		0x034, 0x00082C70,
+		0x034, 0x00081C6D,
+		0x034, 0x00080C6A,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AB,
+		0x034, 0x00088389,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AB,
+		0x034, 0x00088389,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008A794,
+		0x034, 0x00089791,
+		0x034, 0x0008878E,
+		0x034, 0x0008778B,
+		0x034, 0x0008658D,
+		0x034, 0x0008558A,
+		0x034, 0x0008448D,
+		0x034, 0x0008348A,
+		0x034, 0x0008244C,
+		0x034, 0x00081449,
+		0x034, 0x0008042B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x0DF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000040,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0xA0000000,	0x00000000,
+		0x035, 0x00000484,
+		0x035, 0x00008484,
+		0x035, 0x00010484,
+		0x035, 0x00020584,
+		0x035, 0x00028584,
+		0x035, 0x00030584,
+		0x035, 0x00040584,
+		0x035, 0x00048584,
+		0x035, 0x00050584,
+		0x035, 0x000805FB,
+		0x035, 0x000885FB,
+		0x035, 0x000905FB,
+		0x035, 0x000A05FB,
+		0x035, 0x000A85FB,
+		0x035, 0x000B05FB,
+		0x035, 0x000C05FB,
+		0x035, 0x000C85FB,
+		0x035, 0x000D05FB,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x0DF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000010,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000473,
+		0x036, 0x00008473,
+		0x036, 0x00010473,
+		0x036, 0x00020473,
+		0x036, 0x00028473,
+		0x036, 0x00030473,
+		0x036, 0x00040473,
+		0x036, 0x00048473,
+		0x036, 0x00050473,
+		0x036, 0x00080473,
+		0x036, 0x00088473,
+		0x036, 0x00090473,
+		0x036, 0x000A0473,
+		0x036, 0x000A8473,
+		0x036, 0x000B0473,
+		0x036, 0x000C0473,
+		0x036, 0x000C8473,
+		0x036, 0x000D0473,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0xA0000000,	0x00000000,
+		0x036, 0x00000474,
+		0x036, 0x00008474,
+		0x036, 0x00010474,
+		0x036, 0x00020474,
+		0x036, 0x00028474,
+		0x036, 0x00030474,
+		0x036, 0x00040474,
+		0x036, 0x00048474,
+		0x036, 0x00050474,
+		0x036, 0x00080474,
+		0x036, 0x00088474,
+		0x036, 0x00090474,
+		0x036, 0x000A0474,
+		0x036, 0x000A8474,
+		0x036, 0x000B0474,
+		0x036, 0x000C0474,
+		0x036, 0x000C8474,
+		0x036, 0x000D0474,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x0EF, 0x00000004,
+		0x037, 0x00000000,
+		0x038, 0x0000514E,
+		0x037, 0x00004000,
+		0x038, 0x0000514E,
+		0x037, 0x00008000,
+		0x038, 0x0000514E,
+		0x037, 0x00010000,
+		0x038, 0x0000514E,
+		0x037, 0x00014000,
+		0x038, 0x0000514E,
+		0x037, 0x00018000,
+		0x038, 0x0000514E,
+		0x037, 0x0001C000,
+		0x038, 0x0000514E,
+		0x037, 0x00020000,
+		0x038, 0x0000514E,
+		0x037, 0x00024000,
+		0x038, 0x0000514E,
+		0x037, 0x00028000,
+		0x038, 0x0000514E,
+		0x037, 0x0002C000,
+		0x038, 0x0000714E,
+		0x037, 0x00030000,
+		0x038, 0x0000514E,
+		0x037, 0x00034000,
+		0x038, 0x0000514E,
+		0x037, 0x00038000,
+		0x038, 0x0000514E,
+		0x037, 0x0003C000,
+		0x038, 0x0000514E,
+		0x037, 0x00040000,
+		0x038, 0x0000514E,
+		0x037, 0x00044000,
+		0x038, 0x0000514E,
+		0x037, 0x00048000,
+		0x038, 0x0000514E,
+		0x037, 0x00080000,
+		0x038, 0x00005ECE,
+		0x037, 0x00084000,
+		0x038, 0x00005ECE,
+		0x037, 0x00088000,
+		0x038, 0x00005ECE,
+		0x037, 0x00090000,
+		0x038, 0x00005ECE,
+		0x037, 0x00094000,
+		0x038, 0x00005ECE,
+		0x037, 0x00098000,
+		0x038, 0x00005ECE,
+		0x037, 0x0009C000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000AC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000BC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C8000,
+		0x038, 0x00005ECE,
+		0x0EF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000008,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000007D,
+		0x03C, 0x0000047D,
+		0x03C, 0x0000087D,
+		0x03C, 0x0000107D,
+		0x03C, 0x0000147D,
+		0x03C, 0x0000187D,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000541,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001541,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002541,
+		0x03C, 0x00002821,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027D,
+		0x03C, 0x00000546,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127D,
+		0x03C, 0x00001546,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227D,
+		0x03C, 0x00002546,
+		0x03C, 0x00002821,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000037E,
+		0x03C, 0x00000575,
+		0x03C, 0x00000971,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001575,
+		0x03C, 0x00001871,
+		0x03C, 0x0000217E,
+		0x03C, 0x00002575,
+		0x03C, 0x00002871,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x061, 0x000C0D47,
+		0x062, 0x0000133C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0xA0000000,	0x00000000,
+		0x063, 0x0007D0E7,
+	0xB0000000,	0x00000000,
+		0x064, 0x00014FEC,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0xA0000000,	0x00000000,
+		0x065, 0x000923FF,
+	0xB0000000,	0x00000000,
+		0x066, 0x00000040,
+		0x057, 0x00050000,
+		0x056, 0x00051DF0,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x055, 0x00082060,
+	0xB0000000,	0x00000000,
+};
+
+RTW_DECL_TABLE_RF_RADIO(rtw8814a_rf_c, C);
+
+static const u32 rtw8814a_rf_d[] = {
+		0x018, 0x00013124,
+		0x040, 0x00000C00,
+		0x058, 0x00000F98,
+		0x07F, 0x00068004,
+		0x018, 0x00000006,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x086, 0x000E335A,
+		0x087, 0x00079F80,
+	0xA0000000,	0x00000000,
+		0x086, 0x000E4B58,
+		0x087, 0x00049F80,
+	0xB0000000,	0x00000000,
+		0x0DF, 0x00000008,
+		0x0EF, 0x00002000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017803,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F09B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017803,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017803,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F09B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017803,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017803,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017803,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0003F19B,
+		0x03B, 0x00037A5B,
+		0x03B, 0x0002A433,
+		0x03B, 0x00027BD3,
+		0x03B, 0x0001F80B,
+		0x03B, 0x00017803,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0003F258,
+		0x03B, 0x00030A58,
+		0x03B, 0x0002FA58,
+		0x03B, 0x00022590,
+		0x03B, 0x0001FA50,
+		0x03B, 0x00010248,
+		0x03B, 0x00008240,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000100,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000ADF6,
+		0x034, 0x00009DF3,
+		0x034, 0x00008DF0,
+		0x034, 0x00007DED,
+		0x034, 0x00006DEA,
+		0x034, 0x00005CED,
+		0x034, 0x00004CEA,
+		0x034, 0x000034EA,
+		0x034, 0x000024E7,
+		0x034, 0x0000146A,
+		0x034, 0x0000006B,
+	0xB0000000,	0x00000000,
+	0x80000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A0D0,
+		0x034, 0x000090CD,
+		0x034, 0x000080CA,
+		0x034, 0x0000704D,
+		0x034, 0x0000604A,
+		0x034, 0x00005047,
+		0x034, 0x0000400A,
+		0x034, 0x00003007,
+		0x034, 0x00002004,
+		0x034, 0x00001001,
+		0x034, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008ADF6,
+		0x034, 0x00089DF3,
+		0x034, 0x00088DF0,
+		0x034, 0x00087DED,
+		0x034, 0x00086DEA,
+		0x034, 0x00085CED,
+		0x034, 0x00084CEA,
+		0x034, 0x000834EA,
+		0x034, 0x000824E7,
+		0x034, 0x0008146A,
+		0x034, 0x0008006B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x000020A2,
+		0x0DF, 0x00000080,
+		0x035, 0x00000192,
+		0x035, 0x00008192,
+		0x035, 0x00010192,
+		0x036, 0x00000024,
+		0x036, 0x00008024,
+		0x036, 0x00010024,
+		0x036, 0x00018024,
+		0x0EF, 0x00000000,
+		0x051, 0x00000C21,
+		0x052, 0x000006D9,
+		0x053, 0x000FC649,
+		0x054, 0x0000017E,
+		0x018, 0x0001012A,
+		0x081, 0x0007FC00,
+		0x089, 0x00050110,
+		0x08A, 0x00043E50,
+		0x08B, 0x0002E180,
+		0x08C, 0x00093C3C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xA0000000,	0x00000000,
+		0x085, 0x000F8000,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0xA0000000,	0x00000000,
+		0x08D, 0x000FFFF0,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00001000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00038023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00048000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00040000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00088000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00048000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00030023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00028623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00021633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0001C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00010293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00009593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x00000F8B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0000078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00078023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00044000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00024000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00070023,
+		0x03C, 0x00048000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00068623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00061633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0005C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00050293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00049593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x00040F8B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0004078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xA0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B8023,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00004000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00060000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00004000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00060000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00024000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00004000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000B0023,
+	0x80000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x00020000,
+	0xB0000000,	0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A8623,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x000A1633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x0009C633,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00090293,
+		0x03C, 0x00000000,
+		0x03A, 0x0000013C,
+		0x03B, 0x00089593,
+		0x03C, 0x00000000,
+		0x03A, 0x00000148,
+	0x80000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03B, 0x0008138B,
+	0xA0000000,	0x00000000,
+		0x03B, 0x0008078B,
+	0xB0000000,	0x00000000,
+		0x03C, 0x00000000,
+		0x0EF, 0x00000000,
+		0x0EF, 0x00000800,
+		0x03B, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001003,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001803,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000803,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00000803,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00040000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001002,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000001,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000802,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001803,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001000,
+	0xB0000000,	0x00000000,
+		0x03B, 0x00080000,
+	0x80000007,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001802,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00001000,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x03A, 0x00000802,
+	0xA0000000,	0x00000000,
+		0x03A, 0x00001002,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000006,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x00013124,
+		0x0EF, 0x00000100,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3EB,
+		0x034, 0x0004938B,
+		0x034, 0x000481AC,
+		0x034, 0x000471A9,
+		0x034, 0x000460AC,
+		0x034, 0x000450A9,
+		0x034, 0x0004402E,
+		0x034, 0x0004302B,
+		0x034, 0x00042028,
+		0x034, 0x0004100B,
+		0x034, 0x00040008,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3AD,
+		0x034, 0x0004938A,
+		0x034, 0x0004818C,
+		0x034, 0x00047189,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3AD,
+		0x034, 0x0004938A,
+		0x034, 0x0004818C,
+		0x034, 0x00047189,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3AD,
+		0x034, 0x0004938A,
+		0x034, 0x0004818C,
+		0x034, 0x00047189,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3AD,
+		0x034, 0x0004938A,
+		0x034, 0x0004818C,
+		0x034, 0x00047189,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3F4,
+		0x034, 0x000493D2,
+		0x034, 0x000482D1,
+		0x034, 0x000471F1,
+		0x034, 0x000461EE,
+		0x034, 0x000451EB,
+		0x034, 0x000441E8,
+		0x034, 0x0004314B,
+		0x034, 0x00042148,
+		0x034, 0x0004104B,
+		0x034, 0x00040048,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004AFF7,
+		0x034, 0x00049FF6,
+		0x034, 0x00048FF3,
+		0x034, 0x00047FF0,
+		0x034, 0x00046FED,
+		0x034, 0x00045FEA,
+		0x034, 0x00044FE7,
+		0x034, 0x00043CB1,
+		0x034, 0x00042CAE,
+		0x034, 0x00041CAB,
+		0x034, 0x00040CA8,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3AD,
+		0x034, 0x0004938A,
+		0x034, 0x0004818C,
+		0x034, 0x00047189,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0004A3AD,
+		0x034, 0x0004938A,
+		0x034, 0x0004818C,
+		0x034, 0x00047189,
+		0x034, 0x0004606D,
+		0x034, 0x0004506A,
+		0x034, 0x0004402C,
+		0x034, 0x00043029,
+		0x034, 0x00042026,
+		0x034, 0x00041009,
+		0x034, 0x00040006,
+	0xA0000000,	0x00000000,
+		0x034, 0x0004AFF4,
+		0x034, 0x00049FF1,
+		0x034, 0x00048FEE,
+		0x034, 0x00047FEB,
+		0x034, 0x00046FE8,
+		0x034, 0x00045DEA,
+		0x034, 0x00044CED,
+		0x034, 0x00043CEA,
+		0x034, 0x00042C6C,
+		0x034, 0x00041C69,
+		0x034, 0x00040C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3EE,
+		0x034, 0x000293EB,
+		0x034, 0x0002838B,
+		0x034, 0x000271AC,
+		0x034, 0x000261A9,
+		0x034, 0x000250AC,
+		0x034, 0x000240A9,
+		0x034, 0x000230A6,
+		0x034, 0x0002202C,
+		0x034, 0x00021029,
+		0x034, 0x00020026,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3AD,
+		0x034, 0x0002938A,
+		0x034, 0x0002818C,
+		0x034, 0x00027189,
+		0x034, 0x0002606D,
+		0x034, 0x0002504C,
+		0x034, 0x0002402C,
+		0x034, 0x00023029,
+		0x034, 0x00022026,
+		0x034, 0x00021023,
+		0x034, 0x00020006,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3AD,
+		0x034, 0x0002938A,
+		0x034, 0x0002818C,
+		0x034, 0x00027189,
+		0x034, 0x0002606D,
+		0x034, 0x0002504C,
+		0x034, 0x0002402C,
+		0x034, 0x00023029,
+		0x034, 0x00022026,
+		0x034, 0x00021023,
+		0x034, 0x00020006,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3AD,
+		0x034, 0x0002938A,
+		0x034, 0x0002818C,
+		0x034, 0x00027189,
+		0x034, 0x0002606D,
+		0x034, 0x0002504C,
+		0x034, 0x0002402C,
+		0x034, 0x00023029,
+		0x034, 0x00022026,
+		0x034, 0x00021023,
+		0x034, 0x00020006,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3AD,
+		0x034, 0x0002938A,
+		0x034, 0x0002818C,
+		0x034, 0x00027189,
+		0x034, 0x0002606D,
+		0x034, 0x0002504C,
+		0x034, 0x0002402C,
+		0x034, 0x00023029,
+		0x034, 0x00022026,
+		0x034, 0x00021023,
+		0x034, 0x00020006,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3F5,
+		0x034, 0x000293D2,
+		0x034, 0x000283CE,
+		0x034, 0x00027290,
+		0x034, 0x0002628D,
+		0x034, 0x0002528A,
+		0x034, 0x00024287,
+		0x034, 0x0002308D,
+		0x034, 0x0002208A,
+		0x034, 0x00021087,
+		0x034, 0x00020048,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002AFF7,
+		0x034, 0x00029FF6,
+		0x034, 0x00028FF3,
+		0x034, 0x00027FF0,
+		0x034, 0x00026FED,
+		0x034, 0x00025FEA,
+		0x034, 0x00024FE7,
+		0x034, 0x00023DEA,
+		0x034, 0x00022DE7,
+		0x034, 0x00021DE4,
+		0x034, 0x00020D48,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3AD,
+		0x034, 0x0002938A,
+		0x034, 0x0002818C,
+		0x034, 0x00027189,
+		0x034, 0x0002606D,
+		0x034, 0x0002504C,
+		0x034, 0x0002402C,
+		0x034, 0x00023029,
+		0x034, 0x00022026,
+		0x034, 0x00021023,
+		0x034, 0x00020006,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0002A3AD,
+		0x034, 0x0002938A,
+		0x034, 0x0002818C,
+		0x034, 0x00027189,
+		0x034, 0x0002606D,
+		0x034, 0x0002504C,
+		0x034, 0x0002402C,
+		0x034, 0x00023029,
+		0x034, 0x00022026,
+		0x034, 0x00021023,
+		0x034, 0x00020006,
+	0xA0000000,	0x00000000,
+		0x034, 0x0002AFF4,
+		0x034, 0x00029FF1,
+		0x034, 0x00028FEE,
+		0x034, 0x00027FEB,
+		0x034, 0x00026FE8,
+		0x034, 0x00025DEA,
+		0x034, 0x00024CED,
+		0x034, 0x00023CEA,
+		0x034, 0x00022C6C,
+		0x034, 0x00021C69,
+		0x034, 0x00020C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EF,
+		0x034, 0x000093EC,
+		0x034, 0x0000838C,
+		0x034, 0x000071AD,
+		0x034, 0x000061AA,
+		0x034, 0x000050AD,
+		0x034, 0x000040AA,
+		0x034, 0x0000306A,
+		0x034, 0x0000202D,
+		0x034, 0x0000102A,
+		0x034, 0x00000027,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3F1,
+		0x034, 0x000092B1,
+		0x034, 0x000081CF,
+		0x034, 0x00007170,
+		0x034, 0x0000616D,
+		0x034, 0x0000516A,
+		0x034, 0x00004167,
+		0x034, 0x0000302F,
+		0x034, 0x0000202C,
+		0x034, 0x00001029,
+		0x034, 0x00000026,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000AFF7,
+		0x034, 0x00009FF6,
+		0x034, 0x00008FF3,
+		0x034, 0x00007FF0,
+		0x034, 0x00006FED,
+		0x034, 0x00005FEA,
+		0x034, 0x00004FE7,
+		0x034, 0x00003EC7,
+		0x034, 0x00002EC4,
+		0x034, 0x00001D4B,
+		0x034, 0x00000D48,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0000A3EE,
+		0x034, 0x000093AC,
+		0x034, 0x0000838A,
+		0x034, 0x0000718C,
+		0x034, 0x00006189,
+		0x034, 0x0000506D,
+		0x034, 0x0000406A,
+		0x034, 0x0000302C,
+		0x034, 0x00002029,
+		0x034, 0x00001026,
+		0x034, 0x00000023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0000AFF4,
+		0x034, 0x00009FF1,
+		0x034, 0x00008FEE,
+		0x034, 0x00007FEB,
+		0x034, 0x00006FE8,
+		0x034, 0x00005DEA,
+		0x034, 0x00004CED,
+		0x034, 0x00003CEA,
+		0x034, 0x00002C6C,
+		0x034, 0x00001C69,
+		0x034, 0x00000C2B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3EB,
+		0x034, 0x000C938B,
+		0x034, 0x000C81AC,
+		0x034, 0x000C71A9,
+		0x034, 0x000C60AC,
+		0x034, 0x000C50A9,
+		0x034, 0x000C402E,
+		0x034, 0x000C302B,
+		0x034, 0x000C2028,
+		0x034, 0x000C100B,
+		0x034, 0x000C0008,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3AD,
+		0x034, 0x000C938A,
+		0x034, 0x000C818C,
+		0x034, 0x000C7189,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3AD,
+		0x034, 0x000C938A,
+		0x034, 0x000C818C,
+		0x034, 0x000C7189,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3AD,
+		0x034, 0x000C938A,
+		0x034, 0x000C818C,
+		0x034, 0x000C7189,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3AD,
+		0x034, 0x000C938A,
+		0x034, 0x000C818C,
+		0x034, 0x000C7189,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3F4,
+		0x034, 0x000C93D2,
+		0x034, 0x000C82D1,
+		0x034, 0x000C71F1,
+		0x034, 0x000C61EE,
+		0x034, 0x000C51EB,
+		0x034, 0x000C41E8,
+		0x034, 0x000C314B,
+		0x034, 0x000C2148,
+		0x034, 0x000C104B,
+		0x034, 0x000C0048,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CAFF7,
+		0x034, 0x000C9FF6,
+		0x034, 0x000C8FF3,
+		0x034, 0x000C7FF0,
+		0x034, 0x000C6FED,
+		0x034, 0x000C5FEA,
+		0x034, 0x000C4FE7,
+		0x034, 0x000C3CB1,
+		0x034, 0x000C2CAE,
+		0x034, 0x000C1CAB,
+		0x034, 0x000C0CA8,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3AD,
+		0x034, 0x000C938A,
+		0x034, 0x000C818C,
+		0x034, 0x000C7189,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000CA3AD,
+		0x034, 0x000C938A,
+		0x034, 0x000C818C,
+		0x034, 0x000C7189,
+		0x034, 0x000C606D,
+		0x034, 0x000C506A,
+		0x034, 0x000C402C,
+		0x034, 0x000C3029,
+		0x034, 0x000C2026,
+		0x034, 0x000C1009,
+		0x034, 0x000C0006,
+	0xA0000000,	0x00000000,
+		0x034, 0x000CA794,
+		0x034, 0x000C9791,
+		0x034, 0x000C878E,
+		0x034, 0x000C778B,
+		0x034, 0x000C658D,
+		0x034, 0x000C558A,
+		0x034, 0x000C448D,
+		0x034, 0x000C348A,
+		0x034, 0x000C244C,
+		0x034, 0x000C1449,
+		0x034, 0x000C042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3EE,
+		0x034, 0x000A93EB,
+		0x034, 0x000A838B,
+		0x034, 0x000A71AC,
+		0x034, 0x000A61A9,
+		0x034, 0x000A50AC,
+		0x034, 0x000A40A9,
+		0x034, 0x000A30A6,
+		0x034, 0x000A202C,
+		0x034, 0x000A1029,
+		0x034, 0x000A0026,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3AD,
+		0x034, 0x000A938A,
+		0x034, 0x000A818C,
+		0x034, 0x000A7189,
+		0x034, 0x000A606D,
+		0x034, 0x000A504C,
+		0x034, 0x000A402C,
+		0x034, 0x000A3029,
+		0x034, 0x000A2026,
+		0x034, 0x000A1023,
+		0x034, 0x000A0006,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3AD,
+		0x034, 0x000A938A,
+		0x034, 0x000A818C,
+		0x034, 0x000A7189,
+		0x034, 0x000A606D,
+		0x034, 0x000A504C,
+		0x034, 0x000A402C,
+		0x034, 0x000A3029,
+		0x034, 0x000A2026,
+		0x034, 0x000A1023,
+		0x034, 0x000A0006,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3AD,
+		0x034, 0x000A938A,
+		0x034, 0x000A818C,
+		0x034, 0x000A7189,
+		0x034, 0x000A606D,
+		0x034, 0x000A504C,
+		0x034, 0x000A402C,
+		0x034, 0x000A3029,
+		0x034, 0x000A2026,
+		0x034, 0x000A1023,
+		0x034, 0x000A0006,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3AD,
+		0x034, 0x000A938A,
+		0x034, 0x000A818C,
+		0x034, 0x000A7189,
+		0x034, 0x000A606D,
+		0x034, 0x000A504C,
+		0x034, 0x000A402C,
+		0x034, 0x000A3029,
+		0x034, 0x000A2026,
+		0x034, 0x000A1023,
+		0x034, 0x000A0006,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3F5,
+		0x034, 0x000A93D2,
+		0x034, 0x000A83CE,
+		0x034, 0x000A7290,
+		0x034, 0x000A628D,
+		0x034, 0x000A528A,
+		0x034, 0x000A4287,
+		0x034, 0x000A308D,
+		0x034, 0x000A208A,
+		0x034, 0x000A1087,
+		0x034, 0x000A0048,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AAFF7,
+		0x034, 0x000A9FF6,
+		0x034, 0x000A8FF3,
+		0x034, 0x000A7FF0,
+		0x034, 0x000A6FED,
+		0x034, 0x000A5FEA,
+		0x034, 0x000A4FE7,
+		0x034, 0x000A3DEA,
+		0x034, 0x000A2DE7,
+		0x034, 0x000A1DE4,
+		0x034, 0x000A0D48,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3AD,
+		0x034, 0x000A938A,
+		0x034, 0x000A818C,
+		0x034, 0x000A7189,
+		0x034, 0x000A606D,
+		0x034, 0x000A504C,
+		0x034, 0x000A402C,
+		0x034, 0x000A3029,
+		0x034, 0x000A2026,
+		0x034, 0x000A1023,
+		0x034, 0x000A0006,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x000AA3AD,
+		0x034, 0x000A938A,
+		0x034, 0x000A818C,
+		0x034, 0x000A7189,
+		0x034, 0x000A606D,
+		0x034, 0x000A504C,
+		0x034, 0x000A402C,
+		0x034, 0x000A3029,
+		0x034, 0x000A2026,
+		0x034, 0x000A1023,
+		0x034, 0x000A0006,
+	0xA0000000,	0x00000000,
+		0x034, 0x000AA794,
+		0x034, 0x000A9791,
+		0x034, 0x000A878E,
+		0x034, 0x000A778B,
+		0x034, 0x000A658D,
+		0x034, 0x000A558A,
+		0x034, 0x000A448D,
+		0x034, 0x000A348A,
+		0x034, 0x000A244C,
+		0x034, 0x000A1449,
+		0x034, 0x000A042B,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EF,
+		0x034, 0x000893EC,
+		0x034, 0x0008838C,
+		0x034, 0x000871AD,
+		0x034, 0x000861AA,
+		0x034, 0x000850AD,
+		0x034, 0x000840AA,
+		0x034, 0x0008306A,
+		0x034, 0x0008202D,
+		0x034, 0x0008102A,
+		0x034, 0x00080027,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3F1,
+		0x034, 0x000892B1,
+		0x034, 0x000881CF,
+		0x034, 0x00087170,
+		0x034, 0x0008616D,
+		0x034, 0x0008516A,
+		0x034, 0x00084167,
+		0x034, 0x0008302F,
+		0x034, 0x0008202C,
+		0x034, 0x00081029,
+		0x034, 0x00080026,
+	0x90000009,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008AFF7,
+		0x034, 0x00089FF6,
+		0x034, 0x00088FF3,
+		0x034, 0x00087FF0,
+		0x034, 0x00086FED,
+		0x034, 0x00085FEA,
+		0x034, 0x00084FE7,
+		0x034, 0x00083EC7,
+		0x034, 0x00082EC4,
+		0x034, 0x00081D4B,
+		0x034, 0x00080D48,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x034, 0x0008A3EE,
+		0x034, 0x000893AC,
+		0x034, 0x0008838A,
+		0x034, 0x0008718C,
+		0x034, 0x00086189,
+		0x034, 0x0008506D,
+		0x034, 0x0008406A,
+		0x034, 0x0008302C,
+		0x034, 0x00082029,
+		0x034, 0x00081026,
+		0x034, 0x00080023,
+	0xA0000000,	0x00000000,
+		0x034, 0x0008A794,
+		0x034, 0x00089791,
+		0x034, 0x0008878E,
+		0x034, 0x0008778B,
+		0x034, 0x0008658D,
+		0x034, 0x0008558A,
+		0x034, 0x0008448D,
+		0x034, 0x0008348A,
+		0x034, 0x0008244C,
+		0x034, 0x00081449,
+		0x034, 0x0008042B,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x0DF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000040,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x035, 0x000006CC,
+		0x035, 0x000086CC,
+		0x035, 0x000106CC,
+		0x035, 0x000206CC,
+		0x035, 0x000286CC,
+		0x035, 0x000306CC,
+		0x035, 0x000406CC,
+		0x035, 0x000486CC,
+		0x035, 0x000506CC,
+		0x035, 0x000806CC,
+		0x035, 0x000886CC,
+		0x035, 0x000906CC,
+		0x035, 0x000A06CC,
+		0x035, 0x000A86CC,
+		0x035, 0x000B06CC,
+		0x035, 0x000C06CC,
+		0x035, 0x000C86CC,
+		0x035, 0x000D06CC,
+	0xA0000000,	0x00000000,
+		0x035, 0x00000484,
+		0x035, 0x00008484,
+		0x035, 0x00010484,
+		0x035, 0x00020584,
+		0x035, 0x00028584,
+		0x035, 0x00030584,
+		0x035, 0x00040584,
+		0x035, 0x00048584,
+		0x035, 0x00050584,
+		0x035, 0x000805FB,
+		0x035, 0x000885FB,
+		0x035, 0x000905FB,
+		0x035, 0x000A05FB,
+		0x035, 0x000A85FB,
+		0x035, 0x000B05FB,
+		0x035, 0x000C05FB,
+		0x035, 0x000C85FB,
+		0x035, 0x000D05FB,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x0DF, 0x00000001,
+	0xA0000000,	0x00000000,
+		0x0DF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x018, 0x0001712A,
+		0x0EF, 0x00000010,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000473,
+		0x036, 0x00008473,
+		0x036, 0x00010473,
+		0x036, 0x00020473,
+		0x036, 0x00028473,
+		0x036, 0x00030473,
+		0x036, 0x00040473,
+		0x036, 0x00048473,
+		0x036, 0x00050473,
+		0x036, 0x00080473,
+		0x036, 0x00088473,
+		0x036, 0x00090473,
+		0x036, 0x000A0473,
+		0x036, 0x000A8473,
+		0x036, 0x000B0473,
+		0x036, 0x000C0473,
+		0x036, 0x000C8473,
+		0x036, 0x000D0473,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x036, 0x00000475,
+		0x036, 0x00008475,
+		0x036, 0x00010475,
+		0x036, 0x00020475,
+		0x036, 0x00028475,
+		0x036, 0x00030475,
+		0x036, 0x00040475,
+		0x036, 0x00048475,
+		0x036, 0x00050475,
+		0x036, 0x00080475,
+		0x036, 0x00088475,
+		0x036, 0x00090475,
+		0x036, 0x000A0475,
+		0x036, 0x000A8475,
+		0x036, 0x000B0475,
+		0x036, 0x000C0475,
+		0x036, 0x000C8475,
+		0x036, 0x000D0475,
+	0xA0000000,	0x00000000,
+		0x036, 0x00000474,
+		0x036, 0x00008474,
+		0x036, 0x00010474,
+		0x036, 0x00020474,
+		0x036, 0x00028474,
+		0x036, 0x00030474,
+		0x036, 0x00040474,
+		0x036, 0x00048474,
+		0x036, 0x00050474,
+		0x036, 0x00080474,
+		0x036, 0x00088474,
+		0x036, 0x00090474,
+		0x036, 0x000A0474,
+		0x036, 0x000A8474,
+		0x036, 0x000B0474,
+		0x036, 0x000C0474,
+		0x036, 0x000C8474,
+		0x036, 0x000D0474,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x0EF, 0x00000004,
+		0x037, 0x00000000,
+		0x038, 0x0000514E,
+		0x037, 0x00004000,
+		0x038, 0x0000514E,
+		0x037, 0x00008000,
+		0x038, 0x0000514E,
+		0x037, 0x00010000,
+		0x038, 0x0000514E,
+		0x037, 0x00014000,
+		0x038, 0x0000514E,
+		0x037, 0x00018000,
+		0x038, 0x0000514E,
+		0x037, 0x0001C000,
+		0x038, 0x0000514E,
+		0x037, 0x00020000,
+		0x038, 0x0000514E,
+		0x037, 0x00024000,
+		0x038, 0x0000514E,
+		0x037, 0x00028000,
+		0x038, 0x0000514E,
+		0x037, 0x0002C000,
+		0x038, 0x0000714E,
+		0x037, 0x00030000,
+		0x038, 0x0000514E,
+		0x037, 0x00034000,
+		0x038, 0x0000514E,
+		0x037, 0x00038000,
+		0x038, 0x0000514E,
+		0x037, 0x0003C000,
+		0x038, 0x0000514E,
+		0x037, 0x00040000,
+		0x038, 0x0000514E,
+		0x037, 0x00044000,
+		0x038, 0x0000514E,
+		0x037, 0x00048000,
+		0x038, 0x0000514E,
+		0x037, 0x00080000,
+		0x038, 0x00005ECE,
+		0x037, 0x00084000,
+		0x038, 0x00005ECE,
+		0x037, 0x00088000,
+		0x038, 0x00005ECE,
+		0x037, 0x00090000,
+		0x038, 0x00005ECE,
+		0x037, 0x00094000,
+		0x038, 0x00005ECE,
+		0x037, 0x00098000,
+		0x038, 0x00005ECE,
+		0x037, 0x0009C000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000A8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000AC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000B8000,
+		0x038, 0x00005ECE,
+		0x037, 0x000BC000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C0000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C4000,
+		0x038, 0x00005ECE,
+		0x037, 0x000C8000,
+		0x038, 0x00005ECE,
+		0x0EF, 0x00000000,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000008,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000007D,
+		0x03C, 0x0000047D,
+		0x03C, 0x0000087D,
+		0x03C, 0x0000107D,
+		0x03C, 0x0000147D,
+		0x03C, 0x0000187D,
+	0xB0000000,	0x00000000,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x00000275,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x00001275,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x00002275,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027F,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127F,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227F,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027F,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127F,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227F,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027F,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127F,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227F,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027F,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127F,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227F,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027F,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127F,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227F,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027F,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127F,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227F,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x03C, 0x0000027F,
+		0x03C, 0x00000542,
+		0x03C, 0x00000821,
+		0x03C, 0x0000127F,
+		0x03C, 0x00001542,
+		0x03C, 0x00001821,
+		0x03C, 0x0000227F,
+		0x03C, 0x00002542,
+		0x03C, 0x00002821,
+	0xA0000000,	0x00000000,
+		0x03C, 0x0000037E,
+		0x03C, 0x00000575,
+		0x03C, 0x00000971,
+		0x03C, 0x0000127E,
+		0x03C, 0x00001575,
+		0x03C, 0x00001871,
+		0x03C, 0x0000217E,
+		0x03C, 0x00002575,
+		0x03C, 0x00002871,
+	0xB0000000,	0x00000000,
+		0x0EF, 0x00000000,
+		0x061, 0x000C0D47,
+		0x062, 0x0000133C,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x063, 0x000750E7,
+	0xA0000000,	0x00000000,
+		0x063, 0x0007D0E7,
+	0xB0000000,	0x00000000,
+		0x064, 0x00014FEC,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+		0x065, 0x000920D0,
+	0xA0000000,	0x00000000,
+		0x065, 0x000923FF,
+	0xB0000000,	0x00000000,
+		0x066, 0x00000040,
+		0x057, 0x00050000,
+		0x056, 0x00051DF0,
+	0x80000001,	0x00000000,	0x40000000,	0x00000000,
+	0x90000002,	0x00000000,	0x40000000,	0x00000000,
+	0x90000003,	0x00000000,	0x40000000,	0x00000000,
+	0x90000004,	0x00000000,	0x40000000,	0x00000000,
+	0x90000005,	0x00000000,	0x40000000,	0x00000000,
+	0x90000008,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000a,	0x00000000,	0x40000000,	0x00000000,
+	0x9000000b,	0x00000000,	0x40000000,	0x00000000,
+	0xA0000000,	0x00000000,
+		0x055, 0x00082060,
+	0xB0000000,	0x00000000,
+};
+
+RTW_DECL_TABLE_RF_RADIO(rtw8814a_rf_d, D);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt[] = {
+	{ 0, 0, 0, 0, 1, 36, },
+	{ 2, 0, 0, 0, 1, 32, },
+	{ 1, 0, 0, 0, 1, 32, },
+	{ 0, 0, 0, 0, 2, 36, },
+	{ 2, 0, 0, 0, 2, 32, },
+	{ 1, 0, 0, 0, 2, 32, },
+	{ 0, 0, 0, 0, 3, 36, },
+	{ 2, 0, 0, 0, 3, 32, },
+	{ 1, 0, 0, 0, 3, 32, },
+	{ 0, 0, 0, 0, 4, 36, },
+	{ 2, 0, 0, 0, 4, 32, },
+	{ 1, 0, 0, 0, 4, 32, },
+	{ 0, 0, 0, 0, 5, 36, },
+	{ 2, 0, 0, 0, 5, 32, },
+	{ 1, 0, 0, 0, 5, 32, },
+	{ 0, 0, 0, 0, 6, 36, },
+	{ 2, 0, 0, 0, 6, 32, },
+	{ 1, 0, 0, 0, 6, 32, },
+	{ 0, 0, 0, 0, 7, 36, },
+	{ 2, 0, 0, 0, 7, 32, },
+	{ 1, 0, 0, 0, 7, 32, },
+	{ 0, 0, 0, 0, 8, 36, },
+	{ 2, 0, 0, 0, 8, 32, },
+	{ 1, 0, 0, 0, 8, 32, },
+	{ 0, 0, 0, 0, 9, 36, },
+	{ 2, 0, 0, 0, 9, 32, },
+	{ 1, 0, 0, 0, 9, 32, },
+	{ 0, 0, 0, 0, 10, 36, },
+	{ 2, 0, 0, 0, 10, 32, },
+	{ 1, 0, 0, 0, 10, 32, },
+	{ 0, 0, 0, 0, 11, 36, },
+	{ 2, 0, 0, 0, 11, 32, },
+	{ 1, 0, 0, 0, 11, 32, },
+	{ 0, 0, 0, 0, 12, 63, },
+	{ 2, 0, 0, 0, 12, 32, },
+	{ 1, 0, 0, 0, 12, 32, },
+	{ 0, 0, 0, 0, 13, 63, },
+	{ 2, 0, 0, 0, 13, 32, },
+	{ 1, 0, 0, 0, 13, 32, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 32, },
+	{ 0, 0, 0, 1, 1, 34, },
+	{ 2, 0, 0, 1, 1, 32, },
+	{ 1, 0, 0, 1, 1, 32, },
+	{ 0, 0, 0, 1, 2, 36, },
+	{ 2, 0, 0, 1, 2, 32, },
+	{ 1, 0, 0, 1, 2, 32, },
+	{ 0, 0, 0, 1, 3, 36, },
+	{ 2, 0, 0, 1, 3, 32, },
+	{ 1, 0, 0, 1, 3, 32, },
+	{ 0, 0, 0, 1, 4, 36, },
+	{ 2, 0, 0, 1, 4, 32, },
+	{ 1, 0, 0, 1, 4, 32, },
+	{ 0, 0, 0, 1, 5, 36, },
+	{ 2, 0, 0, 1, 5, 32, },
+	{ 1, 0, 0, 1, 5, 32, },
+	{ 0, 0, 0, 1, 6, 36, },
+	{ 2, 0, 0, 1, 6, 32, },
+	{ 1, 0, 0, 1, 6, 32, },
+	{ 0, 0, 0, 1, 7, 36, },
+	{ 2, 0, 0, 1, 7, 32, },
+	{ 1, 0, 0, 1, 7, 32, },
+	{ 0, 0, 0, 1, 8, 36, },
+	{ 2, 0, 0, 1, 8, 32, },
+	{ 1, 0, 0, 1, 8, 32, },
+	{ 0, 0, 0, 1, 9, 36, },
+	{ 2, 0, 0, 1, 9, 32, },
+	{ 1, 0, 0, 1, 9, 32, },
+	{ 0, 0, 0, 1, 10, 36, },
+	{ 2, 0, 0, 1, 10, 32, },
+	{ 1, 0, 0, 1, 10, 32, },
+	{ 0, 0, 0, 1, 11, 32, },
+	{ 2, 0, 0, 1, 11, 32, },
+	{ 1, 0, 0, 1, 11, 32, },
+	{ 0, 0, 0, 1, 12, 63, },
+	{ 2, 0, 0, 1, 12, 32, },
+	{ 1, 0, 0, 1, 12, 32, },
+	{ 0, 0, 0, 1, 13, 63, },
+	{ 2, 0, 0, 1, 13, 32, },
+	{ 1, 0, 0, 1, 13, 32, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 63, },
+	{ 0, 0, 0, 2, 1, 34, },
+	{ 2, 0, 0, 2, 1, 32, },
+	{ 1, 0, 0, 2, 1, 32, },
+	{ 0, 0, 0, 2, 2, 36, },
+	{ 2, 0, 0, 2, 2, 32, },
+	{ 1, 0, 0, 2, 2, 32, },
+	{ 0, 0, 0, 2, 3, 36, },
+	{ 2, 0, 0, 2, 3, 32, },
+	{ 1, 0, 0, 2, 3, 32, },
+	{ 0, 0, 0, 2, 4, 36, },
+	{ 2, 0, 0, 2, 4, 32, },
+	{ 1, 0, 0, 2, 4, 32, },
+	{ 0, 0, 0, 2, 5, 36, },
+	{ 2, 0, 0, 2, 5, 32, },
+	{ 1, 0, 0, 2, 5, 32, },
+	{ 0, 0, 0, 2, 6, 36, },
+	{ 2, 0, 0, 2, 6, 32, },
+	{ 1, 0, 0, 2, 6, 32, },
+	{ 0, 0, 0, 2, 7, 36, },
+	{ 2, 0, 0, 2, 7, 32, },
+	{ 1, 0, 0, 2, 7, 32, },
+	{ 0, 0, 0, 2, 8, 36, },
+	{ 2, 0, 0, 2, 8, 32, },
+	{ 1, 0, 0, 2, 8, 32, },
+	{ 0, 0, 0, 2, 9, 36, },
+	{ 2, 0, 0, 2, 9, 32, },
+	{ 1, 0, 0, 2, 9, 32, },
+	{ 0, 0, 0, 2, 10, 36, },
+	{ 2, 0, 0, 2, 10, 32, },
+	{ 1, 0, 0, 2, 10, 32, },
+	{ 0, 0, 0, 2, 11, 32, },
+	{ 2, 0, 0, 2, 11, 32, },
+	{ 1, 0, 0, 2, 11, 32, },
+	{ 0, 0, 0, 2, 12, 63, },
+	{ 2, 0, 0, 2, 12, 32, },
+	{ 1, 0, 0, 2, 12, 32, },
+	{ 0, 0, 0, 2, 13, 63, },
+	{ 2, 0, 0, 2, 13, 32, },
+	{ 1, 0, 0, 2, 13, 32, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 63, },
+	{ 0, 0, 0, 3, 1, 32, },
+	{ 2, 0, 0, 3, 1, 30, },
+	{ 1, 0, 0, 3, 1, 30, },
+	{ 0, 0, 0, 3, 2, 34, },
+	{ 2, 0, 0, 3, 2, 30, },
+	{ 1, 0, 0, 3, 2, 30, },
+	{ 0, 0, 0, 3, 3, 34, },
+	{ 2, 0, 0, 3, 3, 30, },
+	{ 1, 0, 0, 3, 3, 30, },
+	{ 0, 0, 0, 3, 4, 34, },
+	{ 2, 0, 0, 3, 4, 30, },
+	{ 1, 0, 0, 3, 4, 30, },
+	{ 0, 0, 0, 3, 5, 34, },
+	{ 2, 0, 0, 3, 5, 30, },
+	{ 1, 0, 0, 3, 5, 30, },
+	{ 0, 0, 0, 3, 6, 34, },
+	{ 2, 0, 0, 3, 6, 30, },
+	{ 1, 0, 0, 3, 6, 30, },
+	{ 0, 0, 0, 3, 7, 34, },
+	{ 2, 0, 0, 3, 7, 30, },
+	{ 1, 0, 0, 3, 7, 30, },
+	{ 0, 0, 0, 3, 8, 34, },
+	{ 2, 0, 0, 3, 8, 30, },
+	{ 1, 0, 0, 3, 8, 30, },
+	{ 0, 0, 0, 3, 9, 34, },
+	{ 2, 0, 0, 3, 9, 30, },
+	{ 1, 0, 0, 3, 9, 30, },
+	{ 0, 0, 0, 3, 10, 34, },
+	{ 2, 0, 0, 3, 10, 30, },
+	{ 1, 0, 0, 3, 10, 30, },
+	{ 0, 0, 0, 3, 11, 30, },
+	{ 2, 0, 0, 3, 11, 30, },
+	{ 1, 0, 0, 3, 11, 30, },
+	{ 0, 0, 0, 3, 12, 63, },
+	{ 2, 0, 0, 3, 12, 30, },
+	{ 1, 0, 0, 3, 12, 30, },
+	{ 0, 0, 0, 3, 13, 63, },
+	{ 2, 0, 0, 3, 13, 30, },
+	{ 1, 0, 0, 3, 13, 30, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 63, },
+	{ 0, 0, 0, 6, 1, 30, },
+	{ 2, 0, 0, 6, 1, 28, },
+	{ 1, 0, 0, 6, 1, 28, },
+	{ 0, 0, 0, 6, 2, 32, },
+	{ 2, 0, 0, 6, 2, 28, },
+	{ 1, 0, 0, 6, 2, 28, },
+	{ 0, 0, 0, 6, 3, 32, },
+	{ 2, 0, 0, 6, 3, 28, },
+	{ 1, 0, 0, 6, 3, 28, },
+	{ 0, 0, 0, 6, 4, 32, },
+	{ 2, 0, 0, 6, 4, 28, },
+	{ 1, 0, 0, 6, 4, 28, },
+	{ 0, 0, 0, 6, 5, 32, },
+	{ 2, 0, 0, 6, 5, 28, },
+	{ 1, 0, 0, 6, 5, 28, },
+	{ 0, 0, 0, 6, 6, 32, },
+	{ 2, 0, 0, 6, 6, 28, },
+	{ 1, 0, 0, 6, 6, 28, },
+	{ 0, 0, 0, 6, 7, 32, },
+	{ 2, 0, 0, 6, 7, 28, },
+	{ 1, 0, 0, 6, 7, 28, },
+	{ 0, 0, 0, 6, 8, 32, },
+	{ 2, 0, 0, 6, 8, 28, },
+	{ 1, 0, 0, 6, 8, 28, },
+	{ 0, 0, 0, 6, 9, 32, },
+	{ 2, 0, 0, 6, 9, 28, },
+	{ 1, 0, 0, 6, 9, 28, },
+	{ 0, 0, 0, 6, 10, 32, },
+	{ 2, 0, 0, 6, 10, 28, },
+	{ 1, 0, 0, 6, 10, 28, },
+	{ 0, 0, 0, 6, 11, 28, },
+	{ 2, 0, 0, 6, 11, 28, },
+	{ 1, 0, 0, 6, 11, 28, },
+	{ 0, 0, 0, 6, 12, 63, },
+	{ 2, 0, 0, 6, 12, 28, },
+	{ 1, 0, 0, 6, 12, 28, },
+	{ 0, 0, 0, 6, 13, 63, },
+	{ 2, 0, 0, 6, 13, 28, },
+	{ 1, 0, 0, 6, 13, 28, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 63, },
+	{ 0, 0, 0, 7, 1, 28, },
+	{ 2, 0, 0, 7, 1, 26, },
+	{ 1, 0, 0, 7, 1, 26, },
+	{ 0, 0, 0, 7, 2, 30, },
+	{ 2, 0, 0, 7, 2, 26, },
+	{ 1, 0, 0, 7, 2, 26, },
+	{ 0, 0, 0, 7, 3, 30, },
+	{ 2, 0, 0, 7, 3, 26, },
+	{ 1, 0, 0, 7, 3, 26, },
+	{ 0, 0, 0, 7, 4, 30, },
+	{ 2, 0, 0, 7, 4, 26, },
+	{ 1, 0, 0, 7, 4, 26, },
+	{ 0, 0, 0, 7, 5, 30, },
+	{ 2, 0, 0, 7, 5, 26, },
+	{ 1, 0, 0, 7, 5, 26, },
+	{ 0, 0, 0, 7, 6, 30, },
+	{ 2, 0, 0, 7, 6, 26, },
+	{ 1, 0, 0, 7, 6, 26, },
+	{ 0, 0, 0, 7, 7, 30, },
+	{ 2, 0, 0, 7, 7, 26, },
+	{ 1, 0, 0, 7, 7, 26, },
+	{ 0, 0, 0, 7, 8, 30, },
+	{ 2, 0, 0, 7, 8, 26, },
+	{ 1, 0, 0, 7, 8, 26, },
+	{ 0, 0, 0, 7, 9, 30, },
+	{ 2, 0, 0, 7, 9, 26, },
+	{ 1, 0, 0, 7, 9, 26, },
+	{ 0, 0, 0, 7, 10, 30, },
+	{ 2, 0, 0, 7, 10, 26, },
+	{ 1, 0, 0, 7, 10, 26, },
+	{ 0, 0, 0, 7, 11, 26, },
+	{ 2, 0, 0, 7, 11, 26, },
+	{ 1, 0, 0, 7, 11, 26, },
+	{ 0, 0, 0, 7, 12, 63, },
+	{ 2, 0, 0, 7, 12, 26, },
+	{ 1, 0, 0, 7, 12, 26, },
+	{ 0, 0, 0, 7, 13, 63, },
+	{ 2, 0, 0, 7, 13, 26, },
+	{ 1, 0, 0, 7, 13, 26, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 63, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 32, },
+	{ 2, 0, 1, 2, 3, 32, },
+	{ 1, 0, 1, 2, 3, 32, },
+	{ 0, 0, 1, 2, 4, 36, },
+	{ 2, 0, 1, 2, 4, 32, },
+	{ 1, 0, 1, 2, 4, 32, },
+	{ 0, 0, 1, 2, 5, 36, },
+	{ 2, 0, 1, 2, 5, 32, },
+	{ 1, 0, 1, 2, 5, 32, },
+	{ 0, 0, 1, 2, 6, 36, },
+	{ 2, 0, 1, 2, 6, 32, },
+	{ 1, 0, 1, 2, 6, 32, },
+	{ 0, 0, 1, 2, 7, 36, },
+	{ 2, 0, 1, 2, 7, 32, },
+	{ 1, 0, 1, 2, 7, 32, },
+	{ 0, 0, 1, 2, 8, 36, },
+	{ 2, 0, 1, 2, 8, 32, },
+	{ 1, 0, 1, 2, 8, 32, },
+	{ 0, 0, 1, 2, 9, 36, },
+	{ 2, 0, 1, 2, 9, 32, },
+	{ 1, 0, 1, 2, 9, 32, },
+	{ 0, 0, 1, 2, 10, 36, },
+	{ 2, 0, 1, 2, 10, 32, },
+	{ 1, 0, 1, 2, 10, 32, },
+	{ 0, 0, 1, 2, 11, 32, },
+	{ 2, 0, 1, 2, 11, 32, },
+	{ 1, 0, 1, 2, 11, 32, },
+	{ 0, 0, 1, 2, 12, 63, },
+	{ 2, 0, 1, 2, 12, 32, },
+	{ 1, 0, 1, 2, 12, 32, },
+	{ 0, 0, 1, 2, 13, 63, },
+	{ 2, 0, 1, 2, 13, 32, },
+	{ 1, 0, 1, 2, 13, 32, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 30, },
+	{ 2, 0, 1, 3, 3, 30, },
+	{ 1, 0, 1, 3, 3, 30, },
+	{ 0, 0, 1, 3, 4, 34, },
+	{ 2, 0, 1, 3, 4, 30, },
+	{ 1, 0, 1, 3, 4, 30, },
+	{ 0, 0, 1, 3, 5, 34, },
+	{ 2, 0, 1, 3, 5, 30, },
+	{ 1, 0, 1, 3, 5, 30, },
+	{ 0, 0, 1, 3, 6, 34, },
+	{ 2, 0, 1, 3, 6, 30, },
+	{ 1, 0, 1, 3, 6, 30, },
+	{ 0, 0, 1, 3, 7, 34, },
+	{ 2, 0, 1, 3, 7, 30, },
+	{ 1, 0, 1, 3, 7, 30, },
+	{ 0, 0, 1, 3, 8, 34, },
+	{ 2, 0, 1, 3, 8, 30, },
+	{ 1, 0, 1, 3, 8, 30, },
+	{ 0, 0, 1, 3, 9, 34, },
+	{ 2, 0, 1, 3, 9, 30, },
+	{ 1, 0, 1, 3, 9, 30, },
+	{ 0, 0, 1, 3, 10, 34, },
+	{ 2, 0, 1, 3, 10, 30, },
+	{ 1, 0, 1, 3, 10, 30, },
+	{ 0, 0, 1, 3, 11, 30, },
+	{ 2, 0, 1, 3, 11, 30, },
+	{ 1, 0, 1, 3, 11, 30, },
+	{ 0, 0, 1, 3, 12, 63, },
+	{ 2, 0, 1, 3, 12, 30, },
+	{ 1, 0, 1, 3, 12, 30, },
+	{ 0, 0, 1, 3, 13, 63, },
+	{ 2, 0, 1, 3, 13, 30, },
+	{ 1, 0, 1, 3, 13, 30, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 28, },
+	{ 2, 0, 1, 6, 3, 28, },
+	{ 1, 0, 1, 6, 3, 28, },
+	{ 0, 0, 1, 6, 4, 32, },
+	{ 2, 0, 1, 6, 4, 28, },
+	{ 1, 0, 1, 6, 4, 28, },
+	{ 0, 0, 1, 6, 5, 32, },
+	{ 2, 0, 1, 6, 5, 28, },
+	{ 1, 0, 1, 6, 5, 28, },
+	{ 0, 0, 1, 6, 6, 32, },
+	{ 2, 0, 1, 6, 6, 28, },
+	{ 1, 0, 1, 6, 6, 28, },
+	{ 0, 0, 1, 6, 7, 32, },
+	{ 2, 0, 1, 6, 7, 28, },
+	{ 1, 0, 1, 6, 7, 28, },
+	{ 0, 0, 1, 6, 8, 32, },
+	{ 2, 0, 1, 6, 8, 28, },
+	{ 1, 0, 1, 6, 8, 28, },
+	{ 0, 0, 1, 6, 9, 32, },
+	{ 2, 0, 1, 6, 9, 28, },
+	{ 1, 0, 1, 6, 9, 28, },
+	{ 0, 0, 1, 6, 10, 32, },
+	{ 2, 0, 1, 6, 10, 28, },
+	{ 1, 0, 1, 6, 10, 28, },
+	{ 0, 0, 1, 6, 11, 28, },
+	{ 2, 0, 1, 6, 11, 28, },
+	{ 1, 0, 1, 6, 11, 28, },
+	{ 0, 0, 1, 6, 12, 63, },
+	{ 2, 0, 1, 6, 12, 28, },
+	{ 1, 0, 1, 6, 12, 28, },
+	{ 0, 0, 1, 6, 13, 63, },
+	{ 2, 0, 1, 6, 13, 28, },
+	{ 1, 0, 1, 6, 13, 28, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 26, },
+	{ 2, 0, 1, 7, 3, 26, },
+	{ 1, 0, 1, 7, 3, 26, },
+	{ 0, 0, 1, 7, 4, 30, },
+	{ 2, 0, 1, 7, 4, 26, },
+	{ 1, 0, 1, 7, 4, 26, },
+	{ 0, 0, 1, 7, 5, 30, },
+	{ 2, 0, 1, 7, 5, 26, },
+	{ 1, 0, 1, 7, 5, 26, },
+	{ 0, 0, 1, 7, 6, 30, },
+	{ 2, 0, 1, 7, 6, 26, },
+	{ 1, 0, 1, 7, 6, 26, },
+	{ 0, 0, 1, 7, 7, 30, },
+	{ 2, 0, 1, 7, 7, 26, },
+	{ 1, 0, 1, 7, 7, 26, },
+	{ 0, 0, 1, 7, 8, 30, },
+	{ 2, 0, 1, 7, 8, 26, },
+	{ 1, 0, 1, 7, 8, 26, },
+	{ 0, 0, 1, 7, 9, 30, },
+	{ 2, 0, 1, 7, 9, 26, },
+	{ 1, 0, 1, 7, 9, 26, },
+	{ 0, 0, 1, 7, 10, 30, },
+	{ 2, 0, 1, 7, 10, 26, },
+	{ 1, 0, 1, 7, 10, 26, },
+	{ 0, 0, 1, 7, 11, 26, },
+	{ 2, 0, 1, 7, 11, 26, },
+	{ 1, 0, 1, 7, 11, 26, },
+	{ 0, 0, 1, 7, 12, 63, },
+	{ 2, 0, 1, 7, 12, 26, },
+	{ 1, 0, 1, 7, 12, 26, },
+	{ 0, 0, 1, 7, 13, 63, },
+	{ 2, 0, 1, 7, 13, 26, },
+	{ 1, 0, 1, 7, 13, 26, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 30, },
+	{ 2, 1, 0, 1, 36, 32, },
+	{ 1, 1, 0, 1, 36, 32, },
+	{ 0, 1, 0, 1, 40, 30, },
+	{ 2, 1, 0, 1, 40, 32, },
+	{ 1, 1, 0, 1, 40, 32, },
+	{ 0, 1, 0, 1, 44, 30, },
+	{ 2, 1, 0, 1, 44, 32, },
+	{ 1, 1, 0, 1, 44, 32, },
+	{ 0, 1, 0, 1, 48, 30, },
+	{ 2, 1, 0, 1, 48, 32, },
+	{ 1, 1, 0, 1, 48, 32, },
+	{ 0, 1, 0, 1, 52, 36, },
+	{ 2, 1, 0, 1, 52, 32, },
+	{ 1, 1, 0, 1, 52, 32, },
+	{ 0, 1, 0, 1, 56, 34, },
+	{ 2, 1, 0, 1, 56, 32, },
+	{ 1, 1, 0, 1, 56, 32, },
+	{ 0, 1, 0, 1, 60, 32, },
+	{ 2, 1, 0, 1, 60, 32, },
+	{ 1, 1, 0, 1, 60, 32, },
+	{ 0, 1, 0, 1, 64, 28, },
+	{ 2, 1, 0, 1, 64, 32, },
+	{ 1, 1, 0, 1, 64, 32, },
+	{ 0, 1, 0, 1, 100, 30, },
+	{ 2, 1, 0, 1, 100, 32, },
+	{ 1, 1, 0, 1, 100, 32, },
+	{ 0, 1, 0, 1, 104, 30, },
+	{ 2, 1, 0, 1, 104, 32, },
+	{ 1, 1, 0, 1, 104, 32, },
+	{ 0, 1, 0, 1, 108, 32, },
+	{ 2, 1, 0, 1, 108, 32, },
+	{ 1, 1, 0, 1, 108, 32, },
+	{ 0, 1, 0, 1, 112, 34, },
+	{ 2, 1, 0, 1, 112, 32, },
+	{ 1, 1, 0, 1, 112, 32, },
+	{ 0, 1, 0, 1, 116, 34, },
+	{ 2, 1, 0, 1, 116, 32, },
+	{ 1, 1, 0, 1, 116, 32, },
+	{ 0, 1, 0, 1, 120, 36, },
+	{ 2, 1, 0, 1, 120, 32, },
+	{ 1, 1, 0, 1, 120, 32, },
+	{ 0, 1, 0, 1, 124, 34, },
+	{ 2, 1, 0, 1, 124, 32, },
+	{ 1, 1, 0, 1, 124, 32, },
+	{ 0, 1, 0, 1, 128, 32, },
+	{ 2, 1, 0, 1, 128, 32, },
+	{ 1, 1, 0, 1, 128, 32, },
+	{ 0, 1, 0, 1, 132, 30, },
+	{ 2, 1, 0, 1, 132, 32, },
+	{ 1, 1, 0, 1, 132, 32, },
+	{ 0, 1, 0, 1, 136, 30, },
+	{ 2, 1, 0, 1, 136, 32, },
+	{ 1, 1, 0, 1, 136, 32, },
+	{ 0, 1, 0, 1, 140, 28, },
+	{ 2, 1, 0, 1, 140, 32, },
+	{ 1, 1, 0, 1, 140, 32, },
+	{ 0, 1, 0, 1, 149, 36, },
+	{ 2, 1, 0, 1, 149, 32, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 36, },
+	{ 2, 1, 0, 1, 153, 32, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 36, },
+	{ 2, 1, 0, 1, 157, 32, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 36, },
+	{ 2, 1, 0, 1, 161, 32, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 36, },
+	{ 2, 1, 0, 1, 165, 32, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 30, },
+	{ 2, 1, 0, 2, 36, 32, },
+	{ 1, 1, 0, 2, 36, 32, },
+	{ 0, 1, 0, 2, 40, 30, },
+	{ 2, 1, 0, 2, 40, 32, },
+	{ 1, 1, 0, 2, 40, 32, },
+	{ 0, 1, 0, 2, 44, 30, },
+	{ 2, 1, 0, 2, 44, 32, },
+	{ 1, 1, 0, 2, 44, 32, },
+	{ 0, 1, 0, 2, 48, 30, },
+	{ 2, 1, 0, 2, 48, 32, },
+	{ 1, 1, 0, 2, 48, 32, },
+	{ 0, 1, 0, 2, 52, 36, },
+	{ 2, 1, 0, 2, 52, 32, },
+	{ 1, 1, 0, 2, 52, 32, },
+	{ 0, 1, 0, 2, 56, 34, },
+	{ 2, 1, 0, 2, 56, 32, },
+	{ 1, 1, 0, 2, 56, 32, },
+	{ 0, 1, 0, 2, 60, 32, },
+	{ 2, 1, 0, 2, 60, 32, },
+	{ 1, 1, 0, 2, 60, 32, },
+	{ 0, 1, 0, 2, 64, 28, },
+	{ 2, 1, 0, 2, 64, 32, },
+	{ 1, 1, 0, 2, 64, 32, },
+	{ 0, 1, 0, 2, 100, 30, },
+	{ 2, 1, 0, 2, 100, 32, },
+	{ 1, 1, 0, 2, 100, 32, },
+	{ 0, 1, 0, 2, 104, 30, },
+	{ 2, 1, 0, 2, 104, 32, },
+	{ 1, 1, 0, 2, 104, 32, },
+	{ 0, 1, 0, 2, 108, 32, },
+	{ 2, 1, 0, 2, 108, 32, },
+	{ 1, 1, 0, 2, 108, 32, },
+	{ 0, 1, 0, 2, 112, 34, },
+	{ 2, 1, 0, 2, 112, 32, },
+	{ 1, 1, 0, 2, 112, 32, },
+	{ 0, 1, 0, 2, 116, 34, },
+	{ 2, 1, 0, 2, 116, 32, },
+	{ 1, 1, 0, 2, 116, 32, },
+	{ 0, 1, 0, 2, 120, 36, },
+	{ 2, 1, 0, 2, 120, 32, },
+	{ 1, 1, 0, 2, 120, 32, },
+	{ 0, 1, 0, 2, 124, 34, },
+	{ 2, 1, 0, 2, 124, 32, },
+	{ 1, 1, 0, 2, 124, 32, },
+	{ 0, 1, 0, 2, 128, 32, },
+	{ 2, 1, 0, 2, 128, 32, },
+	{ 1, 1, 0, 2, 128, 32, },
+	{ 0, 1, 0, 2, 132, 30, },
+	{ 2, 1, 0, 2, 132, 32, },
+	{ 1, 1, 0, 2, 132, 32, },
+	{ 0, 1, 0, 2, 136, 30, },
+	{ 2, 1, 0, 2, 136, 32, },
+	{ 1, 1, 0, 2, 136, 32, },
+	{ 0, 1, 0, 2, 140, 28, },
+	{ 2, 1, 0, 2, 140, 32, },
+	{ 1, 1, 0, 2, 140, 32, },
+	{ 0, 1, 0, 2, 149, 36, },
+	{ 2, 1, 0, 2, 149, 32, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 36, },
+	{ 2, 1, 0, 2, 153, 32, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 36, },
+	{ 2, 1, 0, 2, 157, 32, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 36, },
+	{ 2, 1, 0, 2, 161, 32, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 36, },
+	{ 2, 1, 0, 2, 165, 32, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 28, },
+	{ 2, 1, 0, 3, 36, 30, },
+	{ 1, 1, 0, 3, 36, 30, },
+	{ 0, 1, 0, 3, 40, 28, },
+	{ 2, 1, 0, 3, 40, 30, },
+	{ 1, 1, 0, 3, 40, 30, },
+	{ 0, 1, 0, 3, 44, 28, },
+	{ 2, 1, 0, 3, 44, 30, },
+	{ 1, 1, 0, 3, 44, 30, },
+	{ 0, 1, 0, 3, 48, 28, },
+	{ 2, 1, 0, 3, 48, 30, },
+	{ 1, 1, 0, 3, 48, 30, },
+	{ 0, 1, 0, 3, 52, 34, },
+	{ 2, 1, 0, 3, 52, 30, },
+	{ 1, 1, 0, 3, 52, 30, },
+	{ 0, 1, 0, 3, 56, 32, },
+	{ 2, 1, 0, 3, 56, 30, },
+	{ 1, 1, 0, 3, 56, 30, },
+	{ 0, 1, 0, 3, 60, 30, },
+	{ 2, 1, 0, 3, 60, 30, },
+	{ 1, 1, 0, 3, 60, 30, },
+	{ 0, 1, 0, 3, 64, 26, },
+	{ 2, 1, 0, 3, 64, 30, },
+	{ 1, 1, 0, 3, 64, 30, },
+	{ 0, 1, 0, 3, 100, 28, },
+	{ 2, 1, 0, 3, 100, 30, },
+	{ 1, 1, 0, 3, 100, 30, },
+	{ 0, 1, 0, 3, 104, 28, },
+	{ 2, 1, 0, 3, 104, 30, },
+	{ 1, 1, 0, 3, 104, 30, },
+	{ 0, 1, 0, 3, 108, 30, },
+	{ 2, 1, 0, 3, 108, 30, },
+	{ 1, 1, 0, 3, 108, 30, },
+	{ 0, 1, 0, 3, 112, 32, },
+	{ 2, 1, 0, 3, 112, 30, },
+	{ 1, 1, 0, 3, 112, 30, },
+	{ 0, 1, 0, 3, 116, 32, },
+	{ 2, 1, 0, 3, 116, 30, },
+	{ 1, 1, 0, 3, 116, 30, },
+	{ 0, 1, 0, 3, 120, 34, },
+	{ 2, 1, 0, 3, 120, 30, },
+	{ 1, 1, 0, 3, 120, 30, },
+	{ 0, 1, 0, 3, 124, 32, },
+	{ 2, 1, 0, 3, 124, 30, },
+	{ 1, 1, 0, 3, 124, 30, },
+	{ 0, 1, 0, 3, 128, 30, },
+	{ 2, 1, 0, 3, 128, 30, },
+	{ 1, 1, 0, 3, 128, 30, },
+	{ 0, 1, 0, 3, 132, 28, },
+	{ 2, 1, 0, 3, 132, 30, },
+	{ 1, 1, 0, 3, 132, 30, },
+	{ 0, 1, 0, 3, 136, 28, },
+	{ 2, 1, 0, 3, 136, 30, },
+	{ 1, 1, 0, 3, 136, 30, },
+	{ 0, 1, 0, 3, 140, 26, },
+	{ 2, 1, 0, 3, 140, 30, },
+	{ 1, 1, 0, 3, 140, 30, },
+	{ 0, 1, 0, 3, 149, 34, },
+	{ 2, 1, 0, 3, 149, 30, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 34, },
+	{ 2, 1, 0, 3, 153, 30, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 34, },
+	{ 2, 1, 0, 3, 157, 30, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 34, },
+	{ 2, 1, 0, 3, 161, 30, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 34, },
+	{ 2, 1, 0, 3, 165, 30, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 26, },
+	{ 2, 1, 0, 6, 36, 28, },
+	{ 1, 1, 0, 6, 36, 28, },
+	{ 0, 1, 0, 6, 40, 26, },
+	{ 2, 1, 0, 6, 40, 28, },
+	{ 1, 1, 0, 6, 40, 28, },
+	{ 0, 1, 0, 6, 44, 26, },
+	{ 2, 1, 0, 6, 44, 28, },
+	{ 1, 1, 0, 6, 44, 28, },
+	{ 0, 1, 0, 6, 48, 26, },
+	{ 2, 1, 0, 6, 48, 28, },
+	{ 1, 1, 0, 6, 48, 28, },
+	{ 0, 1, 0, 6, 52, 32, },
+	{ 2, 1, 0, 6, 52, 28, },
+	{ 1, 1, 0, 6, 52, 28, },
+	{ 0, 1, 0, 6, 56, 30, },
+	{ 2, 1, 0, 6, 56, 28, },
+	{ 1, 1, 0, 6, 56, 28, },
+	{ 0, 1, 0, 6, 60, 28, },
+	{ 2, 1, 0, 6, 60, 28, },
+	{ 1, 1, 0, 6, 60, 28, },
+	{ 0, 1, 0, 6, 64, 24, },
+	{ 2, 1, 0, 6, 64, 28, },
+	{ 1, 1, 0, 6, 64, 28, },
+	{ 0, 1, 0, 6, 100, 26, },
+	{ 2, 1, 0, 6, 100, 28, },
+	{ 1, 1, 0, 6, 100, 28, },
+	{ 0, 1, 0, 6, 104, 26, },
+	{ 2, 1, 0, 6, 104, 28, },
+	{ 1, 1, 0, 6, 104, 28, },
+	{ 0, 1, 0, 6, 108, 28, },
+	{ 2, 1, 0, 6, 108, 28, },
+	{ 1, 1, 0, 6, 108, 28, },
+	{ 0, 1, 0, 6, 112, 30, },
+	{ 2, 1, 0, 6, 112, 28, },
+	{ 1, 1, 0, 6, 112, 28, },
+	{ 0, 1, 0, 6, 116, 30, },
+	{ 2, 1, 0, 6, 116, 28, },
+	{ 1, 1, 0, 6, 116, 28, },
+	{ 0, 1, 0, 6, 120, 32, },
+	{ 2, 1, 0, 6, 120, 28, },
+	{ 1, 1, 0, 6, 120, 28, },
+	{ 0, 1, 0, 6, 124, 30, },
+	{ 2, 1, 0, 6, 124, 28, },
+	{ 1, 1, 0, 6, 124, 28, },
+	{ 0, 1, 0, 6, 128, 28, },
+	{ 2, 1, 0, 6, 128, 28, },
+	{ 1, 1, 0, 6, 128, 28, },
+	{ 0, 1, 0, 6, 132, 26, },
+	{ 2, 1, 0, 6, 132, 28, },
+	{ 1, 1, 0, 6, 132, 28, },
+	{ 0, 1, 0, 6, 136, 26, },
+	{ 2, 1, 0, 6, 136, 28, },
+	{ 1, 1, 0, 6, 136, 28, },
+	{ 0, 1, 0, 6, 140, 24, },
+	{ 2, 1, 0, 6, 140, 28, },
+	{ 1, 1, 0, 6, 140, 28, },
+	{ 0, 1, 0, 6, 149, 32, },
+	{ 2, 1, 0, 6, 149, 28, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 32, },
+	{ 2, 1, 0, 6, 153, 28, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 32, },
+	{ 2, 1, 0, 6, 157, 28, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 32, },
+	{ 2, 1, 0, 6, 161, 28, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 32, },
+	{ 2, 1, 0, 6, 165, 28, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 24, },
+	{ 2, 1, 0, 7, 36, 26, },
+	{ 1, 1, 0, 7, 36, 26, },
+	{ 0, 1, 0, 7, 40, 24, },
+	{ 2, 1, 0, 7, 40, 26, },
+	{ 1, 1, 0, 7, 40, 26, },
+	{ 0, 1, 0, 7, 44, 24, },
+	{ 2, 1, 0, 7, 44, 26, },
+	{ 1, 1, 0, 7, 44, 26, },
+	{ 0, 1, 0, 7, 48, 24, },
+	{ 2, 1, 0, 7, 48, 26, },
+	{ 1, 1, 0, 7, 48, 26, },
+	{ 0, 1, 0, 7, 52, 30, },
+	{ 2, 1, 0, 7, 52, 26, },
+	{ 1, 1, 0, 7, 52, 26, },
+	{ 0, 1, 0, 7, 56, 28, },
+	{ 2, 1, 0, 7, 56, 26, },
+	{ 1, 1, 0, 7, 56, 26, },
+	{ 0, 1, 0, 7, 60, 26, },
+	{ 2, 1, 0, 7, 60, 26, },
+	{ 1, 1, 0, 7, 60, 26, },
+	{ 0, 1, 0, 7, 64, 22, },
+	{ 2, 1, 0, 7, 64, 26, },
+	{ 1, 1, 0, 7, 64, 26, },
+	{ 0, 1, 0, 7, 100, 24, },
+	{ 2, 1, 0, 7, 100, 26, },
+	{ 1, 1, 0, 7, 100, 26, },
+	{ 0, 1, 0, 7, 104, 24, },
+	{ 2, 1, 0, 7, 104, 26, },
+	{ 1, 1, 0, 7, 104, 26, },
+	{ 0, 1, 0, 7, 108, 26, },
+	{ 2, 1, 0, 7, 108, 26, },
+	{ 1, 1, 0, 7, 108, 26, },
+	{ 0, 1, 0, 7, 112, 28, },
+	{ 2, 1, 0, 7, 112, 26, },
+	{ 1, 1, 0, 7, 112, 26, },
+	{ 0, 1, 0, 7, 116, 28, },
+	{ 2, 1, 0, 7, 116, 26, },
+	{ 1, 1, 0, 7, 116, 26, },
+	{ 0, 1, 0, 7, 120, 30, },
+	{ 2, 1, 0, 7, 120, 26, },
+	{ 1, 1, 0, 7, 120, 26, },
+	{ 0, 1, 0, 7, 124, 28, },
+	{ 2, 1, 0, 7, 124, 26, },
+	{ 1, 1, 0, 7, 124, 26, },
+	{ 0, 1, 0, 7, 128, 26, },
+	{ 2, 1, 0, 7, 128, 26, },
+	{ 1, 1, 0, 7, 128, 26, },
+	{ 0, 1, 0, 7, 132, 24, },
+	{ 2, 1, 0, 7, 132, 26, },
+	{ 1, 1, 0, 7, 132, 26, },
+	{ 0, 1, 0, 7, 136, 24, },
+	{ 2, 1, 0, 7, 136, 26, },
+	{ 1, 1, 0, 7, 136, 26, },
+	{ 0, 1, 0, 7, 140, 22, },
+	{ 2, 1, 0, 7, 140, 26, },
+	{ 1, 1, 0, 7, 140, 26, },
+	{ 0, 1, 0, 7, 149, 30, },
+	{ 2, 1, 0, 7, 149, 26, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 30, },
+	{ 2, 1, 0, 7, 153, 26, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 30, },
+	{ 2, 1, 0, 7, 157, 26, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 30, },
+	{ 2, 1, 0, 7, 161, 26, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 30, },
+	{ 2, 1, 0, 7, 165, 26, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 30, },
+	{ 2, 1, 1, 2, 38, 32, },
+	{ 1, 1, 1, 2, 38, 32, },
+	{ 0, 1, 1, 2, 46, 30, },
+	{ 2, 1, 1, 2, 46, 32, },
+	{ 1, 1, 1, 2, 46, 32, },
+	{ 0, 1, 1, 2, 54, 32, },
+	{ 2, 1, 1, 2, 54, 32, },
+	{ 1, 1, 1, 2, 54, 32, },
+	{ 0, 1, 1, 2, 62, 32, },
+	{ 2, 1, 1, 2, 62, 32, },
+	{ 1, 1, 1, 2, 62, 32, },
+	{ 0, 1, 1, 2, 102, 28, },
+	{ 2, 1, 1, 2, 102, 32, },
+	{ 1, 1, 1, 2, 102, 32, },
+	{ 0, 1, 1, 2, 110, 32, },
+	{ 2, 1, 1, 2, 110, 32, },
+	{ 1, 1, 1, 2, 110, 32, },
+	{ 0, 1, 1, 2, 118, 36, },
+	{ 2, 1, 1, 2, 118, 32, },
+	{ 1, 1, 1, 2, 118, 32, },
+	{ 0, 1, 1, 2, 126, 34, },
+	{ 2, 1, 1, 2, 126, 32, },
+	{ 1, 1, 1, 2, 126, 32, },
+	{ 0, 1, 1, 2, 134, 32, },
+	{ 2, 1, 1, 2, 134, 32, },
+	{ 1, 1, 1, 2, 134, 32, },
+	{ 0, 1, 1, 2, 151, 36, },
+	{ 2, 1, 1, 2, 151, 32, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 36, },
+	{ 2, 1, 1, 2, 159, 32, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 28, },
+	{ 2, 1, 1, 3, 38, 30, },
+	{ 1, 1, 1, 3, 38, 30, },
+	{ 0, 1, 1, 3, 46, 28, },
+	{ 2, 1, 1, 3, 46, 30, },
+	{ 1, 1, 1, 3, 46, 30, },
+	{ 0, 1, 1, 3, 54, 30, },
+	{ 2, 1, 1, 3, 54, 30, },
+	{ 1, 1, 1, 3, 54, 30, },
+	{ 0, 1, 1, 3, 62, 30, },
+	{ 2, 1, 1, 3, 62, 30, },
+	{ 1, 1, 1, 3, 62, 30, },
+	{ 0, 1, 1, 3, 102, 26, },
+	{ 2, 1, 1, 3, 102, 30, },
+	{ 1, 1, 1, 3, 102, 30, },
+	{ 0, 1, 1, 3, 110, 30, },
+	{ 2, 1, 1, 3, 110, 30, },
+	{ 1, 1, 1, 3, 110, 30, },
+	{ 0, 1, 1, 3, 118, 34, },
+	{ 2, 1, 1, 3, 118, 30, },
+	{ 1, 1, 1, 3, 118, 30, },
+	{ 0, 1, 1, 3, 126, 32, },
+	{ 2, 1, 1, 3, 126, 30, },
+	{ 1, 1, 1, 3, 126, 30, },
+	{ 0, 1, 1, 3, 134, 30, },
+	{ 2, 1, 1, 3, 134, 30, },
+	{ 1, 1, 1, 3, 134, 30, },
+	{ 0, 1, 1, 3, 151, 34, },
+	{ 2, 1, 1, 3, 151, 30, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 34, },
+	{ 2, 1, 1, 3, 159, 30, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 26, },
+	{ 2, 1, 1, 6, 38, 28, },
+	{ 1, 1, 1, 6, 38, 28, },
+	{ 0, 1, 1, 6, 46, 26, },
+	{ 2, 1, 1, 6, 46, 28, },
+	{ 1, 1, 1, 6, 46, 28, },
+	{ 0, 1, 1, 6, 54, 28, },
+	{ 2, 1, 1, 6, 54, 28, },
+	{ 1, 1, 1, 6, 54, 28, },
+	{ 0, 1, 1, 6, 62, 28, },
+	{ 2, 1, 1, 6, 62, 28, },
+	{ 1, 1, 1, 6, 62, 28, },
+	{ 0, 1, 1, 6, 102, 24, },
+	{ 2, 1, 1, 6, 102, 28, },
+	{ 1, 1, 1, 6, 102, 28, },
+	{ 0, 1, 1, 6, 110, 28, },
+	{ 2, 1, 1, 6, 110, 28, },
+	{ 1, 1, 1, 6, 110, 28, },
+	{ 0, 1, 1, 6, 118, 32, },
+	{ 2, 1, 1, 6, 118, 28, },
+	{ 1, 1, 1, 6, 118, 28, },
+	{ 0, 1, 1, 6, 126, 30, },
+	{ 2, 1, 1, 6, 126, 28, },
+	{ 1, 1, 1, 6, 126, 28, },
+	{ 0, 1, 1, 6, 134, 28, },
+	{ 2, 1, 1, 6, 134, 28, },
+	{ 1, 1, 1, 6, 134, 28, },
+	{ 0, 1, 1, 6, 151, 32, },
+	{ 2, 1, 1, 6, 151, 28, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 32, },
+	{ 2, 1, 1, 6, 159, 28, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 24, },
+	{ 2, 1, 1, 7, 38, 26, },
+	{ 1, 1, 1, 7, 38, 26, },
+	{ 0, 1, 1, 7, 46, 24, },
+	{ 2, 1, 1, 7, 46, 26, },
+	{ 1, 1, 1, 7, 46, 26, },
+	{ 0, 1, 1, 7, 54, 26, },
+	{ 2, 1, 1, 7, 54, 26, },
+	{ 1, 1, 1, 7, 54, 26, },
+	{ 0, 1, 1, 7, 62, 26, },
+	{ 2, 1, 1, 7, 62, 26, },
+	{ 1, 1, 1, 7, 62, 26, },
+	{ 0, 1, 1, 7, 102, 22, },
+	{ 2, 1, 1, 7, 102, 26, },
+	{ 1, 1, 1, 7, 102, 26, },
+	{ 0, 1, 1, 7, 110, 26, },
+	{ 2, 1, 1, 7, 110, 26, },
+	{ 1, 1, 1, 7, 110, 26, },
+	{ 0, 1, 1, 7, 118, 30, },
+	{ 2, 1, 1, 7, 118, 26, },
+	{ 1, 1, 1, 7, 118, 26, },
+	{ 0, 1, 1, 7, 126, 28, },
+	{ 2, 1, 1, 7, 126, 26, },
+	{ 1, 1, 1, 7, 126, 26, },
+	{ 0, 1, 1, 7, 134, 26, },
+	{ 2, 1, 1, 7, 134, 26, },
+	{ 1, 1, 1, 7, 134, 26, },
+	{ 0, 1, 1, 7, 151, 30, },
+	{ 2, 1, 1, 7, 151, 26, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 30, },
+	{ 2, 1, 1, 7, 159, 26, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 30, },
+	{ 2, 1, 2, 4, 42, 32, },
+	{ 1, 1, 2, 4, 42, 32, },
+	{ 0, 1, 2, 4, 58, 28, },
+	{ 2, 1, 2, 4, 58, 32, },
+	{ 1, 1, 2, 4, 58, 32, },
+	{ 0, 1, 2, 4, 106, 30, },
+	{ 2, 1, 2, 4, 106, 32, },
+	{ 1, 1, 2, 4, 106, 32, },
+	{ 0, 1, 2, 4, 122, 34, },
+	{ 2, 1, 2, 4, 122, 32, },
+	{ 1, 1, 2, 4, 122, 32, },
+	{ 0, 1, 2, 4, 155, 36, },
+	{ 2, 1, 2, 4, 155, 32, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 28, },
+	{ 2, 1, 2, 5, 42, 30, },
+	{ 1, 1, 2, 5, 42, 30, },
+	{ 0, 1, 2, 5, 58, 26, },
+	{ 2, 1, 2, 5, 58, 30, },
+	{ 1, 1, 2, 5, 58, 30, },
+	{ 0, 1, 2, 5, 106, 28, },
+	{ 2, 1, 2, 5, 106, 30, },
+	{ 1, 1, 2, 5, 106, 30, },
+	{ 0, 1, 2, 5, 122, 32, },
+	{ 2, 1, 2, 5, 122, 30, },
+	{ 1, 1, 2, 5, 122, 30, },
+	{ 0, 1, 2, 5, 155, 34, },
+	{ 2, 1, 2, 5, 155, 30, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 26, },
+	{ 2, 1, 2, 8, 42, 28, },
+	{ 1, 1, 2, 8, 42, 28, },
+	{ 0, 1, 2, 8, 58, 24, },
+	{ 2, 1, 2, 8, 58, 28, },
+	{ 1, 1, 2, 8, 58, 28, },
+	{ 0, 1, 2, 8, 106, 26, },
+	{ 2, 1, 2, 8, 106, 28, },
+	{ 1, 1, 2, 8, 106, 28, },
+	{ 0, 1, 2, 8, 122, 30, },
+	{ 2, 1, 2, 8, 122, 28, },
+	{ 1, 1, 2, 8, 122, 28, },
+	{ 0, 1, 2, 8, 155, 32, },
+	{ 2, 1, 2, 8, 155, 28, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 24, },
+	{ 2, 1, 2, 9, 42, 26, },
+	{ 1, 1, 2, 9, 42, 26, },
+	{ 0, 1, 2, 9, 58, 22, },
+	{ 2, 1, 2, 9, 58, 26, },
+	{ 1, 1, 2, 9, 58, 26, },
+	{ 0, 1, 2, 9, 106, 24, },
+	{ 2, 1, 2, 9, 106, 26, },
+	{ 1, 1, 2, 9, 106, 26, },
+	{ 0, 1, 2, 9, 122, 28, },
+	{ 2, 1, 2, 9, 122, 26, },
+	{ 1, 1, 2, 9, 122, 26, },
+	{ 0, 1, 2, 9, 155, 30, },
+	{ 2, 1, 2, 9, 155, 26, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt_type0[] = {
+	{ 0, 0, 0, 0, 1, 32, },
+	{ 2, 0, 0, 0, 1, 32, },
+	{ 1, 0, 0, 0, 1, 32, },
+	{ 0, 0, 0, 0, 2, 32, },
+	{ 2, 0, 0, 0, 2, 32, },
+	{ 1, 0, 0, 0, 2, 32, },
+	{ 0, 0, 0, 0, 3, 32, },
+	{ 2, 0, 0, 0, 3, 32, },
+	{ 1, 0, 0, 0, 3, 32, },
+	{ 0, 0, 0, 0, 4, 32, },
+	{ 2, 0, 0, 0, 4, 32, },
+	{ 1, 0, 0, 0, 4, 32, },
+	{ 0, 0, 0, 0, 5, 32, },
+	{ 2, 0, 0, 0, 5, 32, },
+	{ 1, 0, 0, 0, 5, 32, },
+	{ 0, 0, 0, 0, 6, 32, },
+	{ 2, 0, 0, 0, 6, 32, },
+	{ 1, 0, 0, 0, 6, 32, },
+	{ 0, 0, 0, 0, 7, 32, },
+	{ 2, 0, 0, 0, 7, 32, },
+	{ 1, 0, 0, 0, 7, 32, },
+	{ 0, 0, 0, 0, 8, 32, },
+	{ 2, 0, 0, 0, 8, 32, },
+	{ 1, 0, 0, 0, 8, 32, },
+	{ 0, 0, 0, 0, 9, 32, },
+	{ 2, 0, 0, 0, 9, 32, },
+	{ 1, 0, 0, 0, 9, 32, },
+	{ 0, 0, 0, 0, 10, 32, },
+	{ 2, 0, 0, 0, 10, 32, },
+	{ 1, 0, 0, 0, 10, 32, },
+	{ 0, 0, 0, 0, 11, 32, },
+	{ 2, 0, 0, 0, 11, 32, },
+	{ 1, 0, 0, 0, 11, 32, },
+	{ 0, 0, 0, 0, 12, 24, },
+	{ 2, 0, 0, 0, 12, 32, },
+	{ 1, 0, 0, 0, 12, 32, },
+	{ 0, 0, 0, 0, 13, 16, },
+	{ 2, 0, 0, 0, 13, 32, },
+	{ 1, 0, 0, 0, 13, 32, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 32, },
+	{ 0, 0, 0, 1, 1, 28, },
+	{ 2, 0, 0, 1, 1, 32, },
+	{ 1, 0, 0, 1, 1, 32, },
+	{ 0, 0, 0, 1, 2, 32, },
+	{ 2, 0, 0, 1, 2, 32, },
+	{ 1, 0, 0, 1, 2, 32, },
+	{ 0, 0, 0, 1, 3, 32, },
+	{ 2, 0, 0, 1, 3, 32, },
+	{ 1, 0, 0, 1, 3, 32, },
+	{ 0, 0, 0, 1, 4, 32, },
+	{ 2, 0, 0, 1, 4, 32, },
+	{ 1, 0, 0, 1, 4, 32, },
+	{ 0, 0, 0, 1, 5, 32, },
+	{ 2, 0, 0, 1, 5, 32, },
+	{ 1, 0, 0, 1, 5, 32, },
+	{ 0, 0, 0, 1, 6, 32, },
+	{ 2, 0, 0, 1, 6, 32, },
+	{ 1, 0, 0, 1, 6, 32, },
+	{ 0, 0, 0, 1, 7, 32, },
+	{ 2, 0, 0, 1, 7, 32, },
+	{ 1, 0, 0, 1, 7, 32, },
+	{ 0, 0, 0, 1, 8, 32, },
+	{ 2, 0, 0, 1, 8, 32, },
+	{ 1, 0, 0, 1, 8, 32, },
+	{ 0, 0, 0, 1, 9, 32, },
+	{ 2, 0, 0, 1, 9, 32, },
+	{ 1, 0, 0, 1, 9, 32, },
+	{ 0, 0, 0, 1, 10, 32, },
+	{ 2, 0, 0, 1, 10, 32, },
+	{ 1, 0, 0, 1, 10, 32, },
+	{ 0, 0, 0, 1, 11, 28, },
+	{ 2, 0, 0, 1, 11, 32, },
+	{ 1, 0, 0, 1, 11, 32, },
+	{ 0, 0, 0, 1, 12, 18, },
+	{ 2, 0, 0, 1, 12, 32, },
+	{ 1, 0, 0, 1, 12, 32, },
+	{ 0, 0, 0, 1, 13, 8, },
+	{ 2, 0, 0, 1, 13, 32, },
+	{ 1, 0, 0, 1, 13, 32, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 63, },
+	{ 0, 0, 0, 2, 1, 26, },
+	{ 2, 0, 0, 2, 1, 32, },
+	{ 1, 0, 0, 2, 1, 32, },
+	{ 0, 0, 0, 2, 2, 32, },
+	{ 2, 0, 0, 2, 2, 32, },
+	{ 1, 0, 0, 2, 2, 32, },
+	{ 0, 0, 0, 2, 3, 32, },
+	{ 2, 0, 0, 2, 3, 32, },
+	{ 1, 0, 0, 2, 3, 32, },
+	{ 0, 0, 0, 2, 4, 32, },
+	{ 2, 0, 0, 2, 4, 32, },
+	{ 1, 0, 0, 2, 4, 32, },
+	{ 0, 0, 0, 2, 5, 32, },
+	{ 2, 0, 0, 2, 5, 32, },
+	{ 1, 0, 0, 2, 5, 32, },
+	{ 0, 0, 0, 2, 6, 32, },
+	{ 2, 0, 0, 2, 6, 32, },
+	{ 1, 0, 0, 2, 6, 32, },
+	{ 0, 0, 0, 2, 7, 32, },
+	{ 2, 0, 0, 2, 7, 32, },
+	{ 1, 0, 0, 2, 7, 32, },
+	{ 0, 0, 0, 2, 8, 32, },
+	{ 2, 0, 0, 2, 8, 32, },
+	{ 1, 0, 0, 2, 8, 32, },
+	{ 0, 0, 0, 2, 9, 32, },
+	{ 2, 0, 0, 2, 9, 32, },
+	{ 1, 0, 0, 2, 9, 32, },
+	{ 0, 0, 0, 2, 10, 32, },
+	{ 2, 0, 0, 2, 10, 32, },
+	{ 1, 0, 0, 2, 10, 32, },
+	{ 0, 0, 0, 2, 11, 26, },
+	{ 2, 0, 0, 2, 11, 32, },
+	{ 1, 0, 0, 2, 11, 32, },
+	{ 0, 0, 0, 2, 12, 16, },
+	{ 2, 0, 0, 2, 12, 32, },
+	{ 1, 0, 0, 2, 12, 32, },
+	{ 0, 0, 0, 2, 13, 6, },
+	{ 2, 0, 0, 2, 13, 32, },
+	{ 1, 0, 0, 2, 13, 32, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 63, },
+	{ 0, 0, 0, 3, 1, 24, },
+	{ 2, 0, 0, 3, 1, 30, },
+	{ 1, 0, 0, 3, 1, 30, },
+	{ 0, 0, 0, 3, 2, 30, },
+	{ 2, 0, 0, 3, 2, 30, },
+	{ 1, 0, 0, 3, 2, 30, },
+	{ 0, 0, 0, 3, 3, 30, },
+	{ 2, 0, 0, 3, 3, 30, },
+	{ 1, 0, 0, 3, 3, 30, },
+	{ 0, 0, 0, 3, 4, 30, },
+	{ 2, 0, 0, 3, 4, 30, },
+	{ 1, 0, 0, 3, 4, 30, },
+	{ 0, 0, 0, 3, 5, 30, },
+	{ 2, 0, 0, 3, 5, 30, },
+	{ 1, 0, 0, 3, 5, 30, },
+	{ 0, 0, 0, 3, 6, 30, },
+	{ 2, 0, 0, 3, 6, 30, },
+	{ 1, 0, 0, 3, 6, 30, },
+	{ 0, 0, 0, 3, 7, 30, },
+	{ 2, 0, 0, 3, 7, 30, },
+	{ 1, 0, 0, 3, 7, 30, },
+	{ 0, 0, 0, 3, 8, 30, },
+	{ 2, 0, 0, 3, 8, 30, },
+	{ 1, 0, 0, 3, 8, 30, },
+	{ 0, 0, 0, 3, 9, 30, },
+	{ 2, 0, 0, 3, 9, 30, },
+	{ 1, 0, 0, 3, 9, 30, },
+	{ 0, 0, 0, 3, 10, 30, },
+	{ 2, 0, 0, 3, 10, 30, },
+	{ 1, 0, 0, 3, 10, 30, },
+	{ 0, 0, 0, 3, 11, 24, },
+	{ 2, 0, 0, 3, 11, 30, },
+	{ 1, 0, 0, 3, 11, 30, },
+	{ 0, 0, 0, 3, 12, 14, },
+	{ 2, 0, 0, 3, 12, 30, },
+	{ 1, 0, 0, 3, 12, 30, },
+	{ 0, 0, 0, 3, 13, 4, },
+	{ 2, 0, 0, 3, 13, 30, },
+	{ 1, 0, 0, 3, 13, 30, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 63, },
+	{ 0, 0, 0, 6, 1, 22, },
+	{ 2, 0, 0, 6, 1, 28, },
+	{ 1, 0, 0, 6, 1, 28, },
+	{ 0, 0, 0, 6, 2, 28, },
+	{ 2, 0, 0, 6, 2, 28, },
+	{ 1, 0, 0, 6, 2, 28, },
+	{ 0, 0, 0, 6, 3, 28, },
+	{ 2, 0, 0, 6, 3, 28, },
+	{ 1, 0, 0, 6, 3, 28, },
+	{ 0, 0, 0, 6, 4, 28, },
+	{ 2, 0, 0, 6, 4, 28, },
+	{ 1, 0, 0, 6, 4, 28, },
+	{ 0, 0, 0, 6, 5, 28, },
+	{ 2, 0, 0, 6, 5, 28, },
+	{ 1, 0, 0, 6, 5, 28, },
+	{ 0, 0, 0, 6, 6, 28, },
+	{ 2, 0, 0, 6, 6, 28, },
+	{ 1, 0, 0, 6, 6, 28, },
+	{ 0, 0, 0, 6, 7, 28, },
+	{ 2, 0, 0, 6, 7, 28, },
+	{ 1, 0, 0, 6, 7, 28, },
+	{ 0, 0, 0, 6, 8, 28, },
+	{ 2, 0, 0, 6, 8, 28, },
+	{ 1, 0, 0, 6, 8, 28, },
+	{ 0, 0, 0, 6, 9, 28, },
+	{ 2, 0, 0, 6, 9, 28, },
+	{ 1, 0, 0, 6, 9, 28, },
+	{ 0, 0, 0, 6, 10, 28, },
+	{ 2, 0, 0, 6, 10, 28, },
+	{ 1, 0, 0, 6, 10, 28, },
+	{ 0, 0, 0, 6, 11, 22, },
+	{ 2, 0, 0, 6, 11, 28, },
+	{ 1, 0, 0, 6, 11, 28, },
+	{ 0, 0, 0, 6, 12, 14, },
+	{ 2, 0, 0, 6, 12, 28, },
+	{ 1, 0, 0, 6, 12, 28, },
+	{ 0, 0, 0, 6, 13, 4, },
+	{ 2, 0, 0, 6, 13, 28, },
+	{ 1, 0, 0, 6, 13, 28, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 63, },
+	{ 0, 0, 0, 7, 1, 20, },
+	{ 2, 0, 0, 7, 1, 26, },
+	{ 1, 0, 0, 7, 1, 26, },
+	{ 0, 0, 0, 7, 2, 26, },
+	{ 2, 0, 0, 7, 2, 26, },
+	{ 1, 0, 0, 7, 2, 26, },
+	{ 0, 0, 0, 7, 3, 26, },
+	{ 2, 0, 0, 7, 3, 26, },
+	{ 1, 0, 0, 7, 3, 26, },
+	{ 0, 0, 0, 7, 4, 26, },
+	{ 2, 0, 0, 7, 4, 26, },
+	{ 1, 0, 0, 7, 4, 26, },
+	{ 0, 0, 0, 7, 5, 26, },
+	{ 2, 0, 0, 7, 5, 26, },
+	{ 1, 0, 0, 7, 5, 26, },
+	{ 0, 0, 0, 7, 6, 26, },
+	{ 2, 0, 0, 7, 6, 26, },
+	{ 1, 0, 0, 7, 6, 26, },
+	{ 0, 0, 0, 7, 7, 26, },
+	{ 2, 0, 0, 7, 7, 26, },
+	{ 1, 0, 0, 7, 7, 26, },
+	{ 0, 0, 0, 7, 8, 26, },
+	{ 2, 0, 0, 7, 8, 26, },
+	{ 1, 0, 0, 7, 8, 26, },
+	{ 0, 0, 0, 7, 9, 26, },
+	{ 2, 0, 0, 7, 9, 26, },
+	{ 1, 0, 0, 7, 9, 26, },
+	{ 0, 0, 0, 7, 10, 26, },
+	{ 2, 0, 0, 7, 10, 26, },
+	{ 1, 0, 0, 7, 10, 26, },
+	{ 0, 0, 0, 7, 11, 20, },
+	{ 2, 0, 0, 7, 11, 26, },
+	{ 1, 0, 0, 7, 11, 26, },
+	{ 0, 0, 0, 7, 12, 14, },
+	{ 2, 0, 0, 7, 12, 26, },
+	{ 1, 0, 0, 7, 12, 26, },
+	{ 0, 0, 0, 7, 13, 4, },
+	{ 2, 0, 0, 7, 13, 26, },
+	{ 1, 0, 0, 7, 13, 26, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 63, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 26, },
+	{ 2, 0, 1, 2, 3, 32, },
+	{ 1, 0, 1, 2, 3, 32, },
+	{ 0, 0, 1, 2, 4, 32, },
+	{ 2, 0, 1, 2, 4, 32, },
+	{ 1, 0, 1, 2, 4, 32, },
+	{ 0, 0, 1, 2, 5, 32, },
+	{ 2, 0, 1, 2, 5, 32, },
+	{ 1, 0, 1, 2, 5, 32, },
+	{ 0, 0, 1, 2, 6, 32, },
+	{ 2, 0, 1, 2, 6, 32, },
+	{ 1, 0, 1, 2, 6, 32, },
+	{ 0, 0, 1, 2, 7, 32, },
+	{ 2, 0, 1, 2, 7, 32, },
+	{ 1, 0, 1, 2, 7, 32, },
+	{ 0, 0, 1, 2, 8, 32, },
+	{ 2, 0, 1, 2, 8, 32, },
+	{ 1, 0, 1, 2, 8, 32, },
+	{ 0, 0, 1, 2, 9, 32, },
+	{ 2, 0, 1, 2, 9, 32, },
+	{ 1, 0, 1, 2, 9, 32, },
+	{ 0, 0, 1, 2, 10, 32, },
+	{ 2, 0, 1, 2, 10, 32, },
+	{ 1, 0, 1, 2, 10, 32, },
+	{ 0, 0, 1, 2, 11, 26, },
+	{ 2, 0, 1, 2, 11, 32, },
+	{ 1, 0, 1, 2, 11, 32, },
+	{ 0, 0, 1, 2, 12, 16, },
+	{ 2, 0, 1, 2, 12, 32, },
+	{ 1, 0, 1, 2, 12, 32, },
+	{ 0, 0, 1, 2, 13, 10, },
+	{ 2, 0, 1, 2, 13, 32, },
+	{ 1, 0, 1, 2, 13, 32, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 24, },
+	{ 2, 0, 1, 3, 3, 30, },
+	{ 1, 0, 1, 3, 3, 30, },
+	{ 0, 0, 1, 3, 4, 30, },
+	{ 2, 0, 1, 3, 4, 30, },
+	{ 1, 0, 1, 3, 4, 30, },
+	{ 0, 0, 1, 3, 5, 30, },
+	{ 2, 0, 1, 3, 5, 30, },
+	{ 1, 0, 1, 3, 5, 30, },
+	{ 0, 0, 1, 3, 6, 30, },
+	{ 2, 0, 1, 3, 6, 30, },
+	{ 1, 0, 1, 3, 6, 30, },
+	{ 0, 0, 1, 3, 7, 30, },
+	{ 2, 0, 1, 3, 7, 30, },
+	{ 1, 0, 1, 3, 7, 30, },
+	{ 0, 0, 1, 3, 8, 30, },
+	{ 2, 0, 1, 3, 8, 30, },
+	{ 1, 0, 1, 3, 8, 30, },
+	{ 0, 0, 1, 3, 9, 30, },
+	{ 2, 0, 1, 3, 9, 30, },
+	{ 1, 0, 1, 3, 9, 30, },
+	{ 0, 0, 1, 3, 10, 30, },
+	{ 2, 0, 1, 3, 10, 30, },
+	{ 1, 0, 1, 3, 10, 30, },
+	{ 0, 0, 1, 3, 11, 24, },
+	{ 2, 0, 1, 3, 11, 30, },
+	{ 1, 0, 1, 3, 11, 30, },
+	{ 0, 0, 1, 3, 12, 14, },
+	{ 2, 0, 1, 3, 12, 30, },
+	{ 1, 0, 1, 3, 12, 30, },
+	{ 0, 0, 1, 3, 13, 8, },
+	{ 2, 0, 1, 3, 13, 30, },
+	{ 1, 0, 1, 3, 13, 30, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 22, },
+	{ 2, 0, 1, 6, 3, 28, },
+	{ 1, 0, 1, 6, 3, 28, },
+	{ 0, 0, 1, 6, 4, 28, },
+	{ 2, 0, 1, 6, 4, 28, },
+	{ 1, 0, 1, 6, 4, 28, },
+	{ 0, 0, 1, 6, 5, 28, },
+	{ 2, 0, 1, 6, 5, 28, },
+	{ 1, 0, 1, 6, 5, 28, },
+	{ 0, 0, 1, 6, 6, 28, },
+	{ 2, 0, 1, 6, 6, 28, },
+	{ 1, 0, 1, 6, 6, 28, },
+	{ 0, 0, 1, 6, 7, 28, },
+	{ 2, 0, 1, 6, 7, 28, },
+	{ 1, 0, 1, 6, 7, 28, },
+	{ 0, 0, 1, 6, 8, 28, },
+	{ 2, 0, 1, 6, 8, 28, },
+	{ 1, 0, 1, 6, 8, 28, },
+	{ 0, 0, 1, 6, 9, 28, },
+	{ 2, 0, 1, 6, 9, 28, },
+	{ 1, 0, 1, 6, 9, 28, },
+	{ 0, 0, 1, 6, 10, 28, },
+	{ 2, 0, 1, 6, 10, 28, },
+	{ 1, 0, 1, 6, 10, 28, },
+	{ 0, 0, 1, 6, 11, 22, },
+	{ 2, 0, 1, 6, 11, 28, },
+	{ 1, 0, 1, 6, 11, 28, },
+	{ 0, 0, 1, 6, 12, 14, },
+	{ 2, 0, 1, 6, 12, 28, },
+	{ 1, 0, 1, 6, 12, 28, },
+	{ 0, 0, 1, 6, 13, 8, },
+	{ 2, 0, 1, 6, 13, 28, },
+	{ 1, 0, 1, 6, 13, 28, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 20, },
+	{ 2, 0, 1, 7, 3, 26, },
+	{ 1, 0, 1, 7, 3, 26, },
+	{ 0, 0, 1, 7, 4, 26, },
+	{ 2, 0, 1, 7, 4, 26, },
+	{ 1, 0, 1, 7, 4, 26, },
+	{ 0, 0, 1, 7, 5, 26, },
+	{ 2, 0, 1, 7, 5, 26, },
+	{ 1, 0, 1, 7, 5, 26, },
+	{ 0, 0, 1, 7, 6, 26, },
+	{ 2, 0, 1, 7, 6, 26, },
+	{ 1, 0, 1, 7, 6, 26, },
+	{ 0, 0, 1, 7, 7, 26, },
+	{ 2, 0, 1, 7, 7, 26, },
+	{ 1, 0, 1, 7, 7, 26, },
+	{ 0, 0, 1, 7, 8, 26, },
+	{ 2, 0, 1, 7, 8, 26, },
+	{ 1, 0, 1, 7, 8, 26, },
+	{ 0, 0, 1, 7, 9, 26, },
+	{ 2, 0, 1, 7, 9, 26, },
+	{ 1, 0, 1, 7, 9, 26, },
+	{ 0, 0, 1, 7, 10, 26, },
+	{ 2, 0, 1, 7, 10, 26, },
+	{ 1, 0, 1, 7, 10, 26, },
+	{ 0, 0, 1, 7, 11, 20, },
+	{ 2, 0, 1, 7, 11, 26, },
+	{ 1, 0, 1, 7, 11, 26, },
+	{ 0, 0, 1, 7, 12, 14, },
+	{ 2, 0, 1, 7, 12, 26, },
+	{ 1, 0, 1, 7, 12, 26, },
+	{ 0, 0, 1, 7, 13, 8, },
+	{ 2, 0, 1, 7, 13, 26, },
+	{ 1, 0, 1, 7, 13, 26, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 28, },
+	{ 2, 1, 0, 1, 36, 32, },
+	{ 1, 1, 0, 1, 36, 32, },
+	{ 0, 1, 0, 1, 40, 32, },
+	{ 2, 1, 0, 1, 40, 32, },
+	{ 1, 1, 0, 1, 40, 32, },
+	{ 0, 1, 0, 1, 44, 32, },
+	{ 2, 1, 0, 1, 44, 32, },
+	{ 1, 1, 0, 1, 44, 32, },
+	{ 0, 1, 0, 1, 48, 32, },
+	{ 2, 1, 0, 1, 48, 32, },
+	{ 1, 1, 0, 1, 48, 32, },
+	{ 0, 1, 0, 1, 52, 32, },
+	{ 2, 1, 0, 1, 52, 32, },
+	{ 1, 1, 0, 1, 52, 32, },
+	{ 0, 1, 0, 1, 56, 32, },
+	{ 2, 1, 0, 1, 56, 32, },
+	{ 1, 1, 0, 1, 56, 32, },
+	{ 0, 1, 0, 1, 60, 32, },
+	{ 2, 1, 0, 1, 60, 32, },
+	{ 1, 1, 0, 1, 60, 32, },
+	{ 0, 1, 0, 1, 64, 28, },
+	{ 2, 1, 0, 1, 64, 32, },
+	{ 1, 1, 0, 1, 64, 32, },
+	{ 0, 1, 0, 1, 100, 28, },
+	{ 2, 1, 0, 1, 100, 32, },
+	{ 1, 1, 0, 1, 100, 32, },
+	{ 0, 1, 0, 1, 104, 32, },
+	{ 2, 1, 0, 1, 104, 32, },
+	{ 1, 1, 0, 1, 104, 32, },
+	{ 0, 1, 0, 1, 108, 32, },
+	{ 2, 1, 0, 1, 108, 32, },
+	{ 1, 1, 0, 1, 108, 32, },
+	{ 0, 1, 0, 1, 112, 32, },
+	{ 2, 1, 0, 1, 112, 32, },
+	{ 1, 1, 0, 1, 112, 32, },
+	{ 0, 1, 0, 1, 116, 32, },
+	{ 2, 1, 0, 1, 116, 32, },
+	{ 1, 1, 0, 1, 116, 32, },
+	{ 0, 1, 0, 1, 120, 32, },
+	{ 2, 1, 0, 1, 120, 32, },
+	{ 1, 1, 0, 1, 120, 32, },
+	{ 0, 1, 0, 1, 124, 32, },
+	{ 2, 1, 0, 1, 124, 32, },
+	{ 1, 1, 0, 1, 124, 32, },
+	{ 0, 1, 0, 1, 128, 32, },
+	{ 2, 1, 0, 1, 128, 32, },
+	{ 1, 1, 0, 1, 128, 32, },
+	{ 0, 1, 0, 1, 132, 32, },
+	{ 2, 1, 0, 1, 132, 32, },
+	{ 1, 1, 0, 1, 132, 32, },
+	{ 0, 1, 0, 1, 136, 32, },
+	{ 2, 1, 0, 1, 136, 32, },
+	{ 1, 1, 0, 1, 136, 32, },
+	{ 0, 1, 0, 1, 140, 28, },
+	{ 2, 1, 0, 1, 140, 32, },
+	{ 1, 1, 0, 1, 140, 32, },
+	{ 0, 1, 0, 1, 149, 28, },
+	{ 2, 1, 0, 1, 149, 32, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 32, },
+	{ 2, 1, 0, 1, 153, 32, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 32, },
+	{ 2, 1, 0, 1, 157, 32, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 32, },
+	{ 2, 1, 0, 1, 161, 32, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 32, },
+	{ 2, 1, 0, 1, 165, 32, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 26, },
+	{ 2, 1, 0, 2, 36, 32, },
+	{ 1, 1, 0, 2, 36, 32, },
+	{ 0, 1, 0, 2, 40, 32, },
+	{ 2, 1, 0, 2, 40, 32, },
+	{ 1, 1, 0, 2, 40, 32, },
+	{ 0, 1, 0, 2, 44, 32, },
+	{ 2, 1, 0, 2, 44, 32, },
+	{ 1, 1, 0, 2, 44, 32, },
+	{ 0, 1, 0, 2, 48, 32, },
+	{ 2, 1, 0, 2, 48, 32, },
+	{ 1, 1, 0, 2, 48, 32, },
+	{ 0, 1, 0, 2, 52, 32, },
+	{ 2, 1, 0, 2, 52, 32, },
+	{ 1, 1, 0, 2, 52, 32, },
+	{ 0, 1, 0, 2, 56, 32, },
+	{ 2, 1, 0, 2, 56, 32, },
+	{ 1, 1, 0, 2, 56, 32, },
+	{ 0, 1, 0, 2, 60, 32, },
+	{ 2, 1, 0, 2, 60, 32, },
+	{ 1, 1, 0, 2, 60, 32, },
+	{ 0, 1, 0, 2, 64, 26, },
+	{ 2, 1, 0, 2, 64, 32, },
+	{ 1, 1, 0, 2, 64, 32, },
+	{ 0, 1, 0, 2, 100, 26, },
+	{ 2, 1, 0, 2, 100, 32, },
+	{ 1, 1, 0, 2, 100, 32, },
+	{ 0, 1, 0, 2, 104, 32, },
+	{ 2, 1, 0, 2, 104, 32, },
+	{ 1, 1, 0, 2, 104, 32, },
+	{ 0, 1, 0, 2, 108, 32, },
+	{ 2, 1, 0, 2, 108, 32, },
+	{ 1, 1, 0, 2, 108, 32, },
+	{ 0, 1, 0, 2, 112, 32, },
+	{ 2, 1, 0, 2, 112, 32, },
+	{ 1, 1, 0, 2, 112, 32, },
+	{ 0, 1, 0, 2, 116, 32, },
+	{ 2, 1, 0, 2, 116, 32, },
+	{ 1, 1, 0, 2, 116, 32, },
+	{ 0, 1, 0, 2, 120, 32, },
+	{ 2, 1, 0, 2, 120, 32, },
+	{ 1, 1, 0, 2, 120, 32, },
+	{ 0, 1, 0, 2, 124, 32, },
+	{ 2, 1, 0, 2, 124, 32, },
+	{ 1, 1, 0, 2, 124, 32, },
+	{ 0, 1, 0, 2, 128, 32, },
+	{ 2, 1, 0, 2, 128, 32, },
+	{ 1, 1, 0, 2, 128, 32, },
+	{ 0, 1, 0, 2, 132, 32, },
+	{ 2, 1, 0, 2, 132, 32, },
+	{ 1, 1, 0, 2, 132, 32, },
+	{ 0, 1, 0, 2, 136, 32, },
+	{ 2, 1, 0, 2, 136, 32, },
+	{ 1, 1, 0, 2, 136, 32, },
+	{ 0, 1, 0, 2, 140, 26, },
+	{ 2, 1, 0, 2, 140, 32, },
+	{ 1, 1, 0, 2, 140, 32, },
+	{ 0, 1, 0, 2, 149, 26, },
+	{ 2, 1, 0, 2, 149, 32, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 32, },
+	{ 2, 1, 0, 2, 153, 32, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 32, },
+	{ 2, 1, 0, 2, 157, 32, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 32, },
+	{ 2, 1, 0, 2, 161, 32, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 32, },
+	{ 2, 1, 0, 2, 165, 32, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 24, },
+	{ 2, 1, 0, 3, 36, 28, },
+	{ 1, 1, 0, 3, 36, 28, },
+	{ 0, 1, 0, 3, 40, 28, },
+	{ 2, 1, 0, 3, 40, 28, },
+	{ 1, 1, 0, 3, 40, 28, },
+	{ 0, 1, 0, 3, 44, 28, },
+	{ 2, 1, 0, 3, 44, 28, },
+	{ 1, 1, 0, 3, 44, 28, },
+	{ 0, 1, 0, 3, 48, 28, },
+	{ 2, 1, 0, 3, 48, 28, },
+	{ 1, 1, 0, 3, 48, 28, },
+	{ 0, 1, 0, 3, 52, 28, },
+	{ 2, 1, 0, 3, 52, 28, },
+	{ 1, 1, 0, 3, 52, 28, },
+	{ 0, 1, 0, 3, 56, 28, },
+	{ 2, 1, 0, 3, 56, 28, },
+	{ 1, 1, 0, 3, 56, 28, },
+	{ 0, 1, 0, 3, 60, 28, },
+	{ 2, 1, 0, 3, 60, 28, },
+	{ 1, 1, 0, 3, 60, 28, },
+	{ 0, 1, 0, 3, 64, 24, },
+	{ 2, 1, 0, 3, 64, 28, },
+	{ 1, 1, 0, 3, 64, 28, },
+	{ 0, 1, 0, 3, 100, 24, },
+	{ 2, 1, 0, 3, 100, 28, },
+	{ 1, 1, 0, 3, 100, 28, },
+	{ 0, 1, 0, 3, 104, 28, },
+	{ 2, 1, 0, 3, 104, 28, },
+	{ 1, 1, 0, 3, 104, 28, },
+	{ 0, 1, 0, 3, 108, 28, },
+	{ 2, 1, 0, 3, 108, 28, },
+	{ 1, 1, 0, 3, 108, 28, },
+	{ 0, 1, 0, 3, 112, 28, },
+	{ 2, 1, 0, 3, 112, 28, },
+	{ 1, 1, 0, 3, 112, 28, },
+	{ 0, 1, 0, 3, 116, 28, },
+	{ 2, 1, 0, 3, 116, 28, },
+	{ 1, 1, 0, 3, 116, 28, },
+	{ 0, 1, 0, 3, 120, 28, },
+	{ 2, 1, 0, 3, 120, 28, },
+	{ 1, 1, 0, 3, 120, 28, },
+	{ 0, 1, 0, 3, 124, 28, },
+	{ 2, 1, 0, 3, 124, 28, },
+	{ 1, 1, 0, 3, 124, 28, },
+	{ 0, 1, 0, 3, 128, 28, },
+	{ 2, 1, 0, 3, 128, 28, },
+	{ 1, 1, 0, 3, 128, 28, },
+	{ 0, 1, 0, 3, 132, 28, },
+	{ 2, 1, 0, 3, 132, 28, },
+	{ 1, 1, 0, 3, 132, 28, },
+	{ 0, 1, 0, 3, 136, 28, },
+	{ 2, 1, 0, 3, 136, 28, },
+	{ 1, 1, 0, 3, 136, 28, },
+	{ 0, 1, 0, 3, 140, 24, },
+	{ 2, 1, 0, 3, 140, 28, },
+	{ 1, 1, 0, 3, 140, 28, },
+	{ 0, 1, 0, 3, 149, 24, },
+	{ 2, 1, 0, 3, 149, 28, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 28, },
+	{ 2, 1, 0, 3, 153, 28, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 28, },
+	{ 2, 1, 0, 3, 157, 28, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 28, },
+	{ 2, 1, 0, 3, 161, 28, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 28, },
+	{ 2, 1, 0, 3, 165, 28, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 22, },
+	{ 2, 1, 0, 6, 36, 26, },
+	{ 1, 1, 0, 6, 36, 26, },
+	{ 0, 1, 0, 6, 40, 26, },
+	{ 2, 1, 0, 6, 40, 26, },
+	{ 1, 1, 0, 6, 40, 26, },
+	{ 0, 1, 0, 6, 44, 26, },
+	{ 2, 1, 0, 6, 44, 26, },
+	{ 1, 1, 0, 6, 44, 26, },
+	{ 0, 1, 0, 6, 48, 26, },
+	{ 2, 1, 0, 6, 48, 26, },
+	{ 1, 1, 0, 6, 48, 26, },
+	{ 0, 1, 0, 6, 52, 26, },
+	{ 2, 1, 0, 6, 52, 26, },
+	{ 1, 1, 0, 6, 52, 26, },
+	{ 0, 1, 0, 6, 56, 26, },
+	{ 2, 1, 0, 6, 56, 26, },
+	{ 1, 1, 0, 6, 56, 26, },
+	{ 0, 1, 0, 6, 60, 26, },
+	{ 2, 1, 0, 6, 60, 26, },
+	{ 1, 1, 0, 6, 60, 26, },
+	{ 0, 1, 0, 6, 64, 22, },
+	{ 2, 1, 0, 6, 64, 26, },
+	{ 1, 1, 0, 6, 64, 26, },
+	{ 0, 1, 0, 6, 100, 22, },
+	{ 2, 1, 0, 6, 100, 26, },
+	{ 1, 1, 0, 6, 100, 26, },
+	{ 0, 1, 0, 6, 104, 26, },
+	{ 2, 1, 0, 6, 104, 26, },
+	{ 1, 1, 0, 6, 104, 26, },
+	{ 0, 1, 0, 6, 108, 26, },
+	{ 2, 1, 0, 6, 108, 26, },
+	{ 1, 1, 0, 6, 108, 26, },
+	{ 0, 1, 0, 6, 112, 26, },
+	{ 2, 1, 0, 6, 112, 26, },
+	{ 1, 1, 0, 6, 112, 26, },
+	{ 0, 1, 0, 6, 116, 26, },
+	{ 2, 1, 0, 6, 116, 26, },
+	{ 1, 1, 0, 6, 116, 26, },
+	{ 0, 1, 0, 6, 120, 26, },
+	{ 2, 1, 0, 6, 120, 26, },
+	{ 1, 1, 0, 6, 120, 26, },
+	{ 0, 1, 0, 6, 124, 26, },
+	{ 2, 1, 0, 6, 124, 26, },
+	{ 1, 1, 0, 6, 124, 26, },
+	{ 0, 1, 0, 6, 128, 26, },
+	{ 2, 1, 0, 6, 128, 26, },
+	{ 1, 1, 0, 6, 128, 26, },
+	{ 0, 1, 0, 6, 132, 26, },
+	{ 2, 1, 0, 6, 132, 26, },
+	{ 1, 1, 0, 6, 132, 26, },
+	{ 0, 1, 0, 6, 136, 26, },
+	{ 2, 1, 0, 6, 136, 26, },
+	{ 1, 1, 0, 6, 136, 26, },
+	{ 0, 1, 0, 6, 140, 22, },
+	{ 2, 1, 0, 6, 140, 26, },
+	{ 1, 1, 0, 6, 140, 26, },
+	{ 0, 1, 0, 6, 149, 22, },
+	{ 2, 1, 0, 6, 149, 26, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 26, },
+	{ 2, 1, 0, 6, 153, 26, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 26, },
+	{ 2, 1, 0, 6, 157, 26, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 26, },
+	{ 2, 1, 0, 6, 161, 26, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 26, },
+	{ 2, 1, 0, 6, 165, 26, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 20, },
+	{ 2, 1, 0, 7, 36, 24, },
+	{ 1, 1, 0, 7, 36, 24, },
+	{ 0, 1, 0, 7, 40, 24, },
+	{ 2, 1, 0, 7, 40, 24, },
+	{ 1, 1, 0, 7, 40, 24, },
+	{ 0, 1, 0, 7, 44, 24, },
+	{ 2, 1, 0, 7, 44, 24, },
+	{ 1, 1, 0, 7, 44, 24, },
+	{ 0, 1, 0, 7, 48, 24, },
+	{ 2, 1, 0, 7, 48, 24, },
+	{ 1, 1, 0, 7, 48, 24, },
+	{ 0, 1, 0, 7, 52, 24, },
+	{ 2, 1, 0, 7, 52, 24, },
+	{ 1, 1, 0, 7, 52, 24, },
+	{ 0, 1, 0, 7, 56, 24, },
+	{ 2, 1, 0, 7, 56, 24, },
+	{ 1, 1, 0, 7, 56, 24, },
+	{ 0, 1, 0, 7, 60, 24, },
+	{ 2, 1, 0, 7, 60, 24, },
+	{ 1, 1, 0, 7, 60, 24, },
+	{ 0, 1, 0, 7, 64, 20, },
+	{ 2, 1, 0, 7, 64, 24, },
+	{ 1, 1, 0, 7, 64, 24, },
+	{ 0, 1, 0, 7, 100, 20, },
+	{ 2, 1, 0, 7, 100, 24, },
+	{ 1, 1, 0, 7, 100, 24, },
+	{ 0, 1, 0, 7, 104, 24, },
+	{ 2, 1, 0, 7, 104, 24, },
+	{ 1, 1, 0, 7, 104, 24, },
+	{ 0, 1, 0, 7, 108, 24, },
+	{ 2, 1, 0, 7, 108, 24, },
+	{ 1, 1, 0, 7, 108, 24, },
+	{ 0, 1, 0, 7, 112, 24, },
+	{ 2, 1, 0, 7, 112, 24, },
+	{ 1, 1, 0, 7, 112, 24, },
+	{ 0, 1, 0, 7, 116, 24, },
+	{ 2, 1, 0, 7, 116, 24, },
+	{ 1, 1, 0, 7, 116, 24, },
+	{ 0, 1, 0, 7, 120, 24, },
+	{ 2, 1, 0, 7, 120, 24, },
+	{ 1, 1, 0, 7, 120, 24, },
+	{ 0, 1, 0, 7, 124, 24, },
+	{ 2, 1, 0, 7, 124, 24, },
+	{ 1, 1, 0, 7, 124, 24, },
+	{ 0, 1, 0, 7, 128, 24, },
+	{ 2, 1, 0, 7, 128, 24, },
+	{ 1, 1, 0, 7, 128, 24, },
+	{ 0, 1, 0, 7, 132, 24, },
+	{ 2, 1, 0, 7, 132, 24, },
+	{ 1, 1, 0, 7, 132, 24, },
+	{ 0, 1, 0, 7, 136, 24, },
+	{ 2, 1, 0, 7, 136, 24, },
+	{ 1, 1, 0, 7, 136, 24, },
+	{ 0, 1, 0, 7, 140, 20, },
+	{ 2, 1, 0, 7, 140, 24, },
+	{ 1, 1, 0, 7, 140, 24, },
+	{ 0, 1, 0, 7, 149, 20, },
+	{ 2, 1, 0, 7, 149, 24, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 24, },
+	{ 2, 1, 0, 7, 153, 24, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 24, },
+	{ 2, 1, 0, 7, 157, 24, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 24, },
+	{ 2, 1, 0, 7, 161, 24, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 24, },
+	{ 2, 1, 0, 7, 165, 24, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 26, },
+	{ 2, 1, 1, 2, 38, 32, },
+	{ 1, 1, 1, 2, 38, 32, },
+	{ 0, 1, 1, 2, 46, 32, },
+	{ 2, 1, 1, 2, 46, 32, },
+	{ 1, 1, 1, 2, 46, 32, },
+	{ 0, 1, 1, 2, 54, 32, },
+	{ 2, 1, 1, 2, 54, 32, },
+	{ 1, 1, 1, 2, 54, 32, },
+	{ 0, 1, 1, 2, 62, 26, },
+	{ 2, 1, 1, 2, 62, 32, },
+	{ 1, 1, 1, 2, 62, 32, },
+	{ 0, 1, 1, 2, 102, 26, },
+	{ 2, 1, 1, 2, 102, 32, },
+	{ 1, 1, 1, 2, 102, 32, },
+	{ 0, 1, 1, 2, 110, 32, },
+	{ 2, 1, 1, 2, 110, 32, },
+	{ 1, 1, 1, 2, 110, 32, },
+	{ 0, 1, 1, 2, 118, 32, },
+	{ 2, 1, 1, 2, 118, 32, },
+	{ 1, 1, 1, 2, 118, 32, },
+	{ 0, 1, 1, 2, 126, 32, },
+	{ 2, 1, 1, 2, 126, 32, },
+	{ 1, 1, 1, 2, 126, 32, },
+	{ 0, 1, 1, 2, 134, 32, },
+	{ 2, 1, 1, 2, 134, 32, },
+	{ 1, 1, 1, 2, 134, 32, },
+	{ 0, 1, 1, 2, 151, 26, },
+	{ 2, 1, 1, 2, 151, 32, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 32, },
+	{ 2, 1, 1, 2, 159, 32, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 24, },
+	{ 2, 1, 1, 3, 38, 28, },
+	{ 1, 1, 1, 3, 38, 28, },
+	{ 0, 1, 1, 3, 46, 28, },
+	{ 2, 1, 1, 3, 46, 28, },
+	{ 1, 1, 1, 3, 46, 28, },
+	{ 0, 1, 1, 3, 54, 28, },
+	{ 2, 1, 1, 3, 54, 28, },
+	{ 1, 1, 1, 3, 54, 28, },
+	{ 0, 1, 1, 3, 62, 24, },
+	{ 2, 1, 1, 3, 62, 28, },
+	{ 1, 1, 1, 3, 62, 28, },
+	{ 0, 1, 1, 3, 102, 24, },
+	{ 2, 1, 1, 3, 102, 28, },
+	{ 1, 1, 1, 3, 102, 28, },
+	{ 0, 1, 1, 3, 110, 28, },
+	{ 2, 1, 1, 3, 110, 28, },
+	{ 1, 1, 1, 3, 110, 28, },
+	{ 0, 1, 1, 3, 118, 28, },
+	{ 2, 1, 1, 3, 118, 28, },
+	{ 1, 1, 1, 3, 118, 28, },
+	{ 0, 1, 1, 3, 126, 28, },
+	{ 2, 1, 1, 3, 126, 28, },
+	{ 1, 1, 1, 3, 126, 28, },
+	{ 0, 1, 1, 3, 134, 28, },
+	{ 2, 1, 1, 3, 134, 28, },
+	{ 1, 1, 1, 3, 134, 28, },
+	{ 0, 1, 1, 3, 151, 24, },
+	{ 2, 1, 1, 3, 151, 28, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 28, },
+	{ 2, 1, 1, 3, 159, 28, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 20, },
+	{ 2, 1, 1, 6, 38, 26, },
+	{ 1, 1, 1, 6, 38, 26, },
+	{ 0, 1, 1, 6, 46, 26, },
+	{ 2, 1, 1, 6, 46, 26, },
+	{ 1, 1, 1, 6, 46, 26, },
+	{ 0, 1, 1, 6, 54, 26, },
+	{ 2, 1, 1, 6, 54, 26, },
+	{ 1, 1, 1, 6, 54, 26, },
+	{ 0, 1, 1, 6, 62, 20, },
+	{ 2, 1, 1, 6, 62, 26, },
+	{ 1, 1, 1, 6, 62, 26, },
+	{ 0, 1, 1, 6, 102, 20, },
+	{ 2, 1, 1, 6, 102, 26, },
+	{ 1, 1, 1, 6, 102, 26, },
+	{ 0, 1, 1, 6, 110, 26, },
+	{ 2, 1, 1, 6, 110, 26, },
+	{ 1, 1, 1, 6, 110, 26, },
+	{ 0, 1, 1, 6, 118, 26, },
+	{ 2, 1, 1, 6, 118, 26, },
+	{ 1, 1, 1, 6, 118, 26, },
+	{ 0, 1, 1, 6, 126, 26, },
+	{ 2, 1, 1, 6, 126, 26, },
+	{ 1, 1, 1, 6, 126, 26, },
+	{ 0, 1, 1, 6, 134, 26, },
+	{ 2, 1, 1, 6, 134, 26, },
+	{ 1, 1, 1, 6, 134, 26, },
+	{ 0, 1, 1, 6, 151, 20, },
+	{ 2, 1, 1, 6, 151, 26, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 26, },
+	{ 2, 1, 1, 6, 159, 26, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 18, },
+	{ 2, 1, 1, 7, 38, 24, },
+	{ 1, 1, 1, 7, 38, 24, },
+	{ 0, 1, 1, 7, 46, 24, },
+	{ 2, 1, 1, 7, 46, 24, },
+	{ 1, 1, 1, 7, 46, 24, },
+	{ 0, 1, 1, 7, 54, 24, },
+	{ 2, 1, 1, 7, 54, 24, },
+	{ 1, 1, 1, 7, 54, 24, },
+	{ 0, 1, 1, 7, 62, 18, },
+	{ 2, 1, 1, 7, 62, 24, },
+	{ 1, 1, 1, 7, 62, 24, },
+	{ 0, 1, 1, 7, 102, 18, },
+	{ 2, 1, 1, 7, 102, 24, },
+	{ 1, 1, 1, 7, 102, 24, },
+	{ 0, 1, 1, 7, 110, 24, },
+	{ 2, 1, 1, 7, 110, 24, },
+	{ 1, 1, 1, 7, 110, 24, },
+	{ 0, 1, 1, 7, 118, 24, },
+	{ 2, 1, 1, 7, 118, 24, },
+	{ 1, 1, 1, 7, 118, 24, },
+	{ 0, 1, 1, 7, 126, 24, },
+	{ 2, 1, 1, 7, 126, 24, },
+	{ 1, 1, 1, 7, 126, 24, },
+	{ 0, 1, 1, 7, 134, 24, },
+	{ 2, 1, 1, 7, 134, 24, },
+	{ 1, 1, 1, 7, 134, 24, },
+	{ 0, 1, 1, 7, 151, 18, },
+	{ 2, 1, 1, 7, 151, 24, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 24, },
+	{ 2, 1, 1, 7, 159, 24, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 22, },
+	{ 2, 1, 2, 4, 42, 30, },
+	{ 1, 1, 2, 4, 42, 30, },
+	{ 0, 1, 2, 4, 58, 22, },
+	{ 2, 1, 2, 4, 58, 30, },
+	{ 1, 1, 2, 4, 58, 30, },
+	{ 0, 1, 2, 4, 106, 22, },
+	{ 2, 1, 2, 4, 106, 30, },
+	{ 1, 1, 2, 4, 106, 30, },
+	{ 0, 1, 2, 4, 122, 30, },
+	{ 2, 1, 2, 4, 122, 30, },
+	{ 1, 1, 2, 4, 122, 30, },
+	{ 0, 1, 2, 4, 155, 22, },
+	{ 2, 1, 2, 4, 155, 30, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 20, },
+	{ 2, 1, 2, 5, 42, 28, },
+	{ 1, 1, 2, 5, 42, 28, },
+	{ 0, 1, 2, 5, 58, 20, },
+	{ 2, 1, 2, 5, 58, 28, },
+	{ 1, 1, 2, 5, 58, 28, },
+	{ 0, 1, 2, 5, 106, 20, },
+	{ 2, 1, 2, 5, 106, 28, },
+	{ 1, 1, 2, 5, 106, 28, },
+	{ 0, 1, 2, 5, 122, 28, },
+	{ 2, 1, 2, 5, 122, 28, },
+	{ 1, 1, 2, 5, 122, 28, },
+	{ 0, 1, 2, 5, 155, 20, },
+	{ 2, 1, 2, 5, 155, 28, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 18, },
+	{ 2, 1, 2, 8, 42, 26, },
+	{ 1, 1, 2, 8, 42, 26, },
+	{ 0, 1, 2, 8, 58, 18, },
+	{ 2, 1, 2, 8, 58, 26, },
+	{ 1, 1, 2, 8, 58, 26, },
+	{ 0, 1, 2, 8, 106, 18, },
+	{ 2, 1, 2, 8, 106, 26, },
+	{ 1, 1, 2, 8, 106, 26, },
+	{ 0, 1, 2, 8, 122, 26, },
+	{ 2, 1, 2, 8, 122, 26, },
+	{ 1, 1, 2, 8, 122, 26, },
+	{ 0, 1, 2, 8, 155, 18, },
+	{ 2, 1, 2, 8, 155, 26, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 16, },
+	{ 2, 1, 2, 9, 42, 24, },
+	{ 1, 1, 2, 9, 42, 24, },
+	{ 0, 1, 2, 9, 58, 16, },
+	{ 2, 1, 2, 9, 58, 24, },
+	{ 1, 1, 2, 9, 58, 24, },
+	{ 0, 1, 2, 9, 106, 16, },
+	{ 2, 1, 2, 9, 106, 24, },
+	{ 1, 1, 2, 9, 106, 24, },
+	{ 0, 1, 2, 9, 122, 24, },
+	{ 2, 1, 2, 9, 122, 24, },
+	{ 1, 1, 2, 9, 122, 24, },
+	{ 0, 1, 2, 9, 155, 16, },
+	{ 2, 1, 2, 9, 155, 24, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt_type0);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt_type1[] = {
+		{ 0, 0, 0, 0, 1, 34, },
+	{ 2, 0, 0, 0, 1, 32, },
+	{ 1, 0, 0, 0, 1, 32, },
+	{ 0, 0, 0, 0, 2, 34, },
+	{ 2, 0, 0, 0, 2, 32, },
+	{ 1, 0, 0, 0, 2, 32, },
+	{ 0, 0, 0, 0, 3, 34, },
+	{ 2, 0, 0, 0, 3, 32, },
+	{ 1, 0, 0, 0, 3, 32, },
+	{ 0, 0, 0, 0, 4, 34, },
+	{ 2, 0, 0, 0, 4, 32, },
+	{ 1, 0, 0, 0, 4, 32, },
+	{ 0, 0, 0, 0, 5, 34, },
+	{ 2, 0, 0, 0, 5, 32, },
+	{ 1, 0, 0, 0, 5, 32, },
+	{ 0, 0, 0, 0, 6, 34, },
+	{ 2, 0, 0, 0, 6, 32, },
+	{ 1, 0, 0, 0, 6, 32, },
+	{ 0, 0, 0, 0, 7, 34, },
+	{ 2, 0, 0, 0, 7, 32, },
+	{ 1, 0, 0, 0, 7, 32, },
+	{ 0, 0, 0, 0, 8, 34, },
+	{ 2, 0, 0, 0, 8, 32, },
+	{ 1, 0, 0, 0, 8, 32, },
+	{ 0, 0, 0, 0, 9, 34, },
+	{ 2, 0, 0, 0, 9, 32, },
+	{ 1, 0, 0, 0, 9, 32, },
+	{ 0, 0, 0, 0, 10, 34, },
+	{ 2, 0, 0, 0, 10, 32, },
+	{ 1, 0, 0, 0, 10, 32, },
+	{ 0, 0, 0, 0, 11, 34, },
+	{ 2, 0, 0, 0, 11, 32, },
+	{ 1, 0, 0, 0, 11, 32, },
+	{ 0, 0, 0, 0, 12, 24, },
+	{ 2, 0, 0, 0, 12, 32, },
+	{ 1, 0, 0, 0, 12, 32, },
+	{ 0, 0, 0, 0, 13, 16, },
+	{ 2, 0, 0, 0, 13, 32, },
+	{ 1, 0, 0, 0, 13, 32, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 32, },
+	{ 0, 0, 0, 1, 1, 30, },
+	{ 2, 0, 0, 1, 1, 32, },
+	{ 1, 0, 0, 1, 1, 32, },
+	{ 0, 0, 0, 1, 2, 32, },
+	{ 2, 0, 0, 1, 2, 32, },
+	{ 1, 0, 0, 1, 2, 32, },
+	{ 0, 0, 0, 1, 3, 32, },
+	{ 2, 0, 0, 1, 3, 32, },
+	{ 1, 0, 0, 1, 3, 32, },
+	{ 0, 0, 0, 1, 4, 32, },
+	{ 2, 0, 0, 1, 4, 32, },
+	{ 1, 0, 0, 1, 4, 32, },
+	{ 0, 0, 0, 1, 5, 32, },
+	{ 2, 0, 0, 1, 5, 32, },
+	{ 1, 0, 0, 1, 5, 32, },
+	{ 0, 0, 0, 1, 6, 32, },
+	{ 2, 0, 0, 1, 6, 32, },
+	{ 1, 0, 0, 1, 6, 32, },
+	{ 0, 0, 0, 1, 7, 32, },
+	{ 2, 0, 0, 1, 7, 32, },
+	{ 1, 0, 0, 1, 7, 32, },
+	{ 0, 0, 0, 1, 8, 32, },
+	{ 2, 0, 0, 1, 8, 32, },
+	{ 1, 0, 0, 1, 8, 32, },
+	{ 0, 0, 0, 1, 9, 32, },
+	{ 2, 0, 0, 1, 9, 32, },
+	{ 1, 0, 0, 1, 9, 32, },
+	{ 0, 0, 0, 1, 10, 32, },
+	{ 2, 0, 0, 1, 10, 32, },
+	{ 1, 0, 0, 1, 10, 32, },
+	{ 0, 0, 0, 1, 11, 30, },
+	{ 2, 0, 0, 1, 11, 32, },
+	{ 1, 0, 0, 1, 11, 32, },
+	{ 0, 0, 0, 1, 12, 18, },
+	{ 2, 0, 0, 1, 12, 32, },
+	{ 1, 0, 0, 1, 12, 32, },
+	{ 0, 0, 0, 1, 13, 8, },
+	{ 2, 0, 0, 1, 13, 32, },
+	{ 1, 0, 0, 1, 13, 32, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 63, },
+	{ 0, 0, 0, 2, 1, 28, },
+	{ 2, 0, 0, 2, 1, 32, },
+	{ 1, 0, 0, 2, 1, 32, },
+	{ 0, 0, 0, 2, 2, 32, },
+	{ 2, 0, 0, 2, 2, 32, },
+	{ 1, 0, 0, 2, 2, 32, },
+	{ 0, 0, 0, 2, 3, 32, },
+	{ 2, 0, 0, 2, 3, 32, },
+	{ 1, 0, 0, 2, 3, 32, },
+	{ 0, 0, 0, 2, 4, 32, },
+	{ 2, 0, 0, 2, 4, 32, },
+	{ 1, 0, 0, 2, 4, 32, },
+	{ 0, 0, 0, 2, 5, 32, },
+	{ 2, 0, 0, 2, 5, 32, },
+	{ 1, 0, 0, 2, 5, 32, },
+	{ 0, 0, 0, 2, 6, 32, },
+	{ 2, 0, 0, 2, 6, 32, },
+	{ 1, 0, 0, 2, 6, 32, },
+	{ 0, 0, 0, 2, 7, 32, },
+	{ 2, 0, 0, 2, 7, 32, },
+	{ 1, 0, 0, 2, 7, 32, },
+	{ 0, 0, 0, 2, 8, 32, },
+	{ 2, 0, 0, 2, 8, 32, },
+	{ 1, 0, 0, 2, 8, 32, },
+	{ 0, 0, 0, 2, 9, 32, },
+	{ 2, 0, 0, 2, 9, 32, },
+	{ 1, 0, 0, 2, 9, 32, },
+	{ 0, 0, 0, 2, 10, 32, },
+	{ 2, 0, 0, 2, 10, 32, },
+	{ 1, 0, 0, 2, 10, 32, },
+	{ 0, 0, 0, 2, 11, 28, },
+	{ 2, 0, 0, 2, 11, 32, },
+	{ 1, 0, 0, 2, 11, 32, },
+	{ 0, 0, 0, 2, 12, 16, },
+	{ 2, 0, 0, 2, 12, 32, },
+	{ 1, 0, 0, 2, 12, 32, },
+	{ 0, 0, 0, 2, 13, 6, },
+	{ 2, 0, 0, 2, 13, 32, },
+	{ 1, 0, 0, 2, 13, 32, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 63, },
+	{ 0, 0, 0, 3, 1, 26, },
+	{ 2, 0, 0, 3, 1, 30, },
+	{ 1, 0, 0, 3, 1, 30, },
+	{ 0, 0, 0, 3, 2, 30, },
+	{ 2, 0, 0, 3, 2, 30, },
+	{ 1, 0, 0, 3, 2, 30, },
+	{ 0, 0, 0, 3, 3, 30, },
+	{ 2, 0, 0, 3, 3, 30, },
+	{ 1, 0, 0, 3, 3, 30, },
+	{ 0, 0, 0, 3, 4, 30, },
+	{ 2, 0, 0, 3, 4, 30, },
+	{ 1, 0, 0, 3, 4, 30, },
+	{ 0, 0, 0, 3, 5, 30, },
+	{ 2, 0, 0, 3, 5, 30, },
+	{ 1, 0, 0, 3, 5, 30, },
+	{ 0, 0, 0, 3, 6, 30, },
+	{ 2, 0, 0, 3, 6, 30, },
+	{ 1, 0, 0, 3, 6, 30, },
+	{ 0, 0, 0, 3, 7, 30, },
+	{ 2, 0, 0, 3, 7, 30, },
+	{ 1, 0, 0, 3, 7, 30, },
+	{ 0, 0, 0, 3, 8, 30, },
+	{ 2, 0, 0, 3, 8, 30, },
+	{ 1, 0, 0, 3, 8, 30, },
+	{ 0, 0, 0, 3, 9, 30, },
+	{ 2, 0, 0, 3, 9, 30, },
+	{ 1, 0, 0, 3, 9, 30, },
+	{ 0, 0, 0, 3, 10, 30, },
+	{ 2, 0, 0, 3, 10, 30, },
+	{ 1, 0, 0, 3, 10, 30, },
+	{ 0, 0, 0, 3, 11, 26, },
+	{ 2, 0, 0, 3, 11, 30, },
+	{ 1, 0, 0, 3, 11, 30, },
+	{ 0, 0, 0, 3, 12, 14, },
+	{ 2, 0, 0, 3, 12, 30, },
+	{ 1, 0, 0, 3, 12, 30, },
+	{ 0, 0, 0, 3, 13, 4, },
+	{ 2, 0, 0, 3, 13, 30, },
+	{ 1, 0, 0, 3, 13, 30, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 63, },
+	{ 0, 0, 0, 6, 1, 24, },
+	{ 2, 0, 0, 6, 1, 28, },
+	{ 1, 0, 0, 6, 1, 28, },
+	{ 0, 0, 0, 6, 2, 28, },
+	{ 2, 0, 0, 6, 2, 28, },
+	{ 1, 0, 0, 6, 2, 28, },
+	{ 0, 0, 0, 6, 3, 28, },
+	{ 2, 0, 0, 6, 3, 28, },
+	{ 1, 0, 0, 6, 3, 28, },
+	{ 0, 0, 0, 6, 4, 28, },
+	{ 2, 0, 0, 6, 4, 28, },
+	{ 1, 0, 0, 6, 4, 28, },
+	{ 0, 0, 0, 6, 5, 28, },
+	{ 2, 0, 0, 6, 5, 28, },
+	{ 1, 0, 0, 6, 5, 28, },
+	{ 0, 0, 0, 6, 6, 28, },
+	{ 2, 0, 0, 6, 6, 28, },
+	{ 1, 0, 0, 6, 6, 28, },
+	{ 0, 0, 0, 6, 7, 28, },
+	{ 2, 0, 0, 6, 7, 28, },
+	{ 1, 0, 0, 6, 7, 28, },
+	{ 0, 0, 0, 6, 8, 28, },
+	{ 2, 0, 0, 6, 8, 28, },
+	{ 1, 0, 0, 6, 8, 28, },
+	{ 0, 0, 0, 6, 9, 28, },
+	{ 2, 0, 0, 6, 9, 28, },
+	{ 1, 0, 0, 6, 9, 28, },
+	{ 0, 0, 0, 6, 10, 28, },
+	{ 2, 0, 0, 6, 10, 28, },
+	{ 1, 0, 0, 6, 10, 28, },
+	{ 0, 0, 0, 6, 11, 24, },
+	{ 2, 0, 0, 6, 11, 28, },
+	{ 1, 0, 0, 6, 11, 28, },
+	{ 0, 0, 0, 6, 12, 14, },
+	{ 2, 0, 0, 6, 12, 28, },
+	{ 1, 0, 0, 6, 12, 28, },
+	{ 0, 0, 0, 6, 13, 4, },
+	{ 2, 0, 0, 6, 13, 28, },
+	{ 1, 0, 0, 6, 13, 28, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 63, },
+	{ 0, 0, 0, 7, 1, 22, },
+	{ 2, 0, 0, 7, 1, 26, },
+	{ 1, 0, 0, 7, 1, 26, },
+	{ 0, 0, 0, 7, 2, 26, },
+	{ 2, 0, 0, 7, 2, 26, },
+	{ 1, 0, 0, 7, 2, 26, },
+	{ 0, 0, 0, 7, 3, 26, },
+	{ 2, 0, 0, 7, 3, 26, },
+	{ 1, 0, 0, 7, 3, 26, },
+	{ 0, 0, 0, 7, 4, 26, },
+	{ 2, 0, 0, 7, 4, 26, },
+	{ 1, 0, 0, 7, 4, 26, },
+	{ 0, 0, 0, 7, 5, 26, },
+	{ 2, 0, 0, 7, 5, 26, },
+	{ 1, 0, 0, 7, 5, 26, },
+	{ 0, 0, 0, 7, 6, 26, },
+	{ 2, 0, 0, 7, 6, 26, },
+	{ 1, 0, 0, 7, 6, 26, },
+	{ 0, 0, 0, 7, 7, 26, },
+	{ 2, 0, 0, 7, 7, 26, },
+	{ 1, 0, 0, 7, 7, 26, },
+	{ 0, 0, 0, 7, 8, 26, },
+	{ 2, 0, 0, 7, 8, 26, },
+	{ 1, 0, 0, 7, 8, 26, },
+	{ 0, 0, 0, 7, 9, 26, },
+	{ 2, 0, 0, 7, 9, 26, },
+	{ 1, 0, 0, 7, 9, 26, },
+	{ 0, 0, 0, 7, 10, 26, },
+	{ 2, 0, 0, 7, 10, 26, },
+	{ 1, 0, 0, 7, 10, 26, },
+	{ 0, 0, 0, 7, 11, 22, },
+	{ 2, 0, 0, 7, 11, 26, },
+	{ 1, 0, 0, 7, 11, 26, },
+	{ 0, 0, 0, 7, 12, 14, },
+	{ 2, 0, 0, 7, 12, 26, },
+	{ 1, 0, 0, 7, 12, 26, },
+	{ 0, 0, 0, 7, 13, 4, },
+	{ 2, 0, 0, 7, 13, 26, },
+	{ 1, 0, 0, 7, 13, 26, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 63, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 28, },
+	{ 2, 0, 1, 2, 3, 32, },
+	{ 1, 0, 1, 2, 3, 32, },
+	{ 0, 0, 1, 2, 4, 32, },
+	{ 2, 0, 1, 2, 4, 32, },
+	{ 1, 0, 1, 2, 4, 32, },
+	{ 0, 0, 1, 2, 5, 32, },
+	{ 2, 0, 1, 2, 5, 32, },
+	{ 1, 0, 1, 2, 5, 32, },
+	{ 0, 0, 1, 2, 6, 32, },
+	{ 2, 0, 1, 2, 6, 32, },
+	{ 1, 0, 1, 2, 6, 32, },
+	{ 0, 0, 1, 2, 7, 32, },
+	{ 2, 0, 1, 2, 7, 32, },
+	{ 1, 0, 1, 2, 7, 32, },
+	{ 0, 0, 1, 2, 8, 32, },
+	{ 2, 0, 1, 2, 8, 32, },
+	{ 1, 0, 1, 2, 8, 32, },
+	{ 0, 0, 1, 2, 9, 32, },
+	{ 2, 0, 1, 2, 9, 32, },
+	{ 1, 0, 1, 2, 9, 32, },
+	{ 0, 0, 1, 2, 10, 32, },
+	{ 2, 0, 1, 2, 10, 32, },
+	{ 1, 0, 1, 2, 10, 32, },
+	{ 0, 0, 1, 2, 11, 28, },
+	{ 2, 0, 1, 2, 11, 32, },
+	{ 1, 0, 1, 2, 11, 32, },
+	{ 0, 0, 1, 2, 12, 16, },
+	{ 2, 0, 1, 2, 12, 32, },
+	{ 1, 0, 1, 2, 12, 32, },
+	{ 0, 0, 1, 2, 13, 10, },
+	{ 2, 0, 1, 2, 13, 32, },
+	{ 1, 0, 1, 2, 13, 32, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 26, },
+	{ 2, 0, 1, 3, 3, 30, },
+	{ 1, 0, 1, 3, 3, 30, },
+	{ 0, 0, 1, 3, 4, 30, },
+	{ 2, 0, 1, 3, 4, 30, },
+	{ 1, 0, 1, 3, 4, 30, },
+	{ 0, 0, 1, 3, 5, 30, },
+	{ 2, 0, 1, 3, 5, 30, },
+	{ 1, 0, 1, 3, 5, 30, },
+	{ 0, 0, 1, 3, 6, 30, },
+	{ 2, 0, 1, 3, 6, 30, },
+	{ 1, 0, 1, 3, 6, 30, },
+	{ 0, 0, 1, 3, 7, 30, },
+	{ 2, 0, 1, 3, 7, 30, },
+	{ 1, 0, 1, 3, 7, 30, },
+	{ 0, 0, 1, 3, 8, 30, },
+	{ 2, 0, 1, 3, 8, 30, },
+	{ 1, 0, 1, 3, 8, 30, },
+	{ 0, 0, 1, 3, 9, 30, },
+	{ 2, 0, 1, 3, 9, 30, },
+	{ 1, 0, 1, 3, 9, 30, },
+	{ 0, 0, 1, 3, 10, 30, },
+	{ 2, 0, 1, 3, 10, 30, },
+	{ 1, 0, 1, 3, 10, 30, },
+	{ 0, 0, 1, 3, 11, 26, },
+	{ 2, 0, 1, 3, 11, 30, },
+	{ 1, 0, 1, 3, 11, 30, },
+	{ 0, 0, 1, 3, 12, 14, },
+	{ 2, 0, 1, 3, 12, 30, },
+	{ 1, 0, 1, 3, 12, 30, },
+	{ 0, 0, 1, 3, 13, 8, },
+	{ 2, 0, 1, 3, 13, 30, },
+	{ 1, 0, 1, 3, 13, 30, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 24, },
+	{ 2, 0, 1, 6, 3, 28, },
+	{ 1, 0, 1, 6, 3, 28, },
+	{ 0, 0, 1, 6, 4, 28, },
+	{ 2, 0, 1, 6, 4, 28, },
+	{ 1, 0, 1, 6, 4, 28, },
+	{ 0, 0, 1, 6, 5, 28, },
+	{ 2, 0, 1, 6, 5, 28, },
+	{ 1, 0, 1, 6, 5, 28, },
+	{ 0, 0, 1, 6, 6, 28, },
+	{ 2, 0, 1, 6, 6, 28, },
+	{ 1, 0, 1, 6, 6, 28, },
+	{ 0, 0, 1, 6, 7, 28, },
+	{ 2, 0, 1, 6, 7, 28, },
+	{ 1, 0, 1, 6, 7, 28, },
+	{ 0, 0, 1, 6, 8, 28, },
+	{ 2, 0, 1, 6, 8, 28, },
+	{ 1, 0, 1, 6, 8, 28, },
+	{ 0, 0, 1, 6, 9, 28, },
+	{ 2, 0, 1, 6, 9, 28, },
+	{ 1, 0, 1, 6, 9, 28, },
+	{ 0, 0, 1, 6, 10, 28, },
+	{ 2, 0, 1, 6, 10, 28, },
+	{ 1, 0, 1, 6, 10, 28, },
+	{ 0, 0, 1, 6, 11, 24, },
+	{ 2, 0, 1, 6, 11, 28, },
+	{ 1, 0, 1, 6, 11, 28, },
+	{ 0, 0, 1, 6, 12, 14, },
+	{ 2, 0, 1, 6, 12, 28, },
+	{ 1, 0, 1, 6, 12, 28, },
+	{ 0, 0, 1, 6, 13, 8, },
+	{ 2, 0, 1, 6, 13, 28, },
+	{ 1, 0, 1, 6, 13, 28, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 22, },
+	{ 2, 0, 1, 7, 3, 26, },
+	{ 1, 0, 1, 7, 3, 26, },
+	{ 0, 0, 1, 7, 4, 26, },
+	{ 2, 0, 1, 7, 4, 26, },
+	{ 1, 0, 1, 7, 4, 26, },
+	{ 0, 0, 1, 7, 5, 26, },
+	{ 2, 0, 1, 7, 5, 26, },
+	{ 1, 0, 1, 7, 5, 26, },
+	{ 0, 0, 1, 7, 6, 26, },
+	{ 2, 0, 1, 7, 6, 26, },
+	{ 1, 0, 1, 7, 6, 26, },
+	{ 0, 0, 1, 7, 7, 26, },
+	{ 2, 0, 1, 7, 7, 26, },
+	{ 1, 0, 1, 7, 7, 26, },
+	{ 0, 0, 1, 7, 8, 26, },
+	{ 2, 0, 1, 7, 8, 26, },
+	{ 1, 0, 1, 7, 8, 26, },
+	{ 0, 0, 1, 7, 9, 26, },
+	{ 2, 0, 1, 7, 9, 26, },
+	{ 1, 0, 1, 7, 9, 26, },
+	{ 0, 0, 1, 7, 10, 26, },
+	{ 2, 0, 1, 7, 10, 26, },
+	{ 1, 0, 1, 7, 10, 26, },
+	{ 0, 0, 1, 7, 11, 22, },
+	{ 2, 0, 1, 7, 11, 26, },
+	{ 1, 0, 1, 7, 11, 26, },
+	{ 0, 0, 1, 7, 12, 14, },
+	{ 2, 0, 1, 7, 12, 26, },
+	{ 1, 0, 1, 7, 12, 26, },
+	{ 0, 0, 1, 7, 13, 8, },
+	{ 2, 0, 1, 7, 13, 26, },
+	{ 1, 0, 1, 7, 13, 26, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 28, },
+	{ 2, 1, 0, 1, 36, 32, },
+	{ 1, 1, 0, 1, 36, 32, },
+	{ 0, 1, 0, 1, 40, 32, },
+	{ 2, 1, 0, 1, 40, 32, },
+	{ 1, 1, 0, 1, 40, 32, },
+	{ 0, 1, 0, 1, 44, 32, },
+	{ 2, 1, 0, 1, 44, 32, },
+	{ 1, 1, 0, 1, 44, 32, },
+	{ 0, 1, 0, 1, 48, 32, },
+	{ 2, 1, 0, 1, 48, 32, },
+	{ 1, 1, 0, 1, 48, 32, },
+	{ 0, 1, 0, 1, 52, 32, },
+	{ 2, 1, 0, 1, 52, 32, },
+	{ 1, 1, 0, 1, 52, 32, },
+	{ 0, 1, 0, 1, 56, 32, },
+	{ 2, 1, 0, 1, 56, 32, },
+	{ 1, 1, 0, 1, 56, 32, },
+	{ 0, 1, 0, 1, 60, 32, },
+	{ 2, 1, 0, 1, 60, 32, },
+	{ 1, 1, 0, 1, 60, 32, },
+	{ 0, 1, 0, 1, 64, 30, },
+	{ 2, 1, 0, 1, 64, 32, },
+	{ 1, 1, 0, 1, 64, 32, },
+	{ 0, 1, 0, 1, 100, 28, },
+	{ 2, 1, 0, 1, 100, 32, },
+	{ 1, 1, 0, 1, 100, 32, },
+	{ 0, 1, 0, 1, 104, 32, },
+	{ 2, 1, 0, 1, 104, 32, },
+	{ 1, 1, 0, 1, 104, 32, },
+	{ 0, 1, 0, 1, 108, 32, },
+	{ 2, 1, 0, 1, 108, 32, },
+	{ 1, 1, 0, 1, 108, 32, },
+	{ 0, 1, 0, 1, 112, 32, },
+	{ 2, 1, 0, 1, 112, 32, },
+	{ 1, 1, 0, 1, 112, 32, },
+	{ 0, 1, 0, 1, 116, 32, },
+	{ 2, 1, 0, 1, 116, 32, },
+	{ 1, 1, 0, 1, 116, 32, },
+	{ 0, 1, 0, 1, 120, 32, },
+	{ 2, 1, 0, 1, 120, 32, },
+	{ 1, 1, 0, 1, 120, 32, },
+	{ 0, 1, 0, 1, 124, 32, },
+	{ 2, 1, 0, 1, 124, 32, },
+	{ 1, 1, 0, 1, 124, 32, },
+	{ 0, 1, 0, 1, 128, 32, },
+	{ 2, 1, 0, 1, 128, 32, },
+	{ 1, 1, 0, 1, 128, 32, },
+	{ 0, 1, 0, 1, 132, 32, },
+	{ 2, 1, 0, 1, 132, 32, },
+	{ 1, 1, 0, 1, 132, 32, },
+	{ 0, 1, 0, 1, 136, 32, },
+	{ 2, 1, 0, 1, 136, 32, },
+	{ 1, 1, 0, 1, 136, 32, },
+	{ 0, 1, 0, 1, 140, 28, },
+	{ 2, 1, 0, 1, 140, 32, },
+	{ 1, 1, 0, 1, 140, 32, },
+	{ 0, 1, 0, 1, 149, 28, },
+	{ 2, 1, 0, 1, 149, 32, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 32, },
+	{ 2, 1, 0, 1, 153, 32, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 32, },
+	{ 2, 1, 0, 1, 157, 32, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 32, },
+	{ 2, 1, 0, 1, 161, 32, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 32, },
+	{ 2, 1, 0, 1, 165, 32, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 28, },
+	{ 2, 1, 0, 2, 36, 32, },
+	{ 1, 1, 0, 2, 36, 32, },
+	{ 0, 1, 0, 2, 40, 32, },
+	{ 2, 1, 0, 2, 40, 32, },
+	{ 1, 1, 0, 2, 40, 32, },
+	{ 0, 1, 0, 2, 44, 32, },
+	{ 2, 1, 0, 2, 44, 32, },
+	{ 1, 1, 0, 2, 44, 32, },
+	{ 0, 1, 0, 2, 48, 32, },
+	{ 2, 1, 0, 2, 48, 32, },
+	{ 1, 1, 0, 2, 48, 32, },
+	{ 0, 1, 0, 2, 52, 32, },
+	{ 2, 1, 0, 2, 52, 32, },
+	{ 1, 1, 0, 2, 52, 32, },
+	{ 0, 1, 0, 2, 56, 32, },
+	{ 2, 1, 0, 2, 56, 32, },
+	{ 1, 1, 0, 2, 56, 32, },
+	{ 0, 1, 0, 2, 60, 32, },
+	{ 2, 1, 0, 2, 60, 32, },
+	{ 1, 1, 0, 2, 60, 32, },
+	{ 0, 1, 0, 2, 64, 30, },
+	{ 2, 1, 0, 2, 64, 32, },
+	{ 1, 1, 0, 2, 64, 32, },
+	{ 0, 1, 0, 2, 100, 28, },
+	{ 2, 1, 0, 2, 100, 32, },
+	{ 1, 1, 0, 2, 100, 32, },
+	{ 0, 1, 0, 2, 104, 32, },
+	{ 2, 1, 0, 2, 104, 32, },
+	{ 1, 1, 0, 2, 104, 32, },
+	{ 0, 1, 0, 2, 108, 32, },
+	{ 2, 1, 0, 2, 108, 32, },
+	{ 1, 1, 0, 2, 108, 32, },
+	{ 0, 1, 0, 2, 112, 32, },
+	{ 2, 1, 0, 2, 112, 32, },
+	{ 1, 1, 0, 2, 112, 32, },
+	{ 0, 1, 0, 2, 116, 32, },
+	{ 2, 1, 0, 2, 116, 32, },
+	{ 1, 1, 0, 2, 116, 32, },
+	{ 0, 1, 0, 2, 120, 32, },
+	{ 2, 1, 0, 2, 120, 32, },
+	{ 1, 1, 0, 2, 120, 32, },
+	{ 0, 1, 0, 2, 124, 32, },
+	{ 2, 1, 0, 2, 124, 32, },
+	{ 1, 1, 0, 2, 124, 32, },
+	{ 0, 1, 0, 2, 128, 32, },
+	{ 2, 1, 0, 2, 128, 32, },
+	{ 1, 1, 0, 2, 128, 32, },
+	{ 0, 1, 0, 2, 132, 32, },
+	{ 2, 1, 0, 2, 132, 32, },
+	{ 1, 1, 0, 2, 132, 32, },
+	{ 0, 1, 0, 2, 136, 32, },
+	{ 2, 1, 0, 2, 136, 32, },
+	{ 1, 1, 0, 2, 136, 32, },
+	{ 0, 1, 0, 2, 140, 28, },
+	{ 2, 1, 0, 2, 140, 32, },
+	{ 1, 1, 0, 2, 140, 32, },
+	{ 0, 1, 0, 2, 149, 28, },
+	{ 2, 1, 0, 2, 149, 32, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 32, },
+	{ 2, 1, 0, 2, 153, 32, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 32, },
+	{ 2, 1, 0, 2, 157, 32, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 32, },
+	{ 2, 1, 0, 2, 161, 32, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 32, },
+	{ 2, 1, 0, 2, 165, 32, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 26, },
+	{ 2, 1, 0, 3, 36, 30, },
+	{ 1, 1, 0, 3, 36, 30, },
+	{ 0, 1, 0, 3, 40, 30, },
+	{ 2, 1, 0, 3, 40, 30, },
+	{ 1, 1, 0, 3, 40, 30, },
+	{ 0, 1, 0, 3, 44, 30, },
+	{ 2, 1, 0, 3, 44, 30, },
+	{ 1, 1, 0, 3, 44, 30, },
+	{ 0, 1, 0, 3, 48, 30, },
+	{ 2, 1, 0, 3, 48, 30, },
+	{ 1, 1, 0, 3, 48, 30, },
+	{ 0, 1, 0, 3, 52, 30, },
+	{ 2, 1, 0, 3, 52, 30, },
+	{ 1, 1, 0, 3, 52, 30, },
+	{ 0, 1, 0, 3, 56, 30, },
+	{ 2, 1, 0, 3, 56, 30, },
+	{ 1, 1, 0, 3, 56, 30, },
+	{ 0, 1, 0, 3, 60, 30, },
+	{ 2, 1, 0, 3, 60, 30, },
+	{ 1, 1, 0, 3, 60, 30, },
+	{ 0, 1, 0, 3, 64, 28, },
+	{ 2, 1, 0, 3, 64, 30, },
+	{ 1, 1, 0, 3, 64, 30, },
+	{ 0, 1, 0, 3, 100, 28, },
+	{ 2, 1, 0, 3, 100, 30, },
+	{ 1, 1, 0, 3, 100, 30, },
+	{ 0, 1, 0, 3, 104, 30, },
+	{ 2, 1, 0, 3, 104, 30, },
+	{ 1, 1, 0, 3, 104, 30, },
+	{ 0, 1, 0, 3, 108, 30, },
+	{ 2, 1, 0, 3, 108, 30, },
+	{ 1, 1, 0, 3, 108, 30, },
+	{ 0, 1, 0, 3, 112, 30, },
+	{ 2, 1, 0, 3, 112, 30, },
+	{ 1, 1, 0, 3, 112, 30, },
+	{ 0, 1, 0, 3, 116, 30, },
+	{ 2, 1, 0, 3, 116, 30, },
+	{ 1, 1, 0, 3, 116, 30, },
+	{ 0, 1, 0, 3, 120, 30, },
+	{ 2, 1, 0, 3, 120, 30, },
+	{ 1, 1, 0, 3, 120, 30, },
+	{ 0, 1, 0, 3, 124, 30, },
+	{ 2, 1, 0, 3, 124, 30, },
+	{ 1, 1, 0, 3, 124, 30, },
+	{ 0, 1, 0, 3, 128, 30, },
+	{ 2, 1, 0, 3, 128, 30, },
+	{ 1, 1, 0, 3, 128, 30, },
+	{ 0, 1, 0, 3, 132, 30, },
+	{ 2, 1, 0, 3, 132, 30, },
+	{ 1, 1, 0, 3, 132, 30, },
+	{ 0, 1, 0, 3, 136, 30, },
+	{ 2, 1, 0, 3, 136, 30, },
+	{ 1, 1, 0, 3, 136, 30, },
+	{ 0, 1, 0, 3, 140, 26, },
+	{ 2, 1, 0, 3, 140, 30, },
+	{ 1, 1, 0, 3, 140, 30, },
+	{ 0, 1, 0, 3, 149, 26, },
+	{ 2, 1, 0, 3, 149, 30, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 30, },
+	{ 2, 1, 0, 3, 153, 30, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 30, },
+	{ 2, 1, 0, 3, 157, 30, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 30, },
+	{ 2, 1, 0, 3, 161, 30, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 30, },
+	{ 2, 1, 0, 3, 165, 30, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 24, },
+	{ 2, 1, 0, 6, 36, 28, },
+	{ 1, 1, 0, 6, 36, 28, },
+	{ 0, 1, 0, 6, 40, 28, },
+	{ 2, 1, 0, 6, 40, 28, },
+	{ 1, 1, 0, 6, 40, 28, },
+	{ 0, 1, 0, 6, 44, 28, },
+	{ 2, 1, 0, 6, 44, 28, },
+	{ 1, 1, 0, 6, 44, 28, },
+	{ 0, 1, 0, 6, 48, 28, },
+	{ 2, 1, 0, 6, 48, 28, },
+	{ 1, 1, 0, 6, 48, 28, },
+	{ 0, 1, 0, 6, 52, 28, },
+	{ 2, 1, 0, 6, 52, 28, },
+	{ 1, 1, 0, 6, 52, 28, },
+	{ 0, 1, 0, 6, 56, 28, },
+	{ 2, 1, 0, 6, 56, 28, },
+	{ 1, 1, 0, 6, 56, 28, },
+	{ 0, 1, 0, 6, 60, 28, },
+	{ 2, 1, 0, 6, 60, 28, },
+	{ 1, 1, 0, 6, 60, 28, },
+	{ 0, 1, 0, 6, 64, 26, },
+	{ 2, 1, 0, 6, 64, 28, },
+	{ 1, 1, 0, 6, 64, 28, },
+	{ 0, 1, 0, 6, 100, 24, },
+	{ 2, 1, 0, 6, 100, 28, },
+	{ 1, 1, 0, 6, 100, 28, },
+	{ 0, 1, 0, 6, 104, 28, },
+	{ 2, 1, 0, 6, 104, 28, },
+	{ 1, 1, 0, 6, 104, 28, },
+	{ 0, 1, 0, 6, 108, 28, },
+	{ 2, 1, 0, 6, 108, 28, },
+	{ 1, 1, 0, 6, 108, 28, },
+	{ 0, 1, 0, 6, 112, 28, },
+	{ 2, 1, 0, 6, 112, 28, },
+	{ 1, 1, 0, 6, 112, 28, },
+	{ 0, 1, 0, 6, 116, 28, },
+	{ 2, 1, 0, 6, 116, 28, },
+	{ 1, 1, 0, 6, 116, 28, },
+	{ 0, 1, 0, 6, 120, 28, },
+	{ 2, 1, 0, 6, 120, 28, },
+	{ 1, 1, 0, 6, 120, 28, },
+	{ 0, 1, 0, 6, 124, 28, },
+	{ 2, 1, 0, 6, 124, 28, },
+	{ 1, 1, 0, 6, 124, 28, },
+	{ 0, 1, 0, 6, 128, 28, },
+	{ 2, 1, 0, 6, 128, 28, },
+	{ 1, 1, 0, 6, 128, 28, },
+	{ 0, 1, 0, 6, 132, 28, },
+	{ 2, 1, 0, 6, 132, 28, },
+	{ 1, 1, 0, 6, 132, 28, },
+	{ 0, 1, 0, 6, 136, 28, },
+	{ 2, 1, 0, 6, 136, 28, },
+	{ 1, 1, 0, 6, 136, 28, },
+	{ 0, 1, 0, 6, 140, 24, },
+	{ 2, 1, 0, 6, 140, 28, },
+	{ 1, 1, 0, 6, 140, 28, },
+	{ 0, 1, 0, 6, 149, 24, },
+	{ 2, 1, 0, 6, 149, 28, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 28, },
+	{ 2, 1, 0, 6, 153, 28, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 28, },
+	{ 2, 1, 0, 6, 157, 28, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 28, },
+	{ 2, 1, 0, 6, 161, 28, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 28, },
+	{ 2, 1, 0, 6, 165, 28, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 22, },
+	{ 2, 1, 0, 7, 36, 26, },
+	{ 1, 1, 0, 7, 36, 26, },
+	{ 0, 1, 0, 7, 40, 26, },
+	{ 2, 1, 0, 7, 40, 26, },
+	{ 1, 1, 0, 7, 40, 26, },
+	{ 0, 1, 0, 7, 44, 26, },
+	{ 2, 1, 0, 7, 44, 26, },
+	{ 1, 1, 0, 7, 44, 26, },
+	{ 0, 1, 0, 7, 48, 26, },
+	{ 2, 1, 0, 7, 48, 26, },
+	{ 1, 1, 0, 7, 48, 26, },
+	{ 0, 1, 0, 7, 52, 26, },
+	{ 2, 1, 0, 7, 52, 26, },
+	{ 1, 1, 0, 7, 52, 26, },
+	{ 0, 1, 0, 7, 56, 26, },
+	{ 2, 1, 0, 7, 56, 26, },
+	{ 1, 1, 0, 7, 56, 26, },
+	{ 0, 1, 0, 7, 60, 26, },
+	{ 2, 1, 0, 7, 60, 26, },
+	{ 1, 1, 0, 7, 60, 26, },
+	{ 0, 1, 0, 7, 64, 24, },
+	{ 2, 1, 0, 7, 64, 26, },
+	{ 1, 1, 0, 7, 64, 26, },
+	{ 0, 1, 0, 7, 100, 22, },
+	{ 2, 1, 0, 7, 100, 26, },
+	{ 1, 1, 0, 7, 100, 26, },
+	{ 0, 1, 0, 7, 104, 26, },
+	{ 2, 1, 0, 7, 104, 26, },
+	{ 1, 1, 0, 7, 104, 26, },
+	{ 0, 1, 0, 7, 108, 26, },
+	{ 2, 1, 0, 7, 108, 26, },
+	{ 1, 1, 0, 7, 108, 26, },
+	{ 0, 1, 0, 7, 112, 26, },
+	{ 2, 1, 0, 7, 112, 26, },
+	{ 1, 1, 0, 7, 112, 26, },
+	{ 0, 1, 0, 7, 116, 26, },
+	{ 2, 1, 0, 7, 116, 26, },
+	{ 1, 1, 0, 7, 116, 26, },
+	{ 0, 1, 0, 7, 120, 26, },
+	{ 2, 1, 0, 7, 120, 26, },
+	{ 1, 1, 0, 7, 120, 26, },
+	{ 0, 1, 0, 7, 124, 26, },
+	{ 2, 1, 0, 7, 124, 26, },
+	{ 1, 1, 0, 7, 124, 26, },
+	{ 0, 1, 0, 7, 128, 26, },
+	{ 2, 1, 0, 7, 128, 26, },
+	{ 1, 1, 0, 7, 128, 26, },
+	{ 0, 1, 0, 7, 132, 26, },
+	{ 2, 1, 0, 7, 132, 26, },
+	{ 1, 1, 0, 7, 132, 26, },
+	{ 0, 1, 0, 7, 136, 26, },
+	{ 2, 1, 0, 7, 136, 26, },
+	{ 1, 1, 0, 7, 136, 26, },
+	{ 0, 1, 0, 7, 140, 22, },
+	{ 2, 1, 0, 7, 140, 26, },
+	{ 1, 1, 0, 7, 140, 26, },
+	{ 0, 1, 0, 7, 149, 22, },
+	{ 2, 1, 0, 7, 149, 26, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 26, },
+	{ 2, 1, 0, 7, 153, 26, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 26, },
+	{ 2, 1, 0, 7, 157, 26, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 26, },
+	{ 2, 1, 0, 7, 161, 26, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 26, },
+	{ 2, 1, 0, 7, 165, 26, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 28, },
+	{ 2, 1, 1, 2, 38, 32, },
+	{ 1, 1, 1, 2, 38, 32, },
+	{ 0, 1, 1, 2, 46, 32, },
+	{ 2, 1, 1, 2, 46, 32, },
+	{ 1, 1, 1, 2, 46, 32, },
+	{ 0, 1, 1, 2, 54, 32, },
+	{ 2, 1, 1, 2, 54, 32, },
+	{ 1, 1, 1, 2, 54, 32, },
+	{ 0, 1, 1, 2, 62, 28, },
+	{ 2, 1, 1, 2, 62, 32, },
+	{ 1, 1, 1, 2, 62, 32, },
+	{ 0, 1, 1, 2, 102, 28, },
+	{ 2, 1, 1, 2, 102, 32, },
+	{ 1, 1, 1, 2, 102, 32, },
+	{ 0, 1, 1, 2, 110, 32, },
+	{ 2, 1, 1, 2, 110, 32, },
+	{ 1, 1, 1, 2, 110, 32, },
+	{ 0, 1, 1, 2, 118, 32, },
+	{ 2, 1, 1, 2, 118, 32, },
+	{ 1, 1, 1, 2, 118, 32, },
+	{ 0, 1, 1, 2, 126, 32, },
+	{ 2, 1, 1, 2, 126, 32, },
+	{ 1, 1, 1, 2, 126, 32, },
+	{ 0, 1, 1, 2, 134, 30, },
+	{ 2, 1, 1, 2, 134, 32, },
+	{ 1, 1, 1, 2, 134, 32, },
+	{ 0, 1, 1, 2, 151, 28, },
+	{ 2, 1, 1, 2, 151, 32, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 32, },
+	{ 2, 1, 1, 2, 159, 32, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 26, },
+	{ 2, 1, 1, 3, 38, 30, },
+	{ 1, 1, 1, 3, 38, 30, },
+	{ 0, 1, 1, 3, 46, 30, },
+	{ 2, 1, 1, 3, 46, 30, },
+	{ 1, 1, 1, 3, 46, 30, },
+	{ 0, 1, 1, 3, 54, 30, },
+	{ 2, 1, 1, 3, 54, 30, },
+	{ 1, 1, 1, 3, 54, 30, },
+	{ 0, 1, 1, 3, 62, 26, },
+	{ 2, 1, 1, 3, 62, 30, },
+	{ 1, 1, 1, 3, 62, 30, },
+	{ 0, 1, 1, 3, 102, 26, },
+	{ 2, 1, 1, 3, 102, 30, },
+	{ 1, 1, 1, 3, 102, 30, },
+	{ 0, 1, 1, 3, 110, 30, },
+	{ 2, 1, 1, 3, 110, 30, },
+	{ 1, 1, 1, 3, 110, 30, },
+	{ 0, 1, 1, 3, 118, 30, },
+	{ 2, 1, 1, 3, 118, 30, },
+	{ 1, 1, 1, 3, 118, 30, },
+	{ 0, 1, 1, 3, 126, 30, },
+	{ 2, 1, 1, 3, 126, 30, },
+	{ 1, 1, 1, 3, 126, 30, },
+	{ 0, 1, 1, 3, 134, 28, },
+	{ 2, 1, 1, 3, 134, 30, },
+	{ 1, 1, 1, 3, 134, 30, },
+	{ 0, 1, 1, 3, 151, 26, },
+	{ 2, 1, 1, 3, 151, 30, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 30, },
+	{ 2, 1, 1, 3, 159, 30, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 20, },
+	{ 2, 1, 1, 6, 38, 28, },
+	{ 1, 1, 1, 6, 38, 28, },
+	{ 0, 1, 1, 6, 46, 28, },
+	{ 2, 1, 1, 6, 46, 28, },
+	{ 1, 1, 1, 6, 46, 28, },
+	{ 0, 1, 1, 6, 54, 28, },
+	{ 2, 1, 1, 6, 54, 28, },
+	{ 1, 1, 1, 6, 54, 28, },
+	{ 0, 1, 1, 6, 62, 20, },
+	{ 2, 1, 1, 6, 62, 28, },
+	{ 1, 1, 1, 6, 62, 28, },
+	{ 0, 1, 1, 6, 102, 22, },
+	{ 2, 1, 1, 6, 102, 28, },
+	{ 1, 1, 1, 6, 102, 28, },
+	{ 0, 1, 1, 6, 110, 28, },
+	{ 2, 1, 1, 6, 110, 28, },
+	{ 1, 1, 1, 6, 110, 28, },
+	{ 0, 1, 1, 6, 118, 28, },
+	{ 2, 1, 1, 6, 118, 28, },
+	{ 1, 1, 1, 6, 118, 28, },
+	{ 0, 1, 1, 6, 126, 28, },
+	{ 2, 1, 1, 6, 126, 28, },
+	{ 1, 1, 1, 6, 126, 28, },
+	{ 0, 1, 1, 6, 134, 26, },
+	{ 2, 1, 1, 6, 134, 28, },
+	{ 1, 1, 1, 6, 134, 28, },
+	{ 0, 1, 1, 6, 151, 22, },
+	{ 2, 1, 1, 6, 151, 28, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 28, },
+	{ 2, 1, 1, 6, 159, 28, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 18, },
+	{ 2, 1, 1, 7, 38, 26, },
+	{ 1, 1, 1, 7, 38, 26, },
+	{ 0, 1, 1, 7, 46, 26, },
+	{ 2, 1, 1, 7, 46, 26, },
+	{ 1, 1, 1, 7, 46, 26, },
+	{ 0, 1, 1, 7, 54, 26, },
+	{ 2, 1, 1, 7, 54, 26, },
+	{ 1, 1, 1, 7, 54, 26, },
+	{ 0, 1, 1, 7, 62, 18, },
+	{ 2, 1, 1, 7, 62, 26, },
+	{ 1, 1, 1, 7, 62, 26, },
+	{ 0, 1, 1, 7, 102, 20, },
+	{ 2, 1, 1, 7, 102, 26, },
+	{ 1, 1, 1, 7, 102, 26, },
+	{ 0, 1, 1, 7, 110, 26, },
+	{ 2, 1, 1, 7, 110, 26, },
+	{ 1, 1, 1, 7, 110, 26, },
+	{ 0, 1, 1, 7, 118, 26, },
+	{ 2, 1, 1, 7, 118, 26, },
+	{ 1, 1, 1, 7, 118, 26, },
+	{ 0, 1, 1, 7, 126, 26, },
+	{ 2, 1, 1, 7, 126, 26, },
+	{ 1, 1, 1, 7, 126, 26, },
+	{ 0, 1, 1, 7, 134, 24, },
+	{ 2, 1, 1, 7, 134, 26, },
+	{ 1, 1, 1, 7, 134, 26, },
+	{ 0, 1, 1, 7, 151, 20, },
+	{ 2, 1, 1, 7, 151, 26, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 26, },
+	{ 2, 1, 1, 7, 159, 26, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 24, },
+	{ 2, 1, 2, 4, 42, 32, },
+	{ 1, 1, 2, 4, 42, 32, },
+	{ 0, 1, 2, 4, 58, 24, },
+	{ 2, 1, 2, 4, 58, 32, },
+	{ 1, 1, 2, 4, 58, 32, },
+	{ 0, 1, 2, 4, 106, 24, },
+	{ 2, 1, 2, 4, 106, 32, },
+	{ 1, 1, 2, 4, 106, 32, },
+	{ 0, 1, 2, 4, 122, 32, },
+	{ 2, 1, 2, 4, 122, 32, },
+	{ 1, 1, 2, 4, 122, 32, },
+	{ 0, 1, 2, 4, 155, 26, },
+	{ 2, 1, 2, 4, 155, 32, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 22, },
+	{ 2, 1, 2, 5, 42, 30, },
+	{ 1, 1, 2, 5, 42, 30, },
+	{ 0, 1, 2, 5, 58, 22, },
+	{ 2, 1, 2, 5, 58, 30, },
+	{ 1, 1, 2, 5, 58, 30, },
+	{ 0, 1, 2, 5, 106, 22, },
+	{ 2, 1, 2, 5, 106, 30, },
+	{ 1, 1, 2, 5, 106, 30, },
+	{ 0, 1, 2, 5, 122, 30, },
+	{ 2, 1, 2, 5, 122, 30, },
+	{ 1, 1, 2, 5, 122, 30, },
+	{ 0, 1, 2, 5, 155, 24, },
+	{ 2, 1, 2, 5, 155, 30, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 20, },
+	{ 2, 1, 2, 8, 42, 28, },
+	{ 1, 1, 2, 8, 42, 28, },
+	{ 0, 1, 2, 8, 58, 20, },
+	{ 2, 1, 2, 8, 58, 28, },
+	{ 1, 1, 2, 8, 58, 28, },
+	{ 0, 1, 2, 8, 106, 20, },
+	{ 2, 1, 2, 8, 106, 28, },
+	{ 1, 1, 2, 8, 106, 28, },
+	{ 0, 1, 2, 8, 122, 28, },
+	{ 2, 1, 2, 8, 122, 28, },
+	{ 1, 1, 2, 8, 122, 28, },
+	{ 0, 1, 2, 8, 155, 20, },
+	{ 2, 1, 2, 8, 155, 28, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 18, },
+	{ 2, 1, 2, 9, 42, 26, },
+	{ 1, 1, 2, 9, 42, 26, },
+	{ 0, 1, 2, 9, 58, 18, },
+	{ 2, 1, 2, 9, 58, 26, },
+	{ 1, 1, 2, 9, 58, 26, },
+	{ 0, 1, 2, 9, 106, 18, },
+	{ 2, 1, 2, 9, 106, 26, },
+	{ 1, 1, 2, 9, 106, 26, },
+	{ 0, 1, 2, 9, 122, 26, },
+	{ 2, 1, 2, 9, 122, 26, },
+	{ 1, 1, 2, 9, 122, 26, },
+	{ 0, 1, 2, 9, 155, 18, },
+	{ 2, 1, 2, 9, 155, 26, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt_type1);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt_type2[] = {
+	{ 0, 0, 0, 0, 1, 42, },
+	{ 2, 0, 0, 0, 1, 42, },
+	{ 1, 0, 0, 0, 1, 42, },
+	{ 0, 0, 0, 0, 2, 50, },
+	{ 2, 0, 0, 0, 2, 42, },
+	{ 1, 0, 0, 0, 2, 42, },
+	{ 0, 0, 0, 0, 3, 50, },
+	{ 2, 0, 0, 0, 3, 42, },
+	{ 1, 0, 0, 0, 3, 42, },
+	{ 0, 0, 0, 0, 4, 50, },
+	{ 2, 0, 0, 0, 4, 42, },
+	{ 1, 0, 0, 0, 4, 42, },
+	{ 0, 0, 0, 0, 5, 50, },
+	{ 2, 0, 0, 0, 5, 42, },
+	{ 1, 0, 0, 0, 5, 42, },
+	{ 0, 0, 0, 0, 6, 50, },
+	{ 2, 0, 0, 0, 6, 42, },
+	{ 1, 0, 0, 0, 6, 42, },
+	{ 0, 0, 0, 0, 7, 50, },
+	{ 2, 0, 0, 0, 7, 42, },
+	{ 1, 0, 0, 0, 7, 42, },
+	{ 0, 0, 0, 0, 8, 50, },
+	{ 2, 0, 0, 0, 8, 42, },
+	{ 1, 0, 0, 0, 8, 42, },
+	{ 0, 0, 0, 0, 9, 50, },
+	{ 2, 0, 0, 0, 9, 42, },
+	{ 1, 0, 0, 0, 9, 42, },
+	{ 0, 0, 0, 0, 10, 50, },
+	{ 2, 0, 0, 0, 10, 42, },
+	{ 1, 0, 0, 0, 10, 42, },
+	{ 0, 0, 0, 0, 11, 44, },
+	{ 2, 0, 0, 0, 11, 42, },
+	{ 1, 0, 0, 0, 11, 42, },
+	{ 0, 0, 0, 0, 12, 63, },
+	{ 2, 0, 0, 0, 12, 42, },
+	{ 1, 0, 0, 0, 12, 42, },
+	{ 0, 0, 0, 0, 13, 63, },
+	{ 2, 0, 0, 0, 13, 42, },
+	{ 1, 0, 0, 0, 13, 42, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 42, },
+	{ 0, 0, 0, 1, 1, 32, },
+	{ 2, 0, 0, 1, 1, 42, },
+	{ 1, 0, 0, 1, 1, 42, },
+	{ 0, 0, 0, 1, 2, 42, },
+	{ 2, 0, 0, 1, 2, 42, },
+	{ 1, 0, 0, 1, 2, 42, },
+	{ 0, 0, 0, 1, 3, 42, },
+	{ 2, 0, 0, 1, 3, 42, },
+	{ 1, 0, 0, 1, 3, 42, },
+	{ 0, 0, 0, 1, 4, 42, },
+	{ 2, 0, 0, 1, 4, 42, },
+	{ 1, 0, 0, 1, 4, 42, },
+	{ 0, 0, 0, 1, 5, 42, },
+	{ 2, 0, 0, 1, 5, 42, },
+	{ 1, 0, 0, 1, 5, 42, },
+	{ 0, 0, 0, 1, 6, 42, },
+	{ 2, 0, 0, 1, 6, 42, },
+	{ 1, 0, 0, 1, 6, 42, },
+	{ 0, 0, 0, 1, 7, 42, },
+	{ 2, 0, 0, 1, 7, 42, },
+	{ 1, 0, 0, 1, 7, 42, },
+	{ 0, 0, 0, 1, 8, 42, },
+	{ 2, 0, 0, 1, 8, 42, },
+	{ 1, 0, 0, 1, 8, 42, },
+	{ 0, 0, 0, 1, 9, 42, },
+	{ 2, 0, 0, 1, 9, 42, },
+	{ 1, 0, 0, 1, 9, 42, },
+	{ 0, 0, 0, 1, 10, 42, },
+	{ 2, 0, 0, 1, 10, 42, },
+	{ 1, 0, 0, 1, 10, 42, },
+	{ 0, 0, 0, 1, 11, 32, },
+	{ 2, 0, 0, 1, 11, 42, },
+	{ 1, 0, 0, 1, 11, 42, },
+	{ 0, 0, 0, 1, 12, 63, },
+	{ 2, 0, 0, 1, 12, 42, },
+	{ 1, 0, 0, 1, 12, 42, },
+	{ 0, 0, 0, 1, 13, 63, },
+	{ 2, 0, 0, 1, 13, 42, },
+	{ 1, 0, 0, 1, 13, 42, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 63, },
+	{ 0, 0, 0, 2, 1, 32, },
+	{ 2, 0, 0, 2, 1, 42, },
+	{ 1, 0, 0, 2, 1, 42, },
+	{ 0, 0, 0, 2, 2, 40, },
+	{ 2, 0, 0, 2, 2, 42, },
+	{ 1, 0, 0, 2, 2, 42, },
+	{ 0, 0, 0, 2, 3, 40, },
+	{ 2, 0, 0, 2, 3, 42, },
+	{ 1, 0, 0, 2, 3, 42, },
+	{ 0, 0, 0, 2, 4, 40, },
+	{ 2, 0, 0, 2, 4, 42, },
+	{ 1, 0, 0, 2, 4, 42, },
+	{ 0, 0, 0, 2, 5, 40, },
+	{ 2, 0, 0, 2, 5, 42, },
+	{ 1, 0, 0, 2, 5, 42, },
+	{ 0, 0, 0, 2, 6, 40, },
+	{ 2, 0, 0, 2, 6, 42, },
+	{ 1, 0, 0, 2, 6, 42, },
+	{ 0, 0, 0, 2, 7, 40, },
+	{ 2, 0, 0, 2, 7, 42, },
+	{ 1, 0, 0, 2, 7, 42, },
+	{ 0, 0, 0, 2, 8, 40, },
+	{ 2, 0, 0, 2, 8, 42, },
+	{ 1, 0, 0, 2, 8, 42, },
+	{ 0, 0, 0, 2, 9, 40, },
+	{ 2, 0, 0, 2, 9, 42, },
+	{ 1, 0, 0, 2, 9, 42, },
+	{ 0, 0, 0, 2, 10, 40, },
+	{ 2, 0, 0, 2, 10, 42, },
+	{ 1, 0, 0, 2, 10, 42, },
+	{ 0, 0, 0, 2, 11, 28, },
+	{ 2, 0, 0, 2, 11, 42, },
+	{ 1, 0, 0, 2, 11, 42, },
+	{ 0, 0, 0, 2, 12, 63, },
+	{ 2, 0, 0, 2, 12, 42, },
+	{ 1, 0, 0, 2, 12, 42, },
+	{ 0, 0, 0, 2, 13, 63, },
+	{ 2, 0, 0, 2, 13, 42, },
+	{ 1, 0, 0, 2, 13, 42, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 63, },
+	{ 0, 0, 0, 3, 1, 32, },
+	{ 2, 0, 0, 3, 1, 40, },
+	{ 1, 0, 0, 3, 1, 40, },
+	{ 0, 0, 0, 3, 2, 40, },
+	{ 2, 0, 0, 3, 2, 40, },
+	{ 1, 0, 0, 3, 2, 40, },
+	{ 0, 0, 0, 3, 3, 40, },
+	{ 2, 0, 0, 3, 3, 40, },
+	{ 1, 0, 0, 3, 3, 40, },
+	{ 0, 0, 0, 3, 4, 40, },
+	{ 2, 0, 0, 3, 4, 40, },
+	{ 1, 0, 0, 3, 4, 40, },
+	{ 0, 0, 0, 3, 5, 40, },
+	{ 2, 0, 0, 3, 5, 40, },
+	{ 1, 0, 0, 3, 5, 40, },
+	{ 0, 0, 0, 3, 6, 40, },
+	{ 2, 0, 0, 3, 6, 40, },
+	{ 1, 0, 0, 3, 6, 40, },
+	{ 0, 0, 0, 3, 7, 40, },
+	{ 2, 0, 0, 3, 7, 40, },
+	{ 1, 0, 0, 3, 7, 40, },
+	{ 0, 0, 0, 3, 8, 40, },
+	{ 2, 0, 0, 3, 8, 40, },
+	{ 1, 0, 0, 3, 8, 40, },
+	{ 0, 0, 0, 3, 9, 40, },
+	{ 2, 0, 0, 3, 9, 40, },
+	{ 1, 0, 0, 3, 9, 40, },
+	{ 0, 0, 0, 3, 10, 40, },
+	{ 2, 0, 0, 3, 10, 40, },
+	{ 1, 0, 0, 3, 10, 40, },
+	{ 0, 0, 0, 3, 11, 28, },
+	{ 2, 0, 0, 3, 11, 40, },
+	{ 1, 0, 0, 3, 11, 40, },
+	{ 0, 0, 0, 3, 12, 63, },
+	{ 2, 0, 0, 3, 12, 40, },
+	{ 1, 0, 0, 3, 12, 40, },
+	{ 0, 0, 0, 3, 13, 63, },
+	{ 2, 0, 0, 3, 13, 40, },
+	{ 1, 0, 0, 3, 13, 40, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 63, },
+	{ 0, 0, 0, 6, 1, 32, },
+	{ 2, 0, 0, 6, 1, 40, },
+	{ 1, 0, 0, 6, 1, 40, },
+	{ 0, 0, 0, 6, 2, 40, },
+	{ 2, 0, 0, 6, 2, 40, },
+	{ 1, 0, 0, 6, 2, 40, },
+	{ 0, 0, 0, 6, 3, 40, },
+	{ 2, 0, 0, 6, 3, 40, },
+	{ 1, 0, 0, 6, 3, 40, },
+	{ 0, 0, 0, 6, 4, 40, },
+	{ 2, 0, 0, 6, 4, 40, },
+	{ 1, 0, 0, 6, 4, 40, },
+	{ 0, 0, 0, 6, 5, 40, },
+	{ 2, 0, 0, 6, 5, 40, },
+	{ 1, 0, 0, 6, 5, 40, },
+	{ 0, 0, 0, 6, 6, 40, },
+	{ 2, 0, 0, 6, 6, 40, },
+	{ 1, 0, 0, 6, 6, 40, },
+	{ 0, 0, 0, 6, 7, 40, },
+	{ 2, 0, 0, 6, 7, 40, },
+	{ 1, 0, 0, 6, 7, 40, },
+	{ 0, 0, 0, 6, 8, 40, },
+	{ 2, 0, 0, 6, 8, 40, },
+	{ 1, 0, 0, 6, 8, 40, },
+	{ 0, 0, 0, 6, 9, 40, },
+	{ 2, 0, 0, 6, 9, 40, },
+	{ 1, 0, 0, 6, 9, 40, },
+	{ 0, 0, 0, 6, 10, 40, },
+	{ 2, 0, 0, 6, 10, 40, },
+	{ 1, 0, 0, 6, 10, 40, },
+	{ 0, 0, 0, 6, 11, 28, },
+	{ 2, 0, 0, 6, 11, 40, },
+	{ 1, 0, 0, 6, 11, 40, },
+	{ 0, 0, 0, 6, 12, 63, },
+	{ 2, 0, 0, 6, 12, 40, },
+	{ 1, 0, 0, 6, 12, 40, },
+	{ 0, 0, 0, 6, 13, 63, },
+	{ 2, 0, 0, 6, 13, 40, },
+	{ 1, 0, 0, 6, 13, 40, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 63, },
+	{ 0, 0, 0, 7, 1, 32, },
+	{ 2, 0, 0, 7, 1, 40, },
+	{ 1, 0, 0, 7, 1, 40, },
+	{ 0, 0, 0, 7, 2, 40, },
+	{ 2, 0, 0, 7, 2, 40, },
+	{ 1, 0, 0, 7, 2, 40, },
+	{ 0, 0, 0, 7, 3, 40, },
+	{ 2, 0, 0, 7, 3, 40, },
+	{ 1, 0, 0, 7, 3, 40, },
+	{ 0, 0, 0, 7, 4, 40, },
+	{ 2, 0, 0, 7, 4, 40, },
+	{ 1, 0, 0, 7, 4, 40, },
+	{ 0, 0, 0, 7, 5, 40, },
+	{ 2, 0, 0, 7, 5, 40, },
+	{ 1, 0, 0, 7, 5, 40, },
+	{ 0, 0, 0, 7, 6, 40, },
+	{ 2, 0, 0, 7, 6, 40, },
+	{ 1, 0, 0, 7, 6, 40, },
+	{ 0, 0, 0, 7, 7, 40, },
+	{ 2, 0, 0, 7, 7, 40, },
+	{ 1, 0, 0, 7, 7, 40, },
+	{ 0, 0, 0, 7, 8, 40, },
+	{ 2, 0, 0, 7, 8, 40, },
+	{ 1, 0, 0, 7, 8, 40, },
+	{ 0, 0, 0, 7, 9, 40, },
+	{ 2, 0, 0, 7, 9, 40, },
+	{ 1, 0, 0, 7, 9, 40, },
+	{ 0, 0, 0, 7, 10, 40, },
+	{ 2, 0, 0, 7, 10, 40, },
+	{ 1, 0, 0, 7, 10, 40, },
+	{ 0, 0, 0, 7, 11, 28, },
+	{ 2, 0, 0, 7, 11, 40, },
+	{ 1, 0, 0, 7, 11, 40, },
+	{ 0, 0, 0, 7, 12, 63, },
+	{ 2, 0, 0, 7, 12, 40, },
+	{ 1, 0, 0, 7, 12, 40, },
+	{ 0, 0, 0, 7, 13, 63, },
+	{ 2, 0, 0, 7, 13, 40, },
+	{ 1, 0, 0, 7, 13, 40, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 63, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 30, },
+	{ 2, 0, 1, 2, 3, 34, },
+	{ 1, 0, 1, 2, 3, 34, },
+	{ 0, 0, 1, 2, 4, 34, },
+	{ 2, 0, 1, 2, 4, 34, },
+	{ 1, 0, 1, 2, 4, 34, },
+	{ 0, 0, 1, 2, 5, 34, },
+	{ 2, 0, 1, 2, 5, 34, },
+	{ 1, 0, 1, 2, 5, 34, },
+	{ 0, 0, 1, 2, 6, 34, },
+	{ 2, 0, 1, 2, 6, 34, },
+	{ 1, 0, 1, 2, 6, 34, },
+	{ 0, 0, 1, 2, 7, 34, },
+	{ 2, 0, 1, 2, 7, 34, },
+	{ 1, 0, 1, 2, 7, 34, },
+	{ 0, 0, 1, 2, 8, 34, },
+	{ 2, 0, 1, 2, 8, 34, },
+	{ 1, 0, 1, 2, 8, 34, },
+	{ 0, 0, 1, 2, 9, 34, },
+	{ 2, 0, 1, 2, 9, 34, },
+	{ 1, 0, 1, 2, 9, 34, },
+	{ 0, 0, 1, 2, 10, 34, },
+	{ 2, 0, 1, 2, 10, 34, },
+	{ 1, 0, 1, 2, 10, 34, },
+	{ 0, 0, 1, 2, 11, 28, },
+	{ 2, 0, 1, 2, 11, 34, },
+	{ 1, 0, 1, 2, 11, 34, },
+	{ 0, 0, 1, 2, 12, 63, },
+	{ 2, 0, 1, 2, 12, 34, },
+	{ 1, 0, 1, 2, 12, 34, },
+	{ 0, 0, 1, 2, 13, 63, },
+	{ 2, 0, 1, 2, 13, 34, },
+	{ 1, 0, 1, 2, 13, 34, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 30, },
+	{ 2, 0, 1, 3, 3, 34, },
+	{ 1, 0, 1, 3, 3, 34, },
+	{ 0, 0, 1, 3, 4, 34, },
+	{ 2, 0, 1, 3, 4, 34, },
+	{ 1, 0, 1, 3, 4, 34, },
+	{ 0, 0, 1, 3, 5, 34, },
+	{ 2, 0, 1, 3, 5, 34, },
+	{ 1, 0, 1, 3, 5, 34, },
+	{ 0, 0, 1, 3, 6, 34, },
+	{ 2, 0, 1, 3, 6, 34, },
+	{ 1, 0, 1, 3, 6, 34, },
+	{ 0, 0, 1, 3, 7, 34, },
+	{ 2, 0, 1, 3, 7, 34, },
+	{ 1, 0, 1, 3, 7, 34, },
+	{ 0, 0, 1, 3, 8, 34, },
+	{ 2, 0, 1, 3, 8, 34, },
+	{ 1, 0, 1, 3, 8, 34, },
+	{ 0, 0, 1, 3, 9, 34, },
+	{ 2, 0, 1, 3, 9, 34, },
+	{ 1, 0, 1, 3, 9, 34, },
+	{ 0, 0, 1, 3, 10, 34, },
+	{ 2, 0, 1, 3, 10, 34, },
+	{ 1, 0, 1, 3, 10, 34, },
+	{ 0, 0, 1, 3, 11, 28, },
+	{ 2, 0, 1, 3, 11, 34, },
+	{ 1, 0, 1, 3, 11, 34, },
+	{ 0, 0, 1, 3, 12, 63, },
+	{ 2, 0, 1, 3, 12, 34, },
+	{ 1, 0, 1, 3, 12, 34, },
+	{ 0, 0, 1, 3, 13, 63, },
+	{ 2, 0, 1, 3, 13, 34, },
+	{ 1, 0, 1, 3, 13, 34, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 30, },
+	{ 2, 0, 1, 6, 3, 34, },
+	{ 1, 0, 1, 6, 3, 34, },
+	{ 0, 0, 1, 6, 4, 34, },
+	{ 2, 0, 1, 6, 4, 34, },
+	{ 1, 0, 1, 6, 4, 34, },
+	{ 0, 0, 1, 6, 5, 34, },
+	{ 2, 0, 1, 6, 5, 34, },
+	{ 1, 0, 1, 6, 5, 34, },
+	{ 0, 0, 1, 6, 6, 34, },
+	{ 2, 0, 1, 6, 6, 34, },
+	{ 1, 0, 1, 6, 6, 34, },
+	{ 0, 0, 1, 6, 7, 34, },
+	{ 2, 0, 1, 6, 7, 34, },
+	{ 1, 0, 1, 6, 7, 34, },
+	{ 0, 0, 1, 6, 8, 34, },
+	{ 2, 0, 1, 6, 8, 34, },
+	{ 1, 0, 1, 6, 8, 34, },
+	{ 0, 0, 1, 6, 9, 34, },
+	{ 2, 0, 1, 6, 9, 34, },
+	{ 1, 0, 1, 6, 9, 34, },
+	{ 0, 0, 1, 6, 10, 34, },
+	{ 2, 0, 1, 6, 10, 34, },
+	{ 1, 0, 1, 6, 10, 34, },
+	{ 0, 0, 1, 6, 11, 28, },
+	{ 2, 0, 1, 6, 11, 34, },
+	{ 1, 0, 1, 6, 11, 34, },
+	{ 0, 0, 1, 6, 12, 63, },
+	{ 2, 0, 1, 6, 12, 34, },
+	{ 1, 0, 1, 6, 12, 34, },
+	{ 0, 0, 1, 6, 13, 63, },
+	{ 2, 0, 1, 6, 13, 34, },
+	{ 1, 0, 1, 6, 13, 34, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 30, },
+	{ 2, 0, 1, 7, 3, 34, },
+	{ 1, 0, 1, 7, 3, 34, },
+	{ 0, 0, 1, 7, 4, 34, },
+	{ 2, 0, 1, 7, 4, 34, },
+	{ 1, 0, 1, 7, 4, 34, },
+	{ 0, 0, 1, 7, 5, 34, },
+	{ 2, 0, 1, 7, 5, 34, },
+	{ 1, 0, 1, 7, 5, 34, },
+	{ 0, 0, 1, 7, 6, 34, },
+	{ 2, 0, 1, 7, 6, 34, },
+	{ 1, 0, 1, 7, 6, 34, },
+	{ 0, 0, 1, 7, 7, 34, },
+	{ 2, 0, 1, 7, 7, 34, },
+	{ 1, 0, 1, 7, 7, 34, },
+	{ 0, 0, 1, 7, 8, 34, },
+	{ 2, 0, 1, 7, 8, 34, },
+	{ 1, 0, 1, 7, 8, 34, },
+	{ 0, 0, 1, 7, 9, 34, },
+	{ 2, 0, 1, 7, 9, 34, },
+	{ 1, 0, 1, 7, 9, 34, },
+	{ 0, 0, 1, 7, 10, 34, },
+	{ 2, 0, 1, 7, 10, 34, },
+	{ 1, 0, 1, 7, 10, 34, },
+	{ 0, 0, 1, 7, 11, 28, },
+	{ 2, 0, 1, 7, 11, 34, },
+	{ 1, 0, 1, 7, 11, 34, },
+	{ 0, 0, 1, 7, 12, 63, },
+	{ 2, 0, 1, 7, 12, 34, },
+	{ 1, 0, 1, 7, 12, 34, },
+	{ 0, 0, 1, 7, 13, 63, },
+	{ 2, 0, 1, 7, 13, 34, },
+	{ 1, 0, 1, 7, 13, 34, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 42, },
+	{ 2, 1, 0, 1, 36, 42, },
+	{ 1, 1, 0, 1, 36, 42, },
+	{ 0, 1, 0, 1, 40, 42, },
+	{ 2, 1, 0, 1, 40, 42, },
+	{ 1, 1, 0, 1, 40, 42, },
+	{ 0, 1, 0, 1, 44, 42, },
+	{ 2, 1, 0, 1, 44, 42, },
+	{ 1, 1, 0, 1, 44, 42, },
+	{ 0, 1, 0, 1, 48, 42, },
+	{ 2, 1, 0, 1, 48, 42, },
+	{ 1, 1, 0, 1, 48, 42, },
+	{ 0, 1, 0, 1, 52, 42, },
+	{ 2, 1, 0, 1, 52, 42, },
+	{ 1, 1, 0, 1, 52, 42, },
+	{ 0, 1, 0, 1, 56, 42, },
+	{ 2, 1, 0, 1, 56, 42, },
+	{ 1, 1, 0, 1, 56, 42, },
+	{ 0, 1, 0, 1, 60, 42, },
+	{ 2, 1, 0, 1, 60, 42, },
+	{ 1, 1, 0, 1, 60, 42, },
+	{ 0, 1, 0, 1, 64, 42, },
+	{ 2, 1, 0, 1, 64, 42, },
+	{ 1, 1, 0, 1, 64, 42, },
+	{ 0, 1, 0, 1, 100, 42, },
+	{ 2, 1, 0, 1, 100, 42, },
+	{ 1, 1, 0, 1, 100, 42, },
+	{ 0, 1, 0, 1, 104, 42, },
+	{ 2, 1, 0, 1, 104, 42, },
+	{ 1, 1, 0, 1, 104, 42, },
+	{ 0, 1, 0, 1, 108, 42, },
+	{ 2, 1, 0, 1, 108, 42, },
+	{ 1, 1, 0, 1, 108, 42, },
+	{ 0, 1, 0, 1, 112, 42, },
+	{ 2, 1, 0, 1, 112, 42, },
+	{ 1, 1, 0, 1, 112, 42, },
+	{ 0, 1, 0, 1, 116, 42, },
+	{ 2, 1, 0, 1, 116, 42, },
+	{ 1, 1, 0, 1, 116, 42, },
+	{ 0, 1, 0, 1, 120, 42, },
+	{ 2, 1, 0, 1, 120, 42, },
+	{ 1, 1, 0, 1, 120, 42, },
+	{ 0, 1, 0, 1, 124, 42, },
+	{ 2, 1, 0, 1, 124, 42, },
+	{ 1, 1, 0, 1, 124, 42, },
+	{ 0, 1, 0, 1, 128, 42, },
+	{ 2, 1, 0, 1, 128, 42, },
+	{ 1, 1, 0, 1, 128, 42, },
+	{ 0, 1, 0, 1, 132, 42, },
+	{ 2, 1, 0, 1, 132, 42, },
+	{ 1, 1, 0, 1, 132, 42, },
+	{ 0, 1, 0, 1, 136, 42, },
+	{ 2, 1, 0, 1, 136, 42, },
+	{ 1, 1, 0, 1, 136, 42, },
+	{ 0, 1, 0, 1, 140, 40, },
+	{ 2, 1, 0, 1, 140, 40, },
+	{ 1, 1, 0, 1, 140, 40, },
+	{ 0, 1, 0, 1, 149, 44, },
+	{ 2, 1, 0, 1, 149, 44, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 44, },
+	{ 2, 1, 0, 1, 153, 44, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 44, },
+	{ 2, 1, 0, 1, 157, 44, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 44, },
+	{ 2, 1, 0, 1, 161, 44, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 44, },
+	{ 2, 1, 0, 1, 165, 44, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 32, },
+	{ 2, 1, 0, 2, 36, 32, },
+	{ 1, 1, 0, 2, 36, 32, },
+	{ 0, 1, 0, 2, 40, 32, },
+	{ 2, 1, 0, 2, 40, 32, },
+	{ 1, 1, 0, 2, 40, 32, },
+	{ 0, 1, 0, 2, 44, 32, },
+	{ 2, 1, 0, 2, 44, 32, },
+	{ 1, 1, 0, 2, 44, 32, },
+	{ 0, 1, 0, 2, 48, 36, },
+	{ 2, 1, 0, 2, 48, 36, },
+	{ 1, 1, 0, 2, 48, 36, },
+	{ 0, 1, 0, 2, 52, 36, },
+	{ 2, 1, 0, 2, 52, 36, },
+	{ 1, 1, 0, 2, 52, 36, },
+	{ 0, 1, 0, 2, 56, 32, },
+	{ 2, 1, 0, 2, 56, 32, },
+	{ 1, 1, 0, 2, 56, 32, },
+	{ 0, 1, 0, 2, 60, 32, },
+	{ 2, 1, 0, 2, 60, 32, },
+	{ 1, 1, 0, 2, 60, 32, },
+	{ 0, 1, 0, 2, 64, 32, },
+	{ 2, 1, 0, 2, 64, 32, },
+	{ 1, 1, 0, 2, 64, 32, },
+	{ 0, 1, 0, 2, 100, 32, },
+	{ 2, 1, 0, 2, 100, 32, },
+	{ 1, 1, 0, 2, 100, 32, },
+	{ 0, 1, 0, 2, 104, 32, },
+	{ 2, 1, 0, 2, 104, 32, },
+	{ 1, 1, 0, 2, 104, 32, },
+	{ 0, 1, 0, 2, 108, 32, },
+	{ 2, 1, 0, 2, 108, 32, },
+	{ 1, 1, 0, 2, 108, 32, },
+	{ 0, 1, 0, 2, 112, 32, },
+	{ 2, 1, 0, 2, 112, 32, },
+	{ 1, 1, 0, 2, 112, 32, },
+	{ 0, 1, 0, 2, 116, 32, },
+	{ 2, 1, 0, 2, 116, 32, },
+	{ 1, 1, 0, 2, 116, 32, },
+	{ 0, 1, 0, 2, 120, 32, },
+	{ 2, 1, 0, 2, 120, 32, },
+	{ 1, 1, 0, 2, 120, 32, },
+	{ 0, 1, 0, 2, 124, 32, },
+	{ 2, 1, 0, 2, 124, 32, },
+	{ 1, 1, 0, 2, 124, 32, },
+	{ 0, 1, 0, 2, 128, 32, },
+	{ 2, 1, 0, 2, 128, 32, },
+	{ 1, 1, 0, 2, 128, 32, },
+	{ 0, 1, 0, 2, 132, 32, },
+	{ 2, 1, 0, 2, 132, 32, },
+	{ 1, 1, 0, 2, 132, 32, },
+	{ 0, 1, 0, 2, 136, 32, },
+	{ 2, 1, 0, 2, 136, 32, },
+	{ 1, 1, 0, 2, 136, 32, },
+	{ 0, 1, 0, 2, 140, 30, },
+	{ 2, 1, 0, 2, 140, 30, },
+	{ 1, 1, 0, 2, 140, 30, },
+	{ 0, 1, 0, 2, 149, 40, },
+	{ 2, 1, 0, 2, 149, 40, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 40, },
+	{ 2, 1, 0, 2, 153, 40, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 40, },
+	{ 2, 1, 0, 2, 157, 40, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 40, },
+	{ 2, 1, 0, 2, 161, 40, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 42, },
+	{ 2, 1, 0, 2, 165, 42, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 32, },
+	{ 2, 1, 0, 3, 36, 32, },
+	{ 1, 1, 0, 3, 36, 32, },
+	{ 0, 1, 0, 3, 40, 32, },
+	{ 2, 1, 0, 3, 40, 32, },
+	{ 1, 1, 0, 3, 40, 32, },
+	{ 0, 1, 0, 3, 44, 32, },
+	{ 2, 1, 0, 3, 44, 32, },
+	{ 1, 1, 0, 3, 44, 32, },
+	{ 0, 1, 0, 3, 48, 36, },
+	{ 2, 1, 0, 3, 48, 36, },
+	{ 1, 1, 0, 3, 48, 36, },
+	{ 0, 1, 0, 3, 52, 36, },
+	{ 2, 1, 0, 3, 52, 36, },
+	{ 1, 1, 0, 3, 52, 36, },
+	{ 0, 1, 0, 3, 56, 32, },
+	{ 2, 1, 0, 3, 56, 32, },
+	{ 1, 1, 0, 3, 56, 32, },
+	{ 0, 1, 0, 3, 60, 32, },
+	{ 2, 1, 0, 3, 60, 32, },
+	{ 1, 1, 0, 3, 60, 32, },
+	{ 0, 1, 0, 3, 64, 32, },
+	{ 2, 1, 0, 3, 64, 32, },
+	{ 1, 1, 0, 3, 64, 32, },
+	{ 0, 1, 0, 3, 100, 32, },
+	{ 2, 1, 0, 3, 100, 32, },
+	{ 1, 1, 0, 3, 100, 32, },
+	{ 0, 1, 0, 3, 104, 32, },
+	{ 2, 1, 0, 3, 104, 32, },
+	{ 1, 1, 0, 3, 104, 32, },
+	{ 0, 1, 0, 3, 108, 32, },
+	{ 2, 1, 0, 3, 108, 32, },
+	{ 1, 1, 0, 3, 108, 32, },
+	{ 0, 1, 0, 3, 112, 32, },
+	{ 2, 1, 0, 3, 112, 32, },
+	{ 1, 1, 0, 3, 112, 32, },
+	{ 0, 1, 0, 3, 116, 32, },
+	{ 2, 1, 0, 3, 116, 32, },
+	{ 1, 1, 0, 3, 116, 32, },
+	{ 0, 1, 0, 3, 120, 32, },
+	{ 2, 1, 0, 3, 120, 32, },
+	{ 1, 1, 0, 3, 120, 32, },
+	{ 0, 1, 0, 3, 124, 32, },
+	{ 2, 1, 0, 3, 124, 32, },
+	{ 1, 1, 0, 3, 124, 32, },
+	{ 0, 1, 0, 3, 128, 32, },
+	{ 2, 1, 0, 3, 128, 32, },
+	{ 1, 1, 0, 3, 128, 32, },
+	{ 0, 1, 0, 3, 132, 32, },
+	{ 2, 1, 0, 3, 132, 32, },
+	{ 1, 1, 0, 3, 132, 32, },
+	{ 0, 1, 0, 3, 136, 32, },
+	{ 2, 1, 0, 3, 136, 32, },
+	{ 1, 1, 0, 3, 136, 32, },
+	{ 0, 1, 0, 3, 140, 30, },
+	{ 2, 1, 0, 3, 140, 30, },
+	{ 1, 1, 0, 3, 140, 30, },
+	{ 0, 1, 0, 3, 149, 40, },
+	{ 2, 1, 0, 3, 149, 40, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 40, },
+	{ 2, 1, 0, 3, 153, 40, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 40, },
+	{ 2, 1, 0, 3, 157, 40, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 40, },
+	{ 2, 1, 0, 3, 161, 40, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 42, },
+	{ 2, 1, 0, 3, 165, 42, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 32, },
+	{ 2, 1, 0, 6, 36, 32, },
+	{ 1, 1, 0, 6, 36, 32, },
+	{ 0, 1, 0, 6, 40, 32, },
+	{ 2, 1, 0, 6, 40, 32, },
+	{ 1, 1, 0, 6, 40, 32, },
+	{ 0, 1, 0, 6, 44, 32, },
+	{ 2, 1, 0, 6, 44, 32, },
+	{ 1, 1, 0, 6, 44, 32, },
+	{ 0, 1, 0, 6, 48, 36, },
+	{ 2, 1, 0, 6, 48, 36, },
+	{ 1, 1, 0, 6, 48, 36, },
+	{ 0, 1, 0, 6, 52, 36, },
+	{ 2, 1, 0, 6, 52, 36, },
+	{ 1, 1, 0, 6, 52, 36, },
+	{ 0, 1, 0, 6, 56, 32, },
+	{ 2, 1, 0, 6, 56, 32, },
+	{ 1, 1, 0, 6, 56, 32, },
+	{ 0, 1, 0, 6, 60, 32, },
+	{ 2, 1, 0, 6, 60, 32, },
+	{ 1, 1, 0, 6, 60, 32, },
+	{ 0, 1, 0, 6, 64, 32, },
+	{ 2, 1, 0, 6, 64, 32, },
+	{ 1, 1, 0, 6, 64, 32, },
+	{ 0, 1, 0, 6, 100, 32, },
+	{ 2, 1, 0, 6, 100, 32, },
+	{ 1, 1, 0, 6, 100, 32, },
+	{ 0, 1, 0, 6, 104, 32, },
+	{ 2, 1, 0, 6, 104, 32, },
+	{ 1, 1, 0, 6, 104, 32, },
+	{ 0, 1, 0, 6, 108, 32, },
+	{ 2, 1, 0, 6, 108, 32, },
+	{ 1, 1, 0, 6, 108, 32, },
+	{ 0, 1, 0, 6, 112, 32, },
+	{ 2, 1, 0, 6, 112, 32, },
+	{ 1, 1, 0, 6, 112, 32, },
+	{ 0, 1, 0, 6, 116, 32, },
+	{ 2, 1, 0, 6, 116, 32, },
+	{ 1, 1, 0, 6, 116, 32, },
+	{ 0, 1, 0, 6, 120, 32, },
+	{ 2, 1, 0, 6, 120, 32, },
+	{ 1, 1, 0, 6, 120, 32, },
+	{ 0, 1, 0, 6, 124, 32, },
+	{ 2, 1, 0, 6, 124, 32, },
+	{ 1, 1, 0, 6, 124, 32, },
+	{ 0, 1, 0, 6, 128, 32, },
+	{ 2, 1, 0, 6, 128, 32, },
+	{ 1, 1, 0, 6, 128, 32, },
+	{ 0, 1, 0, 6, 132, 32, },
+	{ 2, 1, 0, 6, 132, 32, },
+	{ 1, 1, 0, 6, 132, 32, },
+	{ 0, 1, 0, 6, 136, 32, },
+	{ 2, 1, 0, 6, 136, 32, },
+	{ 1, 1, 0, 6, 136, 32, },
+	{ 0, 1, 0, 6, 140, 30, },
+	{ 2, 1, 0, 6, 140, 30, },
+	{ 1, 1, 0, 6, 140, 30, },
+	{ 0, 1, 0, 6, 149, 40, },
+	{ 2, 1, 0, 6, 149, 40, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 40, },
+	{ 2, 1, 0, 6, 153, 40, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 40, },
+	{ 2, 1, 0, 6, 157, 40, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 40, },
+	{ 2, 1, 0, 6, 161, 40, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 42, },
+	{ 2, 1, 0, 6, 165, 42, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 32, },
+	{ 2, 1, 0, 7, 36, 32, },
+	{ 1, 1, 0, 7, 36, 32, },
+	{ 0, 1, 0, 7, 40, 32, },
+	{ 2, 1, 0, 7, 40, 32, },
+	{ 1, 1, 0, 7, 40, 32, },
+	{ 0, 1, 0, 7, 44, 32, },
+	{ 2, 1, 0, 7, 44, 32, },
+	{ 1, 1, 0, 7, 44, 32, },
+	{ 0, 1, 0, 7, 48, 36, },
+	{ 2, 1, 0, 7, 48, 36, },
+	{ 1, 1, 0, 7, 48, 36, },
+	{ 0, 1, 0, 7, 52, 36, },
+	{ 2, 1, 0, 7, 52, 36, },
+	{ 1, 1, 0, 7, 52, 36, },
+	{ 0, 1, 0, 7, 56, 32, },
+	{ 2, 1, 0, 7, 56, 32, },
+	{ 1, 1, 0, 7, 56, 32, },
+	{ 0, 1, 0, 7, 60, 32, },
+	{ 2, 1, 0, 7, 60, 32, },
+	{ 1, 1, 0, 7, 60, 32, },
+	{ 0, 1, 0, 7, 64, 32, },
+	{ 2, 1, 0, 7, 64, 32, },
+	{ 1, 1, 0, 7, 64, 32, },
+	{ 0, 1, 0, 7, 100, 32, },
+	{ 2, 1, 0, 7, 100, 32, },
+	{ 1, 1, 0, 7, 100, 32, },
+	{ 0, 1, 0, 7, 104, 32, },
+	{ 2, 1, 0, 7, 104, 32, },
+	{ 1, 1, 0, 7, 104, 32, },
+	{ 0, 1, 0, 7, 108, 32, },
+	{ 2, 1, 0, 7, 108, 32, },
+	{ 1, 1, 0, 7, 108, 32, },
+	{ 0, 1, 0, 7, 112, 32, },
+	{ 2, 1, 0, 7, 112, 32, },
+	{ 1, 1, 0, 7, 112, 32, },
+	{ 0, 1, 0, 7, 116, 32, },
+	{ 2, 1, 0, 7, 116, 32, },
+	{ 1, 1, 0, 7, 116, 32, },
+	{ 0, 1, 0, 7, 120, 32, },
+	{ 2, 1, 0, 7, 120, 32, },
+	{ 1, 1, 0, 7, 120, 32, },
+	{ 0, 1, 0, 7, 124, 32, },
+	{ 2, 1, 0, 7, 124, 32, },
+	{ 1, 1, 0, 7, 124, 32, },
+	{ 0, 1, 0, 7, 128, 32, },
+	{ 2, 1, 0, 7, 128, 32, },
+	{ 1, 1, 0, 7, 128, 32, },
+	{ 0, 1, 0, 7, 132, 32, },
+	{ 2, 1, 0, 7, 132, 32, },
+	{ 1, 1, 0, 7, 132, 32, },
+	{ 0, 1, 0, 7, 136, 32, },
+	{ 2, 1, 0, 7, 136, 32, },
+	{ 1, 1, 0, 7, 136, 32, },
+	{ 0, 1, 0, 7, 140, 30, },
+	{ 2, 1, 0, 7, 140, 30, },
+	{ 1, 1, 0, 7, 140, 30, },
+	{ 0, 1, 0, 7, 149, 40, },
+	{ 2, 1, 0, 7, 149, 40, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 40, },
+	{ 2, 1, 0, 7, 153, 40, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 40, },
+	{ 2, 1, 0, 7, 157, 40, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 40, },
+	{ 2, 1, 0, 7, 161, 40, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 42, },
+	{ 2, 1, 0, 7, 165, 42, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 32, },
+	{ 2, 1, 1, 2, 38, 32, },
+	{ 1, 1, 1, 2, 38, 32, },
+	{ 0, 1, 1, 2, 46, 36, },
+	{ 2, 1, 1, 2, 46, 36, },
+	{ 1, 1, 1, 2, 46, 36, },
+	{ 0, 1, 1, 2, 54, 36, },
+	{ 2, 1, 1, 2, 54, 36, },
+	{ 1, 1, 1, 2, 54, 36, },
+	{ 0, 1, 1, 2, 62, 32, },
+	{ 2, 1, 1, 2, 62, 32, },
+	{ 1, 1, 1, 2, 62, 32, },
+	{ 0, 1, 1, 2, 102, 30, },
+	{ 2, 1, 1, 2, 102, 30, },
+	{ 1, 1, 1, 2, 102, 30, },
+	{ 0, 1, 1, 2, 110, 32, },
+	{ 2, 1, 1, 2, 110, 32, },
+	{ 1, 1, 1, 2, 110, 32, },
+	{ 0, 1, 1, 2, 118, 32, },
+	{ 2, 1, 1, 2, 118, 32, },
+	{ 1, 1, 1, 2, 118, 32, },
+	{ 0, 1, 1, 2, 126, 32, },
+	{ 2, 1, 1, 2, 126, 32, },
+	{ 1, 1, 1, 2, 126, 32, },
+	{ 0, 1, 1, 2, 134, 36, },
+	{ 2, 1, 1, 2, 134, 36, },
+	{ 1, 1, 1, 2, 134, 36, },
+	{ 0, 1, 1, 2, 151, 36, },
+	{ 2, 1, 1, 2, 151, 36, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 40, },
+	{ 2, 1, 1, 2, 159, 40, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 32, },
+	{ 2, 1, 1, 3, 38, 32, },
+	{ 1, 1, 1, 3, 38, 32, },
+	{ 0, 1, 1, 3, 46, 36, },
+	{ 2, 1, 1, 3, 46, 36, },
+	{ 1, 1, 1, 3, 46, 36, },
+	{ 0, 1, 1, 3, 54, 36, },
+	{ 2, 1, 1, 3, 54, 36, },
+	{ 1, 1, 1, 3, 54, 36, },
+	{ 0, 1, 1, 3, 62, 32, },
+	{ 2, 1, 1, 3, 62, 32, },
+	{ 1, 1, 1, 3, 62, 32, },
+	{ 0, 1, 1, 3, 102, 30, },
+	{ 2, 1, 1, 3, 102, 30, },
+	{ 1, 1, 1, 3, 102, 30, },
+	{ 0, 1, 1, 3, 110, 32, },
+	{ 2, 1, 1, 3, 110, 32, },
+	{ 1, 1, 1, 3, 110, 32, },
+	{ 0, 1, 1, 3, 118, 32, },
+	{ 2, 1, 1, 3, 118, 32, },
+	{ 1, 1, 1, 3, 118, 32, },
+	{ 0, 1, 1, 3, 126, 32, },
+	{ 2, 1, 1, 3, 126, 32, },
+	{ 1, 1, 1, 3, 126, 32, },
+	{ 0, 1, 1, 3, 134, 36, },
+	{ 2, 1, 1, 3, 134, 36, },
+	{ 1, 1, 1, 3, 134, 36, },
+	{ 0, 1, 1, 3, 151, 36, },
+	{ 2, 1, 1, 3, 151, 36, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 40, },
+	{ 2, 1, 1, 3, 159, 40, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 32, },
+	{ 2, 1, 1, 6, 38, 32, },
+	{ 1, 1, 1, 6, 38, 32, },
+	{ 0, 1, 1, 6, 46, 36, },
+	{ 2, 1, 1, 6, 46, 36, },
+	{ 1, 1, 1, 6, 46, 36, },
+	{ 0, 1, 1, 6, 54, 36, },
+	{ 2, 1, 1, 6, 54, 36, },
+	{ 1, 1, 1, 6, 54, 36, },
+	{ 0, 1, 1, 6, 62, 32, },
+	{ 2, 1, 1, 6, 62, 32, },
+	{ 1, 1, 1, 6, 62, 32, },
+	{ 0, 1, 1, 6, 102, 30, },
+	{ 2, 1, 1, 6, 102, 30, },
+	{ 1, 1, 1, 6, 102, 30, },
+	{ 0, 1, 1, 6, 110, 32, },
+	{ 2, 1, 1, 6, 110, 32, },
+	{ 1, 1, 1, 6, 110, 32, },
+	{ 0, 1, 1, 6, 118, 32, },
+	{ 2, 1, 1, 6, 118, 32, },
+	{ 1, 1, 1, 6, 118, 32, },
+	{ 0, 1, 1, 6, 126, 32, },
+	{ 2, 1, 1, 6, 126, 32, },
+	{ 1, 1, 1, 6, 126, 32, },
+	{ 0, 1, 1, 6, 134, 36, },
+	{ 2, 1, 1, 6, 134, 36, },
+	{ 1, 1, 1, 6, 134, 36, },
+	{ 0, 1, 1, 6, 151, 36, },
+	{ 2, 1, 1, 6, 151, 36, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 40, },
+	{ 2, 1, 1, 6, 159, 40, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 32, },
+	{ 2, 1, 1, 7, 38, 32, },
+	{ 1, 1, 1, 7, 38, 32, },
+	{ 0, 1, 1, 7, 46, 36, },
+	{ 2, 1, 1, 7, 46, 36, },
+	{ 1, 1, 1, 7, 46, 36, },
+	{ 0, 1, 1, 7, 54, 36, },
+	{ 2, 1, 1, 7, 54, 36, },
+	{ 1, 1, 1, 7, 54, 36, },
+	{ 0, 1, 1, 7, 62, 32, },
+	{ 2, 1, 1, 7, 62, 32, },
+	{ 1, 1, 1, 7, 62, 32, },
+	{ 0, 1, 1, 7, 102, 30, },
+	{ 2, 1, 1, 7, 102, 30, },
+	{ 1, 1, 1, 7, 102, 30, },
+	{ 0, 1, 1, 7, 110, 32, },
+	{ 2, 1, 1, 7, 110, 32, },
+	{ 1, 1, 1, 7, 110, 32, },
+	{ 0, 1, 1, 7, 118, 32, },
+	{ 2, 1, 1, 7, 118, 32, },
+	{ 1, 1, 1, 7, 118, 32, },
+	{ 0, 1, 1, 7, 126, 32, },
+	{ 2, 1, 1, 7, 126, 32, },
+	{ 1, 1, 1, 7, 126, 32, },
+	{ 0, 1, 1, 7, 134, 36, },
+	{ 2, 1, 1, 7, 134, 36, },
+	{ 1, 1, 1, 7, 134, 36, },
+	{ 0, 1, 1, 7, 151, 36, },
+	{ 2, 1, 1, 7, 151, 36, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 40, },
+	{ 2, 1, 1, 7, 159, 40, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 34, },
+	{ 2, 1, 2, 4, 42, 34, },
+	{ 1, 1, 2, 4, 42, 34, },
+	{ 0, 1, 2, 4, 58, 34, },
+	{ 2, 1, 2, 4, 58, 34, },
+	{ 1, 1, 2, 4, 58, 34, },
+	{ 0, 1, 2, 4, 106, 32, },
+	{ 2, 1, 2, 4, 106, 32, },
+	{ 1, 1, 2, 4, 106, 32, },
+	{ 0, 1, 2, 4, 122, 34, },
+	{ 2, 1, 2, 4, 122, 34, },
+	{ 1, 1, 2, 4, 122, 34, },
+	{ 0, 1, 2, 4, 155, 34, },
+	{ 2, 1, 2, 4, 155, 34, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 34, },
+	{ 2, 1, 2, 5, 42, 34, },
+	{ 1, 1, 2, 5, 42, 34, },
+	{ 0, 1, 2, 5, 58, 34, },
+	{ 2, 1, 2, 5, 58, 34, },
+	{ 1, 1, 2, 5, 58, 34, },
+	{ 0, 1, 2, 5, 106, 32, },
+	{ 2, 1, 2, 5, 106, 32, },
+	{ 1, 1, 2, 5, 106, 32, },
+	{ 0, 1, 2, 5, 122, 34, },
+	{ 2, 1, 2, 5, 122, 34, },
+	{ 1, 1, 2, 5, 122, 34, },
+	{ 0, 1, 2, 5, 155, 34, },
+	{ 2, 1, 2, 5, 155, 34, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 34, },
+	{ 2, 1, 2, 8, 42, 34, },
+	{ 1, 1, 2, 8, 42, 34, },
+	{ 0, 1, 2, 8, 58, 34, },
+	{ 2, 1, 2, 8, 58, 34, },
+	{ 1, 1, 2, 8, 58, 34, },
+	{ 0, 1, 2, 8, 106, 32, },
+	{ 2, 1, 2, 8, 106, 32, },
+	{ 1, 1, 2, 8, 106, 32, },
+	{ 0, 1, 2, 8, 122, 34, },
+	{ 2, 1, 2, 8, 122, 34, },
+	{ 1, 1, 2, 8, 122, 34, },
+	{ 0, 1, 2, 8, 155, 34, },
+	{ 2, 1, 2, 8, 155, 34, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 34, },
+	{ 2, 1, 2, 9, 42, 34, },
+	{ 1, 1, 2, 9, 42, 34, },
+	{ 0, 1, 2, 9, 58, 34, },
+	{ 2, 1, 2, 9, 58, 34, },
+	{ 1, 1, 2, 9, 58, 34, },
+	{ 0, 1, 2, 9, 106, 32, },
+	{ 2, 1, 2, 9, 106, 32, },
+	{ 1, 1, 2, 9, 106, 32, },
+	{ 0, 1, 2, 9, 122, 34, },
+	{ 2, 1, 2, 9, 122, 34, },
+	{ 1, 1, 2, 9, 122, 34, },
+	{ 0, 1, 2, 9, 155, 34, },
+	{ 2, 1, 2, 9, 155, 34, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt_type2);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt_type3[] = {
+	{ 0, 0, 0, 0, 1, 46, },
+	{ 2, 0, 0, 0, 1, 40, },
+	{ 1, 0, 0, 0, 1, 40, },
+	{ 0, 0, 0, 0, 2, 46, },
+	{ 2, 0, 0, 0, 2, 40, },
+	{ 1, 0, 0, 0, 2, 40, },
+	{ 0, 0, 0, 0, 3, 46, },
+	{ 2, 0, 0, 0, 3, 40, },
+	{ 1, 0, 0, 0, 3, 40, },
+	{ 0, 0, 0, 0, 4, 46, },
+	{ 2, 0, 0, 0, 4, 40, },
+	{ 1, 0, 0, 0, 4, 40, },
+	{ 0, 0, 0, 0, 5, 46, },
+	{ 2, 0, 0, 0, 5, 40, },
+	{ 1, 0, 0, 0, 5, 40, },
+	{ 0, 0, 0, 0, 6, 46, },
+	{ 2, 0, 0, 0, 6, 40, },
+	{ 1, 0, 0, 0, 6, 40, },
+	{ 0, 0, 0, 0, 7, 46, },
+	{ 2, 0, 0, 0, 7, 40, },
+	{ 1, 0, 0, 0, 7, 40, },
+	{ 0, 0, 0, 0, 8, 46, },
+	{ 2, 0, 0, 0, 8, 40, },
+	{ 1, 0, 0, 0, 8, 40, },
+	{ 0, 0, 0, 0, 9, 46, },
+	{ 2, 0, 0, 0, 9, 40, },
+	{ 1, 0, 0, 0, 9, 40, },
+	{ 0, 0, 0, 0, 10, 46, },
+	{ 2, 0, 0, 0, 10, 40, },
+	{ 1, 0, 0, 0, 10, 40, },
+	{ 0, 0, 0, 0, 11, 46, },
+	{ 2, 0, 0, 0, 11, 40, },
+	{ 1, 0, 0, 0, 11, 40, },
+	{ 0, 0, 0, 0, 12, 63, },
+	{ 2, 0, 0, 0, 12, 40, },
+	{ 1, 0, 0, 0, 12, 40, },
+	{ 0, 0, 0, 0, 13, 63, },
+	{ 2, 0, 0, 0, 13, 40, },
+	{ 1, 0, 0, 0, 13, 40, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 40, },
+	{ 0, 0, 0, 1, 1, 46, },
+	{ 2, 0, 0, 1, 1, 40, },
+	{ 1, 0, 0, 1, 1, 40, },
+	{ 0, 0, 0, 1, 2, 46, },
+	{ 2, 0, 0, 1, 2, 40, },
+	{ 1, 0, 0, 1, 2, 40, },
+	{ 0, 0, 0, 1, 3, 46, },
+	{ 2, 0, 0, 1, 3, 40, },
+	{ 1, 0, 0, 1, 3, 40, },
+	{ 0, 0, 0, 1, 4, 46, },
+	{ 2, 0, 0, 1, 4, 40, },
+	{ 1, 0, 0, 1, 4, 40, },
+	{ 0, 0, 0, 1, 5, 46, },
+	{ 2, 0, 0, 1, 5, 40, },
+	{ 1, 0, 0, 1, 5, 40, },
+	{ 0, 0, 0, 1, 6, 46, },
+	{ 2, 0, 0, 1, 6, 40, },
+	{ 1, 0, 0, 1, 6, 40, },
+	{ 0, 0, 0, 1, 7, 46, },
+	{ 2, 0, 0, 1, 7, 40, },
+	{ 1, 0, 0, 1, 7, 40, },
+	{ 0, 0, 0, 1, 8, 46, },
+	{ 2, 0, 0, 1, 8, 40, },
+	{ 1, 0, 0, 1, 8, 40, },
+	{ 0, 0, 0, 1, 9, 46, },
+	{ 2, 0, 0, 1, 9, 40, },
+	{ 1, 0, 0, 1, 9, 40, },
+	{ 0, 0, 0, 1, 10, 46, },
+	{ 2, 0, 0, 1, 10, 40, },
+	{ 1, 0, 0, 1, 10, 40, },
+	{ 0, 0, 0, 1, 11, 46, },
+	{ 2, 0, 0, 1, 11, 40, },
+	{ 1, 0, 0, 1, 11, 40, },
+	{ 0, 0, 0, 1, 12, 63, },
+	{ 2, 0, 0, 1, 12, 40, },
+	{ 1, 0, 0, 1, 12, 40, },
+	{ 0, 0, 0, 1, 13, 63, },
+	{ 2, 0, 0, 1, 13, 40, },
+	{ 1, 0, 0, 1, 13, 40, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 63, },
+	{ 0, 0, 0, 2, 1, 46, },
+	{ 2, 0, 0, 2, 1, 40, },
+	{ 1, 0, 0, 2, 1, 40, },
+	{ 0, 0, 0, 2, 2, 46, },
+	{ 2, 0, 0, 2, 2, 40, },
+	{ 1, 0, 0, 2, 2, 40, },
+	{ 0, 0, 0, 2, 3, 46, },
+	{ 2, 0, 0, 2, 3, 40, },
+	{ 1, 0, 0, 2, 3, 40, },
+	{ 0, 0, 0, 2, 4, 46, },
+	{ 2, 0, 0, 2, 4, 40, },
+	{ 1, 0, 0, 2, 4, 40, },
+	{ 0, 0, 0, 2, 5, 46, },
+	{ 2, 0, 0, 2, 5, 40, },
+	{ 1, 0, 0, 2, 5, 40, },
+	{ 0, 0, 0, 2, 6, 46, },
+	{ 2, 0, 0, 2, 6, 40, },
+	{ 1, 0, 0, 2, 6, 40, },
+	{ 0, 0, 0, 2, 7, 46, },
+	{ 2, 0, 0, 2, 7, 40, },
+	{ 1, 0, 0, 2, 7, 40, },
+	{ 0, 0, 0, 2, 8, 46, },
+	{ 2, 0, 0, 2, 8, 40, },
+	{ 1, 0, 0, 2, 8, 40, },
+	{ 0, 0, 0, 2, 9, 46, },
+	{ 2, 0, 0, 2, 9, 40, },
+	{ 1, 0, 0, 2, 9, 40, },
+	{ 0, 0, 0, 2, 10, 46, },
+	{ 2, 0, 0, 2, 10, 40, },
+	{ 1, 0, 0, 2, 10, 40, },
+	{ 0, 0, 0, 2, 11, 46, },
+	{ 2, 0, 0, 2, 11, 40, },
+	{ 1, 0, 0, 2, 11, 40, },
+	{ 0, 0, 0, 2, 12, 63, },
+	{ 2, 0, 0, 2, 12, 40, },
+	{ 1, 0, 0, 2, 12, 40, },
+	{ 0, 0, 0, 2, 13, 63, },
+	{ 2, 0, 0, 2, 13, 40, },
+	{ 1, 0, 0, 2, 13, 40, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 63, },
+	{ 0, 0, 0, 3, 1, 46, },
+	{ 2, 0, 0, 3, 1, 40, },
+	{ 1, 0, 0, 3, 1, 40, },
+	{ 0, 0, 0, 3, 2, 46, },
+	{ 2, 0, 0, 3, 2, 40, },
+	{ 1, 0, 0, 3, 2, 40, },
+	{ 0, 0, 0, 3, 3, 46, },
+	{ 2, 0, 0, 3, 3, 40, },
+	{ 1, 0, 0, 3, 3, 40, },
+	{ 0, 0, 0, 3, 4, 46, },
+	{ 2, 0, 0, 3, 4, 40, },
+	{ 1, 0, 0, 3, 4, 40, },
+	{ 0, 0, 0, 3, 5, 46, },
+	{ 2, 0, 0, 3, 5, 40, },
+	{ 1, 0, 0, 3, 5, 40, },
+	{ 0, 0, 0, 3, 6, 46, },
+	{ 2, 0, 0, 3, 6, 40, },
+	{ 1, 0, 0, 3, 6, 40, },
+	{ 0, 0, 0, 3, 7, 46, },
+	{ 2, 0, 0, 3, 7, 40, },
+	{ 1, 0, 0, 3, 7, 40, },
+	{ 0, 0, 0, 3, 8, 46, },
+	{ 2, 0, 0, 3, 8, 40, },
+	{ 1, 0, 0, 3, 8, 40, },
+	{ 0, 0, 0, 3, 9, 46, },
+	{ 2, 0, 0, 3, 9, 40, },
+	{ 1, 0, 0, 3, 9, 40, },
+	{ 0, 0, 0, 3, 10, 46, },
+	{ 2, 0, 0, 3, 10, 40, },
+	{ 1, 0, 0, 3, 10, 40, },
+	{ 0, 0, 0, 3, 11, 46, },
+	{ 2, 0, 0, 3, 11, 40, },
+	{ 1, 0, 0, 3, 11, 40, },
+	{ 0, 0, 0, 3, 12, 63, },
+	{ 2, 0, 0, 3, 12, 40, },
+	{ 1, 0, 0, 3, 12, 40, },
+	{ 0, 0, 0, 3, 13, 63, },
+	{ 2, 0, 0, 3, 13, 40, },
+	{ 1, 0, 0, 3, 13, 40, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 63, },
+	{ 0, 0, 0, 6, 1, 46, },
+	{ 2, 0, 0, 6, 1, 40, },
+	{ 1, 0, 0, 6, 1, 40, },
+	{ 0, 0, 0, 6, 2, 46, },
+	{ 2, 0, 0, 6, 2, 40, },
+	{ 1, 0, 0, 6, 2, 40, },
+	{ 0, 0, 0, 6, 3, 46, },
+	{ 2, 0, 0, 6, 3, 40, },
+	{ 1, 0, 0, 6, 3, 40, },
+	{ 0, 0, 0, 6, 4, 46, },
+	{ 2, 0, 0, 6, 4, 40, },
+	{ 1, 0, 0, 6, 4, 40, },
+	{ 0, 0, 0, 6, 5, 46, },
+	{ 2, 0, 0, 6, 5, 40, },
+	{ 1, 0, 0, 6, 5, 40, },
+	{ 0, 0, 0, 6, 6, 46, },
+	{ 2, 0, 0, 6, 6, 40, },
+	{ 1, 0, 0, 6, 6, 40, },
+	{ 0, 0, 0, 6, 7, 46, },
+	{ 2, 0, 0, 6, 7, 40, },
+	{ 1, 0, 0, 6, 7, 40, },
+	{ 0, 0, 0, 6, 8, 46, },
+	{ 2, 0, 0, 6, 8, 40, },
+	{ 1, 0, 0, 6, 8, 40, },
+	{ 0, 0, 0, 6, 9, 46, },
+	{ 2, 0, 0, 6, 9, 40, },
+	{ 1, 0, 0, 6, 9, 40, },
+	{ 0, 0, 0, 6, 10, 46, },
+	{ 2, 0, 0, 6, 10, 40, },
+	{ 1, 0, 0, 6, 10, 40, },
+	{ 0, 0, 0, 6, 11, 46, },
+	{ 2, 0, 0, 6, 11, 40, },
+	{ 1, 0, 0, 6, 11, 40, },
+	{ 0, 0, 0, 6, 12, 63, },
+	{ 2, 0, 0, 6, 12, 40, },
+	{ 1, 0, 0, 6, 12, 40, },
+	{ 0, 0, 0, 6, 13, 63, },
+	{ 2, 0, 0, 6, 13, 40, },
+	{ 1, 0, 0, 6, 13, 40, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 63, },
+	{ 0, 0, 0, 7, 1, 46, },
+	{ 2, 0, 0, 7, 1, 40, },
+	{ 1, 0, 0, 7, 1, 40, },
+	{ 0, 0, 0, 7, 2, 46, },
+	{ 2, 0, 0, 7, 2, 40, },
+	{ 1, 0, 0, 7, 2, 40, },
+	{ 0, 0, 0, 7, 3, 46, },
+	{ 2, 0, 0, 7, 3, 40, },
+	{ 1, 0, 0, 7, 3, 40, },
+	{ 0, 0, 0, 7, 4, 46, },
+	{ 2, 0, 0, 7, 4, 40, },
+	{ 1, 0, 0, 7, 4, 40, },
+	{ 0, 0, 0, 7, 5, 46, },
+	{ 2, 0, 0, 7, 5, 40, },
+	{ 1, 0, 0, 7, 5, 40, },
+	{ 0, 0, 0, 7, 6, 46, },
+	{ 2, 0, 0, 7, 6, 40, },
+	{ 1, 0, 0, 7, 6, 40, },
+	{ 0, 0, 0, 7, 7, 46, },
+	{ 2, 0, 0, 7, 7, 40, },
+	{ 1, 0, 0, 7, 7, 40, },
+	{ 0, 0, 0, 7, 8, 46, },
+	{ 2, 0, 0, 7, 8, 40, },
+	{ 1, 0, 0, 7, 8, 40, },
+	{ 0, 0, 0, 7, 9, 46, },
+	{ 2, 0, 0, 7, 9, 40, },
+	{ 1, 0, 0, 7, 9, 40, },
+	{ 0, 0, 0, 7, 10, 46, },
+	{ 2, 0, 0, 7, 10, 40, },
+	{ 1, 0, 0, 7, 10, 40, },
+	{ 0, 0, 0, 7, 11, 46, },
+	{ 2, 0, 0, 7, 11, 40, },
+	{ 1, 0, 0, 7, 11, 40, },
+	{ 0, 0, 0, 7, 12, 63, },
+	{ 2, 0, 0, 7, 12, 40, },
+	{ 1, 0, 0, 7, 12, 40, },
+	{ 0, 0, 0, 7, 13, 63, },
+	{ 2, 0, 0, 7, 13, 40, },
+	{ 1, 0, 0, 7, 13, 40, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 63, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 46, },
+	{ 2, 0, 1, 2, 3, 40, },
+	{ 1, 0, 1, 2, 3, 40, },
+	{ 0, 0, 1, 2, 4, 46, },
+	{ 2, 0, 1, 2, 4, 40, },
+	{ 1, 0, 1, 2, 4, 40, },
+	{ 0, 0, 1, 2, 5, 46, },
+	{ 2, 0, 1, 2, 5, 40, },
+	{ 1, 0, 1, 2, 5, 40, },
+	{ 0, 0, 1, 2, 6, 46, },
+	{ 2, 0, 1, 2, 6, 40, },
+	{ 1, 0, 1, 2, 6, 40, },
+	{ 0, 0, 1, 2, 7, 46, },
+	{ 2, 0, 1, 2, 7, 40, },
+	{ 1, 0, 1, 2, 7, 40, },
+	{ 0, 0, 1, 2, 8, 46, },
+	{ 2, 0, 1, 2, 8, 40, },
+	{ 1, 0, 1, 2, 8, 40, },
+	{ 0, 0, 1, 2, 9, 46, },
+	{ 2, 0, 1, 2, 9, 40, },
+	{ 1, 0, 1, 2, 9, 40, },
+	{ 0, 0, 1, 2, 10, 46, },
+	{ 2, 0, 1, 2, 10, 40, },
+	{ 1, 0, 1, 2, 10, 40, },
+	{ 0, 0, 1, 2, 11, 46, },
+	{ 2, 0, 1, 2, 11, 40, },
+	{ 1, 0, 1, 2, 11, 40, },
+	{ 0, 0, 1, 2, 12, 63, },
+	{ 2, 0, 1, 2, 12, 40, },
+	{ 1, 0, 1, 2, 12, 40, },
+	{ 0, 0, 1, 2, 13, 63, },
+	{ 2, 0, 1, 2, 13, 40, },
+	{ 1, 0, 1, 2, 13, 40, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 46, },
+	{ 2, 0, 1, 3, 3, 40, },
+	{ 1, 0, 1, 3, 3, 40, },
+	{ 0, 0, 1, 3, 4, 46, },
+	{ 2, 0, 1, 3, 4, 40, },
+	{ 1, 0, 1, 3, 4, 40, },
+	{ 0, 0, 1, 3, 5, 46, },
+	{ 2, 0, 1, 3, 5, 40, },
+	{ 1, 0, 1, 3, 5, 40, },
+	{ 0, 0, 1, 3, 6, 46, },
+	{ 2, 0, 1, 3, 6, 40, },
+	{ 1, 0, 1, 3, 6, 40, },
+	{ 0, 0, 1, 3, 7, 46, },
+	{ 2, 0, 1, 3, 7, 40, },
+	{ 1, 0, 1, 3, 7, 40, },
+	{ 0, 0, 1, 3, 8, 46, },
+	{ 2, 0, 1, 3, 8, 40, },
+	{ 1, 0, 1, 3, 8, 40, },
+	{ 0, 0, 1, 3, 9, 46, },
+	{ 2, 0, 1, 3, 9, 40, },
+	{ 1, 0, 1, 3, 9, 40, },
+	{ 0, 0, 1, 3, 10, 46, },
+	{ 2, 0, 1, 3, 10, 40, },
+	{ 1, 0, 1, 3, 10, 40, },
+	{ 0, 0, 1, 3, 11, 46, },
+	{ 2, 0, 1, 3, 11, 40, },
+	{ 1, 0, 1, 3, 11, 40, },
+	{ 0, 0, 1, 3, 12, 63, },
+	{ 2, 0, 1, 3, 12, 40, },
+	{ 1, 0, 1, 3, 12, 40, },
+	{ 0, 0, 1, 3, 13, 63, },
+	{ 2, 0, 1, 3, 13, 40, },
+	{ 1, 0, 1, 3, 13, 40, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 46, },
+	{ 2, 0, 1, 6, 3, 40, },
+	{ 1, 0, 1, 6, 3, 40, },
+	{ 0, 0, 1, 6, 4, 46, },
+	{ 2, 0, 1, 6, 4, 40, },
+	{ 1, 0, 1, 6, 4, 40, },
+	{ 0, 0, 1, 6, 5, 46, },
+	{ 2, 0, 1, 6, 5, 40, },
+	{ 1, 0, 1, 6, 5, 40, },
+	{ 0, 0, 1, 6, 6, 46, },
+	{ 2, 0, 1, 6, 6, 40, },
+	{ 1, 0, 1, 6, 6, 40, },
+	{ 0, 0, 1, 6, 7, 46, },
+	{ 2, 0, 1, 6, 7, 40, },
+	{ 1, 0, 1, 6, 7, 40, },
+	{ 0, 0, 1, 6, 8, 46, },
+	{ 2, 0, 1, 6, 8, 40, },
+	{ 1, 0, 1, 6, 8, 40, },
+	{ 0, 0, 1, 6, 9, 46, },
+	{ 2, 0, 1, 6, 9, 40, },
+	{ 1, 0, 1, 6, 9, 40, },
+	{ 0, 0, 1, 6, 10, 46, },
+	{ 2, 0, 1, 6, 10, 40, },
+	{ 1, 0, 1, 6, 10, 40, },
+	{ 0, 0, 1, 6, 11, 46, },
+	{ 2, 0, 1, 6, 11, 40, },
+	{ 1, 0, 1, 6, 11, 40, },
+	{ 0, 0, 1, 6, 12, 63, },
+	{ 2, 0, 1, 6, 12, 40, },
+	{ 1, 0, 1, 6, 12, 40, },
+	{ 0, 0, 1, 6, 13, 63, },
+	{ 2, 0, 1, 6, 13, 40, },
+	{ 1, 0, 1, 6, 13, 40, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 46, },
+	{ 2, 0, 1, 7, 3, 40, },
+	{ 1, 0, 1, 7, 3, 40, },
+	{ 0, 0, 1, 7, 4, 46, },
+	{ 2, 0, 1, 7, 4, 40, },
+	{ 1, 0, 1, 7, 4, 40, },
+	{ 0, 0, 1, 7, 5, 46, },
+	{ 2, 0, 1, 7, 5, 40, },
+	{ 1, 0, 1, 7, 5, 40, },
+	{ 0, 0, 1, 7, 6, 46, },
+	{ 2, 0, 1, 7, 6, 40, },
+	{ 1, 0, 1, 7, 6, 40, },
+	{ 0, 0, 1, 7, 7, 46, },
+	{ 2, 0, 1, 7, 7, 40, },
+	{ 1, 0, 1, 7, 7, 40, },
+	{ 0, 0, 1, 7, 8, 46, },
+	{ 2, 0, 1, 7, 8, 40, },
+	{ 1, 0, 1, 7, 8, 40, },
+	{ 0, 0, 1, 7, 9, 46, },
+	{ 2, 0, 1, 7, 9, 40, },
+	{ 1, 0, 1, 7, 9, 40, },
+	{ 0, 0, 1, 7, 10, 46, },
+	{ 2, 0, 1, 7, 10, 40, },
+	{ 1, 0, 1, 7, 10, 40, },
+	{ 0, 0, 1, 7, 11, 46, },
+	{ 2, 0, 1, 7, 11, 40, },
+	{ 1, 0, 1, 7, 11, 40, },
+	{ 0, 0, 1, 7, 12, 63, },
+	{ 2, 0, 1, 7, 12, 40, },
+	{ 1, 0, 1, 7, 12, 40, },
+	{ 0, 0, 1, 7, 13, 63, },
+	{ 2, 0, 1, 7, 13, 40, },
+	{ 1, 0, 1, 7, 13, 40, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 46, },
+	{ 2, 1, 0, 1, 36, 40, },
+	{ 1, 1, 0, 1, 36, 40, },
+	{ 0, 1, 0, 1, 40, 46, },
+	{ 2, 1, 0, 1, 40, 40, },
+	{ 1, 1, 0, 1, 40, 40, },
+	{ 0, 1, 0, 1, 44, 46, },
+	{ 2, 1, 0, 1, 44, 40, },
+	{ 1, 1, 0, 1, 44, 40, },
+	{ 0, 1, 0, 1, 48, 46, },
+	{ 2, 1, 0, 1, 48, 40, },
+	{ 1, 1, 0, 1, 48, 40, },
+	{ 0, 1, 0, 1, 52, 46, },
+	{ 2, 1, 0, 1, 52, 40, },
+	{ 1, 1, 0, 1, 52, 40, },
+	{ 0, 1, 0, 1, 56, 46, },
+	{ 2, 1, 0, 1, 56, 40, },
+	{ 1, 1, 0, 1, 56, 40, },
+	{ 0, 1, 0, 1, 60, 46, },
+	{ 2, 1, 0, 1, 60, 40, },
+	{ 1, 1, 0, 1, 60, 40, },
+	{ 0, 1, 0, 1, 64, 46, },
+	{ 2, 1, 0, 1, 64, 40, },
+	{ 1, 1, 0, 1, 64, 40, },
+	{ 0, 1, 0, 1, 100, 46, },
+	{ 2, 1, 0, 1, 100, 40, },
+	{ 1, 1, 0, 1, 100, 40, },
+	{ 0, 1, 0, 1, 104, 46, },
+	{ 2, 1, 0, 1, 104, 40, },
+	{ 1, 1, 0, 1, 104, 40, },
+	{ 0, 1, 0, 1, 108, 46, },
+	{ 2, 1, 0, 1, 108, 40, },
+	{ 1, 1, 0, 1, 108, 40, },
+	{ 0, 1, 0, 1, 112, 46, },
+	{ 2, 1, 0, 1, 112, 40, },
+	{ 1, 1, 0, 1, 112, 40, },
+	{ 0, 1, 0, 1, 116, 46, },
+	{ 2, 1, 0, 1, 116, 40, },
+	{ 1, 1, 0, 1, 116, 40, },
+	{ 0, 1, 0, 1, 120, 46, },
+	{ 2, 1, 0, 1, 120, 40, },
+	{ 1, 1, 0, 1, 120, 40, },
+	{ 0, 1, 0, 1, 124, 46, },
+	{ 2, 1, 0, 1, 124, 40, },
+	{ 1, 1, 0, 1, 124, 40, },
+	{ 0, 1, 0, 1, 128, 46, },
+	{ 2, 1, 0, 1, 128, 40, },
+	{ 1, 1, 0, 1, 128, 40, },
+	{ 0, 1, 0, 1, 132, 46, },
+	{ 2, 1, 0, 1, 132, 40, },
+	{ 1, 1, 0, 1, 132, 40, },
+	{ 0, 1, 0, 1, 136, 46, },
+	{ 2, 1, 0, 1, 136, 40, },
+	{ 1, 1, 0, 1, 136, 40, },
+	{ 0, 1, 0, 1, 140, 46, },
+	{ 2, 1, 0, 1, 140, 40, },
+	{ 1, 1, 0, 1, 140, 40, },
+	{ 0, 1, 0, 1, 149, 46, },
+	{ 2, 1, 0, 1, 149, 40, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 46, },
+	{ 2, 1, 0, 1, 153, 40, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 46, },
+	{ 2, 1, 0, 1, 157, 40, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 46, },
+	{ 2, 1, 0, 1, 161, 40, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 46, },
+	{ 2, 1, 0, 1, 165, 40, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 46, },
+	{ 2, 1, 0, 2, 36, 40, },
+	{ 1, 1, 0, 2, 36, 40, },
+	{ 0, 1, 0, 2, 40, 46, },
+	{ 2, 1, 0, 2, 40, 40, },
+	{ 1, 1, 0, 2, 40, 40, },
+	{ 0, 1, 0, 2, 44, 46, },
+	{ 2, 1, 0, 2, 44, 40, },
+	{ 1, 1, 0, 2, 44, 40, },
+	{ 0, 1, 0, 2, 48, 46, },
+	{ 2, 1, 0, 2, 48, 40, },
+	{ 1, 1, 0, 2, 48, 40, },
+	{ 0, 1, 0, 2, 52, 46, },
+	{ 2, 1, 0, 2, 52, 40, },
+	{ 1, 1, 0, 2, 52, 40, },
+	{ 0, 1, 0, 2, 56, 46, },
+	{ 2, 1, 0, 2, 56, 40, },
+	{ 1, 1, 0, 2, 56, 40, },
+	{ 0, 1, 0, 2, 60, 46, },
+	{ 2, 1, 0, 2, 60, 40, },
+	{ 1, 1, 0, 2, 60, 40, },
+	{ 0, 1, 0, 2, 64, 46, },
+	{ 2, 1, 0, 2, 64, 40, },
+	{ 1, 1, 0, 2, 64, 40, },
+	{ 0, 1, 0, 2, 100, 46, },
+	{ 2, 1, 0, 2, 100, 40, },
+	{ 1, 1, 0, 2, 100, 40, },
+	{ 0, 1, 0, 2, 104, 46, },
+	{ 2, 1, 0, 2, 104, 40, },
+	{ 1, 1, 0, 2, 104, 40, },
+	{ 0, 1, 0, 2, 108, 46, },
+	{ 2, 1, 0, 2, 108, 40, },
+	{ 1, 1, 0, 2, 108, 40, },
+	{ 0, 1, 0, 2, 112, 46, },
+	{ 2, 1, 0, 2, 112, 40, },
+	{ 1, 1, 0, 2, 112, 40, },
+	{ 0, 1, 0, 2, 116, 46, },
+	{ 2, 1, 0, 2, 116, 40, },
+	{ 1, 1, 0, 2, 116, 40, },
+	{ 0, 1, 0, 2, 120, 46, },
+	{ 2, 1, 0, 2, 120, 40, },
+	{ 1, 1, 0, 2, 120, 40, },
+	{ 0, 1, 0, 2, 124, 46, },
+	{ 2, 1, 0, 2, 124, 40, },
+	{ 1, 1, 0, 2, 124, 40, },
+	{ 0, 1, 0, 2, 128, 46, },
+	{ 2, 1, 0, 2, 128, 40, },
+	{ 1, 1, 0, 2, 128, 40, },
+	{ 0, 1, 0, 2, 132, 46, },
+	{ 2, 1, 0, 2, 132, 40, },
+	{ 1, 1, 0, 2, 132, 40, },
+	{ 0, 1, 0, 2, 136, 46, },
+	{ 2, 1, 0, 2, 136, 40, },
+	{ 1, 1, 0, 2, 136, 40, },
+	{ 0, 1, 0, 2, 140, 46, },
+	{ 2, 1, 0, 2, 140, 40, },
+	{ 1, 1, 0, 2, 140, 40, },
+	{ 0, 1, 0, 2, 149, 46, },
+	{ 2, 1, 0, 2, 149, 40, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 46, },
+	{ 2, 1, 0, 2, 153, 40, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 46, },
+	{ 2, 1, 0, 2, 157, 40, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 46, },
+	{ 2, 1, 0, 2, 161, 40, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 46, },
+	{ 2, 1, 0, 2, 165, 40, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 46, },
+	{ 2, 1, 0, 3, 36, 40, },
+	{ 1, 1, 0, 3, 36, 40, },
+	{ 0, 1, 0, 3, 40, 46, },
+	{ 2, 1, 0, 3, 40, 40, },
+	{ 1, 1, 0, 3, 40, 40, },
+	{ 0, 1, 0, 3, 44, 46, },
+	{ 2, 1, 0, 3, 44, 40, },
+	{ 1, 1, 0, 3, 44, 40, },
+	{ 0, 1, 0, 3, 48, 46, },
+	{ 2, 1, 0, 3, 48, 40, },
+	{ 1, 1, 0, 3, 48, 40, },
+	{ 0, 1, 0, 3, 52, 46, },
+	{ 2, 1, 0, 3, 52, 40, },
+	{ 1, 1, 0, 3, 52, 40, },
+	{ 0, 1, 0, 3, 56, 46, },
+	{ 2, 1, 0, 3, 56, 40, },
+	{ 1, 1, 0, 3, 56, 40, },
+	{ 0, 1, 0, 3, 60, 46, },
+	{ 2, 1, 0, 3, 60, 40, },
+	{ 1, 1, 0, 3, 60, 40, },
+	{ 0, 1, 0, 3, 64, 46, },
+	{ 2, 1, 0, 3, 64, 40, },
+	{ 1, 1, 0, 3, 64, 40, },
+	{ 0, 1, 0, 3, 100, 46, },
+	{ 2, 1, 0, 3, 100, 40, },
+	{ 1, 1, 0, 3, 100, 40, },
+	{ 0, 1, 0, 3, 104, 46, },
+	{ 2, 1, 0, 3, 104, 40, },
+	{ 1, 1, 0, 3, 104, 40, },
+	{ 0, 1, 0, 3, 108, 46, },
+	{ 2, 1, 0, 3, 108, 40, },
+	{ 1, 1, 0, 3, 108, 40, },
+	{ 0, 1, 0, 3, 112, 46, },
+	{ 2, 1, 0, 3, 112, 40, },
+	{ 1, 1, 0, 3, 112, 40, },
+	{ 0, 1, 0, 3, 116, 46, },
+	{ 2, 1, 0, 3, 116, 40, },
+	{ 1, 1, 0, 3, 116, 40, },
+	{ 0, 1, 0, 3, 120, 46, },
+	{ 2, 1, 0, 3, 120, 40, },
+	{ 1, 1, 0, 3, 120, 40, },
+	{ 0, 1, 0, 3, 124, 46, },
+	{ 2, 1, 0, 3, 124, 40, },
+	{ 1, 1, 0, 3, 124, 40, },
+	{ 0, 1, 0, 3, 128, 46, },
+	{ 2, 1, 0, 3, 128, 40, },
+	{ 1, 1, 0, 3, 128, 40, },
+	{ 0, 1, 0, 3, 132, 46, },
+	{ 2, 1, 0, 3, 132, 40, },
+	{ 1, 1, 0, 3, 132, 40, },
+	{ 0, 1, 0, 3, 136, 46, },
+	{ 2, 1, 0, 3, 136, 40, },
+	{ 1, 1, 0, 3, 136, 40, },
+	{ 0, 1, 0, 3, 140, 46, },
+	{ 2, 1, 0, 3, 140, 40, },
+	{ 1, 1, 0, 3, 140, 40, },
+	{ 0, 1, 0, 3, 149, 46, },
+	{ 2, 1, 0, 3, 149, 40, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 46, },
+	{ 2, 1, 0, 3, 153, 40, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 46, },
+	{ 2, 1, 0, 3, 157, 40, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 46, },
+	{ 2, 1, 0, 3, 161, 40, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 46, },
+	{ 2, 1, 0, 3, 165, 40, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 46, },
+	{ 2, 1, 0, 6, 36, 40, },
+	{ 1, 1, 0, 6, 36, 40, },
+	{ 0, 1, 0, 6, 40, 46, },
+	{ 2, 1, 0, 6, 40, 40, },
+	{ 1, 1, 0, 6, 40, 40, },
+	{ 0, 1, 0, 6, 44, 46, },
+	{ 2, 1, 0, 6, 44, 40, },
+	{ 1, 1, 0, 6, 44, 40, },
+	{ 0, 1, 0, 6, 48, 46, },
+	{ 2, 1, 0, 6, 48, 40, },
+	{ 1, 1, 0, 6, 48, 40, },
+	{ 0, 1, 0, 6, 52, 46, },
+	{ 2, 1, 0, 6, 52, 40, },
+	{ 1, 1, 0, 6, 52, 40, },
+	{ 0, 1, 0, 6, 56, 46, },
+	{ 2, 1, 0, 6, 56, 40, },
+	{ 1, 1, 0, 6, 56, 40, },
+	{ 0, 1, 0, 6, 60, 46, },
+	{ 2, 1, 0, 6, 60, 40, },
+	{ 1, 1, 0, 6, 60, 40, },
+	{ 0, 1, 0, 6, 64, 46, },
+	{ 2, 1, 0, 6, 64, 40, },
+	{ 1, 1, 0, 6, 64, 40, },
+	{ 0, 1, 0, 6, 100, 46, },
+	{ 2, 1, 0, 6, 100, 40, },
+	{ 1, 1, 0, 6, 100, 40, },
+	{ 0, 1, 0, 6, 104, 46, },
+	{ 2, 1, 0, 6, 104, 40, },
+	{ 1, 1, 0, 6, 104, 40, },
+	{ 0, 1, 0, 6, 108, 46, },
+	{ 2, 1, 0, 6, 108, 40, },
+	{ 1, 1, 0, 6, 108, 40, },
+	{ 0, 1, 0, 6, 112, 46, },
+	{ 2, 1, 0, 6, 112, 40, },
+	{ 1, 1, 0, 6, 112, 40, },
+	{ 0, 1, 0, 6, 116, 46, },
+	{ 2, 1, 0, 6, 116, 40, },
+	{ 1, 1, 0, 6, 116, 40, },
+	{ 0, 1, 0, 6, 120, 46, },
+	{ 2, 1, 0, 6, 120, 40, },
+	{ 1, 1, 0, 6, 120, 40, },
+	{ 0, 1, 0, 6, 124, 46, },
+	{ 2, 1, 0, 6, 124, 40, },
+	{ 1, 1, 0, 6, 124, 40, },
+	{ 0, 1, 0, 6, 128, 46, },
+	{ 2, 1, 0, 6, 128, 40, },
+	{ 1, 1, 0, 6, 128, 40, },
+	{ 0, 1, 0, 6, 132, 46, },
+	{ 2, 1, 0, 6, 132, 40, },
+	{ 1, 1, 0, 6, 132, 40, },
+	{ 0, 1, 0, 6, 136, 46, },
+	{ 2, 1, 0, 6, 136, 40, },
+	{ 1, 1, 0, 6, 136, 40, },
+	{ 0, 1, 0, 6, 140, 46, },
+	{ 2, 1, 0, 6, 140, 40, },
+	{ 1, 1, 0, 6, 140, 40, },
+	{ 0, 1, 0, 6, 149, 46, },
+	{ 2, 1, 0, 6, 149, 40, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 46, },
+	{ 2, 1, 0, 6, 153, 40, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 46, },
+	{ 2, 1, 0, 6, 157, 40, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 46, },
+	{ 2, 1, 0, 6, 161, 40, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 46, },
+	{ 2, 1, 0, 6, 165, 40, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 46, },
+	{ 2, 1, 0, 7, 36, 40, },
+	{ 1, 1, 0, 7, 36, 40, },
+	{ 0, 1, 0, 7, 40, 46, },
+	{ 2, 1, 0, 7, 40, 40, },
+	{ 1, 1, 0, 7, 40, 40, },
+	{ 0, 1, 0, 7, 44, 46, },
+	{ 2, 1, 0, 7, 44, 40, },
+	{ 1, 1, 0, 7, 44, 40, },
+	{ 0, 1, 0, 7, 48, 46, },
+	{ 2, 1, 0, 7, 48, 40, },
+	{ 1, 1, 0, 7, 48, 40, },
+	{ 0, 1, 0, 7, 52, 46, },
+	{ 2, 1, 0, 7, 52, 40, },
+	{ 1, 1, 0, 7, 52, 40, },
+	{ 0, 1, 0, 7, 56, 46, },
+	{ 2, 1, 0, 7, 56, 40, },
+	{ 1, 1, 0, 7, 56, 40, },
+	{ 0, 1, 0, 7, 60, 46, },
+	{ 2, 1, 0, 7, 60, 40, },
+	{ 1, 1, 0, 7, 60, 40, },
+	{ 0, 1, 0, 7, 64, 46, },
+	{ 2, 1, 0, 7, 64, 40, },
+	{ 1, 1, 0, 7, 64, 40, },
+	{ 0, 1, 0, 7, 100, 46, },
+	{ 2, 1, 0, 7, 100, 40, },
+	{ 1, 1, 0, 7, 100, 40, },
+	{ 0, 1, 0, 7, 104, 46, },
+	{ 2, 1, 0, 7, 104, 40, },
+	{ 1, 1, 0, 7, 104, 40, },
+	{ 0, 1, 0, 7, 108, 46, },
+	{ 2, 1, 0, 7, 108, 40, },
+	{ 1, 1, 0, 7, 108, 40, },
+	{ 0, 1, 0, 7, 112, 46, },
+	{ 2, 1, 0, 7, 112, 40, },
+	{ 1, 1, 0, 7, 112, 40, },
+	{ 0, 1, 0, 7, 116, 46, },
+	{ 2, 1, 0, 7, 116, 40, },
+	{ 1, 1, 0, 7, 116, 40, },
+	{ 0, 1, 0, 7, 120, 46, },
+	{ 2, 1, 0, 7, 120, 40, },
+	{ 1, 1, 0, 7, 120, 40, },
+	{ 0, 1, 0, 7, 124, 46, },
+	{ 2, 1, 0, 7, 124, 40, },
+	{ 1, 1, 0, 7, 124, 40, },
+	{ 0, 1, 0, 7, 128, 46, },
+	{ 2, 1, 0, 7, 128, 40, },
+	{ 1, 1, 0, 7, 128, 40, },
+	{ 0, 1, 0, 7, 132, 46, },
+	{ 2, 1, 0, 7, 132, 40, },
+	{ 1, 1, 0, 7, 132, 40, },
+	{ 0, 1, 0, 7, 136, 46, },
+	{ 2, 1, 0, 7, 136, 40, },
+	{ 1, 1, 0, 7, 136, 40, },
+	{ 0, 1, 0, 7, 140, 46, },
+	{ 2, 1, 0, 7, 140, 40, },
+	{ 1, 1, 0, 7, 140, 40, },
+	{ 0, 1, 0, 7, 149, 46, },
+	{ 2, 1, 0, 7, 149, 40, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 46, },
+	{ 2, 1, 0, 7, 153, 40, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 46, },
+	{ 2, 1, 0, 7, 157, 40, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 46, },
+	{ 2, 1, 0, 7, 161, 40, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 46, },
+	{ 2, 1, 0, 7, 165, 40, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 46, },
+	{ 2, 1, 1, 2, 38, 40, },
+	{ 1, 1, 1, 2, 38, 40, },
+	{ 0, 1, 1, 2, 46, 46, },
+	{ 2, 1, 1, 2, 46, 40, },
+	{ 1, 1, 1, 2, 46, 40, },
+	{ 0, 1, 1, 2, 54, 46, },
+	{ 2, 1, 1, 2, 54, 40, },
+	{ 1, 1, 1, 2, 54, 40, },
+	{ 0, 1, 1, 2, 62, 46, },
+	{ 2, 1, 1, 2, 62, 40, },
+	{ 1, 1, 1, 2, 62, 40, },
+	{ 0, 1, 1, 2, 102, 46, },
+	{ 2, 1, 1, 2, 102, 40, },
+	{ 1, 1, 1, 2, 102, 40, },
+	{ 0, 1, 1, 2, 110, 46, },
+	{ 2, 1, 1, 2, 110, 40, },
+	{ 1, 1, 1, 2, 110, 40, },
+	{ 0, 1, 1, 2, 118, 46, },
+	{ 2, 1, 1, 2, 118, 40, },
+	{ 1, 1, 1, 2, 118, 40, },
+	{ 0, 1, 1, 2, 126, 46, },
+	{ 2, 1, 1, 2, 126, 40, },
+	{ 1, 1, 1, 2, 126, 40, },
+	{ 0, 1, 1, 2, 134, 46, },
+	{ 2, 1, 1, 2, 134, 40, },
+	{ 1, 1, 1, 2, 134, 40, },
+	{ 0, 1, 1, 2, 151, 46, },
+	{ 2, 1, 1, 2, 151, 40, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 46, },
+	{ 2, 1, 1, 2, 159, 40, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 46, },
+	{ 2, 1, 1, 3, 38, 40, },
+	{ 1, 1, 1, 3, 38, 40, },
+	{ 0, 1, 1, 3, 46, 46, },
+	{ 2, 1, 1, 3, 46, 40, },
+	{ 1, 1, 1, 3, 46, 40, },
+	{ 0, 1, 1, 3, 54, 46, },
+	{ 2, 1, 1, 3, 54, 40, },
+	{ 1, 1, 1, 3, 54, 40, },
+	{ 0, 1, 1, 3, 62, 46, },
+	{ 2, 1, 1, 3, 62, 40, },
+	{ 1, 1, 1, 3, 62, 40, },
+	{ 0, 1, 1, 3, 102, 46, },
+	{ 2, 1, 1, 3, 102, 40, },
+	{ 1, 1, 1, 3, 102, 40, },
+	{ 0, 1, 1, 3, 110, 46, },
+	{ 2, 1, 1, 3, 110, 40, },
+	{ 1, 1, 1, 3, 110, 40, },
+	{ 0, 1, 1, 3, 118, 46, },
+	{ 2, 1, 1, 3, 118, 40, },
+	{ 1, 1, 1, 3, 118, 40, },
+	{ 0, 1, 1, 3, 126, 46, },
+	{ 2, 1, 1, 3, 126, 40, },
+	{ 1, 1, 1, 3, 126, 40, },
+	{ 0, 1, 1, 3, 134, 46, },
+	{ 2, 1, 1, 3, 134, 40, },
+	{ 1, 1, 1, 3, 134, 40, },
+	{ 0, 1, 1, 3, 151, 46, },
+	{ 2, 1, 1, 3, 151, 40, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 46, },
+	{ 2, 1, 1, 3, 159, 40, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 46, },
+	{ 2, 1, 1, 6, 38, 40, },
+	{ 1, 1, 1, 6, 38, 40, },
+	{ 0, 1, 1, 6, 46, 46, },
+	{ 2, 1, 1, 6, 46, 40, },
+	{ 1, 1, 1, 6, 46, 40, },
+	{ 0, 1, 1, 6, 54, 46, },
+	{ 2, 1, 1, 6, 54, 40, },
+	{ 1, 1, 1, 6, 54, 40, },
+	{ 0, 1, 1, 6, 62, 46, },
+	{ 2, 1, 1, 6, 62, 40, },
+	{ 1, 1, 1, 6, 62, 40, },
+	{ 0, 1, 1, 6, 102, 46, },
+	{ 2, 1, 1, 6, 102, 40, },
+	{ 1, 1, 1, 6, 102, 40, },
+	{ 0, 1, 1, 6, 110, 46, },
+	{ 2, 1, 1, 6, 110, 40, },
+	{ 1, 1, 1, 6, 110, 40, },
+	{ 0, 1, 1, 6, 118, 46, },
+	{ 2, 1, 1, 6, 118, 40, },
+	{ 1, 1, 1, 6, 118, 40, },
+	{ 0, 1, 1, 6, 126, 46, },
+	{ 2, 1, 1, 6, 126, 40, },
+	{ 1, 1, 1, 6, 126, 40, },
+	{ 0, 1, 1, 6, 134, 46, },
+	{ 2, 1, 1, 6, 134, 40, },
+	{ 1, 1, 1, 6, 134, 40, },
+	{ 0, 1, 1, 6, 151, 46, },
+	{ 2, 1, 1, 6, 151, 40, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 46, },
+	{ 2, 1, 1, 6, 159, 40, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 46, },
+	{ 2, 1, 1, 7, 38, 40, },
+	{ 1, 1, 1, 7, 38, 40, },
+	{ 0, 1, 1, 7, 46, 46, },
+	{ 2, 1, 1, 7, 46, 40, },
+	{ 1, 1, 1, 7, 46, 40, },
+	{ 0, 1, 1, 7, 54, 46, },
+	{ 2, 1, 1, 7, 54, 40, },
+	{ 1, 1, 1, 7, 54, 40, },
+	{ 0, 1, 1, 7, 62, 46, },
+	{ 2, 1, 1, 7, 62, 40, },
+	{ 1, 1, 1, 7, 62, 40, },
+	{ 0, 1, 1, 7, 102, 46, },
+	{ 2, 1, 1, 7, 102, 40, },
+	{ 1, 1, 1, 7, 102, 40, },
+	{ 0, 1, 1, 7, 110, 46, },
+	{ 2, 1, 1, 7, 110, 40, },
+	{ 1, 1, 1, 7, 110, 40, },
+	{ 0, 1, 1, 7, 118, 46, },
+	{ 2, 1, 1, 7, 118, 40, },
+	{ 1, 1, 1, 7, 118, 40, },
+	{ 0, 1, 1, 7, 126, 46, },
+	{ 2, 1, 1, 7, 126, 40, },
+	{ 1, 1, 1, 7, 126, 40, },
+	{ 0, 1, 1, 7, 134, 46, },
+	{ 2, 1, 1, 7, 134, 40, },
+	{ 1, 1, 1, 7, 134, 40, },
+	{ 0, 1, 1, 7, 151, 46, },
+	{ 2, 1, 1, 7, 151, 40, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 46, },
+	{ 2, 1, 1, 7, 159, 40, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 46, },
+	{ 2, 1, 2, 4, 42, 40, },
+	{ 1, 1, 2, 4, 42, 40, },
+	{ 0, 1, 2, 4, 58, 46, },
+	{ 2, 1, 2, 4, 58, 40, },
+	{ 1, 1, 2, 4, 58, 40, },
+	{ 0, 1, 2, 4, 106, 46, },
+	{ 2, 1, 2, 4, 106, 40, },
+	{ 1, 1, 2, 4, 106, 40, },
+	{ 0, 1, 2, 4, 122, 46, },
+	{ 2, 1, 2, 4, 122, 40, },
+	{ 1, 1, 2, 4, 122, 40, },
+	{ 0, 1, 2, 4, 155, 46, },
+	{ 2, 1, 2, 4, 155, 40, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 46, },
+	{ 2, 1, 2, 5, 42, 40, },
+	{ 1, 1, 2, 5, 42, 40, },
+	{ 0, 1, 2, 5, 58, 46, },
+	{ 2, 1, 2, 5, 58, 40, },
+	{ 1, 1, 2, 5, 58, 40, },
+	{ 0, 1, 2, 5, 106, 46, },
+	{ 2, 1, 2, 5, 106, 40, },
+	{ 1, 1, 2, 5, 106, 40, },
+	{ 0, 1, 2, 5, 122, 46, },
+	{ 2, 1, 2, 5, 122, 40, },
+	{ 1, 1, 2, 5, 122, 40, },
+	{ 0, 1, 2, 5, 155, 46, },
+	{ 2, 1, 2, 5, 155, 40, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 46, },
+	{ 2, 1, 2, 8, 42, 40, },
+	{ 1, 1, 2, 8, 42, 40, },
+	{ 0, 1, 2, 8, 58, 46, },
+	{ 2, 1, 2, 8, 58, 40, },
+	{ 1, 1, 2, 8, 58, 40, },
+	{ 0, 1, 2, 8, 106, 46, },
+	{ 2, 1, 2, 8, 106, 40, },
+	{ 1, 1, 2, 8, 106, 40, },
+	{ 0, 1, 2, 8, 122, 46, },
+	{ 2, 1, 2, 8, 122, 40, },
+	{ 1, 1, 2, 8, 122, 40, },
+	{ 0, 1, 2, 8, 155, 46, },
+	{ 2, 1, 2, 8, 155, 40, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 46, },
+	{ 2, 1, 2, 9, 42, 40, },
+	{ 1, 1, 2, 9, 42, 40, },
+	{ 0, 1, 2, 9, 58, 46, },
+	{ 2, 1, 2, 9, 58, 40, },
+	{ 1, 1, 2, 9, 58, 40, },
+	{ 0, 1, 2, 9, 106, 46, },
+	{ 2, 1, 2, 9, 106, 40, },
+	{ 1, 1, 2, 9, 106, 40, },
+	{ 0, 1, 2, 9, 122, 46, },
+	{ 2, 1, 2, 9, 122, 40, },
+	{ 1, 1, 2, 9, 122, 40, },
+	{ 0, 1, 2, 9, 155, 46, },
+	{ 2, 1, 2, 9, 155, 40, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt_type3);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt_type5[] = {
+	{ 0, 0, 0, 0, 1, 46, },
+	{ 2, 0, 0, 0, 1, 40, },
+	{ 1, 0, 0, 0, 1, 40, },
+	{ 0, 0, 0, 0, 2, 46, },
+	{ 2, 0, 0, 0, 2, 40, },
+	{ 1, 0, 0, 0, 2, 40, },
+	{ 0, 0, 0, 0, 3, 46, },
+	{ 2, 0, 0, 0, 3, 40, },
+	{ 1, 0, 0, 0, 3, 40, },
+	{ 0, 0, 0, 0, 4, 46, },
+	{ 2, 0, 0, 0, 4, 40, },
+	{ 1, 0, 0, 0, 4, 40, },
+	{ 0, 0, 0, 0, 5, 46, },
+	{ 2, 0, 0, 0, 5, 40, },
+	{ 1, 0, 0, 0, 5, 40, },
+	{ 0, 0, 0, 0, 6, 46, },
+	{ 2, 0, 0, 0, 6, 40, },
+	{ 1, 0, 0, 0, 6, 40, },
+	{ 0, 0, 0, 0, 7, 46, },
+	{ 2, 0, 0, 0, 7, 40, },
+	{ 1, 0, 0, 0, 7, 40, },
+	{ 0, 0, 0, 0, 8, 46, },
+	{ 2, 0, 0, 0, 8, 40, },
+	{ 1, 0, 0, 0, 8, 40, },
+	{ 0, 0, 0, 0, 9, 46, },
+	{ 2, 0, 0, 0, 9, 40, },
+	{ 1, 0, 0, 0, 9, 40, },
+	{ 0, 0, 0, 0, 10, 46, },
+	{ 2, 0, 0, 0, 10, 40, },
+	{ 1, 0, 0, 0, 10, 40, },
+	{ 0, 0, 0, 0, 11, 46, },
+	{ 2, 0, 0, 0, 11, 40, },
+	{ 1, 0, 0, 0, 11, 40, },
+	{ 0, 0, 0, 0, 12, 63, },
+	{ 2, 0, 0, 0, 12, 40, },
+	{ 1, 0, 0, 0, 12, 40, },
+	{ 0, 0, 0, 0, 13, 63, },
+	{ 2, 0, 0, 0, 13, 40, },
+	{ 1, 0, 0, 0, 13, 40, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 40, },
+	{ 0, 0, 0, 1, 1, 46, },
+	{ 2, 0, 0, 1, 1, 40, },
+	{ 1, 0, 0, 1, 1, 40, },
+	{ 0, 0, 0, 1, 2, 46, },
+	{ 2, 0, 0, 1, 2, 40, },
+	{ 1, 0, 0, 1, 2, 40, },
+	{ 0, 0, 0, 1, 3, 46, },
+	{ 2, 0, 0, 1, 3, 40, },
+	{ 1, 0, 0, 1, 3, 40, },
+	{ 0, 0, 0, 1, 4, 46, },
+	{ 2, 0, 0, 1, 4, 40, },
+	{ 1, 0, 0, 1, 4, 40, },
+	{ 0, 0, 0, 1, 5, 46, },
+	{ 2, 0, 0, 1, 5, 40, },
+	{ 1, 0, 0, 1, 5, 40, },
+	{ 0, 0, 0, 1, 6, 46, },
+	{ 2, 0, 0, 1, 6, 40, },
+	{ 1, 0, 0, 1, 6, 40, },
+	{ 0, 0, 0, 1, 7, 46, },
+	{ 2, 0, 0, 1, 7, 40, },
+	{ 1, 0, 0, 1, 7, 40, },
+	{ 0, 0, 0, 1, 8, 46, },
+	{ 2, 0, 0, 1, 8, 40, },
+	{ 1, 0, 0, 1, 8, 40, },
+	{ 0, 0, 0, 1, 9, 46, },
+	{ 2, 0, 0, 1, 9, 40, },
+	{ 1, 0, 0, 1, 9, 40, },
+	{ 0, 0, 0, 1, 10, 46, },
+	{ 2, 0, 0, 1, 10, 40, },
+	{ 1, 0, 0, 1, 10, 40, },
+	{ 0, 0, 0, 1, 11, 46, },
+	{ 2, 0, 0, 1, 11, 40, },
+	{ 1, 0, 0, 1, 11, 40, },
+	{ 0, 0, 0, 1, 12, 63, },
+	{ 2, 0, 0, 1, 12, 40, },
+	{ 1, 0, 0, 1, 12, 40, },
+	{ 0, 0, 0, 1, 13, 63, },
+	{ 2, 0, 0, 1, 13, 40, },
+	{ 1, 0, 0, 1, 13, 40, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 63, },
+	{ 0, 0, 0, 2, 1, 46, },
+	{ 2, 0, 0, 2, 1, 40, },
+	{ 1, 0, 0, 2, 1, 40, },
+	{ 0, 0, 0, 2, 2, 46, },
+	{ 2, 0, 0, 2, 2, 40, },
+	{ 1, 0, 0, 2, 2, 40, },
+	{ 0, 0, 0, 2, 3, 46, },
+	{ 2, 0, 0, 2, 3, 40, },
+	{ 1, 0, 0, 2, 3, 40, },
+	{ 0, 0, 0, 2, 4, 46, },
+	{ 2, 0, 0, 2, 4, 40, },
+	{ 1, 0, 0, 2, 4, 40, },
+	{ 0, 0, 0, 2, 5, 46, },
+	{ 2, 0, 0, 2, 5, 40, },
+	{ 1, 0, 0, 2, 5, 40, },
+	{ 0, 0, 0, 2, 6, 46, },
+	{ 2, 0, 0, 2, 6, 40, },
+	{ 1, 0, 0, 2, 6, 40, },
+	{ 0, 0, 0, 2, 7, 46, },
+	{ 2, 0, 0, 2, 7, 40, },
+	{ 1, 0, 0, 2, 7, 40, },
+	{ 0, 0, 0, 2, 8, 46, },
+	{ 2, 0, 0, 2, 8, 40, },
+	{ 1, 0, 0, 2, 8, 40, },
+	{ 0, 0, 0, 2, 9, 46, },
+	{ 2, 0, 0, 2, 9, 40, },
+	{ 1, 0, 0, 2, 9, 40, },
+	{ 0, 0, 0, 2, 10, 46, },
+	{ 2, 0, 0, 2, 10, 40, },
+	{ 1, 0, 0, 2, 10, 40, },
+	{ 0, 0, 0, 2, 11, 46, },
+	{ 2, 0, 0, 2, 11, 40, },
+	{ 1, 0, 0, 2, 11, 40, },
+	{ 0, 0, 0, 2, 12, 63, },
+	{ 2, 0, 0, 2, 12, 40, },
+	{ 1, 0, 0, 2, 12, 40, },
+	{ 0, 0, 0, 2, 13, 63, },
+	{ 2, 0, 0, 2, 13, 40, },
+	{ 1, 0, 0, 2, 13, 40, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 63, },
+	{ 0, 0, 0, 3, 1, 46, },
+	{ 2, 0, 0, 3, 1, 40, },
+	{ 1, 0, 0, 3, 1, 40, },
+	{ 0, 0, 0, 3, 2, 46, },
+	{ 2, 0, 0, 3, 2, 40, },
+	{ 1, 0, 0, 3, 2, 40, },
+	{ 0, 0, 0, 3, 3, 46, },
+	{ 2, 0, 0, 3, 3, 40, },
+	{ 1, 0, 0, 3, 3, 40, },
+	{ 0, 0, 0, 3, 4, 46, },
+	{ 2, 0, 0, 3, 4, 40, },
+	{ 1, 0, 0, 3, 4, 40, },
+	{ 0, 0, 0, 3, 5, 46, },
+	{ 2, 0, 0, 3, 5, 40, },
+	{ 1, 0, 0, 3, 5, 40, },
+	{ 0, 0, 0, 3, 6, 46, },
+	{ 2, 0, 0, 3, 6, 40, },
+	{ 1, 0, 0, 3, 6, 40, },
+	{ 0, 0, 0, 3, 7, 46, },
+	{ 2, 0, 0, 3, 7, 40, },
+	{ 1, 0, 0, 3, 7, 40, },
+	{ 0, 0, 0, 3, 8, 46, },
+	{ 2, 0, 0, 3, 8, 40, },
+	{ 1, 0, 0, 3, 8, 40, },
+	{ 0, 0, 0, 3, 9, 46, },
+	{ 2, 0, 0, 3, 9, 40, },
+	{ 1, 0, 0, 3, 9, 40, },
+	{ 0, 0, 0, 3, 10, 46, },
+	{ 2, 0, 0, 3, 10, 40, },
+	{ 1, 0, 0, 3, 10, 40, },
+	{ 0, 0, 0, 3, 11, 46, },
+	{ 2, 0, 0, 3, 11, 40, },
+	{ 1, 0, 0, 3, 11, 40, },
+	{ 0, 0, 0, 3, 12, 63, },
+	{ 2, 0, 0, 3, 12, 40, },
+	{ 1, 0, 0, 3, 12, 40, },
+	{ 0, 0, 0, 3, 13, 63, },
+	{ 2, 0, 0, 3, 13, 40, },
+	{ 1, 0, 0, 3, 13, 40, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 63, },
+	{ 0, 0, 0, 6, 1, 46, },
+	{ 2, 0, 0, 6, 1, 40, },
+	{ 1, 0, 0, 6, 1, 40, },
+	{ 0, 0, 0, 6, 2, 46, },
+	{ 2, 0, 0, 6, 2, 40, },
+	{ 1, 0, 0, 6, 2, 40, },
+	{ 0, 0, 0, 6, 3, 46, },
+	{ 2, 0, 0, 6, 3, 40, },
+	{ 1, 0, 0, 6, 3, 40, },
+	{ 0, 0, 0, 6, 4, 46, },
+	{ 2, 0, 0, 6, 4, 40, },
+	{ 1, 0, 0, 6, 4, 40, },
+	{ 0, 0, 0, 6, 5, 46, },
+	{ 2, 0, 0, 6, 5, 40, },
+	{ 1, 0, 0, 6, 5, 40, },
+	{ 0, 0, 0, 6, 6, 46, },
+	{ 2, 0, 0, 6, 6, 40, },
+	{ 1, 0, 0, 6, 6, 40, },
+	{ 0, 0, 0, 6, 7, 46, },
+	{ 2, 0, 0, 6, 7, 40, },
+	{ 1, 0, 0, 6, 7, 40, },
+	{ 0, 0, 0, 6, 8, 46, },
+	{ 2, 0, 0, 6, 8, 40, },
+	{ 1, 0, 0, 6, 8, 40, },
+	{ 0, 0, 0, 6, 9, 46, },
+	{ 2, 0, 0, 6, 9, 40, },
+	{ 1, 0, 0, 6, 9, 40, },
+	{ 0, 0, 0, 6, 10, 46, },
+	{ 2, 0, 0, 6, 10, 40, },
+	{ 1, 0, 0, 6, 10, 40, },
+	{ 0, 0, 0, 6, 11, 46, },
+	{ 2, 0, 0, 6, 11, 40, },
+	{ 1, 0, 0, 6, 11, 40, },
+	{ 0, 0, 0, 6, 12, 63, },
+	{ 2, 0, 0, 6, 12, 40, },
+	{ 1, 0, 0, 6, 12, 40, },
+	{ 0, 0, 0, 6, 13, 63, },
+	{ 2, 0, 0, 6, 13, 40, },
+	{ 1, 0, 0, 6, 13, 40, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 63, },
+	{ 0, 0, 0, 7, 1, 46, },
+	{ 2, 0, 0, 7, 1, 40, },
+	{ 1, 0, 0, 7, 1, 40, },
+	{ 0, 0, 0, 7, 2, 46, },
+	{ 2, 0, 0, 7, 2, 40, },
+	{ 1, 0, 0, 7, 2, 40, },
+	{ 0, 0, 0, 7, 3, 46, },
+	{ 2, 0, 0, 7, 3, 40, },
+	{ 1, 0, 0, 7, 3, 40, },
+	{ 0, 0, 0, 7, 4, 46, },
+	{ 2, 0, 0, 7, 4, 40, },
+	{ 1, 0, 0, 7, 4, 40, },
+	{ 0, 0, 0, 7, 5, 46, },
+	{ 2, 0, 0, 7, 5, 40, },
+	{ 1, 0, 0, 7, 5, 40, },
+	{ 0, 0, 0, 7, 6, 46, },
+	{ 2, 0, 0, 7, 6, 40, },
+	{ 1, 0, 0, 7, 6, 40, },
+	{ 0, 0, 0, 7, 7, 46, },
+	{ 2, 0, 0, 7, 7, 40, },
+	{ 1, 0, 0, 7, 7, 40, },
+	{ 0, 0, 0, 7, 8, 46, },
+	{ 2, 0, 0, 7, 8, 40, },
+	{ 1, 0, 0, 7, 8, 40, },
+	{ 0, 0, 0, 7, 9, 46, },
+	{ 2, 0, 0, 7, 9, 40, },
+	{ 1, 0, 0, 7, 9, 40, },
+	{ 0, 0, 0, 7, 10, 46, },
+	{ 2, 0, 0, 7, 10, 40, },
+	{ 1, 0, 0, 7, 10, 40, },
+	{ 0, 0, 0, 7, 11, 46, },
+	{ 2, 0, 0, 7, 11, 40, },
+	{ 1, 0, 0, 7, 11, 40, },
+	{ 0, 0, 0, 7, 12, 63, },
+	{ 2, 0, 0, 7, 12, 40, },
+	{ 1, 0, 0, 7, 12, 40, },
+	{ 0, 0, 0, 7, 13, 63, },
+	{ 2, 0, 0, 7, 13, 40, },
+	{ 1, 0, 0, 7, 13, 40, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 63, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 46, },
+	{ 2, 0, 1, 2, 3, 40, },
+	{ 1, 0, 1, 2, 3, 40, },
+	{ 0, 0, 1, 2, 4, 46, },
+	{ 2, 0, 1, 2, 4, 40, },
+	{ 1, 0, 1, 2, 4, 40, },
+	{ 0, 0, 1, 2, 5, 46, },
+	{ 2, 0, 1, 2, 5, 40, },
+	{ 1, 0, 1, 2, 5, 40, },
+	{ 0, 0, 1, 2, 6, 46, },
+	{ 2, 0, 1, 2, 6, 40, },
+	{ 1, 0, 1, 2, 6, 40, },
+	{ 0, 0, 1, 2, 7, 46, },
+	{ 2, 0, 1, 2, 7, 40, },
+	{ 1, 0, 1, 2, 7, 40, },
+	{ 0, 0, 1, 2, 8, 46, },
+	{ 2, 0, 1, 2, 8, 40, },
+	{ 1, 0, 1, 2, 8, 40, },
+	{ 0, 0, 1, 2, 9, 46, },
+	{ 2, 0, 1, 2, 9, 40, },
+	{ 1, 0, 1, 2, 9, 40, },
+	{ 0, 0, 1, 2, 10, 46, },
+	{ 2, 0, 1, 2, 10, 40, },
+	{ 1, 0, 1, 2, 10, 40, },
+	{ 0, 0, 1, 2, 11, 46, },
+	{ 2, 0, 1, 2, 11, 40, },
+	{ 1, 0, 1, 2, 11, 40, },
+	{ 0, 0, 1, 2, 12, 63, },
+	{ 2, 0, 1, 2, 12, 40, },
+	{ 1, 0, 1, 2, 12, 40, },
+	{ 0, 0, 1, 2, 13, 63, },
+	{ 2, 0, 1, 2, 13, 40, },
+	{ 1, 0, 1, 2, 13, 40, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 46, },
+	{ 2, 0, 1, 3, 3, 40, },
+	{ 1, 0, 1, 3, 3, 40, },
+	{ 0, 0, 1, 3, 4, 46, },
+	{ 2, 0, 1, 3, 4, 40, },
+	{ 1, 0, 1, 3, 4, 40, },
+	{ 0, 0, 1, 3, 5, 46, },
+	{ 2, 0, 1, 3, 5, 40, },
+	{ 1, 0, 1, 3, 5, 40, },
+	{ 0, 0, 1, 3, 6, 46, },
+	{ 2, 0, 1, 3, 6, 40, },
+	{ 1, 0, 1, 3, 6, 40, },
+	{ 0, 0, 1, 3, 7, 46, },
+	{ 2, 0, 1, 3, 7, 40, },
+	{ 1, 0, 1, 3, 7, 40, },
+	{ 0, 0, 1, 3, 8, 46, },
+	{ 2, 0, 1, 3, 8, 40, },
+	{ 1, 0, 1, 3, 8, 40, },
+	{ 0, 0, 1, 3, 9, 46, },
+	{ 2, 0, 1, 3, 9, 40, },
+	{ 1, 0, 1, 3, 9, 40, },
+	{ 0, 0, 1, 3, 10, 46, },
+	{ 2, 0, 1, 3, 10, 40, },
+	{ 1, 0, 1, 3, 10, 40, },
+	{ 0, 0, 1, 3, 11, 46, },
+	{ 2, 0, 1, 3, 11, 40, },
+	{ 1, 0, 1, 3, 11, 40, },
+	{ 0, 0, 1, 3, 12, 63, },
+	{ 2, 0, 1, 3, 12, 40, },
+	{ 1, 0, 1, 3, 12, 40, },
+	{ 0, 0, 1, 3, 13, 63, },
+	{ 2, 0, 1, 3, 13, 40, },
+	{ 1, 0, 1, 3, 13, 40, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 46, },
+	{ 2, 0, 1, 6, 3, 40, },
+	{ 1, 0, 1, 6, 3, 40, },
+	{ 0, 0, 1, 6, 4, 46, },
+	{ 2, 0, 1, 6, 4, 40, },
+	{ 1, 0, 1, 6, 4, 40, },
+	{ 0, 0, 1, 6, 5, 46, },
+	{ 2, 0, 1, 6, 5, 40, },
+	{ 1, 0, 1, 6, 5, 40, },
+	{ 0, 0, 1, 6, 6, 46, },
+	{ 2, 0, 1, 6, 6, 40, },
+	{ 1, 0, 1, 6, 6, 40, },
+	{ 0, 0, 1, 6, 7, 46, },
+	{ 2, 0, 1, 6, 7, 40, },
+	{ 1, 0, 1, 6, 7, 40, },
+	{ 0, 0, 1, 6, 8, 46, },
+	{ 2, 0, 1, 6, 8, 40, },
+	{ 1, 0, 1, 6, 8, 40, },
+	{ 0, 0, 1, 6, 9, 46, },
+	{ 2, 0, 1, 6, 9, 40, },
+	{ 1, 0, 1, 6, 9, 40, },
+	{ 0, 0, 1, 6, 10, 46, },
+	{ 2, 0, 1, 6, 10, 40, },
+	{ 1, 0, 1, 6, 10, 40, },
+	{ 0, 0, 1, 6, 11, 46, },
+	{ 2, 0, 1, 6, 11, 40, },
+	{ 1, 0, 1, 6, 11, 40, },
+	{ 0, 0, 1, 6, 12, 63, },
+	{ 2, 0, 1, 6, 12, 40, },
+	{ 1, 0, 1, 6, 12, 40, },
+	{ 0, 0, 1, 6, 13, 63, },
+	{ 2, 0, 1, 6, 13, 40, },
+	{ 1, 0, 1, 6, 13, 40, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 46, },
+	{ 2, 0, 1, 7, 3, 40, },
+	{ 1, 0, 1, 7, 3, 40, },
+	{ 0, 0, 1, 7, 4, 46, },
+	{ 2, 0, 1, 7, 4, 40, },
+	{ 1, 0, 1, 7, 4, 40, },
+	{ 0, 0, 1, 7, 5, 46, },
+	{ 2, 0, 1, 7, 5, 40, },
+	{ 1, 0, 1, 7, 5, 40, },
+	{ 0, 0, 1, 7, 6, 46, },
+	{ 2, 0, 1, 7, 6, 40, },
+	{ 1, 0, 1, 7, 6, 40, },
+	{ 0, 0, 1, 7, 7, 46, },
+	{ 2, 0, 1, 7, 7, 40, },
+	{ 1, 0, 1, 7, 7, 40, },
+	{ 0, 0, 1, 7, 8, 46, },
+	{ 2, 0, 1, 7, 8, 40, },
+	{ 1, 0, 1, 7, 8, 40, },
+	{ 0, 0, 1, 7, 9, 46, },
+	{ 2, 0, 1, 7, 9, 40, },
+	{ 1, 0, 1, 7, 9, 40, },
+	{ 0, 0, 1, 7, 10, 46, },
+	{ 2, 0, 1, 7, 10, 40, },
+	{ 1, 0, 1, 7, 10, 40, },
+	{ 0, 0, 1, 7, 11, 46, },
+	{ 2, 0, 1, 7, 11, 40, },
+	{ 1, 0, 1, 7, 11, 40, },
+	{ 0, 0, 1, 7, 12, 63, },
+	{ 2, 0, 1, 7, 12, 40, },
+	{ 1, 0, 1, 7, 12, 40, },
+	{ 0, 0, 1, 7, 13, 63, },
+	{ 2, 0, 1, 7, 13, 40, },
+	{ 1, 0, 1, 7, 13, 40, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 46, },
+	{ 2, 1, 0, 1, 36, 40, },
+	{ 1, 1, 0, 1, 36, 40, },
+	{ 0, 1, 0, 1, 40, 46, },
+	{ 2, 1, 0, 1, 40, 40, },
+	{ 1, 1, 0, 1, 40, 40, },
+	{ 0, 1, 0, 1, 44, 46, },
+	{ 2, 1, 0, 1, 44, 40, },
+	{ 1, 1, 0, 1, 44, 40, },
+	{ 0, 1, 0, 1, 48, 46, },
+	{ 2, 1, 0, 1, 48, 40, },
+	{ 1, 1, 0, 1, 48, 40, },
+	{ 0, 1, 0, 1, 52, 46, },
+	{ 2, 1, 0, 1, 52, 40, },
+	{ 1, 1, 0, 1, 52, 40, },
+	{ 0, 1, 0, 1, 56, 46, },
+	{ 2, 1, 0, 1, 56, 40, },
+	{ 1, 1, 0, 1, 56, 40, },
+	{ 0, 1, 0, 1, 60, 46, },
+	{ 2, 1, 0, 1, 60, 40, },
+	{ 1, 1, 0, 1, 60, 40, },
+	{ 0, 1, 0, 1, 64, 46, },
+	{ 2, 1, 0, 1, 64, 40, },
+	{ 1, 1, 0, 1, 64, 40, },
+	{ 0, 1, 0, 1, 100, 46, },
+	{ 2, 1, 0, 1, 100, 40, },
+	{ 1, 1, 0, 1, 100, 40, },
+	{ 0, 1, 0, 1, 104, 46, },
+	{ 2, 1, 0, 1, 104, 40, },
+	{ 1, 1, 0, 1, 104, 40, },
+	{ 0, 1, 0, 1, 108, 46, },
+	{ 2, 1, 0, 1, 108, 40, },
+	{ 1, 1, 0, 1, 108, 40, },
+	{ 0, 1, 0, 1, 112, 46, },
+	{ 2, 1, 0, 1, 112, 40, },
+	{ 1, 1, 0, 1, 112, 40, },
+	{ 0, 1, 0, 1, 116, 46, },
+	{ 2, 1, 0, 1, 116, 40, },
+	{ 1, 1, 0, 1, 116, 40, },
+	{ 0, 1, 0, 1, 120, 46, },
+	{ 2, 1, 0, 1, 120, 40, },
+	{ 1, 1, 0, 1, 120, 40, },
+	{ 0, 1, 0, 1, 124, 46, },
+	{ 2, 1, 0, 1, 124, 40, },
+	{ 1, 1, 0, 1, 124, 40, },
+	{ 0, 1, 0, 1, 128, 46, },
+	{ 2, 1, 0, 1, 128, 40, },
+	{ 1, 1, 0, 1, 128, 40, },
+	{ 0, 1, 0, 1, 132, 46, },
+	{ 2, 1, 0, 1, 132, 40, },
+	{ 1, 1, 0, 1, 132, 40, },
+	{ 0, 1, 0, 1, 136, 46, },
+	{ 2, 1, 0, 1, 136, 40, },
+	{ 1, 1, 0, 1, 136, 40, },
+	{ 0, 1, 0, 1, 140, 46, },
+	{ 2, 1, 0, 1, 140, 40, },
+	{ 1, 1, 0, 1, 140, 40, },
+	{ 0, 1, 0, 1, 149, 46, },
+	{ 2, 1, 0, 1, 149, 40, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 46, },
+	{ 2, 1, 0, 1, 153, 40, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 46, },
+	{ 2, 1, 0, 1, 157, 40, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 46, },
+	{ 2, 1, 0, 1, 161, 40, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 46, },
+	{ 2, 1, 0, 1, 165, 40, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 46, },
+	{ 2, 1, 0, 2, 36, 40, },
+	{ 1, 1, 0, 2, 36, 40, },
+	{ 0, 1, 0, 2, 40, 46, },
+	{ 2, 1, 0, 2, 40, 40, },
+	{ 1, 1, 0, 2, 40, 40, },
+	{ 0, 1, 0, 2, 44, 46, },
+	{ 2, 1, 0, 2, 44, 40, },
+	{ 1, 1, 0, 2, 44, 40, },
+	{ 0, 1, 0, 2, 48, 46, },
+	{ 2, 1, 0, 2, 48, 40, },
+	{ 1, 1, 0, 2, 48, 40, },
+	{ 0, 1, 0, 2, 52, 46, },
+	{ 2, 1, 0, 2, 52, 40, },
+	{ 1, 1, 0, 2, 52, 40, },
+	{ 0, 1, 0, 2, 56, 46, },
+	{ 2, 1, 0, 2, 56, 40, },
+	{ 1, 1, 0, 2, 56, 40, },
+	{ 0, 1, 0, 2, 60, 46, },
+	{ 2, 1, 0, 2, 60, 40, },
+	{ 1, 1, 0, 2, 60, 40, },
+	{ 0, 1, 0, 2, 64, 46, },
+	{ 2, 1, 0, 2, 64, 40, },
+	{ 1, 1, 0, 2, 64, 40, },
+	{ 0, 1, 0, 2, 100, 46, },
+	{ 2, 1, 0, 2, 100, 40, },
+	{ 1, 1, 0, 2, 100, 40, },
+	{ 0, 1, 0, 2, 104, 46, },
+	{ 2, 1, 0, 2, 104, 40, },
+	{ 1, 1, 0, 2, 104, 40, },
+	{ 0, 1, 0, 2, 108, 46, },
+	{ 2, 1, 0, 2, 108, 40, },
+	{ 1, 1, 0, 2, 108, 40, },
+	{ 0, 1, 0, 2, 112, 46, },
+	{ 2, 1, 0, 2, 112, 40, },
+	{ 1, 1, 0, 2, 112, 40, },
+	{ 0, 1, 0, 2, 116, 46, },
+	{ 2, 1, 0, 2, 116, 40, },
+	{ 1, 1, 0, 2, 116, 40, },
+	{ 0, 1, 0, 2, 120, 46, },
+	{ 2, 1, 0, 2, 120, 40, },
+	{ 1, 1, 0, 2, 120, 40, },
+	{ 0, 1, 0, 2, 124, 46, },
+	{ 2, 1, 0, 2, 124, 40, },
+	{ 1, 1, 0, 2, 124, 40, },
+	{ 0, 1, 0, 2, 128, 46, },
+	{ 2, 1, 0, 2, 128, 40, },
+	{ 1, 1, 0, 2, 128, 40, },
+	{ 0, 1, 0, 2, 132, 46, },
+	{ 2, 1, 0, 2, 132, 40, },
+	{ 1, 1, 0, 2, 132, 40, },
+	{ 0, 1, 0, 2, 136, 46, },
+	{ 2, 1, 0, 2, 136, 40, },
+	{ 1, 1, 0, 2, 136, 40, },
+	{ 0, 1, 0, 2, 140, 46, },
+	{ 2, 1, 0, 2, 140, 40, },
+	{ 1, 1, 0, 2, 140, 40, },
+	{ 0, 1, 0, 2, 149, 46, },
+	{ 2, 1, 0, 2, 149, 40, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 46, },
+	{ 2, 1, 0, 2, 153, 40, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 46, },
+	{ 2, 1, 0, 2, 157, 40, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 46, },
+	{ 2, 1, 0, 2, 161, 40, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 46, },
+	{ 2, 1, 0, 2, 165, 40, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 46, },
+	{ 2, 1, 0, 3, 36, 40, },
+	{ 1, 1, 0, 3, 36, 40, },
+	{ 0, 1, 0, 3, 40, 46, },
+	{ 2, 1, 0, 3, 40, 40, },
+	{ 1, 1, 0, 3, 40, 40, },
+	{ 0, 1, 0, 3, 44, 46, },
+	{ 2, 1, 0, 3, 44, 40, },
+	{ 1, 1, 0, 3, 44, 40, },
+	{ 0, 1, 0, 3, 48, 46, },
+	{ 2, 1, 0, 3, 48, 40, },
+	{ 1, 1, 0, 3, 48, 40, },
+	{ 0, 1, 0, 3, 52, 46, },
+	{ 2, 1, 0, 3, 52, 40, },
+	{ 1, 1, 0, 3, 52, 40, },
+	{ 0, 1, 0, 3, 56, 46, },
+	{ 2, 1, 0, 3, 56, 40, },
+	{ 1, 1, 0, 3, 56, 40, },
+	{ 0, 1, 0, 3, 60, 46, },
+	{ 2, 1, 0, 3, 60, 40, },
+	{ 1, 1, 0, 3, 60, 40, },
+	{ 0, 1, 0, 3, 64, 46, },
+	{ 2, 1, 0, 3, 64, 40, },
+	{ 1, 1, 0, 3, 64, 40, },
+	{ 0, 1, 0, 3, 100, 46, },
+	{ 2, 1, 0, 3, 100, 40, },
+	{ 1, 1, 0, 3, 100, 40, },
+	{ 0, 1, 0, 3, 104, 46, },
+	{ 2, 1, 0, 3, 104, 40, },
+	{ 1, 1, 0, 3, 104, 40, },
+	{ 0, 1, 0, 3, 108, 46, },
+	{ 2, 1, 0, 3, 108, 40, },
+	{ 1, 1, 0, 3, 108, 40, },
+	{ 0, 1, 0, 3, 112, 46, },
+	{ 2, 1, 0, 3, 112, 40, },
+	{ 1, 1, 0, 3, 112, 40, },
+	{ 0, 1, 0, 3, 116, 46, },
+	{ 2, 1, 0, 3, 116, 40, },
+	{ 1, 1, 0, 3, 116, 40, },
+	{ 0, 1, 0, 3, 120, 46, },
+	{ 2, 1, 0, 3, 120, 40, },
+	{ 1, 1, 0, 3, 120, 40, },
+	{ 0, 1, 0, 3, 124, 46, },
+	{ 2, 1, 0, 3, 124, 40, },
+	{ 1, 1, 0, 3, 124, 40, },
+	{ 0, 1, 0, 3, 128, 46, },
+	{ 2, 1, 0, 3, 128, 40, },
+	{ 1, 1, 0, 3, 128, 40, },
+	{ 0, 1, 0, 3, 132, 46, },
+	{ 2, 1, 0, 3, 132, 40, },
+	{ 1, 1, 0, 3, 132, 40, },
+	{ 0, 1, 0, 3, 136, 46, },
+	{ 2, 1, 0, 3, 136, 40, },
+	{ 1, 1, 0, 3, 136, 40, },
+	{ 0, 1, 0, 3, 140, 46, },
+	{ 2, 1, 0, 3, 140, 40, },
+	{ 1, 1, 0, 3, 140, 40, },
+	{ 0, 1, 0, 3, 149, 46, },
+	{ 2, 1, 0, 3, 149, 40, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 46, },
+	{ 2, 1, 0, 3, 153, 40, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 46, },
+	{ 2, 1, 0, 3, 157, 40, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 46, },
+	{ 2, 1, 0, 3, 161, 40, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 46, },
+	{ 2, 1, 0, 3, 165, 40, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 46, },
+	{ 2, 1, 0, 6, 36, 40, },
+	{ 1, 1, 0, 6, 36, 40, },
+	{ 0, 1, 0, 6, 40, 46, },
+	{ 2, 1, 0, 6, 40, 40, },
+	{ 1, 1, 0, 6, 40, 40, },
+	{ 0, 1, 0, 6, 44, 46, },
+	{ 2, 1, 0, 6, 44, 40, },
+	{ 1, 1, 0, 6, 44, 40, },
+	{ 0, 1, 0, 6, 48, 46, },
+	{ 2, 1, 0, 6, 48, 40, },
+	{ 1, 1, 0, 6, 48, 40, },
+	{ 0, 1, 0, 6, 52, 46, },
+	{ 2, 1, 0, 6, 52, 40, },
+	{ 1, 1, 0, 6, 52, 40, },
+	{ 0, 1, 0, 6, 56, 46, },
+	{ 2, 1, 0, 6, 56, 40, },
+	{ 1, 1, 0, 6, 56, 40, },
+	{ 0, 1, 0, 6, 60, 46, },
+	{ 2, 1, 0, 6, 60, 40, },
+	{ 1, 1, 0, 6, 60, 40, },
+	{ 0, 1, 0, 6, 64, 46, },
+	{ 2, 1, 0, 6, 64, 40, },
+	{ 1, 1, 0, 6, 64, 40, },
+	{ 0, 1, 0, 6, 100, 46, },
+	{ 2, 1, 0, 6, 100, 40, },
+	{ 1, 1, 0, 6, 100, 40, },
+	{ 0, 1, 0, 6, 104, 46, },
+	{ 2, 1, 0, 6, 104, 40, },
+	{ 1, 1, 0, 6, 104, 40, },
+	{ 0, 1, 0, 6, 108, 46, },
+	{ 2, 1, 0, 6, 108, 40, },
+	{ 1, 1, 0, 6, 108, 40, },
+	{ 0, 1, 0, 6, 112, 46, },
+	{ 2, 1, 0, 6, 112, 40, },
+	{ 1, 1, 0, 6, 112, 40, },
+	{ 0, 1, 0, 6, 116, 46, },
+	{ 2, 1, 0, 6, 116, 40, },
+	{ 1, 1, 0, 6, 116, 40, },
+	{ 0, 1, 0, 6, 120, 46, },
+	{ 2, 1, 0, 6, 120, 40, },
+	{ 1, 1, 0, 6, 120, 40, },
+	{ 0, 1, 0, 6, 124, 46, },
+	{ 2, 1, 0, 6, 124, 40, },
+	{ 1, 1, 0, 6, 124, 40, },
+	{ 0, 1, 0, 6, 128, 46, },
+	{ 2, 1, 0, 6, 128, 40, },
+	{ 1, 1, 0, 6, 128, 40, },
+	{ 0, 1, 0, 6, 132, 46, },
+	{ 2, 1, 0, 6, 132, 40, },
+	{ 1, 1, 0, 6, 132, 40, },
+	{ 0, 1, 0, 6, 136, 46, },
+	{ 2, 1, 0, 6, 136, 40, },
+	{ 1, 1, 0, 6, 136, 40, },
+	{ 0, 1, 0, 6, 140, 46, },
+	{ 2, 1, 0, 6, 140, 40, },
+	{ 1, 1, 0, 6, 140, 40, },
+	{ 0, 1, 0, 6, 149, 46, },
+	{ 2, 1, 0, 6, 149, 40, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 46, },
+	{ 2, 1, 0, 6, 153, 40, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 46, },
+	{ 2, 1, 0, 6, 157, 40, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 46, },
+	{ 2, 1, 0, 6, 161, 40, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 46, },
+	{ 2, 1, 0, 6, 165, 40, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 46, },
+	{ 2, 1, 0, 7, 36, 40, },
+	{ 1, 1, 0, 7, 36, 40, },
+	{ 0, 1, 0, 7, 40, 46, },
+	{ 2, 1, 0, 7, 40, 40, },
+	{ 1, 1, 0, 7, 40, 40, },
+	{ 0, 1, 0, 7, 44, 46, },
+	{ 2, 1, 0, 7, 44, 40, },
+	{ 1, 1, 0, 7, 44, 40, },
+	{ 0, 1, 0, 7, 48, 46, },
+	{ 2, 1, 0, 7, 48, 40, },
+	{ 1, 1, 0, 7, 48, 40, },
+	{ 0, 1, 0, 7, 52, 46, },
+	{ 2, 1, 0, 7, 52, 40, },
+	{ 1, 1, 0, 7, 52, 40, },
+	{ 0, 1, 0, 7, 56, 46, },
+	{ 2, 1, 0, 7, 56, 40, },
+	{ 1, 1, 0, 7, 56, 40, },
+	{ 0, 1, 0, 7, 60, 46, },
+	{ 2, 1, 0, 7, 60, 40, },
+	{ 1, 1, 0, 7, 60, 40, },
+	{ 0, 1, 0, 7, 64, 46, },
+	{ 2, 1, 0, 7, 64, 40, },
+	{ 1, 1, 0, 7, 64, 40, },
+	{ 0, 1, 0, 7, 100, 46, },
+	{ 2, 1, 0, 7, 100, 40, },
+	{ 1, 1, 0, 7, 100, 40, },
+	{ 0, 1, 0, 7, 104, 46, },
+	{ 2, 1, 0, 7, 104, 40, },
+	{ 1, 1, 0, 7, 104, 40, },
+	{ 0, 1, 0, 7, 108, 46, },
+	{ 2, 1, 0, 7, 108, 40, },
+	{ 1, 1, 0, 7, 108, 40, },
+	{ 0, 1, 0, 7, 112, 46, },
+	{ 2, 1, 0, 7, 112, 40, },
+	{ 1, 1, 0, 7, 112, 40, },
+	{ 0, 1, 0, 7, 116, 46, },
+	{ 2, 1, 0, 7, 116, 40, },
+	{ 1, 1, 0, 7, 116, 40, },
+	{ 0, 1, 0, 7, 120, 46, },
+	{ 2, 1, 0, 7, 120, 40, },
+	{ 1, 1, 0, 7, 120, 40, },
+	{ 0, 1, 0, 7, 124, 46, },
+	{ 2, 1, 0, 7, 124, 40, },
+	{ 1, 1, 0, 7, 124, 40, },
+	{ 0, 1, 0, 7, 128, 46, },
+	{ 2, 1, 0, 7, 128, 40, },
+	{ 1, 1, 0, 7, 128, 40, },
+	{ 0, 1, 0, 7, 132, 46, },
+	{ 2, 1, 0, 7, 132, 40, },
+	{ 1, 1, 0, 7, 132, 40, },
+	{ 0, 1, 0, 7, 136, 46, },
+	{ 2, 1, 0, 7, 136, 40, },
+	{ 1, 1, 0, 7, 136, 40, },
+	{ 0, 1, 0, 7, 140, 46, },
+	{ 2, 1, 0, 7, 140, 40, },
+	{ 1, 1, 0, 7, 140, 40, },
+	{ 0, 1, 0, 7, 149, 46, },
+	{ 2, 1, 0, 7, 149, 40, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 46, },
+	{ 2, 1, 0, 7, 153, 40, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 46, },
+	{ 2, 1, 0, 7, 157, 40, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 46, },
+	{ 2, 1, 0, 7, 161, 40, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 46, },
+	{ 2, 1, 0, 7, 165, 40, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 46, },
+	{ 2, 1, 1, 2, 38, 40, },
+	{ 1, 1, 1, 2, 38, 40, },
+	{ 0, 1, 1, 2, 46, 46, },
+	{ 2, 1, 1, 2, 46, 40, },
+	{ 1, 1, 1, 2, 46, 40, },
+	{ 0, 1, 1, 2, 54, 46, },
+	{ 2, 1, 1, 2, 54, 40, },
+	{ 1, 1, 1, 2, 54, 40, },
+	{ 0, 1, 1, 2, 62, 46, },
+	{ 2, 1, 1, 2, 62, 40, },
+	{ 1, 1, 1, 2, 62, 40, },
+	{ 0, 1, 1, 2, 102, 46, },
+	{ 2, 1, 1, 2, 102, 40, },
+	{ 1, 1, 1, 2, 102, 40, },
+	{ 0, 1, 1, 2, 110, 46, },
+	{ 2, 1, 1, 2, 110, 40, },
+	{ 1, 1, 1, 2, 110, 40, },
+	{ 0, 1, 1, 2, 118, 46, },
+	{ 2, 1, 1, 2, 118, 40, },
+	{ 1, 1, 1, 2, 118, 40, },
+	{ 0, 1, 1, 2, 126, 46, },
+	{ 2, 1, 1, 2, 126, 40, },
+	{ 1, 1, 1, 2, 126, 40, },
+	{ 0, 1, 1, 2, 134, 46, },
+	{ 2, 1, 1, 2, 134, 40, },
+	{ 1, 1, 1, 2, 134, 40, },
+	{ 0, 1, 1, 2, 151, 46, },
+	{ 2, 1, 1, 2, 151, 40, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 46, },
+	{ 2, 1, 1, 2, 159, 40, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 46, },
+	{ 2, 1, 1, 3, 38, 40, },
+	{ 1, 1, 1, 3, 38, 40, },
+	{ 0, 1, 1, 3, 46, 46, },
+	{ 2, 1, 1, 3, 46, 40, },
+	{ 1, 1, 1, 3, 46, 40, },
+	{ 0, 1, 1, 3, 54, 46, },
+	{ 2, 1, 1, 3, 54, 40, },
+	{ 1, 1, 1, 3, 54, 40, },
+	{ 0, 1, 1, 3, 62, 46, },
+	{ 2, 1, 1, 3, 62, 40, },
+	{ 1, 1, 1, 3, 62, 40, },
+	{ 0, 1, 1, 3, 102, 46, },
+	{ 2, 1, 1, 3, 102, 40, },
+	{ 1, 1, 1, 3, 102, 40, },
+	{ 0, 1, 1, 3, 110, 46, },
+	{ 2, 1, 1, 3, 110, 40, },
+	{ 1, 1, 1, 3, 110, 40, },
+	{ 0, 1, 1, 3, 118, 46, },
+	{ 2, 1, 1, 3, 118, 40, },
+	{ 1, 1, 1, 3, 118, 40, },
+	{ 0, 1, 1, 3, 126, 46, },
+	{ 2, 1, 1, 3, 126, 40, },
+	{ 1, 1, 1, 3, 126, 40, },
+	{ 0, 1, 1, 3, 134, 46, },
+	{ 2, 1, 1, 3, 134, 40, },
+	{ 1, 1, 1, 3, 134, 40, },
+	{ 0, 1, 1, 3, 151, 46, },
+	{ 2, 1, 1, 3, 151, 40, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 46, },
+	{ 2, 1, 1, 3, 159, 40, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 46, },
+	{ 2, 1, 1, 6, 38, 40, },
+	{ 1, 1, 1, 6, 38, 40, },
+	{ 0, 1, 1, 6, 46, 46, },
+	{ 2, 1, 1, 6, 46, 40, },
+	{ 1, 1, 1, 6, 46, 40, },
+	{ 0, 1, 1, 6, 54, 46, },
+	{ 2, 1, 1, 6, 54, 40, },
+	{ 1, 1, 1, 6, 54, 40, },
+	{ 0, 1, 1, 6, 62, 46, },
+	{ 2, 1, 1, 6, 62, 40, },
+	{ 1, 1, 1, 6, 62, 40, },
+	{ 0, 1, 1, 6, 102, 46, },
+	{ 2, 1, 1, 6, 102, 40, },
+	{ 1, 1, 1, 6, 102, 40, },
+	{ 0, 1, 1, 6, 110, 46, },
+	{ 2, 1, 1, 6, 110, 40, },
+	{ 1, 1, 1, 6, 110, 40, },
+	{ 0, 1, 1, 6, 118, 46, },
+	{ 2, 1, 1, 6, 118, 40, },
+	{ 1, 1, 1, 6, 118, 40, },
+	{ 0, 1, 1, 6, 126, 46, },
+	{ 2, 1, 1, 6, 126, 40, },
+	{ 1, 1, 1, 6, 126, 40, },
+	{ 0, 1, 1, 6, 134, 46, },
+	{ 2, 1, 1, 6, 134, 40, },
+	{ 1, 1, 1, 6, 134, 40, },
+	{ 0, 1, 1, 6, 151, 46, },
+	{ 2, 1, 1, 6, 151, 40, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 46, },
+	{ 2, 1, 1, 6, 159, 40, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 46, },
+	{ 2, 1, 1, 7, 38, 40, },
+	{ 1, 1, 1, 7, 38, 40, },
+	{ 0, 1, 1, 7, 46, 46, },
+	{ 2, 1, 1, 7, 46, 40, },
+	{ 1, 1, 1, 7, 46, 40, },
+	{ 0, 1, 1, 7, 54, 46, },
+	{ 2, 1, 1, 7, 54, 40, },
+	{ 1, 1, 1, 7, 54, 40, },
+	{ 0, 1, 1, 7, 62, 46, },
+	{ 2, 1, 1, 7, 62, 40, },
+	{ 1, 1, 1, 7, 62, 40, },
+	{ 0, 1, 1, 7, 102, 46, },
+	{ 2, 1, 1, 7, 102, 40, },
+	{ 1, 1, 1, 7, 102, 40, },
+	{ 0, 1, 1, 7, 110, 46, },
+	{ 2, 1, 1, 7, 110, 40, },
+	{ 1, 1, 1, 7, 110, 40, },
+	{ 0, 1, 1, 7, 118, 46, },
+	{ 2, 1, 1, 7, 118, 40, },
+	{ 1, 1, 1, 7, 118, 40, },
+	{ 0, 1, 1, 7, 126, 46, },
+	{ 2, 1, 1, 7, 126, 40, },
+	{ 1, 1, 1, 7, 126, 40, },
+	{ 0, 1, 1, 7, 134, 46, },
+	{ 2, 1, 1, 7, 134, 40, },
+	{ 1, 1, 1, 7, 134, 40, },
+	{ 0, 1, 1, 7, 151, 46, },
+	{ 2, 1, 1, 7, 151, 40, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 46, },
+	{ 2, 1, 1, 7, 159, 40, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 46, },
+	{ 2, 1, 2, 4, 42, 40, },
+	{ 1, 1, 2, 4, 42, 40, },
+	{ 0, 1, 2, 4, 58, 46, },
+	{ 2, 1, 2, 4, 58, 40, },
+	{ 1, 1, 2, 4, 58, 40, },
+	{ 0, 1, 2, 4, 106, 46, },
+	{ 2, 1, 2, 4, 106, 40, },
+	{ 1, 1, 2, 4, 106, 40, },
+	{ 0, 1, 2, 4, 122, 46, },
+	{ 2, 1, 2, 4, 122, 40, },
+	{ 1, 1, 2, 4, 122, 40, },
+	{ 0, 1, 2, 4, 155, 46, },
+	{ 2, 1, 2, 4, 155, 40, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 46, },
+	{ 2, 1, 2, 5, 42, 40, },
+	{ 1, 1, 2, 5, 42, 40, },
+	{ 0, 1, 2, 5, 58, 46, },
+	{ 2, 1, 2, 5, 58, 40, },
+	{ 1, 1, 2, 5, 58, 40, },
+	{ 0, 1, 2, 5, 106, 46, },
+	{ 2, 1, 2, 5, 106, 40, },
+	{ 1, 1, 2, 5, 106, 40, },
+	{ 0, 1, 2, 5, 122, 46, },
+	{ 2, 1, 2, 5, 122, 40, },
+	{ 1, 1, 2, 5, 122, 40, },
+	{ 0, 1, 2, 5, 155, 46, },
+	{ 2, 1, 2, 5, 155, 40, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 46, },
+	{ 2, 1, 2, 8, 42, 40, },
+	{ 1, 1, 2, 8, 42, 40, },
+	{ 0, 1, 2, 8, 58, 46, },
+	{ 2, 1, 2, 8, 58, 40, },
+	{ 1, 1, 2, 8, 58, 40, },
+	{ 0, 1, 2, 8, 106, 46, },
+	{ 2, 1, 2, 8, 106, 40, },
+	{ 1, 1, 2, 8, 106, 40, },
+	{ 0, 1, 2, 8, 122, 46, },
+	{ 2, 1, 2, 8, 122, 40, },
+	{ 1, 1, 2, 8, 122, 40, },
+	{ 0, 1, 2, 8, 155, 46, },
+	{ 2, 1, 2, 8, 155, 40, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 46, },
+	{ 2, 1, 2, 9, 42, 40, },
+	{ 1, 1, 2, 9, 42, 40, },
+	{ 0, 1, 2, 9, 58, 46, },
+	{ 2, 1, 2, 9, 58, 40, },
+	{ 1, 1, 2, 9, 58, 40, },
+	{ 0, 1, 2, 9, 106, 46, },
+	{ 2, 1, 2, 9, 106, 40, },
+	{ 1, 1, 2, 9, 106, 40, },
+	{ 0, 1, 2, 9, 122, 46, },
+	{ 2, 1, 2, 9, 122, 40, },
+	{ 1, 1, 2, 9, 122, 40, },
+	{ 0, 1, 2, 9, 155, 46, },
+	{ 2, 1, 2, 9, 155, 40, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt_type5);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt_type7[] = {
+	{ 0, 0, 0, 0, 1, 44, },
+	{ 2, 0, 0, 0, 1, 32, },
+	{ 1, 0, 0, 0, 1, 32, },
+	{ 0, 0, 0, 0, 2, 52, },
+	{ 2, 0, 0, 0, 2, 32, },
+	{ 1, 0, 0, 0, 2, 32, },
+	{ 0, 0, 0, 0, 3, 52, },
+	{ 2, 0, 0, 0, 3, 32, },
+	{ 1, 0, 0, 0, 3, 32, },
+	{ 0, 0, 0, 0, 4, 52, },
+	{ 2, 0, 0, 0, 4, 32, },
+	{ 1, 0, 0, 0, 4, 32, },
+	{ 0, 0, 0, 0, 5, 52, },
+	{ 2, 0, 0, 0, 5, 32, },
+	{ 1, 0, 0, 0, 5, 32, },
+	{ 0, 0, 0, 0, 6, 52, },
+	{ 2, 0, 0, 0, 6, 32, },
+	{ 1, 0, 0, 0, 6, 32, },
+	{ 0, 0, 0, 0, 7, 52, },
+	{ 2, 0, 0, 0, 7, 32, },
+	{ 1, 0, 0, 0, 7, 32, },
+	{ 0, 0, 0, 0, 8, 52, },
+	{ 2, 0, 0, 0, 8, 32, },
+	{ 1, 0, 0, 0, 8, 32, },
+	{ 0, 0, 0, 0, 9, 52, },
+	{ 2, 0, 0, 0, 9, 32, },
+	{ 1, 0, 0, 0, 9, 32, },
+	{ 0, 0, 0, 0, 10, 52, },
+	{ 2, 0, 0, 0, 10, 32, },
+	{ 1, 0, 0, 0, 10, 32, },
+	{ 0, 0, 0, 0, 11, 44, },
+	{ 2, 0, 0, 0, 11, 32, },
+	{ 1, 0, 0, 0, 11, 32, },
+	{ 0, 0, 0, 0, 12, 63, },
+	{ 2, 0, 0, 0, 12, 32, },
+	{ 1, 0, 0, 0, 12, 32, },
+	{ 0, 0, 0, 0, 13, 63, },
+	{ 2, 0, 0, 0, 13, 32, },
+	{ 1, 0, 0, 0, 13, 32, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 32, },
+	{ 0, 0, 0, 1, 1, 38, },
+	{ 2, 0, 0, 1, 1, 32, },
+	{ 1, 0, 0, 1, 1, 32, },
+	{ 0, 0, 0, 1, 2, 46, },
+	{ 2, 0, 0, 1, 2, 32, },
+	{ 1, 0, 0, 1, 2, 32, },
+	{ 0, 0, 0, 1, 3, 46, },
+	{ 2, 0, 0, 1, 3, 32, },
+	{ 1, 0, 0, 1, 3, 32, },
+	{ 0, 0, 0, 1, 4, 46, },
+	{ 2, 0, 0, 1, 4, 32, },
+	{ 1, 0, 0, 1, 4, 32, },
+	{ 0, 0, 0, 1, 5, 46, },
+	{ 2, 0, 0, 1, 5, 32, },
+	{ 1, 0, 0, 1, 5, 32, },
+	{ 0, 0, 0, 1, 6, 46, },
+	{ 2, 0, 0, 1, 6, 32, },
+	{ 1, 0, 0, 1, 6, 32, },
+	{ 0, 0, 0, 1, 7, 46, },
+	{ 2, 0, 0, 1, 7, 32, },
+	{ 1, 0, 0, 1, 7, 32, },
+	{ 0, 0, 0, 1, 8, 46, },
+	{ 2, 0, 0, 1, 8, 32, },
+	{ 1, 0, 0, 1, 8, 32, },
+	{ 0, 0, 0, 1, 9, 46, },
+	{ 2, 0, 0, 1, 9, 32, },
+	{ 1, 0, 0, 1, 9, 32, },
+	{ 0, 0, 0, 1, 10, 46, },
+	{ 2, 0, 0, 1, 10, 32, },
+	{ 1, 0, 0, 1, 10, 32, },
+	{ 0, 0, 0, 1, 11, 38, },
+	{ 2, 0, 0, 1, 11, 32, },
+	{ 1, 0, 0, 1, 11, 32, },
+	{ 0, 0, 0, 1, 12, 63, },
+	{ 2, 0, 0, 1, 12, 32, },
+	{ 1, 0, 0, 1, 12, 32, },
+	{ 0, 0, 0, 1, 13, 63, },
+	{ 2, 0, 0, 1, 13, 32, },
+	{ 1, 0, 0, 1, 13, 32, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 63, },
+	{ 0, 0, 0, 2, 1, 34, },
+	{ 2, 0, 0, 2, 1, 32, },
+	{ 1, 0, 0, 2, 1, 32, },
+	{ 0, 0, 0, 2, 2, 46, },
+	{ 2, 0, 0, 2, 2, 32, },
+	{ 1, 0, 0, 2, 2, 32, },
+	{ 0, 0, 0, 2, 3, 46, },
+	{ 2, 0, 0, 2, 3, 32, },
+	{ 1, 0, 0, 2, 3, 32, },
+	{ 0, 0, 0, 2, 4, 46, },
+	{ 2, 0, 0, 2, 4, 32, },
+	{ 1, 0, 0, 2, 4, 32, },
+	{ 0, 0, 0, 2, 5, 46, },
+	{ 2, 0, 0, 2, 5, 32, },
+	{ 1, 0, 0, 2, 5, 32, },
+	{ 0, 0, 0, 2, 6, 46, },
+	{ 2, 0, 0, 2, 6, 32, },
+	{ 1, 0, 0, 2, 6, 32, },
+	{ 0, 0, 0, 2, 7, 46, },
+	{ 2, 0, 0, 2, 7, 32, },
+	{ 1, 0, 0, 2, 7, 32, },
+	{ 0, 0, 0, 2, 8, 46, },
+	{ 2, 0, 0, 2, 8, 32, },
+	{ 1, 0, 0, 2, 8, 32, },
+	{ 0, 0, 0, 2, 9, 46, },
+	{ 2, 0, 0, 2, 9, 32, },
+	{ 1, 0, 0, 2, 9, 32, },
+	{ 0, 0, 0, 2, 10, 46, },
+	{ 2, 0, 0, 2, 10, 32, },
+	{ 1, 0, 0, 2, 10, 32, },
+	{ 0, 0, 0, 2, 11, 34, },
+	{ 2, 0, 0, 2, 11, 32, },
+	{ 1, 0, 0, 2, 11, 32, },
+	{ 0, 0, 0, 2, 12, 63, },
+	{ 2, 0, 0, 2, 12, 32, },
+	{ 1, 0, 0, 2, 12, 32, },
+	{ 0, 0, 0, 2, 13, 63, },
+	{ 2, 0, 0, 2, 13, 32, },
+	{ 1, 0, 0, 2, 13, 32, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 63, },
+	{ 0, 0, 0, 3, 1, 32, },
+	{ 2, 0, 0, 3, 1, 30, },
+	{ 1, 0, 0, 3, 1, 30, },
+	{ 0, 0, 0, 3, 2, 44, },
+	{ 2, 0, 0, 3, 2, 30, },
+	{ 1, 0, 0, 3, 2, 30, },
+	{ 0, 0, 0, 3, 3, 44, },
+	{ 2, 0, 0, 3, 3, 30, },
+	{ 1, 0, 0, 3, 3, 30, },
+	{ 0, 0, 0, 3, 4, 44, },
+	{ 2, 0, 0, 3, 4, 30, },
+	{ 1, 0, 0, 3, 4, 30, },
+	{ 0, 0, 0, 3, 5, 44, },
+	{ 2, 0, 0, 3, 5, 30, },
+	{ 1, 0, 0, 3, 5, 30, },
+	{ 0, 0, 0, 3, 6, 44, },
+	{ 2, 0, 0, 3, 6, 30, },
+	{ 1, 0, 0, 3, 6, 30, },
+	{ 0, 0, 0, 3, 7, 44, },
+	{ 2, 0, 0, 3, 7, 30, },
+	{ 1, 0, 0, 3, 7, 30, },
+	{ 0, 0, 0, 3, 8, 44, },
+	{ 2, 0, 0, 3, 8, 30, },
+	{ 1, 0, 0, 3, 8, 30, },
+	{ 0, 0, 0, 3, 9, 44, },
+	{ 2, 0, 0, 3, 9, 30, },
+	{ 1, 0, 0, 3, 9, 30, },
+	{ 0, 0, 0, 3, 10, 44, },
+	{ 2, 0, 0, 3, 10, 30, },
+	{ 1, 0, 0, 3, 10, 30, },
+	{ 0, 0, 0, 3, 11, 32, },
+	{ 2, 0, 0, 3, 11, 30, },
+	{ 1, 0, 0, 3, 11, 30, },
+	{ 0, 0, 0, 3, 12, 63, },
+	{ 2, 0, 0, 3, 12, 30, },
+	{ 1, 0, 0, 3, 12, 30, },
+	{ 0, 0, 0, 3, 13, 63, },
+	{ 2, 0, 0, 3, 13, 30, },
+	{ 1, 0, 0, 3, 13, 30, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 63, },
+	{ 0, 0, 0, 6, 1, 30, },
+	{ 2, 0, 0, 6, 1, 28, },
+	{ 1, 0, 0, 6, 1, 28, },
+	{ 0, 0, 0, 6, 2, 42, },
+	{ 2, 0, 0, 6, 2, 28, },
+	{ 1, 0, 0, 6, 2, 28, },
+	{ 0, 0, 0, 6, 3, 42, },
+	{ 2, 0, 0, 6, 3, 28, },
+	{ 1, 0, 0, 6, 3, 28, },
+	{ 0, 0, 0, 6, 4, 42, },
+	{ 2, 0, 0, 6, 4, 28, },
+	{ 1, 0, 0, 6, 4, 28, },
+	{ 0, 0, 0, 6, 5, 42, },
+	{ 2, 0, 0, 6, 5, 28, },
+	{ 1, 0, 0, 6, 5, 28, },
+	{ 0, 0, 0, 6, 6, 42, },
+	{ 2, 0, 0, 6, 6, 28, },
+	{ 1, 0, 0, 6, 6, 28, },
+	{ 0, 0, 0, 6, 7, 42, },
+	{ 2, 0, 0, 6, 7, 28, },
+	{ 1, 0, 0, 6, 7, 28, },
+	{ 0, 0, 0, 6, 8, 42, },
+	{ 2, 0, 0, 6, 8, 28, },
+	{ 1, 0, 0, 6, 8, 28, },
+	{ 0, 0, 0, 6, 9, 42, },
+	{ 2, 0, 0, 6, 9, 28, },
+	{ 1, 0, 0, 6, 9, 28, },
+	{ 0, 0, 0, 6, 10, 42, },
+	{ 2, 0, 0, 6, 10, 28, },
+	{ 1, 0, 0, 6, 10, 28, },
+	{ 0, 0, 0, 6, 11, 30, },
+	{ 2, 0, 0, 6, 11, 28, },
+	{ 1, 0, 0, 6, 11, 28, },
+	{ 0, 0, 0, 6, 12, 63, },
+	{ 2, 0, 0, 6, 12, 28, },
+	{ 1, 0, 0, 6, 12, 28, },
+	{ 0, 0, 0, 6, 13, 63, },
+	{ 2, 0, 0, 6, 13, 28, },
+	{ 1, 0, 0, 6, 13, 28, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 63, },
+	{ 0, 0, 0, 7, 1, 28, },
+	{ 2, 0, 0, 7, 1, 26, },
+	{ 1, 0, 0, 7, 1, 26, },
+	{ 0, 0, 0, 7, 2, 40, },
+	{ 2, 0, 0, 7, 2, 26, },
+	{ 1, 0, 0, 7, 2, 26, },
+	{ 0, 0, 0, 7, 3, 40, },
+	{ 2, 0, 0, 7, 3, 26, },
+	{ 1, 0, 0, 7, 3, 26, },
+	{ 0, 0, 0, 7, 4, 40, },
+	{ 2, 0, 0, 7, 4, 26, },
+	{ 1, 0, 0, 7, 4, 26, },
+	{ 0, 0, 0, 7, 5, 40, },
+	{ 2, 0, 0, 7, 5, 26, },
+	{ 1, 0, 0, 7, 5, 26, },
+	{ 0, 0, 0, 7, 6, 40, },
+	{ 2, 0, 0, 7, 6, 26, },
+	{ 1, 0, 0, 7, 6, 26, },
+	{ 0, 0, 0, 7, 7, 40, },
+	{ 2, 0, 0, 7, 7, 26, },
+	{ 1, 0, 0, 7, 7, 26, },
+	{ 0, 0, 0, 7, 8, 40, },
+	{ 2, 0, 0, 7, 8, 26, },
+	{ 1, 0, 0, 7, 8, 26, },
+	{ 0, 0, 0, 7, 9, 40, },
+	{ 2, 0, 0, 7, 9, 26, },
+	{ 1, 0, 0, 7, 9, 26, },
+	{ 0, 0, 0, 7, 10, 40, },
+	{ 2, 0, 0, 7, 10, 26, },
+	{ 1, 0, 0, 7, 10, 26, },
+	{ 0, 0, 0, 7, 11, 28, },
+	{ 2, 0, 0, 7, 11, 26, },
+	{ 1, 0, 0, 7, 11, 26, },
+	{ 0, 0, 0, 7, 12, 63, },
+	{ 2, 0, 0, 7, 12, 26, },
+	{ 1, 0, 0, 7, 12, 26, },
+	{ 0, 0, 0, 7, 13, 63, },
+	{ 2, 0, 0, 7, 13, 26, },
+	{ 1, 0, 0, 7, 13, 26, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 63, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 36, },
+	{ 2, 0, 1, 2, 3, 32, },
+	{ 1, 0, 1, 2, 3, 32, },
+	{ 0, 0, 1, 2, 4, 40, },
+	{ 2, 0, 1, 2, 4, 32, },
+	{ 1, 0, 1, 2, 4, 32, },
+	{ 0, 0, 1, 2, 5, 40, },
+	{ 2, 0, 1, 2, 5, 32, },
+	{ 1, 0, 1, 2, 5, 32, },
+	{ 0, 0, 1, 2, 6, 40, },
+	{ 2, 0, 1, 2, 6, 32, },
+	{ 1, 0, 1, 2, 6, 32, },
+	{ 0, 0, 1, 2, 7, 40, },
+	{ 2, 0, 1, 2, 7, 32, },
+	{ 1, 0, 1, 2, 7, 32, },
+	{ 0, 0, 1, 2, 8, 40, },
+	{ 2, 0, 1, 2, 8, 32, },
+	{ 1, 0, 1, 2, 8, 32, },
+	{ 0, 0, 1, 2, 9, 40, },
+	{ 2, 0, 1, 2, 9, 32, },
+	{ 1, 0, 1, 2, 9, 32, },
+	{ 0, 0, 1, 2, 10, 40, },
+	{ 2, 0, 1, 2, 10, 32, },
+	{ 1, 0, 1, 2, 10, 32, },
+	{ 0, 0, 1, 2, 11, 34, },
+	{ 2, 0, 1, 2, 11, 32, },
+	{ 1, 0, 1, 2, 11, 32, },
+	{ 0, 0, 1, 2, 12, 63, },
+	{ 2, 0, 1, 2, 12, 32, },
+	{ 1, 0, 1, 2, 12, 32, },
+	{ 0, 0, 1, 2, 13, 63, },
+	{ 2, 0, 1, 2, 13, 32, },
+	{ 1, 0, 1, 2, 13, 32, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 34, },
+	{ 2, 0, 1, 3, 3, 30, },
+	{ 1, 0, 1, 3, 3, 30, },
+	{ 0, 0, 1, 3, 4, 38, },
+	{ 2, 0, 1, 3, 4, 30, },
+	{ 1, 0, 1, 3, 4, 30, },
+	{ 0, 0, 1, 3, 5, 38, },
+	{ 2, 0, 1, 3, 5, 30, },
+	{ 1, 0, 1, 3, 5, 30, },
+	{ 0, 0, 1, 3, 6, 38, },
+	{ 2, 0, 1, 3, 6, 30, },
+	{ 1, 0, 1, 3, 6, 30, },
+	{ 0, 0, 1, 3, 7, 38, },
+	{ 2, 0, 1, 3, 7, 30, },
+	{ 1, 0, 1, 3, 7, 30, },
+	{ 0, 0, 1, 3, 8, 38, },
+	{ 2, 0, 1, 3, 8, 30, },
+	{ 1, 0, 1, 3, 8, 30, },
+	{ 0, 0, 1, 3, 9, 38, },
+	{ 2, 0, 1, 3, 9, 30, },
+	{ 1, 0, 1, 3, 9, 30, },
+	{ 0, 0, 1, 3, 10, 38, },
+	{ 2, 0, 1, 3, 10, 30, },
+	{ 1, 0, 1, 3, 10, 30, },
+	{ 0, 0, 1, 3, 11, 32, },
+	{ 2, 0, 1, 3, 11, 30, },
+	{ 1, 0, 1, 3, 11, 30, },
+	{ 0, 0, 1, 3, 12, 63, },
+	{ 2, 0, 1, 3, 12, 30, },
+	{ 1, 0, 1, 3, 12, 30, },
+	{ 0, 0, 1, 3, 13, 63, },
+	{ 2, 0, 1, 3, 13, 30, },
+	{ 1, 0, 1, 3, 13, 30, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 32, },
+	{ 2, 0, 1, 6, 3, 28, },
+	{ 1, 0, 1, 6, 3, 28, },
+	{ 0, 0, 1, 6, 4, 36, },
+	{ 2, 0, 1, 6, 4, 28, },
+	{ 1, 0, 1, 6, 4, 28, },
+	{ 0, 0, 1, 6, 5, 36, },
+	{ 2, 0, 1, 6, 5, 28, },
+	{ 1, 0, 1, 6, 5, 28, },
+	{ 0, 0, 1, 6, 6, 36, },
+	{ 2, 0, 1, 6, 6, 28, },
+	{ 1, 0, 1, 6, 6, 28, },
+	{ 0, 0, 1, 6, 7, 36, },
+	{ 2, 0, 1, 6, 7, 28, },
+	{ 1, 0, 1, 6, 7, 28, },
+	{ 0, 0, 1, 6, 8, 36, },
+	{ 2, 0, 1, 6, 8, 28, },
+	{ 1, 0, 1, 6, 8, 28, },
+	{ 0, 0, 1, 6, 9, 36, },
+	{ 2, 0, 1, 6, 9, 28, },
+	{ 1, 0, 1, 6, 9, 28, },
+	{ 0, 0, 1, 6, 10, 36, },
+	{ 2, 0, 1, 6, 10, 28, },
+	{ 1, 0, 1, 6, 10, 28, },
+	{ 0, 0, 1, 6, 11, 30, },
+	{ 2, 0, 1, 6, 11, 28, },
+	{ 1, 0, 1, 6, 11, 28, },
+	{ 0, 0, 1, 6, 12, 63, },
+	{ 2, 0, 1, 6, 12, 28, },
+	{ 1, 0, 1, 6, 12, 28, },
+	{ 0, 0, 1, 6, 13, 63, },
+	{ 2, 0, 1, 6, 13, 28, },
+	{ 1, 0, 1, 6, 13, 28, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 32, },
+	{ 2, 0, 1, 7, 3, 26, },
+	{ 1, 0, 1, 7, 3, 26, },
+	{ 0, 0, 1, 7, 4, 36, },
+	{ 2, 0, 1, 7, 4, 26, },
+	{ 1, 0, 1, 7, 4, 26, },
+	{ 0, 0, 1, 7, 5, 36, },
+	{ 2, 0, 1, 7, 5, 26, },
+	{ 1, 0, 1, 7, 5, 26, },
+	{ 0, 0, 1, 7, 6, 36, },
+	{ 2, 0, 1, 7, 6, 26, },
+	{ 1, 0, 1, 7, 6, 26, },
+	{ 0, 0, 1, 7, 7, 36, },
+	{ 2, 0, 1, 7, 7, 26, },
+	{ 1, 0, 1, 7, 7, 26, },
+	{ 0, 0, 1, 7, 8, 36, },
+	{ 2, 0, 1, 7, 8, 26, },
+	{ 1, 0, 1, 7, 8, 26, },
+	{ 0, 0, 1, 7, 9, 36, },
+	{ 2, 0, 1, 7, 9, 26, },
+	{ 1, 0, 1, 7, 9, 26, },
+	{ 0, 0, 1, 7, 10, 36, },
+	{ 2, 0, 1, 7, 10, 26, },
+	{ 1, 0, 1, 7, 10, 26, },
+	{ 0, 0, 1, 7, 11, 30, },
+	{ 2, 0, 1, 7, 11, 26, },
+	{ 1, 0, 1, 7, 11, 26, },
+	{ 0, 0, 1, 7, 12, 63, },
+	{ 2, 0, 1, 7, 12, 26, },
+	{ 1, 0, 1, 7, 12, 26, },
+	{ 0, 0, 1, 7, 13, 63, },
+	{ 2, 0, 1, 7, 13, 26, },
+	{ 1, 0, 1, 7, 13, 26, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 38, },
+	{ 2, 1, 0, 1, 36, 32, },
+	{ 1, 1, 0, 1, 36, 32, },
+	{ 0, 1, 0, 1, 40, 38, },
+	{ 2, 1, 0, 1, 40, 32, },
+	{ 1, 1, 0, 1, 40, 32, },
+	{ 0, 1, 0, 1, 44, 38, },
+	{ 2, 1, 0, 1, 44, 32, },
+	{ 1, 1, 0, 1, 44, 32, },
+	{ 0, 1, 0, 1, 48, 38, },
+	{ 2, 1, 0, 1, 48, 32, },
+	{ 1, 1, 0, 1, 48, 32, },
+	{ 0, 1, 0, 1, 52, 38, },
+	{ 2, 1, 0, 1, 52, 32, },
+	{ 1, 1, 0, 1, 52, 32, },
+	{ 0, 1, 0, 1, 56, 38, },
+	{ 2, 1, 0, 1, 56, 32, },
+	{ 1, 1, 0, 1, 56, 32, },
+	{ 0, 1, 0, 1, 60, 38, },
+	{ 2, 1, 0, 1, 60, 32, },
+	{ 1, 1, 0, 1, 60, 32, },
+	{ 0, 1, 0, 1, 64, 38, },
+	{ 2, 1, 0, 1, 64, 32, },
+	{ 1, 1, 0, 1, 64, 32, },
+	{ 0, 1, 0, 1, 100, 36, },
+	{ 2, 1, 0, 1, 100, 32, },
+	{ 1, 1, 0, 1, 100, 32, },
+	{ 0, 1, 0, 1, 104, 36, },
+	{ 2, 1, 0, 1, 104, 32, },
+	{ 1, 1, 0, 1, 104, 32, },
+	{ 0, 1, 0, 1, 108, 36, },
+	{ 2, 1, 0, 1, 108, 32, },
+	{ 1, 1, 0, 1, 108, 32, },
+	{ 0, 1, 0, 1, 112, 36, },
+	{ 2, 1, 0, 1, 112, 32, },
+	{ 1, 1, 0, 1, 112, 32, },
+	{ 0, 1, 0, 1, 116, 36, },
+	{ 2, 1, 0, 1, 116, 32, },
+	{ 1, 1, 0, 1, 116, 32, },
+	{ 0, 1, 0, 1, 120, 36, },
+	{ 2, 1, 0, 1, 120, 32, },
+	{ 1, 1, 0, 1, 120, 32, },
+	{ 0, 1, 0, 1, 124, 36, },
+	{ 2, 1, 0, 1, 124, 32, },
+	{ 1, 1, 0, 1, 124, 32, },
+	{ 0, 1, 0, 1, 128, 36, },
+	{ 2, 1, 0, 1, 128, 32, },
+	{ 1, 1, 0, 1, 128, 32, },
+	{ 0, 1, 0, 1, 132, 36, },
+	{ 2, 1, 0, 1, 132, 32, },
+	{ 1, 1, 0, 1, 132, 32, },
+	{ 0, 1, 0, 1, 136, 36, },
+	{ 2, 1, 0, 1, 136, 32, },
+	{ 1, 1, 0, 1, 136, 32, },
+	{ 0, 1, 0, 1, 140, 36, },
+	{ 2, 1, 0, 1, 140, 32, },
+	{ 1, 1, 0, 1, 140, 32, },
+	{ 0, 1, 0, 1, 149, 36, },
+	{ 2, 1, 0, 1, 149, 32, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 36, },
+	{ 2, 1, 0, 1, 153, 32, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 36, },
+	{ 2, 1, 0, 1, 157, 32, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 36, },
+	{ 2, 1, 0, 1, 161, 32, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 36, },
+	{ 2, 1, 0, 1, 165, 32, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 36, },
+	{ 2, 1, 0, 2, 36, 32, },
+	{ 1, 1, 0, 2, 36, 32, },
+	{ 0, 1, 0, 2, 40, 36, },
+	{ 2, 1, 0, 2, 40, 32, },
+	{ 1, 1, 0, 2, 40, 32, },
+	{ 0, 1, 0, 2, 44, 36, },
+	{ 2, 1, 0, 2, 44, 32, },
+	{ 1, 1, 0, 2, 44, 32, },
+	{ 0, 1, 0, 2, 48, 36, },
+	{ 2, 1, 0, 2, 48, 32, },
+	{ 1, 1, 0, 2, 48, 32, },
+	{ 0, 1, 0, 2, 52, 36, },
+	{ 2, 1, 0, 2, 52, 32, },
+	{ 1, 1, 0, 2, 52, 32, },
+	{ 0, 1, 0, 2, 56, 36, },
+	{ 2, 1, 0, 2, 56, 32, },
+	{ 1, 1, 0, 2, 56, 32, },
+	{ 0, 1, 0, 2, 60, 36, },
+	{ 2, 1, 0, 2, 60, 32, },
+	{ 1, 1, 0, 2, 60, 32, },
+	{ 0, 1, 0, 2, 64, 36, },
+	{ 2, 1, 0, 2, 64, 32, },
+	{ 1, 1, 0, 2, 64, 32, },
+	{ 0, 1, 0, 2, 100, 36, },
+	{ 2, 1, 0, 2, 100, 32, },
+	{ 1, 1, 0, 2, 100, 32, },
+	{ 0, 1, 0, 2, 104, 36, },
+	{ 2, 1, 0, 2, 104, 32, },
+	{ 1, 1, 0, 2, 104, 32, },
+	{ 0, 1, 0, 2, 108, 36, },
+	{ 2, 1, 0, 2, 108, 32, },
+	{ 1, 1, 0, 2, 108, 32, },
+	{ 0, 1, 0, 2, 112, 36, },
+	{ 2, 1, 0, 2, 112, 32, },
+	{ 1, 1, 0, 2, 112, 32, },
+	{ 0, 1, 0, 2, 116, 36, },
+	{ 2, 1, 0, 2, 116, 32, },
+	{ 1, 1, 0, 2, 116, 32, },
+	{ 0, 1, 0, 2, 120, 36, },
+	{ 2, 1, 0, 2, 120, 32, },
+	{ 1, 1, 0, 2, 120, 32, },
+	{ 0, 1, 0, 2, 124, 36, },
+	{ 2, 1, 0, 2, 124, 32, },
+	{ 1, 1, 0, 2, 124, 32, },
+	{ 0, 1, 0, 2, 128, 36, },
+	{ 2, 1, 0, 2, 128, 32, },
+	{ 1, 1, 0, 2, 128, 32, },
+	{ 0, 1, 0, 2, 132, 36, },
+	{ 2, 1, 0, 2, 132, 32, },
+	{ 1, 1, 0, 2, 132, 32, },
+	{ 0, 1, 0, 2, 136, 36, },
+	{ 2, 1, 0, 2, 136, 32, },
+	{ 1, 1, 0, 2, 136, 32, },
+	{ 0, 1, 0, 2, 140, 34, },
+	{ 2, 1, 0, 2, 140, 32, },
+	{ 1, 1, 0, 2, 140, 32, },
+	{ 0, 1, 0, 2, 149, 32, },
+	{ 2, 1, 0, 2, 149, 32, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 38, },
+	{ 2, 1, 0, 2, 153, 32, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 38, },
+	{ 2, 1, 0, 2, 157, 32, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 38, },
+	{ 2, 1, 0, 2, 161, 32, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 38, },
+	{ 2, 1, 0, 2, 165, 32, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 34, },
+	{ 2, 1, 0, 3, 36, 30, },
+	{ 1, 1, 0, 3, 36, 30, },
+	{ 0, 1, 0, 3, 40, 34, },
+	{ 2, 1, 0, 3, 40, 30, },
+	{ 1, 1, 0, 3, 40, 30, },
+	{ 0, 1, 0, 3, 44, 34, },
+	{ 2, 1, 0, 3, 44, 30, },
+	{ 1, 1, 0, 3, 44, 30, },
+	{ 0, 1, 0, 3, 48, 34, },
+	{ 2, 1, 0, 3, 48, 30, },
+	{ 1, 1, 0, 3, 48, 30, },
+	{ 0, 1, 0, 3, 52, 34, },
+	{ 2, 1, 0, 3, 52, 30, },
+	{ 1, 1, 0, 3, 52, 30, },
+	{ 0, 1, 0, 3, 56, 34, },
+	{ 2, 1, 0, 3, 56, 30, },
+	{ 1, 1, 0, 3, 56, 30, },
+	{ 0, 1, 0, 3, 60, 34, },
+	{ 2, 1, 0, 3, 60, 30, },
+	{ 1, 1, 0, 3, 60, 30, },
+	{ 0, 1, 0, 3, 64, 34, },
+	{ 2, 1, 0, 3, 64, 30, },
+	{ 1, 1, 0, 3, 64, 30, },
+	{ 0, 1, 0, 3, 100, 34, },
+	{ 2, 1, 0, 3, 100, 30, },
+	{ 1, 1, 0, 3, 100, 30, },
+	{ 0, 1, 0, 3, 104, 34, },
+	{ 2, 1, 0, 3, 104, 30, },
+	{ 1, 1, 0, 3, 104, 30, },
+	{ 0, 1, 0, 3, 108, 34, },
+	{ 2, 1, 0, 3, 108, 30, },
+	{ 1, 1, 0, 3, 108, 30, },
+	{ 0, 1, 0, 3, 112, 34, },
+	{ 2, 1, 0, 3, 112, 30, },
+	{ 1, 1, 0, 3, 112, 30, },
+	{ 0, 1, 0, 3, 116, 34, },
+	{ 2, 1, 0, 3, 116, 30, },
+	{ 1, 1, 0, 3, 116, 30, },
+	{ 0, 1, 0, 3, 120, 34, },
+	{ 2, 1, 0, 3, 120, 30, },
+	{ 1, 1, 0, 3, 120, 30, },
+	{ 0, 1, 0, 3, 124, 34, },
+	{ 2, 1, 0, 3, 124, 30, },
+	{ 1, 1, 0, 3, 124, 30, },
+	{ 0, 1, 0, 3, 128, 34, },
+	{ 2, 1, 0, 3, 128, 30, },
+	{ 1, 1, 0, 3, 128, 30, },
+	{ 0, 1, 0, 3, 132, 34, },
+	{ 2, 1, 0, 3, 132, 30, },
+	{ 1, 1, 0, 3, 132, 30, },
+	{ 0, 1, 0, 3, 136, 34, },
+	{ 2, 1, 0, 3, 136, 30, },
+	{ 1, 1, 0, 3, 136, 30, },
+	{ 0, 1, 0, 3, 140, 32, },
+	{ 2, 1, 0, 3, 140, 30, },
+	{ 1, 1, 0, 3, 140, 30, },
+	{ 0, 1, 0, 3, 149, 30, },
+	{ 2, 1, 0, 3, 149, 30, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 36, },
+	{ 2, 1, 0, 3, 153, 30, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 36, },
+	{ 2, 1, 0, 3, 157, 30, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 36, },
+	{ 2, 1, 0, 3, 161, 30, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 36, },
+	{ 2, 1, 0, 3, 165, 30, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 32, },
+	{ 2, 1, 0, 6, 36, 28, },
+	{ 1, 1, 0, 6, 36, 28, },
+	{ 0, 1, 0, 6, 40, 32, },
+	{ 2, 1, 0, 6, 40, 28, },
+	{ 1, 1, 0, 6, 40, 28, },
+	{ 0, 1, 0, 6, 44, 32, },
+	{ 2, 1, 0, 6, 44, 28, },
+	{ 1, 1, 0, 6, 44, 28, },
+	{ 0, 1, 0, 6, 48, 32, },
+	{ 2, 1, 0, 6, 48, 28, },
+	{ 1, 1, 0, 6, 48, 28, },
+	{ 0, 1, 0, 6, 52, 32, },
+	{ 2, 1, 0, 6, 52, 28, },
+	{ 1, 1, 0, 6, 52, 28, },
+	{ 0, 1, 0, 6, 56, 32, },
+	{ 2, 1, 0, 6, 56, 28, },
+	{ 1, 1, 0, 6, 56, 28, },
+	{ 0, 1, 0, 6, 60, 32, },
+	{ 2, 1, 0, 6, 60, 28, },
+	{ 1, 1, 0, 6, 60, 28, },
+	{ 0, 1, 0, 6, 64, 32, },
+	{ 2, 1, 0, 6, 64, 28, },
+	{ 1, 1, 0, 6, 64, 28, },
+	{ 0, 1, 0, 6, 100, 32, },
+	{ 2, 1, 0, 6, 100, 28, },
+	{ 1, 1, 0, 6, 100, 28, },
+	{ 0, 1, 0, 6, 104, 32, },
+	{ 2, 1, 0, 6, 104, 28, },
+	{ 1, 1, 0, 6, 104, 28, },
+	{ 0, 1, 0, 6, 108, 32, },
+	{ 2, 1, 0, 6, 108, 28, },
+	{ 1, 1, 0, 6, 108, 28, },
+	{ 0, 1, 0, 6, 112, 32, },
+	{ 2, 1, 0, 6, 112, 28, },
+	{ 1, 1, 0, 6, 112, 28, },
+	{ 0, 1, 0, 6, 116, 32, },
+	{ 2, 1, 0, 6, 116, 28, },
+	{ 1, 1, 0, 6, 116, 28, },
+	{ 0, 1, 0, 6, 120, 32, },
+	{ 2, 1, 0, 6, 120, 28, },
+	{ 1, 1, 0, 6, 120, 28, },
+	{ 0, 1, 0, 6, 124, 32, },
+	{ 2, 1, 0, 6, 124, 28, },
+	{ 1, 1, 0, 6, 124, 28, },
+	{ 0, 1, 0, 6, 128, 32, },
+	{ 2, 1, 0, 6, 128, 28, },
+	{ 1, 1, 0, 6, 128, 28, },
+	{ 0, 1, 0, 6, 132, 32, },
+	{ 2, 1, 0, 6, 132, 28, },
+	{ 1, 1, 0, 6, 132, 28, },
+	{ 0, 1, 0, 6, 136, 32, },
+	{ 2, 1, 0, 6, 136, 28, },
+	{ 1, 1, 0, 6, 136, 28, },
+	{ 0, 1, 0, 6, 140, 30, },
+	{ 2, 1, 0, 6, 140, 28, },
+	{ 1, 1, 0, 6, 140, 28, },
+	{ 0, 1, 0, 6, 149, 28, },
+	{ 2, 1, 0, 6, 149, 28, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 34, },
+	{ 2, 1, 0, 6, 153, 28, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 34, },
+	{ 2, 1, 0, 6, 157, 28, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 34, },
+	{ 2, 1, 0, 6, 161, 28, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 34, },
+	{ 2, 1, 0, 6, 165, 28, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 30, },
+	{ 2, 1, 0, 7, 36, 26, },
+	{ 1, 1, 0, 7, 36, 26, },
+	{ 0, 1, 0, 7, 40, 30, },
+	{ 2, 1, 0, 7, 40, 26, },
+	{ 1, 1, 0, 7, 40, 26, },
+	{ 0, 1, 0, 7, 44, 30, },
+	{ 2, 1, 0, 7, 44, 26, },
+	{ 1, 1, 0, 7, 44, 26, },
+	{ 0, 1, 0, 7, 48, 30, },
+	{ 2, 1, 0, 7, 48, 26, },
+	{ 1, 1, 0, 7, 48, 26, },
+	{ 0, 1, 0, 7, 52, 30, },
+	{ 2, 1, 0, 7, 52, 26, },
+	{ 1, 1, 0, 7, 52, 26, },
+	{ 0, 1, 0, 7, 56, 30, },
+	{ 2, 1, 0, 7, 56, 26, },
+	{ 1, 1, 0, 7, 56, 26, },
+	{ 0, 1, 0, 7, 60, 30, },
+	{ 2, 1, 0, 7, 60, 26, },
+	{ 1, 1, 0, 7, 60, 26, },
+	{ 0, 1, 0, 7, 64, 30, },
+	{ 2, 1, 0, 7, 64, 26, },
+	{ 1, 1, 0, 7, 64, 26, },
+	{ 0, 1, 0, 7, 100, 30, },
+	{ 2, 1, 0, 7, 100, 26, },
+	{ 1, 1, 0, 7, 100, 26, },
+	{ 0, 1, 0, 7, 104, 30, },
+	{ 2, 1, 0, 7, 104, 26, },
+	{ 1, 1, 0, 7, 104, 26, },
+	{ 0, 1, 0, 7, 108, 30, },
+	{ 2, 1, 0, 7, 108, 26, },
+	{ 1, 1, 0, 7, 108, 26, },
+	{ 0, 1, 0, 7, 112, 30, },
+	{ 2, 1, 0, 7, 112, 26, },
+	{ 1, 1, 0, 7, 112, 26, },
+	{ 0, 1, 0, 7, 116, 30, },
+	{ 2, 1, 0, 7, 116, 26, },
+	{ 1, 1, 0, 7, 116, 26, },
+	{ 0, 1, 0, 7, 120, 30, },
+	{ 2, 1, 0, 7, 120, 26, },
+	{ 1, 1, 0, 7, 120, 26, },
+	{ 0, 1, 0, 7, 124, 30, },
+	{ 2, 1, 0, 7, 124, 26, },
+	{ 1, 1, 0, 7, 124, 26, },
+	{ 0, 1, 0, 7, 128, 30, },
+	{ 2, 1, 0, 7, 128, 26, },
+	{ 1, 1, 0, 7, 128, 26, },
+	{ 0, 1, 0, 7, 132, 30, },
+	{ 2, 1, 0, 7, 132, 26, },
+	{ 1, 1, 0, 7, 132, 26, },
+	{ 0, 1, 0, 7, 136, 30, },
+	{ 2, 1, 0, 7, 136, 26, },
+	{ 1, 1, 0, 7, 136, 26, },
+	{ 0, 1, 0, 7, 140, 28, },
+	{ 2, 1, 0, 7, 140, 26, },
+	{ 1, 1, 0, 7, 140, 26, },
+	{ 0, 1, 0, 7, 149, 26, },
+	{ 2, 1, 0, 7, 149, 26, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 32, },
+	{ 2, 1, 0, 7, 153, 26, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 32, },
+	{ 2, 1, 0, 7, 157, 26, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 32, },
+	{ 2, 1, 0, 7, 161, 26, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 32, },
+	{ 2, 1, 0, 7, 165, 26, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 32, },
+	{ 2, 1, 1, 2, 38, 32, },
+	{ 1, 1, 1, 2, 38, 32, },
+	{ 0, 1, 1, 2, 46, 32, },
+	{ 2, 1, 1, 2, 46, 32, },
+	{ 1, 1, 1, 2, 46, 32, },
+	{ 0, 1, 1, 2, 54, 32, },
+	{ 2, 1, 1, 2, 54, 32, },
+	{ 1, 1, 1, 2, 54, 32, },
+	{ 0, 1, 1, 2, 62, 30, },
+	{ 2, 1, 1, 2, 62, 32, },
+	{ 1, 1, 1, 2, 62, 32, },
+	{ 0, 1, 1, 2, 102, 30, },
+	{ 2, 1, 1, 2, 102, 32, },
+	{ 1, 1, 1, 2, 102, 32, },
+	{ 0, 1, 1, 2, 110, 38, },
+	{ 2, 1, 1, 2, 110, 32, },
+	{ 1, 1, 1, 2, 110, 32, },
+	{ 0, 1, 1, 2, 118, 38, },
+	{ 2, 1, 1, 2, 118, 32, },
+	{ 1, 1, 1, 2, 118, 32, },
+	{ 0, 1, 1, 2, 126, 38, },
+	{ 2, 1, 1, 2, 126, 32, },
+	{ 1, 1, 1, 2, 126, 32, },
+	{ 0, 1, 1, 2, 134, 38, },
+	{ 2, 1, 1, 2, 134, 32, },
+	{ 1, 1, 1, 2, 134, 32, },
+	{ 0, 1, 1, 2, 151, 32, },
+	{ 2, 1, 1, 2, 151, 32, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 32, },
+	{ 2, 1, 1, 2, 159, 32, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 30, },
+	{ 2, 1, 1, 3, 38, 30, },
+	{ 1, 1, 1, 3, 38, 30, },
+	{ 0, 1, 1, 3, 46, 30, },
+	{ 2, 1, 1, 3, 46, 30, },
+	{ 1, 1, 1, 3, 46, 30, },
+	{ 0, 1, 1, 3, 54, 30, },
+	{ 2, 1, 1, 3, 54, 30, },
+	{ 1, 1, 1, 3, 54, 30, },
+	{ 0, 1, 1, 3, 62, 28, },
+	{ 2, 1, 1, 3, 62, 30, },
+	{ 1, 1, 1, 3, 62, 30, },
+	{ 0, 1, 1, 3, 102, 28, },
+	{ 2, 1, 1, 3, 102, 30, },
+	{ 1, 1, 1, 3, 102, 30, },
+	{ 0, 1, 1, 3, 110, 36, },
+	{ 2, 1, 1, 3, 110, 30, },
+	{ 1, 1, 1, 3, 110, 30, },
+	{ 0, 1, 1, 3, 118, 36, },
+	{ 2, 1, 1, 3, 118, 30, },
+	{ 1, 1, 1, 3, 118, 30, },
+	{ 0, 1, 1, 3, 126, 36, },
+	{ 2, 1, 1, 3, 126, 30, },
+	{ 1, 1, 1, 3, 126, 30, },
+	{ 0, 1, 1, 3, 134, 36, },
+	{ 2, 1, 1, 3, 134, 30, },
+	{ 1, 1, 1, 3, 134, 30, },
+	{ 0, 1, 1, 3, 151, 30, },
+	{ 2, 1, 1, 3, 151, 30, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 30, },
+	{ 2, 1, 1, 3, 159, 30, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 28, },
+	{ 2, 1, 1, 6, 38, 28, },
+	{ 1, 1, 1, 6, 38, 28, },
+	{ 0, 1, 1, 6, 46, 28, },
+	{ 2, 1, 1, 6, 46, 28, },
+	{ 1, 1, 1, 6, 46, 28, },
+	{ 0, 1, 1, 6, 54, 28, },
+	{ 2, 1, 1, 6, 54, 28, },
+	{ 1, 1, 1, 6, 54, 28, },
+	{ 0, 1, 1, 6, 62, 26, },
+	{ 2, 1, 1, 6, 62, 28, },
+	{ 1, 1, 1, 6, 62, 28, },
+	{ 0, 1, 1, 6, 102, 26, },
+	{ 2, 1, 1, 6, 102, 28, },
+	{ 1, 1, 1, 6, 102, 28, },
+	{ 0, 1, 1, 6, 110, 34, },
+	{ 2, 1, 1, 6, 110, 28, },
+	{ 1, 1, 1, 6, 110, 28, },
+	{ 0, 1, 1, 6, 118, 34, },
+	{ 2, 1, 1, 6, 118, 28, },
+	{ 1, 1, 1, 6, 118, 28, },
+	{ 0, 1, 1, 6, 126, 34, },
+	{ 2, 1, 1, 6, 126, 28, },
+	{ 1, 1, 1, 6, 126, 28, },
+	{ 0, 1, 1, 6, 134, 34, },
+	{ 2, 1, 1, 6, 134, 28, },
+	{ 1, 1, 1, 6, 134, 28, },
+	{ 0, 1, 1, 6, 151, 28, },
+	{ 2, 1, 1, 6, 151, 28, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 28, },
+	{ 2, 1, 1, 6, 159, 28, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 26, },
+	{ 2, 1, 1, 7, 38, 26, },
+	{ 1, 1, 1, 7, 38, 26, },
+	{ 0, 1, 1, 7, 46, 26, },
+	{ 2, 1, 1, 7, 46, 26, },
+	{ 1, 1, 1, 7, 46, 26, },
+	{ 0, 1, 1, 7, 54, 26, },
+	{ 2, 1, 1, 7, 54, 26, },
+	{ 1, 1, 1, 7, 54, 26, },
+	{ 0, 1, 1, 7, 62, 24, },
+	{ 2, 1, 1, 7, 62, 26, },
+	{ 1, 1, 1, 7, 62, 26, },
+	{ 0, 1, 1, 7, 102, 24, },
+	{ 2, 1, 1, 7, 102, 26, },
+	{ 1, 1, 1, 7, 102, 26, },
+	{ 0, 1, 1, 7, 110, 32, },
+	{ 2, 1, 1, 7, 110, 26, },
+	{ 1, 1, 1, 7, 110, 26, },
+	{ 0, 1, 1, 7, 118, 32, },
+	{ 2, 1, 1, 7, 118, 26, },
+	{ 1, 1, 1, 7, 118, 26, },
+	{ 0, 1, 1, 7, 126, 32, },
+	{ 2, 1, 1, 7, 126, 26, },
+	{ 1, 1, 1, 7, 126, 26, },
+	{ 0, 1, 1, 7, 134, 32, },
+	{ 2, 1, 1, 7, 134, 26, },
+	{ 1, 1, 1, 7, 134, 26, },
+	{ 0, 1, 1, 7, 151, 26, },
+	{ 2, 1, 1, 7, 151, 26, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 26, },
+	{ 2, 1, 1, 7, 159, 26, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 26, },
+	{ 2, 1, 2, 4, 42, 32, },
+	{ 1, 1, 2, 4, 42, 32, },
+	{ 0, 1, 2, 4, 58, 26, },
+	{ 2, 1, 2, 4, 58, 32, },
+	{ 1, 1, 2, 4, 58, 32, },
+	{ 0, 1, 2, 4, 106, 28, },
+	{ 2, 1, 2, 4, 106, 32, },
+	{ 1, 1, 2, 4, 106, 32, },
+	{ 0, 1, 2, 4, 122, 28, },
+	{ 2, 1, 2, 4, 122, 32, },
+	{ 1, 1, 2, 4, 122, 32, },
+	{ 0, 1, 2, 4, 155, 28, },
+	{ 2, 1, 2, 4, 155, 32, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 24, },
+	{ 2, 1, 2, 5, 42, 30, },
+	{ 1, 1, 2, 5, 42, 30, },
+	{ 0, 1, 2, 5, 58, 24, },
+	{ 2, 1, 2, 5, 58, 30, },
+	{ 1, 1, 2, 5, 58, 30, },
+	{ 0, 1, 2, 5, 106, 26, },
+	{ 2, 1, 2, 5, 106, 30, },
+	{ 1, 1, 2, 5, 106, 30, },
+	{ 0, 1, 2, 5, 122, 26, },
+	{ 2, 1, 2, 5, 122, 30, },
+	{ 1, 1, 2, 5, 122, 30, },
+	{ 0, 1, 2, 5, 155, 26, },
+	{ 2, 1, 2, 5, 155, 30, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 22, },
+	{ 2, 1, 2, 8, 42, 28, },
+	{ 1, 1, 2, 8, 42, 28, },
+	{ 0, 1, 2, 8, 58, 22, },
+	{ 2, 1, 2, 8, 58, 28, },
+	{ 1, 1, 2, 8, 58, 28, },
+	{ 0, 1, 2, 8, 106, 24, },
+	{ 2, 1, 2, 8, 106, 28, },
+	{ 1, 1, 2, 8, 106, 28, },
+	{ 0, 1, 2, 8, 122, 24, },
+	{ 2, 1, 2, 8, 122, 28, },
+	{ 1, 1, 2, 8, 122, 28, },
+	{ 0, 1, 2, 8, 155, 24, },
+	{ 2, 1, 2, 8, 155, 28, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 20, },
+	{ 2, 1, 2, 9, 42, 26, },
+	{ 1, 1, 2, 9, 42, 26, },
+	{ 0, 1, 2, 9, 58, 20, },
+	{ 2, 1, 2, 9, 58, 26, },
+	{ 1, 1, 2, 9, 58, 26, },
+	{ 0, 1, 2, 9, 106, 22, },
+	{ 2, 1, 2, 9, 106, 26, },
+	{ 1, 1, 2, 9, 106, 26, },
+	{ 0, 1, 2, 9, 122, 22, },
+	{ 2, 1, 2, 9, 122, 26, },
+	{ 1, 1, 2, 9, 122, 26, },
+	{ 0, 1, 2, 9, 155, 22, },
+	{ 2, 1, 2, 9, 155, 26, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt_type7);
+
+static const struct rtw_txpwr_lmt_cfg_pair rtw8814a_txpwr_lmt_type8[] = {
+	{ 0, 0, 0, 0, 1, 46, },
+	{ 2, 0, 0, 0, 1, 46, },
+	{ 1, 0, 0, 0, 1, 46, },
+	{ 0, 0, 0, 0, 2, 46, },
+	{ 2, 0, 0, 0, 2, 46, },
+	{ 1, 0, 0, 0, 2, 46, },
+	{ 0, 0, 0, 0, 3, 46, },
+	{ 2, 0, 0, 0, 3, 46, },
+	{ 1, 0, 0, 0, 3, 46, },
+	{ 0, 0, 0, 0, 4, 46, },
+	{ 2, 0, 0, 0, 4, 46, },
+	{ 1, 0, 0, 0, 4, 46, },
+	{ 0, 0, 0, 0, 5, 46, },
+	{ 2, 0, 0, 0, 5, 46, },
+	{ 1, 0, 0, 0, 5, 46, },
+	{ 0, 0, 0, 0, 6, 46, },
+	{ 2, 0, 0, 0, 6, 46, },
+	{ 1, 0, 0, 0, 6, 46, },
+	{ 0, 0, 0, 0, 7, 46, },
+	{ 2, 0, 0, 0, 7, 46, },
+	{ 1, 0, 0, 0, 7, 46, },
+	{ 0, 0, 0, 0, 8, 46, },
+	{ 2, 0, 0, 0, 8, 46, },
+	{ 1, 0, 0, 0, 8, 46, },
+	{ 0, 0, 0, 0, 9, 46, },
+	{ 2, 0, 0, 0, 9, 46, },
+	{ 1, 0, 0, 0, 9, 46, },
+	{ 0, 0, 0, 0, 10, 46, },
+	{ 2, 0, 0, 0, 10, 46, },
+	{ 1, 0, 0, 0, 10, 46, },
+	{ 0, 0, 0, 0, 11, 46, },
+	{ 2, 0, 0, 0, 11, 46, },
+	{ 1, 0, 0, 0, 11, 46, },
+	{ 0, 0, 0, 0, 12, 63, },
+	{ 2, 0, 0, 0, 12, 46, },
+	{ 1, 0, 0, 0, 12, 46, },
+	{ 0, 0, 0, 0, 13, 63, },
+	{ 2, 0, 0, 0, 13, 46, },
+	{ 1, 0, 0, 0, 13, 46, },
+	{ 0, 0, 0, 0, 14, 63, },
+	{ 2, 0, 0, 0, 14, 63, },
+	{ 1, 0, 0, 0, 14, 46, },
+	{ 0, 0, 0, 1, 1, 46, },
+	{ 2, 0, 0, 1, 1, 46, },
+	{ 1, 0, 0, 1, 1, 46, },
+	{ 0, 0, 0, 1, 2, 46, },
+	{ 2, 0, 0, 1, 2, 46, },
+	{ 1, 0, 0, 1, 2, 46, },
+	{ 0, 0, 0, 1, 3, 46, },
+	{ 2, 0, 0, 1, 3, 46, },
+	{ 1, 0, 0, 1, 3, 46, },
+	{ 0, 0, 0, 1, 4, 46, },
+	{ 2, 0, 0, 1, 4, 46, },
+	{ 1, 0, 0, 1, 4, 46, },
+	{ 0, 0, 0, 1, 5, 46, },
+	{ 2, 0, 0, 1, 5, 46, },
+	{ 1, 0, 0, 1, 5, 46, },
+	{ 0, 0, 0, 1, 6, 46, },
+	{ 2, 0, 0, 1, 6, 46, },
+	{ 1, 0, 0, 1, 6, 46, },
+	{ 0, 0, 0, 1, 7, 46, },
+	{ 2, 0, 0, 1, 7, 46, },
+	{ 1, 0, 0, 1, 7, 46, },
+	{ 0, 0, 0, 1, 8, 46, },
+	{ 2, 0, 0, 1, 8, 46, },
+	{ 1, 0, 0, 1, 8, 46, },
+	{ 0, 0, 0, 1, 9, 46, },
+	{ 2, 0, 0, 1, 9, 46, },
+	{ 1, 0, 0, 1, 9, 46, },
+	{ 0, 0, 0, 1, 10, 46, },
+	{ 2, 0, 0, 1, 10, 46, },
+	{ 1, 0, 0, 1, 10, 46, },
+	{ 0, 0, 0, 1, 11, 46, },
+	{ 2, 0, 0, 1, 11, 46, },
+	{ 1, 0, 0, 1, 11, 46, },
+	{ 0, 0, 0, 1, 12, 63, },
+	{ 2, 0, 0, 1, 12, 46, },
+	{ 1, 0, 0, 1, 12, 46, },
+	{ 0, 0, 0, 1, 13, 63, },
+	{ 2, 0, 0, 1, 13, 46, },
+	{ 1, 0, 0, 1, 13, 46, },
+	{ 0, 0, 0, 1, 14, 63, },
+	{ 2, 0, 0, 1, 14, 63, },
+	{ 1, 0, 0, 1, 14, 46, },
+	{ 0, 0, 0, 2, 1, 46, },
+	{ 2, 0, 0, 2, 1, 46, },
+	{ 1, 0, 0, 2, 1, 46, },
+	{ 0, 0, 0, 2, 2, 46, },
+	{ 2, 0, 0, 2, 2, 46, },
+	{ 1, 0, 0, 2, 2, 46, },
+	{ 0, 0, 0, 2, 3, 46, },
+	{ 2, 0, 0, 2, 3, 46, },
+	{ 1, 0, 0, 2, 3, 46, },
+	{ 0, 0, 0, 2, 4, 46, },
+	{ 2, 0, 0, 2, 4, 46, },
+	{ 1, 0, 0, 2, 4, 46, },
+	{ 0, 0, 0, 2, 5, 46, },
+	{ 2, 0, 0, 2, 5, 46, },
+	{ 1, 0, 0, 2, 5, 46, },
+	{ 0, 0, 0, 2, 6, 46, },
+	{ 2, 0, 0, 2, 6, 46, },
+	{ 1, 0, 0, 2, 6, 46, },
+	{ 0, 0, 0, 2, 7, 46, },
+	{ 2, 0, 0, 2, 7, 46, },
+	{ 1, 0, 0, 2, 7, 46, },
+	{ 0, 0, 0, 2, 8, 46, },
+	{ 2, 0, 0, 2, 8, 46, },
+	{ 1, 0, 0, 2, 8, 46, },
+	{ 0, 0, 0, 2, 9, 46, },
+	{ 2, 0, 0, 2, 9, 46, },
+	{ 1, 0, 0, 2, 9, 46, },
+	{ 0, 0, 0, 2, 10, 46, },
+	{ 2, 0, 0, 2, 10, 46, },
+	{ 1, 0, 0, 2, 10, 46, },
+	{ 0, 0, 0, 2, 11, 46, },
+	{ 2, 0, 0, 2, 11, 46, },
+	{ 1, 0, 0, 2, 11, 46, },
+	{ 0, 0, 0, 2, 12, 63, },
+	{ 2, 0, 0, 2, 12, 46, },
+	{ 1, 0, 0, 2, 12, 46, },
+	{ 0, 0, 0, 2, 13, 63, },
+	{ 2, 0, 0, 2, 13, 46, },
+	{ 1, 0, 0, 2, 13, 46, },
+	{ 0, 0, 0, 2, 14, 63, },
+	{ 2, 0, 0, 2, 14, 63, },
+	{ 1, 0, 0, 2, 14, 46, },
+	{ 0, 0, 0, 3, 1, 46, },
+	{ 2, 0, 0, 3, 1, 46, },
+	{ 1, 0, 0, 3, 1, 46, },
+	{ 0, 0, 0, 3, 2, 46, },
+	{ 2, 0, 0, 3, 2, 46, },
+	{ 1, 0, 0, 3, 2, 46, },
+	{ 0, 0, 0, 3, 3, 46, },
+	{ 2, 0, 0, 3, 3, 46, },
+	{ 1, 0, 0, 3, 3, 46, },
+	{ 0, 0, 0, 3, 4, 46, },
+	{ 2, 0, 0, 3, 4, 46, },
+	{ 1, 0, 0, 3, 4, 46, },
+	{ 0, 0, 0, 3, 5, 46, },
+	{ 2, 0, 0, 3, 5, 46, },
+	{ 1, 0, 0, 3, 5, 46, },
+	{ 0, 0, 0, 3, 6, 46, },
+	{ 2, 0, 0, 3, 6, 46, },
+	{ 1, 0, 0, 3, 6, 46, },
+	{ 0, 0, 0, 3, 7, 46, },
+	{ 2, 0, 0, 3, 7, 46, },
+	{ 1, 0, 0, 3, 7, 46, },
+	{ 0, 0, 0, 3, 8, 46, },
+	{ 2, 0, 0, 3, 8, 46, },
+	{ 1, 0, 0, 3, 8, 46, },
+	{ 0, 0, 0, 3, 9, 46, },
+	{ 2, 0, 0, 3, 9, 46, },
+	{ 1, 0, 0, 3, 9, 46, },
+	{ 0, 0, 0, 3, 10, 46, },
+	{ 2, 0, 0, 3, 10, 46, },
+	{ 1, 0, 0, 3, 10, 46, },
+	{ 0, 0, 0, 3, 11, 46, },
+	{ 2, 0, 0, 3, 11, 46, },
+	{ 1, 0, 0, 3, 11, 46, },
+	{ 0, 0, 0, 3, 12, 63, },
+	{ 2, 0, 0, 3, 12, 46, },
+	{ 1, 0, 0, 3, 12, 46, },
+	{ 0, 0, 0, 3, 13, 63, },
+	{ 2, 0, 0, 3, 13, 46, },
+	{ 1, 0, 0, 3, 13, 46, },
+	{ 0, 0, 0, 3, 14, 63, },
+	{ 2, 0, 0, 3, 14, 63, },
+	{ 1, 0, 0, 3, 14, 46, },
+	{ 0, 0, 0, 6, 1, 46, },
+	{ 2, 0, 0, 6, 1, 46, },
+	{ 1, 0, 0, 6, 1, 46, },
+	{ 0, 0, 0, 6, 2, 46, },
+	{ 2, 0, 0, 6, 2, 46, },
+	{ 1, 0, 0, 6, 2, 46, },
+	{ 0, 0, 0, 6, 3, 46, },
+	{ 2, 0, 0, 6, 3, 46, },
+	{ 1, 0, 0, 6, 3, 46, },
+	{ 0, 0, 0, 6, 4, 46, },
+	{ 2, 0, 0, 6, 4, 46, },
+	{ 1, 0, 0, 6, 4, 46, },
+	{ 0, 0, 0, 6, 5, 46, },
+	{ 2, 0, 0, 6, 5, 46, },
+	{ 1, 0, 0, 6, 5, 46, },
+	{ 0, 0, 0, 6, 6, 46, },
+	{ 2, 0, 0, 6, 6, 46, },
+	{ 1, 0, 0, 6, 6, 46, },
+	{ 0, 0, 0, 6, 7, 46, },
+	{ 2, 0, 0, 6, 7, 46, },
+	{ 1, 0, 0, 6, 7, 46, },
+	{ 0, 0, 0, 6, 8, 46, },
+	{ 2, 0, 0, 6, 8, 46, },
+	{ 1, 0, 0, 6, 8, 46, },
+	{ 0, 0, 0, 6, 9, 46, },
+	{ 2, 0, 0, 6, 9, 46, },
+	{ 1, 0, 0, 6, 9, 46, },
+	{ 0, 0, 0, 6, 10, 46, },
+	{ 2, 0, 0, 6, 10, 46, },
+	{ 1, 0, 0, 6, 10, 46, },
+	{ 0, 0, 0, 6, 11, 46, },
+	{ 2, 0, 0, 6, 11, 46, },
+	{ 1, 0, 0, 6, 11, 46, },
+	{ 0, 0, 0, 6, 12, 63, },
+	{ 2, 0, 0, 6, 12, 46, },
+	{ 1, 0, 0, 6, 12, 46, },
+	{ 0, 0, 0, 6, 13, 63, },
+	{ 2, 0, 0, 6, 13, 46, },
+	{ 1, 0, 0, 6, 13, 46, },
+	{ 0, 0, 0, 6, 14, 63, },
+	{ 2, 0, 0, 6, 14, 63, },
+	{ 1, 0, 0, 6, 14, 46, },
+	{ 0, 0, 0, 7, 1, 46, },
+	{ 2, 0, 0, 7, 1, 46, },
+	{ 1, 0, 0, 7, 1, 46, },
+	{ 0, 0, 0, 7, 2, 46, },
+	{ 2, 0, 0, 7, 2, 46, },
+	{ 1, 0, 0, 7, 2, 46, },
+	{ 0, 0, 0, 7, 3, 46, },
+	{ 2, 0, 0, 7, 3, 46, },
+	{ 1, 0, 0, 7, 3, 46, },
+	{ 0, 0, 0, 7, 4, 46, },
+	{ 2, 0, 0, 7, 4, 46, },
+	{ 1, 0, 0, 7, 4, 46, },
+	{ 0, 0, 0, 7, 5, 46, },
+	{ 2, 0, 0, 7, 5, 46, },
+	{ 1, 0, 0, 7, 5, 46, },
+	{ 0, 0, 0, 7, 6, 46, },
+	{ 2, 0, 0, 7, 6, 46, },
+	{ 1, 0, 0, 7, 6, 46, },
+	{ 0, 0, 0, 7, 7, 46, },
+	{ 2, 0, 0, 7, 7, 46, },
+	{ 1, 0, 0, 7, 7, 46, },
+	{ 0, 0, 0, 7, 8, 46, },
+	{ 2, 0, 0, 7, 8, 46, },
+	{ 1, 0, 0, 7, 8, 46, },
+	{ 0, 0, 0, 7, 9, 46, },
+	{ 2, 0, 0, 7, 9, 46, },
+	{ 1, 0, 0, 7, 9, 46, },
+	{ 0, 0, 0, 7, 10, 46, },
+	{ 2, 0, 0, 7, 10, 46, },
+	{ 1, 0, 0, 7, 10, 46, },
+	{ 0, 0, 0, 7, 11, 46, },
+	{ 2, 0, 0, 7, 11, 46, },
+	{ 1, 0, 0, 7, 11, 46, },
+	{ 0, 0, 0, 7, 12, 63, },
+	{ 2, 0, 0, 7, 12, 46, },
+	{ 1, 0, 0, 7, 12, 46, },
+	{ 0, 0, 0, 7, 13, 63, },
+	{ 2, 0, 0, 7, 13, 46, },
+	{ 1, 0, 0, 7, 13, 46, },
+	{ 0, 0, 0, 7, 14, 63, },
+	{ 2, 0, 0, 7, 14, 63, },
+	{ 1, 0, 0, 7, 14, 46, },
+	{ 0, 0, 1, 2, 1, 63, },
+	{ 2, 0, 1, 2, 1, 63, },
+	{ 1, 0, 1, 2, 1, 63, },
+	{ 0, 0, 1, 2, 2, 63, },
+	{ 2, 0, 1, 2, 2, 63, },
+	{ 1, 0, 1, 2, 2, 63, },
+	{ 0, 0, 1, 2, 3, 30, },
+	{ 2, 0, 1, 2, 3, 34, },
+	{ 1, 0, 1, 2, 3, 34, },
+	{ 0, 0, 1, 2, 4, 34, },
+	{ 2, 0, 1, 2, 4, 34, },
+	{ 1, 0, 1, 2, 4, 34, },
+	{ 0, 0, 1, 2, 5, 34, },
+	{ 2, 0, 1, 2, 5, 34, },
+	{ 1, 0, 1, 2, 5, 34, },
+	{ 0, 0, 1, 2, 6, 34, },
+	{ 2, 0, 1, 2, 6, 34, },
+	{ 1, 0, 1, 2, 6, 34, },
+	{ 0, 0, 1, 2, 7, 34, },
+	{ 2, 0, 1, 2, 7, 34, },
+	{ 1, 0, 1, 2, 7, 34, },
+	{ 0, 0, 1, 2, 8, 34, },
+	{ 2, 0, 1, 2, 8, 34, },
+	{ 1, 0, 1, 2, 8, 34, },
+	{ 0, 0, 1, 2, 9, 34, },
+	{ 2, 0, 1, 2, 9, 34, },
+	{ 1, 0, 1, 2, 9, 34, },
+	{ 0, 0, 1, 2, 10, 34, },
+	{ 2, 0, 1, 2, 10, 34, },
+	{ 1, 0, 1, 2, 10, 34, },
+	{ 0, 0, 1, 2, 11, 28, },
+	{ 2, 0, 1, 2, 11, 34, },
+	{ 1, 0, 1, 2, 11, 34, },
+	{ 0, 0, 1, 2, 12, 63, },
+	{ 2, 0, 1, 2, 12, 34, },
+	{ 1, 0, 1, 2, 12, 34, },
+	{ 0, 0, 1, 2, 13, 63, },
+	{ 2, 0, 1, 2, 13, 34, },
+	{ 1, 0, 1, 2, 13, 34, },
+	{ 0, 0, 1, 2, 14, 63, },
+	{ 2, 0, 1, 2, 14, 63, },
+	{ 1, 0, 1, 2, 14, 63, },
+	{ 0, 0, 1, 3, 1, 63, },
+	{ 2, 0, 1, 3, 1, 63, },
+	{ 1, 0, 1, 3, 1, 63, },
+	{ 0, 0, 1, 3, 2, 63, },
+	{ 2, 0, 1, 3, 2, 63, },
+	{ 1, 0, 1, 3, 2, 63, },
+	{ 0, 0, 1, 3, 3, 30, },
+	{ 2, 0, 1, 3, 3, 34, },
+	{ 1, 0, 1, 3, 3, 34, },
+	{ 0, 0, 1, 3, 4, 34, },
+	{ 2, 0, 1, 3, 4, 34, },
+	{ 1, 0, 1, 3, 4, 34, },
+	{ 0, 0, 1, 3, 5, 34, },
+	{ 2, 0, 1, 3, 5, 34, },
+	{ 1, 0, 1, 3, 5, 34, },
+	{ 0, 0, 1, 3, 6, 34, },
+	{ 2, 0, 1, 3, 6, 34, },
+	{ 1, 0, 1, 3, 6, 34, },
+	{ 0, 0, 1, 3, 7, 34, },
+	{ 2, 0, 1, 3, 7, 34, },
+	{ 1, 0, 1, 3, 7, 34, },
+	{ 0, 0, 1, 3, 8, 34, },
+	{ 2, 0, 1, 3, 8, 34, },
+	{ 1, 0, 1, 3, 8, 34, },
+	{ 0, 0, 1, 3, 9, 34, },
+	{ 2, 0, 1, 3, 9, 34, },
+	{ 1, 0, 1, 3, 9, 34, },
+	{ 0, 0, 1, 3, 10, 34, },
+	{ 2, 0, 1, 3, 10, 34, },
+	{ 1, 0, 1, 3, 10, 34, },
+	{ 0, 0, 1, 3, 11, 28, },
+	{ 2, 0, 1, 3, 11, 34, },
+	{ 1, 0, 1, 3, 11, 34, },
+	{ 0, 0, 1, 3, 12, 63, },
+	{ 2, 0, 1, 3, 12, 34, },
+	{ 1, 0, 1, 3, 12, 34, },
+	{ 0, 0, 1, 3, 13, 63, },
+	{ 2, 0, 1, 3, 13, 34, },
+	{ 1, 0, 1, 3, 13, 34, },
+	{ 0, 0, 1, 3, 14, 63, },
+	{ 2, 0, 1, 3, 14, 63, },
+	{ 1, 0, 1, 3, 14, 63, },
+	{ 0, 0, 1, 6, 1, 63, },
+	{ 2, 0, 1, 6, 1, 63, },
+	{ 1, 0, 1, 6, 1, 63, },
+	{ 0, 0, 1, 6, 2, 63, },
+	{ 2, 0, 1, 6, 2, 63, },
+	{ 1, 0, 1, 6, 2, 63, },
+	{ 0, 0, 1, 6, 3, 30, },
+	{ 2, 0, 1, 6, 3, 34, },
+	{ 1, 0, 1, 6, 3, 34, },
+	{ 0, 0, 1, 6, 4, 34, },
+	{ 2, 0, 1, 6, 4, 34, },
+	{ 1, 0, 1, 6, 4, 34, },
+	{ 0, 0, 1, 6, 5, 34, },
+	{ 2, 0, 1, 6, 5, 34, },
+	{ 1, 0, 1, 6, 5, 34, },
+	{ 0, 0, 1, 6, 6, 34, },
+	{ 2, 0, 1, 6, 6, 34, },
+	{ 1, 0, 1, 6, 6, 34, },
+	{ 0, 0, 1, 6, 7, 34, },
+	{ 2, 0, 1, 6, 7, 34, },
+	{ 1, 0, 1, 6, 7, 34, },
+	{ 0, 0, 1, 6, 8, 34, },
+	{ 2, 0, 1, 6, 8, 34, },
+	{ 1, 0, 1, 6, 8, 34, },
+	{ 0, 0, 1, 6, 9, 34, },
+	{ 2, 0, 1, 6, 9, 34, },
+	{ 1, 0, 1, 6, 9, 34, },
+	{ 0, 0, 1, 6, 10, 34, },
+	{ 2, 0, 1, 6, 10, 34, },
+	{ 1, 0, 1, 6, 10, 34, },
+	{ 0, 0, 1, 6, 11, 28, },
+	{ 2, 0, 1, 6, 11, 34, },
+	{ 1, 0, 1, 6, 11, 34, },
+	{ 0, 0, 1, 6, 12, 63, },
+	{ 2, 0, 1, 6, 12, 34, },
+	{ 1, 0, 1, 6, 12, 34, },
+	{ 0, 0, 1, 6, 13, 63, },
+	{ 2, 0, 1, 6, 13, 34, },
+	{ 1, 0, 1, 6, 13, 34, },
+	{ 0, 0, 1, 6, 14, 63, },
+	{ 2, 0, 1, 6, 14, 63, },
+	{ 1, 0, 1, 6, 14, 63, },
+	{ 0, 0, 1, 7, 1, 63, },
+	{ 2, 0, 1, 7, 1, 63, },
+	{ 1, 0, 1, 7, 1, 63, },
+	{ 0, 0, 1, 7, 2, 63, },
+	{ 2, 0, 1, 7, 2, 63, },
+	{ 1, 0, 1, 7, 2, 63, },
+	{ 0, 0, 1, 7, 3, 30, },
+	{ 2, 0, 1, 7, 3, 34, },
+	{ 1, 0, 1, 7, 3, 34, },
+	{ 0, 0, 1, 7, 4, 34, },
+	{ 2, 0, 1, 7, 4, 34, },
+	{ 1, 0, 1, 7, 4, 34, },
+	{ 0, 0, 1, 7, 5, 34, },
+	{ 2, 0, 1, 7, 5, 34, },
+	{ 1, 0, 1, 7, 5, 34, },
+	{ 0, 0, 1, 7, 6, 34, },
+	{ 2, 0, 1, 7, 6, 34, },
+	{ 1, 0, 1, 7, 6, 34, },
+	{ 0, 0, 1, 7, 7, 34, },
+	{ 2, 0, 1, 7, 7, 34, },
+	{ 1, 0, 1, 7, 7, 34, },
+	{ 0, 0, 1, 7, 8, 34, },
+	{ 2, 0, 1, 7, 8, 34, },
+	{ 1, 0, 1, 7, 8, 34, },
+	{ 0, 0, 1, 7, 9, 34, },
+	{ 2, 0, 1, 7, 9, 34, },
+	{ 1, 0, 1, 7, 9, 34, },
+	{ 0, 0, 1, 7, 10, 34, },
+	{ 2, 0, 1, 7, 10, 34, },
+	{ 1, 0, 1, 7, 10, 34, },
+	{ 0, 0, 1, 7, 11, 28, },
+	{ 2, 0, 1, 7, 11, 34, },
+	{ 1, 0, 1, 7, 11, 34, },
+	{ 0, 0, 1, 7, 12, 63, },
+	{ 2, 0, 1, 7, 12, 34, },
+	{ 1, 0, 1, 7, 12, 34, },
+	{ 0, 0, 1, 7, 13, 63, },
+	{ 2, 0, 1, 7, 13, 34, },
+	{ 1, 0, 1, 7, 13, 34, },
+	{ 0, 0, 1, 7, 14, 63, },
+	{ 2, 0, 1, 7, 14, 63, },
+	{ 1, 0, 1, 7, 14, 63, },
+	{ 0, 1, 0, 1, 36, 46, },
+	{ 2, 1, 0, 1, 36, 46, },
+	{ 1, 1, 0, 1, 36, 46, },
+	{ 0, 1, 0, 1, 40, 46, },
+	{ 2, 1, 0, 1, 40, 46, },
+	{ 1, 1, 0, 1, 40, 46, },
+	{ 0, 1, 0, 1, 44, 46, },
+	{ 2, 1, 0, 1, 44, 46, },
+	{ 1, 1, 0, 1, 44, 46, },
+	{ 0, 1, 0, 1, 48, 46, },
+	{ 2, 1, 0, 1, 48, 46, },
+	{ 1, 1, 0, 1, 48, 46, },
+	{ 0, 1, 0, 1, 52, 46, },
+	{ 2, 1, 0, 1, 52, 46, },
+	{ 1, 1, 0, 1, 52, 46, },
+	{ 0, 1, 0, 1, 56, 46, },
+	{ 2, 1, 0, 1, 56, 46, },
+	{ 1, 1, 0, 1, 56, 46, },
+	{ 0, 1, 0, 1, 60, 46, },
+	{ 2, 1, 0, 1, 60, 46, },
+	{ 1, 1, 0, 1, 60, 46, },
+	{ 0, 1, 0, 1, 64, 46, },
+	{ 2, 1, 0, 1, 64, 46, },
+	{ 1, 1, 0, 1, 64, 46, },
+	{ 0, 1, 0, 1, 100, 46, },
+	{ 2, 1, 0, 1, 100, 46, },
+	{ 1, 1, 0, 1, 100, 46, },
+	{ 0, 1, 0, 1, 104, 46, },
+	{ 2, 1, 0, 1, 104, 46, },
+	{ 1, 1, 0, 1, 104, 46, },
+	{ 0, 1, 0, 1, 108, 46, },
+	{ 2, 1, 0, 1, 108, 46, },
+	{ 1, 1, 0, 1, 108, 46, },
+	{ 0, 1, 0, 1, 112, 46, },
+	{ 2, 1, 0, 1, 112, 46, },
+	{ 1, 1, 0, 1, 112, 46, },
+	{ 0, 1, 0, 1, 116, 46, },
+	{ 2, 1, 0, 1, 116, 46, },
+	{ 1, 1, 0, 1, 116, 46, },
+	{ 0, 1, 0, 1, 120, 46, },
+	{ 2, 1, 0, 1, 120, 46, },
+	{ 1, 1, 0, 1, 120, 46, },
+	{ 0, 1, 0, 1, 124, 46, },
+	{ 2, 1, 0, 1, 124, 46, },
+	{ 1, 1, 0, 1, 124, 46, },
+	{ 0, 1, 0, 1, 128, 46, },
+	{ 2, 1, 0, 1, 128, 46, },
+	{ 1, 1, 0, 1, 128, 46, },
+	{ 0, 1, 0, 1, 132, 46, },
+	{ 2, 1, 0, 1, 132, 46, },
+	{ 1, 1, 0, 1, 132, 46, },
+	{ 0, 1, 0, 1, 136, 46, },
+	{ 2, 1, 0, 1, 136, 46, },
+	{ 1, 1, 0, 1, 136, 46, },
+	{ 0, 1, 0, 1, 140, 46, },
+	{ 2, 1, 0, 1, 140, 46, },
+	{ 1, 1, 0, 1, 140, 46, },
+	{ 0, 1, 0, 1, 149, 46, },
+	{ 2, 1, 0, 1, 149, 46, },
+	{ 1, 1, 0, 1, 149, 63, },
+	{ 0, 1, 0, 1, 153, 46, },
+	{ 2, 1, 0, 1, 153, 46, },
+	{ 1, 1, 0, 1, 153, 63, },
+	{ 0, 1, 0, 1, 157, 46, },
+	{ 2, 1, 0, 1, 157, 46, },
+	{ 1, 1, 0, 1, 157, 63, },
+	{ 0, 1, 0, 1, 161, 46, },
+	{ 2, 1, 0, 1, 161, 46, },
+	{ 1, 1, 0, 1, 161, 63, },
+	{ 0, 1, 0, 1, 165, 46, },
+	{ 2, 1, 0, 1, 165, 46, },
+	{ 1, 1, 0, 1, 165, 63, },
+	{ 0, 1, 0, 2, 36, 46, },
+	{ 2, 1, 0, 2, 36, 46, },
+	{ 1, 1, 0, 2, 36, 46, },
+	{ 0, 1, 0, 2, 40, 46, },
+	{ 2, 1, 0, 2, 40, 46, },
+	{ 1, 1, 0, 2, 40, 46, },
+	{ 0, 1, 0, 2, 44, 46, },
+	{ 2, 1, 0, 2, 44, 46, },
+	{ 1, 1, 0, 2, 44, 46, },
+	{ 0, 1, 0, 2, 48, 46, },
+	{ 2, 1, 0, 2, 48, 46, },
+	{ 1, 1, 0, 2, 48, 46, },
+	{ 0, 1, 0, 2, 52, 46, },
+	{ 2, 1, 0, 2, 52, 46, },
+	{ 1, 1, 0, 2, 52, 46, },
+	{ 0, 1, 0, 2, 56, 46, },
+	{ 2, 1, 0, 2, 56, 46, },
+	{ 1, 1, 0, 2, 56, 46, },
+	{ 0, 1, 0, 2, 60, 46, },
+	{ 2, 1, 0, 2, 60, 46, },
+	{ 1, 1, 0, 2, 60, 46, },
+	{ 0, 1, 0, 2, 64, 46, },
+	{ 2, 1, 0, 2, 64, 46, },
+	{ 1, 1, 0, 2, 64, 46, },
+	{ 0, 1, 0, 2, 100, 46, },
+	{ 2, 1, 0, 2, 100, 46, },
+	{ 1, 1, 0, 2, 100, 46, },
+	{ 0, 1, 0, 2, 104, 46, },
+	{ 2, 1, 0, 2, 104, 46, },
+	{ 1, 1, 0, 2, 104, 46, },
+	{ 0, 1, 0, 2, 108, 46, },
+	{ 2, 1, 0, 2, 108, 46, },
+	{ 1, 1, 0, 2, 108, 46, },
+	{ 0, 1, 0, 2, 112, 46, },
+	{ 2, 1, 0, 2, 112, 46, },
+	{ 1, 1, 0, 2, 112, 46, },
+	{ 0, 1, 0, 2, 116, 46, },
+	{ 2, 1, 0, 2, 116, 46, },
+	{ 1, 1, 0, 2, 116, 46, },
+	{ 0, 1, 0, 2, 120, 46, },
+	{ 2, 1, 0, 2, 120, 46, },
+	{ 1, 1, 0, 2, 120, 46, },
+	{ 0, 1, 0, 2, 124, 46, },
+	{ 2, 1, 0, 2, 124, 46, },
+	{ 1, 1, 0, 2, 124, 46, },
+	{ 0, 1, 0, 2, 128, 46, },
+	{ 2, 1, 0, 2, 128, 46, },
+	{ 1, 1, 0, 2, 128, 46, },
+	{ 0, 1, 0, 2, 132, 46, },
+	{ 2, 1, 0, 2, 132, 46, },
+	{ 1, 1, 0, 2, 132, 46, },
+	{ 0, 1, 0, 2, 136, 46, },
+	{ 2, 1, 0, 2, 136, 46, },
+	{ 1, 1, 0, 2, 136, 46, },
+	{ 0, 1, 0, 2, 140, 46, },
+	{ 2, 1, 0, 2, 140, 46, },
+	{ 1, 1, 0, 2, 140, 46, },
+	{ 0, 1, 0, 2, 149, 46, },
+	{ 2, 1, 0, 2, 149, 46, },
+	{ 1, 1, 0, 2, 149, 63, },
+	{ 0, 1, 0, 2, 153, 46, },
+	{ 2, 1, 0, 2, 153, 46, },
+	{ 1, 1, 0, 2, 153, 63, },
+	{ 0, 1, 0, 2, 157, 46, },
+	{ 2, 1, 0, 2, 157, 46, },
+	{ 1, 1, 0, 2, 157, 63, },
+	{ 0, 1, 0, 2, 161, 46, },
+	{ 2, 1, 0, 2, 161, 46, },
+	{ 1, 1, 0, 2, 161, 63, },
+	{ 0, 1, 0, 2, 165, 46, },
+	{ 2, 1, 0, 2, 165, 46, },
+	{ 1, 1, 0, 2, 165, 63, },
+	{ 0, 1, 0, 3, 36, 46, },
+	{ 2, 1, 0, 3, 36, 46, },
+	{ 1, 1, 0, 3, 36, 46, },
+	{ 0, 1, 0, 3, 40, 46, },
+	{ 2, 1, 0, 3, 40, 46, },
+	{ 1, 1, 0, 3, 40, 46, },
+	{ 0, 1, 0, 3, 44, 46, },
+	{ 2, 1, 0, 3, 44, 46, },
+	{ 1, 1, 0, 3, 44, 46, },
+	{ 0, 1, 0, 3, 48, 46, },
+	{ 2, 1, 0, 3, 48, 46, },
+	{ 1, 1, 0, 3, 48, 46, },
+	{ 0, 1, 0, 3, 52, 46, },
+	{ 2, 1, 0, 3, 52, 46, },
+	{ 1, 1, 0, 3, 52, 46, },
+	{ 0, 1, 0, 3, 56, 46, },
+	{ 2, 1, 0, 3, 56, 46, },
+	{ 1, 1, 0, 3, 56, 46, },
+	{ 0, 1, 0, 3, 60, 46, },
+	{ 2, 1, 0, 3, 60, 46, },
+	{ 1, 1, 0, 3, 60, 46, },
+	{ 0, 1, 0, 3, 64, 46, },
+	{ 2, 1, 0, 3, 64, 46, },
+	{ 1, 1, 0, 3, 64, 46, },
+	{ 0, 1, 0, 3, 100, 46, },
+	{ 2, 1, 0, 3, 100, 46, },
+	{ 1, 1, 0, 3, 100, 46, },
+	{ 0, 1, 0, 3, 104, 46, },
+	{ 2, 1, 0, 3, 104, 46, },
+	{ 1, 1, 0, 3, 104, 46, },
+	{ 0, 1, 0, 3, 108, 46, },
+	{ 2, 1, 0, 3, 108, 46, },
+	{ 1, 1, 0, 3, 108, 46, },
+	{ 0, 1, 0, 3, 112, 46, },
+	{ 2, 1, 0, 3, 112, 46, },
+	{ 1, 1, 0, 3, 112, 46, },
+	{ 0, 1, 0, 3, 116, 46, },
+	{ 2, 1, 0, 3, 116, 46, },
+	{ 1, 1, 0, 3, 116, 46, },
+	{ 0, 1, 0, 3, 120, 46, },
+	{ 2, 1, 0, 3, 120, 46, },
+	{ 1, 1, 0, 3, 120, 46, },
+	{ 0, 1, 0, 3, 124, 46, },
+	{ 2, 1, 0, 3, 124, 46, },
+	{ 1, 1, 0, 3, 124, 46, },
+	{ 0, 1, 0, 3, 128, 46, },
+	{ 2, 1, 0, 3, 128, 46, },
+	{ 1, 1, 0, 3, 128, 46, },
+	{ 0, 1, 0, 3, 132, 46, },
+	{ 2, 1, 0, 3, 132, 46, },
+	{ 1, 1, 0, 3, 132, 46, },
+	{ 0, 1, 0, 3, 136, 46, },
+	{ 2, 1, 0, 3, 136, 46, },
+	{ 1, 1, 0, 3, 136, 46, },
+	{ 0, 1, 0, 3, 140, 46, },
+	{ 2, 1, 0, 3, 140, 46, },
+	{ 1, 1, 0, 3, 140, 46, },
+	{ 0, 1, 0, 3, 149, 46, },
+	{ 2, 1, 0, 3, 149, 46, },
+	{ 1, 1, 0, 3, 149, 63, },
+	{ 0, 1, 0, 3, 153, 46, },
+	{ 2, 1, 0, 3, 153, 46, },
+	{ 1, 1, 0, 3, 153, 63, },
+	{ 0, 1, 0, 3, 157, 46, },
+	{ 2, 1, 0, 3, 157, 46, },
+	{ 1, 1, 0, 3, 157, 63, },
+	{ 0, 1, 0, 3, 161, 46, },
+	{ 2, 1, 0, 3, 161, 46, },
+	{ 1, 1, 0, 3, 161, 63, },
+	{ 0, 1, 0, 3, 165, 46, },
+	{ 2, 1, 0, 3, 165, 46, },
+	{ 1, 1, 0, 3, 165, 63, },
+	{ 0, 1, 0, 6, 36, 46, },
+	{ 2, 1, 0, 6, 36, 46, },
+	{ 1, 1, 0, 6, 36, 46, },
+	{ 0, 1, 0, 6, 40, 46, },
+	{ 2, 1, 0, 6, 40, 46, },
+	{ 1, 1, 0, 6, 40, 46, },
+	{ 0, 1, 0, 6, 44, 46, },
+	{ 2, 1, 0, 6, 44, 46, },
+	{ 1, 1, 0, 6, 44, 46, },
+	{ 0, 1, 0, 6, 48, 46, },
+	{ 2, 1, 0, 6, 48, 46, },
+	{ 1, 1, 0, 6, 48, 46, },
+	{ 0, 1, 0, 6, 52, 46, },
+	{ 2, 1, 0, 6, 52, 46, },
+	{ 1, 1, 0, 6, 52, 46, },
+	{ 0, 1, 0, 6, 56, 46, },
+	{ 2, 1, 0, 6, 56, 46, },
+	{ 1, 1, 0, 6, 56, 46, },
+	{ 0, 1, 0, 6, 60, 46, },
+	{ 2, 1, 0, 6, 60, 46, },
+	{ 1, 1, 0, 6, 60, 46, },
+	{ 0, 1, 0, 6, 64, 46, },
+	{ 2, 1, 0, 6, 64, 46, },
+	{ 1, 1, 0, 6, 64, 46, },
+	{ 0, 1, 0, 6, 100, 46, },
+	{ 2, 1, 0, 6, 100, 46, },
+	{ 1, 1, 0, 6, 100, 46, },
+	{ 0, 1, 0, 6, 104, 46, },
+	{ 2, 1, 0, 6, 104, 46, },
+	{ 1, 1, 0, 6, 104, 46, },
+	{ 0, 1, 0, 6, 108, 46, },
+	{ 2, 1, 0, 6, 108, 46, },
+	{ 1, 1, 0, 6, 108, 46, },
+	{ 0, 1, 0, 6, 112, 46, },
+	{ 2, 1, 0, 6, 112, 46, },
+	{ 1, 1, 0, 6, 112, 46, },
+	{ 0, 1, 0, 6, 116, 46, },
+	{ 2, 1, 0, 6, 116, 46, },
+	{ 1, 1, 0, 6, 116, 46, },
+	{ 0, 1, 0, 6, 120, 46, },
+	{ 2, 1, 0, 6, 120, 46, },
+	{ 1, 1, 0, 6, 120, 46, },
+	{ 0, 1, 0, 6, 124, 46, },
+	{ 2, 1, 0, 6, 124, 46, },
+	{ 1, 1, 0, 6, 124, 46, },
+	{ 0, 1, 0, 6, 128, 46, },
+	{ 2, 1, 0, 6, 128, 46, },
+	{ 1, 1, 0, 6, 128, 46, },
+	{ 0, 1, 0, 6, 132, 46, },
+	{ 2, 1, 0, 6, 132, 46, },
+	{ 1, 1, 0, 6, 132, 46, },
+	{ 0, 1, 0, 6, 136, 46, },
+	{ 2, 1, 0, 6, 136, 46, },
+	{ 1, 1, 0, 6, 136, 46, },
+	{ 0, 1, 0, 6, 140, 46, },
+	{ 2, 1, 0, 6, 140, 46, },
+	{ 1, 1, 0, 6, 140, 46, },
+	{ 0, 1, 0, 6, 149, 46, },
+	{ 2, 1, 0, 6, 149, 46, },
+	{ 1, 1, 0, 6, 149, 63, },
+	{ 0, 1, 0, 6, 153, 46, },
+	{ 2, 1, 0, 6, 153, 46, },
+	{ 1, 1, 0, 6, 153, 63, },
+	{ 0, 1, 0, 6, 157, 46, },
+	{ 2, 1, 0, 6, 157, 46, },
+	{ 1, 1, 0, 6, 157, 63, },
+	{ 0, 1, 0, 6, 161, 46, },
+	{ 2, 1, 0, 6, 161, 46, },
+	{ 1, 1, 0, 6, 161, 63, },
+	{ 0, 1, 0, 6, 165, 46, },
+	{ 2, 1, 0, 6, 165, 46, },
+	{ 1, 1, 0, 6, 165, 63, },
+	{ 0, 1, 0, 7, 36, 46, },
+	{ 2, 1, 0, 7, 36, 46, },
+	{ 1, 1, 0, 7, 36, 46, },
+	{ 0, 1, 0, 7, 40, 46, },
+	{ 2, 1, 0, 7, 40, 46, },
+	{ 1, 1, 0, 7, 40, 46, },
+	{ 0, 1, 0, 7, 44, 46, },
+	{ 2, 1, 0, 7, 44, 46, },
+	{ 1, 1, 0, 7, 44, 46, },
+	{ 0, 1, 0, 7, 48, 46, },
+	{ 2, 1, 0, 7, 48, 46, },
+	{ 1, 1, 0, 7, 48, 46, },
+	{ 0, 1, 0, 7, 52, 46, },
+	{ 2, 1, 0, 7, 52, 46, },
+	{ 1, 1, 0, 7, 52, 46, },
+	{ 0, 1, 0, 7, 56, 46, },
+	{ 2, 1, 0, 7, 56, 46, },
+	{ 1, 1, 0, 7, 56, 46, },
+	{ 0, 1, 0, 7, 60, 46, },
+	{ 2, 1, 0, 7, 60, 46, },
+	{ 1, 1, 0, 7, 60, 46, },
+	{ 0, 1, 0, 7, 64, 46, },
+	{ 2, 1, 0, 7, 64, 46, },
+	{ 1, 1, 0, 7, 64, 46, },
+	{ 0, 1, 0, 7, 100, 46, },
+	{ 2, 1, 0, 7, 100, 46, },
+	{ 1, 1, 0, 7, 100, 46, },
+	{ 0, 1, 0, 7, 104, 46, },
+	{ 2, 1, 0, 7, 104, 46, },
+	{ 1, 1, 0, 7, 104, 46, },
+	{ 0, 1, 0, 7, 108, 46, },
+	{ 2, 1, 0, 7, 108, 46, },
+	{ 1, 1, 0, 7, 108, 46, },
+	{ 0, 1, 0, 7, 112, 46, },
+	{ 2, 1, 0, 7, 112, 46, },
+	{ 1, 1, 0, 7, 112, 46, },
+	{ 0, 1, 0, 7, 116, 46, },
+	{ 2, 1, 0, 7, 116, 46, },
+	{ 1, 1, 0, 7, 116, 46, },
+	{ 0, 1, 0, 7, 120, 46, },
+	{ 2, 1, 0, 7, 120, 46, },
+	{ 1, 1, 0, 7, 120, 46, },
+	{ 0, 1, 0, 7, 124, 46, },
+	{ 2, 1, 0, 7, 124, 46, },
+	{ 1, 1, 0, 7, 124, 46, },
+	{ 0, 1, 0, 7, 128, 46, },
+	{ 2, 1, 0, 7, 128, 46, },
+	{ 1, 1, 0, 7, 128, 46, },
+	{ 0, 1, 0, 7, 132, 46, },
+	{ 2, 1, 0, 7, 132, 46, },
+	{ 1, 1, 0, 7, 132, 46, },
+	{ 0, 1, 0, 7, 136, 46, },
+	{ 2, 1, 0, 7, 136, 46, },
+	{ 1, 1, 0, 7, 136, 46, },
+	{ 0, 1, 0, 7, 140, 46, },
+	{ 2, 1, 0, 7, 140, 46, },
+	{ 1, 1, 0, 7, 140, 46, },
+	{ 0, 1, 0, 7, 149, 46, },
+	{ 2, 1, 0, 7, 149, 46, },
+	{ 1, 1, 0, 7, 149, 63, },
+	{ 0, 1, 0, 7, 153, 46, },
+	{ 2, 1, 0, 7, 153, 46, },
+	{ 1, 1, 0, 7, 153, 63, },
+	{ 0, 1, 0, 7, 157, 46, },
+	{ 2, 1, 0, 7, 157, 46, },
+	{ 1, 1, 0, 7, 157, 63, },
+	{ 0, 1, 0, 7, 161, 46, },
+	{ 2, 1, 0, 7, 161, 46, },
+	{ 1, 1, 0, 7, 161, 63, },
+	{ 0, 1, 0, 7, 165, 46, },
+	{ 2, 1, 0, 7, 165, 46, },
+	{ 1, 1, 0, 7, 165, 63, },
+	{ 0, 1, 1, 2, 38, 46, },
+	{ 2, 1, 1, 2, 38, 46, },
+	{ 1, 1, 1, 2, 38, 46, },
+	{ 0, 1, 1, 2, 46, 46, },
+	{ 2, 1, 1, 2, 46, 46, },
+	{ 1, 1, 1, 2, 46, 46, },
+	{ 0, 1, 1, 2, 54, 46, },
+	{ 2, 1, 1, 2, 54, 46, },
+	{ 1, 1, 1, 2, 54, 46, },
+	{ 0, 1, 1, 2, 62, 46, },
+	{ 2, 1, 1, 2, 62, 46, },
+	{ 1, 1, 1, 2, 62, 46, },
+	{ 0, 1, 1, 2, 102, 46, },
+	{ 2, 1, 1, 2, 102, 46, },
+	{ 1, 1, 1, 2, 102, 46, },
+	{ 0, 1, 1, 2, 110, 46, },
+	{ 2, 1, 1, 2, 110, 46, },
+	{ 1, 1, 1, 2, 110, 46, },
+	{ 0, 1, 1, 2, 118, 46, },
+	{ 2, 1, 1, 2, 118, 46, },
+	{ 1, 1, 1, 2, 118, 46, },
+	{ 0, 1, 1, 2, 126, 46, },
+	{ 2, 1, 1, 2, 126, 46, },
+	{ 1, 1, 1, 2, 126, 46, },
+	{ 0, 1, 1, 2, 134, 46, },
+	{ 2, 1, 1, 2, 134, 46, },
+	{ 1, 1, 1, 2, 134, 46, },
+	{ 0, 1, 1, 2, 151, 46, },
+	{ 2, 1, 1, 2, 151, 46, },
+	{ 1, 1, 1, 2, 151, 63, },
+	{ 0, 1, 1, 2, 159, 46, },
+	{ 2, 1, 1, 2, 159, 46, },
+	{ 1, 1, 1, 2, 159, 63, },
+	{ 0, 1, 1, 3, 38, 46, },
+	{ 2, 1, 1, 3, 38, 46, },
+	{ 1, 1, 1, 3, 38, 46, },
+	{ 0, 1, 1, 3, 46, 46, },
+	{ 2, 1, 1, 3, 46, 46, },
+	{ 1, 1, 1, 3, 46, 46, },
+	{ 0, 1, 1, 3, 54, 46, },
+	{ 2, 1, 1, 3, 54, 46, },
+	{ 1, 1, 1, 3, 54, 46, },
+	{ 0, 1, 1, 3, 62, 46, },
+	{ 2, 1, 1, 3, 62, 46, },
+	{ 1, 1, 1, 3, 62, 46, },
+	{ 0, 1, 1, 3, 102, 46, },
+	{ 2, 1, 1, 3, 102, 46, },
+	{ 1, 1, 1, 3, 102, 46, },
+	{ 0, 1, 1, 3, 110, 46, },
+	{ 2, 1, 1, 3, 110, 46, },
+	{ 1, 1, 1, 3, 110, 46, },
+	{ 0, 1, 1, 3, 118, 46, },
+	{ 2, 1, 1, 3, 118, 46, },
+	{ 1, 1, 1, 3, 118, 46, },
+	{ 0, 1, 1, 3, 126, 46, },
+	{ 2, 1, 1, 3, 126, 46, },
+	{ 1, 1, 1, 3, 126, 46, },
+	{ 0, 1, 1, 3, 134, 46, },
+	{ 2, 1, 1, 3, 134, 46, },
+	{ 1, 1, 1, 3, 134, 46, },
+	{ 0, 1, 1, 3, 151, 46, },
+	{ 2, 1, 1, 3, 151, 46, },
+	{ 1, 1, 1, 3, 151, 63, },
+	{ 0, 1, 1, 3, 159, 46, },
+	{ 2, 1, 1, 3, 159, 46, },
+	{ 1, 1, 1, 3, 159, 63, },
+	{ 0, 1, 1, 6, 38, 46, },
+	{ 2, 1, 1, 6, 38, 46, },
+	{ 1, 1, 1, 6, 38, 46, },
+	{ 0, 1, 1, 6, 46, 46, },
+	{ 2, 1, 1, 6, 46, 46, },
+	{ 1, 1, 1, 6, 46, 46, },
+	{ 0, 1, 1, 6, 54, 46, },
+	{ 2, 1, 1, 6, 54, 46, },
+	{ 1, 1, 1, 6, 54, 46, },
+	{ 0, 1, 1, 6, 62, 46, },
+	{ 2, 1, 1, 6, 62, 46, },
+	{ 1, 1, 1, 6, 62, 46, },
+	{ 0, 1, 1, 6, 102, 46, },
+	{ 2, 1, 1, 6, 102, 46, },
+	{ 1, 1, 1, 6, 102, 46, },
+	{ 0, 1, 1, 6, 110, 46, },
+	{ 2, 1, 1, 6, 110, 46, },
+	{ 1, 1, 1, 6, 110, 46, },
+	{ 0, 1, 1, 6, 118, 46, },
+	{ 2, 1, 1, 6, 118, 46, },
+	{ 1, 1, 1, 6, 118, 46, },
+	{ 0, 1, 1, 6, 126, 46, },
+	{ 2, 1, 1, 6, 126, 46, },
+	{ 1, 1, 1, 6, 126, 46, },
+	{ 0, 1, 1, 6, 134, 46, },
+	{ 2, 1, 1, 6, 134, 46, },
+	{ 1, 1, 1, 6, 134, 46, },
+	{ 0, 1, 1, 6, 151, 46, },
+	{ 2, 1, 1, 6, 151, 46, },
+	{ 1, 1, 1, 6, 151, 63, },
+	{ 0, 1, 1, 6, 159, 46, },
+	{ 2, 1, 1, 6, 159, 46, },
+	{ 1, 1, 1, 6, 159, 63, },
+	{ 0, 1, 1, 7, 38, 46, },
+	{ 2, 1, 1, 7, 38, 46, },
+	{ 1, 1, 1, 7, 38, 46, },
+	{ 0, 1, 1, 7, 46, 46, },
+	{ 2, 1, 1, 7, 46, 46, },
+	{ 1, 1, 1, 7, 46, 46, },
+	{ 0, 1, 1, 7, 54, 46, },
+	{ 2, 1, 1, 7, 54, 46, },
+	{ 1, 1, 1, 7, 54, 46, },
+	{ 0, 1, 1, 7, 62, 46, },
+	{ 2, 1, 1, 7, 62, 46, },
+	{ 1, 1, 1, 7, 62, 46, },
+	{ 0, 1, 1, 7, 102, 46, },
+	{ 2, 1, 1, 7, 102, 46, },
+	{ 1, 1, 1, 7, 102, 46, },
+	{ 0, 1, 1, 7, 110, 46, },
+	{ 2, 1, 1, 7, 110, 46, },
+	{ 1, 1, 1, 7, 110, 46, },
+	{ 0, 1, 1, 7, 118, 46, },
+	{ 2, 1, 1, 7, 118, 46, },
+	{ 1, 1, 1, 7, 118, 46, },
+	{ 0, 1, 1, 7, 126, 46, },
+	{ 2, 1, 1, 7, 126, 46, },
+	{ 1, 1, 1, 7, 126, 46, },
+	{ 0, 1, 1, 7, 134, 46, },
+	{ 2, 1, 1, 7, 134, 46, },
+	{ 1, 1, 1, 7, 134, 46, },
+	{ 0, 1, 1, 7, 151, 46, },
+	{ 2, 1, 1, 7, 151, 46, },
+	{ 1, 1, 1, 7, 151, 63, },
+	{ 0, 1, 1, 7, 159, 46, },
+	{ 2, 1, 1, 7, 159, 46, },
+	{ 1, 1, 1, 7, 159, 63, },
+	{ 0, 1, 2, 4, 42, 46, },
+	{ 2, 1, 2, 4, 42, 46, },
+	{ 1, 1, 2, 4, 42, 46, },
+	{ 0, 1, 2, 4, 58, 46, },
+	{ 2, 1, 2, 4, 58, 46, },
+	{ 1, 1, 2, 4, 58, 46, },
+	{ 0, 1, 2, 4, 106, 46, },
+	{ 2, 1, 2, 4, 106, 46, },
+	{ 1, 1, 2, 4, 106, 46, },
+	{ 0, 1, 2, 4, 122, 46, },
+	{ 2, 1, 2, 4, 122, 46, },
+	{ 1, 1, 2, 4, 122, 46, },
+	{ 0, 1, 2, 4, 155, 46, },
+	{ 2, 1, 2, 4, 155, 46, },
+	{ 1, 1, 2, 4, 155, 63, },
+	{ 0, 1, 2, 5, 42, 46, },
+	{ 2, 1, 2, 5, 42, 46, },
+	{ 1, 1, 2, 5, 42, 46, },
+	{ 0, 1, 2, 5, 58, 46, },
+	{ 2, 1, 2, 5, 58, 46, },
+	{ 1, 1, 2, 5, 58, 46, },
+	{ 0, 1, 2, 5, 106, 46, },
+	{ 2, 1, 2, 5, 106, 46, },
+	{ 1, 1, 2, 5, 106, 46, },
+	{ 0, 1, 2, 5, 122, 46, },
+	{ 2, 1, 2, 5, 122, 46, },
+	{ 1, 1, 2, 5, 122, 46, },
+	{ 0, 1, 2, 5, 155, 46, },
+	{ 2, 1, 2, 5, 155, 46, },
+	{ 1, 1, 2, 5, 155, 63, },
+	{ 0, 1, 2, 8, 42, 46, },
+	{ 2, 1, 2, 8, 42, 46, },
+	{ 1, 1, 2, 8, 42, 46, },
+	{ 0, 1, 2, 8, 58, 46, },
+	{ 2, 1, 2, 8, 58, 46, },
+	{ 1, 1, 2, 8, 58, 46, },
+	{ 0, 1, 2, 8, 106, 46, },
+	{ 2, 1, 2, 8, 106, 46, },
+	{ 1, 1, 2, 8, 106, 46, },
+	{ 0, 1, 2, 8, 122, 46, },
+	{ 2, 1, 2, 8, 122, 46, },
+	{ 1, 1, 2, 8, 122, 46, },
+	{ 0, 1, 2, 8, 155, 46, },
+	{ 2, 1, 2, 8, 155, 46, },
+	{ 1, 1, 2, 8, 155, 63, },
+	{ 0, 1, 2, 9, 42, 46, },
+	{ 2, 1, 2, 9, 42, 46, },
+	{ 1, 1, 2, 9, 42, 46, },
+	{ 0, 1, 2, 9, 58, 46, },
+	{ 2, 1, 2, 9, 58, 46, },
+	{ 1, 1, 2, 9, 58, 46, },
+	{ 0, 1, 2, 9, 106, 46, },
+	{ 2, 1, 2, 9, 106, 46, },
+	{ 1, 1, 2, 9, 106, 46, },
+	{ 0, 1, 2, 9, 122, 46, },
+	{ 2, 1, 2, 9, 122, 46, },
+	{ 1, 1, 2, 9, 122, 46, },
+	{ 0, 1, 2, 9, 155, 46, },
+	{ 2, 1, 2, 9, 155, 46, },
+	{ 1, 1, 2, 9, 155, 63, },
+};
+
+RTW_DECL_TABLE_TXPWR_LMT(rtw8814a_txpwr_lmt_type8);
+
+static const u8
+rtw8814a_pwrtrk_5gd_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9, 9, 10, 11,
+	 11, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18, 19, 19},
+	{0, 1, 1, 2, 2, 3, 4, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10, 10,
+	 11, 12, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18},
+	{0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 12, 12, 13, 14, 14, 15, 16, 16, 17, 17, 18, 19},
+};
+
+static const u8
+rtw8814a_pwrtrk_5gd_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 18, 19, 20, 21, 22, 23, 24, 25, 25, 25, 25, 25},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 17, 18, 19, 20, 21, 22, 23, 24, 25, 25, 25, 25},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 17, 18, 19, 20, 21, 22, 23, 24, 25, 25, 25, 25},
+};
+
+static const u8
+rtw8814a_pwrtrk_5gc_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 9, 10, 10,
+	 11, 12, 13, 14, 15, 15, 15, 15, 16, 16, 17, 18},
+	{0, 1, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 8, 9, 10, 10, 11,
+	 12, 12, 13, 14, 15, 15, 16, 17, 17, 18, 19, 19, 20},
+	{0, 1, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11,
+	 12, 13, 13, 14, 14, 15, 16, 17, 18, 18, 19, 20, 20},
+};
+
+static const u8
+rtw8814a_pwrtrk_5gc_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	 16, 16, 17, 18, 19, 20, 21, 21, 22, 23, 24, 25, 25},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 17, 18, 19, 20, 21, 22, 23, 24, 24, 25, 25, 25},
+	{0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	 16, 16, 17, 18, 19, 20, 21, 22, 23, 23, 24, 25, 25},
+};
+
+static const u8
+rtw8814a_pwrtrk_5gb_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17, 17},
+	{0, 1, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 8, 9, 10, 10, 11,
+	 12, 12, 13, 14, 15, 15, 16, 17, 17, 18, 19, 19, 20},
+	{0, 1, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11,
+	 12, 13, 13, 14, 14, 15, 16, 17, 18, 18, 19, 20, 20},
+};
+
+static const u8
+rtw8814a_pwrtrk_5gb_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, 13, 14,
+	 15, 15, 16, 17, 18, 18, 19, 20, 21, 22, 23, 23, 24},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 18, 18, 19, 20, 21, 22, 23, 24, 25, 25, 25, 25},
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	 16, 16, 17, 18, 19, 20, 20, 21, 22, 23, 24, 25, 25},
+};
+
+static const u8
+rtw8814a_pwrtrk_5ga_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
+	 8, 9, 9, 10, 11, 11, 11, 11, 12, 12, 13, 13, 14},
+	{0, 1, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 9, 9, 10, 11, 11,
+	 12, 13, 14, 14, 15, 16, 16, 17, 18, 19, 19, 20, 21},
+	{0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 12, 12, 13, 14, 14, 15, 16, 16, 17, 17, 18, 19},
+};
+
+static const u8
+rtw8814a_pwrtrk_5ga_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 2, 3, 4, 5, 6, 7, 7, 8, 9, 10, 11, 12, 12, 13,
+	 14, 15, 16, 16, 17, 18, 19, 20, 21, 21, 22, 23, 24},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 18, 19, 20, 21, 22, 23, 23, 24, 25, 25, 25, 25},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 17, 18, 19, 20, 21, 22, 23, 24, 25, 25, 25, 25},
+};
+
+static const u8 rtw8814a_pwrtrk_2gd_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7,
+	 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_2gd_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_2gc_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 6, 7,
+	 7, 7, 8, 8, 9, 9, 10, 10, 10, 11, 11, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_2gc_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 13
+};
+
+static const u8 rtw8814a_pwrtrk_2gb_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 6, 7,
+	 7, 8, 8, 8, 9, 9, 10, 10, 11, 11, 11, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_2gb_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_2ga_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7,
+	 7, 8, 8, 9, 9, 10, 10, 11, 11, 11, 12, 12, 13
+};
+
+static const u8 rtw8814a_pwrtrk_2ga_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
+	 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_d_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 6, 7,
+	 7, 8, 8, 8, 9, 9, 10, 10, 10, 11, 11, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_d_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_c_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6,
+	 7, 7, 8, 8, 8, 9, 9, 10, 10, 10, 11, 11, 12
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_c_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7,
+	 7, 8, 8, 9, 9, 10, 10, 11, 11, 11, 12, 12, 13
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_b_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6,
+	 6, 6, 7, 7, 8, 8, 8, 9, 9, 10, 10, 10, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_b_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7,
+	 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_a_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 6, 7,
+	 7, 8, 8, 8, 9, 9, 10, 10, 11, 11, 11, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_2g_cck_a_p[RTW_PWR_TRK_TBL_SZ] = {
+	 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7,
+	 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
+};
+
+const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_tbl = {
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5gd_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5gd_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5gd_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5gd_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5gd_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5gd_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5gc_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5gc_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5gc_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5gc_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5gc_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5gc_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5gb_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5gb_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5gb_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5gb_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5gb_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5gb_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5ga_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5ga_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5ga_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_5ga_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_5ga_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_5ga_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_2gd_n = rtw8814a_pwrtrk_2gd_n,
+	.pwrtrk_2gd_p = rtw8814a_pwrtrk_2gd_p,
+	.pwrtrk_2gc_n = rtw8814a_pwrtrk_2gc_n,
+	.pwrtrk_2gc_p = rtw8814a_pwrtrk_2gc_p,
+	.pwrtrk_2gb_n = rtw8814a_pwrtrk_2gb_n,
+	.pwrtrk_2gb_p = rtw8814a_pwrtrk_2gb_p,
+	.pwrtrk_2ga_n = rtw8814a_pwrtrk_2ga_n,
+	.pwrtrk_2ga_p = rtw8814a_pwrtrk_2ga_p,
+	.pwrtrk_2g_cckd_n = rtw8814a_pwrtrk_2g_cck_d_n,
+	.pwrtrk_2g_cckd_p = rtw8814a_pwrtrk_2g_cck_d_p,
+	.pwrtrk_2g_cckc_n = rtw8814a_pwrtrk_2g_cck_c_n,
+	.pwrtrk_2g_cckc_p = rtw8814a_pwrtrk_2g_cck_c_p,
+	.pwrtrk_2g_cckb_n = rtw8814a_pwrtrk_2g_cck_b_n,
+	.pwrtrk_2g_cckb_p = rtw8814a_pwrtrk_2g_cck_b_p,
+	.pwrtrk_2g_ccka_n = rtw8814a_pwrtrk_2g_cck_a_n,
+	.pwrtrk_2g_ccka_p = rtw8814a_pwrtrk_2g_cck_a_p,
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5gd_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5gd_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5gc_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5gc_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5gb_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5gb_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5ga_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type0_5ga_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2gd_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2gd_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2gc_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2gc_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2gb_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2gb_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2ga_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2ga_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_d_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_d_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_c_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_c_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_b_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_b_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_a_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type0_2g_cck_a_p[RTW_PWR_TRK_TBL_SZ] = {
+	 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type0_tbl = {
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5gd_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5gd_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5gd_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5gd_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5gd_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5gd_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5gc_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5gc_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5gc_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5gc_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5gc_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5gc_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5gb_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5gb_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5gb_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5gb_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5gb_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5gb_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5ga_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5ga_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5ga_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type0_5ga_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type0_5ga_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type0_5ga_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_2gd_n = rtw8814a_pwrtrk_type0_2gd_n,
+	.pwrtrk_2gd_p = rtw8814a_pwrtrk_type0_2gd_p,
+	.pwrtrk_2gc_n = rtw8814a_pwrtrk_type0_2gc_n,
+	.pwrtrk_2gc_p = rtw8814a_pwrtrk_type0_2gc_p,
+	.pwrtrk_2gb_n = rtw8814a_pwrtrk_type0_2gb_n,
+	.pwrtrk_2gb_p = rtw8814a_pwrtrk_type0_2gb_p,
+	.pwrtrk_2ga_n = rtw8814a_pwrtrk_type0_2ga_n,
+	.pwrtrk_2ga_p = rtw8814a_pwrtrk_type0_2ga_p,
+	.pwrtrk_2g_cckd_n = rtw8814a_pwrtrk_type0_2g_cck_d_n,
+	.pwrtrk_2g_cckd_p = rtw8814a_pwrtrk_type0_2g_cck_d_p,
+	.pwrtrk_2g_cckc_n = rtw8814a_pwrtrk_type0_2g_cck_c_n,
+	.pwrtrk_2g_cckc_p = rtw8814a_pwrtrk_type0_2g_cck_c_p,
+	.pwrtrk_2g_cckb_n = rtw8814a_pwrtrk_type0_2g_cck_b_n,
+	.pwrtrk_2g_cckb_p = rtw8814a_pwrtrk_type0_2g_cck_b_p,
+	.pwrtrk_2g_ccka_n = rtw8814a_pwrtrk_type0_2g_cck_a_n,
+	.pwrtrk_2g_ccka_p = rtw8814a_pwrtrk_type0_2g_cck_a_p,
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5gd_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 11, 12, 12,
+	 12, 13, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16},
+	{0, 1, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10, 10, 10, 10,
+	 11, 11, 12, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 10, 11, 11, 11, 12,
+	 12, 13, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5gd_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, 13, 14,
+	 15, 16, 17, 18, 19, 20, 21, 22, 22, 22, 22, 22, 22},
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 11, 12, 13, 14,
+	 15, 16, 17, 18, 19, 20, 21, 22, 22, 22, 22, 22, 22},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+	 17, 18, 19, 20, 21, 22, 23, 23, 23, 23, 23, 23, 23},
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5gc_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 3, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 11, 12,
+	 12, 13, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12,
+	 13, 13, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 9, 10, 11, 11,
+	 12, 12, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5gc_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 5, 6, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14,
+	 15, 16, 17, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21},
+	{0, 1, 2, 3, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, 12, 13, 14,
+	 15, 16, 17, 18, 19, 20, 20, 20, 21, 21, 21, 21, 21},
+	{0, 1, 2, 3, 3, 4, 5, 6, 7, 7, 8, 9, 10, 11, 12, 13, 14,
+	 15, 16, 17, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21},
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5gb_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 11, 12, 13,
+	 13, 13, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12,
+	 13, 13, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 9, 10, 11, 11,
+	 12, 12, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5gb_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14, 15,
+	 16, 17, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21},
+	{0, 0, 1, 2, 3, 4, 5, 6, 6, 7, 8, 9, 10, 10, 11, 12, 13,
+	 14, 15, 16, 17, 18, 19, 20, 20, 20, 20, 20, 20, 20},
+	{0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14,
+	 15, 16, 17, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21},
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5ga_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 5, 5, 6, 7, 7, 8, 9, 10, 10, 11, 11, 11,
+	 12, 13, 13, 13, 13, 14, 15, 15, 15, 15, 15, 15, 15},
+	{0, 1, 2, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 10, 11, 12, 12,
+	 12, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 11, 11, 11, 12,
+	 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+};
+
+static const u8
+rtw8814a_pwrtrk_type2_5ga_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10, 10, 11, 12, 13, 14,
+	 15, 15, 16, 17, 18, 18, 19, 19, 20, 20, 20, 20, 20},
+	{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 12, 13, 14,
+	 15, 16, 16, 17, 18, 19, 20, 20, 20, 20, 20, 20, 20},
+	{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14, 15,
+	 15, 16, 17, 18, 19, 19, 20, 20, 20, 20, 20, 20, 20},
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2gd_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5,
+	 6, 6, 7, 8, 9, 10, 11, 11, 11, 11, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2gd_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2gc_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2gc_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 7, 8, 8, 9, 10, 10,
+	 11, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2gb_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8,
+	 9, 9, 10, 10, 11, 11, 12, 12, 12, 13, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2gb_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 13, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2ga_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9, 9, 9,
+	 10, 11, 11, 12, 12, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2ga_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 6, 6, 7, 7, 8, 8, 9, 10,
+	 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_d_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 8,
+	 9, 9, 9, 9, 9, 9, 10, 10, 11, 11, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_d_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 10, 11, 12, 12, 13, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_c_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 2, 2, 3, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9, 9, 10,
+	 10, 11, 11, 11, 12, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_c_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_b_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 9, 10,
+	 10, 10, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_b_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 1, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_a_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 7, 8, 9, 9, 9, 9,
+	 10, 10, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type2_2g_cck_a_p[RTW_PWR_TRK_TBL_SZ] = {
+	 0, 0, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 10,
+	 11, 11, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13
+};
+
+const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type2_tbl = {
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5gd_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5gd_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5gd_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5gd_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5gd_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5gd_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5gc_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5gc_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5gc_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5gc_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5gc_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5gc_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5gb_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5gb_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5gb_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5gb_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5gb_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5gb_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5ga_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5ga_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5ga_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type2_5ga_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type2_5ga_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type2_5ga_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_2gd_n = rtw8814a_pwrtrk_type2_2gd_n,
+	.pwrtrk_2gd_p = rtw8814a_pwrtrk_type2_2gd_p,
+	.pwrtrk_2gc_n = rtw8814a_pwrtrk_type2_2gc_n,
+	.pwrtrk_2gc_p = rtw8814a_pwrtrk_type2_2gc_p,
+	.pwrtrk_2gb_n = rtw8814a_pwrtrk_type2_2gb_n,
+	.pwrtrk_2gb_p = rtw8814a_pwrtrk_type2_2gb_p,
+	.pwrtrk_2ga_n = rtw8814a_pwrtrk_type2_2ga_n,
+	.pwrtrk_2ga_p = rtw8814a_pwrtrk_type2_2ga_p,
+	.pwrtrk_2g_cckd_n = rtw8814a_pwrtrk_type2_2g_cck_d_n,
+	.pwrtrk_2g_cckd_p = rtw8814a_pwrtrk_type2_2g_cck_d_p,
+	.pwrtrk_2g_cckc_n = rtw8814a_pwrtrk_type2_2g_cck_c_n,
+	.pwrtrk_2g_cckc_p = rtw8814a_pwrtrk_type2_2g_cck_c_p,
+	.pwrtrk_2g_cckb_n = rtw8814a_pwrtrk_type2_2g_cck_b_n,
+	.pwrtrk_2g_cckb_p = rtw8814a_pwrtrk_type2_2g_cck_b_p,
+	.pwrtrk_2g_ccka_n = rtw8814a_pwrtrk_type2_2g_cck_a_n,
+	.pwrtrk_2g_ccka_p = rtw8814a_pwrtrk_type2_2g_cck_a_p,
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5gd_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 3, 3, 3, 4, 6, 6, 7, 7, 8, 9, 10, 11, 12, 13, 13, 14,
+	 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 4, 5, 6, 7, 7, 8, 7, 8, 10, 11, 12, 12, 13, 13, 14, 14,
+	 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 5, 5, 6, 7, 8, 9, 9, 10, 11, 12, 13, 13, 14, 15, 15, 16,
+	 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16},
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5gd_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 9, 10, 10, 11, 12,
+	 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+	{0, 0, 0, 1, 1, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 11, 12,
+	 12, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 0, 1, 1, 2, 2, 3, 5, 7, 8, 9, 10, 11, 12, 13, 13,
+	 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5gc_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 1, 2, 3, 3, 4, 6, 7, 7, 8, 9, 9, 9, 10, 10, 10,
+	 10, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12},
+	{0, 1, 2, 3, 3, 7, 7, 8, 8, 9, 11, 12, 12, 13, 14, 14, 15,
+	 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 0, 1, 2, 3, 4, 5, 7, 8, 8, 10, 11, 12, 12, 13, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5gc_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 0, 1, 1, 2, 2, 3, 4, 5, 6, 6, 7, 8, 9, 11, 11,
+	 11, 11, 11, 11, 12, 12, 12, 12, 12, 13, 13, 13, 13},
+	{0, 0, 1, 2, 3, 3, 5, 5, 6, 8, 8, 9, 10, 11, 13, 13, 13,
+	 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 1, 2, 3, 4, 4, 5, 7, 8, 9, 9, 10, 11, 12, 12, 12,
+	 12, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5gb_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 1, 2, 2, 2, 3, 4, 5, 6, 7, 9, 10, 10, 10, 10, 10,
+	 10, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12},
+	{0, 1, 2, 3, 3, 7, 7, 8, 8, 9, 11, 12, 12, 13, 14, 14, 15,
+	 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 0, 1, 2, 3, 4, 5, 7, 8, 8, 10, 11, 12, 12, 13, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5gb_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 1, 1, 2, 2, 4, 5, 6, 6, 7, 8, 9, 10, 11, 11, 11,
+	 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 0, 2, 3, 4, 5, 6, 8, 8, 9, 9, 11, 12, 13, 13, 13,
+	 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 0, 1, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 12, 12,
+	 12, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5ga_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 0, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 10, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12},
+	{0, 2, 3, 4, 5, 7, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 17,
+	 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18},
+	{0, 1, 2, 3, 3, 4, 6, 7, 8, 8, 10, 11, 11, 12, 13, 13, 13,
+	 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type5_5ga_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 1, 3, 3, 3, 5, 5, 6, 6, 8, 8, 9, 10, 11, 11, 11,
+	 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 8, 9, 11, 12, 13, 14, 15,
+	 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 0, 1, 3, 3, 4, 5, 5, 6, 7, 7, 8, 10, 10, 11, 11, 11,
+	 11, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13},
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2gd_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 4, 5, 6, 6, 7, 7, 7, 8, 9, 9, 9,
+	 9, 9, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2gd_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 2, 3, 4, 4, 5, 6, 6, 7, 7, 8, 8,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2gc_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 10, 10,
+	 10, 10, 10, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2gc_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8, 8, 9, 9, 9,
+	 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2gb_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 4, 5, 6, 6, 7, 7, 8, 9, 10, 10, 10,
+	 10, 10, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2gb_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8, 9, 9, 9, 9,
+	 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2ga_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 9, 10, 10, 10,
+	 10, 11, 11, 11, 11, 111, 12, 12, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2ga_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 8, 9, 9, 9,
+	 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_d_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 8, 9, 9, 9, 9,
+	 10, 10, 10, 10, 10, 11, 11, 10, 11, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_d_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 8,
+	 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_c_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 8, 8, 9, 10, 10, 10,
+	 10, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_c_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 8,
+	 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_b_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 2, 3, 4, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 10, 10, 10,
+	 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_b_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 8,
+	 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_a_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 9, 9, 9, 9, 10,
+	 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
+};
+
+static const u8 rtw8814a_pwrtrk_type5_2g_cck_a_p[RTW_PWR_TRK_TBL_SZ] = {
+	 0, 1, 1, 2, 2, 2, 3, 4, 4, 5, 6, 7, 8, 8, 9, 9, 9,
+	 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10
+};
+
+const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type5_tbl = {
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5gd_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5gd_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5gd_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5gd_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5gd_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5gd_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5gc_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5gc_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5gc_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5gc_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5gc_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5gc_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5gb_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5gb_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5gb_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5gb_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5gb_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5gb_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5ga_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5ga_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5ga_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type5_5ga_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type5_5ga_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type5_5ga_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_2gd_n = rtw8814a_pwrtrk_type5_2gd_n,
+	.pwrtrk_2gd_p = rtw8814a_pwrtrk_type5_2gd_p,
+	.pwrtrk_2gc_n = rtw8814a_pwrtrk_type5_2gc_n,
+	.pwrtrk_2gc_p = rtw8814a_pwrtrk_type5_2gc_p,
+	.pwrtrk_2gb_n = rtw8814a_pwrtrk_type5_2gb_n,
+	.pwrtrk_2gb_p = rtw8814a_pwrtrk_type5_2gb_p,
+	.pwrtrk_2ga_n = rtw8814a_pwrtrk_type5_2ga_n,
+	.pwrtrk_2ga_p = rtw8814a_pwrtrk_type5_2ga_p,
+	.pwrtrk_2g_cckd_n = rtw8814a_pwrtrk_type5_2g_cck_d_n,
+	.pwrtrk_2g_cckd_p = rtw8814a_pwrtrk_type5_2g_cck_d_p,
+	.pwrtrk_2g_cckc_n = rtw8814a_pwrtrk_type5_2g_cck_c_n,
+	.pwrtrk_2g_cckc_p = rtw8814a_pwrtrk_type5_2g_cck_c_p,
+	.pwrtrk_2g_cckb_n = rtw8814a_pwrtrk_type5_2g_cck_b_n,
+	.pwrtrk_2g_cckb_p = rtw8814a_pwrtrk_type5_2g_cck_b_p,
+	.pwrtrk_2g_ccka_n = rtw8814a_pwrtrk_type5_2g_cck_a_n,
+	.pwrtrk_2g_ccka_p = rtw8814a_pwrtrk_type5_2g_cck_a_p,
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5gd_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5gd_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5gc_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5gc_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5gb_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5gb_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5ga_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+	{0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	 8, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
+};
+
+static const u8
+rtw8814a_pwrtrk_type7_5ga_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 0, 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2gd_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2gd_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2gc_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2gc_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2gb_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2gb_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2ga_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2ga_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_d_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_d_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_c_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_c_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_b_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_b_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_a_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7,
+	 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+static const u8 rtw8814a_pwrtrk_type7_2g_cck_a_p[RTW_PWR_TRK_TBL_SZ] = {
+	 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+	 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
+};
+
+const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type7_tbl = {
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5gd_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5gd_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5gd_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5gd_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5gd_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5gd_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5gc_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5gc_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5gc_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5gc_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5gc_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5gc_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5gb_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5gb_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5gb_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5gb_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5gb_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5gb_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5ga_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5ga_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5ga_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type7_5ga_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type7_5ga_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type7_5ga_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_2gd_n = rtw8814a_pwrtrk_type7_2gd_n,
+	.pwrtrk_2gd_p = rtw8814a_pwrtrk_type7_2gd_p,
+	.pwrtrk_2gc_n = rtw8814a_pwrtrk_type7_2gc_n,
+	.pwrtrk_2gc_p = rtw8814a_pwrtrk_type7_2gc_p,
+	.pwrtrk_2gb_n = rtw8814a_pwrtrk_type7_2gb_n,
+	.pwrtrk_2gb_p = rtw8814a_pwrtrk_type7_2gb_p,
+	.pwrtrk_2ga_n = rtw8814a_pwrtrk_type7_2ga_n,
+	.pwrtrk_2ga_p = rtw8814a_pwrtrk_type7_2ga_p,
+	.pwrtrk_2g_cckd_n = rtw8814a_pwrtrk_type7_2g_cck_d_n,
+	.pwrtrk_2g_cckd_p = rtw8814a_pwrtrk_type7_2g_cck_d_p,
+	.pwrtrk_2g_cckc_n = rtw8814a_pwrtrk_type7_2g_cck_c_n,
+	.pwrtrk_2g_cckc_p = rtw8814a_pwrtrk_type7_2g_cck_c_p,
+	.pwrtrk_2g_cckb_n = rtw8814a_pwrtrk_type7_2g_cck_b_n,
+	.pwrtrk_2g_cckb_p = rtw8814a_pwrtrk_type7_2g_cck_b_p,
+	.pwrtrk_2g_ccka_n = rtw8814a_pwrtrk_type7_2g_cck_a_n,
+	.pwrtrk_2g_ccka_p = rtw8814a_pwrtrk_type7_2g_cck_a_p,
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5gd_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 4, 6, 7, 7, 8, 8, 9, 10, 10, 11, 11, 12,
+	 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+	{0, 1, 2, 3, 4, 4, 6, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12,
+	 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+	{0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 7, 8, 8, 8, 9, 10, 10,
+	 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11},
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5gd_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 14,
+	 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 0, 1, 2, 3, 5, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 1, 2, 3, 4, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5gc_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 4, 4, 5, 6, 7, 7, 8, 8, 9, 9, 10, 11, 11, 12,
+	 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+	{0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 9, 9, 10, 10, 10, 11, 11,
+	 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 8, 9, 10, 10, 11, 11,
+	 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5gc_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 4, 4, 5, 6, 7, 7, 8, 9, 10, 11, 12, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 1, 3, 4, 4, 5, 6, 7, 7, 8, 10, 11, 12, 13, 14,
+	 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15},
+	{0, 0, 1, 2, 4, 5, 5, 6, 6, 7, 7, 9, 10, 11, 12, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5gb_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 8, 9, 9, 10, 11, 11,
+	 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 9, 9, 10, 10, 10, 11, 11,
+	 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 8, 9, 10, 10, 11, 11,
+	 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5gb_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 10, 11, 11, 13, 13,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 10, 10, 11, 13, 13,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5ga_n[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 9, 10, 11, 11,
+	 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12},
+	{0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 10, 10, 11, 11,
+	 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+	{0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 10, 11, 12,
+	 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13},
+};
+
+static const u8
+rtw8814a_pwrtrk_type8_5ga_p[RTW_PWR_TRK_5G_NUM][RTW_PWR_TRK_TBL_SZ] = {
+	{0, 0, 1, 2, 3, 3, 4, 5, 6, 7, 7, 9, 10, 11, 12, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 13,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+	{0, 0, 1, 2, 3, 3, 4, 4, 6, 7, 7, 9, 10, 11, 12, 13, 14,
+	 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2gd_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5,
+	 6, 6, 7, 8, 9, 10, 11, 11, 11, 11, 11, 11, 11
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2gd_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2gc_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2gc_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 7, 8, 8, 9, 10, 10,
+	 11, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2gb_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8,
+	 9, 9, 10, 10, 11, 11, 12, 12, 12, 13, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2gb_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 13, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2ga_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9, 9, 9,
+	 10, 11, 11, 12, 12, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2ga_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 6, 6, 7, 7, 8, 8, 9, 10,
+	 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_d_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 8,
+	 9, 9, 9, 9, 9, 9, 10, 10, 11, 11, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_d_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 10,
+	 10, 11, 12, 12, 13, 14, 14, 14, 14, 14, 14, 14
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_c_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 2, 2, 3, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9, 9, 10,
+	 10, 11, 11, 11, 12, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_c_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_b_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 9, 10,
+	 10, 10, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_b_p[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 1, 1, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10,
+	 10, 11, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_a_n[RTW_PWR_TRK_TBL_SZ] = {
+	0, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 7, 8, 9, 9, 9, 9,
+	 10, 10, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12
+};
+
+static const u8 rtw8814a_pwrtrk_type8_2g_cck_a_p[RTW_PWR_TRK_TBL_SZ] = {
+	 0, 0, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 10,
+	 11, 11, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13
+};
+
+const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type8_tbl = {
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5gd_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5gd_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5gd_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5gd_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5gd_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gd_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5gd_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5gc_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5gc_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5gc_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5gc_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5gc_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gc_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5gc_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5gb_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5gb_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5gb_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5gb_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5gb_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5gb_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5gb_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5ga_n[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5ga_n[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_n[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5ga_n[RTW_PWR_TRK_5G_3],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_1] = rtw8814a_pwrtrk_type8_5ga_p[RTW_PWR_TRK_5G_1],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_2] = rtw8814a_pwrtrk_type8_5ga_p[RTW_PWR_TRK_5G_2],
+	.pwrtrk_5ga_p[RTW_PWR_TRK_5G_3] = rtw8814a_pwrtrk_type8_5ga_p[RTW_PWR_TRK_5G_3],
+	.pwrtrk_2gd_n = rtw8814a_pwrtrk_type8_2gd_n,
+	.pwrtrk_2gd_p = rtw8814a_pwrtrk_type8_2gd_p,
+	.pwrtrk_2gc_n = rtw8814a_pwrtrk_type8_2gc_n,
+	.pwrtrk_2gc_p = rtw8814a_pwrtrk_type8_2gc_p,
+	.pwrtrk_2gb_n = rtw8814a_pwrtrk_type8_2gb_n,
+	.pwrtrk_2gb_p = rtw8814a_pwrtrk_type8_2gb_p,
+	.pwrtrk_2ga_n = rtw8814a_pwrtrk_type8_2ga_n,
+	.pwrtrk_2ga_p = rtw8814a_pwrtrk_type8_2ga_p,
+	.pwrtrk_2g_cckd_n = rtw8814a_pwrtrk_type8_2g_cck_d_n,
+	.pwrtrk_2g_cckd_p = rtw8814a_pwrtrk_type8_2g_cck_d_p,
+	.pwrtrk_2g_cckc_n = rtw8814a_pwrtrk_type8_2g_cck_c_n,
+	.pwrtrk_2g_cckc_p = rtw8814a_pwrtrk_type8_2g_cck_c_p,
+	.pwrtrk_2g_cckb_n = rtw8814a_pwrtrk_type8_2g_cck_b_n,
+	.pwrtrk_2g_cckb_p = rtw8814a_pwrtrk_type8_2g_cck_b_p,
+	.pwrtrk_2g_ccka_n = rtw8814a_pwrtrk_type8_2g_cck_a_n,
+	.pwrtrk_2g_ccka_p = rtw8814a_pwrtrk_type8_2g_cck_a_p,
+};
+
+static const struct rtw_pwr_seq_cmd init_power_on_8814a[] = {
+	{0x10c2,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_USB_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), BIT(1)},
+	{0xFFFF,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 0,
+	 RTW_PWR_CMD_END, 0, 0},
+};
+
+static const struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8814a[] = {
+	{0x0012,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(6), BIT(6)},
+	{0x0015,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(5), 0},
+	{0x0015,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(6), 0},
+	{0x0023,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(4), 0},
+	{0x0046,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x00},
+	{0x0062,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x00},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_PCI_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(2), 0},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(3), 0},
+	{0x0301,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_PCI_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0},
+	{0x0071,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_PCI_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(2), 0},
+	{0xFFFF,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 0,
+	 RTW_PWR_CMD_END, 0, 0},
+};
+
+static const struct rtw_pwr_seq_cmd trans_cardemu_to_act_8814a[] = {
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(2), 0},
+	{0x0006,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_POLLING, BIT(1), BIT(1)},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(3), 0},
+	{0x00F0,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(7), 0},
+	{0x0081,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0x30, 0x20},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(0), BIT(0)},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_POLLING, BIT(0), 0},
+	{0xFFFF,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 0,
+	 RTW_PWR_CMD_END, 0, 0},
+};
+
+static const struct rtw_pwr_seq_cmd trans_act_to_cardemu_8814a[] = {
+	{0x0c00,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x04},
+	{0x0e00,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x04},
+	{0x1002,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(0), 0},
+	{0x0002,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_DELAY, 1, RTW_PWR_DELAY_US},
+	{0x1002,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_PCI_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), 0},
+	{0x001F,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0},
+	{0x0007,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x28},
+	{0x0008,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_USB_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0x02, 0},
+	{0x0066,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_USB_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(7), 0},
+	{0x0041,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_USB_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(4), 0},
+	{0x0042,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_USB_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), 0},
+	{0x004e,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_USB_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(5), BIT(5)},
+	{0x0041,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_USB_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(0), 0},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), BIT(1)},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_POLLING, BIT(1), 0},
+	{0xFFFF,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 0,
+	 RTW_PWR_CMD_END, 0, 0},
+};
+
+static const struct rtw_pwr_seq_cmd trans_cardemu_to_carddis_8814a[] = {
+	{0x0003,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(2), 0},
+	{0x0080,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x01},
+	{0x0081,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x30},
+	{0x0045,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x00},
+	{0x0046,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0xff},
+	{0x0047,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0},
+	{0x0015,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(6), BIT(6)},
+	{0x0015,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(5), BIT(5)},
+	{0x0012,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(6), 0},
+	{0x0023,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(4), BIT(4)},
+	{0x0008,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), 0},
+	{0x0007,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xFF, 0x20},
+	{0x001f,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), 0},
+	{0x0020,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), 0},
+	{0x0021,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), 0},
+	{0x0076,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(1), 0},
+	{0x0091,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, 0xA0, 0xA0},
+	{0x0070,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(3), BIT(3)},
+	{0x0005,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 RTW_PWR_ADDR_MAC,
+	 RTW_PWR_CMD_WRITE, BIT(3), BIT(3)},
+	{0xFFFF,
+	 RTW_PWR_CUT_ALL_MSK,
+	 RTW_PWR_INTF_ALL_MSK,
+	 0,
+	 RTW_PWR_CMD_END, 0, 0},
+};
+
+const struct rtw_pwr_seq_cmd * const card_enable_flow_8814a[] = {
+	init_power_on_8814a,
+	trans_carddis_to_cardemu_8814a,
+	trans_cardemu_to_act_8814a,
+	NULL
+};
+
+const struct rtw_pwr_seq_cmd * const card_disable_flow_8814a[] = {
+	trans_act_to_cardemu_8814a,
+	trans_cardemu_to_carddis_8814a,
+	NULL
+};
diff --git a/sys/contrib/dev/rtw88/rtw8814a_table.h b/sys/contrib/dev/rtw88/rtw8814a_table.h
new file mode 100644
index 000000000000..69fb87e36c48
--- /dev/null
+++ b/sys/contrib/dev/rtw88/rtw8814a_table.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#ifndef __RTW8814A_TABLE_H__
+#define __RTW8814A_TABLE_H__
+
+extern const struct rtw_table rtw8814a_mac_tbl;
+extern const struct rtw_table rtw8814a_agc_tbl;
+extern const struct rtw_table rtw8814a_bb_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_type0_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_type2_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_type3_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_type4_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_type5_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_type7_tbl;
+extern const struct rtw_table rtw8814a_bb_pg_type8_tbl;
+extern const struct rtw_table rtw8814a_rf_a_tbl;
+extern const struct rtw_table rtw8814a_rf_b_tbl;
+extern const struct rtw_table rtw8814a_rf_c_tbl;
+extern const struct rtw_table rtw8814a_rf_d_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_type0_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_type1_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_type2_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_type3_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_type5_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_type7_tbl;
+extern const struct rtw_table rtw8814a_txpwr_lmt_type8_tbl;
+extern const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_tbl;
+extern const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type0_tbl;
+extern const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type2_tbl;
+extern const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type5_tbl;
+extern const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type7_tbl;
+extern const struct rtw_pwr_track_tbl rtw8814a_rtw_pwrtrk_type8_tbl;
+extern const struct rtw_pwr_seq_cmd * const card_disable_flow_8814a[];
+extern const struct rtw_pwr_seq_cmd * const card_enable_flow_8814a[];
+
+#endif
diff --git a/sys/contrib/dev/rtw88/rtw8814ae.c b/sys/contrib/dev/rtw88/rtw8814ae.c
new file mode 100644
index 000000000000..9d789a81ea21
--- /dev/null
+++ b/sys/contrib/dev/rtw88/rtw8814ae.c
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include "pci.h"
+#include "rtw8814a.h"
+
+static const struct pci_device_id rtw_8814ae_id_table[] = {
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8813),
+		.driver_data = (kernel_ulong_t)&rtw8814a_hw_spec
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(pci, rtw_8814ae_id_table);
+
+static struct pci_driver rtw_8814ae_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = rtw_8814ae_id_table,
+	.probe = rtw_pci_probe,
+	.remove = rtw_pci_remove,
+	.driver.pm = &rtw_pm_ops,
+	.shutdown = rtw_pci_shutdown,
+#if defined(__FreeBSD__)
+	.bsddriver.name = KBUILD_MODNAME,
+#endif
+};
+module_pci_driver(rtw_8814ae_driver);
+
+MODULE_AUTHOR("Bitterblue Smith <rtl8821cerfe2@gmail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8814ae driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sys/contrib/dev/rtw88/rtw8814au.c b/sys/contrib/dev/rtw88/rtw8814au.c
new file mode 100644
index 000000000000..1a0886ec17dd
--- /dev/null
+++ b/sys/contrib/dev/rtw88/rtw8814au.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/usb.h>
+#include "main.h"
+#include "rtw8814a.h"
+#include "usb.h"
+
+static const struct usb_device_id rtw_8814au_id_table[] = {
+	{ USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, 0x8813, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x056e, 0x400b, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x056e, 0x400d, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9054, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0b05, 0x1817, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0b05, 0x1852, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0b05, 0x1853, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0e66, 0x0026, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x331a, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x20f4, 0x809a, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x20f4, 0x809b, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2357, 0x0106, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x7392, 0xa834, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x7392, 0xa833, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8814a_hw_spec) },
+	{},
+};
+MODULE_DEVICE_TABLE(usb, rtw_8814au_id_table);
+
+static struct usb_driver rtw_8814au_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = rtw_8814au_id_table,
+	.probe = rtw_usb_probe,
+	.disconnect = rtw_usb_disconnect,
+};
+module_usb_driver(rtw_8814au_driver);
+
+MODULE_AUTHOR("Bitterblue Smith <rtl8821cerfe2@gmail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8814au driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sys/contrib/dev/rtw88/rtw8821a.c b/sys/contrib/dev/rtw88/rtw8821a.c
index f68239b07319..414b77eef07c 100644
--- a/sys/contrib/dev/rtw88/rtw8821a.c
+++ b/sys/contrib/dev/rtw88/rtw8821a.c
@@ -865,12 +865,14 @@ static const struct rtw_chip_ops rtw8821a_ops = {
 	.query_phy_status	= rtw8821a_query_phy_status,
 	.set_channel		= rtw88xxa_set_channel,
 	.mac_init		= NULL,
+	.mac_postinit		= NULL,
 	.read_rf		= rtw88xxa_phy_read_rf,
 	.write_rf		= rtw_phy_write_rf_reg_sipi,
 	.set_antenna		= NULL,
 	.set_tx_power_index	= rtw88xxa_set_tx_power_index,
 	.cfg_ldo25		= rtw8821a_cfg_ldo25,
 	.efuse_grant		= rtw88xxa_efuse_grant,
+	.set_ampdu_factor	= NULL,
 	.false_alarm_statistics	= rtw88xxa_false_alarm_statistics,
 	.phy_calibration	= rtw8821a_phy_calibration,
 	.cck_pd_set		= rtw88xxa_phy_cck_pd_set,
@@ -1137,7 +1139,7 @@ const struct rtw_chip_info rtw8821a_hw_spec = {
 	.ops = &rtw8821a_ops,
 	.id = RTW_CHIP_TYPE_8821A,
 	.fw_name = "rtw88/rtw8821a_fw.bin",
-	.wlan_cpu = RTW_WCPU_11N,
+	.wlan_cpu = RTW_WCPU_8051,
 	.tx_pkt_desc_sz = 40,
 	.tx_buf_desc_sz = 16,
 	.rx_pkt_desc_sz = 24,
@@ -1175,6 +1177,7 @@ const struct rtw_chip_info rtw8821a_hw_spec = {
 	.rfe_defs = rtw8821a_rfe_defs,
 	.rfe_defs_size = ARRAY_SIZE(rtw8821a_rfe_defs),
 	.rx_ldpc = false,
+	.amsdu_in_ampdu = true,
 	.hw_feature_report = false,
 	.c2h_ra_report_size = 4,
 	.old_datarate_fb_limit = true,
diff --git a/sys/contrib/dev/rtw88/rtw8821au.c b/sys/contrib/dev/rtw88/rtw8821au.c
index a01744b64e8d..28c281b32978 100644
--- a/sys/contrib/dev/rtw88/rtw8821au.c
+++ b/sys/contrib/dev/rtw88/rtw8821au.c
@@ -66,7 +66,7 @@ static const struct usb_device_id rtw_8821au_id_table[] = {
 MODULE_DEVICE_TABLE(usb, rtw_8821au_id_table);
 
 static struct usb_driver rtw_8821au_driver = {
-	.name = "rtw_8821au",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8821au_id_table,
 	.probe = rtw_usb_probe,
 	.disconnect = rtw_usb_disconnect,
diff --git a/sys/contrib/dev/rtw88/rtw8821c.c b/sys/contrib/dev/rtw88/rtw8821c.c
index 0fe02f0b5d91..a6401e6172ba 100644
--- a/sys/contrib/dev/rtw88/rtw8821c.c
+++ b/sys/contrib/dev/rtw88/rtw8821c.c
@@ -680,11 +680,11 @@ static void query_phy_status(struct rtw_dev *rtwdev, u8 *phy_status,
 }
 
 static void
-rtw8821c_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs)
+rtw8821c_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path,
+				    u8 rs, u32 *phy_pwr_idx)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
 	static const u32 offset_txagc[2] = {0x1d00, 0x1d80};
-	static u32 phy_pwr_idx;
 	u8 rate, rate_idx, pwr_index, shift;
 	int j;
 
@@ -692,12 +692,12 @@ rtw8821c_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs)
 		rate = rtw_rate_section[rs][j];
 		pwr_index = hal->tx_pwr_tbl[path][rate];
 		shift = rate & 0x3;
-		phy_pwr_idx |= ((u32)pwr_index << (shift * 8));
+		*phy_pwr_idx |= ((u32)pwr_index << (shift * 8));
 		if (shift == 0x3 || rate == DESC_RATEVHT1SS_MCS9) {
 			rate_idx = rate & 0xfc;
 			rtw_write32(rtwdev, offset_txagc[path] + rate_idx,
-				    phy_pwr_idx);
-			phy_pwr_idx = 0;
+				    *phy_pwr_idx);
+			*phy_pwr_idx = 0;
 		}
 	}
 }
@@ -705,14 +705,16 @@ rtw8821c_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs)
 static void rtw8821c_set_tx_power_index(struct rtw_dev *rtwdev)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
+	u32 phy_pwr_idx = 0;
 	int rs, path;
 
 	for (path = 0; path < hal->rf_path_num; path++) {
-		for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++) {
+		for (rs = 0; rs <= __RTW_RATE_SECTION_2SS_MAX; rs++) {
 			if (rs == RTW_RATE_SECTION_HT_2S ||
 			    rs == RTW_RATE_SECTION_VHT_2S)
 				continue;
-			rtw8821c_set_tx_power_index_by_rate(rtwdev, path, rs);
+			rtw8821c_set_tx_power_index_by_rate(rtwdev, path, rs,
+							    &phy_pwr_idx);
 		}
 	}
 }
@@ -1661,11 +1663,13 @@ static const struct rtw_chip_ops rtw8821c_ops = {
 	.query_phy_status	= query_phy_status,
 	.set_channel		= rtw8821c_set_channel,
 	.mac_init		= rtw8821c_mac_init,
+	.mac_postinit		= NULL,
 	.read_rf		= rtw_phy_read_rf,
 	.write_rf		= rtw_phy_write_rf_reg_sipi,
 	.set_antenna		= NULL,
 	.set_tx_power_index	= rtw8821c_set_tx_power_index,
 	.cfg_ldo25		= rtw8821c_cfg_ldo25,
+	.set_ampdu_factor	= NULL,
 	.false_alarm_statistics	= rtw8821c_false_alarm_statistics,
 	.phy_calibration	= rtw8821c_phy_calibration,
 	.cck_pd_set		= rtw8821c_phy_cck_pd_set,
@@ -1974,7 +1978,7 @@ const struct rtw_chip_info rtw8821c_hw_spec = {
 	.ops = &rtw8821c_ops,
 	.id = RTW_CHIP_TYPE_8821C,
 	.fw_name = "rtw88/rtw8821c_fw.bin",
-	.wlan_cpu = RTW_WCPU_11AC,
+	.wlan_cpu = RTW_WCPU_3081,
 	.tx_pkt_desc_sz = 48,
 	.tx_buf_desc_sz = 16,
 	.rx_pkt_desc_sz = 24,
@@ -1992,6 +1996,7 @@ const struct rtw_chip_info rtw8821c_hw_spec = {
 	.band = RTW_BAND_2G | RTW_BAND_5G,
 	.page_size = TX_PAGE_SIZE,
 	.dig_min = 0x1c,
+	.amsdu_in_ampdu = true,
 	.usb_tx_agg_desc_num = 3,
 	.hw_feature_report = true,
 	.c2h_ra_report_size = 7,
diff --git a/sys/contrib/dev/rtw88/rtw8821ce.c b/sys/contrib/dev/rtw88/rtw8821ce.c
index 1cdb01dce255..9569a88d236a 100644
--- a/sys/contrib/dev/rtw88/rtw8821ce.c
+++ b/sys/contrib/dev/rtw88/rtw8821ce.c
@@ -21,12 +21,13 @@ static const struct pci_device_id rtw_8821ce_id_table[] = {
 MODULE_DEVICE_TABLE(pci, rtw_8821ce_id_table);
 
 static struct pci_driver rtw_8821ce_driver = {
-	.name = "rtw_8821ce",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8821ce_id_table,
 	.probe = rtw_pci_probe,
 	.remove = rtw_pci_remove,
 	.driver.pm = &rtw_pm_ops,
 	.shutdown = rtw_pci_shutdown,
+	.err_handler  = &rtw_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name = KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw88/rtw8821cs.c b/sys/contrib/dev/rtw88/rtw8821cs.c
index a359413369a4..6d94162213c6 100644
--- a/sys/contrib/dev/rtw88/rtw8821cs.c
+++ b/sys/contrib/dev/rtw88/rtw8821cs.c
@@ -20,7 +20,7 @@ static const struct sdio_device_id rtw_8821cs_id_table[] =  {
 MODULE_DEVICE_TABLE(sdio, rtw_8821cs_id_table);
 
 static struct sdio_driver rtw_8821cs_driver = {
-	.name = "rtw_8821cs",
+	.name = KBUILD_MODNAME,
 	.probe = rtw_sdio_probe,
 	.remove = rtw_sdio_remove,
 	.id_table = rtw_8821cs_id_table,
diff --git a/sys/contrib/dev/rtw88/rtw8821cu.c b/sys/contrib/dev/rtw88/rtw8821cu.c
index 46ea9acc4941..3386c54f2646 100644
--- a/sys/contrib/dev/rtw88/rtw8821cu.c
+++ b/sys/contrib/dev/rtw88/rtw8821cu.c
@@ -48,7 +48,7 @@ static int rtw_8821cu_probe(struct usb_interface *intf,
 }
 
 static struct usb_driver rtw_8821cu_driver = {
-	.name = "rtw_8821cu",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8821cu_id_table,
 	.probe = rtw_8821cu_probe,
 	.disconnect = rtw_usb_disconnect,
diff --git a/sys/contrib/dev/rtw88/rtw8822b.c b/sys/contrib/dev/rtw88/rtw8822b.c
index c097224e1530..b37b301fc866 100644
--- a/sys/contrib/dev/rtw88/rtw8822b.c
+++ b/sys/contrib/dev/rtw88/rtw8822b.c
@@ -935,11 +935,11 @@ static void query_phy_status(struct rtw_dev *rtwdev, u8 *phy_status,
 }
 
 static void
-rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs)
+rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path,
+				    u8 rs, u32 *phy_pwr_idx)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
 	static const u32 offset_txagc[2] = {0x1d00, 0x1d80};
-	static u32 phy_pwr_idx;
 	u8 rate, rate_idx, pwr_index, shift;
 	int j;
 
@@ -947,12 +947,12 @@ rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs)
 		rate = rtw_rate_section[rs][j];
 		pwr_index = hal->tx_pwr_tbl[path][rate];
 		shift = rate & 0x3;
-		phy_pwr_idx |= ((u32)pwr_index << (shift * 8));
+		*phy_pwr_idx |= ((u32)pwr_index << (shift * 8));
 		if (shift == 0x3) {
 			rate_idx = rate & 0xfc;
 			rtw_write32(rtwdev, offset_txagc[path] + rate_idx,
-				    phy_pwr_idx);
-			phy_pwr_idx = 0;
+				    *phy_pwr_idx);
+			*phy_pwr_idx = 0;
 		}
 	}
 }
@@ -960,11 +960,13 @@ rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs)
 static void rtw8822b_set_tx_power_index(struct rtw_dev *rtwdev)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
+	u32 phy_pwr_idx = 0;
 	int rs, path;
 
 	for (path = 0; path < hal->rf_path_num; path++) {
-		for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++)
-			rtw8822b_set_tx_power_index_by_rate(rtwdev, path, rs);
+		for (rs = 0; rs <= __RTW_RATE_SECTION_2SS_MAX; rs++)
+			rtw8822b_set_tx_power_index_by_rate(rtwdev, path, rs,
+							    &phy_pwr_idx);
 	}
 }
 
@@ -981,6 +983,7 @@ static bool rtw8822b_check_rf_path(u8 antenna)
 }
 
 static int rtw8822b_set_antenna(struct rtw_dev *rtwdev,
+				int radio_idx,
 				u32 antenna_tx,
 				u32 antenna_rx)
 {
@@ -2151,11 +2154,13 @@ static const struct rtw_chip_ops rtw8822b_ops = {
 	.query_phy_status	= query_phy_status,
 	.set_channel		= rtw8822b_set_channel,
 	.mac_init		= rtw8822b_mac_init,
+	.mac_postinit		= NULL,
 	.read_rf		= rtw_phy_read_rf,
 	.write_rf		= rtw_phy_write_rf_reg_sipi,
 	.set_tx_power_index	= rtw8822b_set_tx_power_index,
 	.set_antenna		= rtw8822b_set_antenna,
 	.cfg_ldo25		= rtw8822b_cfg_ldo25,
+	.set_ampdu_factor	= NULL,
 	.false_alarm_statistics	= rtw8822b_false_alarm_statistics,
 	.phy_calibration	= rtw8822b_phy_calibration,
 	.pwr_track		= rtw8822b_pwr_track,
@@ -2514,7 +2519,7 @@ const struct rtw_chip_info rtw8822b_hw_spec = {
 	.ops = &rtw8822b_ops,
 	.id = RTW_CHIP_TYPE_8822B,
 	.fw_name = "rtw88/rtw8822b_fw.bin",
-	.wlan_cpu = RTW_WCPU_11AC,
+	.wlan_cpu = RTW_WCPU_3081,
 	.tx_pkt_desc_sz = 48,
 	.tx_buf_desc_sz = 16,
 	.rx_pkt_desc_sz = 24,
@@ -2533,6 +2538,7 @@ const struct rtw_chip_info rtw8822b_hw_spec = {
 	.band = RTW_BAND_2G | RTW_BAND_5G,
 	.page_size = TX_PAGE_SIZE,
 	.dig_min = 0x1c,
+	.amsdu_in_ampdu = true,
 	.usb_tx_agg_desc_num = 3,
 	.hw_feature_report = true,
 	.c2h_ra_report_size = 7,
diff --git a/sys/contrib/dev/rtw88/rtw8822be.c b/sys/contrib/dev/rtw88/rtw8822be.c
index 9d1924e4d5ca..611b0a5d32c7 100644
--- a/sys/contrib/dev/rtw88/rtw8822be.c
+++ b/sys/contrib/dev/rtw88/rtw8822be.c
@@ -17,12 +17,13 @@ static const struct pci_device_id rtw_8822be_id_table[] = {
 MODULE_DEVICE_TABLE(pci, rtw_8822be_id_table);
 
 static struct pci_driver rtw_8822be_driver = {
-	.name = "rtw_8822be",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8822be_id_table,
 	.probe = rtw_pci_probe,
 	.remove = rtw_pci_remove,
 	.driver.pm = &rtw_pm_ops,
 	.shutdown = rtw_pci_shutdown,
+	.err_handler  = &rtw_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name = KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw88/rtw8822bs.c b/sys/contrib/dev/rtw88/rtw8822bs.c
index 31d8645f83bd..744781dcb419 100644
--- a/sys/contrib/dev/rtw88/rtw8822bs.c
+++ b/sys/contrib/dev/rtw88/rtw8822bs.c
@@ -20,7 +20,7 @@ static const struct sdio_device_id rtw_8822bs_id_table[] =  {
 MODULE_DEVICE_TABLE(sdio, rtw_8822bs_id_table);
 
 static struct sdio_driver rtw_8822bs_driver = {
-	.name = "rtw_8822bs",
+	.name = KBUILD_MODNAME,
 	.probe = rtw_sdio_probe,
 	.remove = rtw_sdio_remove,
 	.id_table = rtw_8822bs_id_table,
diff --git a/sys/contrib/dev/rtw88/rtw8822bu.c b/sys/contrib/dev/rtw88/rtw8822bu.c
index 70aa730dfef0..efda9887cc41 100644
--- a/sys/contrib/dev/rtw88/rtw8822bu.c
+++ b/sys/contrib/dev/rtw88/rtw8822bu.c
@@ -73,6 +73,12 @@ static const struct usb_device_id rtw_8822bu_id_table[] = {
 	  .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* ELECOM WDB-867DU3S */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x2c4e, 0x0107, 0xff, 0xff, 0xff),
 	  .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* Mercusys MA30H */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2c4e, 0x010a, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* Mercusys MA30N */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3322, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* D-Link DWA-T185 rev. A1 */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0411, 0x03d1, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, /* BUFFALO WI-U2-866DM */
 	{},
 };
 MODULE_DEVICE_TABLE(usb, rtw_8822bu_id_table);
@@ -84,7 +90,7 @@ static int rtw8822bu_probe(struct usb_interface *intf,
 }
 
 static struct usb_driver rtw_8822bu_driver = {
-	.name = "rtw_8822bu",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8822bu_id_table,
 	.probe = rtw8822bu_probe,
 	.disconnect = rtw_usb_disconnect,
diff --git a/sys/contrib/dev/rtw88/rtw8822c.c b/sys/contrib/dev/rtw88/rtw8822c.c
index 5b4201bf9561..22294461ea6d 100644
--- a/sys/contrib/dev/rtw88/rtw8822c.c
+++ b/sys/contrib/dev/rtw88/rtw8822c.c
@@ -2750,7 +2750,7 @@ static void rtw8822c_set_tx_power_index(struct rtw_dev *rtwdev)
 	s8 diff_idx[4];
 
 	rtw8822c_set_write_tx_power_ref(rtwdev, pwr_ref_cck, pwr_ref_ofdm);
-	for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++) {
+	for (rs = 0; rs <= __RTW_RATE_SECTION_2SS_MAX; rs++) {
 		for (j = 0; j < rtw_rate_size[rs]; j++) {
 			rate = rtw_rate_section[rs][j];
 			pwr_a = hal->tx_pwr_tbl[RF_PATH_A][rate];
@@ -2771,6 +2771,7 @@ static void rtw8822c_set_tx_power_index(struct rtw_dev *rtwdev)
 }
 
 static int rtw8822c_set_antenna(struct rtw_dev *rtwdev,
+				int radio_idx,
 				u32 antenna_tx,
 				u32 antenna_rx)
 {
@@ -3959,7 +3960,8 @@ static void rtw8822c_dpk_cal_coef1(struct rtw_dev *rtwdev)
 	rtw_write32(rtwdev, REG_NCTL0, 0x00001148);
 	rtw_write32(rtwdev, REG_NCTL0, 0x00001149);
 
-	check_hw_ready(rtwdev, 0x2d9c, MASKBYTE0, 0x55);
+	if (!check_hw_ready(rtwdev, 0x2d9c, MASKBYTE0, 0x55))
+		rtw_warn(rtwdev, "DPK stuck, performance may be suboptimal");
 
 	rtw_write8(rtwdev, 0x1b10, 0x0);
 	rtw_write32_mask(rtwdev, REG_NCTL0, BIT_SUBPAGE, 0x0000000c);
@@ -4974,12 +4976,14 @@ static const struct rtw_chip_ops rtw8822c_ops = {
 	.query_phy_status	= query_phy_status,
 	.set_channel		= rtw8822c_set_channel,
 	.mac_init		= rtw8822c_mac_init,
+	.mac_postinit		= NULL,
 	.dump_fw_crash		= rtw8822c_dump_fw_crash,
 	.read_rf		= rtw_phy_read_rf,
 	.write_rf		= rtw_phy_write_rf_reg_mix,
 	.set_tx_power_index	= rtw8822c_set_tx_power_index,
 	.set_antenna		= rtw8822c_set_antenna,
 	.cfg_ldo25		= rtw8822c_cfg_ldo25,
+	.set_ampdu_factor	= NULL,
 	.false_alarm_statistics	= rtw8822c_false_alarm_statistics,
 	.dpk_track		= rtw8822c_dpk_track,
 	.phy_calibration	= rtw8822c_phy_calibration,
@@ -5346,7 +5350,7 @@ const struct rtw_chip_info rtw8822c_hw_spec = {
 	.ops = &rtw8822c_ops,
 	.id = RTW_CHIP_TYPE_8822C,
 	.fw_name = "rtw88/rtw8822c_fw.bin",
-	.wlan_cpu = RTW_WCPU_11AC,
+	.wlan_cpu = RTW_WCPU_3081,
 	.tx_pkt_desc_sz = 48,
 	.tx_buf_desc_sz = 16,
 	.rx_pkt_desc_sz = 24,
@@ -5365,6 +5369,7 @@ const struct rtw_chip_info rtw8822c_hw_spec = {
 	.band = RTW_BAND_2G | RTW_BAND_5G,
 	.page_size = TX_PAGE_SIZE,
 	.dig_min = 0x20,
+	.amsdu_in_ampdu = true,
 	.usb_tx_agg_desc_num = 3,
 	.hw_feature_report = true,
 	.c2h_ra_report_size = 7,
diff --git a/sys/contrib/dev/rtw88/rtw8822ce.c b/sys/contrib/dev/rtw88/rtw8822ce.c
index cd600233df5e..2acf8a7970f0 100644
--- a/sys/contrib/dev/rtw88/rtw8822ce.c
+++ b/sys/contrib/dev/rtw88/rtw8822ce.c
@@ -21,12 +21,13 @@ static const struct pci_device_id rtw_8822ce_id_table[] = {
 MODULE_DEVICE_TABLE(pci, rtw_8822ce_id_table);
 
 static struct pci_driver rtw_8822ce_driver = {
-	.name = "rtw_8822ce",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8822ce_id_table,
 	.probe = rtw_pci_probe,
 	.remove = rtw_pci_remove,
 	.driver.pm = &rtw_pm_ops,
 	.shutdown = rtw_pci_shutdown,
+	.err_handler  = &rtw_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name = KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw88/rtw8822cs.c b/sys/contrib/dev/rtw88/rtw8822cs.c
index 975e81c824f2..322281e07eb8 100644
--- a/sys/contrib/dev/rtw88/rtw8822cs.c
+++ b/sys/contrib/dev/rtw88/rtw8822cs.c
@@ -20,7 +20,7 @@ static const struct sdio_device_id rtw_8822cs_id_table[] =  {
 MODULE_DEVICE_TABLE(sdio, rtw_8822cs_id_table);
 
 static struct sdio_driver rtw_8822cs_driver = {
-	.name = "rtw_8822cs",
+	.name = KBUILD_MODNAME,
 	.probe = rtw_sdio_probe,
 	.remove = rtw_sdio_remove,
 	.id_table = rtw_8822cs_id_table,
diff --git a/sys/contrib/dev/rtw88/rtw8822cu.c b/sys/contrib/dev/rtw88/rtw8822cu.c
index 2d4dc2f12021..90fcbb8ec629 100644
--- a/sys/contrib/dev/rtw88/rtw8822cu.c
+++ b/sys/contrib/dev/rtw88/rtw8822cu.c
@@ -32,7 +32,7 @@ static int rtw8822cu_probe(struct usb_interface *intf,
 }
 
 static struct usb_driver rtw_8822cu_driver = {
-	.name = "rtw_8822cu",
+	.name = KBUILD_MODNAME,
 	.id_table = rtw_8822cu_id_table,
 	.probe = rtw8822cu_probe,
 	.disconnect = rtw_usb_disconnect,
diff --git a/sys/contrib/dev/rtw88/rtw88xxa.c b/sys/contrib/dev/rtw88/rtw88xxa.c
index 71e61b9c0bec..0fa943271fb6 100644
--- a/sys/contrib/dev/rtw88/rtw88xxa.c
+++ b/sys/contrib/dev/rtw88/rtw88xxa.c
@@ -1637,7 +1637,7 @@ void rtw88xxa_set_tx_power_index(struct rtw_dev *rtwdev)
 	int rs, path;
 
 	for (path = 0; path < hal->rf_path_num; path++) {
-		for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++) {
+		for (rs = 0; rs <= __RTW_RATE_SECTION_2SS_MAX; rs++) {
 			if (hal->rf_path_num == 1 &&
 			    (rs == RTW_RATE_SECTION_HT_2S ||
 			     rs == RTW_RATE_SECTION_VHT_2S))
diff --git a/sys/contrib/dev/rtw88/rx.c b/sys/contrib/dev/rtw88/rx.c
index 219fd1292be4..310917c6eda0 100644
--- a/sys/contrib/dev/rtw88/rx.c
+++ b/sys/contrib/dev/rtw88/rx.c
@@ -73,6 +73,12 @@ static void rtw_rx_phy_stat(struct rtw_dev *rtwdev,
 		rate_ss_evm = 2;
 		evm_id = RTW_EVM_2SS_A;
 		break;
+	case DESC_RATEMCS16...DESC_RATEMCS23:
+	case DESC_RATEVHT3SS_MCS0...DESC_RATEVHT3SS_MCS9:
+		rate_ss = 3;
+		rate_ss_evm = 3;
+		evm_id = RTW_EVM_3SS_A;
+		break;
 	default:
 		rtw_warn(rtwdev, "unknown pkt rate = %d\n", pkt_stat->rate);
 		return;
diff --git a/sys/contrib/dev/rtw88/sar.c b/sys/contrib/dev/rtw88/sar.c
index 529c69fe3bf3..68cf6eb99b5e 100644
--- a/sys/contrib/dev/rtw88/sar.c
+++ b/sys/contrib/dev/rtw88/sar.c
@@ -101,7 +101,7 @@ int rtw_set_sar_specs(struct rtw_dev *rtwdev,
 			power, BIT(RTW_COMMON_SAR_FCT));
 
 		for (j = 0; j < RTW_RF_PATH_MAX; j++) {
-			for (k = 0; k < RTW_RATE_SECTION_MAX; k++) {
+			for (k = 0; k < RTW_RATE_SECTION_NUM; k++) {
 				arg = (struct rtw_sar_arg){
 					.sar_band = idx,
 					.path = j,
diff --git a/sys/contrib/dev/rtw88/sdio.c b/sys/contrib/dev/rtw88/sdio.c
index e024061bdbf7..cc2d4fef3587 100644
--- a/sys/contrib/dev/rtw88/sdio.c
+++ b/sys/contrib/dev/rtw88/sdio.c
@@ -10,6 +10,7 @@
 #include <linux/mmc/host.h>
 #include <linux/mmc/sdio_func.h>
 #include "main.h"
+#include "mac.h"
 #include "debug.h"
 #include "fw.h"
 #include "ps.h"
@@ -546,7 +547,7 @@ static int rtw_sdio_check_free_txpg(struct rtw_dev *rtwdev, u8 queue,
 {
 	unsigned int pages_free, pages_needed;
 
-	if (rtw_chip_wcpu_11n(rtwdev)) {
+	if (rtw_chip_wcpu_8051(rtwdev)) {
 		u32 free_txpg;
 
 		free_txpg = rtw_sdio_read32(rtwdev, REG_SDIO_FREE_TXPG);
@@ -677,12 +678,22 @@ static void rtw_sdio_enable_rx_aggregation(struct rtw_dev *rtwdev)
 {
 	u8 size, timeout;
 
-	if (rtw_chip_wcpu_11n(rtwdev)) {
+	switch (rtwdev->chip->id) {
+	case RTW_CHIP_TYPE_8703B:
+	case RTW_CHIP_TYPE_8821A:
+	case RTW_CHIP_TYPE_8812A:
 		size = 0x6;
 		timeout = 0x6;
-	} else {
+		break;
+	case RTW_CHIP_TYPE_8723D:
+		size = 0xa;
+		timeout = 0x3;
+		rtw_write8_set(rtwdev, REG_RXDMA_AGG_PG_TH + 3, BIT(7));
+		break;
+	default:
 		size = 0xff;
 		timeout = 0x1;
+		break;
 	}
 
 	/* Make the firmware honor the size limit configured below */
@@ -718,10 +729,7 @@ static u8 rtw_sdio_get_tx_qsel(struct rtw_dev *rtwdev, struct sk_buff *skb,
 	case RTW_TX_QUEUE_H2C:
 		return TX_DESC_QSEL_H2C;
 	case RTW_TX_QUEUE_MGMT:
-		if (rtw_chip_wcpu_11n(rtwdev))
-			return TX_DESC_QSEL_HIGH;
-		else
-			return TX_DESC_QSEL_MGMT;
+		return TX_DESC_QSEL_MGMT;
 	case RTW_TX_QUEUE_HI0:
 		return TX_DESC_QSEL_HIGH;
 	default:
@@ -1022,7 +1030,7 @@ static void rtw_sdio_rx_isr(struct rtw_dev *rtwdev)
 	u32 rx_len, hisr, total_rx_bytes = 0;
 
 	do {
-		if (rtw_chip_wcpu_11n(rtwdev))
+		if (rtw_chip_wcpu_8051(rtwdev))
 			rx_len = rtw_read16(rtwdev, REG_SDIO_RX0_REQ_LEN);
 		else
 			rx_len = rtw_read32(rtwdev, REG_SDIO_RX0_REQ_LEN);
@@ -1034,7 +1042,7 @@ static void rtw_sdio_rx_isr(struct rtw_dev *rtwdev)
 
 		total_rx_bytes += rx_len;
 
-		if (rtw_chip_wcpu_11n(rtwdev)) {
+		if (rtw_chip_wcpu_8051(rtwdev)) {
 			/* Stop if no more RX requests are pending, even if
 			 * rx_len could be greater than zero in the next
 			 * iteration. This is needed because the RX buffer may
@@ -1046,7 +1054,7 @@ static void rtw_sdio_rx_isr(struct rtw_dev *rtwdev)
 			 */
 			hisr = rtw_read32(rtwdev, REG_SDIO_HISR);
 		} else {
-			/* RTW_WCPU_11AC chips have improved hardware or
+			/* RTW_WCPU_3081 chips have improved hardware or
 			 * firmware and can use rx_len unconditionally.
 			 */
 			hisr = REG_SDIO_HISR_RX_REQUEST;
@@ -1147,7 +1155,7 @@ static void rtw_sdio_declaim(struct rtw_dev *rtwdev,
 	sdio_release_host(sdio_func);
 }
 
-static struct rtw_hci_ops rtw_sdio_ops = {
+static const struct rtw_hci_ops rtw_sdio_ops = {
 	.tx_write = rtw_sdio_tx_write,
 	.tx_kick_off = rtw_sdio_tx_kick_off,
 	.setup = rtw_sdio_setup,
@@ -1157,6 +1165,7 @@ static struct rtw_hci_ops rtw_sdio_ops = {
 	.link_ps = rtw_sdio_link_ps,
 	.interface_cfg = rtw_sdio_interface_cfg,
 	.dynamic_rx_agg = NULL,
+	.write_firmware_page = rtw_write_firmware_page,
 
 	.read8 = rtw_sdio_read8,
 	.read16 = rtw_sdio_read16,
@@ -1227,10 +1236,7 @@ static void rtw_sdio_process_tx_queue(struct rtw_dev *rtwdev,
 		return;
 	}
 
-	if (queue <= RTW_TX_QUEUE_VO)
-		rtw_sdio_indicate_tx_status(rtwdev, skb);
-	else
-		dev_kfree_skb_any(skb);
+	rtw_sdio_indicate_tx_status(rtwdev, skb);
 }
 
 static void rtw_sdio_tx_handler(struct work_struct *work)
@@ -1298,7 +1304,6 @@ static void rtw_sdio_deinit_tx(struct rtw_dev *rtwdev)
 	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
 	int i;
 
-	flush_workqueue(rtwsdio->txwq);
 	destroy_workqueue(rtwsdio->txwq);
 	kfree(rtwsdio->tx_handler_data);
 
diff --git a/sys/contrib/dev/rtw88/tx.c b/sys/contrib/dev/rtw88/tx.c
index ebde62c1e48b..310df18a3abf 100644
--- a/sys/contrib/dev/rtw88/tx.c
+++ b/sys/contrib/dev/rtw88/tx.c
@@ -178,7 +178,8 @@ static void rtw_tx_report_enable(struct rtw_dev *rtwdev,
 
 void rtw_tx_report_purge_timer(struct timer_list *t)
 {
-	struct rtw_dev *rtwdev = from_timer(rtwdev, t, tx_report.purge_timer);
+	struct rtw_dev *rtwdev = timer_container_of(rtwdev, t,
+						    tx_report.purge_timer);
 	struct rtw_tx_report *tx_report = &rtwdev->tx_report;
 	unsigned long flags;
 
diff --git a/sys/contrib/dev/rtw88/usb.c b/sys/contrib/dev/rtw88/usb.c
index 6ce68c7c4d4a..9fe00af9fc0e 100644
--- a/sys/contrib/dev/rtw88/usb.c
+++ b/sys/contrib/dev/rtw88/usb.c
@@ -139,7 +139,7 @@ static void rtw_usb_write(struct rtw_dev *rtwdev, u32 addr, u32 val, int len)
 
 	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
 			      RTW_USB_CMD_REQ, RTW_USB_CMD_WRITE,
-			      addr, 0, data, len, 30000);
+			      addr, 0, data, len, 500);
 	if (ret < 0 && ret != -ENODEV && count++ < 4)
 		rtw_err(rtwdev, "write register 0x%x failed with %d\n",
 			addr, ret);
@@ -165,6 +165,60 @@ static void rtw_usb_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
 	rtw_usb_write(rtwdev, addr, val, 4);
 }
 
+static void rtw_usb_write_firmware_page(struct rtw_dev *rtwdev, u32 page,
+					const u8 *data, u32 size)
+{
+	struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev);
+	struct usb_device *udev = rtwusb->udev;
+	u32 addr = FW_START_ADDR_LEGACY;
+	u8 *data_dup, *buf;
+	u32 n, block_size;
+	int ret;
+
+	switch (rtwdev->chip->id) {
+	case RTW_CHIP_TYPE_8723D:
+		block_size = 254;
+		break;
+	default:
+		block_size = 196;
+		break;
+	}
+
+	data_dup = kmemdup(data, size, GFP_KERNEL);
+	if (!data_dup)
+		return;
+
+	buf = data_dup;
+
+	rtw_write32_mask(rtwdev, REG_MCUFW_CTRL, BIT_ROM_PGE, page);
+
+	while (size > 0) {
+		if (size >= block_size)
+			n = block_size;
+		else if (size >= 8)
+			n = 8;
+		else
+			n = 1;
+
+		ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				      RTW_USB_CMD_REQ, RTW_USB_CMD_WRITE,
+				      addr, 0, buf, n, 500);
+		if (ret != n) {
+			if (ret != -ENODEV)
+				rtw_err(rtwdev,
+					"write 0x%x len %d failed: %d\n",
+					addr, n, ret);
+			break;
+		}
+
+		addr += n;
+		buf += n;
+		size -= n;
+	}
+
+	kfree(data_dup);
+}
+
 static int dma_mapping_to_ep(enum rtw_dma_mapping dma_mapping)
 {
 	switch (dma_mapping) {
@@ -866,6 +920,7 @@ static void rtw_usb_dynamic_rx_agg(struct rtw_dev *rtwdev, bool enable)
 	case RTW_CHIP_TYPE_8822C:
 	case RTW_CHIP_TYPE_8822B:
 	case RTW_CHIP_TYPE_8821C:
+	case RTW_CHIP_TYPE_8814A:
 		rtw_usb_dynamic_rx_agg_v1(rtwdev, enable);
 		break;
 	case RTW_CHIP_TYPE_8821A:
@@ -881,7 +936,7 @@ static void rtw_usb_dynamic_rx_agg(struct rtw_dev *rtwdev, bool enable)
 	}
 }
 
-static struct rtw_hci_ops rtw_usb_ops = {
+static const struct rtw_hci_ops rtw_usb_ops = {
 	.tx_write = rtw_usb_tx_write,
 	.tx_kick_off = rtw_usb_tx_kick_off,
 	.setup = rtw_usb_setup,
@@ -891,6 +946,7 @@ static struct rtw_hci_ops rtw_usb_ops = {
 	.link_ps = rtw_usb_link_ps,
 	.interface_cfg = rtw_usb_interface_cfg,
 	.dynamic_rx_agg = rtw_usb_dynamic_rx_agg,
+	.write_firmware_page = rtw_usb_write_firmware_page,
 
 	.write8  = rtw_usb_write8,
 	.write16 = rtw_usb_write16,
@@ -948,7 +1004,6 @@ static void rtw_usb_deinit_rx(struct rtw_dev *rtwdev)
 
 	skb_queue_purge(&rtwusb->rx_queue);
 
-	flush_workqueue(rtwusb->rxwq);
 	destroy_workqueue(rtwusb->rxwq);
 
 	skb_queue_purge(&rtwusb->rx_free_queue);
@@ -977,7 +1032,6 @@ static void rtw_usb_deinit_tx(struct rtw_dev *rtwdev)
 {
 	struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev);
 
-	flush_workqueue(rtwusb->txwq);
 	destroy_workqueue(rtwusb->txwq);
 	rtw_usb_tx_queue_purge(rtwusb);
 }
@@ -1094,7 +1148,8 @@ static int rtw_usb_switch_mode_new(struct rtw_dev *rtwdev)
 
 static bool rtw_usb3_chip_old(u8 chip_id)
 {
-	return chip_id == RTW_CHIP_TYPE_8812A;
+	return chip_id == RTW_CHIP_TYPE_8812A ||
+	       chip_id == RTW_CHIP_TYPE_8814A;
 }
 
 static bool rtw_usb3_chip_new(u8 chip_id)
diff --git a/sys/contrib/dev/rtw88/util.c b/sys/contrib/dev/rtw88/util.c
index e222d3c01a77..66819f694405 100644
--- a/sys/contrib/dev/rtw88/util.c
+++ b/sys/contrib/dev/rtw88/util.c
@@ -101,7 +101,8 @@ void rtw_desc_to_mcsrate(u16 rate, u8 *mcs, u8 *nss)
 		*nss = 4;
 		*mcs = rate - DESC_RATEVHT4SS_MCS0;
 	} else if (rate >= DESC_RATEMCS0 &&
-		   rate <= DESC_RATEMCS15) {
+		   rate <= DESC_RATEMCS31) {
+		*nss = 0;
 		*mcs = rate - DESC_RATEMCS0;
 	}
 }
diff --git a/sys/contrib/dev/rtw89/Kconfig b/sys/contrib/dev/rtw89/Kconfig
index b1c86cdd9c0e..4288c30b400a 100644
--- a/sys/contrib/dev/rtw89/Kconfig
+++ b/sys/contrib/dev/rtw89/Kconfig
@@ -17,6 +17,9 @@ config RTW89_CORE
 config RTW89_PCI
 	tristate
 
+config RTW89_USB
+	tristate
+
 config RTW89_8851B
 	tristate
 
@@ -49,6 +52,17 @@ config RTW89_8851BE
 
 	  802.11ax PCIe wireless network (Wi-Fi 6) adapter
 
+config RTW89_8851BU
+	tristate "Realtek 8851BU USB wireless network (Wi-Fi 6) adapter"
+	depends on USB
+	select RTW89_CORE
+	select RTW89_USB
+	select RTW89_8851B
+	help
+	  Select this option will enable support for 8851BU chipset
+
+	  802.11ax USB wireless network (Wi-Fi 6) adapter
+
 config RTW89_8852AE
 	tristate "Realtek 8852AE PCI wireless network (Wi-Fi 6) adapter"
 	depends on PCI
@@ -72,6 +86,18 @@ config RTW89_8852BE
 
 	  802.11ax PCIe wireless network (Wi-Fi 6) adapter
 
+config RTW89_8852BU
+	tristate "Realtek 8852BU USB wireless network (Wi-Fi 6) adapter"
+	depends on USB
+	select RTW89_CORE
+	select RTW89_USB
+	select RTW89_8852B
+	select RTW89_8852B_COMMON
+	help
+	  Select this option will enable support for 8852BU chipset
+
+	  802.11ax USB wireless network (Wi-Fi 6) adapter
+
 config RTW89_8852BTE
 	tristate "Realtek 8852BE-VT PCI wireless network (Wi-Fi 6) adapter"
 	depends on PCI
@@ -123,7 +149,7 @@ config RTW89_DEBUGMSG
 
 config RTW89_DEBUGFS
 	bool "Realtek rtw89 debugfs support"
-	depends on RTW89_CORE
+	depends on RTW89_CORE && CFG80211_DEBUGFS
 	select RTW89_DEBUG
 	help
 	  Enable debugfs support
diff --git a/sys/contrib/dev/rtw89/Makefile b/sys/contrib/dev/rtw89/Makefile
index c751013e811e..23e43c444f69 100644
--- a/sys/contrib/dev/rtw89/Makefile
+++ b/sys/contrib/dev/rtw89/Makefile
@@ -31,6 +31,9 @@ rtw89_8851b-objs := rtw8851b.o \
 obj-$(CONFIG_RTW89_8851BE) += rtw89_8851be.o
 rtw89_8851be-objs := rtw8851be.o
 
+obj-$(CONFIG_RTW89_8851BU) += rtw89_8851bu.o
+rtw89_8851bu-objs := rtw8851bu.o
+
 obj-$(CONFIG_RTW89_8852A) += rtw89_8852a.o
 rtw89_8852a-objs := rtw8852a.o \
 		    rtw8852a_table.o \
@@ -52,6 +55,9 @@ rtw89_8852b-objs := rtw8852b.o \
 obj-$(CONFIG_RTW89_8852BE) += rtw89_8852be.o
 rtw89_8852be-objs := rtw8852be.o
 
+obj-$(CONFIG_RTW89_8852BU) += rtw89_8852bu.o
+rtw89_8852bu-objs := rtw8852bu.o
+
 obj-$(CONFIG_RTW89_8852BT) += rtw89_8852bt.o
 rtw89_8852bt-objs := rtw8852bt.o \
 		    rtw8852bt_rfk.o \
@@ -81,3 +87,6 @@ rtw89_core-$(CONFIG_RTW89_DEBUG) += debug.o
 obj-$(CONFIG_RTW89_PCI) += rtw89_pci.o
 rtw89_pci-y := pci.o pci_be.o
 
+obj-$(CONFIG_RTW89_USB) += rtw89_usb.o
+rtw89_usb-y := usb.o
+
diff --git a/sys/contrib/dev/rtw89/acpi.c b/sys/contrib/dev/rtw89/acpi.c
index f5dedb12c129..fdba1ea46ec6 100644
--- a/sys/contrib/dev/rtw89/acpi.c
+++ b/sys/contrib/dev/rtw89/acpi.c
@@ -12,6 +12,125 @@ static const guid_t rtw89_guid = GUID_INIT(0xD2A8C3E8, 0x4B69, 0x4F00,
 					   0x82, 0xBD, 0xFE, 0x86,
 					   0x07, 0x80, 0x3A, 0xA7);
 
+static u32 rtw89_acpi_traversal_object(struct rtw89_dev *rtwdev,
+				       const union acpi_object *obj, u8 *pos)
+{
+	const union acpi_object *elm;
+	unsigned int i;
+	u32 sub_len;
+	u32 len = 0;
+	u8 *tmp;
+
+	switch (obj->type) {
+	case ACPI_TYPE_INTEGER:
+		if (pos)
+			pos[len] = obj->integer.value;
+
+		len++;
+		break;
+	case ACPI_TYPE_BUFFER:
+		if (unlikely(obj->buffer.length == 0)) {
+			rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+				    "%s: invalid buffer type\n", __func__);
+			goto err;
+		}
+
+		if (pos)
+			memcpy(pos, obj->buffer.pointer, obj->buffer.length);
+
+		len += obj->buffer.length;
+		break;
+	case ACPI_TYPE_PACKAGE:
+		if (unlikely(obj->package.count == 0)) {
+			rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+				    "%s: invalid package type\n", __func__);
+			goto err;
+		}
+
+		for (i = 0; i < obj->package.count; i++) {
+			elm = &obj->package.elements[i];
+			tmp = pos ? pos + len : NULL;
+
+			sub_len = rtw89_acpi_traversal_object(rtwdev, elm, tmp);
+			if (unlikely(sub_len == 0))
+				goto err;
+
+			len += sub_len;
+		}
+		break;
+	default:
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: unhandled type: %d\n",
+			    __func__, obj->type);
+		goto err;
+	}
+
+	return len;
+
+err:
+	return 0;
+}
+
+static u32 rtw89_acpi_calculate_object_length(struct rtw89_dev *rtwdev,
+					      const union acpi_object *obj)
+{
+	return rtw89_acpi_traversal_object(rtwdev, obj, NULL);
+}
+
+static struct rtw89_acpi_data *
+rtw89_acpi_evaluate_method(struct rtw89_dev *rtwdev, const char *method)
+{
+	struct acpi_buffer buf = {ACPI_ALLOCATE_BUFFER, NULL};
+	struct rtw89_acpi_data *data = NULL;
+	acpi_handle root, handle;
+	union acpi_object *obj;
+	acpi_status status;
+	u32 len;
+
+	root = ACPI_HANDLE(rtwdev->dev);
+	if (!root) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "acpi (%s): failed to get root\n", method);
+		return NULL;
+	}
+
+#if defined(__linux__)
+	status = acpi_get_handle(root, (acpi_string)method, &handle);
+#elif defined(__FreeBSD__)
+	status = acpi_get_handle(root, method, &handle);
+#endif
+	if (ACPI_FAILURE(status)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "acpi (%s): failed to get handle\n", method);
+		return NULL;
+	}
+
+	status = acpi_evaluate_object(handle, NULL, NULL, &buf);
+	if (ACPI_FAILURE(status)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "acpi (%s): failed to evaluate object\n", method);
+		return NULL;
+	}
+
+	obj = buf.pointer;
+	len = rtw89_acpi_calculate_object_length(rtwdev, obj);
+	if (unlikely(len == 0)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "acpi (%s): failed to traversal obj len\n", method);
+		goto out;
+	}
+
+	data = kzalloc(struct_size(data, buf, len), GFP_KERNEL);
+	if (!data)
+		goto out;
+
+	data->len = len;
+	rtw89_acpi_traversal_object(rtwdev, obj, data->buf);
+
+out:
+	ACPI_FREE(obj);
+	return data;
+}
+
 static
 int rtw89_acpi_dsm_get_value(struct rtw89_dev *rtwdev, union acpi_object *obj,
 			     u8 *value)
@@ -121,6 +240,138 @@ int rtw89_acpi_dsm_get_policy_6ghz_sp(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static bool chk_acpi_policy_6ghz_vlp_sig(const struct rtw89_acpi_policy_6ghz_vlp *p)
+{
+	return p->signature[0] == 0x52 &&
+	       p->signature[1] == 0x54 &&
+	       p->signature[2] == 0x4B &&
+	       p->signature[3] == 0x0B;
+}
+
+static
+int rtw89_acpi_dsm_get_policy_6ghz_vlp(struct rtw89_dev *rtwdev,
+				       union acpi_object *obj,
+				       struct rtw89_acpi_policy_6ghz_vlp **policy)
+{
+	const struct rtw89_acpi_policy_6ghz_vlp *ptr;
+	u32 buf_len;
+
+	if (obj->type != ACPI_TYPE_BUFFER) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "acpi: expect buffer but type: %d\n", obj->type);
+		return -EINVAL;
+	}
+
+	buf_len = obj->buffer.length;
+	if (buf_len < sizeof(*ptr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: invalid buffer length: %u\n",
+			    __func__, buf_len);
+		return -EINVAL;
+	}
+
+	ptr = (typeof(ptr))obj->buffer.pointer;
+	if (!chk_acpi_policy_6ghz_vlp_sig(ptr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: bad signature\n", __func__);
+		return -EINVAL;
+	}
+
+	*policy = kmemdup(ptr, sizeof(*ptr), GFP_KERNEL);
+	if (!*policy)
+		return -ENOMEM;
+
+	rtw89_hex_dump(rtwdev, RTW89_DBG_ACPI, "policy_6ghz_vlp: ", *policy,
+		       sizeof(*ptr));
+	return 0;
+}
+
+static bool chk_acpi_policy_tas_sig(const struct rtw89_acpi_policy_tas *p)
+{
+	return p->signature[0] == 0x52 &&
+	       p->signature[1] == 0x54 &&
+	       p->signature[2] == 0x4B &&
+	       p->signature[3] == 0x05;
+}
+
+static int rtw89_acpi_dsm_get_policy_tas(struct rtw89_dev *rtwdev,
+					 union acpi_object *obj,
+					 struct rtw89_acpi_policy_tas **policy)
+{
+	const struct rtw89_acpi_policy_tas *ptr;
+	u32 buf_len;
+
+	if (obj->type != ACPI_TYPE_BUFFER) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "acpi: expect buffer but type: %d\n", obj->type);
+		return -EINVAL;
+	}
+
+	buf_len = obj->buffer.length;
+	if (buf_len < sizeof(*ptr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: invalid buffer length: %u\n",
+			    __func__, buf_len);
+		return -EINVAL;
+	}
+
+	ptr = (typeof(ptr))obj->buffer.pointer;
+	if (!chk_acpi_policy_tas_sig(ptr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: bad signature\n", __func__);
+		return -EINVAL;
+	}
+
+	*policy = kmemdup(ptr, sizeof(*ptr), GFP_KERNEL);
+	if (!*policy)
+		return -ENOMEM;
+
+	rtw89_hex_dump(rtwdev, RTW89_DBG_ACPI, "policy_tas: ", *policy,
+		       sizeof(*ptr));
+	return 0;
+}
+
+static
+bool chk_acpi_policy_reg_rules_sig(const struct rtw89_acpi_policy_reg_rules *p)
+{
+	return p->signature[0] == 0x52 &&
+	       p->signature[1] == 0x54 &&
+	       p->signature[2] == 0x4B &&
+	       p->signature[3] == 0x0A;
+}
+
+static
+int rtw89_acpi_dsm_get_policy_reg_rules(struct rtw89_dev *rtwdev,
+					union acpi_object *obj,
+					struct rtw89_acpi_policy_reg_rules **policy)
+{
+	const struct rtw89_acpi_policy_reg_rules *ptr;
+	u32 buf_len;
+
+	if (obj->type != ACPI_TYPE_BUFFER) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "acpi: expect buffer but type: %d\n", obj->type);
+		return -EINVAL;
+	}
+
+	buf_len = obj->buffer.length;
+	if (buf_len < sizeof(*ptr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: invalid buffer length: %u\n",
+			    __func__, buf_len);
+		return -EINVAL;
+	}
+
+	ptr = (typeof(ptr))obj->buffer.pointer;
+	if (!chk_acpi_policy_reg_rules_sig(ptr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: bad signature\n", __func__);
+		return -EINVAL;
+	}
+
+	*policy = kmemdup(ptr, sizeof(*ptr), GFP_KERNEL);
+	if (!*policy)
+		return -ENOMEM;
+
+	rtw89_hex_dump(rtwdev, RTW89_DBG_ACPI, "policy_reg_rules: ", *policy,
+		       sizeof(*ptr));
+	return 0;
+}
+
 int rtw89_acpi_evaluate_dsm(struct rtw89_dev *rtwdev,
 			    enum rtw89_acpi_dsm_func func,
 			    struct rtw89_acpi_dsm_result *res)
@@ -142,6 +393,14 @@ int rtw89_acpi_evaluate_dsm(struct rtw89_dev *rtwdev,
 	else if (func == RTW89_ACPI_DSM_FUNC_6GHZ_SP_SUP)
 		ret = rtw89_acpi_dsm_get_policy_6ghz_sp(rtwdev, obj,
 							&res->u.policy_6ghz_sp);
+	else if (func == RTW89_ACPI_DSM_FUNC_6GHZ_VLP_SUP)
+		ret = rtw89_acpi_dsm_get_policy_6ghz_vlp(rtwdev, obj,
+							 &res->u.policy_6ghz_vlp);
+	else if (func == RTW89_ACPI_DSM_FUNC_TAS_EN)
+		ret = rtw89_acpi_dsm_get_policy_tas(rtwdev, obj, &res->u.policy_tas);
+	else if (func == RTW89_ACPI_DSM_FUNC_REG_RULES_EN)
+		ret = rtw89_acpi_dsm_get_policy_reg_rules(rtwdev, obj,
+							  &res->u.policy_reg_rules);
 	else
 		ret = rtw89_acpi_dsm_get_value(rtwdev, obj, &res->u.value);
 
@@ -152,46 +411,879 @@ int rtw89_acpi_evaluate_dsm(struct rtw89_dev *rtwdev,
 int rtw89_acpi_evaluate_rtag(struct rtw89_dev *rtwdev,
 			     struct rtw89_acpi_rtag_result *res)
 {
-	struct acpi_buffer buf = {ACPI_ALLOCATE_BUFFER, NULL};
-	acpi_handle root, handle;
-	union acpi_object *obj;
-	acpi_status status;
+#if defined(__linux__)
+	const struct rtw89_acpi_data *data;
+#elif defined(__FreeBSD__)
+	struct rtw89_acpi_data *data;
+#endif
 	u32 buf_len;
 	int ret = 0;
 
-	root = ACPI_HANDLE(rtwdev->dev);
-	if (!root)
-		return -EOPNOTSUPP;
-
-	status = acpi_get_handle(root, (acpi_string)"RTAG", &handle);
-	if (ACPI_FAILURE(status))
+	data = rtw89_acpi_evaluate_method(rtwdev, "RTAG");
+	if (!data)
 		return -EIO;
 
-	status = acpi_evaluate_object(handle, NULL, NULL, &buf);
-	if (ACPI_FAILURE(status))
-		return -EIO;
+	buf_len = data->len;
+	if (buf_len != sizeof(*res)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: invalid buffer length: %u\n",
+			    __func__, buf_len);
+		ret = -EINVAL;
+		goto out;
+	}
 
-	obj = buf.pointer;
-	if (obj->type != ACPI_TYPE_BUFFER) {
+	*res = *(struct rtw89_acpi_rtag_result *)data->buf;
+
+	rtw89_hex_dump(rtwdev, RTW89_DBG_ACPI, "antenna_gain: ", res, sizeof(*res));
+
+out:
+	kfree(data);
+	return ret;
+}
+
+enum rtw89_acpi_sar_subband rtw89_acpi_sar_get_subband(struct rtw89_dev *rtwdev,
+						       u32 center_freq)
+{
+	switch (center_freq) {
+	default:
 		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
-			    "acpi: expect buffer but type: %d\n", obj->type);
-		ret = -EINVAL;
+			    "center freq %u to ACPI SAR subband is unhandled\n",
+			    center_freq);
+		fallthrough;
+	case 2412 ... 2484:
+		return RTW89_ACPI_SAR_2GHZ_SUBBAND;
+	case 5180 ... 5240:
+		return RTW89_ACPI_SAR_5GHZ_SUBBAND_1;
+	case 5250 ... 5320:
+		return RTW89_ACPI_SAR_5GHZ_SUBBAND_2;
+	case 5500 ... 5720:
+		return RTW89_ACPI_SAR_5GHZ_SUBBAND_2E;
+	case 5745 ... 5885:
+		return RTW89_ACPI_SAR_5GHZ_SUBBAND_3_4;
+	case 5955 ... 6155:
+		return RTW89_ACPI_SAR_6GHZ_SUBBAND_5_L;
+	case 6175 ... 6415:
+		return RTW89_ACPI_SAR_6GHZ_SUBBAND_5_H;
+	case 6435 ... 6515:
+		return RTW89_ACPI_SAR_6GHZ_SUBBAND_6;
+	case 6535 ... 6695:
+		return RTW89_ACPI_SAR_6GHZ_SUBBAND_7_L;
+	case 6715 ... 6855:
+		return RTW89_ACPI_SAR_6GHZ_SUBBAND_7_H;
+
+	/* freq 6875 (ch 185, 20MHz) spans RTW89_ACPI_SAR_6GHZ_SUBBAND_7_H
+	 * and RTW89_ACPI_SAR_6GHZ_SUBBAND_8, so directly describe it with
+	 * struct rtw89_6ghz_span.
+	 */
+
+	case 6895 ... 7115:
+		return RTW89_ACPI_SAR_6GHZ_SUBBAND_8;
+	}
+}
+
+enum rtw89_band rtw89_acpi_sar_subband_to_band(struct rtw89_dev *rtwdev,
+					       enum rtw89_acpi_sar_subband subband)
+{
+	switch (subband) {
+	default:
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+			    "ACPI SAR subband %u to band is unhandled\n", subband);
+		fallthrough;
+	case RTW89_ACPI_SAR_2GHZ_SUBBAND:
+		return RTW89_BAND_2G;
+	case RTW89_ACPI_SAR_5GHZ_SUBBAND_1:
+		return RTW89_BAND_5G;
+	case RTW89_ACPI_SAR_5GHZ_SUBBAND_2:
+		return RTW89_BAND_5G;
+	case RTW89_ACPI_SAR_5GHZ_SUBBAND_2E:
+		return RTW89_BAND_5G;
+	case RTW89_ACPI_SAR_5GHZ_SUBBAND_3_4:
+		return RTW89_BAND_5G;
+	case RTW89_ACPI_SAR_6GHZ_SUBBAND_5_L:
+		return RTW89_BAND_6G;
+	case RTW89_ACPI_SAR_6GHZ_SUBBAND_5_H:
+		return RTW89_BAND_6G;
+	case RTW89_ACPI_SAR_6GHZ_SUBBAND_6:
+		return RTW89_BAND_6G;
+	case RTW89_ACPI_SAR_6GHZ_SUBBAND_7_L:
+		return RTW89_BAND_6G;
+	case RTW89_ACPI_SAR_6GHZ_SUBBAND_7_H:
+		return RTW89_BAND_6G;
+	case RTW89_ACPI_SAR_6GHZ_SUBBAND_8:
+		return RTW89_BAND_6G;
+	}
+}
+
+static u8 rtw89_acpi_sar_rfpath_to_hp_antidx(enum rtw89_rf_path rfpath)
+{
+	switch (rfpath) {
+	default:
+	case RF_PATH_B:
+		return 0;
+	case RF_PATH_A:
+		return 1;
+	}
+}
+
+static u8 rtw89_acpi_sar_rfpath_to_rt_antidx(enum rtw89_rf_path rfpath)
+{
+	switch (rfpath) {
+	default:
+	case RF_PATH_A:
+		return 0;
+	case RF_PATH_B:
+		return 1;
+	}
+}
+
+static s16 rtw89_acpi_sar_normalize_hp_val(u8 v)
+{
+	static const u8 bias = 10;
+	static const u8 fct = 1;
+	u16 res;
+
+	BUILD_BUG_ON(fct > TXPWR_FACTOR_OF_RTW89_ACPI_SAR);
+
+	res = (bias << TXPWR_FACTOR_OF_RTW89_ACPI_SAR) +
+	      (v << (TXPWR_FACTOR_OF_RTW89_ACPI_SAR - fct));
+
+	return min_t(s32, res, MAX_VAL_OF_RTW89_ACPI_SAR);
+}
+
+static s16 rtw89_acpi_sar_normalize_rt_val(u8 v)
+{
+	static const u8 fct = 3;
+	u16 res;
+
+	BUILD_BUG_ON(fct > TXPWR_FACTOR_OF_RTW89_ACPI_SAR);
+
+	res = v << (TXPWR_FACTOR_OF_RTW89_ACPI_SAR - fct);
+
+	return min_t(s32, res, MAX_VAL_OF_RTW89_ACPI_SAR);
+}
+
+static
+void rtw89_acpi_sar_load_std_legacy(struct rtw89_dev *rtwdev,
+				    const struct rtw89_acpi_sar_recognition *rec,
+				    const void *content,
+				    struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_sar_std_legacy *ptr = content;
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++) {
+			u8 antidx = rec->rfpath_to_antidx(path);
+
+			if (subband < RTW89_ACPI_SAR_SUBBAND_NR_LEGACY)
+				ent->v[subband][path] =
+					rec->normalize(ptr->v[antidx][subband]);
+			else
+				ent->v[subband][path] = MAX_VAL_OF_RTW89_ACPI_SAR;
+		}
+	}
+}
+
+static
+void rtw89_acpi_sar_load_std_has_6ghz(struct rtw89_dev *rtwdev,
+				      const struct rtw89_acpi_sar_recognition *rec,
+				      const void *content,
+				      struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_sar_std_has_6ghz *ptr = content;
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+
+	BUILD_BUG_ON(RTW89_ACPI_SAR_SUBBAND_NR_HAS_6GHZ != NUM_OF_RTW89_ACPI_SAR_SUBBAND);
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++) {
+			u8 antidx = rec->rfpath_to_antidx(path);
+
+			ent->v[subband][path] = rec->normalize(ptr->v[antidx][subband]);
+		}
+	}
+}
+
+static
+void rtw89_acpi_sar_load_sml_legacy(struct rtw89_dev *rtwdev,
+				    const struct rtw89_acpi_sar_recognition *rec,
+				    const void *content,
+				    struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_sar_sml_legacy *ptr = content;
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++) {
+			u8 antidx = rec->rfpath_to_antidx(path);
+
+			if (subband < RTW89_ACPI_SAR_SUBBAND_NR_LEGACY)
+				ent->v[subband][path] =
+					rec->normalize(ptr->v[antidx][subband]);
+			else
+				ent->v[subband][path] = MAX_VAL_OF_RTW89_ACPI_SAR;
+		}
+	}
+}
+
+static
+void rtw89_acpi_sar_load_sml_has_6ghz(struct rtw89_dev *rtwdev,
+				      const struct rtw89_acpi_sar_recognition *rec,
+				      const void *content,
+				      struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_sar_sml_has_6ghz *ptr = content;
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+
+	BUILD_BUG_ON(RTW89_ACPI_SAR_SUBBAND_NR_HAS_6GHZ != NUM_OF_RTW89_ACPI_SAR_SUBBAND);
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++) {
+			u8 antidx = rec->rfpath_to_antidx(path);
+
+			ent->v[subband][path] = rec->normalize(ptr->v[antidx][subband]);
+		}
+	}
+}
+
+static s16 rtw89_acpi_geo_sar_normalize_delta(s8 delta)
+{
+	static const u8 fct = 1;
+
+	BUILD_BUG_ON(fct > TXPWR_FACTOR_OF_RTW89_ACPI_SAR);
+
+	return delta << (TXPWR_FACTOR_OF_RTW89_ACPI_SAR - fct);
+}
+
+static enum rtw89_acpi_geo_sar_regd_hp
+rtw89_acpi_geo_sar_regd_convert_hp_idx(enum rtw89_regulation_type regd)
+{
+	switch (regd) {
+	case RTW89_FCC:
+	case RTW89_IC:
+	case RTW89_NCC:
+	case RTW89_CHILE:
+	case RTW89_MEXICO:
+		return RTW89_ACPI_GEO_SAR_REGD_HP_FCC;
+	case RTW89_ETSI:
+	case RTW89_MKK:
+	case RTW89_ACMA:
+		return RTW89_ACPI_GEO_SAR_REGD_HP_ETSI;
+	default:
+	case RTW89_WW:
+	case RTW89_NA:
+	case RTW89_KCC:
+		return RTW89_ACPI_GEO_SAR_REGD_HP_WW;
+	}
+}
+
+static enum rtw89_acpi_geo_sar_regd_rt
+rtw89_acpi_geo_sar_regd_convert_rt_idx(enum rtw89_regulation_type regd)
+{
+	switch (regd) {
+	case RTW89_FCC:
+	case RTW89_NCC:
+	case RTW89_CHILE:
+	case RTW89_MEXICO:
+		return RTW89_ACPI_GEO_SAR_REGD_RT_FCC;
+	case RTW89_ETSI:
+	case RTW89_ACMA:
+		return RTW89_ACPI_GEO_SAR_REGD_RT_ETSI;
+	case RTW89_MKK:
+		return RTW89_ACPI_GEO_SAR_REGD_RT_MKK;
+	case RTW89_IC:
+		return RTW89_ACPI_GEO_SAR_REGD_RT_IC;
+	case RTW89_KCC:
+		return RTW89_ACPI_GEO_SAR_REGD_RT_KCC;
+	default:
+	case RTW89_WW:
+	case RTW89_NA:
+		return RTW89_ACPI_GEO_SAR_REGD_RT_WW;
+	}
+}
+
+static
+void rtw89_acpi_geo_sar_load_by_hp(struct rtw89_dev *rtwdev,
+				   const struct rtw89_acpi_geo_sar_hp_val *ptr,
+				   enum rtw89_rf_path path, s16 *val)
+{
+	u8 antidx = rtw89_acpi_sar_rfpath_to_hp_antidx(path);
+	s16 delta = rtw89_acpi_geo_sar_normalize_delta(ptr->delta[antidx]);
+	s16 max = rtw89_acpi_sar_normalize_hp_val(ptr->max);
+
+	*val = clamp_t(s32, (*val) + delta, MIN_VAL_OF_RTW89_ACPI_SAR, max);
+}
+
+static
+void rtw89_acpi_geo_sar_load_by_rt(struct rtw89_dev *rtwdev,
+				   const struct rtw89_acpi_geo_sar_rt_val *ptr,
+				   s16 *val)
+{
+	s16 delta = rtw89_acpi_geo_sar_normalize_delta(ptr->delta);
+	s16 max = rtw89_acpi_sar_normalize_rt_val(ptr->max);
+
+	*val = clamp_t(s32, (*val) + delta, MIN_VAL_OF_RTW89_ACPI_SAR, max);
+}
+
+static
+void rtw89_acpi_geo_sar_load_hp_legacy(struct rtw89_dev *rtwdev,
+				       const void *content,
+				       enum rtw89_regulation_type regd,
+				       struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_geo_sar_hp_legacy *ptr = content;
+	const struct rtw89_acpi_geo_sar_hp_legacy_entry *ptr_ent;
+	const struct rtw89_acpi_geo_sar_hp_val *ptr_ent_val;
+	enum rtw89_acpi_geo_sar_regd_hp geo_idx =
+		rtw89_acpi_geo_sar_regd_convert_hp_idx(regd);
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+	enum rtw89_band band;
+
+	ptr_ent = &ptr->entries[geo_idx];
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		band = rtw89_acpi_sar_subband_to_band(rtwdev, subband);
+		switch (band) {
+		case RTW89_BAND_2G:
+			ptr_ent_val = &ptr_ent->val_2ghz;
+			break;
+		case RTW89_BAND_5G:
+			ptr_ent_val = &ptr_ent->val_5ghz;
+			break;
+		default:
+		case RTW89_BAND_6G:
+			ptr_ent_val = NULL;
+			break;
+		}
+
+		if (!ptr_ent_val)
+			continue;
+
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++)
+			rtw89_acpi_geo_sar_load_by_hp(rtwdev, ptr_ent_val, path,
+						      &ent->v[subband][path]);
+	}
+}
+
+static
+void rtw89_acpi_geo_sar_load_hp_has_6ghz(struct rtw89_dev *rtwdev,
+					 const void *content,
+					 enum rtw89_regulation_type regd,
+					 struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_geo_sar_hp_has_6ghz *ptr = content;
+	const struct rtw89_acpi_geo_sar_hp_has_6ghz_entry *ptr_ent;
+	const struct rtw89_acpi_geo_sar_hp_val *ptr_ent_val;
+	enum rtw89_acpi_geo_sar_regd_hp geo_idx =
+		rtw89_acpi_geo_sar_regd_convert_hp_idx(regd);
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+	enum rtw89_band band;
+
+	ptr_ent = &ptr->entries[geo_idx];
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		band = rtw89_acpi_sar_subband_to_band(rtwdev, subband);
+		switch (band) {
+		case RTW89_BAND_2G:
+			ptr_ent_val = &ptr_ent->val_2ghz;
+			break;
+		case RTW89_BAND_5G:
+			ptr_ent_val = &ptr_ent->val_5ghz;
+			break;
+		case RTW89_BAND_6G:
+			ptr_ent_val = &ptr_ent->val_6ghz;
+			break;
+		default:
+			ptr_ent_val = NULL;
+			break;
+		}
+
+		if (!ptr_ent_val)
+			continue;
+
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++)
+			rtw89_acpi_geo_sar_load_by_hp(rtwdev, ptr_ent_val, path,
+						      &ent->v[subband][path]);
+	}
+}
+
+static
+void rtw89_acpi_geo_sar_load_rt_legacy(struct rtw89_dev *rtwdev,
+				       const void *content,
+				       enum rtw89_regulation_type regd,
+				       struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_geo_sar_rt_legacy *ptr = content;
+	const struct rtw89_acpi_geo_sar_rt_legacy_entry *ptr_ent;
+	const struct rtw89_acpi_geo_sar_rt_val *ptr_ent_val;
+	enum rtw89_acpi_geo_sar_regd_rt geo_idx =
+		rtw89_acpi_geo_sar_regd_convert_rt_idx(regd);
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+	enum rtw89_band band;
+
+	ptr_ent = &ptr->entries[geo_idx];
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		band = rtw89_acpi_sar_subband_to_band(rtwdev, subband);
+		switch (band) {
+		case RTW89_BAND_2G:
+			ptr_ent_val = &ptr_ent->val_2ghz;
+			break;
+		case RTW89_BAND_5G:
+			ptr_ent_val = &ptr_ent->val_5ghz;
+			break;
+		default:
+		case RTW89_BAND_6G:
+			ptr_ent_val = NULL;
+			break;
+		}
+
+		if (!ptr_ent_val)
+			continue;
+
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++)
+			rtw89_acpi_geo_sar_load_by_rt(rtwdev, ptr_ent_val,
+						      &ent->v[subband][path]);
+	}
+}
+
+static
+void rtw89_acpi_geo_sar_load_rt_has_6ghz(struct rtw89_dev *rtwdev,
+					 const void *content,
+					 enum rtw89_regulation_type regd,
+					 struct rtw89_sar_entry_from_acpi *ent)
+{
+	const struct rtw89_acpi_geo_sar_rt_has_6ghz *ptr = content;
+	const struct rtw89_acpi_geo_sar_rt_has_6ghz_entry *ptr_ent;
+	const struct rtw89_acpi_geo_sar_rt_val *ptr_ent_val;
+	enum rtw89_acpi_geo_sar_regd_rt geo_idx =
+		rtw89_acpi_geo_sar_regd_convert_rt_idx(regd);
+	enum rtw89_acpi_sar_subband subband;
+	enum rtw89_rf_path path;
+	enum rtw89_band band;
+
+	ptr_ent = &ptr->entries[geo_idx];
+
+	for (subband = 0; subband < NUM_OF_RTW89_ACPI_SAR_SUBBAND; subband++) {
+		band = rtw89_acpi_sar_subband_to_band(rtwdev, subband);
+		switch (band) {
+		case RTW89_BAND_2G:
+			ptr_ent_val = &ptr_ent->val_2ghz;
+			break;
+		case RTW89_BAND_5G:
+			ptr_ent_val = &ptr_ent->val_5ghz;
+			break;
+		case RTW89_BAND_6G:
+			ptr_ent_val = &ptr_ent->val_6ghz;
+			break;
+		default:
+			ptr_ent_val = NULL;
+			break;
+		}
+
+		if (!ptr_ent_val)
+			continue;
+
+		for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++)
+			rtw89_acpi_geo_sar_load_by_rt(rtwdev, ptr_ent_val,
+						      &ent->v[subband][path]);
+	}
+}
+
+#define RTW89_ACPI_GEO_SAR_DECL_HANDLER(type) \
+static const struct rtw89_acpi_geo_sar_handler \
+rtw89_acpi_geo_sar_handler_ ## type = { \
+	.data_size = RTW89_ACPI_GEO_SAR_SIZE_OF(type), \
+	.load = rtw89_acpi_geo_sar_load_ ## type, \
+}
+
+RTW89_ACPI_GEO_SAR_DECL_HANDLER(hp_legacy);
+RTW89_ACPI_GEO_SAR_DECL_HANDLER(hp_has_6ghz);
+RTW89_ACPI_GEO_SAR_DECL_HANDLER(rt_legacy);
+RTW89_ACPI_GEO_SAR_DECL_HANDLER(rt_has_6ghz);
+
+static const struct rtw89_acpi_sar_recognition rtw89_acpi_sar_recs[] = {
+	{
+		.id = {
+			.cid = RTW89_ACPI_SAR_CID_HP,
+			.rev = RTW89_ACPI_SAR_REV_LEGACY,
+			.size = RTW89_ACPI_SAR_SIZE_OF(std_legacy),
+		},
+		.geo = &rtw89_acpi_geo_sar_handler_hp_legacy,
+
+		.rfpath_to_antidx = rtw89_acpi_sar_rfpath_to_hp_antidx,
+		.normalize = rtw89_acpi_sar_normalize_hp_val,
+		.load = rtw89_acpi_sar_load_std_legacy,
+	},
+	{
+		.id = {
+			.cid = RTW89_ACPI_SAR_CID_HP,
+			.rev = RTW89_ACPI_SAR_REV_HAS_6GHZ,
+			.size = RTW89_ACPI_SAR_SIZE_OF(std_has_6ghz),
+		},
+		.geo = &rtw89_acpi_geo_sar_handler_hp_has_6ghz,
+
+		.rfpath_to_antidx = rtw89_acpi_sar_rfpath_to_hp_antidx,
+		.normalize = rtw89_acpi_sar_normalize_hp_val,
+		.load = rtw89_acpi_sar_load_std_has_6ghz,
+	},
+	{
+		.id = {
+			.cid = RTW89_ACPI_SAR_CID_RT,
+			.rev = RTW89_ACPI_SAR_REV_LEGACY,
+			.size = RTW89_ACPI_SAR_SIZE_OF(std_legacy),
+		},
+		.geo = &rtw89_acpi_geo_sar_handler_rt_legacy,
+
+		.rfpath_to_antidx = rtw89_acpi_sar_rfpath_to_rt_antidx,
+		.normalize = rtw89_acpi_sar_normalize_rt_val,
+		.load = rtw89_acpi_sar_load_std_legacy,
+	},
+	{
+		.id = {
+			.cid = RTW89_ACPI_SAR_CID_RT,
+			.rev = RTW89_ACPI_SAR_REV_HAS_6GHZ,
+			.size = RTW89_ACPI_SAR_SIZE_OF(std_has_6ghz),
+		},
+		.geo = &rtw89_acpi_geo_sar_handler_rt_has_6ghz,
+
+		.rfpath_to_antidx = rtw89_acpi_sar_rfpath_to_rt_antidx,
+		.normalize = rtw89_acpi_sar_normalize_rt_val,
+		.load = rtw89_acpi_sar_load_std_has_6ghz,
+	},
+	{
+		.id = {
+			.cid = RTW89_ACPI_SAR_CID_RT,
+			.rev = RTW89_ACPI_SAR_REV_LEGACY,
+			.size = RTW89_ACPI_SAR_SIZE_OF(sml_legacy),
+		},
+		.geo = &rtw89_acpi_geo_sar_handler_rt_legacy,
+
+		.rfpath_to_antidx = rtw89_acpi_sar_rfpath_to_rt_antidx,
+		.normalize = rtw89_acpi_sar_normalize_rt_val,
+		.load = rtw89_acpi_sar_load_sml_legacy,
+	},
+	{
+		.id = {
+			.cid = RTW89_ACPI_SAR_CID_RT,
+			.rev = RTW89_ACPI_SAR_REV_HAS_6GHZ,
+			.size = RTW89_ACPI_SAR_SIZE_OF(sml_has_6ghz),
+		},
+		.geo = &rtw89_acpi_geo_sar_handler_rt_has_6ghz,
+
+		.rfpath_to_antidx = rtw89_acpi_sar_rfpath_to_rt_antidx,
+		.normalize = rtw89_acpi_sar_normalize_rt_val,
+		.load = rtw89_acpi_sar_load_sml_has_6ghz,
+	},
+};
+
+struct rtw89_acpi_sar_rec_parm {
+	u32 pld_len;
+	u8 tbl_cnt;
+	u16 cid;
+	u8 rev;
+};
+
+static const struct rtw89_acpi_sar_recognition *
+rtw89_acpi_sar_recognize(struct rtw89_dev *rtwdev,
+			 const struct rtw89_acpi_sar_rec_parm *parm)
+{
+	const u32 tbl_len = parm->pld_len / parm->tbl_cnt;
+	const struct rtw89_acpi_sar_recognition *rec;
+	struct rtw89_acpi_sar_identifier id = {};
+
+	rtw89_debug(rtwdev, RTW89_DBG_ACPI,
+		    "%s: cid %u, rev %u, tbl len %u, tbl cnt %u\n",
+		    __func__, parm->cid, parm->rev, tbl_len, parm->tbl_cnt);
+
+	if (unlikely(parm->pld_len % parm->tbl_cnt)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid pld len %u\n",
+			    parm->pld_len);
+		return NULL;
+	}
+
+	if (unlikely(tbl_len > RTW89_ACPI_SAR_SIZE_MAX)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid tbl len %u\n",
+			    tbl_len);
+		return NULL;
+	}
+
+	if (unlikely(parm->tbl_cnt > MAX_NUM_OF_RTW89_ACPI_SAR_TBL)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid tbl cnt %u\n",
+			    parm->tbl_cnt);
+		return NULL;
+	}
+
+	switch (parm->cid) {
+	case RTW89_ACPI_SAR_CID_HP:
+	case RTW89_ACPI_SAR_CID_RT:
+		id.cid = parm->cid;
+		break;
+	default:
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid cid 0x%x\n",
+			    parm->cid);
+		return NULL;
+	}
+
+	switch (parm->rev) {
+	case RTW89_ACPI_SAR_REV_LEGACY:
+	case RTW89_ACPI_SAR_REV_HAS_6GHZ:
+		id.rev = parm->rev;
+		break;
+	default:
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid rev %u\n",
+			    parm->rev);
+		return NULL;
+	}
+
+	id.size = tbl_len;
+	for (unsigned int i = 0; i < ARRAY_SIZE(rtw89_acpi_sar_recs); i++) {
+		rec = &rtw89_acpi_sar_recs[i];
+		if (memcmp(&rec->id, &id, sizeof(rec->id)) == 0)
+			return rec;
+	}
+
+	rtw89_debug(rtwdev, RTW89_DBG_ACPI, "failed to recognize\n");
+	return NULL;
+}
+
+static const struct rtw89_acpi_sar_recognition *
+rtw89_acpi_evaluate_static_sar(struct rtw89_dev *rtwdev,
+			       struct rtw89_sar_cfg_acpi *cfg)
+{
+	const struct rtw89_acpi_sar_recognition *rec = NULL;
+	const struct rtw89_acpi_static_sar_hdr *hdr;
+	struct rtw89_sar_entry_from_acpi tmp = {};
+	struct rtw89_acpi_sar_rec_parm parm = {};
+	struct rtw89_sar_table_from_acpi *tbl;
+	const struct rtw89_acpi_data *data;
+	u32 len;
+
+	data = rtw89_acpi_evaluate_method(rtwdev, RTW89_ACPI_METHOD_STATIC_SAR);
+	if (!data)
+		return NULL;
+
+	rtw89_debug(rtwdev, RTW89_DBG_ACPI, "acpi load static sar\n");
+
+	len = data->len;
+	if (len <= sizeof(*hdr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid buf len %u\n", len);
 		goto out;
 	}
 
-	buf_len = obj->buffer.length;
-	if (buf_len != sizeof(*res)) {
-		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "%s: invalid buffer length: %u\n",
-			    __func__, buf_len);
+	hdr = (typeof(hdr))data->buf;
+
+	parm.cid = le16_to_cpu(hdr->cid);
+	parm.rev = hdr->rev;
+	parm.tbl_cnt = 1;
+	parm.pld_len = len - sizeof(*hdr);
+
+	rec = rtw89_acpi_sar_recognize(rtwdev, &parm);
+	if (!rec)
+		goto out;
+
+	rec->load(rtwdev, rec, hdr->content, &tmp);
+
+	tbl = &cfg->tables[0];
+	for (u8 regd = 0; regd < RTW89_REGD_NUM; regd++)
+		tbl->entries[regd] = tmp;
+
+	cfg->valid_num = 1;
+
+out:
+	kfree(data);
+	return rec;
+}
+
+static const struct rtw89_acpi_sar_recognition *
+rtw89_acpi_evaluate_dynamic_sar(struct rtw89_dev *rtwdev,
+				struct rtw89_sar_cfg_acpi *cfg)
+{
+	const struct rtw89_acpi_sar_recognition *rec = NULL;
+	const struct rtw89_acpi_dynamic_sar_hdr *hdr;
+	struct rtw89_acpi_sar_rec_parm parm = {};
+	struct rtw89_sar_table_from_acpi *tbl;
+	const struct rtw89_acpi_data *data;
+	u32 len;
+
+	data = rtw89_acpi_evaluate_method(rtwdev, RTW89_ACPI_METHOD_DYNAMIC_SAR);
+	if (!data)
+		return NULL;
+
+	rtw89_debug(rtwdev, RTW89_DBG_ACPI, "acpi load dynamic sar\n");
+
+	len = data->len;
+	if (len <= sizeof(*hdr)) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid buf len %u\n", len);
+		goto out;
+	}
+
+	hdr = (typeof(hdr))data->buf;
+
+	parm.cid = le16_to_cpu(hdr->cid);
+	parm.rev = hdr->rev;
+	parm.tbl_cnt = hdr->cnt;
+	parm.pld_len = len - sizeof(*hdr);
+
+	rec = rtw89_acpi_sar_recognize(rtwdev, &parm);
+	if (!rec)
+		goto out;
+
+	for (unsigned int i = 0; i < hdr->cnt; i++) {
+		const u8 *content = hdr->content + rec->id.size * i;
+		struct rtw89_sar_entry_from_acpi tmp = {};
+
+		rec->load(rtwdev, rec, content, &tmp);
+
+		tbl = &cfg->tables[i];
+		for (u8 regd = 0; regd < RTW89_REGD_NUM; regd++)
+			tbl->entries[regd] = tmp;
+	}
+
+	cfg->valid_num = hdr->cnt;
+
+out:
+	kfree(data);
+	return rec;
+}
+
+int rtw89_acpi_evaluate_dynamic_sar_indicator(struct rtw89_dev *rtwdev,
+					      struct rtw89_sar_cfg_acpi *cfg,
+					      bool *poll_changed)
+{
+	struct rtw89_sar_indicator_from_acpi *ind = &cfg->indicator;
+	struct rtw89_sar_indicator_from_acpi tmp = *ind;
+	const struct rtw89_acpi_data *data;
+	const u8 *tbl_base1_by_ant;
+	enum rtw89_rf_path path;
+	int ret = 0;
+	u32 len;
+
+	data = rtw89_acpi_evaluate_method(rtwdev, RTW89_ACPI_METHOD_DYNAMIC_SAR_INDICATOR);
+	if (!data)
+		return -EFAULT;
+
+	if (!poll_changed)
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "acpi load dynamic sar indicator\n");
+
+	len = data->len;
+	if (len != ind->fields) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid buf len %u\n", len);
 		ret = -EINVAL;
 		goto out;
 	}
 
-	*res = *(struct rtw89_acpi_rtag_result *)obj->buffer.pointer;
+	tbl_base1_by_ant = data->buf;
 
-	rtw89_hex_dump(rtwdev, RTW89_DBG_ACPI, "antenna_gain: ", res, sizeof(*res));
+	for (path = 0; path < NUM_OF_RTW89_ACPI_SAR_RF_PATH; path++) {
+		u8 antidx = ind->rfpath_to_antidx(path);
+		u8 sel;
+
+		if (antidx >= ind->fields)
+			antidx = 0;
+
+		/* convert the table index from 1-based to 0-based */
+		sel = tbl_base1_by_ant[antidx] - 1;
+		if (sel >= cfg->valid_num)
+			sel = 0;
+
+		tmp.tblsel[path] = sel;
+	}
+
+	if (memcmp(ind, &tmp, sizeof(*ind)) == 0) {
+		if (poll_changed)
+			*poll_changed = false;
+	} else {
+		if (poll_changed)
+			*poll_changed = true;
+
+		*ind = tmp;
+	}
 
 out:
-	ACPI_FREE(obj);
+	kfree(data);
 	return ret;
 }
+
+static
+void rtw89_acpi_evaluate_geo_sar(struct rtw89_dev *rtwdev,
+				 const struct rtw89_acpi_geo_sar_handler *hdl,
+				 struct rtw89_sar_cfg_acpi *cfg)
+{
+	const struct rtw89_acpi_data *data;
+	u32 len;
+
+	data = rtw89_acpi_evaluate_method(rtwdev, RTW89_ACPI_METHOD_GEO_SAR);
+	if (!data)
+		return;
+
+	rtw89_debug(rtwdev, RTW89_DBG_ACPI, "acpi load geo sar\n");
+
+	len = data->len;
+	if (len != hdl->data_size) {
+		rtw89_debug(rtwdev, RTW89_DBG_ACPI, "invalid buf len %u (expected %u)\n",
+			    len, hdl->data_size);
+		goto out;
+	}
+
+	for (unsigned int i = 0; i < cfg->valid_num; i++)
+		for (u8 regd = 0; regd < RTW89_REGD_NUM; regd++)
+			hdl->load(rtwdev, data->buf, regd, &cfg->tables[i].entries[regd]);
+
+out:
+	kfree(data);
+}
+
+int rtw89_acpi_evaluate_sar(struct rtw89_dev *rtwdev,
+			    struct rtw89_sar_cfg_acpi *cfg)
+{
+	struct rtw89_sar_indicator_from_acpi *ind = &cfg->indicator;
+	const struct rtw89_acpi_sar_recognition *rec;
+	bool fetch_indicator = false;
+	int ret;
+
+	rec = rtw89_acpi_evaluate_static_sar(rtwdev, cfg);
+	if (rec)
+		goto recognized;
+
+	rec = rtw89_acpi_evaluate_dynamic_sar(rtwdev, cfg);
+	if (!rec)
+		return -ENOENT;
+
+	fetch_indicator = true;
+
+recognized:
+	rtw89_acpi_evaluate_geo_sar(rtwdev, rec->geo, cfg);
+
+	switch (rec->id.cid) {
+	case RTW89_ACPI_SAR_CID_HP:
+		cfg->downgrade_2tx = 3 << TXPWR_FACTOR_OF_RTW89_ACPI_SAR;
+		ind->fields = RTW89_ACPI_SAR_ANT_NR_STD;
+		break;
+	case RTW89_ACPI_SAR_CID_RT:
+		cfg->downgrade_2tx = 0;
+		ind->fields = 1;
+		break;
+	default:
+		return -EFAULT;
+	}
+
+	if (fetch_indicator) {
+		ind->rfpath_to_antidx = rec->rfpath_to_antidx;
+		ret = rtw89_acpi_evaluate_dynamic_sar_indicator(rtwdev, cfg, NULL);
+		if (ret)
+			fetch_indicator = false;
+	}
+
+	if (!fetch_indicator)
+		memset(ind->tblsel, 0, sizeof(ind->tblsel));
+
+	ind->enable_sync = fetch_indicator;
+	return 0;
+}
diff --git a/sys/contrib/dev/rtw89/acpi.h b/sys/contrib/dev/rtw89/acpi.h
index b43ab106e44d..48a46f2005b1 100644
--- a/sys/contrib/dev/rtw89/acpi.h
+++ b/sys/contrib/dev/rtw89/acpi.h
@@ -7,6 +7,11 @@
 
 #include "core.h"
 
+struct rtw89_acpi_data {
+	u32 len;
+	u8 buf[] __counted_by(len);
+};
+
 enum rtw89_acpi_dsm_func {
 	RTW89_ACPI_DSM_FUNC_IDN_BAND_SUP = 2,
 	RTW89_ACPI_DSM_FUNC_6G_DIS = 3,
@@ -14,11 +19,13 @@ enum rtw89_acpi_dsm_func {
 	RTW89_ACPI_DSM_FUNC_TAS_EN = 5,
 	RTW89_ACPI_DSM_FUNC_UNII4_SUP = 6,
 	RTW89_ACPI_DSM_FUNC_6GHZ_SP_SUP = 7,
+	RTW89_ACPI_DSM_FUNC_REG_RULES_EN = 10,
+	RTW89_ACPI_DSM_FUNC_6GHZ_VLP_SUP = 11,
 };
 
 enum rtw89_acpi_conf_unii4 {
-	RTW89_ACPI_CONF_UNII4_FCC = BIT(0),
-	RTW89_ACPI_CONF_UNII4_IC = BIT(1),
+	RTW89_ACPI_CONF_UNII4_US = BIT(0),
+	RTW89_ACPI_CONF_UNII4_CA = BIT(1),
 };
 
 enum rtw89_acpi_policy_mode {
@@ -26,6 +33,13 @@ enum rtw89_acpi_policy_mode {
 	RTW89_ACPI_POLICY_ALLOW = 1,
 };
 
+enum rtw89_acpi_conf_tas {
+	RTW89_ACPI_CONF_TAS_US = BIT(0),
+	RTW89_ACPI_CONF_TAS_CA = BIT(1),
+	RTW89_ACPI_CONF_TAS_KR = BIT(2),
+	RTW89_ACPI_CONF_TAS_OTHERS = BIT(7),
+};
+
 struct rtw89_acpi_country_code {
 	/* below are allowed:
 	 * * ISO alpha2 country code
@@ -44,6 +58,7 @@ struct rtw89_acpi_policy_6ghz {
 
 enum rtw89_acpi_conf_6ghz_sp {
 	RTW89_ACPI_CONF_6GHZ_SP_US = BIT(0),
+	RTW89_ACPI_CONF_6GHZ_SP_CA = BIT(1),
 };
 
 struct rtw89_acpi_policy_6ghz_sp {
@@ -54,12 +69,47 @@ struct rtw89_acpi_policy_6ghz_sp {
 	u8 rsvd;
 } __packed;
 
+enum rtw89_acpi_conf_6ghz_vlp {
+	RTW89_ACPI_CONF_6GHZ_VLP_US = BIT(0),
+	RTW89_ACPI_CONF_6GHZ_VLP_CA = BIT(1),
+};
+
+struct rtw89_acpi_policy_6ghz_vlp {
+	u8 signature[4];
+	u8 revision;
+	u8 override;
+	u8 conf;
+	u8 rsvd;
+} __packed;
+
+struct rtw89_acpi_policy_tas {
+	u8 signature[4];
+	u8 revision;
+	u8 enable;
+	u8 enabled_countries;
+	u8 rsvd[3];
+} __packed;
+
+enum rtw89_acpi_conf_reg_rules {
+	RTW89_ACPI_CONF_REG_RULE_REGD_UK = BIT(0),
+};
+
+struct rtw89_acpi_policy_reg_rules {
+	u8 signature[4];
+	u8 revision;
+	u8 conf;
+	u8 rsvd[3];
+} __packed;
+
 struct rtw89_acpi_dsm_result {
 	union {
 		u8 value;
 		/* caller needs to free it after using */
 		struct rtw89_acpi_policy_6ghz *policy_6ghz;
 		struct rtw89_acpi_policy_6ghz_sp *policy_6ghz_sp;
+		struct rtw89_acpi_policy_6ghz_vlp *policy_6ghz_vlp;
+		struct rtw89_acpi_policy_tas *policy_tas;
+		struct rtw89_acpi_policy_reg_rules *policy_reg_rules;
 	} u;
 };
 
@@ -70,10 +120,179 @@ struct rtw89_acpi_rtag_result {
 	u8 ant_gain_table[RTW89_ANT_GAIN_CHAIN_NUM][RTW89_ANT_GAIN_SUBBAND_NR];
 } __packed;
 
+enum rtw89_acpi_sar_cid {
+	RTW89_ACPI_SAR_CID_HP = 0x5048,
+	RTW89_ACPI_SAR_CID_RT = 0x5452,
+};
+
+enum rtw89_acpi_sar_rev {
+	RTW89_ACPI_SAR_REV_LEGACY = 1,
+	RTW89_ACPI_SAR_REV_HAS_6GHZ = 2,
+};
+
+#define RTW89_ACPI_SAR_ANT_NR_STD 4
+#define RTW89_ACPI_SAR_ANT_NR_SML 2
+
+#define RTW89_ACPI_METHOD_STATIC_SAR "WRDS"
+#define RTW89_ACPI_METHOD_DYNAMIC_SAR "RWRD"
+#define RTW89_ACPI_METHOD_DYNAMIC_SAR_INDICATOR "RWSI"
+#define RTW89_ACPI_METHOD_GEO_SAR "RWGS"
+
+struct rtw89_acpi_sar_std_legacy {
+	u8 v[RTW89_ACPI_SAR_ANT_NR_STD][RTW89_ACPI_SAR_SUBBAND_NR_LEGACY];
+} __packed;
+
+struct rtw89_acpi_sar_std_has_6ghz {
+	u8 v[RTW89_ACPI_SAR_ANT_NR_STD][RTW89_ACPI_SAR_SUBBAND_NR_HAS_6GHZ];
+} __packed;
+
+struct rtw89_acpi_sar_sml_legacy {
+	u8 v[RTW89_ACPI_SAR_ANT_NR_SML][RTW89_ACPI_SAR_SUBBAND_NR_LEGACY];
+} __packed;
+
+struct rtw89_acpi_sar_sml_has_6ghz {
+	u8 v[RTW89_ACPI_SAR_ANT_NR_SML][RTW89_ACPI_SAR_SUBBAND_NR_HAS_6GHZ];
+} __packed;
+
+struct rtw89_acpi_static_sar_hdr {
+	__le16 cid;
+	u8 rev;
+	u8 content[];
+} __packed;
+
+struct rtw89_acpi_dynamic_sar_hdr {
+	__le16 cid;
+	u8 rev;
+	u8 cnt;
+	u8 content[];
+} __packed;
+
+struct rtw89_acpi_sar_identifier {
+	enum rtw89_acpi_sar_cid cid;
+	enum rtw89_acpi_sar_rev rev;
+	u8 size;
+};
+
+/* for rtw89_acpi_sar_identifier::size */
+#define RTW89_ACPI_SAR_SIZE_MAX U8_MAX
+#define RTW89_ACPI_SAR_SIZE_OF(type) \
+	(BUILD_BUG_ON_ZERO(sizeof(struct rtw89_acpi_sar_ ## type) > \
+			   RTW89_ACPI_SAR_SIZE_MAX) + \
+	 sizeof(struct rtw89_acpi_sar_ ## type))
+
+struct rtw89_acpi_sar_recognition {
+	struct rtw89_acpi_sar_identifier id;
+	const struct rtw89_acpi_geo_sar_handler *geo;
+
+	u8 (*rfpath_to_antidx)(enum rtw89_rf_path rfpath);
+	s16 (*normalize)(u8 v);
+	void (*load)(struct rtw89_dev *rtwdev,
+		     const struct rtw89_acpi_sar_recognition *rec,
+		     const void *content,
+		     struct rtw89_sar_entry_from_acpi *ent);
+};
+
+struct rtw89_acpi_geo_sar_hp_val {
+	u8 max;
+	s8 delta[RTW89_ACPI_SAR_ANT_NR_STD];
+} __packed;
+
+struct rtw89_acpi_geo_sar_hp_legacy_entry {
+	struct rtw89_acpi_geo_sar_hp_val val_2ghz;
+	struct rtw89_acpi_geo_sar_hp_val val_5ghz;
+} __packed;
+
+struct rtw89_acpi_geo_sar_hp_has_6ghz_entry {
+	struct rtw89_acpi_geo_sar_hp_val val_2ghz;
+	struct rtw89_acpi_geo_sar_hp_val val_5ghz;
+	struct rtw89_acpi_geo_sar_hp_val val_6ghz;
+} __packed;
+
+enum rtw89_acpi_geo_sar_regd_hp {
+	RTW89_ACPI_GEO_SAR_REGD_HP_FCC = 0,
+	RTW89_ACPI_GEO_SAR_REGD_HP_ETSI = 1,
+	RTW89_ACPI_GEO_SAR_REGD_HP_WW = 2,
+
+	RTW89_ACPI_GEO_SAR_REGD_NR_HP,
+};
+
+struct rtw89_acpi_geo_sar_hp_legacy {
+	struct rtw89_acpi_geo_sar_hp_legacy_entry
+		entries[RTW89_ACPI_GEO_SAR_REGD_NR_HP];
+} __packed;
+
+struct rtw89_acpi_geo_sar_hp_has_6ghz {
+	struct rtw89_acpi_geo_sar_hp_has_6ghz_entry
+		entries[RTW89_ACPI_GEO_SAR_REGD_NR_HP];
+} __packed;
+
+struct rtw89_acpi_geo_sar_rt_val {
+	u8 max;
+	s8 delta;
+} __packed;
+
+struct rtw89_acpi_geo_sar_rt_legacy_entry {
+	struct rtw89_acpi_geo_sar_rt_val val_2ghz;
+	struct rtw89_acpi_geo_sar_rt_val val_5ghz;
+} __packed;
+
+struct rtw89_acpi_geo_sar_rt_has_6ghz_entry {
+	struct rtw89_acpi_geo_sar_rt_val val_2ghz;
+	struct rtw89_acpi_geo_sar_rt_val val_5ghz;
+	struct rtw89_acpi_geo_sar_rt_val val_6ghz;
+} __packed;
+
+enum rtw89_acpi_geo_sar_regd_rt {
+	RTW89_ACPI_GEO_SAR_REGD_RT_FCC = 0,
+	RTW89_ACPI_GEO_SAR_REGD_RT_ETSI = 1,
+	RTW89_ACPI_GEO_SAR_REGD_RT_MKK = 2,
+	RTW89_ACPI_GEO_SAR_REGD_RT_IC = 3,
+	RTW89_ACPI_GEO_SAR_REGD_RT_KCC = 4,
+	RTW89_ACPI_GEO_SAR_REGD_RT_WW = 5,
+
+	RTW89_ACPI_GEO_SAR_REGD_NR_RT,
+};
+
+struct rtw89_acpi_geo_sar_rt_legacy {
+	struct rtw89_acpi_geo_sar_rt_legacy_entry
+		entries[RTW89_ACPI_GEO_SAR_REGD_NR_RT];
+} __packed;
+
+struct rtw89_acpi_geo_sar_rt_has_6ghz {
+	struct rtw89_acpi_geo_sar_rt_has_6ghz_entry
+		entries[RTW89_ACPI_GEO_SAR_REGD_NR_RT];
+} __packed;
+
+struct rtw89_acpi_geo_sar_handler {
+	u8 data_size;
+
+	void (*load)(struct rtw89_dev *rtwdev,
+		     const void *content,
+		     enum rtw89_regulation_type regd,
+		     struct rtw89_sar_entry_from_acpi *ent);
+};
+
+/* for rtw89_acpi_geo_sar_handler::data_size */
+#define RTW89_ACPI_GEO_SAR_SIZE_MAX U8_MAX
+#define RTW89_ACPI_GEO_SAR_SIZE_OF(type) \
+	(BUILD_BUG_ON_ZERO(sizeof(struct rtw89_acpi_geo_sar_ ## type) > \
+			   RTW89_ACPI_GEO_SAR_SIZE_MAX) + \
+	 sizeof(struct rtw89_acpi_geo_sar_ ## type))
+
+enum rtw89_acpi_sar_subband rtw89_acpi_sar_get_subband(struct rtw89_dev *rtwdev,
+						       u32 center_freq);
+enum rtw89_band rtw89_acpi_sar_subband_to_band(struct rtw89_dev *rtwdev,
+					       enum rtw89_acpi_sar_subband subband);
+
 int rtw89_acpi_evaluate_dsm(struct rtw89_dev *rtwdev,
 			    enum rtw89_acpi_dsm_func func,
 			    struct rtw89_acpi_dsm_result *res);
 int rtw89_acpi_evaluate_rtag(struct rtw89_dev *rtwdev,
 			     struct rtw89_acpi_rtag_result *res);
+int rtw89_acpi_evaluate_sar(struct rtw89_dev *rtwdev,
+			    struct rtw89_sar_cfg_acpi *cfg);
+int rtw89_acpi_evaluate_dynamic_sar_indicator(struct rtw89_dev *rtwdev,
+					      struct rtw89_sar_cfg_acpi *cfg,
+					      bool *changed);
 
 #endif
diff --git a/sys/contrib/dev/rtw89/cam.c b/sys/contrib/dev/rtw89/cam.c
index 8fa1e6c1ce13..385a238fe5cc 100644
--- a/sys/contrib/dev/rtw89/cam.c
+++ b/sys/contrib/dev/rtw89/cam.c
@@ -6,6 +6,7 @@
 #include "debug.h"
 #include "fw.h"
 #include "mac.h"
+#include "ps.h"
 
 static struct sk_buff *
 rtw89_cam_get_sec_key_cmd(struct rtw89_dev *rtwdev,
@@ -469,13 +470,25 @@ int rtw89_cam_sec_key_add(struct rtw89_dev *rtwdev,
 	bool ext_key = false;
 	int ret;
 
+	if (ieee80211_vif_is_mld(vif) && !chip->hw_mlo_bmc_crypto &&
+	    !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
+		return -EOPNOTSUPP;
+
 	switch (key->cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
+		rtw89_leave_ips_by_hwflags(rtwdev);
 		hw_key_type = RTW89_SEC_KEY_TYPE_WEP40;
 		break;
 	case WLAN_CIPHER_SUITE_WEP104:
+		rtw89_leave_ips_by_hwflags(rtwdev);
 		hw_key_type = RTW89_SEC_KEY_TYPE_WEP104;
 		break;
+	case WLAN_CIPHER_SUITE_TKIP:
+		if (!chip->hw_tkip_crypto)
+			return -EOPNOTSUPP;
+		hw_key_type = RTW89_SEC_KEY_TYPE_TKIP;
+		key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIC;
+		break;
 	case WLAN_CIPHER_SUITE_CCMP:
 		hw_key_type = RTW89_SEC_KEY_TYPE_CCMP128;
 		if (!chip->hw_mgmt_tx_encrypt)
diff --git a/sys/contrib/dev/rtw89/chan.c b/sys/contrib/dev/rtw89/chan.c
index 257331c2de2e..bbdae184a0df 100644
--- a/sys/contrib/dev/rtw89/chan.c
+++ b/sys/contrib/dev/rtw89/chan.c
@@ -7,7 +7,9 @@
 #include "debug.h"
 #include "fw.h"
 #include "mac.h"
+#include "phy.h"
 #include "ps.h"
+#include "sar.h"
 #include "util.h"
 
 static void rtw89_swap_chanctx(struct rtw89_dev *rtwdev,
@@ -127,6 +129,48 @@ void rtw89_chan_create(struct rtw89_chan *chan, u8 center_chan, u8 primary_chan,
 						    bandwidth);
 }
 
+static void _rtw89_chan_update_punctured(struct rtw89_dev *rtwdev,
+					 struct rtw89_vif_link *rtwvif_link,
+					 const struct cfg80211_chan_def *chandef)
+{
+	struct ieee80211_bss_conf *bss_conf;
+
+	if (rtwvif_link->wifi_role != RTW89_WIFI_ROLE_STATION &&
+	    rtwvif_link->wifi_role != RTW89_WIFI_ROLE_P2P_CLIENT)
+		return;
+
+	rcu_read_lock();
+
+	bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
+	if (!bss_conf->eht_support) {
+		rcu_read_unlock();
+		return;
+	}
+
+	rcu_read_unlock();
+
+	rtw89_chip_h2c_punctured_cmac_tbl(rtwdev, rtwvif_link, chandef->punctured);
+}
+
+static void rtw89_chan_update_punctured(struct rtw89_dev *rtwdev,
+					enum rtw89_chanctx_idx idx,
+					const struct cfg80211_chan_def *chandef)
+{
+	struct rtw89_vif_link *rtwvif_link;
+	struct rtw89_vif *rtwvif;
+	unsigned int link_id;
+
+	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
+		rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id) {
+			if (!rtwvif_link->chanctx_assigned ||
+			    rtwvif_link->chanctx_idx != idx)
+				continue;
+
+			_rtw89_chan_update_punctured(rtwdev, rtwvif_link, chandef);
+		}
+	}
+}
+
 bool rtw89_assign_entity_chan(struct rtw89_dev *rtwdev,
 			      enum rtw89_chanctx_idx idx,
 			      const struct rtw89_chan *new)
@@ -155,7 +199,7 @@ int rtw89_iterate_entity_chan(struct rtw89_dev *rtwdev,
 	int ret;
 	u8 idx;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	for_each_set_bit(idx,  hal->entity_map, NUM_OF_RTW89_CHANCTX) {
 		chan = rtw89_chan_get(rtwdev, idx);
@@ -169,28 +213,33 @@ int rtw89_iterate_entity_chan(struct rtw89_dev *rtwdev,
 
 static void __rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
 					  enum rtw89_chanctx_idx idx,
-					  const struct cfg80211_chan_def *chandef,
-					  bool from_stack)
+					  const struct cfg80211_chan_def *chandef)
 {
 	struct rtw89_hal *hal = &rtwdev->hal;
 
 	hal->chanctx[idx].chandef = *chandef;
-
-	if (from_stack)
-		set_bit(idx, hal->entity_map);
 }
 
 void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
 				 enum rtw89_chanctx_idx idx,
 				 const struct cfg80211_chan_def *chandef)
 {
-	__rtw89_config_entity_chandef(rtwdev, idx, chandef, true);
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	if (!chandef) {
+		clear_bit(idx, hal->entity_map);
+		return;
+	}
+
+	__rtw89_config_entity_chandef(rtwdev, idx, chandef);
+	set_bit(idx, hal->entity_map);
 }
 
 void rtw89_config_roc_chandef(struct rtw89_dev *rtwdev,
-			      enum rtw89_chanctx_idx idx,
+			      struct rtw89_vif_link *rtwvif_link,
 			      const struct cfg80211_chan_def *chandef)
 {
+	enum rtw89_chanctx_idx idx = rtwvif_link->chanctx_idx;
 	struct rtw89_hal *hal = &rtwdev->hal;
 	enum rtw89_chanctx_idx cur;
 
@@ -204,6 +253,7 @@ void rtw89_config_roc_chandef(struct rtw89_dev *rtwdev,
 		}
 
 		hal->roc_chandef = *chandef;
+		hal->roc_link_index = rtw89_vif_link_inst_get_index(rtwvif_link);
 	} else {
 		cur = atomic_cmpxchg(&hal->roc_chanctx_idx, idx,
 				     RTW89_CHANCTX_IDLE);
@@ -224,7 +274,7 @@ static void rtw89_config_default_chandef(struct rtw89_dev *rtwdev)
 	struct cfg80211_chan_def chandef = {0};
 
 	rtw89_get_default_chandef(&chandef);
-	__rtw89_config_entity_chandef(rtwdev, RTW89_CHANCTX_0, &chandef, false);
+	__rtw89_config_entity_chandef(rtwdev, RTW89_CHANCTX_0, &chandef);
 }
 
 void rtw89_entity_init(struct rtw89_dev *rtwdev)
@@ -262,6 +312,8 @@ static void rtw89_entity_calculate_weight(struct rtw89_dev *rtwdev,
 	struct rtw89_vif *rtwvif;
 	int idx;
 
+	w->registered_chanctxs = bitmap_weight(hal->entity_map, NUM_OF_RTW89_CHANCTX);
+
 	for_each_set_bit(idx, hal->entity_map, NUM_OF_RTW89_CHANCTX) {
 		cfg = hal->chanctx[idx].cfg;
 		if (!cfg) {
@@ -310,7 +362,7 @@ const struct rtw89_chan *__rtw89_mgnt_chan_get(struct rtw89_dev *rtwdev,
 	enum rtw89_entity_mode mode;
 	u8 role_index;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (unlikely(link_index >= __RTW89_MLD_MAX_LINK_NUM)) {
 		WARN(1, "link index %u is invalid (max link inst num: %d)\n",
@@ -338,11 +390,10 @@ const struct rtw89_chan *__rtw89_mgnt_chan_get(struct rtw89_dev *rtwdev,
 
 	roc_idx = atomic_read(&hal->roc_chanctx_idx);
 	if (roc_idx != RTW89_CHANCTX_IDLE) {
-		/* ROC is ongoing (given ROC runs on RTW89_ROC_BY_LINK_INDEX).
-		 * If @link_index is the same as RTW89_ROC_BY_LINK_INDEX, get
-		 * the ongoing ROC chanctx.
+		/* ROC is ongoing (given ROC runs on @hal->roc_link_index).
+		 * If @link_index is the same, get the ongoing ROC chanctx.
 		 */
-		if (link_index == RTW89_ROC_BY_LINK_INDEX)
+		if (link_index == hal->roc_link_index)
 			chanctx_idx = roc_idx;
 	}
 
@@ -357,16 +408,45 @@ dflt:
 }
 EXPORT_SYMBOL(__rtw89_mgnt_chan_get);
 
+static enum rtw89_mlo_dbcc_mode
+rtw89_entity_sel_mlo_dbcc_mode(struct rtw89_dev *rtwdev, u8 active_hws)
+{
+	if (rtwdev->chip->chip_gen != RTW89_CHIP_BE)
+		return MLO_DBCC_NOT_SUPPORT;
+
+	switch (active_hws) {
+	case BIT(0):
+		return MLO_2_PLUS_0_1RF;
+	case BIT(1):
+		return MLO_0_PLUS_2_1RF;
+	case BIT(0) | BIT(1):
+	default:
+		return MLO_1_PLUS_1_1RF;
+	}
+}
+
+static
+void rtw89_entity_recalc_mlo_dbcc_mode(struct rtw89_dev *rtwdev, u8 active_hws)
+{
+	enum rtw89_mlo_dbcc_mode mode;
+
+	mode = rtw89_entity_sel_mlo_dbcc_mode(rtwdev, active_hws);
+	rtwdev->mlo_dbcc_mode = mode;
+
+	rtw89_debug(rtwdev, RTW89_DBG_STATE, "recalc mlo dbcc mode to %d\n", mode);
+}
+
 static void rtw89_entity_recalc_mgnt_roles(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_hal *hal = &rtwdev->hal;
 	struct rtw89_entity_mgnt *mgnt = &hal->entity_mgnt;
 	struct rtw89_vif_link *link;
 	struct rtw89_vif *role;
+	u8 active_hws = 0;
 	u8 pos = 0;
 	int i, j;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	for (i = 0; i < RTW89_MAX_INTERFACE_NUM; i++)
 		mgnt->active_roles[i] = NULL;
@@ -411,10 +491,13 @@ fill:
 				continue;
 
 			mgnt->chanctx_tbl[pos][i] = link->chanctx_idx;
+			active_hws |= BIT(i);
 		}
 
 		mgnt->active_roles[pos++] = role;
 	}
+
+	rtw89_entity_recalc_mlo_dbcc_mode(rtwdev, active_hws);
 }
 
 enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev)
@@ -427,7 +510,7 @@ enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev)
 	struct rtw89_chan chan;
 	u8 idx;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	bitmap_copy(recalc_map, hal->entity_map, NUM_OF_RTW89_CHANCTX);
 
@@ -439,7 +522,8 @@ enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev)
 		bitmap_zero(recalc_map, NUM_OF_RTW89_CHANCTX);
 		fallthrough;
 	case 0:
-		rtw89_config_default_chandef(rtwdev);
+		if (!w.registered_chanctxs)
+			rtw89_config_default_chandef(rtwdev);
 		set_bit(RTW89_CHANCTX_0, recalc_map);
 		fallthrough;
 	case 1:
@@ -556,7 +640,9 @@ static u32 rtw89_mcc_get_tbtt_ofst(struct rtw89_dev *rtwdev,
 	u64 sync_tsf = READ_ONCE(rtwvif_link->sync_bcn_tsf);
 	u32 remainder;
 
-	if (tsf < sync_tsf) {
+	if (role->is_go) {
+		sync_tsf = 0;
+	} else if (tsf < sync_tsf) {
 		rtw89_debug(rtwdev, RTW89_DBG_CHAN,
 			    "MCC get tbtt ofst: tsf might not update yet\n");
 		sync_tsf = 0;
@@ -690,19 +776,13 @@ static void rtw89_mcc_role_macid_sta_iter(void *data, struct ieee80211_sta *sta)
 	struct rtw89_vif *target = mcc_role->rtwvif_link->rtwvif;
 	struct rtw89_sta *rtwsta = sta_to_rtwsta(sta);
 	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
-	struct rtw89_dev *rtwdev = rtwsta->rtwdev;
-	struct rtw89_sta_link *rtwsta_link;
+	u8 macid;
 
 	if (rtwvif != target)
 		return;
 
-	rtwsta_link = rtw89_sta_get_link_inst(rtwsta, 0);
-	if (unlikely(!rtwsta_link)) {
-		rtw89_err(rtwdev, "mcc sta macid: find no link on HW-0\n");
-		return;
-	}
-
-	rtw89_mcc_role_fw_macid_bitmap_set_bit(mcc_role, rtwsta_link->mac_id);
+	macid = rtw89_sta_get_main_macid(rtwsta);
+	rtw89_mcc_role_fw_macid_bitmap_set_bit(mcc_role, macid);
 }
 
 static void rtw89_mcc_fill_role_macid_bitmap(struct rtw89_dev *rtwdev,
@@ -746,9 +826,11 @@ static void rtw89_mcc_fill_role_limit(struct rtw89_dev *rtwdev,
 	int ret;
 	int i;
 
-	if (!mcc_role->is_go && !mcc_role->is_gc)
+	if (!mcc_role->is_gc)
 		return;
 
+	rtw89_p2p_noa_once_recalc(rtwvif_link);
+
 	rcu_read_lock();
 
 	bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
@@ -784,6 +866,9 @@ fill:
 	}
 
 	tsf_lmt = (tsf & GENMASK_ULL(63, 32)) | start_time;
+	if (tsf_lmt < tsf)
+		tsf_lmt += roundup_u64(tsf - tsf_lmt, interval);
+
 	max_toa_us = rtw89_mcc_get_tbtt_ofst(rtwdev, mcc_role, tsf_lmt);
 	max_dur_us = interval - duration;
 	max_tob_us = max_dur_us - max_toa_us;
@@ -918,6 +1003,7 @@ static int rtw89_mcc_fill_all_roles(struct rtw89_dev *rtwdev)
 		}
 
 		sel.bind_vif[i] = rtwvif_link;
+		rtw89_p2p_disable_all_noa(rtwdev, rtwvif_link, NULL);
 	}
 
 	ret = rtw89_iterate_mcc_roles(rtwdev, rtw89_mcc_fill_role_iterator, &sel);
@@ -928,6 +1014,15 @@ static int rtw89_mcc_fill_all_roles(struct rtw89_dev *rtwdev)
 	return 0;
 }
 
+static bool rtw89_mcc_can_courtesy(const struct rtw89_mcc_role *provider,
+				   const struct rtw89_mcc_role *receiver)
+{
+	if (provider->is_go || receiver->is_gc)
+		return false;
+
+	return true;
+}
+
 static void rtw89_mcc_assign_pattern(struct rtw89_dev *rtwdev,
 				     const struct rtw89_mcc_pattern *new)
 {
@@ -936,37 +1031,54 @@ static void rtw89_mcc_assign_pattern(struct rtw89_dev *rtwdev,
 	struct rtw89_mcc_role *aux = &mcc->role_aux;
 	struct rtw89_mcc_config *config = &mcc->config;
 	struct rtw89_mcc_pattern *pattern = &config->pattern;
+	struct rtw89_mcc_courtesy_cfg *crtz;
 
 	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
 		    "MCC assign pattern: ref {%d | %d}, aux {%d | %d}\n",
 		    new->tob_ref, new->toa_ref, new->tob_aux, new->toa_aux);
 
+	rtw89_debug(rtwdev, RTW89_DBG_CHAN, "MCC pattern plan: %d\n", new->plan);
+
 	*pattern = *new;
 	memset(&pattern->courtesy, 0, sizeof(pattern->courtesy));
 
-	if (pattern->tob_aux <= 0 || pattern->toa_aux <= 0) {
-		pattern->courtesy.macid_tgt = aux->rtwvif_link->mac_id;
-		pattern->courtesy.macid_src = ref->rtwvif_link->mac_id;
-		pattern->courtesy.slot_num = RTW89_MCC_DFLT_COURTESY_SLOT;
-		pattern->courtesy.enable = true;
-	} else if (pattern->tob_ref <= 0 || pattern->toa_ref <= 0) {
-		pattern->courtesy.macid_tgt = ref->rtwvif_link->mac_id;
-		pattern->courtesy.macid_src = aux->rtwvif_link->mac_id;
-		pattern->courtesy.slot_num = RTW89_MCC_DFLT_COURTESY_SLOT;
-		pattern->courtesy.enable = true;
+	if (RTW89_MCC_REQ_COURTESY(pattern, aux) && aux->is_gc)
+		aux->ignore_bcn = true;
+	else
+		aux->ignore_bcn = false;
+
+	if (RTW89_MCC_REQ_COURTESY(pattern, aux) && rtw89_mcc_can_courtesy(ref, aux)) {
+		crtz = &pattern->courtesy.ref;
+		ref->crtz = crtz;
+
+		crtz->macid_tgt = aux->rtwvif_link->mac_id;
+		crtz->slot_num = RTW89_MCC_DFLT_COURTESY_SLOT;
+
+		rtw89_debug(rtwdev, RTW89_DBG_CHAN,
+			    "MCC courtesy ref: tgt %d, slot %d\n",
+			    crtz->macid_tgt, crtz->slot_num);
+	} else {
+		ref->crtz = NULL;
 	}
 
-	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
-		    "MCC pattern flags: plan %d, courtesy_en %d\n",
-		    pattern->plan, pattern->courtesy.enable);
+	if (RTW89_MCC_REQ_COURTESY(pattern, ref) && ref->is_gc)
+		ref->ignore_bcn = true;
+	else
+		ref->ignore_bcn = false;
 
-	if (!pattern->courtesy.enable)
-		return;
+	if (RTW89_MCC_REQ_COURTESY(pattern, ref) && rtw89_mcc_can_courtesy(aux, ref)) {
+		crtz = &pattern->courtesy.aux;
+		aux->crtz = crtz;
 
-	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
-		    "MCC pattern courtesy: tgt %d, src %d, slot %d\n",
-		    pattern->courtesy.macid_tgt, pattern->courtesy.macid_src,
-		    pattern->courtesy.slot_num);
+		crtz->macid_tgt = ref->rtwvif_link->mac_id;
+		crtz->slot_num = RTW89_MCC_DFLT_COURTESY_SLOT;
+
+		rtw89_debug(rtwdev, RTW89_DBG_CHAN,
+			    "MCC courtesy aux: tgt %d, slot %d\n",
+			    crtz->macid_tgt, crtz->slot_num);
+	} else {
+		aux->crtz = NULL;
+	}
 }
 
 /* The follow-up roughly shows the relationship between the parameters
@@ -991,6 +1103,7 @@ static void __rtw89_mcc_calc_pattern_loose(struct rtw89_dev *rtwdev,
 	struct rtw89_mcc_role *ref = &mcc->role_ref;
 	struct rtw89_mcc_role *aux = &mcc->role_aux;
 	struct rtw89_mcc_config *config = &mcc->config;
+	u16 mcc_intvl = config->mcc_interval;
 	u16 bcn_ofst = config->beacon_offset;
 	u16 bt_dur_in_mid = 0;
 	u16 max_bcn_ofst;
@@ -1024,7 +1137,7 @@ calc:
 
 	res = bcn_ofst - bt_dur_in_mid;
 	upper = min_t(s16, ref->duration, res);
-	lower = 0;
+	lower = max_t(s16, 0, ref->duration - (mcc_intvl - bcn_ofst));
 
 	if (ref->limit.enable) {
 		upper = min_t(s16, upper, ref->limit.max_toa);
@@ -1055,7 +1168,7 @@ static int __rtw89_mcc_calc_pattern_strict(struct rtw89_dev *rtwdev,
 	struct rtw89_mcc_role *ref = &mcc->role_ref;
 	struct rtw89_mcc_role *aux = &mcc->role_aux;
 	struct rtw89_mcc_config *config = &mcc->config;
-	u16 min_tob = RTW89_MCC_EARLY_RX_BCN_TIME;
+	u16 min_tob = RTW89_MCC_EARLY_RX_BCN_TIME + RTW89_MCC_SWITCH_CH_TIME;
 	u16 min_toa = RTW89_MCC_MIN_RX_BCN_TIME;
 	u16 bcn_ofst = config->beacon_offset;
 	s16 upper_toa_ref, lower_toa_ref;
@@ -1135,6 +1248,109 @@ static int __rtw89_mcc_calc_pattern_strict(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static void __rtw89_mcc_fill_ptrn_anchor_ref(struct rtw89_dev *rtwdev,
+					     struct rtw89_mcc_pattern *ptrn,
+					     bool small_bcn_ofst)
+{
+	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
+	struct rtw89_mcc_role *ref = &mcc->role_ref;
+	struct rtw89_mcc_role *aux = &mcc->role_aux;
+	struct rtw89_mcc_config *config = &mcc->config;
+	u16 bcn_ofst = config->beacon_offset;
+	u16 ref_tob;
+	u16 ref_toa;
+
+	if (ref->limit.enable) {
+		ref_tob = ref->limit.max_tob;
+		ref_toa = ref->limit.max_toa;
+	} else {
+		ref_tob = ref->duration / 2;
+		ref_toa = ref->duration / 2;
+	}
+
+	if (small_bcn_ofst) {
+		ptrn->toa_ref = ref_toa;
+		ptrn->tob_ref = ref->duration - ptrn->toa_ref;
+	} else {
+		ptrn->tob_ref = ref_tob;
+		ptrn->toa_ref = ref->duration - ptrn->tob_ref;
+	}
+
+	ptrn->tob_aux = bcn_ofst - ptrn->toa_ref;
+	ptrn->toa_aux = aux->duration - ptrn->tob_aux;
+}
+
+static void __rtw89_mcc_fill_ptrn_anchor_aux(struct rtw89_dev *rtwdev,
+					     struct rtw89_mcc_pattern *ptrn,
+					     bool small_bcn_ofst)
+{
+	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
+	struct rtw89_mcc_role *ref = &mcc->role_ref;
+	struct rtw89_mcc_role *aux = &mcc->role_aux;
+	struct rtw89_mcc_config *config = &mcc->config;
+	u16 bcn_ofst = config->beacon_offset;
+	u16 aux_tob;
+	u16 aux_toa;
+
+	if (aux->limit.enable) {
+		aux_tob = aux->limit.max_tob;
+		aux_toa = aux->limit.max_toa;
+	} else {
+		aux_tob = aux->duration / 2;
+		aux_toa = aux->duration / 2;
+	}
+
+	if (small_bcn_ofst) {
+		ptrn->tob_aux = aux_tob;
+		ptrn->toa_aux = aux->duration - ptrn->tob_aux;
+	} else {
+		ptrn->toa_aux = aux_toa;
+		ptrn->tob_aux = aux->duration - ptrn->toa_aux;
+	}
+
+	ptrn->toa_ref = bcn_ofst - ptrn->tob_aux;
+	ptrn->tob_ref = ref->duration - ptrn->toa_ref;
+}
+
+static int __rtw89_mcc_calc_pattern_anchor(struct rtw89_dev *rtwdev,
+					   struct rtw89_mcc_pattern *ptrn,
+					   bool hdl_bt)
+{
+	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
+	struct rtw89_mcc_role *ref = &mcc->role_ref;
+	struct rtw89_mcc_role *aux = &mcc->role_aux;
+	struct rtw89_mcc_config *config = &mcc->config;
+	u16 mcc_intvl = config->mcc_interval;
+	u16 bcn_ofst = config->beacon_offset;
+	bool small_bcn_ofst;
+
+	if (bcn_ofst < RTW89_MCC_MIN_RX_BCN_WITH_SWITCH_CH_TIME)
+		small_bcn_ofst = true;
+	else if (bcn_ofst < aux->duration - aux->limit.max_toa)
+		small_bcn_ofst = true;
+	else if (mcc_intvl - bcn_ofst < RTW89_MCC_MIN_RX_BCN_WITH_SWITCH_CH_TIME)
+		small_bcn_ofst = false;
+	else
+		return -EPERM;
+
+	*ptrn = (typeof(*ptrn)){
+		.plan = hdl_bt ? RTW89_MCC_PLAN_TAIL_BT : RTW89_MCC_PLAN_NO_BT,
+	};
+
+	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
+		    "MCC calc ptrn_ac: plan %d, bcn_ofst %d\n",
+		    ptrn->plan, bcn_ofst);
+
+	if (ref->is_go || ref->is_gc)
+		__rtw89_mcc_fill_ptrn_anchor_ref(rtwdev, ptrn, small_bcn_ofst);
+	else if (aux->is_go || aux->is_gc)
+		__rtw89_mcc_fill_ptrn_anchor_aux(rtwdev, ptrn, small_bcn_ofst);
+	else
+		__rtw89_mcc_fill_ptrn_anchor_ref(rtwdev, ptrn, small_bcn_ofst);
+
+	return 0;
+}
+
 static int rtw89_mcc_calc_pattern(struct rtw89_dev *rtwdev, bool hdl_bt)
 {
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
@@ -1188,6 +1404,10 @@ static int rtw89_mcc_calc_pattern(struct rtw89_dev *rtwdev, bool hdl_bt)
 			goto done;
 	}
 
+	ret = __rtw89_mcc_calc_pattern_anchor(rtwdev, &ptrn, hdl_bt);
+	if (!ret)
+		goto done;
+
 	__rtw89_mcc_calc_pattern_loose(rtwdev, &ptrn, hdl_bt);
 
 done:
@@ -1438,88 +1658,72 @@ static bool rtw89_mcc_duration_decision_on_bt(struct rtw89_dev *rtwdev)
 	return false;
 }
 
-static void rtw89_mcc_sync_tbtt(struct rtw89_dev *rtwdev,
-				struct rtw89_mcc_role *tgt,
-				struct rtw89_mcc_role *src,
-				bool ref_is_src)
+void rtw89_mcc_prepare_done_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
-	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
-	struct rtw89_mcc_config *config = &mcc->config;
-	u16 beacon_offset_us = ieee80211_tu_to_usec(config->beacon_offset);
-	u32 bcn_intvl_src_us = ieee80211_tu_to_usec(src->beacon_interval);
-	u32 cur_tbtt_ofst_src;
-	u32 tsf_ofst_tgt;
-	u32 remainder;
-	u64 tbtt_tgt;
-	u64 tsf_src;
-	int ret;
-
-	ret = rtw89_mac_port_get_tsf(rtwdev, src->rtwvif_link, &tsf_src);
-	if (ret) {
-		rtw89_warn(rtwdev, "MCC failed to get port tsf: %d\n", ret);
-		return;
-	}
-
-	cur_tbtt_ofst_src = rtw89_mcc_get_tbtt_ofst(rtwdev, src, tsf_src);
+	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
+						mcc_prepare_done_work.work);
 
-	if (ref_is_src)
-		tbtt_tgt = tsf_src - cur_tbtt_ofst_src + beacon_offset_us;
-	else
-		tbtt_tgt = tsf_src - cur_tbtt_ofst_src +
-			   (bcn_intvl_src_us - beacon_offset_us);
+	lockdep_assert_wiphy(wiphy);
 
-	div_u64_rem(tbtt_tgt, bcn_intvl_src_us, &remainder);
-	tsf_ofst_tgt = bcn_intvl_src_us - remainder;
+	ieee80211_wake_queues(rtwdev->hw);
+}
 
-	config->sync.macid_tgt = tgt->rtwvif_link->mac_id;
-	config->sync.band_tgt = tgt->rtwvif_link->mac_idx;
-	config->sync.port_tgt = tgt->rtwvif_link->port;
-	config->sync.macid_src = src->rtwvif_link->mac_id;
-	config->sync.band_src = src->rtwvif_link->mac_idx;
-	config->sync.port_src = src->rtwvif_link->port;
-	config->sync.offset = tsf_ofst_tgt / 1024;
-	config->sync.enable = true;
+static void rtw89_mcc_prepare(struct rtw89_dev *rtwdev, bool start)
+{
+	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
+	struct rtw89_mcc_config *config = &mcc->config;
 
-	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
-		    "MCC sync tbtt: tgt %d, src %d, offset %d\n",
-		    config->sync.macid_tgt, config->sync.macid_src,
-		    config->sync.offset);
+	if (start) {
+		ieee80211_stop_queues(rtwdev->hw);
 
-	rtw89_mac_port_tsf_sync(rtwdev, tgt->rtwvif_link, src->rtwvif_link,
-				config->sync.offset);
+		wiphy_delayed_work_queue(rtwdev->hw->wiphy,
+					 &rtwdev->mcc_prepare_done_work,
+					 usecs_to_jiffies(config->prepare_delay));
+	} else {
+		wiphy_delayed_work_queue(rtwdev->hw->wiphy,
+					 &rtwdev->mcc_prepare_done_work, 0);
+		wiphy_delayed_work_flush(rtwdev->hw->wiphy,
+					 &rtwdev->mcc_prepare_done_work);
+	}
 }
 
 static int rtw89_mcc_fill_start_tsf(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
 	struct rtw89_mcc_role *ref = &mcc->role_ref;
+	struct rtw89_mcc_role *aux = &mcc->role_aux;
 	struct rtw89_mcc_config *config = &mcc->config;
 	u32 bcn_intvl_ref_us = ieee80211_tu_to_usec(ref->beacon_interval);
-	u32 tob_ref_us = ieee80211_tu_to_usec(config->pattern.tob_ref);
-	struct rtw89_vif_link *rtwvif_link = ref->rtwvif_link;
+	s32 tob_ref_us = ieee80211_tu_to_usec(config->pattern.tob_ref);
 	u64 tsf, start_tsf;
 	u32 cur_tbtt_ofst;
 	u64 min_time;
+	u64 tsf_aux;
 	int ret;
 
-	ret = rtw89_mac_port_get_tsf(rtwdev, rtwvif_link, &tsf);
-	if (ret) {
-		rtw89_warn(rtwdev, "MCC failed to get port tsf: %d\n", ret);
+	if (rtw89_concurrent_via_mrc(rtwdev))
+		ret = __mrc_fw_req_tsf(rtwdev, &tsf, &tsf_aux);
+	else
+		ret = __mcc_fw_req_tsf(rtwdev, &tsf, &tsf_aux);
+
+	if (ret)
 		return ret;
-	}
 
 	min_time = tsf;
-	if (ref->is_go)
+	if (ref->is_go || aux->is_go)
 		min_time += ieee80211_tu_to_usec(RTW89_MCC_SHORT_TRIGGER_TIME);
 	else
 		min_time += ieee80211_tu_to_usec(RTW89_MCC_LONG_TRIGGER_TIME);
 
 	cur_tbtt_ofst = rtw89_mcc_get_tbtt_ofst(rtwdev, ref, tsf);
 	start_tsf = tsf - cur_tbtt_ofst + bcn_intvl_ref_us - tob_ref_us;
-	while (start_tsf < min_time)
-		start_tsf += bcn_intvl_ref_us;
+	if (start_tsf < min_time)
+		start_tsf += roundup_u64(min_time - start_tsf, bcn_intvl_ref_us);
 
 	config->start_tsf = start_tsf;
+	config->start_tsf_in_aux_domain = tsf_aux + start_tsf - tsf;
+	config->prepare_delay = start_tsf - tsf;
+
 	return 0;
 }
 
@@ -1536,13 +1740,11 @@ static int rtw89_mcc_fill_config(struct rtw89_dev *rtwdev)
 
 	switch (mcc->mode) {
 	case RTW89_MCC_MODE_GO_STA:
-		config->beacon_offset = RTW89_MCC_DFLT_BCN_OFST_TIME;
+		config->beacon_offset = rtw89_mcc_get_bcn_ofst(rtwdev);
 		if (ref->is_go) {
-			rtw89_mcc_sync_tbtt(rtwdev, ref, aux, false);
 			config->mcc_interval = ref->beacon_interval;
 			rtw89_mcc_set_duration_go_sta(rtwdev, ref, aux);
 		} else {
-			rtw89_mcc_sync_tbtt(rtwdev, aux, ref, true);
 			config->mcc_interval = aux->beacon_interval;
 			rtw89_mcc_set_duration_go_sta(rtwdev, aux, ref);
 		}
@@ -1572,10 +1774,8 @@ bottom:
 
 static int __mcc_fw_add_role(struct rtw89_dev *rtwdev, struct rtw89_mcc_role *role)
 {
+	const struct rtw89_mcc_courtesy_cfg *crtz = role->crtz;
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
-	struct rtw89_mcc_config *config = &mcc->config;
-	struct rtw89_mcc_pattern *pattern = &config->pattern;
-	struct rtw89_mcc_courtesy *courtesy = &pattern->courtesy;
 	struct rtw89_mcc_policy *policy = &role->policy;
 	struct rtw89_fw_mcc_add_req req = {};
 	const struct rtw89_chan *chan;
@@ -1598,9 +1798,9 @@ static int __mcc_fw_add_role(struct rtw89_dev *rtwdev, struct rtw89_mcc_role *ro
 	req.duration = role->duration;
 	req.btc_in_2g = false;
 
-	if (courtesy->enable && courtesy->macid_src == req.macid) {
-		req.courtesy_target = courtesy->macid_tgt;
-		req.courtesy_num = courtesy->slot_num;
+	if (crtz) {
+		req.courtesy_target = crtz->macid_tgt;
+		req.courtesy_num = crtz->slot_num;
 		req.courtesy_en = true;
 	}
 
@@ -1780,26 +1980,23 @@ static void __mrc_fw_add_courtesy(struct rtw89_dev *rtwdev,
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
 	struct rtw89_mcc_role *ref = &mcc->role_ref;
 	struct rtw89_mcc_role *aux = &mcc->role_aux;
-	struct rtw89_mcc_config *config = &mcc->config;
-	struct rtw89_mcc_pattern *pattern = &config->pattern;
-	struct rtw89_mcc_courtesy *courtesy = &pattern->courtesy;
 	struct rtw89_fw_mrc_add_slot_arg *slot_arg_src;
-	u8 slot_idx_tgt;
-
-	if (!courtesy->enable)
-		return;
 
-	if (courtesy->macid_src == ref->rtwvif_link->mac_id) {
+	if (ref->crtz) {
 		slot_arg_src = &arg->slots[ref->slot_idx];
-		slot_idx_tgt = aux->slot_idx;
-	} else {
-		slot_arg_src = &arg->slots[aux->slot_idx];
-		slot_idx_tgt = ref->slot_idx;
+
+		slot_arg_src->courtesy_target = aux->slot_idx;
+		slot_arg_src->courtesy_period = ref->crtz->slot_num;
+		slot_arg_src->courtesy_en = true;
 	}
 
-	slot_arg_src->courtesy_target = slot_idx_tgt;
-	slot_arg_src->courtesy_period = courtesy->slot_num;
-	slot_arg_src->courtesy_en = true;
+	if (aux->crtz) {
+		slot_arg_src = &arg->slots[aux->slot_idx];
+
+		slot_arg_src->courtesy_target = ref->slot_idx;
+		slot_arg_src->courtesy_period = aux->crtz->slot_num;
+		slot_arg_src->courtesy_en = true;
+	}
 }
 
 static int __mrc_fw_start(struct rtw89_dev *rtwdev, bool replace)
@@ -1999,30 +2196,24 @@ static void rtw89_mcc_handle_beacon_noa(struct rtw89_dev *rtwdev, bool enable)
 	struct rtw89_mcc_role *ref = &mcc->role_ref;
 	struct rtw89_mcc_role *aux = &mcc->role_aux;
 	struct rtw89_mcc_config *config = &mcc->config;
-	struct rtw89_mcc_pattern *pattern = &config->pattern;
-	struct rtw89_mcc_sync *sync = &config->sync;
 	struct ieee80211_p2p_noa_desc noa_desc = {};
-	u64 start_time = config->start_tsf;
 	u32 interval = config->mcc_interval;
 	struct rtw89_vif_link *rtwvif_go;
+	u64 start_time;
 	u32 duration;
 
 	if (mcc->mode != RTW89_MCC_MODE_GO_STA)
 		return;
 
 	if (ref->is_go) {
+		start_time = config->start_tsf;
 		rtwvif_go = ref->rtwvif_link;
 		start_time += ieee80211_tu_to_usec(ref->duration);
 		duration = config->mcc_interval - ref->duration;
 	} else if (aux->is_go) {
+		start_time = config->start_tsf_in_aux_domain;
 		rtwvif_go = aux->rtwvif_link;
-		start_time += ieee80211_tu_to_usec(pattern->tob_ref) +
-			      ieee80211_tu_to_usec(config->beacon_offset) +
-			      ieee80211_tu_to_usec(pattern->toa_aux);
 		duration = config->mcc_interval - aux->duration;
-
-		/* convert time domain from sta(ref) to GO(aux) */
-		start_time += ieee80211_tu_to_usec(sync->offset);
 	} else {
 		rtw89_debug(rtwdev, RTW89_DBG_CHAN,
 			    "MCC find no GO: skip updating beacon NoA\n");
@@ -2032,6 +2223,7 @@ static void rtw89_mcc_handle_beacon_noa(struct rtw89_dev *rtwdev, bool enable)
 	rtw89_p2p_noa_renew(rtwvif_go);
 
 	if (enable) {
+		duration += RTW89_MCC_SWITCH_CH_TIME;
 		noa_desc.start_time = cpu_to_le32(start_time);
 		noa_desc.interval = cpu_to_le32(ieee80211_tu_to_usec(interval));
 		noa_desc.duration = cpu_to_le32(ieee80211_tu_to_usec(duration));
@@ -2080,6 +2272,18 @@ static void rtw89_mcc_stop_beacon_noa(struct rtw89_dev *rtwdev)
 	rtw89_mcc_handle_beacon_noa(rtwdev, false);
 }
 
+static bool rtw89_mcc_ignore_bcn(struct rtw89_dev *rtwdev, struct rtw89_mcc_role *role)
+{
+	enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
+
+	if (role->is_go)
+		return true;
+	else if (chip_gen == RTW89_CHIP_BE && role->is_gc)
+		return true;
+	else
+		return false;
+}
+
 static int rtw89_mcc_start(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
@@ -2111,6 +2315,15 @@ static int rtw89_mcc_start(struct rtw89_dev *rtwdev)
 	if (ret)
 		return ret;
 
+	if (rtw89_mcc_ignore_bcn(rtwdev, ref) || aux->ignore_bcn) {
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, aux->rtwvif_link, false);
+	} else if (rtw89_mcc_ignore_bcn(rtwdev, aux) || ref->ignore_bcn) {
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, ref->rtwvif_link, false);
+	} else {
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, ref->rtwvif_link, true);
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, aux->rtwvif_link, true);
+	}
+
 	if (rtw89_concurrent_via_mrc(rtwdev))
 		ret = __mrc_fw_start(rtwdev, false);
 	else
@@ -2122,10 +2335,19 @@ static int rtw89_mcc_start(struct rtw89_dev *rtwdev)
 	rtw89_chanctx_notify(rtwdev, RTW89_CHANCTX_STATE_MCC_START);
 
 	rtw89_mcc_start_beacon_noa(rtwdev);
+	rtw89_phy_dig_suspend(rtwdev);
+
+	rtw89_mcc_prepare(rtwdev, true);
 	return 0;
 }
 
 struct rtw89_mcc_stop_sel {
+	struct {
+		const struct rtw89_vif_link *target;
+	} hint;
+
+	/* selection content */
+	bool filled;
 	u8 mac_id;
 	u8 slot_idx;
 };
@@ -2135,6 +2357,7 @@ static void rtw89_mcc_stop_sel_fill(struct rtw89_mcc_stop_sel *sel,
 {
 	sel->mac_id = mcc_role->rtwvif_link->mac_id;
 	sel->slot_idx = mcc_role->slot_idx;
+	sel->filled = true;
 }
 
 static int rtw89_mcc_stop_sel_iterator(struct rtw89_dev *rtwdev,
@@ -2144,23 +2367,49 @@ static int rtw89_mcc_stop_sel_iterator(struct rtw89_dev *rtwdev,
 {
 	struct rtw89_mcc_stop_sel *sel = data;
 
+	if (mcc_role->rtwvif_link == sel->hint.target) {
+		rtw89_mcc_stop_sel_fill(sel, mcc_role);
+		return 1; /* break iteration */
+	}
+
+	if (sel->filled)
+		return 0;
+
 	if (!mcc_role->rtwvif_link->chanctx_assigned)
 		return 0;
 
 	rtw89_mcc_stop_sel_fill(sel, mcc_role);
-	return 1; /* break iteration */
+	return 0;
 }
 
-static void rtw89_mcc_stop(struct rtw89_dev *rtwdev)
+static void rtw89_mcc_stop(struct rtw89_dev *rtwdev,
+			   const struct rtw89_chanctx_pause_parm *pause)
 {
+	struct rtw89_hal *hal = &rtwdev->hal;
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
 	struct rtw89_mcc_role *ref = &mcc->role_ref;
-	struct rtw89_mcc_stop_sel sel;
+	struct rtw89_mcc_role *aux = &mcc->role_aux;
+	struct rtw89_mcc_stop_sel sel = {
+		.hint.target = pause ? pause->trigger : NULL,
+	};
+	bool rsn_scan;
 	int ret;
 
+	if (!pause) {
+		wiphy_delayed_work_cancel(rtwdev->hw->wiphy, &rtwdev->chanctx_work);
+		bitmap_zero(hal->changes, NUM_OF_RTW89_CHANCTX_CHANGES);
+	}
+
+	rsn_scan = pause && pause->rsn == RTW89_CHANCTX_PAUSE_REASON_HW_SCAN;
+	if (rsn_scan && ref->is_go)
+		sel.hint.target = ref->rtwvif_link;
+	else if (rsn_scan && aux->is_go)
+		sel.hint.target = aux->rtwvif_link;
+
 	/* by default, stop at ref */
-	rtw89_mcc_stop_sel_fill(&sel, ref);
 	rtw89_iterate_mcc_roles(rtwdev, rtw89_mcc_stop_sel_iterator, &sel);
+	if (!sel.filled)
+		rtw89_mcc_stop_sel_fill(&sel, ref);
 
 	rtw89_debug(rtwdev, RTW89_DBG_CHAN, "MCC stop at <macid %d>\n", sel.mac_id);
 
@@ -2185,13 +2434,22 @@ static void rtw89_mcc_stop(struct rtw89_dev *rtwdev)
 	rtw89_chanctx_notify(rtwdev, RTW89_CHANCTX_STATE_MCC_STOP);
 
 	rtw89_mcc_stop_beacon_noa(rtwdev);
+	rtw89_fw_h2c_mcc_dig(rtwdev, RTW89_CHANCTX_0, 0, 0, false);
+	rtw89_phy_dig_resume(rtwdev, true);
+
+	rtw89_mcc_prepare(rtwdev, false);
 }
 
 static int rtw89_mcc_update(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
+	bool old_ref_ignore_bcn = mcc->role_ref.ignore_bcn;
+	bool old_aux_ignore_bcn = mcc->role_aux.ignore_bcn;
 	struct rtw89_mcc_config *config = &mcc->config;
+	struct rtw89_mcc_role *ref = &mcc->role_ref;
+	struct rtw89_mcc_role *aux = &mcc->role_aux;
 	struct rtw89_mcc_config old_cfg = *config;
+	bool courtesy_changed;
 	bool sync_changed;
 	int ret;
 
@@ -2204,8 +2462,20 @@ static int rtw89_mcc_update(struct rtw89_dev *rtwdev)
 	if (ret)
 		return ret;
 
+	if (old_ref_ignore_bcn != ref->ignore_bcn)
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, ref->rtwvif_link, !ref->ignore_bcn);
+	else if (old_aux_ignore_bcn != aux->ignore_bcn)
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, aux->rtwvif_link, !aux->ignore_bcn);
+
+	if (memcmp(&old_cfg.pattern.courtesy, &config->pattern.courtesy,
+		   sizeof(old_cfg.pattern.courtesy)) == 0)
+		courtesy_changed = false;
+	else
+		courtesy_changed = true;
+
 	if (old_cfg.pattern.plan != RTW89_MCC_PLAN_NO_BT ||
-	    config->pattern.plan != RTW89_MCC_PLAN_NO_BT) {
+	    config->pattern.plan != RTW89_MCC_PLAN_NO_BT ||
+	    courtesy_changed) {
 		if (rtw89_concurrent_via_mrc(rtwdev))
 			ret = __mrc_fw_start(rtwdev, true);
 		else
@@ -2232,31 +2502,167 @@ static int rtw89_mcc_update(struct rtw89_dev *rtwdev)
 	return 0;
 }
 
+static int rtw89_mcc_search_gc_iterator(struct rtw89_dev *rtwdev,
+					struct rtw89_mcc_role *mcc_role,
+					unsigned int ordered_idx,
+					void *data)
+{
+	struct rtw89_mcc_role **role = data;
+
+	if (mcc_role->is_gc)
+		*role = mcc_role;
+
+	return 0;
+}
+
+static struct rtw89_mcc_role *rtw89_mcc_get_gc_role(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
+	struct rtw89_mcc_role *role = NULL;
+
+	if (mcc->mode != RTW89_MCC_MODE_GC_STA)
+		return NULL;
+
+	rtw89_iterate_mcc_roles(rtwdev, rtw89_mcc_search_gc_iterator, &role);
+
+	return role;
+}
+
+void rtw89_mcc_gc_detect_beacon_work(struct wiphy *wiphy, struct wiphy_work *work)
+{
+	struct rtw89_vif_link *rtwvif_link = container_of(work, struct rtw89_vif_link,
+							  mcc_gc_detect_beacon_work.work);
+	struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link);
+	enum rtw89_entity_mode mode;
+	struct rtw89_dev *rtwdev;
+
+	lockdep_assert_wiphy(wiphy);
+
+	rtwdev = rtwvif_link->rtwvif->rtwdev;
+
+	mode = rtw89_get_entity_mode(rtwdev);
+	if (mode != RTW89_ENTITY_MODE_MCC)
+		return;
+
+	if (READ_ONCE(rtwvif_link->sync_bcn_tsf) > rtwvif_link->last_sync_bcn_tsf)
+		rtwvif_link->detect_bcn_count = 0;
+	else
+		rtwvif_link->detect_bcn_count++;
+
+	if (rtwvif_link->detect_bcn_count < RTW89_MCC_DETECT_BCN_MAX_TRIES)
+		rtw89_chanctx_proceed(rtwdev, NULL);
+	else
+		ieee80211_connection_loss(vif);
+}
+
+bool rtw89_mcc_detect_go_bcn(struct rtw89_dev *rtwdev,
+			     struct rtw89_vif_link *rtwvif_link)
+{
+	enum rtw89_entity_mode mode = rtw89_get_entity_mode(rtwdev);
+	struct rtw89_chanctx_pause_parm pause_parm = {
+		.rsn = RTW89_CHANCTX_PAUSE_REASON_GC_BCN_LOSS,
+		.trigger = rtwvif_link,
+	};
+	struct ieee80211_bss_conf *bss_conf;
+	struct rtw89_mcc_role *role;
+	u16 bcn_int;
+
+	if (mode != RTW89_ENTITY_MODE_MCC)
+		return false;
+
+	role = rtw89_mcc_get_gc_role(rtwdev);
+	if (!role)
+		return false;
+
+	if (role->rtwvif_link != rtwvif_link)
+		return false;
+
+	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
+		    "MCC GC beacon loss, pause MCC to detect GO beacon\n");
+
+	rcu_read_lock();
+
+	bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
+	bcn_int = bss_conf->beacon_int;
+
+	rcu_read_unlock();
+
+	rtw89_chanctx_pause(rtwdev, &pause_parm);
+	rtwvif_link->last_sync_bcn_tsf = READ_ONCE(rtwvif_link->sync_bcn_tsf);
+	wiphy_delayed_work_queue(rtwdev->hw->wiphy,
+				 &rtwvif_link->mcc_gc_detect_beacon_work,
+				 usecs_to_jiffies(ieee80211_tu_to_usec(bcn_int)));
+
+	return true;
+}
+
+static void rtw89_mcc_detect_connection(struct rtw89_dev *rtwdev,
+					struct rtw89_mcc_role *role)
+{
+	struct ieee80211_vif *vif;
+	bool start_detect;
+	int ret;
+
+	ret = rtw89_core_send_nullfunc(rtwdev, role->rtwvif_link, true, false,
+				       RTW89_MCC_PROBE_TIMEOUT);
+	if (ret)
+		role->probe_count++;
+	else
+		role->probe_count = 0;
+
+	if (role->probe_count < RTW89_MCC_PROBE_MAX_TRIES)
+		return;
+
+	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
+		    "MCC <macid %d> can not detect AP/GO\n", role->rtwvif_link->mac_id);
+
+	start_detect = rtw89_mcc_detect_go_bcn(rtwdev, role->rtwvif_link);
+	if (start_detect)
+		return;
+
+	vif = rtwvif_link_to_vif(role->rtwvif_link);
+	ieee80211_connection_loss(vif);
+}
+
 static void rtw89_mcc_track(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_mcc_info *mcc = &rtwdev->mcc;
 	struct rtw89_mcc_config *config = &mcc->config;
 	struct rtw89_mcc_pattern *pattern = &config->pattern;
-	s16 tolerance;
+	struct rtw89_mcc_role *ref = &mcc->role_ref;
+	struct rtw89_mcc_role *aux = &mcc->role_aux;
+	u16 tolerance;
 	u16 bcn_ofst;
 	u16 diff;
 
+	if (rtw89_mcc_ignore_bcn(rtwdev, ref) || aux->ignore_bcn)
+		rtw89_mcc_detect_connection(rtwdev, aux);
+	else if (rtw89_mcc_ignore_bcn(rtwdev, aux) || ref->ignore_bcn)
+		rtw89_mcc_detect_connection(rtwdev, ref);
+
 	if (mcc->mode != RTW89_MCC_MODE_GC_STA)
 		return;
 
 	bcn_ofst = rtw89_mcc_get_bcn_ofst(rtwdev);
+	if (bcn_ofst == config->beacon_offset)
+		return;
+
 	if (bcn_ofst > config->beacon_offset) {
 		diff = bcn_ofst - config->beacon_offset;
 		if (pattern->tob_aux < 0)
 			tolerance = -pattern->tob_aux;
-		else
+		else if (pattern->toa_aux > 0)
 			tolerance = pattern->toa_aux;
+		else
+			return; /* no chance to improve */
 	} else {
 		diff = config->beacon_offset - bcn_ofst;
 		if (pattern->toa_aux < 0)
 			tolerance = -pattern->toa_aux;
-		else
+		else if (pattern->tob_aux > 0)
 			tolerance = pattern->tob_aux;
+		else
+			return; /* no chance to improve */
 	}
 
 	if (diff <= tolerance)
@@ -2390,7 +2796,31 @@ static void rtw89_mcc_update_limit(struct rtw89_dev *rtwdev)
 	rtw89_iterate_mcc_roles(rtwdev, rtw89_mcc_upd_lmt_iterator, NULL);
 }
 
-void rtw89_chanctx_work(struct work_struct *work)
+static int rtw89_mcc_get_links_iterator(struct rtw89_dev *rtwdev,
+					struct rtw89_mcc_role *mcc_role,
+					unsigned int ordered_idx,
+					void *data)
+{
+	struct rtw89_mcc_links_info *info = data;
+
+	info->links[ordered_idx] = mcc_role->rtwvif_link;
+	return 0;
+}
+
+void rtw89_mcc_get_links(struct rtw89_dev *rtwdev, struct rtw89_mcc_links_info *info)
+{
+	enum rtw89_entity_mode mode;
+
+	memset(info, 0, sizeof(*info));
+
+	mode = rtw89_get_entity_mode(rtwdev);
+	if (unlikely(mode != RTW89_ENTITY_MODE_MCC))
+		return;
+
+	rtw89_iterate_mcc_roles(rtwdev, rtw89_mcc_get_links_iterator, info);
+}
+
+void rtw89_chanctx_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						chanctx_work.work);
@@ -2401,12 +2831,10 @@ void rtw89_chanctx_work(struct work_struct *work)
 	int ret;
 	int i;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
 
-	if (hal->entity_pause) {
-		mutex_unlock(&rtwdev->mutex);
+	if (hal->entity_pause)
 		return;
-	}
 
 	for (i = 0; i < NUM_OF_RTW89_CHANCTX_CHANGES; i++) {
 		if (test_and_clear_bit(i, hal->changes))
@@ -2445,8 +2873,6 @@ void rtw89_chanctx_work(struct work_struct *work)
 	default:
 		break;
 	}
-
-	mutex_unlock(&rtwdev->mutex);
 }
 
 void rtw89_queue_chanctx_change(struct rtw89_dev *rtwdev,
@@ -2462,6 +2888,7 @@ void rtw89_queue_chanctx_change(struct rtw89_dev *rtwdev,
 		return;
 	case RTW89_ENTITY_MODE_MCC_PREPARE:
 		delay = ieee80211_tu_to_usec(RTW89_CHANCTX_TIME_MCC_PREPARE);
+		rtw89_phy_dig_suspend(rtwdev);
 		break;
 	case RTW89_ENTITY_MODE_MCC:
 		delay = ieee80211_tu_to_usec(RTW89_CHANCTX_TIME_MCC);
@@ -2477,8 +2904,8 @@ void rtw89_queue_chanctx_change(struct rtw89_dev *rtwdev,
 	rtw89_debug(rtwdev, RTW89_DBG_CHAN,
 		    "queue chanctx work for mode %d with delay %d us\n",
 		    mode, delay);
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->chanctx_work,
-				     usecs_to_jiffies(delay));
+	wiphy_delayed_work_queue(rtwdev->hw->wiphy, &rtwdev->chanctx_work,
+				 usecs_to_jiffies(delay));
 }
 
 void rtw89_queue_chanctx_work(struct rtw89_dev *rtwdev)
@@ -2486,12 +2913,207 @@ void rtw89_queue_chanctx_work(struct rtw89_dev *rtwdev)
 	rtw89_queue_chanctx_change(rtwdev, RTW89_CHANCTX_CHANGE_DFLT);
 }
 
+static enum rtw89_mr_wtype __rtw89_query_mr_wtype(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_entity_mgnt *mgnt = &rtwdev->hal.entity_mgnt;
+	enum rtw89_chanctx_idx chanctx_idx;
+	struct ieee80211_vif *vif;
+	struct rtw89_vif *rtwvif;
+	unsigned int num_mld = 0;
+	unsigned int num_ml = 0;
+	unsigned int cnt = 0;
+	u8 role_idx;
+	u8 idx;
+
+	for (role_idx = 0; role_idx < RTW89_MAX_INTERFACE_NUM; role_idx++) {
+		rtwvif = mgnt->active_roles[role_idx];
+		if (!rtwvif)
+			continue;
+
+		cnt++;
+
+		vif = rtwvif_to_vif(rtwvif);
+		if (!ieee80211_vif_is_mld(vif))
+			continue;
+
+		num_mld++;
+
+		for (idx = 0; idx < __RTW89_MLD_MAX_LINK_NUM; idx++) {
+			chanctx_idx = mgnt->chanctx_tbl[role_idx][idx];
+			if (chanctx_idx != RTW89_CHANCTX_IDLE)
+				num_ml++;
+		}
+	}
+
+	if (num_mld > 1)
+		goto err;
+
+	switch (cnt) {
+	case 0:
+		return RTW89_MR_WTYPE_NONE;
+	case 1:
+		if (!num_mld)
+			return RTW89_MR_WTYPE_NONMLD;
+		switch (num_ml) {
+		case 1:
+			return RTW89_MR_WTYPE_MLD1L1R;
+		case 2:
+			return RTW89_MR_WTYPE_MLD2L1R;
+		default:
+			break;
+		}
+		break;
+	case 2:
+		if (!num_mld)
+			return RTW89_MR_WTYPE_NONMLD_NONMLD;
+		switch (num_ml) {
+		case 1:
+			return RTW89_MR_WTYPE_MLD1L1R_NONMLD;
+		case 2:
+			return RTW89_MR_WTYPE_MLD2L1R_NONMLD;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+
+err:
+	rtw89_warn(rtwdev, "%s: unhandled cnt %u mld %u ml %u\n", __func__,
+		   cnt, num_mld, num_ml);
+	return RTW89_MR_WTYPE_UNKNOWN;
+}
+
+static enum rtw89_mr_wmode __rtw89_query_mr_wmode(struct rtw89_dev *rtwdev,
+						  u8 inst_idx)
+{
+	struct rtw89_entity_mgnt *mgnt = &rtwdev->hal.entity_mgnt;
+	unsigned int num[NUM_NL80211_IFTYPES] = {};
+	enum rtw89_chanctx_idx chanctx_idx;
+	struct ieee80211_vif *vif;
+	struct rtw89_vif *rtwvif;
+	unsigned int cnt = 0;
+	u8 role_idx;
+
+	if (unlikely(inst_idx >= __RTW89_MLD_MAX_LINK_NUM))
+		return RTW89_MR_WMODE_UNKNOWN;
+
+	for (role_idx = 0; role_idx < RTW89_MAX_INTERFACE_NUM; role_idx++) {
+		chanctx_idx = mgnt->chanctx_tbl[role_idx][inst_idx];
+		if (chanctx_idx == RTW89_CHANCTX_IDLE)
+			continue;
+
+		rtwvif = mgnt->active_roles[role_idx];
+		if (unlikely(!rtwvif))
+			continue;
+
+		vif = rtwvif_to_vif(rtwvif);
+		num[vif->type]++;
+		cnt++;
+	}
+
+	switch (cnt) {
+	case 0:
+		return RTW89_MR_WMODE_NONE;
+	case 1:
+		if (num[NL80211_IFTYPE_STATION])
+			return RTW89_MR_WMODE_1CLIENT;
+		if (num[NL80211_IFTYPE_AP])
+			return RTW89_MR_WMODE_1AP;
+		break;
+	case 2:
+		if (num[NL80211_IFTYPE_STATION] == 2)
+			return RTW89_MR_WMODE_2CLIENTS;
+		if (num[NL80211_IFTYPE_AP] == 2)
+			return RTW89_MR_WMODE_2APS;
+		if (num[NL80211_IFTYPE_STATION] && num[NL80211_IFTYPE_AP])
+			return RTW89_MR_WMODE_1AP_1CLIENT;
+		break;
+	default:
+		break;
+	}
+
+	rtw89_warn(rtwdev, "%s: unhandled cnt %u\n", __func__, cnt);
+	return RTW89_MR_WMODE_UNKNOWN;
+}
+
+static enum rtw89_mr_ctxtype __rtw89_query_mr_ctxtype(struct rtw89_dev *rtwdev,
+						      u8 inst_idx)
+{
+	struct rtw89_entity_mgnt *mgnt = &rtwdev->hal.entity_mgnt;
+	DECLARE_BITMAP(map, NUM_OF_RTW89_CHANCTX) = {};
+	unsigned int num[RTW89_BAND_NUM] = {};
+	enum rtw89_chanctx_idx chanctx_idx;
+	const struct rtw89_chan *chan;
+	unsigned int cnt = 0;
+	u8 role_idx;
+
+	if (unlikely(inst_idx >= __RTW89_MLD_MAX_LINK_NUM))
+		return RTW89_MR_CTX_UNKNOWN;
+
+	for (role_idx = 0; role_idx < RTW89_MAX_INTERFACE_NUM; role_idx++) {
+		chanctx_idx = mgnt->chanctx_tbl[role_idx][inst_idx];
+		if (chanctx_idx == RTW89_CHANCTX_IDLE)
+			continue;
+
+		if (__test_and_set_bit(chanctx_idx, map))
+			continue;
+
+		chan = rtw89_chan_get(rtwdev, chanctx_idx);
+		num[chan->band_type]++;
+		cnt++;
+	}
+
+	switch (cnt) {
+	case 0:
+		return RTW89_MR_CTX_NONE;
+	case 1:
+		if (num[RTW89_BAND_2G])
+			return RTW89_MR_CTX1_2GHZ;
+		if (num[RTW89_BAND_5G])
+			return RTW89_MR_CTX1_5GHZ;
+		if (num[RTW89_BAND_6G])
+			return RTW89_MR_CTX1_6GHZ;
+		break;
+	case 2:
+		if (num[RTW89_BAND_2G] == 2)
+			return RTW89_MR_CTX2_2GHZ;
+		if (num[RTW89_BAND_5G] == 2)
+			return RTW89_MR_CTX2_5GHZ;
+		if (num[RTW89_BAND_6G] == 2)
+			return RTW89_MR_CTX2_6GHZ;
+		if (num[RTW89_BAND_2G] && num[RTW89_BAND_5G])
+			return RTW89_MR_CTX2_2GHZ_5GHZ;
+		if (num[RTW89_BAND_2G] && num[RTW89_BAND_6G])
+			return RTW89_MR_CTX2_2GHZ_6GHZ;
+		if (num[RTW89_BAND_5G] && num[RTW89_BAND_6G])
+			return RTW89_MR_CTX2_5GHZ_6GHZ;
+		break;
+	default:
+		break;
+	}
+
+	rtw89_warn(rtwdev, "%s: unhandled cnt %u\n", __func__, cnt);
+	return RTW89_MR_CTX_UNKNOWN;
+}
+
+void rtw89_query_mr_chanctx_info(struct rtw89_dev *rtwdev, u8 inst_idx,
+				 struct rtw89_mr_chanctx_info *info)
+{
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	info->wtype = __rtw89_query_mr_wtype(rtwdev);
+	info->wmode = __rtw89_query_mr_wmode(rtwdev, inst_idx);
+	info->ctxtype = __rtw89_query_mr_ctxtype(rtwdev, inst_idx);
+}
+
 void rtw89_chanctx_track(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_hal *hal = &rtwdev->hal;
 	enum rtw89_entity_mode mode;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (hal->entity_pause)
 		return;
@@ -2507,22 +3129,22 @@ void rtw89_chanctx_track(struct rtw89_dev *rtwdev)
 }
 
 void rtw89_chanctx_pause(struct rtw89_dev *rtwdev,
-			 enum rtw89_chanctx_pause_reasons rsn)
+			 const struct rtw89_chanctx_pause_parm *pause_parm)
 {
 	struct rtw89_hal *hal = &rtwdev->hal;
 	enum rtw89_entity_mode mode;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (hal->entity_pause)
 		return;
 
-	rtw89_debug(rtwdev, RTW89_DBG_CHAN, "chanctx pause (rsn: %d)\n", rsn);
+	rtw89_debug(rtwdev, RTW89_DBG_CHAN, "chanctx pause (rsn: %d)\n", pause_parm->rsn);
 
 	mode = rtw89_get_entity_mode(rtwdev);
 	switch (mode) {
 	case RTW89_ENTITY_MODE_MCC:
-		rtw89_mcc_stop(rtwdev);
+		rtw89_mcc_stop(rtwdev, pause_parm);
 		break;
 	default:
 		break;
@@ -2555,7 +3177,7 @@ void rtw89_chanctx_proceed(struct rtw89_dev *rtwdev,
 	enum rtw89_entity_mode mode;
 	int ret;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (unlikely(!hal->entity_pause)) {
 		rtw89_chanctx_proceed_cb(rtwdev, cb_parm);
@@ -2670,10 +3292,9 @@ int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev,
 void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev,
 			      struct ieee80211_chanctx_conf *ctx)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
 	struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv;
 
-	clear_bit(cfg->idx, hal->entity_map);
+	rtw89_config_entity_chandef(rtwdev, cfg->idx, NULL);
 }
 
 void rtw89_chanctx_ops_change(struct rtw89_dev *rtwdev,
@@ -2687,6 +3308,9 @@ void rtw89_chanctx_ops_change(struct rtw89_dev *rtwdev,
 		rtw89_config_entity_chandef(rtwdev, idx, &ctx->def);
 		rtw89_set_channel(rtwdev);
 	}
+
+	if (changed & IEEE80211_CHANCTX_CHANGE_PUNCTURING)
+		rtw89_chan_update_punctured(rtwdev, idx, &ctx->def);
 }
 
 int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev,
@@ -2698,11 +3322,15 @@ int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev,
 	struct rtw89_hal *hal = &rtwdev->hal;
 	struct rtw89_entity_mgnt *mgnt = &hal->entity_mgnt;
 	struct rtw89_entity_weight w = {};
+	int ret;
 
 	rtwvif_link->chanctx_idx = cfg->idx;
 	rtwvif_link->chanctx_assigned = true;
 	cfg->ref_count++;
 
+	if (rtwdev->scanning)
+		rtw89_hw_scan_abort(rtwdev, rtwdev->scan_info.scanning_vif);
+
 	if (list_empty(&rtwvif->mgnt_entry))
 		list_add_tail(&rtwvif->mgnt_entry, &mgnt->active_list);
 
@@ -2717,7 +3345,13 @@ int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev,
 	rtw89_swap_chanctx(rtwdev, cfg->idx, RTW89_CHANCTX_0);
 
 out:
-	return rtw89_set_channel(rtwdev);
+	ret = rtw89_set_channel(rtwdev);
+	if (ret)
+		return ret;
+
+	rtw89_tas_reset(rtwdev, true);
+
+	return 0;
 }
 
 void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev,
@@ -2736,6 +3370,9 @@ void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev,
 	rtwvif_link->chanctx_assigned = false;
 	cfg->ref_count--;
 
+	if (rtwdev->scanning)
+		rtw89_hw_scan_abort(rtwdev, rtwdev->scan_info.scanning_vif);
+
 	if (!rtw89_vif_is_active_role(rtwvif))
 		list_del_init(&rtwvif->mgnt_entry);
 
@@ -2761,7 +3398,7 @@ out:
 		cur = rtw89_get_entity_mode(rtwdev);
 		switch (cur) {
 		case RTW89_ENTITY_MODE_MCC:
-			rtw89_mcc_stop(rtwdev);
+			rtw89_mcc_stop(rtwdev, NULL);
 			break;
 		default:
 			break;
@@ -2787,3 +3424,37 @@ out:
 		break;
 	}
 }
+
+int rtw89_chanctx_ops_reassign_vif(struct rtw89_dev *rtwdev,
+				   struct rtw89_vif_link *rtwvif_link,
+				   struct ieee80211_chanctx_conf *old_ctx,
+				   struct ieee80211_chanctx_conf *new_ctx,
+				   bool replace)
+{
+	int ret;
+
+	rtw89_chanctx_ops_unassign_vif(rtwdev, rtwvif_link, old_ctx);
+
+	if (!replace)
+		goto assign;
+
+	rtw89_chanctx_ops_remove(rtwdev, old_ctx);
+	ret = rtw89_chanctx_ops_add(rtwdev, new_ctx);
+	if (ret) {
+		rtw89_err(rtwdev, "%s: failed to add chanctx: %d\n",
+			  __func__, ret);
+		return ret;
+	}
+
+assign:
+	ret = rtw89_chanctx_ops_assign_vif(rtwdev, rtwvif_link, new_ctx);
+	if (ret) {
+		rtw89_err(rtwdev, "%s: failed to assign chanctx: %d\n",
+			  __func__, ret);
+		return ret;
+	}
+
+	_rtw89_chan_update_punctured(rtwdev, rtwvif_link, &new_ctx->def);
+
+	return 0;
+}
diff --git a/sys/contrib/dev/rtw89/chan.h b/sys/contrib/dev/rtw89/chan.h
index 092a6f676894..b1175419f92b 100644
--- a/sys/contrib/dev/rtw89/chan.h
+++ b/sys/contrib/dev/rtw89/chan.h
@@ -18,6 +18,12 @@
 #define RTW89_MCC_EARLY_RX_BCN_TIME 5
 #define RTW89_MCC_MIN_RX_BCN_TIME 10
 #define RTW89_MCC_DFLT_BCN_OFST_TIME 40
+#define RTW89_MCC_SWITCH_CH_TIME 3
+
+#define RTW89_MCC_PROBE_TIMEOUT 100
+#define RTW89_MCC_PROBE_MAX_TRIES 3
+
+#define RTW89_MCC_DETECT_BCN_MAX_TRIES 2
 
 #define RTW89_MCC_MIN_GO_DURATION \
 	(RTW89_MCC_EARLY_TX_BCN_TIME + RTW89_MCC_MIN_RX_BCN_TIME)
@@ -25,17 +31,77 @@
 #define RTW89_MCC_MIN_STA_DURATION \
 	(RTW89_MCC_EARLY_RX_BCN_TIME + RTW89_MCC_MIN_RX_BCN_TIME)
 
+#define RTW89_MCC_MIN_RX_BCN_WITH_SWITCH_CH_TIME \
+	(RTW89_MCC_MIN_RX_BCN_TIME + RTW89_MCC_SWITCH_CH_TIME)
+
 #define RTW89_MCC_DFLT_GROUP 0
 #define RTW89_MCC_NEXT_GROUP(cur) (((cur) + 1) % 4)
 
-#define RTW89_MCC_DFLT_TX_NULL_EARLY 3
+#define RTW89_MCC_DFLT_TX_NULL_EARLY 7
 #define RTW89_MCC_DFLT_COURTESY_SLOT 3
 
+#define RTW89_MCC_REQ_COURTESY_TIME 5
+#define RTW89_MCC_REQ_COURTESY(pattern, role)			\
+({								\
+	const struct rtw89_mcc_pattern *p = pattern;		\
+	p->tob_ ## role <= RTW89_MCC_REQ_COURTESY_TIME ||	\
+	p->toa_ ## role <= RTW89_MCC_REQ_COURTESY_TIME;		\
+})
+
 #define NUM_OF_RTW89_MCC_ROLES 2
 
+enum rtw89_mr_wtype {
+	RTW89_MR_WTYPE_NONE,
+	RTW89_MR_WTYPE_NONMLD,
+	RTW89_MR_WTYPE_MLD1L1R,
+	RTW89_MR_WTYPE_MLD2L1R,
+	RTW89_MR_WTYPE_MLD2L2R,
+	RTW89_MR_WTYPE_NONMLD_NONMLD,
+	RTW89_MR_WTYPE_MLD1L1R_NONMLD,
+	RTW89_MR_WTYPE_MLD2L1R_NONMLD,
+	RTW89_MR_WTYPE_MLD2L2R_NONMLD,
+	RTW89_MR_WTYPE_UNKNOWN,
+};
+
+enum rtw89_mr_wmode {
+	RTW89_MR_WMODE_NONE,
+	RTW89_MR_WMODE_1CLIENT,
+	RTW89_MR_WMODE_1AP,
+	RTW89_MR_WMODE_1AP_1CLIENT,
+	RTW89_MR_WMODE_2CLIENTS,
+	RTW89_MR_WMODE_2APS,
+	RTW89_MR_WMODE_UNKNOWN,
+};
+
+enum rtw89_mr_ctxtype {
+	RTW89_MR_CTX_NONE,
+	RTW89_MR_CTX1_2GHZ,
+	RTW89_MR_CTX1_5GHZ,
+	RTW89_MR_CTX1_6GHZ,
+	RTW89_MR_CTX2_2GHZ,
+	RTW89_MR_CTX2_5GHZ,
+	RTW89_MR_CTX2_6GHZ,
+	RTW89_MR_CTX2_2GHZ_5GHZ,
+	RTW89_MR_CTX2_2GHZ_6GHZ,
+	RTW89_MR_CTX2_5GHZ_6GHZ,
+	RTW89_MR_CTX_UNKNOWN,
+};
+
+struct rtw89_mr_chanctx_info {
+	enum rtw89_mr_wtype wtype;
+	enum rtw89_mr_wmode wmode;
+	enum rtw89_mr_ctxtype ctxtype;
+};
+
 enum rtw89_chanctx_pause_reasons {
 	RTW89_CHANCTX_PAUSE_REASON_HW_SCAN,
 	RTW89_CHANCTX_PAUSE_REASON_ROC,
+	RTW89_CHANCTX_PAUSE_REASON_GC_BCN_LOSS,
+};
+
+struct rtw89_chanctx_pause_parm {
+	const struct rtw89_vif_link *trigger;
+	enum rtw89_chanctx_pause_reasons rsn;
 };
 
 struct rtw89_chanctx_cb_parm {
@@ -45,6 +111,7 @@ struct rtw89_chanctx_cb_parm {
 };
 
 struct rtw89_entity_weight {
+	unsigned int registered_chanctxs;
 	unsigned int active_chanctxs;
 	unsigned int active_roles;
 };
@@ -95,17 +162,19 @@ void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
 				 enum rtw89_chanctx_idx idx,
 				 const struct cfg80211_chan_def *chandef);
 void rtw89_config_roc_chandef(struct rtw89_dev *rtwdev,
-			      enum rtw89_chanctx_idx idx,
+			      struct rtw89_vif_link *rtwvif_link,
 			      const struct cfg80211_chan_def *chandef);
 void rtw89_entity_init(struct rtw89_dev *rtwdev);
 enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev);
-void rtw89_chanctx_work(struct work_struct *work);
+void rtw89_chanctx_work(struct wiphy *wiphy, struct wiphy_work *work);
 void rtw89_queue_chanctx_work(struct rtw89_dev *rtwdev);
 void rtw89_queue_chanctx_change(struct rtw89_dev *rtwdev,
 				enum rtw89_chanctx_changes change);
+void rtw89_query_mr_chanctx_info(struct rtw89_dev *rtwdev, u8 inst_idx,
+				 struct rtw89_mr_chanctx_info *info);
 void rtw89_chanctx_track(struct rtw89_dev *rtwdev);
 void rtw89_chanctx_pause(struct rtw89_dev *rtwdev,
-			 enum rtw89_chanctx_pause_reasons rsn);
+			 const struct rtw89_chanctx_pause_parm *parm);
 void rtw89_chanctx_proceed(struct rtw89_dev *rtwdev,
 			   const struct rtw89_chanctx_cb_parm *cb_parm);
 
@@ -116,6 +185,16 @@ const struct rtw89_chan *__rtw89_mgnt_chan_get(struct rtw89_dev *rtwdev,
 #define rtw89_mgnt_chan_get(rtwdev, link_index) \
 	__rtw89_mgnt_chan_get(rtwdev, __func__, link_index)
 
+struct rtw89_mcc_links_info {
+	struct rtw89_vif_link *links[NUM_OF_RTW89_MCC_ROLES];
+};
+
+void rtw89_mcc_get_links(struct rtw89_dev *rtwdev, struct rtw89_mcc_links_info *info);
+void rtw89_mcc_prepare_done_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_mcc_gc_detect_beacon_work(struct wiphy *wiphy, struct wiphy_work *work);
+bool rtw89_mcc_detect_go_bcn(struct rtw89_dev *rtwdev,
+			     struct rtw89_vif_link *rtwvif_link);
+
 int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev,
 			  struct ieee80211_chanctx_conf *ctx);
 void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev,
@@ -129,5 +208,10 @@ int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev,
 void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev,
 				    struct rtw89_vif_link *rtwvif_link,
 				    struct ieee80211_chanctx_conf *ctx);
+int rtw89_chanctx_ops_reassign_vif(struct rtw89_dev *rtwdev,
+				   struct rtw89_vif_link *rtwvif_link,
+				   struct ieee80211_chanctx_conf *old_ctx,
+				   struct ieee80211_chanctx_conf *new_ctx,
+				   bool replace);
 
 #endif
diff --git a/sys/contrib/dev/rtw89/coex.c b/sys/contrib/dev/rtw89/coex.c
index 68316d44b204..e4e6daf51a1b 100644
--- a/sys/contrib/dev/rtw89/coex.c
+++ b/sys/contrib/dev/rtw89/coex.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2019-2020  Realtek Corporation
  */
 
+#include "chan.h"
 #include "coex.h"
 #include "debug.h"
 #include "fw.h"
@@ -10,7 +11,7 @@
 #include "ps.h"
 #include "reg.h"
 
-#define RTW89_COEX_VERSION 0x07000113
+#define RTW89_COEX_VERSION 0x09000013
 #define FCXDEF_STEP 50 /* MUST <= FCXMAX_STEP and match with wl fw*/
 #define BTC_E2G_LIMIT_DEF 80
 
@@ -89,10 +90,10 @@ static const struct rtw89_btc_fbtc_slot s_def[] = {
 	[CXST_B4]	= __DEF_FBTC_SLOT(50,  0xe5555555, SLOT_MIX),
 	[CXST_LK]	= __DEF_FBTC_SLOT(20,  0xea5a5a5a, SLOT_ISO),
 	[CXST_BLK]	= __DEF_FBTC_SLOT(500, 0x55555555, SLOT_MIX),
-	[CXST_E2G]	= __DEF_FBTC_SLOT(0,   0xea5a5a5a, SLOT_MIX),
-	[CXST_E5G]	= __DEF_FBTC_SLOT(0,   0xffffffff, SLOT_ISO),
+	[CXST_E2G]	= __DEF_FBTC_SLOT(5,   0xea5a5a5a, SLOT_MIX),
+	[CXST_E5G]	= __DEF_FBTC_SLOT(5,   0xffffffff, SLOT_ISO),
 	[CXST_EBT]	= __DEF_FBTC_SLOT(5,   0xe5555555, SLOT_MIX),
-	[CXST_ENULL]	= __DEF_FBTC_SLOT(0,   0xaaaaaaaa, SLOT_ISO),
+	[CXST_ENULL]	= __DEF_FBTC_SLOT(5,   0xaaaaaaaa, SLOT_ISO),
 	[CXST_WLK]	= __DEF_FBTC_SLOT(250, 0xea5a5a5a, SLOT_MIX),
 	[CXST_W1FDD]	= __DEF_FBTC_SLOT(50,  0xffffffff, SLOT_ISO),
 	[CXST_B1FDD]	= __DEF_FBTC_SLOT(50,  0xffffdfff, SLOT_ISO),
@@ -132,13 +133,37 @@ static const u32 cxtbl[] = {
 
 static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	/* firmware version must be in decreasing order for each chip */
+	{RTL8852BT, RTW89_FW_VER_CODE(0, 29, 122, 0),
+	 .fcxbtcrpt = 8, .fcxtdma = 7,    .fcxslots = 7, .fcxcysta = 7,
+	 .fcxstep = 7,   .fcxnullsta = 7, .fcxmreg = 7,  .fcxgpiodbg = 7,
+	 .fcxbtver = 7,  .fcxbtscan = 7,  .fcxbtafh = 7, .fcxbtdevinfo = 7,
+	 .fwlrole = 7,   .frptmap = 3,    .fcxctrl = 7,  .fcxinit = 7,
+	 .fwevntrptl = 1, .fwc2hfunc = 2, .drvinfo_type = 1, .info_buf = 1800,
+	 .max_role_num = 6, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 8,
+	},
 	{RTL8852BT, RTW89_FW_VER_CODE(0, 29, 90, 0),
 	 .fcxbtcrpt = 7, .fcxtdma = 7,    .fcxslots = 7, .fcxcysta = 7,
 	 .fcxstep = 7,   .fcxnullsta = 7, .fcxmreg = 7,  .fcxgpiodbg = 7,
 	 .fcxbtver = 7,  .fcxbtscan = 7,  .fcxbtafh = 7, .fcxbtdevinfo = 7,
 	 .fwlrole = 7,   .frptmap = 3,    .fcxctrl = 7,  .fcxinit = 7,
 	 .fwevntrptl = 1, .fwc2hfunc = 2, .drvinfo_type = 1, .info_buf = 1800,
-	 .max_role_num = 6,
+	 .max_role_num = 6, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 8,
+	},
+	{RTL8922A, RTW89_FW_VER_CODE(0, 35, 71, 0),
+	 .fcxbtcrpt = 8, .fcxtdma = 7,    .fcxslots = 7, .fcxcysta = 7,
+	 .fcxstep = 7,   .fcxnullsta = 7, .fcxmreg = 7,  .fcxgpiodbg = 7,
+	 .fcxbtver = 7,  .fcxbtscan = 7,  .fcxbtafh = 7, .fcxbtdevinfo = 7,
+	 .fwlrole = 8,   .frptmap = 4,    .fcxctrl = 7,  .fcxinit = 7,
+	 .fwevntrptl = 1, .fwc2hfunc = 3, .drvinfo_type = 2, .info_buf = 1800,
+	 .max_role_num = 6, .fcxosi = 1,  .fcxmlo = 1,   .bt_desired = 9,
+	},
+	{RTL8922A, RTW89_FW_VER_CODE(0, 35, 63, 0),
+	 .fcxbtcrpt = 8, .fcxtdma = 7,    .fcxslots = 7, .fcxcysta = 7,
+	 .fcxstep = 7,   .fcxnullsta = 7, .fcxmreg = 7,  .fcxgpiodbg = 7,
+	 .fcxbtver = 7,  .fcxbtscan = 7,  .fcxbtafh = 7, .fcxbtdevinfo = 7,
+	 .fwlrole = 8,   .frptmap = 4,    .fcxctrl = 7,  .fcxinit = 7,
+	 .fwevntrptl = 1, .fwc2hfunc = 3, .drvinfo_type = 2, .info_buf = 1800,
+	 .max_role_num = 6, .fcxosi = 1,  .fcxmlo = 1,   .bt_desired = 9,
 	},
 	{RTL8922A, RTW89_FW_VER_CODE(0, 35, 8, 0),
 	 .fcxbtcrpt = 8, .fcxtdma = 7,    .fcxslots = 7, .fcxcysta = 7,
@@ -146,7 +171,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 7,  .fcxbtscan = 7,  .fcxbtafh = 7, .fcxbtdevinfo = 7,
 	 .fwlrole = 8,   .frptmap = 3,    .fcxctrl = 7,  .fcxinit = 7,
 	 .fwevntrptl = 1, .fwc2hfunc = 1, .drvinfo_type = 1, .info_buf = 1800,
-	 .max_role_num = 6,
+	 .max_role_num = 6, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8851B, RTW89_FW_VER_CODE(0, 29, 29, 0),
 	 .fcxbtcrpt = 105, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 5,
@@ -154,7 +179,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 2,  .fcxbtafh = 2, .fcxbtdevinfo = 1,
 	 .fwlrole = 2,   .frptmap = 3,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1800,
-	 .max_role_num = 6,
+	 .max_role_num = 6, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8852C, RTW89_FW_VER_CODE(0, 27, 57, 0),
 	 .fcxbtcrpt = 4, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 3,
@@ -162,7 +187,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 2, .fcxbtdevinfo = 1,
 	 .fwlrole = 1,   .frptmap = 3,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1280,
-	 .max_role_num = 5,
+	 .max_role_num = 5, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8852C, RTW89_FW_VER_CODE(0, 27, 42, 0),
 	 .fcxbtcrpt = 4, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 3,
@@ -170,7 +195,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 2, .fcxbtdevinfo = 1,
 	 .fwlrole = 1,   .frptmap = 2,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1280,
-	 .max_role_num = 5,
+	 .max_role_num = 5, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8852C, RTW89_FW_VER_CODE(0, 27, 0, 0),
 	 .fcxbtcrpt = 4, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 3,
@@ -178,7 +203,15 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 1, .fcxbtdevinfo = 1,
 	 .fwlrole = 1,   .frptmap = 2,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1280,
-	 .max_role_num = 5,
+	 .max_role_num = 5, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
+	},
+	{RTL8852B, RTW89_FW_VER_CODE(0, 29, 122, 0),
+	 .fcxbtcrpt = 8, .fcxtdma = 7,    .fcxslots = 7, .fcxcysta = 7,
+	 .fcxstep = 7,   .fcxnullsta = 7, .fcxmreg = 7,  .fcxgpiodbg = 7,
+	 .fcxbtver = 7,  .fcxbtscan = 7,  .fcxbtafh = 7, .fcxbtdevinfo = 7,
+	 .fwlrole = 7,   .frptmap = 3,    .fcxctrl = 7,  .fcxinit = 7,
+	 .fwevntrptl = 1, .fwc2hfunc = 2, .drvinfo_type = 1, .info_buf = 1800,
+	 .max_role_num = 6, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 8,
 	},
 	{RTL8852B, RTW89_FW_VER_CODE(0, 29, 29, 0),
 	 .fcxbtcrpt = 105, .fcxtdma = 3,  .fcxslots = 1, .fcxcysta = 5,
@@ -186,7 +219,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 2,  .fcxbtafh = 2, .fcxbtdevinfo = 1,
 	 .fwlrole = 2,   .frptmap = 3,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1800,
-	 .max_role_num = 6,
+	 .max_role_num = 6, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8852B, RTW89_FW_VER_CODE(0, 29, 14, 0),
 	 .fcxbtcrpt = 5, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 4,
@@ -194,7 +227,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 2, .fcxbtdevinfo = 1,
 	 .fwlrole = 1,   .frptmap = 3,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1800,
-	 .max_role_num = 6,
+	 .max_role_num = 6, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8852B, RTW89_FW_VER_CODE(0, 27, 0, 0),
 	 .fcxbtcrpt = 4, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 3,
@@ -202,7 +235,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 1, .fcxbtdevinfo = 1,
 	 .fwlrole = 1,   .frptmap = 1,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1280,
-	 .max_role_num = 5,
+	 .max_role_num = 5, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8852A, RTW89_FW_VER_CODE(0, 13, 37, 0),
 	 .fcxbtcrpt = 4, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 3,
@@ -210,7 +243,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 2, .fcxbtdevinfo = 1,
 	 .fwlrole = 1,   .frptmap = 3,    .fcxctrl = 1,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 0, .drvinfo_type = 0, .info_buf = 1280,
-	 .max_role_num = 5,
+	 .max_role_num = 5, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 	{RTL8852A, RTW89_FW_VER_CODE(0, 13, 0, 0),
 	 .fcxbtcrpt = 1, .fcxtdma = 1,    .fcxslots = 1, .fcxcysta = 2,
@@ -218,7 +251,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 1, .fcxbtdevinfo = 1,
 	 .fwlrole = 0,   .frptmap = 0,    .fcxctrl = 0,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 0, .drvinfo_type = 0, .info_buf = 1024,
-	 .max_role_num = 5,
+	 .max_role_num = 5, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 
 	/* keep it to be the last as default entry */
@@ -228,7 +261,7 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fcxbtver = 1,  .fcxbtscan = 1,  .fcxbtafh = 1, .fcxbtdevinfo = 1,
 	 .fwlrole = 0,   .frptmap = 0,    .fcxctrl = 0,  .fcxinit = 0,
 	 .fwevntrptl = 0, .fwc2hfunc = 1, .drvinfo_type = 0, .info_buf = 1024,
-	 .max_role_num = 5,
+	 .max_role_num = 5, .fcxosi = 0,  .fcxmlo = 0,   .bt_desired = 7,
 	},
 };
 
@@ -261,6 +294,39 @@ static u32 chip_id_to_bt_rom_code_id(u32 id)
 	}
 }
 
+#define CASE_BTC_MLME_STATE(e) case MLME_##e: return #e
+
+static const char *id_to_mlme_state(u32 id)
+{
+	switch (id) {
+	CASE_BTC_MLME_STATE(NO_LINK);
+	CASE_BTC_MLME_STATE(LINKING);
+	CASE_BTC_MLME_STATE(LINKED);
+	default:
+		return "unknown";
+	}
+}
+
+static char *chip_id_str(u32 id)
+{
+	switch (id) {
+	case RTL8852A:
+		return "RTL8852A";
+	case RTL8852B:
+		return "RTL8852B";
+	case RTL8852C:
+		return "RTL8852C";
+	case RTL8852BT:
+		return "RTL8852BT";
+	case RTL8851B:
+		return "RTL8851B";
+	case RTL8922A:
+		return "RTL8922A";
+	default:
+		return "UNKNOWN";
+	}
+}
+
 struct rtw89_btc_btf_tlv {
 	u8 type;
 	u8 len;
@@ -283,6 +349,7 @@ enum btc_btf_set_report_en {
 	RPT_EN_BT_DEVICE_INFO,
 	RPT_EN_BT_AFH_MAP,
 	RPT_EN_BT_AFH_MAP_LE,
+	RPT_EN_BT_TX_PWR_LVL,
 	RPT_EN_FW_STEP_INFO,
 	RPT_EN_TEST,
 	RPT_EN_WL_ALL,
@@ -660,6 +727,27 @@ enum btc_wl_link_mode {
 	BTC_WLINK_MAX
 };
 
+#define CASE_BTC_WL_LINK_MODE(e) case BTC_WLINK_## e: return #e
+
+static const char *id_to_linkmode(u8 id)
+{
+	switch (id) {
+	CASE_BTC_WL_LINK_MODE(NOLINK);
+	CASE_BTC_WL_LINK_MODE(2G_STA);
+	CASE_BTC_WL_LINK_MODE(2G_AP);
+	CASE_BTC_WL_LINK_MODE(2G_GO);
+	CASE_BTC_WL_LINK_MODE(2G_GC);
+	CASE_BTC_WL_LINK_MODE(2G_SCC);
+	CASE_BTC_WL_LINK_MODE(2G_MCC);
+	CASE_BTC_WL_LINK_MODE(25G_MCC);
+	CASE_BTC_WL_LINK_MODE(25G_DBCC);
+	CASE_BTC_WL_LINK_MODE(5G);
+	CASE_BTC_WL_LINK_MODE(OTHER);
+	default:
+		return "unknown";
+	}
+}
+
 enum btc_wl_mrole_type {
 	BTC_WLMROLE_NONE = 0x0,
 	BTC_WLMROLE_STA_GC,
@@ -825,6 +913,9 @@ static int _send_fw_cmd(struct rtw89_dev *rtwdev, u8 h2c_class, u8 h2c_func,
 	return ret;
 }
 
+#define BTC_BT_DEF_BR_TX_PWR 4
+#define BTC_BT_DEF_LE_TX_PWR 4
+
 static void _reset_btc_var(struct rtw89_dev *rtwdev, u8 type)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -893,6 +984,9 @@ static void _reset_btc_var(struct rtw89_dev *rtwdev, u8 type)
 
 	if (type & BTC_RESET_MDINFO)
 		memset(&btc->mdinfo, 0, sizeof(btc->mdinfo));
+
+	bt->link_info.bt_txpwr_desc.br_dbm = BTC_BT_DEF_BR_TX_PWR;
+	bt->link_info.bt_txpwr_desc.le_dbm = BTC_BT_DEF_LE_TX_PWR;
 }
 
 static u8 _search_reg_index(struct rtw89_dev *rtwdev, u8 mreg_num, u16 reg_type, u32 target)
@@ -1316,6 +1410,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			   u8 *prptbuf, u32 index)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_ver *fwsubver = &btc->fwinfo.fw_subver;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_dm *dm = &btc->dm;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
@@ -1358,25 +1453,29 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxbtcrpt == 1) {
 			pfinfo = &pfwinfo->rpt_ctrl.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v1);
+			fwsubver->fcxbtcrpt = pfwinfo->rpt_ctrl.finfo.v1.fver;
 		} else if (ver->fcxbtcrpt == 4) {
 			pfinfo = &pfwinfo->rpt_ctrl.finfo.v4;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v4);
+			fwsubver->fcxbtcrpt = pfwinfo->rpt_ctrl.finfo.v4.fver;
 		} else if (ver->fcxbtcrpt == 5) {
 			pfinfo = &pfwinfo->rpt_ctrl.finfo.v5;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v5);
+			fwsubver->fcxbtcrpt = pfwinfo->rpt_ctrl.finfo.v5.fver;
 		} else if (ver->fcxbtcrpt == 105) {
 			pfinfo = &pfwinfo->rpt_ctrl.finfo.v105;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v105);
+			fwsubver->fcxbtcrpt = pfwinfo->rpt_ctrl.finfo.v105.fver;
 			pcinfo->req_fver = 5;
 			break;
 		} else if (ver->fcxbtcrpt == 8) {
 			pfinfo = &pfwinfo->rpt_ctrl.finfo.v8;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v8);
-			break;
+			fwsubver->fcxbtcrpt = pfwinfo->rpt_ctrl.finfo.v8.fver;
 		} else if (ver->fcxbtcrpt == 7) {
 			pfinfo = &pfwinfo->rpt_ctrl.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v7);
-			break;
+			fwsubver->fcxbtcrpt = pfwinfo->rpt_ctrl.finfo.v7.fver;
 		} else {
 			goto err;
 		}
@@ -1387,9 +1486,11 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxtdma == 1) {
 			pfinfo = &pfwinfo->rpt_fbtc_tdma.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo.v1);
+			fwsubver->fcxtdma = 0;
 		} else if (ver->fcxtdma == 3 || ver->fcxtdma == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_tdma.finfo.v3;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo.v3);
+			fwsubver->fcxtdma = pfwinfo->rpt_fbtc_tdma.finfo.v3.fver;
 		} else {
 			goto err;
 		}
@@ -1400,9 +1501,11 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxslots == 1) {
 			pfinfo = &pfwinfo->rpt_fbtc_slots.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_slots.finfo.v1);
+			fwsubver->fcxslots = pfwinfo->rpt_fbtc_slots.finfo.v1.fver;
 		} else if (ver->fcxslots == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_slots.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_slots.finfo.v7);
+			fwsubver->fcxslots = pfwinfo->rpt_fbtc_slots.finfo.v7.fver;
 		} else {
 			goto err;
 		}
@@ -1415,22 +1518,27 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo.v2;
 			pcysta->v2 = pfwinfo->rpt_fbtc_cysta.finfo.v2;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo.v2);
+			fwsubver->fcxcysta = pfwinfo->rpt_fbtc_cysta.finfo.v2.fver;
 		} else if (ver->fcxcysta == 3) {
 			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo.v3;
 			pcysta->v3 = pfwinfo->rpt_fbtc_cysta.finfo.v3;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo.v3);
+			fwsubver->fcxcysta = pfwinfo->rpt_fbtc_cysta.finfo.v3.fver;
 		} else if (ver->fcxcysta == 4) {
 			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo.v4;
 			pcysta->v4 = pfwinfo->rpt_fbtc_cysta.finfo.v4;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo.v4);
+			fwsubver->fcxcysta = pfwinfo->rpt_fbtc_cysta.finfo.v4.fver;
 		} else if (ver->fcxcysta == 5) {
 			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo.v5;
 			pcysta->v5 = pfwinfo->rpt_fbtc_cysta.finfo.v5;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo.v5);
+			fwsubver->fcxcysta = pfwinfo->rpt_fbtc_cysta.finfo.v5.fver;
 		} else if (ver->fcxcysta == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo.v7;
 			pcysta->v7 = pfwinfo->rpt_fbtc_cysta.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo.v7);
+			fwsubver->fcxcysta = pfwinfo->rpt_fbtc_cysta.finfo.v7.fver;
 		} else {
 			goto err;
 		}
@@ -1446,11 +1554,13 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo.v2.step[0]) *
 					  trace_step +
 					  offsetof(struct rtw89_btc_fbtc_steps_v2, step);
+			fwsubver->fcxstep = pfwinfo->rpt_fbtc_step.finfo.v2.fver;
 		} else if (ver->fcxstep == 3) {
 			pfinfo = &pfwinfo->rpt_fbtc_step.finfo.v3;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo.v3.step[0]) *
 					  trace_step +
 					  offsetof(struct rtw89_btc_fbtc_steps_v3, step);
+			fwsubver->fcxstep = pfwinfo->rpt_fbtc_step.finfo.v3.fver;
 		} else {
 			goto err;
 		}
@@ -1461,12 +1571,15 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxnullsta == 1) {
 			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo.v1);
+			fwsubver->fcxnullsta = pfwinfo->rpt_fbtc_nullsta.finfo.v1.fver;
 		} else if (ver->fcxnullsta == 2) {
 			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo.v2;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo.v2);
+			fwsubver->fcxnullsta = pfwinfo->rpt_fbtc_nullsta.finfo.v2.fver;
 		} else if (ver->fcxnullsta == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo.v7);
+			fwsubver->fcxnullsta = pfwinfo->rpt_fbtc_nullsta.finfo.v7.fver;
 		} else {
 			goto err;
 		}
@@ -1477,12 +1590,15 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxmreg == 1) {
 			pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo.v1);
+			fwsubver->fcxmreg = pfwinfo->rpt_fbtc_mregval.finfo.v1.fver;
 		} else if (ver->fcxmreg == 2) {
 			pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo.v2;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo.v2);
+			fwsubver->fcxmreg = pfwinfo->rpt_fbtc_mregval.finfo.v2.fver;
 		} else if (ver->fcxmreg == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo.v7);
+			fwsubver->fcxmreg = pfwinfo->rpt_fbtc_mregval.finfo.v7.fver;
 		} else {
 			goto err;
 		}
@@ -1493,9 +1609,11 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxgpiodbg == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_gpio_dbg.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_gpio_dbg.finfo.v7);
+			fwsubver->fcxgpiodbg = pfwinfo->rpt_fbtc_gpio_dbg.finfo.v7.fver;
 		} else {
 			pfinfo = &pfwinfo->rpt_fbtc_gpio_dbg.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_gpio_dbg.finfo.v1);
+			fwsubver->fcxgpiodbg = pfwinfo->rpt_fbtc_gpio_dbg.finfo.v1.fver;
 		}
 		pcinfo->req_fver = ver->fcxgpiodbg;
 		break;
@@ -1504,9 +1622,11 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxbtver == 1) {
 			pfinfo = &pfwinfo->rpt_fbtc_btver.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btver.finfo.v1);
+			fwsubver->fcxbtver = pfwinfo->rpt_fbtc_btver.finfo.v1.fver;
 		} else if (ver->fcxbtver == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_btver.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btver.finfo.v7);
+			fwsubver->fcxbtver = pfwinfo->rpt_fbtc_btver.finfo.v7.fver;
 		}
 		pcinfo->req_fver = ver->fcxbtver;
 		break;
@@ -1515,12 +1635,15 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxbtscan == 1) {
 			pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo.v1);
+			fwsubver->fcxbtscan = pfwinfo->rpt_fbtc_btscan.finfo.v1.fver;
 		} else if (ver->fcxbtscan == 2) {
 			pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo.v2;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo.v2);
+			fwsubver->fcxbtscan = pfwinfo->rpt_fbtc_btscan.finfo.v2.fver;
 		} else if (ver->fcxbtscan == 7) {
 			pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo.v7;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo.v7);
+			fwsubver->fcxbtscan = pfwinfo->rpt_fbtc_btscan.finfo.v7.fver;
 		} else {
 			goto err;
 		}
@@ -1531,9 +1654,15 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		if (ver->fcxbtafh == 1) {
 			pfinfo = &pfwinfo->rpt_fbtc_btafh.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btafh.finfo.v1);
+			fwsubver->fcxbtafh = pfwinfo->rpt_fbtc_btafh.finfo.v1.fver;
 		} else if (ver->fcxbtafh == 2) {
 			pfinfo = &pfwinfo->rpt_fbtc_btafh.finfo.v2;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btafh.finfo.v2);
+			fwsubver->fcxbtafh = pfwinfo->rpt_fbtc_btafh.finfo.v2.fver;
+		} else if (ver->fcxbtafh == 7) {
+			pfinfo = &pfwinfo->rpt_fbtc_btafh.finfo.v7;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btafh.finfo.v7);
+			fwsubver->fcxbtafh = pfwinfo->rpt_fbtc_btafh.finfo.v7.fver;
 		} else {
 			goto err;
 		}
@@ -1543,6 +1672,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		pcinfo = &pfwinfo->rpt_fbtc_btdev.cinfo;
 		pfinfo = &pfwinfo->rpt_fbtc_btdev.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btdev.finfo);
+		fwsubver->fcxbtdevinfo = pfwinfo->rpt_fbtc_btdev.finfo.fver;
 		pcinfo->req_fver = ver->fcxbtdevinfo;
 		break;
 	default:
@@ -1602,7 +1732,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			wl->ver_info.fw = le32_to_cpu(prpt->v4.wl_fw_info.fw_ver);
 			dm->wl_fw_cx_offload = !!le32_to_cpu(prpt->v4.wl_fw_info.cx_offload);
 
-			for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++)
+			for (i = RTW89_PHY_0; i < RTW89_PHY_NUM; i++)
 				memcpy(&dm->gnt.band[i], &prpt->v4.gnt_val[i],
 				       sizeof(dm->gnt.band[i]));
 
@@ -1634,7 +1764,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			wl->ver_info.fw = le32_to_cpu(prpt->v5.rpt_info.fw_ver);
 			dm->wl_fw_cx_offload = 0;
 
-			for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++)
+			for (i = RTW89_PHY_0; i < RTW89_PHY_NUM; i++)
 				memcpy(&dm->gnt.band[i], &prpt->v5.gnt_val[i][0],
 				       sizeof(dm->gnt.band[i]));
 
@@ -1661,7 +1791,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			wl->ver_info.fw = le32_to_cpu(prpt->v105.rpt_info.fw_ver);
 			dm->wl_fw_cx_offload = 0;
 
-			for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++)
+			for (i = RTW89_PHY_0; i < RTW89_PHY_NUM; i++)
 				memcpy(&dm->gnt.band[i], &prpt->v105.gnt_val[i][0],
 				       sizeof(dm->gnt.band[i]));
 
@@ -1687,7 +1817,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			wl->ver_info.fw_coex = le32_to_cpu(prpt->v7.rpt_info.cx_ver);
 			wl->ver_info.fw = le32_to_cpu(prpt->v7.rpt_info.fw_ver);
 
-			for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++)
+			for (i = RTW89_PHY_0; i < RTW89_PHY_NUM; i++)
 				memcpy(&dm->gnt.band[i], &prpt->v7.gnt_val[i][0],
 				       sizeof(dm->gnt.band[i]));
 
@@ -1719,7 +1849,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			wl->ver_info.fw_coex = le32_to_cpu(prpt->v8.rpt_info.cx_ver);
 			wl->ver_info.fw = le32_to_cpu(prpt->v8.rpt_info.fw_ver);
 
-			for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++)
+			for (i = RTW89_PHY_0; i < RTW89_PHY_NUM; i++)
 				memcpy(&dm->gnt.band[i], &prpt->v8.gnt_val[i][0],
 				       sizeof(dm->gnt.band[i]));
 
@@ -2274,6 +2404,7 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 			bit_map = BIT(6);
 			break;
 		case 3:
+		case 4:
 			bit_map = BIT(5);
 			break;
 		default:
@@ -2288,6 +2419,7 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 			bit_map = BIT(5);
 			break;
 		case 3:
+		case 4:
 			bit_map = BIT(6);
 			break;
 		default:
@@ -2300,12 +2432,27 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 			bit_map = BIT(8);
 			break;
 		case 3:
+		case 4:
 			bit_map = BIT(7);
 			break;
 		default:
 			break;
 		}
 		break;
+	case RPT_EN_BT_TX_PWR_LVL:
+		switch (ver->frptmap) {
+		case 0:
+		case 1:
+		case 2:
+		case 3:
+			break;
+		case 4:
+			bit_map = BIT(8);
+			break;
+		default:
+			break;
+		}
+		break;
 	case RPT_EN_FW_STEP_INFO:
 		switch (ver->frptmap) {
 		case 1:
@@ -2315,6 +2462,9 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 		case 3:
 			bit_map = BIT(8);
 			break;
+		case 4:
+			bit_map = BIT(9);
+			break;
 		default:
 			break;
 		}
@@ -2332,6 +2482,9 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 		case 3:
 			bit_map = GENMASK(2, 0) | BIT(8);
 			break;
+		case 4:
+			bit_map = GENMASK(2, 0) | BIT(9);
+			break;
 		default:
 			break;
 		}
@@ -2348,6 +2501,9 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 		case 3:
 			bit_map = GENMASK(7, 3);
 			break;
+		case 4:
+			bit_map = GENMASK(8, 3);
+			break;
 		default:
 			break;
 		}
@@ -2364,6 +2520,9 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 		case 3:
 			bit_map = GENMASK(8, 0);
 			break;
+		case 4:
+			bit_map = GENMASK(9, 0);
+			break;
 		default:
 			break;
 		}
@@ -2380,6 +2539,9 @@ static u32 rtw89_btc_fw_rpt_ver(struct rtw89_dev *rtwdev, u32 rpt_map)
 		case 3:
 			bit_map = GENMASK(8, 2);
 			break;
+		case 4:
+			bit_map = GENMASK(9, 2);
+			break;
 		default:
 			break;
 		}
@@ -2669,6 +2831,16 @@ static void _fw_set_drv_info(struct rtw89_dev *rtwdev, u8 type)
 	case CXDRVINFO_FDDT:
 	case CXDRVINFO_MLO:
 	case CXDRVINFO_OSI:
+		if (!ver->fcxosi)
+			return;
+
+		if (ver->drvinfo_type == 2)
+			type = 7;
+		else
+			return;
+
+		rtw89_fw_h2c_cxdrv_osi_info(rtwdev, type);
+		break;
 	default:
 		break;
 	}
@@ -2706,7 +2878,7 @@ static void _set_gnt(struct rtw89_dev *rtwdev, u8 phy_map, u8 wl_state, u8 bt_st
 	if (phy_map > BTC_PHY_ALL)
 		return;
 
-	for (i = 0; i < RTW89_PHY_MAX; i++) {
+	for (i = 0; i < RTW89_PHY_NUM; i++) {
 		if (!(phy_map & BIT(i)))
 			continue;
 
@@ -2749,13 +2921,15 @@ static void _set_gnt_v1(struct rtw89_dev *rtwdev, u8 phy_map,
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_fbtc_outsrc_set_info *osi = &dm->ost_info;
+	struct rtw89_mac_ax_wl_act *b = dm->gnt.bt;
 	struct rtw89_mac_ax_gnt *g = dm->gnt.band;
 	u8 i, bt_idx = dm->bt_select + 1;
 
 	if (phy_map > BTC_PHY_ALL)
 		return;
 
-	for (i = 0; i < RTW89_PHY_MAX; i++) {
+	for (i = 0; i < RTW89_PHY_NUM; i++) {
 		if (!(phy_map & BIT(i)))
 			continue;
 
@@ -2797,21 +2971,35 @@ static void _set_gnt_v1(struct rtw89_dev *rtwdev, u8 phy_map,
 
 			switch (wlact_state) {
 			case BTC_WLACT_HW:
-				dm->gnt.bt[i].wlan_act_en = 0;
-				dm->gnt.bt[i].wlan_act = 0;
+				b[i].wlan_act_en = 0;
+				b[i].wlan_act = 0;
 				break;
 			case BTC_WLACT_SW_LO:
-				dm->gnt.bt[i].wlan_act_en = 1;
-				dm->gnt.bt[i].wlan_act = 0;
+				b[i].wlan_act_en = 1;
+				b[i].wlan_act = 0;
 				break;
 			case BTC_WLACT_SW_HI:
-				dm->gnt.bt[i].wlan_act_en = 1;
-				dm->gnt.bt[i].wlan_act = 1;
+				b[i].wlan_act_en = 1;
+				b[i].wlan_act = 1;
 				break;
 			}
 		}
 	}
-	rtw89_mac_cfg_gnt_v2(rtwdev, &dm->gnt);
+
+	if (!btc->ver->fcxosi) {
+		rtw89_mac_cfg_gnt_v2(rtwdev, &dm->gnt);
+		return;
+	}
+
+	memcpy(osi->gnt_set, dm->gnt.band, sizeof(osi->gnt_set));
+	memcpy(osi->wlact_set, dm->gnt.bt, sizeof(osi->wlact_set));
+
+	/* GBT source should be GBT_S1 in 1+1 (HWB0:5G + HWB1:2G) case */
+	if (osi->rf_band[BTC_RF_S0] == 1 &&
+	    osi->rf_band[BTC_RF_S1] == 0)
+		osi->rf_gbt_source = BTC_RF_S1;
+	else
+		osi->rf_gbt_source = BTC_RF_S0;
 }
 
 #define BTC_TDMA_WLROLE_MAX 3
@@ -2955,7 +3143,7 @@ static void _set_rf_trx_para(struct rtw89_dev *rtwdev)
 
 	if (ver->fwlrole == 0) {
 		link_mode = wl->role_info.link_mode;
-		for (i = 0; i < RTW89_PHY_MAX; i++) {
+		for (i = 0; i < RTW89_PHY_NUM; i++) {
 			if (wl->dbcc_info.real_band[i] == RTW89_BAND_2G)
 				dbcc_2g_phy = i;
 		}
@@ -3053,7 +3241,7 @@ static void _update_btc_state_map(struct rtw89_dev *rtwdev)
 	}
 }
 
-static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
+static void _set_bt_afh_info_v0(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -3222,6 +3410,115 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 	btc->cx.cnt_wl[BTC_WCNT_CH_UPDATE]++;
 }
 
+static void _set_bt_afh_info_v1(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_wl_role_info_v8 *wl_rinfo = &wl->role_info_v8;
+	struct rtw89_btc_wl_afh_info *wl_afh = &wl->afh_info;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+	struct rtw89_btc_wl_rlink *rlink;
+	u8 en = 0, ch = 0, bw = 0, buf[3] = {};
+	u8 i, j, link_mode;
+
+	if (btc->manual_ctrl || wl->status.map.scan)
+		return;
+
+	link_mode = wl_rinfo->link_mode;
+
+	for (i = 0; i < btc->ver->max_role_num; i++) {
+		for (j = RTW89_MAC_0; j < RTW89_MAC_NUM; j++) {
+			if (wl->status.map.rf_off || bt->whql_test ||
+			    link_mode == BTC_WLINK_NOLINK ||
+			    link_mode == BTC_WLINK_5G)
+				break;
+
+			rlink = &wl_rinfo->rlink[i][j];
+
+			/* Don't care no-connected/non-2G-band role */
+			if (!rlink->connected || !rlink->active ||
+			    rlink->rf_band != RTW89_BAND_2G)
+				continue;
+
+			en = 1;
+			ch = rlink->ch;
+			bw = rlink->bw;
+
+			if (link_mode == BTC_WLINK_2G_MCC &&
+			    (rlink->role == RTW89_WIFI_ROLE_AP ||
+			     rlink->role == RTW89_WIFI_ROLE_P2P_GO ||
+			     rlink->role == RTW89_WIFI_ROLE_P2P_CLIENT)) {
+				/* for 2.4G MCC, take role = ap/go/gc */
+				break;
+			} else if (link_mode != BTC_WLINK_2G_SCC ||
+				   rlink->bw == RTW89_CHANNEL_WIDTH_40) {
+				/* for 2.4G scc, take bw = 40M */
+				break;
+			}
+		}
+	}
+
+	/* default AFH channel sapn = center-ch +- 6MHz */
+	switch (bw) {
+	case RTW89_CHANNEL_WIDTH_20:
+		if (btc->dm.freerun || btc->dm.fddt_train)
+			bw = 48;
+		else
+			bw = 20 + chip->afh_guard_ch * 2;
+		break;
+	case RTW89_CHANNEL_WIDTH_40:
+		if (btc->dm.freerun)
+			bw = 40 + chip->afh_guard_ch * 2;
+		else
+			bw = 40;
+		break;
+	case RTW89_CHANNEL_WIDTH_5:
+		bw = 5 + chip->afh_guard_ch * 2;
+		break;
+	case RTW89_CHANNEL_WIDTH_10:
+		bw = 10 + chip->afh_guard_ch * 2;
+		break;
+	default:
+		en = false; /* turn off AFH info if invalid BW */
+		bw = 0;
+		ch = 0;
+		break;
+	}
+
+	if (!en || ch > 14 || ch == 0) {
+		en = false;
+		bw = 0;
+		ch = 0;
+	}
+
+	if (wl_afh->en == en &&
+	    wl_afh->ch == ch &&
+	    wl_afh->bw == bw &&
+	    (!bt->enable.now || bt->enable.last))
+		return;
+
+	wl_afh->en = buf[0];
+	wl_afh->ch = buf[1];
+	wl_afh->bw = buf[2];
+
+	if (_send_fw_cmd(rtwdev, BTFC_SET, SET_BT_WL_CH_INFO, &wl->afh_info, 3)) {
+		rtw89_debug(rtwdev, RTW89_DBG_BTC,
+			    "[BTC], %s(): en=%d, ch=%d, bw=%d\n",
+			    __func__, en, ch, bw);
+
+		btc->cx.cnt_wl[BTC_WCNT_CH_UPDATE]++;
+	}
+}
+
+static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
+{
+	if (rtwdev->chip->chip_id == RTL8922A)
+		_set_bt_afh_info_v1(rtwdev);
+	else
+		_set_bt_afh_info_v0(rtwdev);
+}
+
 static bool _check_freerun(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -3707,6 +4004,15 @@ void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type)
 	u32 tbl_w1, tbl_b1, tbl_b4;
 	u16 dur_2;
 
+	if (wl->status.map.lps) {
+		_slot_set_le(btc, CXST_E2G, s_def[CXST_E2G].dur,
+			     s_def[CXST_E2G].cxtbl, s_def[CXST_E2G].cxtype);
+		_slot_set_le(btc, CXST_E5G, s_def[CXST_E5G].dur,
+			     s_def[CXST_E5G].cxtbl, s_def[CXST_E5G].cxtype);
+		_slot_set_le(btc, CXST_EBT, s_def[CXST_EBT].dur,
+			     s_def[CXST_EBT].cxtbl, s_def[CXST_EBT].cxtype);
+	}
+
 	type = FIELD_GET(BTC_CXP_MASK, policy_type);
 
 	if (btc->ant_type == BTC_ANT_SHARED) {
@@ -3827,13 +4133,13 @@ void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type)
 
 		switch (policy_type) {
 		case BTC_CXP_OFFE_2GBWISOB: /* for normal-case */
-			_slot_set(btc, CXST_E2G, 0, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_E2G, 5, tbl_w1, SLOT_ISO);
 			_slot_set_le(btc, CXST_EBT, s_def[CXST_EBT].dur,
 				     s_def[CXST_EBT].cxtbl, s_def[CXST_EBT].cxtype);
 			_slot_set_dur(btc, CXST_EBT, dur_2);
 			break;
 		case BTC_CXP_OFFE_2GISOB: /* for bt no-link */
-			_slot_set(btc, CXST_E2G, 0, cxtbl[1], SLOT_ISO);
+			_slot_set(btc, CXST_E2G, 5, cxtbl[1], SLOT_ISO);
 			_slot_set_le(btc, CXST_EBT, s_def[CXST_EBT].dur,
 				     s_def[CXST_EBT].cxtbl, s_def[CXST_EBT].cxtype);
 			_slot_set_dur(btc, CXST_EBT, dur_2);
@@ -3859,15 +4165,15 @@ void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type)
 			break;
 		case BTC_CXP_OFFE_2GBWMIXB:
 			if (a2dp->exist)
-				_slot_set(btc, CXST_E2G, 0, cxtbl[2], SLOT_MIX);
+				_slot_set(btc, CXST_E2G, 5, cxtbl[2], SLOT_MIX);
 			else
-				_slot_set(btc, CXST_E2G, 0, tbl_w1, SLOT_MIX);
-			_slot_set_le(btc, CXST_EBT, s_def[CXST_EBT].dur,
+				_slot_set(btc, CXST_E2G, 5, tbl_w1, SLOT_MIX);
+			_slot_set_le(btc, CXST_EBT, cpu_to_le16(40),
 				     s_def[CXST_EBT].cxtbl, s_def[CXST_EBT].cxtype);
 			break;
 		case BTC_CXP_OFFE_WL: /* for 4-way */
-			_slot_set(btc, CXST_E2G, 0, cxtbl[1], SLOT_MIX);
-			_slot_set(btc, CXST_EBT, 0, cxtbl[1], SLOT_MIX);
+			_slot_set(btc, CXST_E2G, 5, cxtbl[1], SLOT_MIX);
+			_slot_set(btc, CXST_EBT, 5, cxtbl[1], SLOT_MIX);
 			break;
 		default:
 			break;
@@ -4240,7 +4546,7 @@ static void _set_ant_v0(struct rtw89_dev *rtwdev, bool force_exec,
 	case BTC_ANT_W2G:
 		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
 		if (rtwdev->dbcc_en) {
-			for (i = 0; i < RTW89_PHY_MAX; i++) {
+			for (i = 0; i < RTW89_PHY_NUM; i++) {
 				b2g = (wl_dinfo->real_band[i] == RTW89_BAND_2G);
 
 				gnt_wl_ctrl = b2g ? BTC_GNT_HW : BTC_GNT_SW_HI;
@@ -4583,23 +4889,16 @@ static void _action_bt_hid(struct rtw89_dev *rtwdev)
 static void _action_bt_a2dp(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
-	struct rtw89_btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
-	struct rtw89_btc_bt_a2dp_desc a2dp = bt_linfo->a2dp_desc;
 	struct rtw89_btc_dm *dm = &btc->dm;
 
 	_set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
 
+	dm->slot_dur[CXST_W1] = 20;
+	dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+
 	switch (btc->cx.state_map) {
 	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-A2DP */
-		if (a2dp.vendor_id == 0x4c || dm->leak_ap) {
-			dm->slot_dur[CXST_W1] = 40;
-			dm->slot_dur[CXST_B1] = 200;
-			_set_policy(rtwdev,
-				    BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP);
-		} else {
-			_set_policy(rtwdev,
-				    BTC_CXP_PAUTO_TD50B1, BTC_ACT_BT_A2DP);
-		}
+		_set_policy(rtwdev, BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP);
 		break;
 	case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP */
 		_set_policy(rtwdev, BTC_CXP_PAUTO2_TD3050, BTC_ACT_BT_A2DP);
@@ -4609,15 +4908,10 @@ static void _action_bt_a2dp(struct rtw89_dev *rtwdev)
 		break;
 	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-A2DP */
 	case BTC_WLINKING: /* wl-connecting + bt-A2DP */
-		if (a2dp.vendor_id == 0x4c || dm->leak_ap) {
-			dm->slot_dur[CXST_W1] = 40;
-			dm->slot_dur[CXST_B1] = 200;
-			_set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1,
-				    BTC_ACT_BT_A2DP);
-		} else {
-			_set_policy(rtwdev, BTC_CXP_AUTO_TD50B1,
-				    BTC_ACT_BT_A2DP);
-		}
+		if (btc->cx.wl.rfk_info.con_rfk)
+			_set_policy(rtwdev, BTC_CXP_OFF_BT, BTC_ACT_BT_A2DP);
+		else
+			_set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1, BTC_ACT_BT_A2DP);
 		break;
 	case BTC_WIDLE:  /* wl-idle + bt-A2DP */
 		_set_policy(rtwdev, BTC_CXP_AUTO_TD20B1, BTC_ACT_BT_A2DP);
@@ -4645,7 +4939,10 @@ static void _action_bt_a2dpsink(struct rtw89_dev *rtwdev)
 		_set_policy(rtwdev, BTC_CXP_FIX_TD2060, BTC_ACT_BT_A2DPSINK);
 		break;
 	case BTC_WLINKING: /* wl-connecting + bt-A2dp_Sink */
-		_set_policy(rtwdev, BTC_CXP_FIX_TD3030, BTC_ACT_BT_A2DPSINK);
+		if (btc->cx.wl.rfk_info.con_rfk)
+			_set_policy(rtwdev, BTC_CXP_OFF_BT, BTC_ACT_BT_A2DPSINK);
+		else
+			_set_policy(rtwdev, BTC_CXP_FIX_TD3030, BTC_ACT_BT_A2DPSINK);
 		break;
 	case BTC_WIDLE: /* wl-idle + bt-A2dp_Sink */
 		_set_policy(rtwdev, BTC_CXP_FIX_TD2080, BTC_ACT_BT_A2DPSINK);
@@ -4693,27 +4990,20 @@ static void _action_bt_pan(struct rtw89_dev *rtwdev)
 static void _action_bt_a2dp_hid(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
-	struct rtw89_btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
-	struct rtw89_btc_bt_a2dp_desc a2dp = bt_linfo->a2dp_desc;
 	struct rtw89_btc_dm *dm = &btc->dm;
 
 	_set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
 
+	dm->slot_dur[CXST_W1] = 20;
+	dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+
 	switch (btc->cx.state_map) {
 	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-A2DP+HID */
 	case BTC_WIDLE:  /* wl-idle + bt-A2DP */
-		if (a2dp.vendor_id == 0x4c || dm->leak_ap) {
-			dm->slot_dur[CXST_W1] = 40;
-			dm->slot_dur[CXST_B1] = 200;
-			_set_policy(rtwdev,
-				    BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP_HID);
-		} else {
-			_set_policy(rtwdev,
-				    BTC_CXP_PAUTO_TD50B1, BTC_ACT_BT_A2DP_HID);
-		}
+		_set_policy(rtwdev, BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP_HID);
 		break;
 	case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP+HID */
-		_set_policy(rtwdev, BTC_CXP_PAUTO2_TD3050, BTC_ACT_BT_A2DP_HID);
+		_set_policy(rtwdev, BTC_CXP_PAUTO2_TD3070, BTC_ACT_BT_A2DP_HID);
 		break;
 
 	case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq + bt-A2DP+HID */
@@ -4721,15 +5011,10 @@ static void _action_bt_a2dp_hid(struct rtw89_dev *rtwdev)
 		break;
 	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-A2DP+HID */
 	case BTC_WLINKING: /* wl-connecting + bt-A2DP+HID */
-		if (a2dp.vendor_id == 0x4c || dm->leak_ap) {
-			dm->slot_dur[CXST_W1] = 40;
-			dm->slot_dur[CXST_B1] = 200;
-			_set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1,
-				    BTC_ACT_BT_A2DP_HID);
-		} else {
-			_set_policy(rtwdev, BTC_CXP_AUTO_TD50B1,
-				    BTC_ACT_BT_A2DP_HID);
-		}
+		if (btc->cx.wl.rfk_info.con_rfk)
+			_set_policy(rtwdev, BTC_CXP_OFF_BT, BTC_ACT_BT_A2DP_HID);
+		else
+			_set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1, BTC_ACT_BT_A2DP_HID);
 		break;
 	}
 }
@@ -4876,16 +5161,14 @@ static void _set_btg_ctrl(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	struct rtw89_btc_wl_role_info_v7 *wl_rinfo_v7 = &wl->role_info_v7;
 	struct rtw89_btc_wl_role_info_v8 *wl_rinfo_v8 = &wl->role_info_v8;
+	struct rtw89_btc_fbtc_outsrc_set_info *o_info = &btc->dm.ost_info;
 	struct rtw89_btc_wl_role_info *wl_rinfo_v0 = &wl->role_info;
-	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_btc_dm *dm = &btc->dm;
 	struct _wl_rinfo_now wl_rinfo;
-	u32 run_reason = btc->dm.run_reason;
-	u32 is_btg;
-	u8 i, val;
+	u32 is_btg = BTC_BTGCTRL_DISABLE;
 
 	if (btc->manual_ctrl)
 		return;
@@ -4903,63 +5186,62 @@ static void _set_btg_ctrl(struct rtw89_dev *rtwdev)
 	else
 		return;
 
-	if (rtwdev->dbcc_en) {
-		if (ver->fwlrole == 0) {
-			wl_rinfo.dbcc_2g_phy = RTW89_PHY_MAX;
+	/* notify halbb ignore GNT_BT or not for WL BB Rx-AGC control */
+	if (btc->ant_type == BTC_ANT_SHARED) {
+		if (!(bt->run_patch_code && bt->enable.now))
+			is_btg = BTC_BTGCTRL_DISABLE;
+		else if (wl_rinfo.link_mode != BTC_WLINK_5G)
+			is_btg = BTC_BTGCTRL_ENABLE;
+		else
+			is_btg = BTC_BTGCTRL_DISABLE;
 
-			for (i = 0; i < RTW89_PHY_MAX; i++) {
-				if (wl_dinfo->real_band[i] == RTW89_BAND_2G)
-					wl_rinfo.dbcc_2g_phy = i;
-			}
-		} else if (ver->fwlrole == 1) {
-			wl_rinfo.dbcc_2g_phy = wl_rinfo_v1->dbcc_2g_phy;
-		} else if (ver->fwlrole == 2) {
-			wl_rinfo.dbcc_2g_phy = wl_rinfo_v2->dbcc_2g_phy;
-		} else if (ver->fwlrole == 7) {
-			wl_rinfo.dbcc_2g_phy = wl_rinfo_v7->dbcc_2g_phy;
-		} else if (ver->fwlrole == 8) {
-			wl_rinfo.dbcc_2g_phy = wl_rinfo_v8->dbcc_2g_phy;
-		} else {
-			return;
-		}
+		/* bb call ctrl_btg() in WL FW by slot */
+		if (!ver->fcxosi &&
+		    wl_rinfo.link_mode == BTC_WLINK_25G_MCC)
+			is_btg = BTC_BTGCTRL_BB_GNT_FWCTRL;
 	}
 
-	if (wl_rinfo.link_mode == BTC_WLINK_25G_MCC)
-		is_btg = BTC_BTGCTRL_BB_GNT_FWCTRL;
-	else if (!(bt->run_patch_code && bt->enable.now))
-		is_btg = BTC_BTGCTRL_DISABLE;
-	else if (wl_rinfo.link_mode == BTC_WLINK_5G)
-		is_btg = BTC_BTGCTRL_DISABLE;
-	else if (dm->freerun)
-		is_btg = BTC_BTGCTRL_DISABLE;
-	else if (rtwdev->dbcc_en && wl_rinfo.dbcc_2g_phy != RTW89_PHY_1)
-		is_btg = BTC_BTGCTRL_DISABLE;
+	if (is_btg == dm->wl_btg_rx)
+		return;
 	else
-		is_btg = BTC_BTGCTRL_ENABLE;
+		dm->wl_btg_rx = is_btg;
 
-	if (dm->wl_btg_rx_rb != dm->wl_btg_rx &&
-	    dm->wl_btg_rx_rb != BTC_BTGCTRL_BB_GNT_NOTFOUND) {
-		_get_reg_status(rtwdev, BTC_CSTATUS_BB_GNT_MUX, &val);
-		dm->wl_btg_rx_rb = val;
-	}
+	/* skip setup if btg_ctrl set by wl fw */
+	if (!ver->fcxosi && is_btg > BTC_BTGCTRL_ENABLE)
+		return;
 
-	if (run_reason == BTC_RSN_NTFY_INIT ||
-	    run_reason == BTC_RSN_NTFY_SWBAND ||
-	    dm->wl_btg_rx_rb != dm->wl_btg_rx ||
-	    is_btg != dm->wl_btg_rx) {
+	/* Below flow is for BTC_FEAT_NEW_BBAPI_FLOW = 1 */
+	if (o_info->rf_band[BTC_RF_S0] != o_info->rf_band[BTC_RF_S1]) {/* 1+1 */
+		if (o_info->rf_band[BTC_RF_S0]) /* Non-2G */
+			o_info->btg_rx[BTC_RF_S0] = BTC_BTGCTRL_DISABLE;
+		else
+			o_info->btg_rx[BTC_RF_S0] = is_btg;
 
-		dm->wl_btg_rx = is_btg;
+		if (o_info->rf_band[BTC_RF_S1]) /* Non-2G */
+			o_info->btg_rx[BTC_RF_S1] = BTC_BTGCTRL_DISABLE;
+		else
+			o_info->btg_rx[BTC_RF_S1] = is_btg;
+	} else { /* 2+0 or 0+2 */
+		o_info->btg_rx[BTC_RF_S0] = is_btg;
+		o_info->btg_rx[BTC_RF_S1] = is_btg;
+	}
 
-		if (is_btg > BTC_BTGCTRL_ENABLE)
-			return;
+	if (ver->fcxosi)
+		return;
 
-		chip->ops->ctrl_btg_bt_rx(rtwdev, is_btg, RTW89_PHY_0);
-	}
+	chip->ops->ctrl_btg_bt_rx(rtwdev, o_info->btg_rx[BTC_RF_S0],
+				  RTW89_PHY_0);
+	if (chip->chip_id != RTL8922A)
+		return;
+
+	chip->ops->ctrl_btg_bt_rx(rtwdev, o_info->btg_rx[BTC_RF_S1],
+				  RTW89_PHY_1);
 }
 
 static void _set_wl_preagc_ctrl(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_fbtc_outsrc_set_info *o_info = &btc->dm.ost_info;
 	struct rtw89_btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_role_info_v2 *rinfo_v2 = &wl->role_info_v2;
@@ -4991,9 +5273,7 @@ static void _set_wl_preagc_ctrl(struct rtw89_dev *rtwdev)
 		return;
 	}
 
-	if (link_mode == BTC_WLINK_25G_MCC) {
-		is_preagc = BTC_PREAGC_BB_FWCTRL;
-	} else if (!(bt->run_patch_code && bt->enable.now)) {
+	if (!(bt->run_patch_code && bt->enable.now)) {
 		is_preagc = BTC_PREAGC_DISABLE;
 	} else if (link_mode == BTC_WLINK_5G) {
 		is_preagc = BTC_PREAGC_DISABLE;
@@ -5013,6 +5293,9 @@ static void _set_wl_preagc_ctrl(struct rtw89_dev *rtwdev)
 		is_preagc = BTC_PREAGC_ENABLE;
 	}
 
+	if (!btc->ver->fcxosi && link_mode == BTC_WLINK_25G_MCC)
+		is_preagc = BTC_PREAGC_BB_FWCTRL;
+
 	if (dm->wl_pre_agc_rb != dm->wl_pre_agc &&
 	    dm->wl_pre_agc_rb != BTC_PREAGC_NOTFOUND) {
 		_get_reg_status(rtwdev, BTC_CSTATUS_BB_PRE_AGC, &val);
@@ -5026,9 +5309,34 @@ static void _set_wl_preagc_ctrl(struct rtw89_dev *rtwdev)
 	    is_preagc != dm->wl_pre_agc) {
 		dm->wl_pre_agc = is_preagc;
 
-		if (is_preagc > BTC_PREAGC_ENABLE)
+		if (!btc->ver->fcxosi && is_preagc > BTC_PREAGC_ENABLE)
 			return;
-		chip->ops->ctrl_nbtg_bt_tx(rtwdev, dm->wl_pre_agc, RTW89_PHY_0);
+
+		if (o_info->rf_band[BTC_RF_S0] != o_info->rf_band[BTC_RF_S1]) {/* 1+1 */
+			if (o_info->rf_band[BTC_RF_S0]) /* Non-2G */
+				o_info->nbtg_tx[BTC_RF_S0] = BTC_PREAGC_DISABLE;
+			else
+				o_info->nbtg_tx[BTC_RF_S0] = is_preagc;
+
+			if (o_info->rf_band[BTC_RF_S1]) /* Non-2G */
+				o_info->nbtg_tx[BTC_RF_S1] = BTC_PREAGC_DISABLE;
+			else
+				o_info->nbtg_tx[BTC_RF_S1] = is_preagc;
+
+		} else { /* 2+0 or 0+2 */
+			o_info->nbtg_tx[BTC_RF_S0] = is_preagc;
+			o_info->nbtg_tx[BTC_RF_S1] = is_preagc;
+		}
+
+		if (btc->ver->fcxosi)
+			return;
+
+		chip->ops->ctrl_nbtg_bt_tx(rtwdev, o_info->nbtg_tx[BTC_RF_S0],
+					   RTW89_PHY_0);
+		if (chip->chip_id != RTL8922A)
+			return;
+		chip->ops->ctrl_nbtg_bt_tx(rtwdev, o_info->nbtg_tx[BTC_RF_S1],
+					   RTW89_PHY_1);
 	}
 }
 
@@ -5241,15 +5549,47 @@ static void _set_bt_rx_scan_pri(struct rtw89_dev *rtwdev)
 	_write_scbd(rtwdev, BTC_WSCB_RXSCAN_PRI, (bool)(!!bt->scan_rx_low_pri));
 }
 
+static void _wl_req_mac(struct rtw89_dev *rtwdev, u8 mac)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	u32 add;
+
+	if (mac == wl->pta_req_mac)
+		return;
+
+	dm->ost_info.pta_req_hw_band = mac;
+	wl->pta_req_mac = mac;
+	wl->pta_reg_mac_chg = true;
+
+	if (btc->ver->fcxosi)
+		return;
+
+	if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
+		add = R_BE_BTC_CFG;
+	else
+		add = R_AX_BTC_CFG;
+
+	if (mac == RTW89_MAC_0)
+		rtw89_write32_clr(rtwdev, add, B_AX_WL_SRC);
+	else
+		rtw89_write32_set(rtwdev, add, B_AX_WL_SRC);
+}
+
 static void _action_common(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_wl_role_info_v8 *rinfo_v8 = &wl->role_info_v8;
 	struct rtw89_btc_wl_smap *wl_smap = &wl->status.map;
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_btc_dm *dm = &btc->dm;
 	u32 bt_rom_code_id, bt_fw_ver;
 
+	if (btc->ver->fwlrole == 8)
+		_wl_req_mac(rtwdev, rinfo_v8->pta_req_band);
+
 	_set_btg_ctrl(rtwdev);
 	_set_wl_preagc_ctrl(rtwdev);
 	_set_wl_tx_limit(rtwdev);
@@ -5285,7 +5625,18 @@ static void _action_common(struct rtw89_dev *rtwdev)
 		wl->scbd_change = false;
 		btc->cx.cnt_wl[BTC_WCNT_SCBDUPDATE]++;
 	}
+
+	if (btc->ver->fcxosi) {
+		if (memcmp(&dm->ost_info_last, &dm->ost_info,
+			   sizeof(dm->ost_info_last)) ||
+		    dm->run_reason == BTC_RSN_NTFY_INIT ||
+		    dm->run_reason == BTC_RSN_NTFY_RADIO_STATE) {
+			dm->ost_info_last = dm->ost_info;
+			_fw_set_drv_info(rtwdev, CXDRVINFO_OSI);
+		}
+	}
 	btc->dm.tdma_instant_excute = 0;
+	wl->pta_reg_mac_chg = false;
 }
 
 static void _action_by_bt(struct rtw89_dev *rtwdev)
@@ -5408,7 +5759,8 @@ static void _action_wl_scan(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
 
-	if (RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw)) {
+	if (btc->cx.state_map != BTC_WLINKING &&
+	    RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw)) {
 		_action_wl_25g_mcc(rtwdev);
 		rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], Scan offload!\n");
 	} else if (rtwdev->dbcc_en) {
@@ -5747,14 +6099,6 @@ _update_rssi_state(struct rtw89_dev *rtwdev, u8 pre_state, u8 rssi, u8 thresh)
 	return next_state;
 }
 
-static void _wl_req_mac(struct rtw89_dev *rtwdev, u8 mac)
-{
-	if (mac == RTW89_MAC_0)
-		rtw89_write32_clr(rtwdev, R_AX_BTC_CFG, B_AX_WL_SRC);
-	else
-		rtw89_write32_set(rtwdev, R_AX_BTC_CFG, B_AX_WL_SRC);
-}
-
 static
 void _update_dbcc_band(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx)
 {
@@ -5798,7 +6142,7 @@ static void _update_wl_info(struct rtw89_dev *rtwdev)
 		phy = wl_linfo[i].phy;
 
 		/* check dbcc role */
-		if (rtwdev->dbcc_en && phy < RTW89_PHY_MAX) {
+		if (rtwdev->dbcc_en && phy < RTW89_PHY_NUM) {
 			wl_dinfo->role[phy] = wl_linfo[i].role;
 			wl_dinfo->op_band[phy] = wl_linfo[i].band;
 			_update_dbcc_band(rtwdev, phy);
@@ -5948,7 +6292,7 @@ static void _update_wl_info_v1(struct rtw89_dev *rtwdev)
 
 		phy = wl_linfo[i].phy;
 
-		if (rtwdev->dbcc_en && phy < RTW89_PHY_MAX) {
+		if (rtwdev->dbcc_en && phy < RTW89_PHY_NUM) {
 			wl_dinfo->role[phy] = wl_linfo[i].role;
 			wl_dinfo->op_band[phy] = wl_linfo[i].band;
 			_update_dbcc_band(rtwdev, phy);
@@ -6098,7 +6442,7 @@ static void _update_wl_info_v2(struct rtw89_dev *rtwdev)
 
 		phy = wl_linfo[i].phy;
 
-		if (rtwdev->dbcc_en && phy < RTW89_PHY_MAX) {
+		if (rtwdev->dbcc_en && phy < RTW89_PHY_NUM) {
 			wl_dinfo->role[phy] = wl_linfo[i].role;
 			wl_dinfo->op_band[phy] = wl_linfo[i].band;
 			_update_dbcc_band(rtwdev, phy);
@@ -6250,23 +6594,16 @@ static bool _chk_role_ch_group(const struct rtw89_btc_chdef *r1,
 }
 
 static u8 _chk_dbcc(struct rtw89_dev *rtwdev, struct rtw89_btc_chdef *ch,
-		    u8 *phy, u8 *role, u8 *dbcc_2g_phy)
+		    u8 *phy, u8 *role, u8 link_cnt)
 {
 	struct rtw89_btc_wl_info *wl = &rtwdev->btc.cx.wl;
 	struct rtw89_btc_wl_role_info_v7 *rinfo_v7 = &wl->role_info_v7;
 	struct rtw89_btc_wl_role_info_v8 *rinfo_v8 = &wl->role_info_v8;
 	bool is_2g_ch_exist = false, is_multi_role_in_2g_phy = false;
-	u8 j, k, dbcc_2g_cid, dbcc_2g_cid2, connect_cnt;
-
-	if (rtwdev->btc.ver->fwlrole == 7)
-		connect_cnt = rinfo_v7->connect_cnt;
-	else if (rtwdev->btc.ver->fwlrole == 8)
-		connect_cnt = rinfo_v8->connect_cnt;
-	else
-		return BTC_WLINK_NOLINK;
+	u8 j, k, dbcc_2g_cid, dbcc_2g_cid2, dbcc_2g_phy, pta_req_band;
 
 	/* find out the 2G-PHY by connect-id ->ch  */
-	for (j = 0; j < connect_cnt; j++) {
+	for (j = 0; j < link_cnt; j++) {
 		if (ch[j].center_ch <= 14) {
 			is_2g_ch_exist = true;
 			break;
@@ -6275,21 +6612,33 @@ static u8 _chk_dbcc(struct rtw89_dev *rtwdev, struct rtw89_btc_chdef *ch,
 
 	/* If no any 2G-port exist, it's impossible because 5G-exclude */
 	if (!is_2g_ch_exist)
-		return BTC_WLINK_OTHER;
+		return BTC_WLINK_5G;
 
 	dbcc_2g_cid = j;
-	*dbcc_2g_phy = phy[dbcc_2g_cid];
+	dbcc_2g_phy = phy[dbcc_2g_cid];
+
+	if (dbcc_2g_phy == RTW89_PHY_1)
+		pta_req_band = RTW89_PHY_1;
+	else
+		pta_req_band = RTW89_PHY_0;
+
+	if (rtwdev->btc.ver->fwlrole == 7) {
+		rinfo_v7->dbcc_2g_phy = dbcc_2g_phy;
+	} else if (rtwdev->btc.ver->fwlrole == 8) {
+		rinfo_v8->dbcc_2g_phy = dbcc_2g_phy;
+		rinfo_v8->pta_req_band = pta_req_band;
+	}
 
 	/* connect_cnt <= 2 */
-	if (connect_cnt < BTC_TDMA_WLROLE_MAX)
+	if (link_cnt < BTC_TDMA_WLROLE_MAX)
 		return (_get_role_link_mode((role[dbcc_2g_cid])));
 
 	/* find the other-port in the 2G-PHY, ex: PHY-0:6G, PHY1: mcc/scc */
-	for (k = 0; k < connect_cnt; k++) {
+	for (k = 0; k < link_cnt; k++) {
 		if (k == dbcc_2g_cid)
 			continue;
 
-		if (phy[k] == *dbcc_2g_phy) {
+		if (phy[k] == dbcc_2g_phy) {
 			is_multi_role_in_2g_phy = true;
 			dbcc_2g_cid2 = k;
 			break;
@@ -6389,7 +6738,7 @@ static void _update_wl_info_v7(struct rtw89_dev *rtwdev, u8 rid)
 	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
 	struct rtw89_btc_wl_link_info *wl_linfo = wl->link_info;
 	struct rtw89_btc_wl_active_role_v7 *act_role = NULL;
-	u8 i, mode, cnt = 0, cnt_2g = 0, cnt_5g = 0, phy_now = RTW89_PHY_MAX, phy_dbcc;
+	u8 i, mode, cnt = 0, cnt_2g = 0, cnt_5g = 0, phy_now = RTW89_PHY_NUM, phy_dbcc;
 	bool b2g = false, b5g = false, client_joined = false, client_inc_2g = false;
 	u8 client_cnt_last[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER] = {};
 	u8 cid_role[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER] = {};
@@ -6400,7 +6749,7 @@ static void _update_wl_info_v7(struct rtw89_dev *rtwdev, u8 rid)
 	memset(wl_rinfo, 0, sizeof(*wl_rinfo));
 
 	for (i = 0; i < RTW89_PORT_NUM; i++) {
-		if (!wl_linfo[i].active || wl_linfo[i].phy >= RTW89_PHY_MAX)
+		if (!wl_linfo[i].active || wl_linfo[i].phy >= RTW89_PHY_NUM)
 			continue;
 
 		act_role = &wl_rinfo->active_role[i];
@@ -6491,10 +6840,10 @@ static void _update_wl_info_v7(struct rtw89_dev *rtwdev, u8 rid)
 	} else if (cnt > BTC_TDMA_WLROLE_MAX) {
 		mode = BTC_WLINK_OTHER;
 	} else if (rtwdev->dbcc_en) {
-		mode = _chk_dbcc(rtwdev, cid_ch, cid_phy, cid_role, &dbcc_2g_phy);
+		mode = _chk_dbcc(rtwdev, cid_ch, cid_phy, cid_role, cnt);
 
 		/* correct 2G-located PHY band for gnt ctrl */
-		if (dbcc_2g_phy < RTW89_PHY_MAX)
+		if (dbcc_2g_phy < RTW89_PHY_NUM)
 			wl_dinfo->op_band[dbcc_2g_phy] = RTW89_BAND_2G;
 	} else if (b2g && b5g && cnt == 2) {
 		mode = BTC_WLINK_25G_MCC;
@@ -6536,26 +6885,336 @@ static void _update_wl_info_v7(struct rtw89_dev *rtwdev, u8 rid)
 	_fw_set_drv_info(rtwdev, CXDRVINFO_ROLE);
 }
 
+static u8 _update_wl_link_mode(struct rtw89_dev *rtwdev, u8 hw_band, u8 type)
+{
+	struct rtw89_btc_wl_info *wl = &rtwdev->btc.cx.wl;
+	struct rtw89_btc_wl_mlo_info *mlo_info = &wl->mlo_info;
+	u8 mode = BTC_WLINK_NOLINK;
+
+	switch (type) {
+	case RTW89_MR_WTYPE_NONE: /* no-link */
+		mode = BTC_WLINK_NOLINK;
+		break;
+	case RTW89_MR_WTYPE_NONMLD:  /* Non_MLO 1-role 2+0/0+2 */
+	case RTW89_MR_WTYPE_MLD1L1R: /* MLO only-1 link 2+0/0+2 */
+		if (mlo_info->hwb_rf_band[hw_band] != RTW89_BAND_2G) {
+			mode = BTC_WLINK_5G;
+		} else if (mlo_info->wmode[hw_band] == RTW89_MR_WMODE_1AP) {
+			mode = BTC_WLINK_2G_GO;
+		} else if (mlo_info->wmode[hw_band] == RTW89_MR_WMODE_1CLIENT) {
+			if (wl->role_info_v8.p2p_2g)
+				mode = BTC_WLINK_2G_GC;
+			else
+				mode = BTC_WLINK_2G_STA;
+		}
+		break;
+	case RTW89_MR_WTYPE_NONMLD_NONMLD: /* Non_MLO 2-role 2+0/0+2 */
+	case RTW89_MR_WTYPE_MLD1L1R_NONMLD: /* MLO only-1 link + P2P 2+0/0+2 */
+		if (mlo_info->hwb_rf_band[hw_band] != RTW89_BAND_2G) {
+			mode = BTC_WLINK_5G;
+		} else if (mlo_info->ch_type[hw_band] == RTW89_MR_CTX2_2GHZ_5GHZ ||
+			   mlo_info->ch_type[hw_band] == RTW89_MR_CTX2_2GHZ_6GHZ) {
+			mode = BTC_WLINK_25G_MCC;
+		} else if (mlo_info->ch_type[hw_band] == RTW89_MR_CTX2_2GHZ) {
+			mode = BTC_WLINK_2G_MCC;
+		} else if (mlo_info->ch_type[hw_band] == RTW89_MR_CTX1_2GHZ) {
+			mode = BTC_WLINK_2G_SCC;
+		}
+		break;
+	case RTW89_MR_WTYPE_MLD2L1R: /* MLO_MLSR 2+0/0+2 */
+		if (mlo_info->hwb_rf_band[hw_band] != RTW89_BAND_2G)
+			mode = BTC_WLINK_5G;
+		else if (wl->role_info_v8.p2p_2g)
+			mode = BTC_WLINK_2G_GC;
+		else
+			mode = BTC_WLINK_2G_STA;
+		break;
+	case RTW89_MR_WTYPE_MLD2L1R_NONMLD: /* MLO_MLSR + P2P 2+0/0+2 */
+	case RTW89_MR_WTYPE_MLD2L2R_NONMLD: /* MLO_MLMR + P2P 1+1/2+2 */
+		/* driver may doze 1-link to
+		 * 2G+5G   -> TDMA slot switch by E2G/E5G
+		 * 5G only -> TDMA slot switch by E5G
+		 */
+		mode = BTC_WLINK_25G_MCC;
+		break;
+	case RTW89_MR_WTYPE_MLD2L2R: /* MLO_MLMR  1+1/2+2 */
+		if (mlo_info->hwb_rf_band[hw_band] != RTW89_BAND_2G) {
+			mode = BTC_WLINK_5G;
+		} else if (mlo_info->wmode[hw_band] == RTW89_MR_WMODE_1AP) {
+			mode = BTC_WLINK_2G_GO;
+		} else if (mlo_info->wmode[hw_band] == RTW89_MR_WMODE_1CLIENT) {
+			if (wl->role_info_v8.p2p_2g)
+				mode = BTC_WLINK_2G_GC;
+			else
+				mode = BTC_WLINK_2G_STA;
+		}
+		break;
+	}
+	return mode;
+}
+
+static void _update_wl_mlo_info(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc_wl_info *wl = &rtwdev->btc.cx.wl;
+	struct rtw89_btc_wl_role_info_v8 *wl_rinfo = &wl->role_info_v8;
+	struct rtw89_btc_wl_mlo_info *mlo_info = &wl->mlo_info;
+	struct rtw89_mr_chanctx_info qinfo;
+	u8 track_band = RTW89_PHY_0;
+	u8 rf_band = RTW89_BAND_2G;
+	u8 i, type;
+
+	/* parse MLO info form PHL API for each HW-band */
+	for (i = RTW89_MAC_0; i <= RTW89_MAC_1; i++) {
+		memset(&qinfo, 0, sizeof(qinfo));
+
+		rtw89_query_mr_chanctx_info(rtwdev, i, &qinfo);
+		mlo_info->wmode[i] = qinfo.wmode;
+		mlo_info->ch_type[i] = qinfo.ctxtype;
+		mlo_info->wtype = qinfo.wtype;
+
+		if (mlo_info->ch_type[i] == RTW89_MR_CTX1_5GHZ ||
+		    mlo_info->ch_type[i] == RTW89_MR_CTX2_5GHZ ||
+		    mlo_info->ch_type[i] == RTW89_MR_CTX2_5GHZ_6GHZ)
+			mlo_info->hwb_rf_band[i] = RTW89_BAND_5G;
+		else if (mlo_info->ch_type[i] == RTW89_MR_CTX1_6GHZ ||
+			 mlo_info->ch_type[i] == RTW89_MR_CTX2_6GHZ)
+			mlo_info->hwb_rf_band[i] = RTW89_BAND_6G;
+		else /* check if "2G-included" or unknown in each HW-band */
+			mlo_info->hwb_rf_band[i] = RTW89_BAND_2G;
+	}
+
+	mlo_info->link_status = rtwdev->mlo_dbcc_mode;
+	type = mlo_info->wtype;
+
+	if (mlo_info->wtype == RTW89_MR_WTYPE_MLD1L1R ||
+	    mlo_info->wtype == RTW89_MR_WTYPE_MLD2L1R ||
+	    mlo_info->wtype == RTW89_MR_WTYPE_MLD2L2R ||
+	    mlo_info->wtype == RTW89_MR_WTYPE_MLD1L1R_NONMLD ||
+	    mlo_info->wtype == RTW89_MR_WTYPE_MLD2L1R_NONMLD ||
+	    mlo_info->wtype == RTW89_MR_WTYPE_MLD2L2R_NONMLD)
+		mlo_info->mlo_en = 1;
+	else
+		mlo_info->mlo_en = 0;
+
+	if (mlo_info->ch_type[RTW89_MAC_0] != RTW89_MR_CTX_NONE &&
+	    mlo_info->ch_type[RTW89_MAC_0] != RTW89_MR_CTX_UNKNOWN &&
+	    mlo_info->ch_type[RTW89_MAC_1] != RTW89_MR_CTX_NONE &&
+	    mlo_info->ch_type[RTW89_MAC_1] != RTW89_MR_CTX_UNKNOWN)
+		mlo_info->dual_hw_band_en = 1; /* two HW-hand link exist */
+	else
+		mlo_info->dual_hw_band_en = 0;
+
+	if (mlo_info->link_status == MLO_2_PLUS_0_2RF ||
+	    mlo_info->link_status == MLO_0_PLUS_2_2RF ||
+	    mlo_info->link_status == MLO_2_PLUS_2_2RF)
+		mlo_info->mlo_adie = 2;
+	else
+		mlo_info->mlo_adie = 1;
+
+	switch (mlo_info->link_status) {
+	default:
+	case MLO_2_PLUS_0_1RF: /* 2+0 */
+	case MLO_2_PLUS_0_2RF:
+		mlo_info->rf_combination = BTC_MLO_RF_2_PLUS_0;
+		track_band = RTW89_MAC_0;
+		rf_band = mlo_info->hwb_rf_band[RTW89_MAC_0];
+		mlo_info->path_rf_band[BTC_RF_S0] = rf_band;
+		mlo_info->path_rf_band[BTC_RF_S1] = rf_band;
+
+		wl_rinfo->pta_req_band = RTW89_MAC_0;
+		wl_rinfo->dbcc_2g_phy = RTW89_PHY_0;
+		wl_rinfo->dbcc_en = 0;
+		break;
+	case MLO_0_PLUS_2_1RF: /* 0+2 */
+	case MLO_0_PLUS_2_2RF:
+		mlo_info->rf_combination = BTC_MLO_RF_0_PLUS_2;
+		track_band = RTW89_MAC_1;
+		rf_band = mlo_info->hwb_rf_band[RTW89_MAC_1];
+		mlo_info->path_rf_band[BTC_RF_S0] = rf_band;
+		mlo_info->path_rf_band[BTC_RF_S1] = rf_band;
+
+		wl_rinfo->pta_req_band = RTW89_MAC_1;
+		wl_rinfo->dbcc_2g_phy = RTW89_PHY_1;
+		wl_rinfo->dbcc_en = 0;
+		break;
+	case MLO_1_PLUS_1_1RF: /* 1+1 */
+	case MLO_1_PLUS_1_2RF: /* 1+1 */
+	case MLO_2_PLUS_2_2RF: /* 2+2 */
+	case DBCC_LEGACY: /* DBCC 1+1 */
+		if (mlo_info->link_status == MLO_2_PLUS_2_2RF)
+			mlo_info->rf_combination = BTC_MLO_RF_2_PLUS_2;
+		else
+			mlo_info->rf_combination = BTC_MLO_RF_1_PLUS_1;
+
+		if (mlo_info->hwb_rf_band[RTW89_MAC_0] == RTW89_BAND_2G)
+			track_band = RTW89_MAC_0;
+		else
+			track_band = RTW89_MAC_1;
+
+		mlo_info->path_rf_band[BTC_RF_S0] =
+					mlo_info->hwb_rf_band[RTW89_MAC_0];
+		mlo_info->path_rf_band[BTC_RF_S1] =
+					mlo_info->hwb_rf_band[RTW89_MAC_1];
+
+		/* Check ch count from ch_type @ 2.4G HW-band, and modify type */
+		if (mlo_info->ch_type[track_band] == RTW89_MR_CTX1_2GHZ)
+			type = RTW89_MR_WTYPE_NONMLD; /* only 1-role at 2G */
+		else
+			type = RTW89_MR_WTYPE_NONMLD_NONMLD;
+
+		if (mlo_info->hwb_rf_band[RTW89_MAC_0] == RTW89_BAND_2G) {
+			wl_rinfo->pta_req_band = RTW89_MAC_0;
+			wl_rinfo->dbcc_2g_phy = RTW89_PHY_0;
+		} else {
+			wl_rinfo->pta_req_band = RTW89_MAC_1;
+			wl_rinfo->dbcc_2g_phy = RTW89_PHY_1;
+		}
+
+		if (mlo_info->wmode[RTW89_MAC_0] == RTW89_MR_WMODE_NONE &&
+		    mlo_info->wmode[RTW89_MAC_1] == RTW89_MR_WMODE_NONE)
+			wl_rinfo->dbcc_en = 0;
+		else
+			wl_rinfo->dbcc_en = 1;
+		break;
+	}
+
+	wl_rinfo->link_mode = _update_wl_link_mode(rtwdev, track_band, type);
+
+	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(), mode=%s, pta_band=%d",
+		    __func__, id_to_linkmode(wl_rinfo->link_mode),
+		    wl_rinfo->pta_req_band);
+}
+
+static void _update_wl_non_mlo_info(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc_wl_info *wl = &rtwdev->btc.cx.wl;
+	struct rtw89_btc_wl_rlink *rlink = NULL;
+	struct rtw89_btc_wl_role_info_v8 *wl_rinfo = &wl->role_info_v8;
+	struct rtw89_btc_chdef cid_ch[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER] = {};
+	u8 cid_role[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER] = {};
+	u8 cid_phy[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER] = {};
+	bool b2g = false, b5g = false, outloop = false;
+	u8 mode = BTC_WLINK_NOLINK;
+	u8 cnt_2g = 0, cnt_5g = 0;
+	u8 i, j, cnt = 0;
+
+	for (j = RTW89_PHY_0; j < RTW89_PHY_NUM; j++) {
+		for (i = 0; i < RTW89_BE_BTC_WL_MAX_ROLE_NUMBER; i++) {
+			rlink = &wl_rinfo->rlink[i][j];
+
+			if (!rlink->active || !rlink->connected)
+				continue;
+
+			if (cnt >= RTW89_BE_BTC_WL_MAX_ROLE_NUMBER) {
+				outloop = true;
+				break;
+			}
+
+			cid_ch[cnt] = wl->rlink_info[i][j].chdef;
+			cid_phy[cnt] = rlink->phy;
+			cid_role[cnt] = rlink->role;
+			cnt++;
+
+			if (rlink->rf_band != RTW89_BAND_2G) {
+				cnt_5g++;
+				b5g = true;
+			} else {
+				cnt_2g++;
+				b2g = true;
+			}
+		}
+		if (outloop)
+			break;
+	}
+
+	rtw89_debug(rtwdev, RTW89_DBG_BTC,
+		    "[BTC], %s(): cnt_2g=%d, cnt_5g=%d\n", __func__, cnt_2g, cnt_5g);
+
+	wl_rinfo->dbcc_en = rtwdev->dbcc_en;
+	/* Be careful to change the following sequence!! */
+	if (cnt == 0) {
+		mode = BTC_WLINK_NOLINK;
+	} else if (!b2g && b5g) {
+		mode = BTC_WLINK_5G;
+	} else if (wl_rinfo->dbcc_en) {
+		mode = _chk_dbcc(rtwdev, cid_ch, cid_phy, cid_role, cnt);
+	} else if (b2g && b5g) {
+		mode = BTC_WLINK_25G_MCC;
+	} else if (!b5g && cnt >= 2) {
+		if (_chk_role_ch_group(&cid_ch[0], &cid_ch[1]))
+			mode = BTC_WLINK_2G_SCC;
+		else
+			mode = BTC_WLINK_2G_MCC;
+	} else if (!b5g) { /* cnt_connect = 1 */
+		mode = _get_role_link_mode(cid_role[0]);
+	}
+
+	wl_rinfo->link_mode = mode;
+}
+
+static void _modify_role_link_mode(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc_wl_info *wl = &rtwdev->btc.cx.wl;
+	struct rtw89_btc_wl_role_info_v8 *wl_rinfo = &wl->role_info_v8;
+	u8 go_cleint_exist = wl->go_client_exist;
+	u8 link_mode = wl_rinfo->link_mode;
+	u32 role_map = wl_rinfo->role_map;
+	u8 noa_exist = wl->noa_exist;
+	u32 mrole = BTC_WLMROLE_NONE;
+
+	/* if no client_joined, don't care P2P-GO/AP role */
+	if (((role_map & BIT(RTW89_WIFI_ROLE_P2P_GO)) ||
+	     (role_map & BIT(RTW89_WIFI_ROLE_AP))) && !go_cleint_exist) {
+		if (link_mode == BTC_WLINK_2G_SCC) {
+			wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+		} else if (link_mode == BTC_WLINK_2G_GO ||
+			   link_mode == BTC_WLINK_2G_AP) {
+			wl_rinfo->link_mode = BTC_WLINK_NOLINK;
+		}
+	}
+
+	/* Identify 2-Role type */
+	if  (link_mode == BTC_WLINK_2G_SCC ||
+	     link_mode == BTC_WLINK_2G_MCC ||
+	     link_mode == BTC_WLINK_25G_MCC ||
+	     link_mode == BTC_WLINK_5G) {
+		if ((role_map & BIT(RTW89_WIFI_ROLE_P2P_GO)) ||
+		    (role_map & BIT(RTW89_WIFI_ROLE_AP))) {
+			if (noa_exist)
+				mrole = BTC_WLMROLE_STA_GO_NOA;
+			else
+				mrole = BTC_WLMROLE_STA_GO;
+		} else if (role_map & BIT(RTW89_WIFI_ROLE_P2P_CLIENT)) {
+			if (noa_exist)
+				mrole = BTC_WLMROLE_STA_GC_NOA;
+			else
+				mrole = BTC_WLMROLE_STA_GC;
+		} else {
+			mrole = BTC_WLMROLE_STA_STA;
+		}
+	}
+
+	wl_rinfo->mrole_type = mrole;
+
+	rtw89_debug(rtwdev, RTW89_DBG_BTC,
+		    "[BTC], %s(): link_mode=%s, mrole_type=%d\n", __func__,
+		    id_to_linkmode(wl_rinfo->link_mode), wl_rinfo->mrole_type);
+}
+
 static void _update_wl_info_v8(struct rtw89_dev *rtwdev, u8 role_id, u8 rlink_id,
 			       enum btc_role_state state)
 {
+	struct rtw89_btc_wl_rlink *rlink = NULL;
+	struct rtw89_btc_wl_link_info *wl_linfo;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
-	struct rtw89_btc_chdef cid_ch[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER];
 	struct rtw89_btc_wl_role_info_v8 *wl_rinfo = &wl->role_info_v8;
-	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
-	bool client_joined = false, b2g = false, b5g = false;
-	u8 cid_role[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER] = {};
-	u8 cid_phy[RTW89_BE_BTC_WL_MAX_ROLE_NUMBER] = {};
-	u8 dbcc_en = 0, pta_req_band = RTW89_MAC_0;
-	u8 i, j, cnt = 0, cnt_2g = 0, cnt_5g = 0;
-	struct rtw89_btc_wl_link_info *wl_linfo;
-	struct rtw89_btc_wl_rlink *rlink = NULL;
-	u8 dbcc_2g_phy = RTW89_PHY_0;
-	u8 mode = BTC_WLINK_NOLINK;
-	u32 noa_dur = 0;
+	bool client_joined = false, noa_exist = false, p2p_exist = false;
+	bool is_5g_hi_channel = false, bg_mode = false, dbcc_en_ori;
+	u8 i, j, link_mode_ori;
+	u32 role_map = 0;
 
-	if (role_id >= RTW89_BE_BTC_WL_MAX_ROLE_NUMBER || rlink_id > RTW89_MAC_1)
+	if (role_id >= RTW89_BE_BTC_WL_MAX_ROLE_NUMBER || rlink_id >= RTW89_MAC_NUM)
 		return;
 
 	/* Extract wl->link_info[role_id][rlink_id] to wl->role_info
@@ -6565,10 +7224,8 @@ static void _update_wl_info_v8(struct rtw89_dev *rtwdev, u8 role_id, u8 rlink_id
 	 */
 
 	wl_linfo = &wl->rlink_info[role_id][rlink_id];
-	if (wl_linfo->connected == MLME_LINKING)
-		return;
-
 	rlink = &wl_rinfo->rlink[role_id][rlink_id];
+
 	rlink->role = wl_linfo->role;
 	rlink->active = wl_linfo->active; /* Doze or not */
 	rlink->pid = wl_linfo->pid;
@@ -6584,8 +7241,6 @@ static void _update_wl_info_v8(struct rtw89_dev *rtwdev, u8 role_id, u8 rlink_id
 	switch (wl_linfo->connected) {
 	case MLME_NO_LINK:
 		rlink->connected = 0;
-		if (rlink->role == RTW89_WIFI_ROLE_STATION)
-			btc->dm.leak_ap = 0;
 		break;
 	case MLME_LINKED:
 		rlink->connected = 1;
@@ -6594,133 +7249,75 @@ static void _update_wl_info_v8(struct rtw89_dev *rtwdev, u8 role_id, u8 rlink_id
 		return;
 	}
 
-	wl->is_5g_hi_channel = false;
-	wl->bg_mode = false;
-	wl_rinfo->role_map = 0;
-	wl_rinfo->p2p_2g = 0;
-	memset(cid_ch, 0, sizeof(cid_ch));
-
-	for (i = 0; i < RTW89_BE_BTC_WL_MAX_ROLE_NUMBER; i++) {
-		for (j = RTW89_MAC_0; j <= RTW89_MAC_1; j++) {
+	for (j = RTW89_MAC_0; j <= RTW89_MAC_1; j++) {
+		for (i = 0; i < RTW89_BE_BTC_WL_MAX_ROLE_NUMBER; i++) {
 			rlink = &wl_rinfo->rlink[i][j];
 
 			if (!rlink->active || !rlink->connected)
 				continue;
 
-			cnt++;
-			wl_rinfo->role_map |= BIT(rlink->role);
-
-			/* only if client connect for p2p-Go/AP */
-			if ((rlink->role == RTW89_WIFI_ROLE_P2P_GO ||
-			     rlink->role == RTW89_WIFI_ROLE_AP) &&
-			     rlink->client_cnt > 1)
-				client_joined = true;
-
-			/* Identufy if P2P-Go (GO/GC/AP) exist at 2G band*/
-			if (rlink->rf_band == RTW89_BAND_2G &&
-			    (client_joined || rlink->role == RTW89_WIFI_ROLE_P2P_CLIENT))
-				wl_rinfo->p2p_2g = 1;
+			role_map |= BIT(rlink->role);
 
 			/* only one noa-role exist */
 			if (rlink->noa && rlink->noa_dur > 0)
-				noa_dur = rlink->noa_dur;
+				noa_exist = true;
 
 			/* for WL 5G-Rx interfered with BT issue */
-			if (rlink->rf_band == RTW89_BAND_5G && rlink->ch >= 100)
-				wl->is_5g_hi_channel = 1;
+			if (rlink->rf_band == RTW89_BAND_5G) {
+				if (rlink->ch >= 100)
+					is_5g_hi_channel = true;
 
-			if ((rlink->mode & BIT(BTC_WL_MODE_11B)) ||
-			    (rlink->mode & BIT(BTC_WL_MODE_11G)))
-				wl->bg_mode = 1;
-
-			if (rtwdev->chip->para_ver & BTC_FEAT_MLO_SUPPORT)
 				continue;
+			}
 
-			cid_ch[cnt - 1] = wl_linfo->chdef;
-			cid_phy[cnt - 1] = rlink->phy;
-			cid_role[cnt - 1] = rlink->role;
-
-			if (rlink->rf_band != RTW89_BAND_2G) {
-				cnt_5g++;
-				b5g = true;
-			} else {
-				cnt_2g++;
-				b2g = true;
+			/* only if client connect for p2p-Go/AP */
+			if ((rlink->role == RTW89_WIFI_ROLE_P2P_GO ||
+			     rlink->role == RTW89_WIFI_ROLE_AP) &&
+			     rlink->client_cnt > 1) {
+				p2p_exist = true;
+				client_joined = true;
 			}
-		}
-	}
 
-	if (rtwdev->chip->para_ver & BTC_FEAT_MLO_SUPPORT) {
-		rtw89_debug(rtwdev, RTW89_DBG_BTC,
-			    "[BTC] rlink cnt_2g=%d cnt_5g=%d\n", cnt_2g, cnt_5g);
-		rtw89_warn(rtwdev, "not support MLO feature yet");
-	} else {
-		dbcc_en = rtwdev->dbcc_en;
+			/* Identify if P2P-Go (GO/GC/AP) exist at 2G band */
+			if (rlink->role == RTW89_WIFI_ROLE_P2P_CLIENT)
+				p2p_exist = true;
 
-		/* Be careful to change the following sequence!! */
-		if (cnt == 0) {
-			mode = BTC_WLINK_NOLINK;
-		} else if (!b2g && b5g) {
-			mode = BTC_WLINK_5G;
-		} else if (wl_rinfo->role_map & BIT(RTW89_WIFI_ROLE_NAN)) {
-			mode = BTC_WLINK_2G_NAN;
-		} else if (cnt > BTC_TDMA_WLROLE_MAX) {
-			mode = BTC_WLINK_OTHER;
-		} else if (dbcc_en) {
-			mode = _chk_dbcc(rtwdev, cid_ch, cid_phy, cid_role,
-					 &dbcc_2g_phy);
-		} else if (b2g && b5g && cnt == 2) {
-			mode = BTC_WLINK_25G_MCC;
-		} else if (!b5g && cnt == 2) { /* cnt_connect = 2 */
-			if (_chk_role_ch_group(&cid_ch[0], &cid_ch[cnt - 1]))
-				mode = BTC_WLINK_2G_SCC;
-			else
-				mode = BTC_WLINK_2G_MCC;
-		} else if (!b5g && cnt == 1) { /* cnt_connect = 1 */
-			mode = _get_role_link_mode(cid_role[0]);
+			if ((rlink->mode & BIT(BTC_WL_MODE_11B)) ||
+			    (rlink->mode & BIT(BTC_WL_MODE_11G)))
+				bg_mode = true;
 		}
 	}
 
-	wl_rinfo->link_mode = mode;
-	wl_rinfo->connect_cnt = cnt;
-	if (wl_rinfo->connect_cnt == 0)
-		wl_rinfo->role_map = BIT(RTW89_WIFI_ROLE_NONE);
-	_update_role_link_mode(rtwdev, client_joined, noa_dur);
+	link_mode_ori = wl_rinfo->link_mode;
+	wl->is_5g_hi_channel = is_5g_hi_channel;
+	wl->bg_mode = bg_mode;
+	wl->go_client_exist = client_joined;
+	wl->noa_exist = noa_exist;
+	wl_rinfo->p2p_2g = p2p_exist;
+	wl_rinfo->role_map = role_map;
 
-	wl_rinfo->dbcc_2g_phy = dbcc_2g_phy;
-	if (wl_rinfo->dbcc_en != dbcc_en) {
-		wl_rinfo->dbcc_en = dbcc_en;
-		wl_rinfo->dbcc_chg = 1;
-		btc->cx.cnt_wl[BTC_WCNT_DBCC_CHG]++;
+	dbcc_en_ori = wl_rinfo->dbcc_en;
+
+	if (rtwdev->chip->para_ver & BTC_FEAT_MLO_SUPPORT) {
+		/* for MLO-supported, link-mode from driver directly */
+		_update_wl_mlo_info(rtwdev);
 	} else {
-		wl_rinfo->dbcc_chg = 0;
+		/* for non-MLO-supported, link-mode by BTC */
+		_update_wl_non_mlo_info(rtwdev);
 	}
 
-	if (wl_rinfo->dbcc_en) {
-		memset(wl_dinfo, 0, sizeof(struct rtw89_btc_wl_dbcc_info));
+	_modify_role_link_mode(rtwdev);
 
-		if (mode == BTC_WLINK_5G) {
-			pta_req_band = RTW89_PHY_0;
-			wl_dinfo->op_band[RTW89_PHY_0] = RTW89_BAND_5G;
-			wl_dinfo->op_band[RTW89_PHY_1] = RTW89_BAND_2G;
-		} else if (wl_rinfo->dbcc_2g_phy == RTW89_PHY_1) {
-			pta_req_band = RTW89_PHY_1;
-			wl_dinfo->op_band[RTW89_PHY_0] = RTW89_BAND_5G;
-			wl_dinfo->op_band[RTW89_PHY_1] = RTW89_BAND_2G;
-		} else {
-			pta_req_band = RTW89_PHY_0;
-			wl_dinfo->op_band[RTW89_PHY_0] = RTW89_BAND_2G;
-			wl_dinfo->op_band[RTW89_PHY_1] = RTW89_BAND_5G;
-		}
-		_update_dbcc_band(rtwdev, RTW89_PHY_0);
-		_update_dbcc_band(rtwdev, RTW89_PHY_1);
-	}
+	if (link_mode_ori != wl_rinfo->link_mode)
+		wl->link_mode_chg = true;
 
-	wl_rinfo->pta_req_band = pta_req_band;
-	_fw_set_drv_info(rtwdev, CXDRVINFO_ROLE);
+	if (wl_rinfo->dbcc_en != dbcc_en_ori) {
+		wl->dbcc_chg = true;
+		btc->cx.cnt_wl[BTC_WCNT_DBCC_CHG]++;
+	}
 }
 
-void rtw89_coex_act1_work(struct work_struct *work)
+void rtw89_coex_act1_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						coex_act1_work.work);
@@ -6729,7 +7326,8 @@ void rtw89_coex_act1_work(struct work_struct *work)
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): enter\n", __func__);
 	dm->cnt_notify[BTC_NCNT_TIMER]++;
 	if (wl->status.map._4way)
@@ -6738,10 +7336,9 @@ void rtw89_coex_act1_work(struct work_struct *work)
 		wl->status.map.connecting = false;
 
 	_run_coex(rtwdev, BTC_RSN_ACT1_WORK);
-	mutex_unlock(&rtwdev->mutex);
 }
 
-void rtw89_coex_bt_devinfo_work(struct work_struct *work)
+void rtw89_coex_bt_devinfo_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						coex_bt_devinfo_work.work);
@@ -6749,15 +7346,15 @@ void rtw89_coex_bt_devinfo_work(struct work_struct *work)
 	struct rtw89_btc_dm *dm = &rtwdev->btc.dm;
 	struct rtw89_btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): enter\n", __func__);
 	dm->cnt_notify[BTC_NCNT_TIMER]++;
 	a2dp->play_latency = 0;
 	_run_coex(rtwdev, BTC_RSN_BT_DEVINFO_WORK);
-	mutex_unlock(&rtwdev->mutex);
 }
 
-void rtw89_coex_rfk_chk_work(struct work_struct *work)
+void rtw89_coex_rfk_chk_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						coex_rfk_chk_work.work);
@@ -6766,7 +7363,8 @@ void rtw89_coex_rfk_chk_work(struct work_struct *work)
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): enter\n", __func__);
 	dm->cnt_notify[BTC_NCNT_TIMER]++;
 	if (wl->rfk_info.state != BTC_WRFK_STOP) {
@@ -6778,7 +7376,6 @@ void rtw89_coex_rfk_chk_work(struct work_struct *work)
 		_write_scbd(rtwdev, BTC_WSCB_WLRFK, false);
 		_run_coex(rtwdev, BTC_RSN_RFK_CHK_WORK);
 	}
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static void _update_bt_scbd(struct rtw89_dev *rtwdev, bool only_update)
@@ -6840,12 +7437,33 @@ static void _update_bt_scbd(struct rtw89_dev *rtwdev, bool only_update)
 	bt->rfk_info.map.req = !!(val & BTC_BSCB_RFK_REQ);
 	bt->hi_lna_rx = !!(val & BTC_BSCB_BT_HILNA);
 	bt->link_info.status.map.connect = !!(val & BTC_BSCB_BT_CONNECT);
+	if (bt->run_patch_code != !!(val & BTC_BSCB_PATCH_CODE))
+		status_change = true;
 	bt->run_patch_code = !!(val & BTC_BSCB_PATCH_CODE);
 
 	if (!only_update && status_change)
 		_run_coex(rtwdev, BTC_RSN_UPDATE_BT_SCBD);
 }
 
+#define BTC_BTINFO_PWR_LEN 5
+static void _update_bt_txpwr_info(struct rtw89_dev *rtwdev, u8 *buf, u32 len)
+{
+	struct rtw89_btc_bt_info *bt = &rtwdev->btc.cx.bt;
+	struct rtw89_btc_bt_link_info *b = &bt->link_info;
+
+	if (len != BTC_BTINFO_PWR_LEN)
+		return;
+
+	if (!memcmp(bt->txpwr_info, buf, sizeof(bt->txpwr_info))) {
+		rtw89_debug(rtwdev, RTW89_DBG_BTC,
+			    "[BTC], %s return by info duplicate!\n", __func__);
+		return;
+	}
+
+	memcpy(bt->txpwr_info, buf, BTC_BTINFO_MAX);
+	memcpy(&b->bt_txpwr_desc, &buf[2], sizeof(b->bt_txpwr_desc));
+}
+
 static bool _chk_wl_rfk_request(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -6882,7 +7500,7 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 	struct rtw89_btc_wl_role_info_v8 *wl_rinfo_v8 = &wl->role_info_v8;
 	u8 mode, igno_bt, always_freerun;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	dm->run_reason = reason;
 	_update_dm_step(rtwdev, reason);
@@ -7002,7 +7620,7 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 		goto exit;
 	}
 
-	if (wl->status.val & btc_scanning_map.val) {
+	if (wl->status.val & btc_scanning_map.val && !wl->rfk_info.con_rfk) {
 		_action_wl_scan(rtwdev);
 		bt->scan_rx_low_pri = true;
 		goto exit;
@@ -7187,7 +7805,7 @@ void rtw89_btc_ntfy_scan_start(struct rtw89_dev *rtwdev, u8 phy_idx, u8 band)
 		    "[BTC], %s(): phy_idx=%d, band=%d\n",
 		    __func__, phy_idx, band);
 
-	if (phy_idx >= RTW89_PHY_MAX)
+	if (phy_idx >= RTW89_PHY_NUM)
 		return;
 
 	btc->dm.cnt_notify[BTC_NCNT_SCAN_START]++;
@@ -7223,6 +7841,8 @@ void rtw89_btc_ntfy_scan_finish(struct rtw89_dev *rtwdev, u8 phy_idx)
 		_fw_set_drv_info(rtwdev, CXDRVINFO_DBCC);
 	}
 
+	btc->dm.tdma_instant_excute = 1;
+
 	_run_coex(rtwdev, BTC_RSN_NTFY_SCAN_FINISH);
 }
 
@@ -7235,7 +7855,7 @@ void rtw89_btc_ntfy_switch_band(struct rtw89_dev *rtwdev, u8 phy_idx, u8 band)
 		    "[BTC], %s(): phy_idx=%d, band=%d\n",
 		    __func__, phy_idx, band);
 
-	if (phy_idx >= RTW89_PHY_MAX)
+	if (phy_idx >= RTW89_PHY_NUM)
 		return;
 
 	btc->dm.cnt_notify[BTC_NCNT_SWITCH_BAND]++;
@@ -7284,7 +7904,7 @@ void rtw89_btc_ntfy_specific_packet(struct rtw89_dev *rtwdev,
 			    "[BTC], %s(): EAPOL_End cnt=%d\n",
 			    __func__, cnt);
 		wl->status.map._4way = false;
-		cancel_delayed_work(&rtwdev->coex_act1_work);
+		wiphy_delayed_work_cancel(rtwdev->hw->wiphy, &rtwdev->coex_act1_work);
 		break;
 	case PACKET_ARP:
 		cnt = ++cx->cnt_wl[BTC_WCNT_ARP];
@@ -7303,56 +7923,56 @@ void rtw89_btc_ntfy_specific_packet(struct rtw89_dev *rtwdev,
 	}
 
 	if (delay_work) {
-		cancel_delayed_work(&rtwdev->coex_act1_work);
-		ieee80211_queue_delayed_work(rtwdev->hw,
-					     &rtwdev->coex_act1_work, delay);
+		wiphy_delayed_work_cancel(rtwdev->hw->wiphy, &rtwdev->coex_act1_work);
+		wiphy_delayed_work_queue(rtwdev->hw->wiphy,
+					 &rtwdev->coex_act1_work, delay);
 	}
 
 	btc->dm.cnt_notify[BTC_NCNT_SPECIAL_PACKET]++;
 	_run_coex(rtwdev, BTC_RSN_NTFY_SPECIFIC_PACKET);
 }
 
-void rtw89_btc_ntfy_eapol_packet_work(struct work_struct *work)
+void rtw89_btc_ntfy_eapol_packet_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						btc.eapol_notify_work);
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 	rtw89_btc_ntfy_specific_packet(rtwdev, PACKET_EAPOL);
-	mutex_unlock(&rtwdev->mutex);
 }
 
-void rtw89_btc_ntfy_arp_packet_work(struct work_struct *work)
+void rtw89_btc_ntfy_arp_packet_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						btc.arp_notify_work);
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_btc_ntfy_specific_packet(rtwdev, PACKET_ARP);
-	mutex_unlock(&rtwdev->mutex);
 }
 
-void rtw89_btc_ntfy_dhcp_packet_work(struct work_struct *work)
+void rtw89_btc_ntfy_dhcp_packet_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						btc.dhcp_notify_work);
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 	rtw89_btc_ntfy_specific_packet(rtwdev, PACKET_DHCP);
-	mutex_unlock(&rtwdev->mutex);
 }
 
-void rtw89_btc_ntfy_icmp_packet_work(struct work_struct *work)
+void rtw89_btc_ntfy_icmp_packet_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						btc.icmp_notify_work);
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 	rtw89_btc_ntfy_specific_packet(rtwdev, PACKET_ICMP);
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static u8 _update_bt_rssi_level(struct rtw89_dev *rtwdev, u8 rssi)
@@ -7531,9 +8151,9 @@ static void _update_bt_info(struct rtw89_dev *rtwdev, u8 *buf, u32 len)
 		a2dp->vendor_id = 0;
 		a2dp->flush_time = 0;
 		a2dp->play_latency = 1;
-		ieee80211_queue_delayed_work(rtwdev->hw,
-					     &rtwdev->coex_bt_devinfo_work,
-					     RTW89_COEX_BT_DEVINFO_WORK_PERIOD);
+		wiphy_delayed_work_queue(rtwdev->hw->wiphy,
+					 &rtwdev->coex_bt_devinfo_work,
+					 RTW89_COEX_BT_DEVINFO_WORK_PERIOD);
 	}
 
 	_run_coex(rtwdev, BTC_RSN_UPDATE_BT_INFO);
@@ -7627,7 +8247,6 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev,
 
 	wlinfo = &wl->link_info[r.pid];
 
-	rlink_id = 0; /* to do */
 	if (ver->fwlrole == 0) {
 		*wlinfo = r;
 		_update_wl_info(rtwdev);
@@ -7641,6 +8260,7 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev,
 		*wlinfo = r;
 		_update_wl_info_v7(rtwdev, r.pid);
 	} else if (ver->fwlrole == 8) {
+		rlink_id = rtwvif_link->mac_idx;
 		wlinfo = &wl->rlink_info[r.pid][rlink_id];
 		*wlinfo = r;
 		link_mode_ori = wl->role_info_v8.link_mode;
@@ -7671,7 +8291,8 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev,
 	else
 		wl->status.map.connecting = 0;
 
-	if (state == BTC_ROLE_MSTS_STA_DIS_CONN)
+	if (state == BTC_ROLE_MSTS_STA_DIS_CONN ||
+	    state == BTC_ROLE_MSTS_STA_CONN_END)
 		wl->status.map._4way = false;
 
 	_run_coex(rtwdev, BTC_RSN_NTFY_ROLE_INFO);
@@ -7781,7 +8402,7 @@ static bool _ntfy_wl_rfk(struct rtw89_dev *rtwdev, u8 phy_path,
 		wl->rfk_info.state = BTC_WRFK_STOP;
 
 		_write_scbd(rtwdev, BTC_WSCB_WLRFK, false);
-		cancel_delayed_work(&rtwdev->coex_rfk_chk_work);
+		wiphy_delayed_work_cancel(rtwdev->hw->wiphy, &rtwdev->coex_rfk_chk_work);
 		break;
 	default:
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -7795,9 +8416,9 @@ static bool _ntfy_wl_rfk(struct rtw89_dev *rtwdev, u8 phy_path,
 			_run_coex(rtwdev, BTC_RSN_NTFY_WL_RFK);
 
 		if (wl->rfk_info.state == BTC_WRFK_START)
-			ieee80211_queue_delayed_work(rtwdev->hw,
-						     &rtwdev->coex_rfk_chk_work,
-						     RTW89_COEX_RFK_CHK_WORK_PERIOD);
+			wiphy_delayed_work_queue(rtwdev->hw->wiphy,
+						 &rtwdev->coex_rfk_chk_work,
+						 RTW89_COEX_RFK_CHK_WORK_PERIOD);
 	}
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -7815,6 +8436,8 @@ void rtw89_btc_ntfy_wl_rfk(struct rtw89_dev *rtwdev, u8 phy_map,
 	bool allow;
 	int ret;
 
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
 	band = FIELD_GET(BTC_RFK_BAND_MAP, phy_map);
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK,
@@ -7882,7 +8505,11 @@ void __rtw89_btc_ntfy_wl_sta_iter(struct rtw89_vif_link *rtwvif_link,
 	rssi = ewma_rssi_read(&rtwsta_link->avg_rssi) >> RSSI_FACTOR;
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], rssi=%d\n", rssi);
 
-	link_info = &wl->link_info[port];
+	if (btc->ver->fwlrole != 8)
+		link_info = &wl->link_info[port];
+	else
+		link_info = &wl->rlink_info[port][rtwvif_link->mac_idx];
+
 	link_info->stat.traffic = *stats;
 	link_info_t = &link_info->stat.traffic;
 
@@ -7963,13 +8590,12 @@ void __rtw89_btc_ntfy_wl_sta_iter(struct rtw89_vif_link *rtwvif_link,
 		r1->active_role_v1[port].rx_lvl = stats->rx_tfc_lv;
 		r1->active_role_v1[port].tx_rate = rtwsta_link->ra_report.hw_rate;
 		r1->active_role_v1[port].rx_rate = rtwsta_link->rx_hw_rate;
-	} else if (ver->fwlrole == 2) {
-		dm->trx_info.tx_lvl = stats->tx_tfc_lv;
-		dm->trx_info.rx_lvl = stats->rx_tfc_lv;
-		dm->trx_info.tx_rate = rtwsta_link->ra_report.hw_rate;
-		dm->trx_info.rx_rate = rtwsta_link->rx_hw_rate;
 	}
 
+	dm->trx_info.tx_lvl = stats->tx_tfc_lv;
+	dm->trx_info.rx_lvl = stats->rx_tfc_lv;
+	dm->trx_info.tx_rate = rtwsta_link->ra_report.hw_rate;
+	dm->trx_info.rx_rate = rtwsta_link->rx_hw_rate;
 	dm->trx_info.tx_tp = link_info_t->tx_throughput;
 	dm->trx_info.rx_tp = link_info_t->rx_throughput;
 
@@ -8076,6 +8702,8 @@ static u8 rtw89_btc_c2h_get_index_by_ver(struct rtw89_dev *rtwdev, u8 func)
 			return BTF_EVNT_BUF_OVERFLOW;
 		else if (ver->fwc2hfunc == 2)
 			return func;
+		else if (ver->fwc2hfunc == 3)
+			return BTF_EVNT_BUF_OVERFLOW;
 		else
 			return BTF_EVNT_MAX;
 	case BTF_EVNT_BUF_OVERFLOW:
@@ -8085,11 +8713,20 @@ static u8 rtw89_btc_c2h_get_index_by_ver(struct rtw89_dev *rtwdev, u8 func)
 			return BTF_EVNT_C2H_LOOPBACK;
 		else if (ver->fwc2hfunc == 2)
 			return func;
+		else if (ver->fwc2hfunc == 3)
+			return BTF_EVNT_C2H_LOOPBACK;
 		else
 			return BTF_EVNT_MAX;
 	case BTF_EVNT_C2H_LOOPBACK:
 		if (ver->fwc2hfunc == 2)
 			return func;
+		else if (ver->fwc2hfunc == 3)
+			return BTF_EVNT_BT_LEAUDIO_INFO;
+		else
+			return BTF_EVNT_MAX;
+	case BTF_EVNT_BT_QUERY_TXPWR:
+		if (ver->fwc2hfunc == 3)
+			return func;
 		else
 			return BTF_EVNT_MAX;
 	case BTF_EVNT_MAX:
@@ -8115,6 +8752,7 @@ void rtw89_btc_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 		return;
 
 	func = rtw89_btc_c2h_get_index_by_ver(rtwdev, func);
+	pfwinfo->cnt_c2h++;
 
 	switch (func) {
 	case BTF_EVNT_BUF_OVERFLOW:
@@ -8151,12 +8789,15 @@ void rtw89_btc_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 	case BTF_EVNT_CX_RUNINFO:
 		btc->dm.cnt_dm[BTC_DCNT_CX_RUNINFO]++;
 		break;
+	case BTF_EVNT_BT_QUERY_TXPWR:
+		btc->cx.cnt_bt[BTC_BCNT_BTTXPWR_UPDATE]++;
+		_update_bt_txpwr_info(rtwdev, buf, len);
 	}
 }
 
 #define BTC_CX_FW_OFFLOAD 0
 
-static void _show_cx_info(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_cx_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	union rtw89_btc_module_info *md = &rtwdev->btc.mdinfo;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
@@ -8168,40 +8809,41 @@ static void _show_cx_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	u32 ver_main = 0, ver_sub = 0, ver_hotfix = 0, id_branch = 0;
 	u8 cv, rfe, iso, ant_num, ant_single_pos;
+	char *p = buf, *end = buf + bufsz;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_CX))
-		return;
-
-	dm->cnt_notify[BTC_NCNT_SHOW_COEX_INFO]++;
+		return 0;
 
-	seq_printf(m, "========== [BTC COEX INFO (%d)] ==========\n",
-		   chip->chip_id);
+	p += scnprintf(p, end - p,
+		       "\n========== [BTC COEX INFO (%s)] ==========\n",
+		       chip_id_str(chip->chip_id));
 
 	ver_main = FIELD_GET(GENMASK(31, 24), RTW89_COEX_VERSION);
 	ver_sub = FIELD_GET(GENMASK(23, 16), RTW89_COEX_VERSION);
 	ver_hotfix = FIELD_GET(GENMASK(15, 8), RTW89_COEX_VERSION);
 	id_branch = FIELD_GET(GENMASK(7, 0), RTW89_COEX_VERSION);
-	seq_printf(m, " %-15s : Coex:%d.%d.%d(branch:%d), ",
-		   "[coex_version]", ver_main, ver_sub, ver_hotfix, id_branch);
+	p += scnprintf(p, end - p, " %-15s : Coex:%d.%d.%d(branch:%d), ",
+		       "[coex_version]", ver_main, ver_sub, ver_hotfix,
+		       id_branch);
 
 	ver_main = FIELD_GET(GENMASK(31, 24), wl->ver_info.fw_coex);
 	ver_sub = FIELD_GET(GENMASK(23, 16), wl->ver_info.fw_coex);
 	ver_hotfix = FIELD_GET(GENMASK(15, 8), wl->ver_info.fw_coex);
 	id_branch = FIELD_GET(GENMASK(7, 0), wl->ver_info.fw_coex);
-	seq_printf(m, "WL_FW_coex:%d.%d.%d(branch:%d)",
-		   ver_main, ver_sub, ver_hotfix, id_branch);
+	p += scnprintf(p, end - p, "WL_FW_coex:%d.%d.%d(branch:%d)",
+		       ver_main, ver_sub, ver_hotfix, id_branch);
 
 	ver_main = FIELD_GET(GENMASK(31, 24), chip->wlcx_desired);
 	ver_sub = FIELD_GET(GENMASK(23, 16), chip->wlcx_desired);
 	ver_hotfix = FIELD_GET(GENMASK(15, 8), chip->wlcx_desired);
-	seq_printf(m, "(%s, desired:%d.%d.%d), ",
-		   (wl->ver_info.fw_coex >= chip->wlcx_desired ?
-		   "Match" : "Mismatch"), ver_main, ver_sub, ver_hotfix);
+	p += scnprintf(p, end - p, "(%s, desired:%d.%d.%d), ",
+		       (wl->ver_info.fw_coex >= chip->wlcx_desired ?
+			"Match" : "Mismatch"), ver_main, ver_sub, ver_hotfix);
 
-	seq_printf(m, "BT_FW_coex:%d(%s, desired:%d)\n",
-		   bt->ver_info.fw_coex,
-		   (bt->ver_info.fw_coex >= chip->btcx_desired ?
-		   "Match" : "Mismatch"), chip->btcx_desired);
+	p += scnprintf(p, end - p, "BT_FW_coex:%d(%s, desired:%d)\n",
+		       bt->ver_info.fw_coex,
+		       (bt->ver_info.fw_coex >= ver->bt_desired ?
+			"Match" : "Mismatch"), ver->bt_desired);
 
 	if (bt->enable.now && bt->ver_info.fw == 0)
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_VER_INFO, true);
@@ -8212,10 +8854,11 @@ static void _show_cx_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	ver_sub = FIELD_GET(GENMASK(23, 16), wl->ver_info.fw);
 	ver_hotfix = FIELD_GET(GENMASK(15, 8), wl->ver_info.fw);
 	id_branch = FIELD_GET(GENMASK(7, 0), wl->ver_info.fw);
-	seq_printf(m, " %-15s : WL_FW:%d.%d.%d.%d, BT_FW:0x%x(%s)\n",
-		   "[sub_module]",
-		   ver_main, ver_sub, ver_hotfix, id_branch,
-		   bt->ver_info.fw, bt->run_patch_code ? "patch" : "ROM");
+	p += scnprintf(p, end - p,
+		       " %-15s : WL_FW:%d.%d.%d.%d, BT_FW:0x%x(%s)\n",
+		       "[sub_module]",
+		       ver_main, ver_sub, ver_hotfix, id_branch,
+		       bt->ver_info.fw, bt->run_patch_code ? "patch" : "ROM");
 
 	if (ver->fcxinit == 7) {
 		cv = md->md_v7.kt_ver;
@@ -8231,79 +8874,74 @@ static void _show_cx_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 		ant_single_pos = md->md.ant.single_pos;
 	}
 
-	seq_printf(m, " %-15s : cv:%x, rfe_type:0x%x, ant_iso:%d, ant_pg:%d, %s",
-		   "[hw_info]", cv, rfe, iso, ant_num,
-		   ant_num > 1 ? "" :
-		   ant_single_pos ? "1Ant_Pos:S1, " : "1Ant_Pos:S0, ");
+	p += scnprintf(p, end - p,
+		       " %-15s : cv:%x, rfe_type:0x%x, ant_iso:%d, ant_pg:%d, %s",
+		       "[hw_info]", cv, rfe, iso, ant_num,
+		       ant_num > 1 ? "" :
+		       ant_single_pos ? "1Ant_Pos:S1, " : "1Ant_Pos:S0, ");
+
+	p += scnprintf(p, end - p,
+		       "3rd_coex:%d, dbcc:%d, tx_num:%d, rx_num:%d\n",
+		       btc->cx.other.type, rtwdev->dbcc_en, hal->tx_nss,
+		       hal->rx_nss);
 
-	seq_printf(m, "3rd_coex:%d, dbcc:%d, tx_num:%d, rx_num:%d\n",
-		   btc->cx.other.type, rtwdev->dbcc_en, hal->tx_nss,
-		   hal->rx_nss);
+	return p - buf;
 }
 
-static void _show_wl_role_info(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_wl_role_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_link_info *plink = NULL;
-	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
-	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
 	struct rtw89_traffic_stats *t;
-	u8 i;
+	char *p = buf, *end = buf + bufsz;
+	u8 i, j;
 
-	if (rtwdev->dbcc_en) {
-		seq_printf(m,
-			   " %-15s : PHY0_band(op:%d/scan:%d/real:%d), ",
-			   "[dbcc_info]", wl_dinfo->op_band[RTW89_PHY_0],
-			   wl_dinfo->scan_band[RTW89_PHY_0],
-			   wl_dinfo->real_band[RTW89_PHY_0]);
-		seq_printf(m,
-			   "PHY1_band(op:%d/scan:%d/real:%d)\n",
-			   wl_dinfo->op_band[RTW89_PHY_1],
-			   wl_dinfo->scan_band[RTW89_PHY_1],
-			   wl_dinfo->real_band[RTW89_PHY_1]);
-	}
-
-	for (i = 0; i < RTW89_PORT_NUM; i++) {
-		if (btc->ver->fwlrole == 8)
-			plink = &btc->cx.wl.rlink_info[i][0];
-		else
-			plink = &btc->cx.wl.link_info[i];
+	for (i = 0; i < btc->ver->max_role_num; i++) {
+		for (j = 0; j < RTW89_MAC_NUM; j++) {
+			if (btc->ver->fwlrole == 8)
+				plink = &btc->cx.wl.rlink_info[i][j];
+			else
+				plink = &btc->cx.wl.link_info[i];
 
-		if (!plink->active)
-			continue;
+			if (!plink->active)
+				continue;
 
-		seq_printf(m,
-			   " [port_%d]        : role=%d(phy-%d), connect=%d(client_cnt=%d), mode=%d, center_ch=%d, bw=%d",
-			   plink->pid, (u32)plink->role, plink->phy,
-			   (u32)plink->connected, plink->client_cnt - 1,
-			   (u32)plink->mode, plink->ch, (u32)plink->bw);
+			p += scnprintf(p, end - p,
+				       " [port_%d]        : role=%d(phy-%d), connect=%s(client_cnt=%d), mode=%d, center_ch=%d, bw=%d",
+				       plink->pid, plink->role, plink->phy,
+				       id_to_mlme_state(plink->connected),
+				       plink->client_cnt - 1, plink->mode,
+				       plink->ch, plink->bw);
 
-		if (plink->connected == MLME_NO_LINK)
-			continue;
+			if (plink->connected == MLME_NO_LINK)
+				continue;
 
-		seq_printf(m,
-			   ", mac_id=%d, max_tx_time=%dus, max_tx_retry=%d\n",
-			   plink->mac_id, plink->tx_time, plink->tx_retry);
+			p += scnprintf(p, end - p,
+				       ", mac_id=%d, max_tx_time=%dus, max_tx_retry=%d\n",
+				       plink->mac_id, plink->tx_time, plink->tx_retry);
 
-		seq_printf(m,
-			   " [port_%d]        : rssi=-%ddBm(%d), busy=%d, dir=%s, ",
-			   plink->pid, 110 - plink->stat.rssi,
-			   plink->stat.rssi, plink->busy,
-			   plink->dir == RTW89_TFC_UL ? "UL" : "DL");
+			p += scnprintf(p, end - p,
+				       " [port_%d]        : rssi=-%ddBm(%d), busy=%d, dir=%s, ",
+				       plink->pid, 110 - plink->stat.rssi,
+				       plink->stat.rssi, plink->busy,
+				       plink->dir == RTW89_TFC_UL ? "UL" : "DL");
 
-		t = &plink->stat.traffic;
+			t = &plink->stat.traffic;
 
-		seq_printf(m,
-			   "tx[rate:%d/busy_level:%d], ",
-			   (u32)t->tx_rate, t->tx_tfc_lv);
+			p += scnprintf(p, end - p,
+				       "tx[rate:%d/busy_level:%d], ",
+				       t->tx_rate, t->tx_tfc_lv);
 
-		seq_printf(m, "rx[rate:%d/busy_level:%d/drop:%d]\n",
-			   (u32)t->rx_rate,
-			   t->rx_tfc_lv, plink->rx_rate_drop_cnt);
+			p += scnprintf(p, end - p,
+				       "rx[rate:%d/busy_level:%d/drop:%d]\n",
+				       t->rx_rate,
+				       t->rx_tfc_lv, plink->rx_rate_drop_cnt);
+		}
 	}
+	return p - buf;
 }
 
-static void _show_wl_info(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_wl_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
@@ -8314,12 +8952,13 @@ static void _show_wl_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	struct rtw89_btc_wl_role_info_v7 *wl_rinfo_v7 = &wl->role_info_v7;
 	struct rtw89_btc_wl_role_info_v8 *wl_rinfo_v8 = &wl->role_info_v8;
+	char *p = buf, *end = buf + bufsz;
 	u8 mode;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_WL))
-		return;
+		return 0;
 
-	seq_puts(m, "========== [WL Status] ==========\n");
+	p += scnprintf(p, end - p, "========== [WL Status] ==========\n");
 
 	if (ver->fwlrole == 0)
 		mode = wl_rinfo->link_mode;
@@ -8332,24 +8971,28 @@ static void _show_wl_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	else if (ver->fwlrole == 8)
 		mode = wl_rinfo_v8->link_mode;
 	else
-		return;
+		goto out;
+
+	p += scnprintf(p, end - p, " %-15s : link_mode:%s, ", "[status]",
+		       id_to_linkmode(mode));
 
-	seq_printf(m, " %-15s : link_mode:%d, ", "[status]", mode);
+	p += scnprintf(p, end - p,
+		       "rf_off:%d, power_save:%d, scan:%s(band:%d/phy_map:0x%x), ",
+		       wl->status.map.rf_off, wl->status.map.lps,
+		       wl->status.map.scan ? "Y" : "N",
+		       wl->scan_info.band[RTW89_PHY_0], wl->scan_info.phy_map);
 
-	seq_printf(m,
-		   "rf_off:%d, power_save:%d, scan:%s(band:%d/phy_map:0x%x), ",
-		   wl->status.map.rf_off, wl->status.map.lps,
-		   wl->status.map.scan ? "Y" : "N",
-		   wl->scan_info.band[RTW89_PHY_0], wl->scan_info.phy_map);
+	p += scnprintf(p, end - p,
+		       "connecting:%s, roam:%s, 4way:%s, init_ok:%s\n",
+		       wl->status.map.connecting ? "Y" : "N",
+		       wl->status.map.roaming ?  "Y" : "N",
+		       wl->status.map._4way ? "Y" : "N",
+		       wl->status.map.init_ok ? "Y" : "N");
 
-	seq_printf(m,
-		   "connecting:%s, roam:%s, 4way:%s, init_ok:%s\n",
-		   wl->status.map.connecting ? "Y" : "N",
-		   wl->status.map.roaming ?  "Y" : "N",
-		   wl->status.map._4way ? "Y" : "N",
-		   wl->status.map.init_ok ? "Y" : "N");
+	p += _show_wl_role_info(rtwdev, p, end - p);
 
-	_show_wl_role_info(rtwdev, m);
+out:
+	return p - buf;
 }
 
 enum btc_bt_a2dp_type {
@@ -8358,7 +9001,7 @@ enum btc_bt_a2dp_type {
 	BTC_A2DP_TWS_RELAY = 2,
 };
 
-static void _show_bt_profile_info(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_bt_profile_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
@@ -8366,182 +9009,219 @@ static void _show_bt_profile_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_bt_hid_desc hid = bt_linfo->hid_desc;
 	struct rtw89_btc_bt_a2dp_desc a2dp = bt_linfo->a2dp_desc;
 	struct rtw89_btc_bt_pan_desc pan = bt_linfo->pan_desc;
+	char *p = buf, *end = buf + bufsz;
 
 	if (hfp.exist) {
-		seq_printf(m, " %-15s : type:%s, sut_pwr:%d, golden-rx:%d",
-			   "[HFP]", (hfp.type == 0 ? "SCO" : "eSCO"),
-			   bt_linfo->sut_pwr_level[0],
-			   bt_linfo->golden_rx_shift[0]);
+		p += scnprintf(p, end - p,
+			       " %-15s : type:%s, sut_pwr:%d, golden-rx:%d",
+			       "[HFP]", (hfp.type == 0 ? "SCO" : "eSCO"),
+			       bt_linfo->sut_pwr_level[0],
+			       bt_linfo->golden_rx_shift[0]);
 	}
 
 	if (hid.exist) {
-		seq_printf(m,
-			   "\n\r %-15s : type:%s%s%s%s%s pair-cnt:%d, sut_pwr:%d, golden-rx:%d\n",
-			   "[HID]",
-			   hid.type & BTC_HID_218 ? "2/18," : "",
-			   hid.type & BTC_HID_418 ? "4/18," : "",
-			   hid.type & BTC_HID_BLE ? "BLE," : "",
-			   hid.type & BTC_HID_RCU ? "RCU," : "",
-			   hid.type & BTC_HID_RCU_VOICE ? "RCU-Voice," : "",
-			   hid.pair_cnt, bt_linfo->sut_pwr_level[1],
-			   bt_linfo->golden_rx_shift[1]);
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : type:%s%s%s%s%s pair-cnt:%d, sut_pwr:%d, golden-rx:%d\n",
+			       "[HID]",
+			       hid.type & BTC_HID_218 ? "2/18," : "",
+			       hid.type & BTC_HID_418 ? "4/18," : "",
+			       hid.type & BTC_HID_BLE ? "BLE," : "",
+			       hid.type & BTC_HID_RCU ? "RCU," : "",
+			       hid.type & BTC_HID_RCU_VOICE ? "RCU-Voice," : "",
+			       hid.pair_cnt, bt_linfo->sut_pwr_level[1],
+			       bt_linfo->golden_rx_shift[1]);
 	}
 
 	if (a2dp.exist) {
-		seq_printf(m,
-			   " %-15s : type:%s, bit-pool:%d, flush-time:%d, ",
-			   "[A2DP]",
-			   a2dp.type == BTC_A2DP_LEGACY ? "Legacy" : "TWS",
-			    a2dp.bitpool, a2dp.flush_time);
+		p += scnprintf(p, end - p,
+			       " %-15s : type:%s, bit-pool:%d, flush-time:%d, ",
+			       "[A2DP]",
+			       a2dp.type == BTC_A2DP_LEGACY ? "Legacy" : "TWS",
+			       a2dp.bitpool, a2dp.flush_time);
 
-		seq_printf(m,
-			   "vid:0x%x, Dev-name:0x%x, sut_pwr:%d, golden-rx:%d\n",
-			   a2dp.vendor_id, a2dp.device_name,
-			   bt_linfo->sut_pwr_level[2],
-			   bt_linfo->golden_rx_shift[2]);
+		p += scnprintf(p, end - p,
+			       "vid:0x%x, Dev-name:0x%x, sut_pwr:%d, golden-rx:%d\n",
+			       a2dp.vendor_id, a2dp.device_name,
+			       bt_linfo->sut_pwr_level[2],
+			       bt_linfo->golden_rx_shift[2]);
 	}
 
 	if (pan.exist) {
-		seq_printf(m, " %-15s : sut_pwr:%d, golden-rx:%d\n",
-			   "[PAN]",
-			   bt_linfo->sut_pwr_level[3],
-			   bt_linfo->golden_rx_shift[3]);
+		p += scnprintf(p, end - p,
+			       " %-15s : sut_pwr:%d, golden-rx:%d\n",
+			       "[PAN]",
+			       bt_linfo->sut_pwr_level[3],
+			       bt_linfo->golden_rx_shift[3]);
 	}
+
+	return p - buf;
 }
 
-static void _show_bt_info(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_bt_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_bt_info *bt = &cx->bt;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
+	u32 ver_main = FIELD_GET(GENMASK(31, 24), wl->ver_info.fw_coex);
 	struct rtw89_btc_bt_link_info *bt_linfo = &bt->link_info;
 	union rtw89_btc_module_info *md = &btc->mdinfo;
+	s8 br_dbm = bt->link_info.bt_txpwr_desc.br_dbm;
+	s8 le_dbm = bt->link_info.bt_txpwr_desc.le_dbm;
+	char *p = buf, *end = buf + bufsz;
 	u8 *afh = bt_linfo->afh_map;
 	u8 *afh_le = bt_linfo->afh_map_le;
 	u8 bt_pos;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_BT))
-		return;
+		return 0;
 
 	if (ver->fcxinit == 7)
 		bt_pos = md->md_v7.bt_pos;
 	else
 		bt_pos = md->md.bt_pos;
 
-	seq_puts(m, "========== [BT Status] ==========\n");
-
-	seq_printf(m, " %-15s : enable:%s, btg:%s%s, connect:%s, ",
-		   "[status]", bt->enable.now ? "Y" : "N",
-		   bt->btg_type ? "Y" : "N",
-		   (bt->enable.now && (bt->btg_type != bt_pos) ?
-		   "(efuse-mismatch!!)" : ""),
-		   (bt_linfo->status.map.connect ? "Y" : "N"));
-
-	seq_printf(m, "igno_wl:%s, mailbox_avl:%s, rfk_state:0x%x\n",
-		   bt->igno_wl ? "Y" : "N",
-		   bt->mbx_avl ? "Y" : "N", bt->rfk_info.val);
-
-	seq_printf(m, " %-15s : profile:%s%s%s%s%s ",
-		   "[profile]",
-		   (bt_linfo->profile_cnt.now == 0) ? "None," : "",
-		   bt_linfo->hfp_desc.exist ? "HFP," : "",
-		   bt_linfo->hid_desc.exist ? "HID," : "",
-		   bt_linfo->a2dp_desc.exist ?
-		   (bt_linfo->a2dp_desc.sink ? "A2DP_sink," : "A2DP,") : "",
-		   bt_linfo->pan_desc.exist ? "PAN," : "");
-
-	seq_printf(m,
-		   "multi-link:%s, role:%s, ble-connect:%s, CQDDR:%s, A2DP_active:%s, PAN_active:%s\n",
-		   bt_linfo->multi_link.now ? "Y" : "N",
-		   bt_linfo->slave_role ? "Slave" : "Master",
-		   bt_linfo->status.map.ble_connect ? "Y" : "N",
-		   bt_linfo->cqddr ? "Y" : "N",
-		   bt_linfo->a2dp_desc.active ? "Y" : "N",
-		   bt_linfo->pan_desc.active ? "Y" : "N");
-
-	seq_printf(m,
-		   " %-15s : rssi:%ddBm(lvl:%d), tx_rate:%dM, %s%s%s",
-		   "[link]", bt_linfo->rssi - 100,
-		   bt->rssi_level,
-		   bt_linfo->tx_3m ? 3 : 2,
-		   bt_linfo->status.map.inq_pag ? " inq-page!!" : "",
-		   bt_linfo->status.map.acl_busy ? " acl_busy!!" : "",
-		   bt_linfo->status.map.mesh_busy ? " mesh_busy!!" : "");
-
-	seq_printf(m,
-		   "%s afh_map[%02x%02x_%02x%02x_%02x%02x_%02x%02x_%02x%02x], ",
-		   bt_linfo->relink.now ? " ReLink!!" : "",
-		   afh[0], afh[1], afh[2], afh[3], afh[4],
-		   afh[5], afh[6], afh[7], afh[8], afh[9]);
+	p += scnprintf(p, end - p, "========== [BT Status] ==========\n");
+
+	p += scnprintf(p, end - p,
+		       " %-15s : enable:%s, btg:%s%s, connect:%s, ",
+		       "[status]", bt->enable.now ? "Y" : "N",
+		       bt->btg_type ? "Y" : "N",
+		       (bt->enable.now && (bt->btg_type != bt_pos) ?
+			"(efuse-mismatch!!)" : ""),
+		       (bt_linfo->status.map.connect ? "Y" : "N"));
+
+	p += scnprintf(p, end - p,
+		       "igno_wl:%s, mailbox_avl:%s, rfk_state:0x%x\n",
+		       bt->igno_wl ? "Y" : "N",
+		       bt->mbx_avl ? "Y" : "N", bt->rfk_info.val);
+
+	p += scnprintf(p, end - p, " %-15s : profile:%s%s%s%s%s ",
+		       "[profile]",
+		       (bt_linfo->profile_cnt.now == 0) ? "None," : "",
+		       bt_linfo->hfp_desc.exist ? "HFP," : "",
+		       bt_linfo->hid_desc.exist ? "HID," : "",
+		       bt_linfo->a2dp_desc.exist ?
+		       (bt_linfo->a2dp_desc.sink ? "A2DP_sink," : "A2DP,") : "",
+		       bt_linfo->pan_desc.exist ? "PAN," : "");
+
+	p += scnprintf(p, end - p,
+		       "multi-link:%s, role:%s, ble-connect:%s, CQDDR:%s, A2DP_active:%s, PAN_active:%s\n",
+		       bt_linfo->multi_link.now ? "Y" : "N",
+		       bt_linfo->slave_role ? "Slave" : "Master",
+		       bt_linfo->status.map.ble_connect ? "Y" : "N",
+		       bt_linfo->cqddr ? "Y" : "N",
+		       bt_linfo->a2dp_desc.active ? "Y" : "N",
+		       bt_linfo->pan_desc.active ? "Y" : "N");
+
+	p += scnprintf(p, end - p,
+		       " %-15s : rssi:%ddBm(lvl:%d), tx_rate:%dM, %s%s%s",
+		       "[link]", bt_linfo->rssi - 100,
+		       bt->rssi_level,
+		       bt_linfo->tx_3m ? 3 : 2,
+		       bt_linfo->status.map.inq_pag ? " inq-page!!" : "",
+		       bt_linfo->status.map.acl_busy ? " acl_busy!!" : "",
+		       bt_linfo->status.map.mesh_busy ? " mesh_busy!!" : "");
+
+	p += scnprintf(p, end - p,
+		       "%s afh_map[%02x%02x_%02x%02x_%02x%02x_%02x%02x_%02x%02x], ",
+		       bt_linfo->relink.now ? " ReLink!!" : "",
+		       afh[0], afh[1], afh[2], afh[3], afh[4],
+		       afh[5], afh[6], afh[7], afh[8], afh[9]);
 
 	if (ver->fcxbtafh == 2 && bt_linfo->status.map.ble_connect)
-		seq_printf(m,
-			   "LE[%02x%02x_%02x_%02x%02x]",
-			   afh_le[0], afh_le[1], afh_le[2],
-			   afh_le[3], afh_le[4]);
-
-	seq_printf(m, "wl_ch_map[en:%d/ch:%d/bw:%d]\n",
-		   wl->afh_info.en, wl->afh_info.ch, wl->afh_info.bw);
-
-	seq_printf(m,
-		   " %-15s : retry:%d, relink:%d, rate_chg:%d, reinit:%d, reenable:%d, ",
-		   "[stat_cnt]", cx->cnt_bt[BTC_BCNT_RETRY],
-		   cx->cnt_bt[BTC_BCNT_RELINK], cx->cnt_bt[BTC_BCNT_RATECHG],
-		   cx->cnt_bt[BTC_BCNT_REINIT], cx->cnt_bt[BTC_BCNT_REENABLE]);
-
-	seq_printf(m,
-		   "role-switch:%d, afh:%d, inq_page:%d(inq:%d/page:%d), igno_wl:%d\n",
-		   cx->cnt_bt[BTC_BCNT_ROLESW], cx->cnt_bt[BTC_BCNT_AFH],
-		   cx->cnt_bt[BTC_BCNT_INQPAG], cx->cnt_bt[BTC_BCNT_INQ],
-		   cx->cnt_bt[BTC_BCNT_PAGE], cx->cnt_bt[BTC_BCNT_IGNOWL]);
-
-	_show_bt_profile_info(rtwdev, m);
-
-	seq_printf(m,
-		   " %-15s : raw_data[%02x %02x %02x %02x %02x %02x] (type:%s/cnt:%d/same:%d)\n",
-		   "[bt_info]", bt->raw_info[2], bt->raw_info[3],
-		   bt->raw_info[4], bt->raw_info[5], bt->raw_info[6],
-		   bt->raw_info[7],
-		   bt->raw_info[0] == BTC_BTINFO_AUTO ? "auto" : "reply",
-		   cx->cnt_bt[BTC_BCNT_INFOUPDATE],
-		   cx->cnt_bt[BTC_BCNT_INFOSAME]);
-
-	seq_printf(m,
-		   " %-15s : Hi-rx = %d, Hi-tx = %d, Lo-rx = %d, Lo-tx = %d (bt_polut_wl_tx = %d)",
-		   "[trx_req_cnt]", cx->cnt_bt[BTC_BCNT_HIPRI_RX],
-		   cx->cnt_bt[BTC_BCNT_HIPRI_TX], cx->cnt_bt[BTC_BCNT_LOPRI_RX],
-		   cx->cnt_bt[BTC_BCNT_LOPRI_TX], cx->cnt_bt[BTC_BCNT_POLUT]);
+		p += scnprintf(p, end - p,
+			       "LE[%02x%02x_%02x_%02x%02x]",
+			       afh_le[0], afh_le[1], afh_le[2],
+			       afh_le[3], afh_le[4]);
+
+	p += scnprintf(p, end - p, "wl_ch_map[en:%d/ch:%d/bw:%d]\n",
+		       wl->afh_info.en, wl->afh_info.ch, wl->afh_info.bw);
+
+	p += scnprintf(p, end - p,
+		       " %-15s : retry:%d, relink:%d, rate_chg:%d, reinit:%d, reenable:%d, ",
+		       "[stat_cnt]", cx->cnt_bt[BTC_BCNT_RETRY],
+		       cx->cnt_bt[BTC_BCNT_RELINK],
+		       cx->cnt_bt[BTC_BCNT_RATECHG],
+		       cx->cnt_bt[BTC_BCNT_REINIT],
+		       cx->cnt_bt[BTC_BCNT_REENABLE]);
+
+	p += scnprintf(p, end - p,
+		       "role-switch:%d, afh:%d, inq_page:%d(inq:%d/page:%d), igno_wl:%d\n",
+		       cx->cnt_bt[BTC_BCNT_ROLESW], cx->cnt_bt[BTC_BCNT_AFH],
+		       cx->cnt_bt[BTC_BCNT_INQPAG], cx->cnt_bt[BTC_BCNT_INQ],
+		       cx->cnt_bt[BTC_BCNT_PAGE], cx->cnt_bt[BTC_BCNT_IGNOWL]);
+
+	p += _show_bt_profile_info(rtwdev, p, end - p);
+
+	p += scnprintf(p, end - p,
+		       " %-15s : raw_data[%02x %02x %02x %02x %02x %02x] (type:%s/cnt:%d/same:%d)\n",
+		       "[bt_info]", bt->raw_info[2], bt->raw_info[3],
+		       bt->raw_info[4], bt->raw_info[5], bt->raw_info[6],
+		       bt->raw_info[7],
+		       bt->raw_info[0] == BTC_BTINFO_AUTO ? "auto" : "reply",
+		       cx->cnt_bt[BTC_BCNT_INFOUPDATE],
+		       cx->cnt_bt[BTC_BCNT_INFOSAME]);
+
+	p += scnprintf(p, end - p,
+		       " %-15s : Hi-rx = %d, Hi-tx = %d, Lo-rx = %d, Lo-tx = %d (bt_polut_wl_tx = %d)",
+		       "[trx_req_cnt]", cx->cnt_bt[BTC_BCNT_HIPRI_RX],
+		       cx->cnt_bt[BTC_BCNT_HIPRI_TX],
+		       cx->cnt_bt[BTC_BCNT_LOPRI_RX],
+		       cx->cnt_bt[BTC_BCNT_LOPRI_TX],
+		       cx->cnt_bt[BTC_BCNT_POLUT]);
 
 	if (!bt->scan_info_update) {
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_SCAN_INFO, true);
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
 	} else {
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_SCAN_INFO, false);
 		if (ver->fcxbtscan == 1) {
-			seq_printf(m,
-				   "(INQ:%d-%d/PAGE:%d-%d/LE:%d-%d/INIT:%d-%d)",
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INQ].win),
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INQ].intvl),
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_PAGE].win),
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_PAGE].intvl),
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_BLE].win),
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_BLE].intvl),
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INIT].win),
-				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INIT].intvl));
+			p += scnprintf(p, end - p,
+				       "(INQ:%d-%d/PAGE:%d-%d/LE:%d-%d/INIT:%d-%d)",
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INQ].win),
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INQ].intvl),
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_PAGE].win),
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_PAGE].intvl),
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_BLE].win),
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_BLE].intvl),
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INIT].win),
+				       le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INIT].intvl));
 		} else if (ver->fcxbtscan == 2) {
-			seq_printf(m,
-				   "(BG:%d-%d/INIT:%d-%d/LE:%d-%d)",
-				   le16_to_cpu(bt->scan_info_v2[CXSCAN_BG].win),
-				   le16_to_cpu(bt->scan_info_v2[CXSCAN_BG].intvl),
-				   le16_to_cpu(bt->scan_info_v2[CXSCAN_INIT].win),
-				   le16_to_cpu(bt->scan_info_v2[CXSCAN_INIT].intvl),
-				   le16_to_cpu(bt->scan_info_v2[CXSCAN_LE].win),
-				   le16_to_cpu(bt->scan_info_v2[CXSCAN_LE].intvl));
+			p += scnprintf(p, end - p,
+				       "(BG:%d-%d/INIT:%d-%d/LE:%d-%d)",
+				       le16_to_cpu(bt->scan_info_v2[CXSCAN_BG].win),
+				       le16_to_cpu(bt->scan_info_v2[CXSCAN_BG].intvl),
+				       le16_to_cpu(bt->scan_info_v2[CXSCAN_INIT].win),
+				       le16_to_cpu(bt->scan_info_v2[CXSCAN_INIT].intvl),
+				       le16_to_cpu(bt->scan_info_v2[CXSCAN_LE].win),
+				       le16_to_cpu(bt->scan_info_v2[CXSCAN_LE].intvl));
 		}
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
+	}
+
+	if (ver_main >= 9 && bt_linfo->profile_cnt.now)
+		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_TX_PWR_LVL, true);
+	else
+		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_TX_PWR_LVL, false);
+
+	if (cx->cnt_bt[BTC_BCNT_BTTXPWR_UPDATE]) {
+		p += scnprintf(p, end - p,
+			       " %-15s : br_index:0x%x, le_index:0x%x",
+			       "[bt_txpwr_lvl]",
+			       bt->link_info.bt_txpwr_desc.br_gain_index,
+			       bt->link_info.bt_txpwr_desc.le_gain_index);
+		p += scnprintf(p, end - p, ", br_dbm:%d dBm", br_dbm);
+		p += scnprintf(p, end - p, ", le_dbm:%d dBm", le_dbm);
+	} else {
+		p += scnprintf(p, end - p,
+			       " %-15s : br_index:NA, le_index:NA, br_dbm:%d dBm[def], le_dbm:%d dBm[def]",
+			       "[bt_txpwr_lvl]",
+			       bt->link_info.bt_txpwr_desc.br_dbm,
+			       bt->link_info.bt_txpwr_desc.le_dbm);
 	}
+	p += scnprintf(p, end - p, "\n");
 
 	if (bt_linfo->profile_cnt.now || bt_linfo->status.map.ble_connect)
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_AFH_MAP, true);
@@ -8560,6 +9240,8 @@ static void _show_bt_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_DEVICE_INFO, true);
 	else
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_DEVICE_INFO, false);
+
+	return p - buf;
 }
 
 #define CASE_BTC_RSN_STR(e) case BTC_RSN_ ## e: return #e
@@ -8853,114 +9535,132 @@ static const char *id_to_ant(u32 id)
 }
 
 static
-void seq_print_segment(struct seq_file *m, const char *prefix, u16 *data,
-		       u8 len, u8 seg_len, u8 start_idx, u8 ring_len)
+int scnprintf_segment(char *buf, size_t bufsz, const char *prefix, const u16 *data,
+		      u8 len, u8 seg_len, u8 start_idx, u8 ring_len)
 {
-	u8 i;
+	char *p = buf, *end = buf + bufsz;
 	u8 cur_index;
+	u8 i;
 
 	for (i = 0; i < len ; i++) {
 		if ((i % seg_len) == 0)
-			seq_printf(m, " %-15s : ", prefix);
+			p += scnprintf(p, end - p, " %-15s : ", prefix);
 		cur_index = (start_idx + i) % ring_len;
 		if (i % 3 == 0)
-			seq_printf(m, "-> %-20s",
-				   steps_to_str(*(data + cur_index)));
+			p += scnprintf(p, end - p, "-> %-20s",
+				       steps_to_str(*(data + cur_index)));
 		else if (i % 3 == 1)
-			seq_printf(m, "-> %-15s",
-				   steps_to_str(*(data + cur_index)));
+			p += scnprintf(p, end - p, "-> %-15s",
+				       steps_to_str(*(data + cur_index)));
 		else
-			seq_printf(m, "-> %-13s",
-				   steps_to_str(*(data + cur_index)));
+			p += scnprintf(p, end - p, "-> %-13s",
+				       steps_to_str(*(data + cur_index)));
 		if (i == (len - 1) || (i % seg_len) == (seg_len - 1))
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
+
+	return p - buf;
 }
 
-static void _show_dm_step(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_dm_step(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &btc->dm;
+	char *p = buf, *end = buf + bufsz;
 	u8 start_idx;
 	u8 len;
 
 	len = dm->dm_step.step_ov ? RTW89_BTC_DM_MAXSTEP : dm->dm_step.step_pos;
 	start_idx = dm->dm_step.step_ov ? dm->dm_step.step_pos : 0;
 
-	seq_print_segment(m, "[dm_steps]", dm->dm_step.step, len, 6, start_idx,
-			  ARRAY_SIZE(dm->dm_step.step));
+	p += scnprintf_segment(p, end - p, "[dm_steps]", dm->dm_step.step, len,
+			       6, start_idx, ARRAY_SIZE(dm->dm_step.step));
+
+	return p - buf;
 }
 
-static void _show_dm_info(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_dm_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_dm *dm = &btc->dm;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+	char *p = buf, *end = buf + bufsz;
 	u8 igno_bt;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_DM))
-		return;
+		return 0;
 
-	seq_printf(m, "========== [Mechanism Status %s] ==========\n",
-		   (btc->manual_ctrl ? "(Manual)" : "(Auto)"));
+	p += scnprintf(p, end - p,
+		       "========== [Mechanism Status %s] ==========\n",
+		       (btc->manual_ctrl ? "(Manual)" : "(Auto)"));
 
-	seq_printf(m,
-		   " %-15s : type:%s, reason:%s(), action:%s(), ant_path:%s, init_mode:%s, run_cnt:%d\n",
-		   "[status]",
-		   btc->ant_type == BTC_ANT_SHARED ? "shared" : "dedicated",
-		   steps_to_str(dm->run_reason),
-		   steps_to_str(dm->run_action | BTC_ACT_EXT_BIT),
-		   id_to_ant(FIELD_GET(GENMASK(7, 0), dm->set_ant_path)),
-		   id_to_mode(wl->coex_mode),
-		   dm->cnt_dm[BTC_DCNT_RUN]);
+	p += scnprintf(p, end - p,
+		       " %-15s : type:%s, reason:%s(), action:%s(), ant_path:%s, init_mode:%s, run_cnt:%d\n",
+		       "[status]",
+		       btc->ant_type == BTC_ANT_SHARED ? "shared" : "dedicated",
+		       steps_to_str(dm->run_reason),
+		       steps_to_str(dm->run_action | BTC_ACT_EXT_BIT),
+		       id_to_ant(FIELD_GET(GENMASK(7, 0), dm->set_ant_path)),
+		       id_to_mode(wl->coex_mode),
+		       dm->cnt_dm[BTC_DCNT_RUN]);
 
-	_show_dm_step(rtwdev, m);
+	p += _show_dm_step(rtwdev, p, end - p);
 
 	if (ver->fcxctrl == 7)
 		igno_bt = btc->ctrl.ctrl_v7.igno_bt;
 	else
 		igno_bt = btc->ctrl.ctrl.igno_bt;
 
-	seq_printf(m, " %-15s : wl_only:%d, bt_only:%d, igno_bt:%d, free_run:%d, wl_ps_ctrl:%d, wl_mimo_ps:%d, ",
-		   "[dm_flag]", dm->wl_only, dm->bt_only, igno_bt,
-		   dm->freerun, btc->lps, dm->wl_mimo_ps);
+	p += scnprintf(p, end - p,
+		       " %-15s : wl_only:%d, bt_only:%d, igno_bt:%d, free_run:%d, wl_ps_ctrl:%d, wl_mimo_ps:%d, ",
+		       "[dm_flag]", dm->wl_only, dm->bt_only, igno_bt,
+		       dm->freerun, btc->lps, dm->wl_mimo_ps);
 
-	seq_printf(m, "leak_ap:%d, fw_offload:%s%s\n", dm->leak_ap,
-		   (BTC_CX_FW_OFFLOAD ? "Y" : "N"),
-		   (dm->wl_fw_cx_offload == BTC_CX_FW_OFFLOAD ?
-		    "" : "(Mismatch!!)"));
+	p += scnprintf(p, end - p, "leak_ap:%d, fw_offload:%s%s\n",
+		       dm->leak_ap,
+		       (BTC_CX_FW_OFFLOAD ? "Y" : "N"),
+		       (dm->wl_fw_cx_offload == BTC_CX_FW_OFFLOAD ?
+			"" : "(Mismatch!!)"));
 
 	if (dm->rf_trx_para.wl_tx_power == 0xff)
-		seq_printf(m,
-			   " %-15s : wl_rssi_lvl:%d, para_lvl:%d, wl_tx_pwr:orig, ",
-			   "[trx_ctrl]", wl->rssi_level, dm->trx_para_level);
+		p += scnprintf(p, end - p,
+			       " %-15s : wl_rssi_lvl:%d, para_lvl:%d, wl_tx_pwr:orig, ",
+			       "[trx_ctrl]", wl->rssi_level,
+			       dm->trx_para_level);
 
 	else
-		seq_printf(m,
-			   " %-15s : wl_rssi_lvl:%d, para_lvl:%d, wl_tx_pwr:%d, ",
-			   "[trx_ctrl]", wl->rssi_level, dm->trx_para_level,
-			   dm->rf_trx_para.wl_tx_power);
+		p += scnprintf(p, end - p,
+			       " %-15s : wl_rssi_lvl:%d, para_lvl:%d, wl_tx_pwr:%d, ",
+			       "[trx_ctrl]", wl->rssi_level,
+			       dm->trx_para_level,
+			       dm->rf_trx_para.wl_tx_power);
+
+	p += scnprintf(p, end - p,
+		       "wl_rx_lvl:%d, bt_tx_pwr_dec:%d, bt_rx_lna:%d(%s-tbl), wl_btg_rx:%d\n",
+		       dm->rf_trx_para.wl_rx_gain,
+		       dm->rf_trx_para.bt_tx_power,
+		       dm->rf_trx_para.bt_rx_gain,
+		       (bt->hi_lna_rx ? "Hi" : "Ori"), dm->wl_btg_rx);
 
-	seq_printf(m,
-		   "wl_rx_lvl:%d, bt_tx_pwr_dec:%d, bt_rx_lna:%d(%s-tbl), wl_btg_rx:%d\n",
-		   dm->rf_trx_para.wl_rx_gain, dm->rf_trx_para.bt_tx_power,
-		   dm->rf_trx_para.bt_rx_gain,
-		   (bt->hi_lna_rx ? "Hi" : "Ori"), dm->wl_btg_rx);
+	p += scnprintf(p, end - p,
+		       " %-15s : wl_tx_limit[en:%d/max_t:%dus/max_retry:%d], bt_slot_reg:%d-TU, bt_scan_rx_low_pri:%d\n",
+		       "[dm_ctrl]", dm->wl_tx_limit.enable,
+		       dm->wl_tx_limit.tx_time,
+		       dm->wl_tx_limit.tx_retry, btc->bt_req_len,
+		       bt->scan_rx_low_pri);
 
-	seq_printf(m,
-		   " %-15s : wl_tx_limit[en:%d/max_t:%dus/max_retry:%d], bt_slot_reg:%d-TU, bt_scan_rx_low_pri:%d\n",
-		   "[dm_ctrl]", dm->wl_tx_limit.enable, dm->wl_tx_limit.tx_time,
-		   dm->wl_tx_limit.tx_retry, btc->bt_req_len, bt->scan_rx_low_pri);
+	return p - buf;
 }
 
-static void _show_error(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_error(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
 	union rtw89_btc_fbtc_cysta_info *pcysta;
+	char *p = buf, *end = buf + bufsz;
 	u32 except_cnt, exception_map;
 
 	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo;
@@ -8985,81 +9685,87 @@ static void _show_error(struct rtw89_dev *rtwdev, struct seq_file *m)
 		except_cnt = pcysta->v7.except_cnt;
 		exception_map = le32_to_cpu(pcysta->v7.except_map);
 	} else {
-		return;
+		return 0;
 	}
 
 	if (pfwinfo->event[BTF_EVNT_BUF_OVERFLOW] == 0 && except_cnt == 0 &&
 	    !pfwinfo->len_mismch && !pfwinfo->fver_mismch)
-		return;
+		return 0;
 
-	seq_printf(m, " %-15s : ", "[error]");
+	p += scnprintf(p, end - p, " %-15s : ", "[error]");
 
 	if (pfwinfo->event[BTF_EVNT_BUF_OVERFLOW]) {
-		seq_printf(m,
-			   "overflow-cnt: %d, ",
-			   pfwinfo->event[BTF_EVNT_BUF_OVERFLOW]);
+		p += scnprintf(p, end - p,
+			       "overflow-cnt: %d, ",
+			       pfwinfo->event[BTF_EVNT_BUF_OVERFLOW]);
 	}
 
 	if (pfwinfo->len_mismch) {
-		seq_printf(m,
-			   "len-mismatch: 0x%x, ",
-			   pfwinfo->len_mismch);
+		p += scnprintf(p, end - p,
+			       "len-mismatch: 0x%x, ",
+			       pfwinfo->len_mismch);
 	}
 
 	if (pfwinfo->fver_mismch) {
-		seq_printf(m,
-			   "fver-mismatch: 0x%x, ",
-			   pfwinfo->fver_mismch);
+		p += scnprintf(p, end - p,
+			       "fver-mismatch: 0x%x, ",
+			       pfwinfo->fver_mismch);
 	}
 
 	/* cycle statistics exceptions */
 	if (exception_map || except_cnt) {
-		seq_printf(m,
-			   "exception-type: 0x%x, exception-cnt = %d",
-			   exception_map, except_cnt);
+		p += scnprintf(p, end - p,
+			       "exception-type: 0x%x, exception-cnt = %d",
+			       exception_map, except_cnt);
 	}
-	seq_puts(m, "\n");
+	p += scnprintf(p, end - p, "\n");
+
+	return p - buf;
 }
 
-static void _show_fbtc_tdma(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_tdma(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
 	struct rtw89_btc_fbtc_tdma *t = NULL;
+	char *p = buf, *end = buf + bufsz;
 
 	pcinfo = &pfwinfo->rpt_fbtc_tdma.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	if (ver->fcxtdma == 1)
 		t = &pfwinfo->rpt_fbtc_tdma.finfo.v1;
 	else
 		t = &pfwinfo->rpt_fbtc_tdma.finfo.v3.tdma;
 
-	seq_printf(m,
-		   " %-15s : ", "[tdma_policy]");
-	seq_printf(m,
-		   "type:%d, rx_flow_ctrl:%d, tx_pause:%d, ",
-		   (u32)t->type,
-		   t->rxflctrl, t->txpause);
+	p += scnprintf(p, end - p,
+		       " %-15s : ", "[tdma_policy]");
+	p += scnprintf(p, end - p,
+		       "type:%d, rx_flow_ctrl:%d, tx_pause:%d, ",
+		       (u32)t->type,
+		       t->rxflctrl, t->txpause);
 
-	seq_printf(m,
-		   "wl_toggle_n:%d, leak_n:%d, ext_ctrl:%d, ",
-		   t->wtgle_n, t->leak_n, t->ext_ctrl);
+	p += scnprintf(p, end - p,
+		       "wl_toggle_n:%d, leak_n:%d, ext_ctrl:%d, ",
+		       t->wtgle_n, t->leak_n, t->ext_ctrl);
 
-	seq_printf(m,
-		   "policy_type:%d",
-		   (u32)btc->policy_type);
+	p += scnprintf(p, end - p,
+		       "policy_type:%d",
+		       (u32)btc->policy_type);
 
-	seq_puts(m, "\n");
+	p += scnprintf(p, end - p, "\n");
+
+	return p - buf;
 }
 
-static void _show_fbtc_slots(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_slots(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &btc->dm;
+	char *p = buf, *end = buf + bufsz;
 	u16 dur, cxtype;
 	u32 tbl;
 	u8 i = 0;
@@ -9074,28 +9780,29 @@ static void _show_fbtc_slots(struct rtw89_dev *rtwdev, struct seq_file *m)
 			tbl = le32_to_cpu(dm->slot_now.v7[i].cxtbl);
 			cxtype = le16_to_cpu(dm->slot_now.v7[i].cxtype);
 		} else {
-			return;
+			return 0;
 		}
 
 		if (i % 5 == 0)
-			seq_printf(m,
-				   " %-15s : %5s[%03d/0x%x/%d]",
-				   "[slot_list]",
-				   id_to_slot((u32)i),
-				   dur, tbl, cxtype);
+			p += scnprintf(p, end - p,
+				       " %-15s : %5s[%03d/0x%x/%d]",
+				       "[slot_list]",
+				       id_to_slot((u32)i),
+				       dur, tbl, cxtype);
 		else
-			seq_printf(m,
-				   ", %5s[%03d/0x%x/%d]",
-				   id_to_slot((u32)i),
-				   dur, tbl, cxtype);
+			p += scnprintf(p, end - p,
+				       ", %5s[%03d/0x%x/%d]",
+				       id_to_slot((u32)i),
+				       dur, tbl, cxtype);
 
 		if (i % 5 == 4)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
-	seq_puts(m, "\n");
+
+	return p - buf;
 }
 
-static void _show_fbtc_cysta_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_cysta_v2(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
@@ -9104,63 +9811,64 @@ static void _show_fbtc_cysta_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
 	struct rtw89_btc_fbtc_cysta_v2 *pcysta_le32 = NULL;
 	union rtw89_btc_fbtc_rxflct r;
-	u8 i, cnt = 0, slot_pair;
 	u16 cycle, c_begin, c_end, store_index;
+	char *p = buf, *end = buf + bufsz;
+	u8 i, cnt = 0, slot_pair;
 
 	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	pcysta_le32 = &pfwinfo->rpt_fbtc_cysta.finfo.v2;
-	seq_printf(m,
-		   " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
-		   "[cycle_cnt]",
-		   le16_to_cpu(pcysta_le32->cycles),
-		   le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_ALL]),
-		   le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_ALL_OK]),
-		   le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_BT_SLOT]),
-		   le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_BT_OK]));
+	p += scnprintf(p, end - p,
+		       " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
+		       "[cycle_cnt]",
+		       le16_to_cpu(pcysta_le32->cycles),
+		       le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_ALL]),
+		       le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_ALL_OK]),
+		       le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_BT_SLOT]),
+		       le32_to_cpu(pcysta_le32->bcn_cnt[CXBCN_BT_OK]));
 
 	for (i = 0; i < CXST_MAX; i++) {
 		if (!le32_to_cpu(pcysta_le32->slot_cnt[i]))
 			continue;
-		seq_printf(m, ", %s:%d", id_to_slot((u32)i),
-			   le32_to_cpu(pcysta_le32->slot_cnt[i]));
+		p += scnprintf(p, end - p, ", %s:%d", id_to_slot((u32)i),
+			       le32_to_cpu(pcysta_le32->slot_cnt[i]));
 	}
 
 	if (dm->tdma_now.rxflctrl) {
-		seq_printf(m, ", leak_rx:%d",
-			   le32_to_cpu(pcysta_le32->leakrx_cnt));
+		p += scnprintf(p, end - p, ", leak_rx:%d",
+			       le32_to_cpu(pcysta_le32->leakrx_cnt));
 	}
 
 	if (le32_to_cpu(pcysta_le32->collision_cnt)) {
-		seq_printf(m, ", collision:%d",
-			   le32_to_cpu(pcysta_le32->collision_cnt));
+		p += scnprintf(p, end - p, ", collision:%d",
+			       le32_to_cpu(pcysta_le32->collision_cnt));
 	}
 
 	if (le32_to_cpu(pcysta_le32->skip_cnt)) {
-		seq_printf(m, ", skip:%d",
-			   le32_to_cpu(pcysta_le32->skip_cnt));
-	}
-	seq_puts(m, "\n");
-
-	seq_printf(m, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   "[cycle_time]",
-		   le16_to_cpu(pcysta_le32->tavg_cycle[CXT_WL]),
-		   le16_to_cpu(pcysta_le32->tavg_cycle[CXT_BT]),
-		   le16_to_cpu(pcysta_le32->tavg_lk) / 1000,
-		   le16_to_cpu(pcysta_le32->tavg_lk) % 1000);
-	seq_printf(m, ", max_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   le16_to_cpu(pcysta_le32->tmax_cycle[CXT_WL]),
-		   le16_to_cpu(pcysta_le32->tmax_cycle[CXT_BT]),
-		   le16_to_cpu(pcysta_le32->tmax_lk) / 1000,
-		   le16_to_cpu(pcysta_le32->tmax_lk) % 1000);
-	seq_printf(m, ", maxdiff_t[wl:%d/bt:%d]\n",
-		   le16_to_cpu(pcysta_le32->tmaxdiff_cycle[CXT_WL]),
-		   le16_to_cpu(pcysta_le32->tmaxdiff_cycle[CXT_BT]));
+		p += scnprintf(p, end - p, ", skip:%d",
+			       le32_to_cpu(pcysta_le32->skip_cnt));
+	}
+	p += scnprintf(p, end - p, "\n");
+
+	p += scnprintf(p, end - p, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       "[cycle_time]",
+		       le16_to_cpu(pcysta_le32->tavg_cycle[CXT_WL]),
+		       le16_to_cpu(pcysta_le32->tavg_cycle[CXT_BT]),
+		       le16_to_cpu(pcysta_le32->tavg_lk) / 1000,
+		       le16_to_cpu(pcysta_le32->tavg_lk) % 1000);
+	p += scnprintf(p, end - p, ", max_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       le16_to_cpu(pcysta_le32->tmax_cycle[CXT_WL]),
+		       le16_to_cpu(pcysta_le32->tmax_cycle[CXT_BT]),
+		       le16_to_cpu(pcysta_le32->tmax_lk) / 1000,
+		       le16_to_cpu(pcysta_le32->tmax_lk) % 1000);
+	p += scnprintf(p, end - p, ", maxdiff_t[wl:%d/bt:%d]\n",
+		       le16_to_cpu(pcysta_le32->tmaxdiff_cycle[CXT_WL]),
+		       le16_to_cpu(pcysta_le32->tmaxdiff_cycle[CXT_BT]));
 
 	if (le16_to_cpu(pcysta_le32->cycles) <= 1)
-		return;
+		goto out;
 
 	/* 1 cycle record 1 wl-slot and 1 bt-slot */
 	slot_pair = BTC_CYCLE_SLOT_MAX / 2;
@@ -9177,53 +9885,57 @@ static void _show_fbtc_cysta_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 		store_index = ((cycle - 1) % slot_pair) * 2;
 
 		if (cnt % (BTC_CYCLE_SLOT_MAX / 4) == 1)
-			seq_printf(m,
-				   " %-15s : ->b%02d->w%02d", "[cycle_step]",
-				   le16_to_cpu(pcysta_le32->tslot_cycle[store_index]),
-				   le16_to_cpu(pcysta_le32->tslot_cycle[store_index + 1]));
+			p += scnprintf(p, end - p,
+				       " %-15s : ->b%02d->w%02d",
+				       "[cycle_step]",
+				       le16_to_cpu(pcysta_le32->tslot_cycle[store_index]),
+				       le16_to_cpu(pcysta_le32->tslot_cycle[store_index + 1]));
 		else
-			seq_printf(m,
-				   "->b%02d->w%02d",
-				   le16_to_cpu(pcysta_le32->tslot_cycle[store_index]),
-				   le16_to_cpu(pcysta_le32->tslot_cycle[store_index + 1]));
+			p += scnprintf(p, end - p,
+				       "->b%02d->w%02d",
+				       le16_to_cpu(pcysta_le32->tslot_cycle[store_index]),
+				       le16_to_cpu(pcysta_le32->tslot_cycle[store_index + 1]));
 		if (cnt % (BTC_CYCLE_SLOT_MAX / 4) == 0 || cnt == c_end)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
 
 	if (a2dp->exist) {
-		seq_printf(m,
-			   " %-15s : a2dp_ept:%d, a2dp_late:%d",
-			   "[a2dp_t_sta]",
-			   le16_to_cpu(pcysta_le32->a2dpept),
-			   le16_to_cpu(pcysta_le32->a2dpeptto));
-
-		seq_printf(m,
-			   ", avg_t:%d, max_t:%d",
-			   le16_to_cpu(pcysta_le32->tavg_a2dpept),
-			   le16_to_cpu(pcysta_le32->tmax_a2dpept));
+		p += scnprintf(p, end - p,
+			       " %-15s : a2dp_ept:%d, a2dp_late:%d",
+			       "[a2dp_t_sta]",
+			       le16_to_cpu(pcysta_le32->a2dpept),
+			       le16_to_cpu(pcysta_le32->a2dpeptto));
+
+		p += scnprintf(p, end - p,
+			       ", avg_t:%d, max_t:%d",
+			       le16_to_cpu(pcysta_le32->tavg_a2dpept),
+			       le16_to_cpu(pcysta_le32->tmax_a2dpept));
 		r.val = dm->tdma_now.rxflctrl;
 
 		if (r.type && r.tgln_n) {
-			seq_printf(m,
-				   ", cycle[PSTDMA:%d/TDMA:%d], ",
-				   le16_to_cpu(pcysta_le32->cycles_a2dp[CXT_FLCTRL_ON]),
-				   le16_to_cpu(pcysta_le32->cycles_a2dp[CXT_FLCTRL_OFF]));
-
-			seq_printf(m,
-				   "avg_t[PSTDMA:%d/TDMA:%d], ",
-				   le16_to_cpu(pcysta_le32->tavg_a2dp[CXT_FLCTRL_ON]),
-				   le16_to_cpu(pcysta_le32->tavg_a2dp[CXT_FLCTRL_OFF]));
-
-			seq_printf(m,
-				   "max_t[PSTDMA:%d/TDMA:%d]",
-				   le16_to_cpu(pcysta_le32->tmax_a2dp[CXT_FLCTRL_ON]),
-				   le16_to_cpu(pcysta_le32->tmax_a2dp[CXT_FLCTRL_OFF]));
+			p += scnprintf(p, end - p,
+				       ", cycle[PSTDMA:%d/TDMA:%d], ",
+				       le16_to_cpu(pcysta_le32->cycles_a2dp[CXT_FLCTRL_ON]),
+				       le16_to_cpu(pcysta_le32->cycles_a2dp[CXT_FLCTRL_OFF]));
+
+			p += scnprintf(p, end - p,
+				       "avg_t[PSTDMA:%d/TDMA:%d], ",
+				       le16_to_cpu(pcysta_le32->tavg_a2dp[CXT_FLCTRL_ON]),
+				       le16_to_cpu(pcysta_le32->tavg_a2dp[CXT_FLCTRL_OFF]));
+
+			p += scnprintf(p, end - p,
+				       "max_t[PSTDMA:%d/TDMA:%d]",
+				       le16_to_cpu(pcysta_le32->tmax_a2dp[CXT_FLCTRL_ON]),
+				       le16_to_cpu(pcysta_le32->tmax_a2dp[CXT_FLCTRL_OFF]));
 		}
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
 	}
+
+out:
+	return p - buf;
 }
 
-static void _show_fbtc_cysta_v3(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_cysta_v3(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
@@ -9234,60 +9946,64 @@ static void _show_fbtc_cysta_v3(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_rpt_cmn_info *pcinfo;
 	u8 i, cnt = 0, slot_pair, divide_cnt;
 	u16 cycle, c_begin, c_end, store_index;
+	char *p = buf, *end = buf + bufsz;
 
 	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo.v3;
-	seq_printf(m,
-		   " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
-		   "[cycle_cnt]",
-		   le16_to_cpu(pcysta->cycles),
-		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
-		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
-		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
-		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
+	p += scnprintf(p, end - p,
+		       " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
+		       "[cycle_cnt]",
+		       le16_to_cpu(pcysta->cycles),
+		       le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
+		       le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
+		       le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
+		       le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
 
 	for (i = 0; i < CXST_MAX; i++) {
 		if (!le32_to_cpu(pcysta->slot_cnt[i]))
 			continue;
 
-		seq_printf(m, ", %s:%d", id_to_slot(i),
-			   le32_to_cpu(pcysta->slot_cnt[i]));
+		p += scnprintf(p, end - p, ", %s:%d", id_to_slot(i),
+			       le32_to_cpu(pcysta->slot_cnt[i]));
 	}
 
 	if (dm->tdma_now.rxflctrl)
-		seq_printf(m, ", leak_rx:%d", le32_to_cpu(pcysta->leak_slot.cnt_rximr));
+		p += scnprintf(p, end - p, ", leak_rx:%d",
+			       le32_to_cpu(pcysta->leak_slot.cnt_rximr));
 
 	if (le32_to_cpu(pcysta->collision_cnt))
-		seq_printf(m, ", collision:%d", le32_to_cpu(pcysta->collision_cnt));
+		p += scnprintf(p, end - p, ", collision:%d",
+			       le32_to_cpu(pcysta->collision_cnt));
 
 	if (le32_to_cpu(pcysta->skip_cnt))
-		seq_printf(m, ", skip:%d", le32_to_cpu(pcysta->skip_cnt));
-
-	seq_puts(m, "\n");
-
-	seq_printf(m, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   "[cycle_time]",
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
-		   le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
-	seq_printf(m,
-		   ", max_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
-		   le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
-	seq_printf(m,
-		   ", maxdiff_t[wl:%d/bt:%d]\n",
-		   le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_BT]));
+		p += scnprintf(p, end - p, ", skip:%d",
+			       le32_to_cpu(pcysta->skip_cnt));
+
+	p += scnprintf(p, end - p, "\n");
+
+	p += scnprintf(p, end - p, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       "[cycle_time]",
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
+		       le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
+	p += scnprintf(p, end - p,
+		       ", max_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
+		       le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
+	p += scnprintf(p, end - p,
+		       ", maxdiff_t[wl:%d/bt:%d]\n",
+		       le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_BT]));
 
 	cycle = le16_to_cpu(pcysta->cycles);
 	if (cycle <= 1)
-		return;
+		goto out;
 
 	/* 1 cycle record 1 wl-slot and 1 bt-slot */
 	slot_pair = BTC_CYCLE_SLOT_MAX / 2;
@@ -9309,51 +10025,56 @@ static void _show_fbtc_cysta_v3(struct rtw89_dev *rtwdev, struct seq_file *m)
 		store_index = ((cycle - 1) % slot_pair) * 2;
 
 		if (cnt % divide_cnt == 1)
-			seq_printf(m, " %-15s : ", "[cycle_step]");
+			p += scnprintf(p, end - p, " %-15s : ",
+				       "[cycle_step]");
 
-		seq_printf(m, "->b%02d",
-			   le16_to_cpu(pcysta->slot_step_time[store_index]));
+		p += scnprintf(p, end - p, "->b%02d",
+			       le16_to_cpu(pcysta->slot_step_time[store_index]));
 		if (a2dp->exist) {
 			a2dp_trx = &pcysta->a2dp_trx[store_index];
-			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
-				   a2dp_trx->empty_cnt,
-				   a2dp_trx->retry_cnt,
-				   a2dp_trx->tx_rate ? 3 : 2,
-				   a2dp_trx->tx_cnt,
-				   a2dp_trx->ack_cnt,
-				   a2dp_trx->nack_cnt);
-		}
-		seq_printf(m, "->w%02d",
-			   le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
+			p += scnprintf(p, end - p, "(%d/%d/%dM/%d/%d/%d)",
+				       a2dp_trx->empty_cnt,
+				       a2dp_trx->retry_cnt,
+				       a2dp_trx->tx_rate ? 3 : 2,
+				       a2dp_trx->tx_cnt,
+				       a2dp_trx->ack_cnt,
+				       a2dp_trx->nack_cnt);
+		}
+		p += scnprintf(p, end - p, "->w%02d",
+			       le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
 		if (a2dp->exist) {
 			a2dp_trx = &pcysta->a2dp_trx[store_index + 1];
-			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
-				   a2dp_trx->empty_cnt,
-				   a2dp_trx->retry_cnt,
-				   a2dp_trx->tx_rate ? 3 : 2,
-				   a2dp_trx->tx_cnt,
-				   a2dp_trx->ack_cnt,
-				   a2dp_trx->nack_cnt);
+			p += scnprintf(p, end - p, "(%d/%d/%dM/%d/%d/%d)",
+				       a2dp_trx->empty_cnt,
+				       a2dp_trx->retry_cnt,
+				       a2dp_trx->tx_rate ? 3 : 2,
+				       a2dp_trx->tx_cnt,
+				       a2dp_trx->ack_cnt,
+				       a2dp_trx->nack_cnt);
 		}
 		if (cnt % divide_cnt == 0 || cnt == c_end)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
 
 	if (a2dp->exist) {
-		seq_printf(m, " %-15s : a2dp_ept:%d, a2dp_late:%d",
-			   "[a2dp_t_sta]",
-			   le16_to_cpu(pcysta->a2dp_ept.cnt),
-			   le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
+		p += scnprintf(p, end - p,
+			       " %-15s : a2dp_ept:%d, a2dp_late:%d",
+			       "[a2dp_t_sta]",
+			       le16_to_cpu(pcysta->a2dp_ept.cnt),
+			       le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
 
-		seq_printf(m, ", avg_t:%d, max_t:%d",
-			   le16_to_cpu(pcysta->a2dp_ept.tavg),
-			   le16_to_cpu(pcysta->a2dp_ept.tmax));
+		p += scnprintf(p, end - p, ", avg_t:%d, max_t:%d",
+			       le16_to_cpu(pcysta->a2dp_ept.tavg),
+			       le16_to_cpu(pcysta->a2dp_ept.tmax));
 
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
 	}
+
+out:
+	return p - buf;
 }
 
-static void _show_fbtc_cysta_v4(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_cysta_v4(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
@@ -9364,62 +10085,64 @@ static void _show_fbtc_cysta_v4(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_rpt_cmn_info *pcinfo;
 	u8 i, cnt = 0, slot_pair, divide_cnt;
 	u16 cycle, c_begin, c_end, store_index;
+	char *p = buf, *end = buf + bufsz;
 
 	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo.v4;
-	seq_printf(m,
-		   " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
-		   "[cycle_cnt]",
-		   le16_to_cpu(pcysta->cycles),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
+	p += scnprintf(p, end - p,
+		       " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
+		       "[cycle_cnt]",
+		       le16_to_cpu(pcysta->cycles),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
 
 	for (i = 0; i < CXST_MAX; i++) {
 		if (!le16_to_cpu(pcysta->slot_cnt[i]))
 			continue;
 
-		seq_printf(m, ", %s:%d", id_to_slot(i),
-			   le16_to_cpu(pcysta->slot_cnt[i]));
+		p += scnprintf(p, end - p, ", %s:%d", id_to_slot(i),
+			       le16_to_cpu(pcysta->slot_cnt[i]));
 	}
 
 	if (dm->tdma_now.rxflctrl)
-		seq_printf(m, ", leak_rx:%d",
-			   le32_to_cpu(pcysta->leak_slot.cnt_rximr));
+		p += scnprintf(p, end - p, ", leak_rx:%d",
+			       le32_to_cpu(pcysta->leak_slot.cnt_rximr));
 
 	if (pcysta->collision_cnt)
-		seq_printf(m, ", collision:%d", pcysta->collision_cnt);
+		p += scnprintf(p, end - p, ", collision:%d",
+			       pcysta->collision_cnt);
 
 	if (le16_to_cpu(pcysta->skip_cnt))
-		seq_printf(m, ", skip:%d",
-			   le16_to_cpu(pcysta->skip_cnt));
-
-	seq_puts(m, "\n");
-
-	seq_printf(m, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   "[cycle_time]",
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
-		   le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
-	seq_printf(m,
-		   ", max_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
-		   le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
-	seq_printf(m,
-		   ", maxdiff_t[wl:%d/bt:%d]\n",
-		   le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_BT]));
+		p += scnprintf(p, end - p, ", skip:%d",
+			       le16_to_cpu(pcysta->skip_cnt));
+
+	p += scnprintf(p, end - p, "\n");
+
+	p += scnprintf(p, end - p, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       "[cycle_time]",
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
+		       le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
+	p += scnprintf(p, end - p,
+		       ", max_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
+		       le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
+	p += scnprintf(p, end - p,
+		       ", maxdiff_t[wl:%d/bt:%d]\n",
+		       le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_BT]));
 
 	cycle = le16_to_cpu(pcysta->cycles);
 	if (cycle <= 1)
-		return;
+		goto out;
 
 	/* 1 cycle record 1 wl-slot and 1 bt-slot */
 	slot_pair = BTC_CYCLE_SLOT_MAX / 2;
@@ -9441,51 +10164,56 @@ static void _show_fbtc_cysta_v4(struct rtw89_dev *rtwdev, struct seq_file *m)
 		store_index = ((cycle - 1) % slot_pair) * 2;
 
 		if (cnt % divide_cnt == 1)
-			seq_printf(m, " %-15s : ", "[cycle_step]");
+			p += scnprintf(p, end - p, " %-15s : ",
+				       "[cycle_step]");
 
-		seq_printf(m, "->b%02d",
-			   le16_to_cpu(pcysta->slot_step_time[store_index]));
+		p += scnprintf(p, end - p, "->b%02d",
+			       le16_to_cpu(pcysta->slot_step_time[store_index]));
 		if (a2dp->exist) {
 			a2dp_trx = &pcysta->a2dp_trx[store_index];
-			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
-				   a2dp_trx->empty_cnt,
-				   a2dp_trx->retry_cnt,
-				   a2dp_trx->tx_rate ? 3 : 2,
-				   a2dp_trx->tx_cnt,
-				   a2dp_trx->ack_cnt,
-				   a2dp_trx->nack_cnt);
-		}
-		seq_printf(m, "->w%02d",
-			   le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
+			p += scnprintf(p, end - p, "(%d/%d/%dM/%d/%d/%d)",
+				       a2dp_trx->empty_cnt,
+				       a2dp_trx->retry_cnt,
+				       a2dp_trx->tx_rate ? 3 : 2,
+				       a2dp_trx->tx_cnt,
+				       a2dp_trx->ack_cnt,
+				       a2dp_trx->nack_cnt);
+		}
+		p += scnprintf(p, end - p, "->w%02d",
+			       le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
 		if (a2dp->exist) {
 			a2dp_trx = &pcysta->a2dp_trx[store_index + 1];
-			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
-				   a2dp_trx->empty_cnt,
-				   a2dp_trx->retry_cnt,
-				   a2dp_trx->tx_rate ? 3 : 2,
-				   a2dp_trx->tx_cnt,
-				   a2dp_trx->ack_cnt,
-				   a2dp_trx->nack_cnt);
+			p += scnprintf(p, end - p, "(%d/%d/%dM/%d/%d/%d)",
+				       a2dp_trx->empty_cnt,
+				       a2dp_trx->retry_cnt,
+				       a2dp_trx->tx_rate ? 3 : 2,
+				       a2dp_trx->tx_cnt,
+				       a2dp_trx->ack_cnt,
+				       a2dp_trx->nack_cnt);
 		}
 		if (cnt % divide_cnt == 0 || cnt == c_end)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
 
 	if (a2dp->exist) {
-		seq_printf(m, " %-15s : a2dp_ept:%d, a2dp_late:%d",
-			   "[a2dp_t_sta]",
-			   le16_to_cpu(pcysta->a2dp_ept.cnt),
-			   le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
+		p += scnprintf(p, end - p,
+			       " %-15s : a2dp_ept:%d, a2dp_late:%d",
+			       "[a2dp_t_sta]",
+			       le16_to_cpu(pcysta->a2dp_ept.cnt),
+			       le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
 
-		seq_printf(m, ", avg_t:%d, max_t:%d",
-			   le16_to_cpu(pcysta->a2dp_ept.tavg),
-			   le16_to_cpu(pcysta->a2dp_ept.tmax));
+		p += scnprintf(p, end - p, ", avg_t:%d, max_t:%d",
+			       le16_to_cpu(pcysta->a2dp_ept.tavg),
+			       le16_to_cpu(pcysta->a2dp_ept.tmax));
 
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
 	}
+
+out:
+	return p - buf;
 }
 
-static void _show_fbtc_cysta_v5(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_cysta_v5(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
@@ -9496,58 +10224,60 @@ static void _show_fbtc_cysta_v5(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_rpt_cmn_info *pcinfo;
 	u8 i, cnt = 0, slot_pair, divide_cnt;
 	u16 cycle, c_begin, c_end, store_index;
+	char *p = buf, *end = buf + bufsz;
 
 	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo.v5;
-	seq_printf(m,
-		   " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
-		   "[cycle_cnt]",
-		   le16_to_cpu(pcysta->cycles),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
+	p += scnprintf(p, end - p,
+		       " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
+		       "[cycle_cnt]",
+		       le16_to_cpu(pcysta->cycles),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
 
 	for (i = 0; i < CXST_MAX; i++) {
 		if (!le16_to_cpu(pcysta->slot_cnt[i]))
 			continue;
 
-		seq_printf(m, ", %s:%d", id_to_slot(i),
-			   le16_to_cpu(pcysta->slot_cnt[i]));
+		p += scnprintf(p, end - p, ", %s:%d", id_to_slot(i),
+			       le16_to_cpu(pcysta->slot_cnt[i]));
 	}
 
 	if (dm->tdma_now.rxflctrl)
-		seq_printf(m, ", leak_rx:%d",
-			   le32_to_cpu(pcysta->leak_slot.cnt_rximr));
+		p += scnprintf(p, end - p, ", leak_rx:%d",
+			       le32_to_cpu(pcysta->leak_slot.cnt_rximr));
 
 	if (pcysta->collision_cnt)
-		seq_printf(m, ", collision:%d", pcysta->collision_cnt);
+		p += scnprintf(p, end - p, ", collision:%d",
+			       pcysta->collision_cnt);
 
 	if (le16_to_cpu(pcysta->skip_cnt))
-		seq_printf(m, ", skip:%d",
-			   le16_to_cpu(pcysta->skip_cnt));
-
-	seq_puts(m, "\n");
-
-	seq_printf(m, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   "[cycle_time]",
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
-		   le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
-	seq_printf(m,
-		   ", max_t[wl:%d/bt:%d/lk:%d.%03d]\n",
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
-		   le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
+		p += scnprintf(p, end - p, ", skip:%d",
+			       le16_to_cpu(pcysta->skip_cnt));
+
+	p += scnprintf(p, end - p, "\n");
+
+	p += scnprintf(p, end - p, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       "[cycle_time]",
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
+		       le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
+	p += scnprintf(p, end - p,
+		       ", max_t[wl:%d/bt:%d/lk:%d.%03d]\n",
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
+		       le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
 
 	cycle = le16_to_cpu(pcysta->cycles);
 	if (cycle <= 1)
-		return;
+		goto out;
 
 	/* 1 cycle record 1 wl-slot and 1 bt-slot */
 	slot_pair = BTC_CYCLE_SLOT_MAX / 2;
@@ -9565,58 +10295,63 @@ static void _show_fbtc_cysta_v5(struct rtw89_dev *rtwdev, struct seq_file *m)
 		divide_cnt = BTC_CYCLE_SLOT_MAX / 4;
 
 	if (c_begin > c_end)
-		return;
+		goto out;
 
 	for (cycle = c_begin; cycle <= c_end; cycle++) {
 		cnt++;
 		store_index = ((cycle - 1) % slot_pair) * 2;
 
 		if (cnt % divide_cnt == 1)
-			seq_printf(m, " %-15s : ", "[cycle_step]");
+			p += scnprintf(p, end - p, " %-15s : ",
+				       "[cycle_step]");
 
-		seq_printf(m, "->b%02d",
-			   le16_to_cpu(pcysta->slot_step_time[store_index]));
+		p += scnprintf(p, end - p, "->b%02d",
+			       le16_to_cpu(pcysta->slot_step_time[store_index]));
 		if (a2dp->exist) {
 			a2dp_trx = &pcysta->a2dp_trx[store_index];
-			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
-				   a2dp_trx->empty_cnt,
-				   a2dp_trx->retry_cnt,
-				   a2dp_trx->tx_rate ? 3 : 2,
-				   a2dp_trx->tx_cnt,
-				   a2dp_trx->ack_cnt,
-				   a2dp_trx->nack_cnt);
-		}
-		seq_printf(m, "->w%02d",
-			   le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
+			p += scnprintf(p, end - p, "(%d/%d/%dM/%d/%d/%d)",
+				       a2dp_trx->empty_cnt,
+				       a2dp_trx->retry_cnt,
+				       a2dp_trx->tx_rate ? 3 : 2,
+				       a2dp_trx->tx_cnt,
+				       a2dp_trx->ack_cnt,
+				       a2dp_trx->nack_cnt);
+		}
+		p += scnprintf(p, end - p, "->w%02d",
+			       le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
 		if (a2dp->exist) {
 			a2dp_trx = &pcysta->a2dp_trx[store_index + 1];
-			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
-				   a2dp_trx->empty_cnt,
-				   a2dp_trx->retry_cnt,
-				   a2dp_trx->tx_rate ? 3 : 2,
-				   a2dp_trx->tx_cnt,
-				   a2dp_trx->ack_cnt,
-				   a2dp_trx->nack_cnt);
+			p += scnprintf(p, end - p, "(%d/%d/%dM/%d/%d/%d)",
+				       a2dp_trx->empty_cnt,
+				       a2dp_trx->retry_cnt,
+				       a2dp_trx->tx_rate ? 3 : 2,
+				       a2dp_trx->tx_cnt,
+				       a2dp_trx->ack_cnt,
+				       a2dp_trx->nack_cnt);
 		}
 		if (cnt % divide_cnt == 0 || cnt == c_end)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
 
 	if (a2dp->exist) {
-		seq_printf(m, " %-15s : a2dp_ept:%d, a2dp_late:%d",
-			   "[a2dp_t_sta]",
-			   le16_to_cpu(pcysta->a2dp_ept.cnt),
-			   le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
+		p += scnprintf(p, end - p,
+			       " %-15s : a2dp_ept:%d, a2dp_late:%d",
+			       "[a2dp_t_sta]",
+			       le16_to_cpu(pcysta->a2dp_ept.cnt),
+			       le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
 
-		seq_printf(m, ", avg_t:%d, max_t:%d",
-			   le16_to_cpu(pcysta->a2dp_ept.tavg),
-			   le16_to_cpu(pcysta->a2dp_ept.tmax));
+		p += scnprintf(p, end - p, ", avg_t:%d, max_t:%d",
+			       le16_to_cpu(pcysta->a2dp_ept.tavg),
+			       le16_to_cpu(pcysta->a2dp_ept.tmax));
 
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
 	}
+
+out:
+	return p - buf;
 }
 
-static void _show_fbtc_cysta_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_cysta_v7(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc_bt_info *bt = &rtwdev->btc.cx.bt;
 	struct rtw89_btc_bt_a2dp_desc *a2dp = &bt->link_info.a2dp_desc;
@@ -9624,68 +10359,75 @@ static void _show_fbtc_cysta_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_fbtc_cysta_v7 *pcysta = NULL;
 	struct rtw89_btc_dm *dm = &rtwdev->btc.dm;
 	struct rtw89_btc_rpt_cmn_info *pcinfo;
+	char *p = buf, *end = buf + bufsz;
 	u16 cycle, c_begin, c_end, s_id;
 	u8 i, cnt = 0, divide_cnt;
 	u8 slot_pair;
 
 	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo.v7;
-	seq_printf(m, "\n\r %-15s : cycle:%d", "[slot_stat]",
-		   le16_to_cpu(pcysta->cycles));
+	p += scnprintf(p, end - p, "\n %-15s : cycle:%d", "[slot_stat]",
+		       le16_to_cpu(pcysta->cycles));
 
 	for (i = 0; i < CXST_MAX; i++) {
 		if (!le16_to_cpu(pcysta->slot_cnt[i]))
 			continue;
-		seq_printf(m, ", %s:%d",
-			   id_to_slot(i), le16_to_cpu(pcysta->slot_cnt[i]));
+		p += scnprintf(p, end - p, ", %s:%d",
+			       id_to_slot(i),
+			       le16_to_cpu(pcysta->slot_cnt[i]));
 	}
 
 	if (dm->tdma_now.rxflctrl)
-		seq_printf(m, ", leak_rx:%d",
-			   le32_to_cpu(pcysta->leak_slot.cnt_rximr));
+		p += scnprintf(p, end - p, ", leak_rx:%d",
+			       le32_to_cpu(pcysta->leak_slot.cnt_rximr));
 
 	if (pcysta->collision_cnt)
-		seq_printf(m, ", collision:%d", pcysta->collision_cnt);
+		p += scnprintf(p, end - p, ", collision:%d",
+			       pcysta->collision_cnt);
 
 	if (pcysta->skip_cnt)
-		seq_printf(m, ", skip:%d", le16_to_cpu(pcysta->skip_cnt));
-
-	seq_printf(m, "\n\r %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
-		   "[cycle_stat]",
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
-		   le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
-	seq_printf(m, ", max_t[wl:%d/bt:%d(>%dms:%d)/lk:%d.%03d]",
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
-		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
-		   dm->bt_slot_flood, dm->cnt_dm[BTC_DCNT_BT_SLOT_FLOOD],
-		   le16_to_cpu(pcysta->leak_slot.tamx) / 1000,
-		   le16_to_cpu(pcysta->leak_slot.tamx) % 1000);
-	seq_printf(m, ", bcn[all:%d/ok:%d/in_bt:%d/in_bt_ok:%d]",
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
-		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
+		p += scnprintf(p, end - p, ", skip:%d",
+			       le16_to_cpu(pcysta->skip_cnt));
+
+	p += scnprintf(p, end - p,
+		       "\n\r %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
+		       "[cycle_stat]",
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
+		       le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
+	p += scnprintf(p, end - p,
+		       ", max_t[wl:%d/bt:%d(>%dms:%d)/lk:%d.%03d]",
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
+		       le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
+		       dm->bt_slot_flood, dm->cnt_dm[BTC_DCNT_BT_SLOT_FLOOD],
+		       le16_to_cpu(pcysta->leak_slot.tamx) / 1000,
+		       le16_to_cpu(pcysta->leak_slot.tamx) % 1000);
+	p += scnprintf(p, end - p, ", bcn[all:%d/ok:%d/in_bt:%d/in_bt_ok:%d]",
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
+		       le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
 
 	if (a2dp->exist) {
-		seq_printf(m,
-			   "\n\r %-15s : a2dp_ept:%d, a2dp_late:%d(streak 2S:%d/max:%d)",
-			   "[a2dp_stat]",
-			   le16_to_cpu(pcysta->a2dp_ept.cnt),
-			   le16_to_cpu(pcysta->a2dp_ept.cnt_timeout),
-			   a2dp->no_empty_streak_2s, a2dp->no_empty_streak_max);
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : a2dp_ept:%d, a2dp_late:%d(streak 2S:%d/max:%d)",
+			       "[a2dp_stat]",
+			       le16_to_cpu(pcysta->a2dp_ept.cnt),
+			       le16_to_cpu(pcysta->a2dp_ept.cnt_timeout),
+			       a2dp->no_empty_streak_2s,
+			       a2dp->no_empty_streak_max);
 
-		seq_printf(m, ", avg_t:%d, max_t:%d",
-			   le16_to_cpu(pcysta->a2dp_ept.tavg),
-			   le16_to_cpu(pcysta->a2dp_ept.tmax));
+		p += scnprintf(p, end - p, ", avg_t:%d, max_t:%d",
+			       le16_to_cpu(pcysta->a2dp_ept.tavg),
+			       le16_to_cpu(pcysta->a2dp_ept.tmax));
 	}
 
 	if (le16_to_cpu(pcysta->cycles) <= 1)
-		return;
+		goto out;
 
 	/* 1 cycle = 1 wl-slot + 1 bt-slot */
 	slot_pair = BTC_CYCLE_SLOT_MAX / 2;
@@ -9703,7 +10445,7 @@ static void _show_fbtc_cysta_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
 		divide_cnt = 6;
 
 	if (c_begin > c_end)
-		return;
+		goto out;
 
 	for (cycle = c_begin; cycle <= c_end; cycle++) {
 		cnt++;
@@ -9711,129 +10453,142 @@ static void _show_fbtc_cysta_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 		if (cnt % divide_cnt == 1) {
 			if (a2dp->exist)
-				seq_printf(m, "\n\r %-15s : ", "[slotT_wermtan]");
+				p += scnprintf(p, end - p, "\n\r %-15s : ",
+					       "[slotT_wermtan]");
 			else
-				seq_printf(m, "\n\r %-15s : ", "[slotT_rxerr]");
+				p += scnprintf(p, end - p, "\n\r %-15s : ",
+					       "[slotT_rxerr]");
 		}
 
-		seq_printf(m, "->b%d", le16_to_cpu(pcysta->slot_step_time[s_id]));
+		p += scnprintf(p, end - p, "->b%d",
+			       le16_to_cpu(pcysta->slot_step_time[s_id]));
 
 		if (a2dp->exist)
-			seq_printf(m, "(%d/%d/%d/%dM/%d/%d/%d)",
-				   pcysta->wl_rx_err_ratio[s_id],
-				   pcysta->a2dp_trx[s_id].empty_cnt,
-				   pcysta->a2dp_trx[s_id].retry_cnt,
-				   (pcysta->a2dp_trx[s_id].tx_rate ? 3 : 2),
-				   pcysta->a2dp_trx[s_id].tx_cnt,
-				   pcysta->a2dp_trx[s_id].ack_cnt,
-				   pcysta->a2dp_trx[s_id].nack_cnt);
+			p += scnprintf(p, end - p, "(%d/%d/%d/%dM/%d/%d/%d)",
+				       pcysta->wl_rx_err_ratio[s_id],
+				       pcysta->a2dp_trx[s_id].empty_cnt,
+				       pcysta->a2dp_trx[s_id].retry_cnt,
+				       (pcysta->a2dp_trx[s_id].tx_rate ? 3 : 2),
+				       pcysta->a2dp_trx[s_id].tx_cnt,
+				       pcysta->a2dp_trx[s_id].ack_cnt,
+				       pcysta->a2dp_trx[s_id].nack_cnt);
 		else
-			seq_printf(m, "(%d)", pcysta->wl_rx_err_ratio[s_id]);
+			p += scnprintf(p, end - p, "(%d)",
+				       pcysta->wl_rx_err_ratio[s_id]);
 
-		seq_printf(m, "->w%d", le16_to_cpu(pcysta->slot_step_time[s_id + 1]));
+		p += scnprintf(p, end - p, "->w%d",
+			       le16_to_cpu(pcysta->slot_step_time[s_id + 1]));
 
 		if (a2dp->exist)
-			seq_printf(m, "(%d/%d/%d/%dM/%d/%d/%d)",
-				   pcysta->wl_rx_err_ratio[s_id + 1],
-				   pcysta->a2dp_trx[s_id + 1].empty_cnt,
-				   pcysta->a2dp_trx[s_id + 1].retry_cnt,
-				   (pcysta->a2dp_trx[s_id + 1].tx_rate ? 3 : 2),
-				   pcysta->a2dp_trx[s_id + 1].tx_cnt,
-				   pcysta->a2dp_trx[s_id + 1].ack_cnt,
-				   pcysta->a2dp_trx[s_id + 1].nack_cnt);
+			p += scnprintf(p, end - p, "(%d/%d/%d/%dM/%d/%d/%d)",
+				       pcysta->wl_rx_err_ratio[s_id + 1],
+				       pcysta->a2dp_trx[s_id + 1].empty_cnt,
+				       pcysta->a2dp_trx[s_id + 1].retry_cnt,
+				       (pcysta->a2dp_trx[s_id + 1].tx_rate ? 3 : 2),
+				       pcysta->a2dp_trx[s_id + 1].tx_cnt,
+				       pcysta->a2dp_trx[s_id + 1].ack_cnt,
+				       pcysta->a2dp_trx[s_id + 1].nack_cnt);
 		else
-			seq_printf(m, "(%d)", pcysta->wl_rx_err_ratio[s_id + 1]);
+			p += scnprintf(p, end - p, "(%d)",
+				       pcysta->wl_rx_err_ratio[s_id + 1]);
 	}
+
+out:
+	return p - buf;
 }
 
-static void _show_fbtc_nullsta(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_nullsta(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
 	struct rtw89_btc_rpt_cmn_info *pcinfo;
 	union rtw89_btc_fbtc_cynullsta_info *ns;
+	char *p = buf, *end = buf + bufsz;
 	u8 i = 0;
 
 	if (!btc->dm.tdma_now.rxflctrl)
-		return;
+		return 0;
 
 	pcinfo = &pfwinfo->rpt_fbtc_nullsta.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	ns = &pfwinfo->rpt_fbtc_nullsta.finfo;
 	if (ver->fcxnullsta == 1) {
 		for (i = 0; i < 2; i++) {
-			seq_printf(m, " %-15s : ", "[NULL-STA]");
-			seq_printf(m, "null-%d", i);
-			seq_printf(m, "[ok:%d/",
-				   le32_to_cpu(ns->v1.result[i][1]));
-			seq_printf(m, "fail:%d/",
-				   le32_to_cpu(ns->v1.result[i][0]));
-			seq_printf(m, "on_time:%d/",
-				   le32_to_cpu(ns->v1.result[i][2]));
-			seq_printf(m, "retry:%d/",
-				   le32_to_cpu(ns->v1.result[i][3]));
-			seq_printf(m, "avg_t:%d.%03d/",
-				   le32_to_cpu(ns->v1.avg_t[i]) / 1000,
-				   le32_to_cpu(ns->v1.avg_t[i]) % 1000);
-			seq_printf(m, "max_t:%d.%03d]\n",
-				   le32_to_cpu(ns->v1.max_t[i]) / 1000,
-				   le32_to_cpu(ns->v1.max_t[i]) % 1000);
+			p += scnprintf(p, end - p, " %-15s : ", "\n[NULL-STA]");
+			p += scnprintf(p, end - p, "null-%d", i);
+			p += scnprintf(p, end - p, "[ok:%d/",
+				       le32_to_cpu(ns->v1.result[i][1]));
+			p += scnprintf(p, end - p, "fail:%d/",
+				       le32_to_cpu(ns->v1.result[i][0]));
+			p += scnprintf(p, end - p, "on_time:%d/",
+				       le32_to_cpu(ns->v1.result[i][2]));
+			p += scnprintf(p, end - p, "retry:%d/",
+				       le32_to_cpu(ns->v1.result[i][3]));
+			p += scnprintf(p, end - p, "avg_t:%d.%03d/",
+				       le32_to_cpu(ns->v1.avg_t[i]) / 1000,
+				       le32_to_cpu(ns->v1.avg_t[i]) % 1000);
+			p += scnprintf(p, end - p, "max_t:%d.%03d]",
+				       le32_to_cpu(ns->v1.max_t[i]) / 1000,
+				       le32_to_cpu(ns->v1.max_t[i]) % 1000);
 		}
 	} else if (ver->fcxnullsta == 7) {
 		for (i = 0; i < 2; i++) {
-			seq_printf(m, " %-15s : ", "[NULL-STA]");
-			seq_printf(m, "null-%d", i);
-			seq_printf(m, "[Tx:%d/",
-				   le32_to_cpu(ns->v7.result[i][4]));
-			seq_printf(m, "[ok:%d/",
-				   le32_to_cpu(ns->v7.result[i][1]));
-			seq_printf(m, "fail:%d/",
-				   le32_to_cpu(ns->v7.result[i][0]));
-			seq_printf(m, "on_time:%d/",
-				   le32_to_cpu(ns->v7.result[i][2]));
-			seq_printf(m, "retry:%d/",
-				   le32_to_cpu(ns->v7.result[i][3]));
-			seq_printf(m, "avg_t:%d.%03d/",
-				   le32_to_cpu(ns->v7.tavg[i]) / 1000,
-				   le32_to_cpu(ns->v7.tavg[i]) % 1000);
-			seq_printf(m, "max_t:%d.%03d]\n",
-				   le32_to_cpu(ns->v7.tmax[i]) / 1000,
-				   le32_to_cpu(ns->v7.tmax[i]) % 1000);
+			p += scnprintf(p, end - p, " %-15s : ", "\n[NULL-STA]");
+			p += scnprintf(p, end - p, "null-%d", i);
+			p += scnprintf(p, end - p, "[Tx:%d/",
+				       le32_to_cpu(ns->v7.result[i][4]));
+			p += scnprintf(p, end - p, "[ok:%d/",
+				       le32_to_cpu(ns->v7.result[i][1]));
+			p += scnprintf(p, end - p, "fail:%d/",
+				       le32_to_cpu(ns->v7.result[i][0]));
+			p += scnprintf(p, end - p, "on_time:%d/",
+				       le32_to_cpu(ns->v7.result[i][2]));
+			p += scnprintf(p, end - p, "retry:%d/",
+				       le32_to_cpu(ns->v7.result[i][3]));
+			p += scnprintf(p, end - p, "avg_t:%d.%03d/",
+				       le32_to_cpu(ns->v7.tavg[i]) / 1000,
+				       le32_to_cpu(ns->v7.tavg[i]) % 1000);
+			p += scnprintf(p, end - p, "max_t:%d.%03d]",
+				       le32_to_cpu(ns->v7.tmax[i]) / 1000,
+				       le32_to_cpu(ns->v7.tmax[i]) % 1000);
 		}
 	} else {
 		for (i = 0; i < 2; i++) {
-			seq_printf(m, " %-15s : ", "[NULL-STA]");
-			seq_printf(m, "null-%d", i);
-			seq_printf(m, "[Tx:%d/",
-				   le32_to_cpu(ns->v2.result[i][4]));
-			seq_printf(m, "[ok:%d/",
-				   le32_to_cpu(ns->v2.result[i][1]));
-			seq_printf(m, "fail:%d/",
-				   le32_to_cpu(ns->v2.result[i][0]));
-			seq_printf(m, "on_time:%d/",
-				   le32_to_cpu(ns->v2.result[i][2]));
-			seq_printf(m, "retry:%d/",
-				   le32_to_cpu(ns->v2.result[i][3]));
-			seq_printf(m, "avg_t:%d.%03d/",
-				   le32_to_cpu(ns->v2.avg_t[i]) / 1000,
-				   le32_to_cpu(ns->v2.avg_t[i]) % 1000);
-			seq_printf(m, "max_t:%d.%03d]\n",
-				   le32_to_cpu(ns->v2.max_t[i]) / 1000,
-				   le32_to_cpu(ns->v2.max_t[i]) % 1000);
-		}
-	}
-}
-
-static void _show_fbtc_step_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
+			p += scnprintf(p, end - p, " %-15s : ", "\n[NULL-STA]");
+			p += scnprintf(p, end - p, "null-%d", i);
+			p += scnprintf(p, end - p, "[Tx:%d/",
+				       le32_to_cpu(ns->v2.result[i][4]));
+			p += scnprintf(p, end - p, "[ok:%d/",
+				       le32_to_cpu(ns->v2.result[i][1]));
+			p += scnprintf(p, end - p, "fail:%d/",
+				       le32_to_cpu(ns->v2.result[i][0]));
+			p += scnprintf(p, end - p, "on_time:%d/",
+				       le32_to_cpu(ns->v2.result[i][2]));
+			p += scnprintf(p, end - p, "retry:%d/",
+				       le32_to_cpu(ns->v2.result[i][3]));
+			p += scnprintf(p, end - p, "avg_t:%d.%03d/",
+				       le32_to_cpu(ns->v2.avg_t[i]) / 1000,
+				       le32_to_cpu(ns->v2.avg_t[i]) % 1000);
+			p += scnprintf(p, end - p, "max_t:%d.%03d]",
+				       le32_to_cpu(ns->v2.max_t[i]) / 1000,
+				       le32_to_cpu(ns->v2.max_t[i]) % 1000);
+		}
+	}
+
+	return p - buf;
+}
+
+static int _show_fbtc_step_v2(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
 	struct rtw89_btc_fbtc_steps_v2 *pstep = NULL;
 	const struct rtw89_btc_ver *ver = btc->ver;
+	char *p = buf, *end = buf + bufsz;
 	u8 type, val, cnt = 0, state = 0;
 	bool outloop = false;
 	u16 i, diff_t, n_start = 0, n_stop = 0;
@@ -9841,14 +10596,14 @@ static void _show_fbtc_step_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 	pcinfo = &pfwinfo->rpt_fbtc_step.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	pstep = &pfwinfo->rpt_fbtc_step.finfo.v2;
 	pos_old = le16_to_cpu(pstep->pos_old);
 	pos_new = le16_to_cpu(pstep->pos_new);
 
 	if (pcinfo->req_fver != pstep->fver)
-		return;
+		return 0;
 
 	/* store step info by using ring instead of FIFO*/
 	do {
@@ -9877,13 +10632,15 @@ static void _show_fbtc_step_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 					continue;
 
 				if (cnt % 10 == 0)
-					seq_printf(m, " %-15s : ", "[steps]");
+					p += scnprintf(p, end - p,
+						       " %-15s : ", "[steps]");
 
-				seq_printf(m, "-> %s(%02d)(%02d)",
-					   (type == CXSTEP_SLOT ? "SLT" :
-					    "EVT"), (u32)val, diff_t);
+				p += scnprintf(p, end - p,
+					       "-> %s(%02d)(%02d)",
+					       (type == CXSTEP_SLOT ? "SLT" :
+						"EVT"), (u32)val, diff_t);
 				if (cnt % 10 == 9)
-					seq_puts(m, "\n");
+					p += scnprintf(p, end - p, "\n");
 				cnt++;
 			}
 
@@ -9900,29 +10657,32 @@ static void _show_fbtc_step_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 			break;
 		}
 	} while (!outloop);
+
+	return p - buf;
 }
 
-static void _show_fbtc_step_v3(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fbtc_step_v3(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
 	struct rtw89_btc_rpt_cmn_info *pcinfo;
 	struct rtw89_btc_fbtc_steps_v3 *pstep;
 	u32 i, n_begin, n_end, array_idx, cnt = 0;
+	char *p = buf, *end = buf + bufsz;
 	u8 type, val;
 	u16 diff_t;
 
 	if ((pfwinfo->rpt_en_map &
 	     rtw89_btc_fw_rpt_ver(rtwdev, RPT_EN_FW_STEP_INFO)) == 0)
-		return;
+		return 0;
 
 	pcinfo = &pfwinfo->rpt_fbtc_step.cinfo;
 	if (!pcinfo->valid)
-		return;
+		return 0;
 
 	pstep = &pfwinfo->rpt_fbtc_step.finfo.v3;
 	if (pcinfo->req_fver != pstep->fver)
-		return;
+		return 0;
 
 	if (le32_to_cpu(pstep->cnt) <= FCXDEF_STEP)
 		n_begin = 1;
@@ -9932,7 +10692,7 @@ static void _show_fbtc_step_v3(struct rtw89_dev *rtwdev, struct seq_file *m)
 	n_end = le32_to_cpu(pstep->cnt);
 
 	if (n_begin > n_end)
-		return;
+		return 0;
 
 	/* restore step info by using ring instead of FIFO */
 	for (i = n_begin; i <= n_end; i++) {
@@ -9945,50 +10705,55 @@ static void _show_fbtc_step_v3(struct rtw89_dev *rtwdev, struct seq_file *m)
 			continue;
 
 		if (cnt % 10 == 0)
-			seq_printf(m, " %-15s : ", "[steps]");
+			p += scnprintf(p, end - p, " %-15s : ", "[steps]");
 
-		seq_printf(m, "-> %s(%02d)",
-			   (type == CXSTEP_SLOT ?
-			    id_to_slot((u32)val) :
-			    id_to_evt((u32)val)), diff_t);
+		p += scnprintf(p, end - p, "-> %s(%02d)",
+			       (type == CXSTEP_SLOT ?
+				id_to_slot((u32)val) :
+				id_to_evt((u32)val)), diff_t);
 
 		if (cnt % 10 == 9)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 
 		cnt++;
 	}
+
+	return p - buf;
 }
 
-static void _show_fw_dm_msg(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_fw_dm_msg(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
+	char *p = buf, *end = buf + bufsz;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_DM))
-		return;
+		goto out;
 
-	_show_error(rtwdev, m);
-	_show_fbtc_tdma(rtwdev, m);
-	_show_fbtc_slots(rtwdev, m);
+	p += _show_error(rtwdev, p, end - p);
+	p += _show_fbtc_tdma(rtwdev, p, end - p);
+	p += _show_fbtc_slots(rtwdev, p, end - p);
 
 	if (ver->fcxcysta == 2)
-		_show_fbtc_cysta_v2(rtwdev, m);
+		p += _show_fbtc_cysta_v2(rtwdev, p, end - p);
 	else if (ver->fcxcysta == 3)
-		_show_fbtc_cysta_v3(rtwdev, m);
+		p += _show_fbtc_cysta_v3(rtwdev, p, end - p);
 	else if (ver->fcxcysta == 4)
-		_show_fbtc_cysta_v4(rtwdev, m);
+		p += _show_fbtc_cysta_v4(rtwdev, p, end - p);
 	else if (ver->fcxcysta == 5)
-		_show_fbtc_cysta_v5(rtwdev, m);
+		p += _show_fbtc_cysta_v5(rtwdev, p, end - p);
 	else if (ver->fcxcysta == 7)
-		_show_fbtc_cysta_v7(rtwdev, m);
+		p += _show_fbtc_cysta_v7(rtwdev, p, end - p);
 
-	_show_fbtc_nullsta(rtwdev, m);
+	p += _show_fbtc_nullsta(rtwdev, p, end - p);
 
 	if (ver->fcxstep == 2)
-		_show_fbtc_step_v2(rtwdev, m);
+		p += _show_fbtc_step_v2(rtwdev, p, end - p);
 	else if (ver->fcxstep == 3)
-		_show_fbtc_step_v3(rtwdev, m);
+		p += _show_fbtc_step_v3(rtwdev, p, end - p);
 
+out:
+	return p - buf;
 }
 
 static void _get_gnt(struct rtw89_dev *rtwdev, struct rtw89_mac_ax_coex_gnt *gnt_cfg)
@@ -10033,12 +10798,13 @@ static void _get_gnt(struct rtw89_dev *rtwdev, struct rtw89_mac_ax_coex_gnt *gnt
 	}
 }
 
-static void _show_gpio_dbg(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_gpio_dbg(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &rtwdev->btc.fwinfo;
 	const struct rtw89_btc_ver *ver = rtwdev->btc.ver;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
 	union rtw89_btc_fbtc_gpio_dbg *gdbg = NULL;
+	char *p = buf, *end = buf + bufsz;
 	u8 *gpio_map, i;
 	u32 en_map;
 
@@ -10048,8 +10814,7 @@ static void _show_gpio_dbg(struct rtw89_dev *rtwdev, struct seq_file *m)
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): stop due rpt_fbtc_gpio_dbg.cinfo\n",
 			    __func__);
-		seq_puts(m, "\n");
-		return;
+		goto out;
 	}
 
 	if (ver->fcxgpiodbg == 7) {
@@ -10061,20 +10826,24 @@ static void _show_gpio_dbg(struct rtw89_dev *rtwdev, struct seq_file *m)
 	}
 
 	if (!en_map)
-		return;
+		goto out;
 
-	seq_printf(m, " %-15s : enable_map:0x%08x",
-		   "[gpio_dbg]", en_map);
+	p += scnprintf(p, end - p, " %-15s : enable_map:0x%08x",
+		       "[gpio_dbg]", en_map);
 
 	for (i = 0; i < BTC_DBG_MAX1; i++) {
 		if (!(en_map & BIT(i)))
 			continue;
-		seq_printf(m, ", %s->GPIO%d", id_to_gdbg(i), gpio_map[i]);
+		p += scnprintf(p, end - p, ", %s->GPIO%d", id_to_gdbg(i),
+			       gpio_map[i]);
 	}
-	seq_puts(m, "\n");
+	p += scnprintf(p, end - p, "\n");
+
+out:
+	return p - buf;
 }
 
-static void _show_mreg_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_mreg_v1(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -10086,45 +10855,47 @@ static void _show_mreg_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_mac_ax_coex_gnt gnt_cfg = {};
 	struct rtw89_mac_ax_gnt gnt;
+	char *p = buf, *end = buf + bufsz;
 	u8 i = 0, type = 0, cnt = 0;
 	u32 val, offset;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_MREG))
-		return;
+		return 0;
 
-	seq_puts(m, "========== [HW Status] ==========\n");
+	p += scnprintf(p, end - p, "========== [HW Status] ==========\n");
 
-	seq_printf(m,
-		   " %-15s : WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)\n",
-		   "[scoreboard]", wl->scbd, cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
-		   bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
-		   cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
+	p += scnprintf(p, end - p,
+		       " %-15s : WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)\n",
+		       "[scoreboard]", wl->scbd,
+		       cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
+		       bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
+		       cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
 
 	btc->dm.pta_owner = rtw89_mac_get_ctrl_path(rtwdev);
 	_get_gnt(rtwdev, &gnt_cfg);
 
 	gnt = gnt_cfg.band[0];
-	seq_printf(m,
-		   " %-15s : pta_owner:%s, phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d], ",
-		   "[gnt_status]",
-		   chip->chip_id == RTL8852C ? "HW" :
-		   btc->dm.pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
-		   gnt.gnt_wl_sw_en ? "SW" : "HW", gnt.gnt_wl,
-		   gnt.gnt_bt_sw_en ? "SW" : "HW", gnt.gnt_bt);
+	p += scnprintf(p, end - p,
+		       " %-15s : pta_owner:%s, phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d], ",
+		       "[gnt_status]",
+		       chip->chip_id == RTL8852C ? "HW" :
+		       btc->dm.pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
+		       gnt.gnt_wl_sw_en ? "SW" : "HW", gnt.gnt_wl,
+		       gnt.gnt_bt_sw_en ? "SW" : "HW", gnt.gnt_bt);
 
 	gnt = gnt_cfg.band[1];
-	seq_printf(m, "phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]\n",
-		   gnt.gnt_wl_sw_en ? "SW" : "HW",
-		   gnt.gnt_wl,
-		   gnt.gnt_bt_sw_en ? "SW" : "HW",
-		   gnt.gnt_bt);
+	p += scnprintf(p, end - p, "phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]\n",
+		       gnt.gnt_wl_sw_en ? "SW" : "HW",
+		       gnt.gnt_wl,
+		       gnt.gnt_bt_sw_en ? "SW" : "HW",
+		       gnt.gnt_bt);
 
 	pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
 	if (!pcinfo->valid) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): stop due rpt_fbtc_mregval.cinfo\n",
 			    __func__);
-		return;
+		goto out;
 	}
 
 	pmreg = &pfwinfo->rpt_fbtc_mregval.finfo.v1;
@@ -10138,21 +10909,26 @@ static void _show_mreg_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
 		val = le32_to_cpu(pmreg->mreg_val[i]);
 
 		if (cnt % 6 == 0)
-			seq_printf(m, " %-15s : %d_0x%04x=0x%08x",
-				   "[reg]", (u32)type, offset, val);
+			p += scnprintf(p, end - p,
+				       " %-15s : %d_0x%04x=0x%08x",
+				       "[reg]", (u32)type, offset, val);
 		else
-			seq_printf(m, ", %d_0x%04x=0x%08x", (u32)type,
-				   offset, val);
+			p += scnprintf(p, end - p, ", %d_0x%04x=0x%08x",
+				       (u32)type,
+				       offset, val);
 		if (cnt % 6 == 5)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 		cnt++;
 
 		if (i >= pmreg->reg_num)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
+
+out:
+	return p - buf;
 }
 
-static void _show_mreg_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_mreg_v2(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -10164,46 +10940,48 @@ static void _show_mreg_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_mac_ax_coex_gnt gnt_cfg = {};
 	struct rtw89_mac_ax_gnt gnt;
+	char *p = buf, *end = buf + bufsz;
 	u8 i = 0, type = 0, cnt = 0;
 	u32 val, offset;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_MREG))
-		return;
+		return 0;
 
-	seq_puts(m, "========== [HW Status] ==========\n");
+	p += scnprintf(p, end - p, "========== [HW Status] ==========\n");
 
-	seq_printf(m,
-		   " %-15s : WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)\n",
-		   "[scoreboard]", wl->scbd, cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
-		   bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
-		   cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
+	p += scnprintf(p, end - p,
+		       " %-15s : WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)\n",
+		       "[scoreboard]", wl->scbd,
+		       cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
+		       bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
+		       cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
 
 	btc->dm.pta_owner = rtw89_mac_get_ctrl_path(rtwdev);
 	_get_gnt(rtwdev, &gnt_cfg);
 
 	gnt = gnt_cfg.band[0];
-	seq_printf(m,
-		   " %-15s : pta_owner:%s, phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d], polut_type:%s",
-		   "[gnt_status]",
-		   chip->chip_id == RTL8852C ? "HW" :
-		   btc->dm.pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
-		   gnt.gnt_wl_sw_en ? "SW" : "HW", gnt.gnt_wl,
-		   gnt.gnt_bt_sw_en ? "SW" : "HW", gnt.gnt_bt,
-		   id_to_polut(wl->bt_polut_type[wl->pta_req_mac]));
+	p += scnprintf(p, end - p,
+		       " %-15s : pta_owner:%s, phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d], polut_type:%s",
+		       "[gnt_status]",
+		       chip->chip_id == RTL8852C ? "HW" :
+		       btc->dm.pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
+		       gnt.gnt_wl_sw_en ? "SW" : "HW", gnt.gnt_wl,
+		       gnt.gnt_bt_sw_en ? "SW" : "HW", gnt.gnt_bt,
+		       id_to_polut(wl->bt_polut_type[wl->pta_req_mac]));
 
 	gnt = gnt_cfg.band[1];
-	seq_printf(m, "phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]\n",
-		   gnt.gnt_wl_sw_en ? "SW" : "HW",
-		   gnt.gnt_wl,
-		   gnt.gnt_bt_sw_en ? "SW" : "HW",
-		   gnt.gnt_bt);
+	p += scnprintf(p, end - p, "phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]\n",
+		       gnt.gnt_wl_sw_en ? "SW" : "HW",
+		       gnt.gnt_wl,
+		       gnt.gnt_bt_sw_en ? "SW" : "HW",
+		       gnt.gnt_bt);
 
 	pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
 	if (!pcinfo->valid) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): stop due rpt_fbtc_mregval.cinfo\n",
 			    __func__);
-		return;
+		goto out;
 	}
 
 	pmreg = &pfwinfo->rpt_fbtc_mregval.finfo.v2;
@@ -10217,21 +10995,26 @@ static void _show_mreg_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 		val = le32_to_cpu(pmreg->mreg_val[i]);
 
 		if (cnt % 6 == 0)
-			seq_printf(m, " %-15s : %d_0x%04x=0x%08x",
-				   "[reg]", (u32)type, offset, val);
+			p += scnprintf(p, end - p,
+				       " %-15s : %d_0x%04x=0x%08x",
+				       "[reg]", (u32)type, offset, val);
 		else
-			seq_printf(m, ", %d_0x%04x=0x%08x", (u32)type,
-				   offset, val);
+			p += scnprintf(p, end - p, ", %d_0x%04x=0x%08x",
+				       (u32)type,
+				       offset, val);
 		if (cnt % 6 == 5)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 		cnt++;
 
 		if (i >= pmreg->reg_num)
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 	}
+
+out:
+	return p - buf;
 }
 
-static void _show_mreg_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_mreg_v7(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
@@ -10242,46 +11025,50 @@ static void _show_mreg_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_bt_info *bt = &cx->bt;
 	struct rtw89_mac_ax_gnt *gnt = NULL;
 	struct rtw89_btc_dm *dm = &btc->dm;
+	char *p = buf, *end = buf + bufsz;
 	u8 i, type, cnt = 0;
 	u32 val, offset;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_MREG))
-		return;
+		return 0;
 
-	seq_puts(m, "\n\r========== [HW Status] ==========");
+	p += scnprintf(p, end - p, "\n\r========== [HW Status] ==========");
 
-	seq_printf(m,
-		   "\n\r %-15s : WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)",
-		   "[scoreboard]", wl->scbd, cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
-		   bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
-		   cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
+	p += scnprintf(p, end - p,
+		       "\n\r %-15s : WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)",
+		       "[scoreboard]", wl->scbd,
+		       cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
+		       bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
+		       cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
 
 	/* To avoid I/O if WL LPS or power-off  */
 	dm->pta_owner = rtw89_mac_get_ctrl_path(rtwdev);
 
-	seq_printf(m,
-		   "\n\r %-15s : pta_owner:%s, pta_req_mac:MAC%d, rf_gnt_source: polut_type:%s",
-		   "[gnt_status]",
-		   rtwdev->chip->para_ver & BTC_FEAT_PTA_ONOFF_CTRL ? "HW" :
-		   dm->pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
-		   wl->pta_req_mac, id_to_polut(wl->bt_polut_type[wl->pta_req_mac]));
+	p += scnprintf(p, end - p,
+		       "\n\r %-15s : pta_owner:%s, pta_req_mac:MAC%d, rf_gnt_source: polut_type:%s",
+		       "[gnt_status]",
+		       rtwdev->chip->para_ver & BTC_FEAT_PTA_ONOFF_CTRL ? "HW" :
+		       dm->pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
+		       wl->pta_req_mac,
+		       id_to_polut(wl->bt_polut_type[wl->pta_req_mac]));
 
 	gnt = &dm->gnt.band[RTW89_PHY_0];
 
-	seq_printf(m, ", phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d]",
-		   gnt->gnt_wl_sw_en ? "SW" : "HW", gnt->gnt_wl,
-		   gnt->gnt_bt_sw_en ? "SW" : "HW", gnt->gnt_bt);
+	p += scnprintf(p, end - p, ", phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d]",
+		       gnt->gnt_wl_sw_en ? "SW" : "HW", gnt->gnt_wl,
+		       gnt->gnt_bt_sw_en ? "SW" : "HW", gnt->gnt_bt);
 
 	if (rtwdev->dbcc_en) {
 		gnt = &dm->gnt.band[RTW89_PHY_1];
-		seq_printf(m, ", phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]",
-			   gnt->gnt_wl_sw_en ? "SW" : "HW", gnt->gnt_wl,
-			   gnt->gnt_bt_sw_en ? "SW" : "HW", gnt->gnt_bt);
+		p += scnprintf(p, end - p,
+			       ", phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]",
+			       gnt->gnt_wl_sw_en ? "SW" : "HW", gnt->gnt_wl,
+			       gnt->gnt_bt_sw_en ? "SW" : "HW", gnt->gnt_bt);
 	}
 
 	pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
 	if (!pcinfo->valid)
-		return;
+		goto out;
 
 	pmreg = &pfwinfo->rpt_fbtc_mregval.finfo.v7;
 
@@ -10291,17 +11078,20 @@ static void _show_mreg_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
 		val = le32_to_cpu(pmreg->mreg_val[i]);
 
 		if (cnt % 6 == 0)
-			seq_printf(m, "\n\r %-15s : %s_0x%x=0x%x", "[reg]",
-				   id_to_regtype(type), offset, val);
+			p += scnprintf(p, end - p,
+				       "\n\r %-15s : %s_0x%x=0x%x", "[reg]",
+				       id_to_regtype(type), offset, val);
 		else
-			seq_printf(m, ", %s_0x%x=0x%x",
-				   id_to_regtype(type), offset, val);
+			p += scnprintf(p, end - p, ", %s_0x%x=0x%x",
+				       id_to_regtype(type), offset, val);
 		cnt++;
 	}
-	seq_puts(m, "\n");
+
+out:
+	return p - buf;
 }
 
-static void _show_summary_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_summary_v1(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
@@ -10312,56 +11102,59 @@ static void _show_summary_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	struct rtw89_btc_bt_info *bt = &cx->bt;
 	u32 cnt_sum = 0, *cnt = btc->dm.cnt_notify;
+	char *p = buf, *end = buf + bufsz;
 	u8 i;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
-		return;
+		return 0;
 
-	seq_puts(m, "========== [Statistics] ==========\n");
+	p += scnprintf(p, end - p, "========== [Statistics] ==========\n");
 
 	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
 	if (pcinfo->valid && !wl->status.map.lps && !wl->status.map.rf_off) {
 		prptctrl = &pfwinfo->rpt_ctrl.finfo.v1;
 
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d), ",
-			   "[summary]", pfwinfo->cnt_h2c,
-			   pfwinfo->cnt_h2c_fail, prptctrl->h2c_cnt,
-			   pfwinfo->cnt_c2h, prptctrl->c2h_cnt);
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d), ",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail, prptctrl->h2c_cnt,
+			       pfwinfo->cnt_c2h, prptctrl->c2h_cnt);
 
-		seq_printf(m,
-			   "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x, dm_error_map:0x%x",
-			   pfwinfo->event[BTF_EVNT_RPT], prptctrl->rpt_cnt,
-			   prptctrl->rpt_enable, dm->error.val);
+		p += scnprintf(p, end - p,
+			       "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x, dm_error_map:0x%x",
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       prptctrl->rpt_cnt,
+			       prptctrl->rpt_enable, dm->error.val);
 
 		if (dm->error.map.wl_fw_hang)
-			seq_puts(m, " (WL FW Hang!!)");
-		seq_puts(m, "\n");
-		seq_printf(m,
-			   " %-15s : send_ok:%d, send_fail:%d, recv:%d",
-			   "[mailbox]", prptctrl->mb_send_ok_cnt,
-			   prptctrl->mb_send_fail_cnt, prptctrl->mb_recv_cnt);
-
-		seq_printf(m,
-			   "(A2DP_empty:%d, A2DP_flowstop:%d, A2DP_full:%d)\n",
-			   prptctrl->mb_a2dp_empty_cnt,
-			   prptctrl->mb_a2dp_flct_cnt,
-			   prptctrl->mb_a2dp_full_cnt);
-
-		seq_printf(m,
-			   " %-15s : wl_rfk[req:%d/go:%d/reject:%d/timeout:%d]",
-			   "[RFK]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
-			   cx->cnt_wl[BTC_WCNT_RFK_GO],
-			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
-			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
-
-		seq_printf(m,
-			   ", bt_rfk[req:%d/go:%d/reject:%d/timeout:%d/fail:%d]\n",
-			   prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_REQ],
-			   prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_GO],
-			   prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_REJECT],
-			   prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_TIMEOUT],
-			   prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_FAIL]);
+			p += scnprintf(p, end - p, " (WL FW Hang!!)");
+		p += scnprintf(p, end - p, "\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : send_ok:%d, send_fail:%d, recv:%d",
+			       "[mailbox]", prptctrl->mb_send_ok_cnt,
+			       prptctrl->mb_send_fail_cnt,
+			       prptctrl->mb_recv_cnt);
+
+		p += scnprintf(p, end - p,
+			       "(A2DP_empty:%d, A2DP_flowstop:%d, A2DP_full:%d)\n",
+			       prptctrl->mb_a2dp_empty_cnt,
+			       prptctrl->mb_a2dp_flct_cnt,
+			       prptctrl->mb_a2dp_full_cnt);
+
+		p += scnprintf(p, end - p,
+			       " %-15s : wl_rfk[req:%d/go:%d/reject:%d/timeout:%d]",
+			       "[RFK]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			       cx->cnt_wl[BTC_WCNT_RFK_GO],
+			       cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			       cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
+
+		p += scnprintf(p, end - p,
+			       ", bt_rfk[req:%d/go:%d/reject:%d/timeout:%d/fail:%d]\n",
+			       prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_REQ],
+			       prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_GO],
+			       prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_REJECT],
+			       prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_TIMEOUT],
+			       prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_FAIL]);
 
 		if (prptctrl->bt_rfk_cnt[BTC_BCNT_RFK_TIMEOUT] > 0)
 			bt->rfk_info.map.timeout = 1;
@@ -10370,42 +11163,44 @@ static void _show_summary_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 		dm->error.map.wl_rfk_timeout = bt->rfk_info.map.timeout;
 	} else {
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d, rpt_cnt=%d, rpt_map=0x%x",
-			   "[summary]", pfwinfo->cnt_h2c,
-			   pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h,
-			   pfwinfo->event[BTF_EVNT_RPT],
-			   btc->fwinfo.rpt_en_map);
-		seq_puts(m, " (WL FW report invalid!!)\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d, rpt_cnt=%d, rpt_map=0x%x",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h,
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       btc->fwinfo.rpt_en_map);
+		p += scnprintf(p, end - p, " (WL FW report invalid!!)\n");
 	}
 
 	for (i = 0; i < BTC_NCNT_NUM; i++)
 		cnt_sum += dm->cnt_notify[i];
 
-	seq_printf(m,
-		   " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
-		   "[notify_cnt]", cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
-		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+	p += scnprintf(p, end - p,
+		       " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		       "[notify_cnt]", cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		       cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+
+	p += scnprintf(p, end - p,
+		       "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d\n",
+		       cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		       cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		       cnt[BTC_NCNT_WL_STA]);
 
-	seq_printf(m,
-		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d\n",
-		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
-		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
-		   cnt[BTC_NCNT_WL_STA]);
+	p += scnprintf(p, end - p,
+		       " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
+		       "[notify_cnt]", cnt[BTC_NCNT_SCAN_START],
+		       cnt[BTC_NCNT_SCAN_FINISH], cnt[BTC_NCNT_SWITCH_BAND],
+		       cnt[BTC_NCNT_SPECIAL_PACKET]);
 
-	seq_printf(m,
-		   " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
-		   "[notify_cnt]", cnt[BTC_NCNT_SCAN_START],
-		   cnt[BTC_NCNT_SCAN_FINISH], cnt[BTC_NCNT_SWITCH_BAND],
-		   cnt[BTC_NCNT_SPECIAL_PACKET]);
+	p += scnprintf(p, end - p,
+		       "timer=%d, control=%d, customerize=%d\n",
+		       cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
+		       cnt[BTC_NCNT_CUSTOMERIZE]);
 
-	seq_printf(m,
-		   "timer=%d, control=%d, customerize=%d\n",
-		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
-		   cnt[BTC_NCNT_CUSTOMERIZE]);
+	return p - buf;
 }
 
-static void _show_summary_v4(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_summary_v4(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
@@ -10416,64 +11211,65 @@ static void _show_summary_v4(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	struct rtw89_btc_bt_info *bt = &cx->bt;
 	u32 cnt_sum = 0, *cnt = btc->dm.cnt_notify;
+	char *p = buf, *end = buf + bufsz;
 	u8 i;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
-		return;
+		return 0;
 
-	seq_puts(m, "========== [Statistics] ==========\n");
+	p += scnprintf(p, end - p, "========== [Statistics] ==========\n");
 
 	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
 	if (pcinfo->valid && !wl->status.map.lps && !wl->status.map.rf_off) {
 		prptctrl = &pfwinfo->rpt_ctrl.finfo.v4;
 
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d), ",
-			   "[summary]", pfwinfo->cnt_h2c,
-			   pfwinfo->cnt_h2c_fail,
-			   le32_to_cpu(prptctrl->rpt_info.cnt_h2c),
-			   pfwinfo->cnt_c2h,
-			   le32_to_cpu(prptctrl->rpt_info.cnt_c2h));
-
-		seq_printf(m,
-			   "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x, dm_error_map:0x%x",
-			   pfwinfo->event[BTF_EVNT_RPT],
-			   le32_to_cpu(prptctrl->rpt_info.cnt),
-			   le32_to_cpu(prptctrl->rpt_info.en),
-			   dm->error.val);
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d), ",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail,
+			       le32_to_cpu(prptctrl->rpt_info.cnt_h2c),
+			       pfwinfo->cnt_c2h,
+			       le32_to_cpu(prptctrl->rpt_info.cnt_c2h));
+
+		p += scnprintf(p, end - p,
+			       "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x, dm_error_map:0x%x",
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       le32_to_cpu(prptctrl->rpt_info.cnt),
+			       le32_to_cpu(prptctrl->rpt_info.en),
+			       dm->error.val);
 
 		if (dm->error.map.wl_fw_hang)
-			seq_puts(m, " (WL FW Hang!!)");
-		seq_puts(m, "\n");
-		seq_printf(m,
-			   " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
-			   "[mailbox]",
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
-
-		seq_printf(m,
-			   "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
-
-		seq_printf(m,
-			   " %-15s : wl_rfk[req:%d/go:%d/reject:%d/timeout:%d]",
-			   "[RFK]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
-			   cx->cnt_wl[BTC_WCNT_RFK_GO],
-			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
-			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
-
-		seq_printf(m,
-			   ", bt_rfk[req:%d/go:%d/reject:%d/timeout:%d/fail:%d]\n",
-			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]),
-			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_GO]),
-			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REJECT]),
-			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_TIMEOUT]),
-			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_FAIL]));
+			p += scnprintf(p, end - p, " (WL FW Hang!!)");
+		p += scnprintf(p, end - p, "\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			       "[mailbox]",
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
+
+		p += scnprintf(p, end - p,
+			       "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
+
+		p += scnprintf(p, end - p,
+			       " %-15s : wl_rfk[req:%d/go:%d/reject:%d/timeout:%d]",
+			       "[RFK]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			       cx->cnt_wl[BTC_WCNT_RFK_GO],
+			       cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			       cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
+
+		p += scnprintf(p, end - p,
+			       ", bt_rfk[req:%d/go:%d/reject:%d/timeout:%d/fail:%d]\n",
+			       le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]),
+			       le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_GO]),
+			       le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REJECT]),
+			       le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_TIMEOUT]),
+			       le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_FAIL]));
 
 		if (le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_TIMEOUT]) > 0)
 			bt->rfk_info.map.timeout = 1;
@@ -10482,42 +11278,44 @@ static void _show_summary_v4(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 		dm->error.map.wl_rfk_timeout = bt->rfk_info.map.timeout;
 	} else {
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d, rpt_cnt=%d, rpt_map=0x%x",
-			   "[summary]", pfwinfo->cnt_h2c,
-			   pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h,
-			   pfwinfo->event[BTF_EVNT_RPT],
-			   btc->fwinfo.rpt_en_map);
-		seq_puts(m, " (WL FW report invalid!!)\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d, rpt_cnt=%d, rpt_map=0x%x",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h,
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       btc->fwinfo.rpt_en_map);
+		p += scnprintf(p, end - p, " (WL FW report invalid!!)\n");
 	}
 
 	for (i = 0; i < BTC_NCNT_NUM; i++)
 		cnt_sum += dm->cnt_notify[i];
 
-	seq_printf(m,
-		   " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
-		   "[notify_cnt]", cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
-		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+	p += scnprintf(p, end - p,
+		       " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		       "[notify_cnt]", cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		       cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+
+	p += scnprintf(p, end - p,
+		       "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d\n",
+		       cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		       cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		       cnt[BTC_NCNT_WL_STA]);
 
-	seq_printf(m,
-		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d\n",
-		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
-		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
-		   cnt[BTC_NCNT_WL_STA]);
+	p += scnprintf(p, end - p,
+		       " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
+		       "[notify_cnt]", cnt[BTC_NCNT_SCAN_START],
+		       cnt[BTC_NCNT_SCAN_FINISH], cnt[BTC_NCNT_SWITCH_BAND],
+		       cnt[BTC_NCNT_SPECIAL_PACKET]);
 
-	seq_printf(m,
-		   " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
-		   "[notify_cnt]", cnt[BTC_NCNT_SCAN_START],
-		   cnt[BTC_NCNT_SCAN_FINISH], cnt[BTC_NCNT_SWITCH_BAND],
-		   cnt[BTC_NCNT_SPECIAL_PACKET]);
+	p += scnprintf(p, end - p,
+		       "timer=%d, control=%d, customerize=%d\n",
+		       cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
+		       cnt[BTC_NCNT_CUSTOMERIZE]);
 
-	seq_printf(m,
-		   "timer=%d, control=%d, customerize=%d\n",
-		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
-		   cnt[BTC_NCNT_CUSTOMERIZE]);
+	return p - buf;
 }
 
-static void _show_summary_v5(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_summary_v5(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
@@ -10527,112 +11325,118 @@ static void _show_summary_v5(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_dm *dm = &btc->dm;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	u32 cnt_sum = 0, *cnt = btc->dm.cnt_notify;
+	char *p = buf, *end = buf + bufsz;
 	u8 i;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
-		return;
+		return 0;
 
-	seq_puts(m, "========== [Statistics] ==========\n");
+	p += scnprintf(p, end - p, "========== [Statistics] ==========\n");
 
 	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
 	if (pcinfo->valid && !wl->status.map.lps && !wl->status.map.rf_off) {
 		prptctrl = &pfwinfo->rpt_ctrl.finfo.v5;
 
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d, len:%d), ",
-			   "[summary]", pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_h2c),
-			   pfwinfo->cnt_c2h,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
-			   le16_to_cpu(prptctrl->rpt_info.len_c2h));
-
-		seq_printf(m,
-			   "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
-			   pfwinfo->event[BTF_EVNT_RPT],
-			   le16_to_cpu(prptctrl->rpt_info.cnt),
-			   le32_to_cpu(prptctrl->rpt_info.en));
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d, len:%d), ",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_h2c),
+			       pfwinfo->cnt_c2h,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
+			       le16_to_cpu(prptctrl->rpt_info.len_c2h));
+
+		p += scnprintf(p, end - p,
+			       "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       le16_to_cpu(prptctrl->rpt_info.cnt),
+			       le32_to_cpu(prptctrl->rpt_info.en));
 
 		if (dm->error.map.wl_fw_hang)
-			seq_puts(m, " (WL FW Hang!!)");
-		seq_puts(m, "\n");
-		seq_printf(m,
-			   " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
-			   "[mailbox]",
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
-
-		seq_printf(m,
-			   "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
-
-		seq_printf(m,
-			   " %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d]",
-			   "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
-			   cx->cnt_wl[BTC_WCNT_RFK_GO],
-			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
-			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
-
-		seq_printf(m,
-			   ", bt_rfk[req:%d]",
-			   le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
-
-		seq_printf(m,
-			   ", AOAC[RF_on:%d/RF_off:%d]",
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
+			p += scnprintf(p, end - p, " (WL FW Hang!!)");
+		p += scnprintf(p, end - p, "\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			       "[mailbox]",
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
+
+		p += scnprintf(p, end - p,
+			       "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
+
+		p += scnprintf(p, end - p,
+			       " %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d]",
+			       "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			       cx->cnt_wl[BTC_WCNT_RFK_GO],
+			       cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			       cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
+
+		p += scnprintf(p, end - p,
+			       ", bt_rfk[req:%d]",
+			       le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
+
+		p += scnprintf(p, end - p,
+			       ", AOAC[RF_on:%d/RF_off:%d]",
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
 	} else {
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d",
-			   "[summary]", pfwinfo->cnt_h2c,
-			   pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h);
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h);
 	}
 
 	if (!pcinfo->valid || pfwinfo->len_mismch || pfwinfo->fver_mismch ||
 	    pfwinfo->err[BTFRE_EXCEPTION]) {
-		seq_puts(m, "\n");
-		seq_printf(m,
-			   " %-15s : WL FW rpt error!![rpt_ctrl_valid:%d/len:"
-			   "0x%x/ver:0x%x/ex:%d/lps=%d/rf_off=%d]",
-			   "[ERROR]", pcinfo->valid, pfwinfo->len_mismch,
-			   pfwinfo->fver_mismch, pfwinfo->err[BTFRE_EXCEPTION],
-			   wl->status.map.lps, wl->status.map.rf_off);
+		p += scnprintf(p, end - p, "\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : WL FW rpt error!![rpt_ctrl_valid:%d/len:"
+			       "0x%x/ver:0x%x/ex:%d/lps=%d/rf_off=%d]",
+			       "[ERROR]", pcinfo->valid, pfwinfo->len_mismch,
+			       pfwinfo->fver_mismch,
+			       pfwinfo->err[BTFRE_EXCEPTION],
+			       wl->status.map.lps, wl->status.map.rf_off);
 	}
 
 	for (i = 0; i < BTC_NCNT_NUM; i++)
 		cnt_sum += dm->cnt_notify[i];
 
-	seq_puts(m, "\n");
-	seq_printf(m,
-		   " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
-		   "[notify_cnt]",
-		   cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
-		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+	p += scnprintf(p, end - p, "\n");
+	p += scnprintf(p, end - p,
+		       " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		       "[notify_cnt]",
+		       cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		       cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+
+	p += scnprintf(p, end - p,
+		       "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
+		       cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		       cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		       cnt[BTC_NCNT_WL_STA]);
 
-	seq_printf(m,
-		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
-		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
-		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
-		   cnt[BTC_NCNT_WL_STA]);
+	p += scnprintf(p, end - p, "\n");
+	p += scnprintf(p, end - p,
+		       " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
+		       "[notify_cnt]",
+		       cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
+		       cnt[BTC_NCNT_SWITCH_BAND],
+		       cnt[BTC_NCNT_SPECIAL_PACKET]);
 
-	seq_puts(m, "\n");
-	seq_printf(m,
-		   " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
-		   "[notify_cnt]",
-		   cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
-		   cnt[BTC_NCNT_SWITCH_BAND], cnt[BTC_NCNT_SPECIAL_PACKET]);
+	p += scnprintf(p, end - p,
+		       "timer=%d, control=%d, customerize=%d",
+		       cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
+		       cnt[BTC_NCNT_CUSTOMERIZE]);
 
-	seq_printf(m,
-		   "timer=%d, control=%d, customerize=%d",
-		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
-		   cnt[BTC_NCNT_CUSTOMERIZE]);
+	return p - buf;
 }
 
-static void _show_summary_v105(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_summary_v105(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
@@ -10642,112 +11446,118 @@ static void _show_summary_v105(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_dm *dm = &btc->dm;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	u32 cnt_sum = 0, *cnt = btc->dm.cnt_notify;
+	char *p = buf, *end = buf + bufsz;
 	u8 i;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
-		return;
+		return 0;
 
-	seq_puts(m, "========== [Statistics] ==========\n");
+	p += scnprintf(p, end - p, "========== [Statistics] ==========\n");
 
 	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
 	if (pcinfo->valid && !wl->status.map.lps && !wl->status.map.rf_off) {
 		prptctrl = &pfwinfo->rpt_ctrl.finfo.v105;
 
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d, len:%d), ",
-			   "[summary]", pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_h2c),
-			   pfwinfo->cnt_c2h,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
-			   le16_to_cpu(prptctrl->rpt_info.len_c2h));
-
-		seq_printf(m,
-			   "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
-			   pfwinfo->event[BTF_EVNT_RPT],
-			   le16_to_cpu(prptctrl->rpt_info.cnt),
-			   le32_to_cpu(prptctrl->rpt_info.en));
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d, len:%d), ",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_h2c),
+			       pfwinfo->cnt_c2h,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
+			       le16_to_cpu(prptctrl->rpt_info.len_c2h));
+
+		p += scnprintf(p, end - p,
+			       "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       le16_to_cpu(prptctrl->rpt_info.cnt),
+			       le32_to_cpu(prptctrl->rpt_info.en));
 
 		if (dm->error.map.wl_fw_hang)
-			seq_puts(m, " (WL FW Hang!!)");
-		seq_puts(m, "\n");
-		seq_printf(m,
-			   " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
-			   "[mailbox]",
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
-
-		seq_printf(m,
-			   "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
-
-		seq_printf(m,
-			   " %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d]",
-			   "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
-			   cx->cnt_wl[BTC_WCNT_RFK_GO],
-			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
-			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
-
-		seq_printf(m,
-			   ", bt_rfk[req:%d]",
-			   le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
-
-		seq_printf(m,
-			   ", AOAC[RF_on:%d/RF_off:%d]",
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
+			p += scnprintf(p, end - p, " (WL FW Hang!!)");
+		p += scnprintf(p, end - p, "\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			       "[mailbox]",
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
+
+		p += scnprintf(p, end - p,
+			       "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
+
+		p += scnprintf(p, end - p,
+			       " %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d]",
+			       "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			       cx->cnt_wl[BTC_WCNT_RFK_GO],
+			       cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			       cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
+
+		p += scnprintf(p, end - p,
+			       ", bt_rfk[req:%d]",
+			       le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
+
+		p += scnprintf(p, end - p,
+			       ", AOAC[RF_on:%d/RF_off:%d]",
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
 	} else {
-		seq_printf(m,
-			   " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d",
-			   "[summary]", pfwinfo->cnt_h2c,
-			   pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h);
+		p += scnprintf(p, end - p,
+			       " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h);
 	}
 
 	if (!pcinfo->valid || pfwinfo->len_mismch || pfwinfo->fver_mismch ||
 	    pfwinfo->err[BTFRE_EXCEPTION]) {
-		seq_puts(m, "\n");
-		seq_printf(m,
-			   " %-15s : WL FW rpt error!![rpt_ctrl_valid:%d/len:"
-			   "0x%x/ver:0x%x/ex:%d/lps=%d/rf_off=%d]",
-			   "[ERROR]", pcinfo->valid, pfwinfo->len_mismch,
-			   pfwinfo->fver_mismch, pfwinfo->err[BTFRE_EXCEPTION],
-			   wl->status.map.lps, wl->status.map.rf_off);
+		p += scnprintf(p, end - p, "\n");
+		p += scnprintf(p, end - p,
+			       " %-15s : WL FW rpt error!![rpt_ctrl_valid:%d/len:"
+			       "0x%x/ver:0x%x/ex:%d/lps=%d/rf_off=%d]",
+			       "[ERROR]", pcinfo->valid, pfwinfo->len_mismch,
+			       pfwinfo->fver_mismch,
+			       pfwinfo->err[BTFRE_EXCEPTION],
+			       wl->status.map.lps, wl->status.map.rf_off);
 	}
 
 	for (i = 0; i < BTC_NCNT_NUM; i++)
 		cnt_sum += dm->cnt_notify[i];
 
-	seq_puts(m, "\n");
-	seq_printf(m,
-		   " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
-		   "[notify_cnt]",
-		   cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
-		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+	p += scnprintf(p, end - p, "\n");
+	p += scnprintf(p, end - p,
+		       " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		       "[notify_cnt]",
+		       cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		       cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
 
-	seq_printf(m,
-		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
-		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
-		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
-		   cnt[BTC_NCNT_WL_STA]);
+	p += scnprintf(p, end - p,
+		       "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
+		       cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		       cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		       cnt[BTC_NCNT_WL_STA]);
 
-	seq_puts(m, "\n");
-	seq_printf(m,
-		   " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
-		   "[notify_cnt]",
-		   cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
-		   cnt[BTC_NCNT_SWITCH_BAND], cnt[BTC_NCNT_SPECIAL_PACKET]);
+	p += scnprintf(p, end - p, "\n");
+	p += scnprintf(p, end - p,
+		       " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
+		       "[notify_cnt]",
+		       cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
+		       cnt[BTC_NCNT_SWITCH_BAND],
+		       cnt[BTC_NCNT_SPECIAL_PACKET]);
 
-	seq_printf(m,
-		   "timer=%d, control=%d, customerize=%d",
-		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
-		   cnt[BTC_NCNT_CUSTOMERIZE]);
+	p += scnprintf(p, end - p,
+		       "timer=%d, control=%d, customerize=%d",
+		       cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
+		       cnt[BTC_NCNT_CUSTOMERIZE]);
+
+	return p - buf;
 }
 
-static void _show_summary_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_summary_v7(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &rtwdev->btc.fwinfo;
 	struct rtw89_btc_fbtc_rpt_ctrl_v7 *prptctrl = NULL;
@@ -10756,100 +11566,111 @@ static void _show_summary_v7(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_dm *dm = &rtwdev->btc.dm;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	u32 *cnt = rtwdev->btc.dm.cnt_notify;
+	char *p = buf, *end = buf + bufsz;
 	u32 cnt_sum = 0;
 	u8 i;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
-		return;
+		return 0;
 
-	seq_printf(m, "%s", "\n\r========== [Statistics] ==========");
+	p += scnprintf(p, end - p, "%s",
+		       "\n\r========== [Statistics] ==========");
 
 	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
 	if (pcinfo->valid && wl->status.map.lps != BTC_LPS_RF_OFF &&
 	    !wl->status.map.rf_off) {
 		prptctrl = &pfwinfo->rpt_ctrl.finfo.v7;
 
-		seq_printf(m,
-			   "\n\r %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d),"
-			   "c2h_cnt=%d(fw_send:%d, len:%d, max:%d), ",
-			   "[summary]", pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_h2c), pfwinfo->cnt_c2h,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
-			   le16_to_cpu(prptctrl->rpt_info.len_c2h),
-			   rtwdev->btc.ver->info_buf);
-
-		seq_printf(m, "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
-			   pfwinfo->event[BTF_EVNT_RPT],
-			   le16_to_cpu(prptctrl->rpt_info.cnt),
-			   le32_to_cpu(prptctrl->rpt_info.en));
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d),"
+			       "c2h_cnt=%d(fw_send:%d, len:%d, max:%d), ",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_h2c),
+			       pfwinfo->cnt_c2h,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
+			       le16_to_cpu(prptctrl->rpt_info.len_c2h),
+			       rtwdev->btc.ver->info_buf);
+
+		p += scnprintf(p, end - p,
+			       "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       le16_to_cpu(prptctrl->rpt_info.cnt),
+			       le32_to_cpu(prptctrl->rpt_info.en));
 
 		if (dm->error.map.wl_fw_hang)
-			seq_puts(m, " (WL FW Hang!!)");
-
-		seq_printf(m, "\n\r %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
-			   "[mailbox]", le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
-
-		seq_printf(m, "A2DP_empty:%d(stop:%d/tx:%d/ack:%d/nack:%d)",
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
-
-		seq_printf(m,
-			   "\n\r %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d/time:%dms]",
-			   "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
-			   cx->cnt_wl[BTC_WCNT_RFK_GO],
-			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
-			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT],
-			   wl->rfk_info.proc_time);
-
-		seq_printf(m, ", bt_rfk[req:%d]",
-			   le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
-
-		seq_printf(m, ", AOAC[RF_on:%d/RF_off:%d]",
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
+			p += scnprintf(p, end - p, " (WL FW Hang!!)");
+
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			       "[mailbox]",
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
+
+		p += scnprintf(p, end - p,
+			       "A2DP_empty:%d(stop:%d/tx:%d/ack:%d/nack:%d)",
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
+
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d/time:%dms]",
+			       "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			       cx->cnt_wl[BTC_WCNT_RFK_GO],
+			       cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			       cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT],
+			       wl->rfk_info.proc_time);
+
+		p += scnprintf(p, end - p, ", bt_rfk[req:%d]",
+			       le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
+
+		p += scnprintf(p, end - p, ", AOAC[RF_on:%d/RF_off:%d]",
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
 	} else {
-		seq_printf(m,
-			   "\n\r %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d (lps=%d/rf_off=%d)",
-			   "[summary]",
-			   pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
-			   pfwinfo->cnt_c2h,
-			   wl->status.map.lps, wl->status.map.rf_off);
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d (lps=%d/rf_off=%d)",
+			       "[summary]",
+			       pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
+			       pfwinfo->cnt_c2h,
+			       wl->status.map.lps, wl->status.map.rf_off);
 	}
 
 	for (i = 0; i < BTC_NCNT_NUM; i++)
 		cnt_sum += dm->cnt_notify[i];
 
-	seq_printf(m,
-		   "\n\r %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
-		   "[notify_cnt]",
-		   cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
-		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+	p += scnprintf(p, end - p,
+		       "\n\r %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		       "[notify_cnt]",
+		       cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		       cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+
+	p += scnprintf(p, end - p,
+		       "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
+		       cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		       cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		       cnt[BTC_NCNT_WL_STA]);
 
-	seq_printf(m,
-		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
-		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
-		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
-		   cnt[BTC_NCNT_WL_STA]);
+	p += scnprintf(p, end - p,
+		       "\n\r %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, switch_chbw=%d, special_pkt=%d, ",
+		       "[notify_cnt]",
+		       cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
+		       cnt[BTC_NCNT_SWITCH_BAND], cnt[BTC_NCNT_SWITCH_CHBW],
+		       cnt[BTC_NCNT_SPECIAL_PACKET]);
 
-	seq_printf(m,
-		   "\n\r %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, switch_chbw=%d, special_pkt=%d, ",
-		   "[notify_cnt]",
-		   cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
-		   cnt[BTC_NCNT_SWITCH_BAND], cnt[BTC_NCNT_SWITCH_CHBW],
-		   cnt[BTC_NCNT_SPECIAL_PACKET]);
+	p += scnprintf(p, end - p,
+		       "timer=%d, customerize=%d, hub_msg=%d, chg_fw=%d, send_cc=%d",
+		       cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CUSTOMERIZE],
+		       rtwdev->btc.hubmsg_cnt, cnt[BTC_NCNT_RESUME_DL_FW],
+		       cnt[BTC_NCNT_COUNTRYCODE]);
 
-	seq_printf(m, "timer=%d, customerize=%d, hub_msg=%d, chg_fw=%d, send_cc=%d",
-		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CUSTOMERIZE],
-		   rtwdev->btc.hubmsg_cnt, cnt[BTC_NCNT_RESUME_DL_FW],
-		   cnt[BTC_NCNT_COUNTRYCODE]);
+	return p - buf;
 }
 
-static void _show_summary_v8(struct rtw89_dev *rtwdev, struct seq_file *m)
+static int _show_summary_v8(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &rtwdev->btc.fwinfo;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
@@ -10858,153 +11679,175 @@ static void _show_summary_v8(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_dm *dm = &rtwdev->btc.dm;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	u32 *cnt = rtwdev->btc.dm.cnt_notify;
+	char *p = buf, *end = buf + bufsz;
 	u32 cnt_sum = 0;
 	u8 i;
 
 	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
-		return;
+		return 0;
 
-	seq_printf(m, "%s", "\n\r========== [Statistics] ==========");
+	p += scnprintf(p, end - p, "%s",
+		       "\n\r========== [Statistics] ==========");
 
 	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
 	if (pcinfo->valid && wl->status.map.lps != BTC_LPS_RF_OFF &&
 	    !wl->status.map.rf_off) {
 		prptctrl = &pfwinfo->rpt_ctrl.finfo.v8;
 
-		seq_printf(m,
-			   "\n\r %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d, len:%d, max:fw-%d/drv-%d), ",
-			   "[summary]", pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_h2c), pfwinfo->cnt_c2h,
-			   le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
-			   le16_to_cpu(prptctrl->rpt_info.len_c2h),
-			   (prptctrl->rpt_len_max_h << 8) + prptctrl->rpt_len_max_l,
-			   rtwdev->btc.ver->info_buf);
-
-		seq_printf(m, "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
-			   pfwinfo->event[BTF_EVNT_RPT],
-			   le16_to_cpu(prptctrl->rpt_info.cnt),
-			   le32_to_cpu(prptctrl->rpt_info.en));
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d, len:%d, max:fw-%d/drv-%d), ",
+			       "[summary]", pfwinfo->cnt_h2c,
+			       pfwinfo->cnt_h2c_fail,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_h2c),
+			       pfwinfo->cnt_c2h,
+			       le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
+			       le16_to_cpu(prptctrl->rpt_info.len_c2h),
+			       (prptctrl->rpt_len_max_h << 8) + prptctrl->rpt_len_max_l,
+			       rtwdev->btc.ver->info_buf);
+
+		p += scnprintf(p, end - p,
+			       "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
+			       pfwinfo->event[BTF_EVNT_RPT],
+			       le16_to_cpu(prptctrl->rpt_info.cnt),
+			       le32_to_cpu(prptctrl->rpt_info.en));
 
 		if (dm->error.map.wl_fw_hang)
-			seq_puts(m, " (WL FW Hang!!)");
-
-		seq_printf(m, "\n\r %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
-			   "[mailbox]", le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
-			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
-
-		seq_printf(m, "A2DP_empty:%d(stop:%d/tx:%d/ack:%d/nack:%d)",
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
-			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
-
-		seq_printf(m,
-			   "\n\r %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d/time:%dms]",
-			   "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
-			   cx->cnt_wl[BTC_WCNT_RFK_GO],
-			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
-			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT],
-			   wl->rfk_info.proc_time);
-
-		seq_printf(m, ", bt_rfk[req:%d]",
-			   le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
-
-		seq_printf(m, ", AOAC[RF_on:%d/RF_off:%d]",
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
-			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
+			p += scnprintf(p, end - p, " (WL FW Hang!!)");
+
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			       "[mailbox]",
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
+			       le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
+
+		p += scnprintf(p, end - p,
+			       "A2DP_empty:%d(stop:%d/tx:%d/ack:%d/nack:%d)",
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
+			       le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
+
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d/time:%dms]",
+			       "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			       cx->cnt_wl[BTC_WCNT_RFK_GO],
+			       cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			       cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT],
+			       wl->rfk_info.proc_time);
+
+		p += scnprintf(p, end - p, ", bt_rfk[req:%d]",
+			       le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
+
+		p += scnprintf(p, end - p, ", AOAC[RF_on:%d/RF_off:%d]",
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
+			       le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
 	} else {
-		seq_printf(m,
-			   "\n\r %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d (lps=%d/rf_off=%d)",
-			   "[summary]",
-			   pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
-			   pfwinfo->cnt_c2h,
-			   wl->status.map.lps, wl->status.map.rf_off);
+		p += scnprintf(p, end - p,
+			       "\n\r %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d (lps=%d/rf_off=%d)",
+			       "[summary]",
+			       pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
+			       pfwinfo->cnt_c2h,
+			       wl->status.map.lps, wl->status.map.rf_off);
 	}
 
 	for (i = 0; i < BTC_NCNT_NUM; i++)
 		cnt_sum += dm->cnt_notify[i];
 
-	seq_printf(m,
-		   "\n\r %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
-		   "[notify_cnt]",
-		   cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
-		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+	p += scnprintf(p, end - p,
+		       "\n\r %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		       "[notify_cnt]",
+		       cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		       cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
 
-	seq_printf(m,
-		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
-		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
-		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
-		   cnt[BTC_NCNT_WL_STA]);
+	p += scnprintf(p, end - p,
+		       "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
+		       cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		       cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		       cnt[BTC_NCNT_WL_STA]);
 
-	seq_printf(m,
-		   "\n\r %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, switch_chbw=%d, special_pkt=%d, ",
-		   "[notify_cnt]",
-		   cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
-		   cnt[BTC_NCNT_SWITCH_BAND], cnt[BTC_NCNT_SWITCH_CHBW],
-		   cnt[BTC_NCNT_SPECIAL_PACKET]);
+	p += scnprintf(p, end - p,
+		       "\n\r %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, switch_chbw=%d, special_pkt=%d, ",
+		       "[notify_cnt]",
+		       cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
+		       cnt[BTC_NCNT_SWITCH_BAND], cnt[BTC_NCNT_SWITCH_CHBW],
+		       cnt[BTC_NCNT_SPECIAL_PACKET]);
 
-	seq_printf(m, "timer=%d, customerize=%d, hub_msg=%d, chg_fw=%d, send_cc=%d",
-		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CUSTOMERIZE],
-		   rtwdev->btc.hubmsg_cnt, cnt[BTC_NCNT_RESUME_DL_FW],
-		   cnt[BTC_NCNT_COUNTRYCODE]);
+	p += scnprintf(p, end - p,
+		       "timer=%d, customerize=%d, hub_msg=%d, chg_fw=%d, send_cc=%d",
+		       cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CUSTOMERIZE],
+		       rtwdev->btc.hubmsg_cnt, cnt[BTC_NCNT_RESUME_DL_FW],
+		       cnt[BTC_NCNT_COUNTRYCODE]);
+
+	return p - buf;
 }
 
-void rtw89_btc_dump_info(struct rtw89_dev *rtwdev, struct seq_file *m)
+ssize_t rtw89_btc_dump_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
-	struct rtw89_fw_suit *fw_suit = &rtwdev->fw.normal;
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_ver *fwsubver = &btc->fwinfo.fw_subver;
 	const struct rtw89_btc_ver *ver = btc->ver;
-	struct rtw89_btc_cx *cx = &btc->cx;
-	struct rtw89_btc_bt_info *bt = &cx->bt;
-
-	seq_puts(m, "=========================================\n");
-	seq_printf(m, "WL FW / BT FW		%d.%d.%d.%d / NA\n",
-		   fw_suit->major_ver, fw_suit->minor_ver,
-		   fw_suit->sub_ver, fw_suit->sub_idex);
-	seq_printf(m, "manual			%d\n", btc->manual_ctrl);
-
-	seq_puts(m, "=========================================\n");
-
-	seq_printf(m, "\n\r %-15s : raw_data[%02x %02x %02x %02x %02x %02x] (type:%s/cnt:%d/same:%d)",
-		   "[bt_info]",
-		   bt->raw_info[2], bt->raw_info[3],
-		   bt->raw_info[4], bt->raw_info[5],
-		   bt->raw_info[6], bt->raw_info[7],
-		   bt->raw_info[0] == BTC_BTINFO_AUTO ? "auto" : "reply",
-		   cx->cnt_bt[BTC_BCNT_INFOUPDATE],
-		   cx->cnt_bt[BTC_BCNT_INFOSAME]);
+	struct rtw89_btc_dm *dm = &btc->dm;
+	char *p = buf, *end = buf + bufsz;
 
-	seq_puts(m, "\n=========================================\n");
+	dm->cnt_notify[BTC_NCNT_SHOW_COEX_INFO]++;
 
-	_show_cx_info(rtwdev, m);
-	_show_wl_info(rtwdev, m);
-	_show_bt_info(rtwdev, m);
-	_show_dm_info(rtwdev, m);
-	_show_fw_dm_msg(rtwdev, m);
+	p += scnprintf(p, end - p,
+		       "\n\n\n** Page:%3d/RunCNT:%3d **",
+		       dm->cnt_notify[BTC_NCNT_SHOW_COEX_INFO],
+		       dm->cnt_dm[BTC_DCNT_RUN]);
+	p += scnprintf(p, end - p,
+		       "\n========== [BTC FEATURE SUB VER] ==========");
+	p += scnprintf(p, end - p,
+		       "\n %-15s : fcxbtcrpt[%d/%d], fcxtdma[%d/%d], fcxslots[%d/%d], fcxcysta[%d/%d]",
+		       "[FW/DRV]", fwsubver->fcxbtcrpt, ver->fcxbtcrpt,
+		       fwsubver->fcxtdma, ver->fcxtdma, fwsubver->fcxslots,
+		       ver->fcxslots, fwsubver->fcxcysta, ver->fcxcysta);
+	p += scnprintf(p, end - p,
+		       "\n %-15s : fcxstep[%d/%d], fcxnullsta[%d/%d], fcxmreg[%d/%d], fcxgpiodbg[%d/%d]",
+		       "[FW/DRV]", fwsubver->fcxstep, ver->fcxstep,
+		       fwsubver->fcxnullsta, ver->fcxnullsta, fwsubver->fcxmreg,
+		       ver->fcxmreg, fwsubver->fcxgpiodbg, ver->fcxgpiodbg);
+	p += scnprintf(p, end - p,
+		       "\n %-15s : fcxbtver[%d/%d], fcxbtscan[%d/%d], fcxbtafh[%d/%d], fcxbtdevinfo[%d/%d]",
+		       "[FW/DRV]", fwsubver->fcxbtver, ver->fcxbtver,
+		       fwsubver->fcxbtscan, ver->fcxbtscan, fwsubver->fcxbtafh,
+		       ver->fcxbtafh, fwsubver->fcxbtdevinfo, ver->fcxbtdevinfo);
+	p += scnprintf(p, end - p,
+		       "\n %-15s : fcxosi[%d/%d], fcxmlo[%d/%d],",
+		       "[FW/DRV]", fwsubver->fcxosi, ver->fcxosi,
+		       fwsubver->fcxmlo, ver->fcxmlo);
+
+	p += _show_cx_info(rtwdev, p, end - p);
+	p += _show_wl_info(rtwdev, p, end - p);
+	p += _show_bt_info(rtwdev, p, end - p);
+	p += _show_dm_info(rtwdev, p, end - p);
+	p += _show_fw_dm_msg(rtwdev, p, end - p);
 
 	if (ver->fcxmreg == 1)
-		_show_mreg_v1(rtwdev, m);
+		p += _show_mreg_v1(rtwdev, p, end - p);
 	else if (ver->fcxmreg == 2)
-		_show_mreg_v2(rtwdev, m);
+		p += _show_mreg_v2(rtwdev, p, end - p);
 	else if (ver->fcxmreg == 7)
-		_show_mreg_v7(rtwdev, m);
+		p += _show_mreg_v7(rtwdev, p, end - p);
 
-	_show_gpio_dbg(rtwdev, m);
+	p += _show_gpio_dbg(rtwdev, p, end - p);
 
 	if (ver->fcxbtcrpt == 1)
-		_show_summary_v1(rtwdev, m);
+		p += _show_summary_v1(rtwdev, p, end - p);
 	else if (ver->fcxbtcrpt == 4)
-		_show_summary_v4(rtwdev, m);
+		p += _show_summary_v4(rtwdev, p, end - p);
 	else if (ver->fcxbtcrpt == 5)
-		_show_summary_v5(rtwdev, m);
+		p += _show_summary_v5(rtwdev, p, end - p);
 	else if (ver->fcxbtcrpt == 105)
-		_show_summary_v105(rtwdev, m);
+		p += _show_summary_v105(rtwdev, p, end - p);
 	else if (ver->fcxbtcrpt == 7)
-		_show_summary_v7(rtwdev, m);
+		p += _show_summary_v7(rtwdev, p, end - p);
 	else if (ver->fcxbtcrpt == 8)
-		_show_summary_v8(rtwdev, m);
+		p += _show_summary_v8(rtwdev, p, end - p);
+
+	return p - buf;
 }
 
 void rtw89_coex_recognize_ver(struct rtw89_dev *rtwdev)
@@ -11037,3 +11880,24 @@ out:
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC] use version def[%d] = 0x%08x\n",
 		    (int)(btc->ver - rtw89_btc_ver_defs), btc->ver->fw_ver_code);
 }
+
+void rtw89_btc_ntfy_preserve_bt_time(struct rtw89_dev *rtwdev, u32 ms)
+{
+	struct rtw89_btc_bt_link_info *bt_linfo = &rtwdev->btc.cx.bt.link_info;
+	struct rtw89_btc_bt_a2dp_desc a2dp = bt_linfo->a2dp_desc;
+
+	if (test_bit(RTW89_FLAG_SER_HANDLING, rtwdev->flags))
+		return;
+
+	if (!a2dp.exist)
+		return;
+
+	fsleep(ms * 1000);
+}
+EXPORT_SYMBOL(rtw89_btc_ntfy_preserve_bt_time);
+
+void rtw89_btc_ntfy_conn_rfk(struct rtw89_dev *rtwdev, bool state)
+{
+	rtwdev->btc.cx.wl.rfk_info.con_rfk = state;
+}
+EXPORT_SYMBOL(rtw89_btc_ntfy_conn_rfk);
diff --git a/sys/contrib/dev/rtw89/coex.h b/sys/contrib/dev/rtw89/coex.h
index dbdb56e063ef..ea2c1e5d70f5 100644
--- a/sys/contrib/dev/rtw89/coex.h
+++ b/sys/contrib/dev/rtw89/coex.h
@@ -224,6 +224,13 @@ enum btc_wl_mode {
 	BTC_WL_MODE_NUM,
 };
 
+enum btc_mlo_rf_combin {
+	BTC_MLO_RF_2_PLUS_0 = 0,
+	BTC_MLO_RF_0_PLUS_2 = 1,
+	BTC_MLO_RF_1_PLUS_1 = 2,
+	BTC_MLO_RF_2_PLUS_2 = 3,
+};
+
 enum btc_wl_gpio_debug {
 	BTC_DBG_GNT_BT = 0,
 	BTC_DBG_GNT_WL = 1,
@@ -267,10 +274,10 @@ void rtw89_btc_ntfy_scan_finish(struct rtw89_dev *rtwdev, u8 phy_idx);
 void rtw89_btc_ntfy_switch_band(struct rtw89_dev *rtwdev, u8 phy_idx, u8 band);
 void rtw89_btc_ntfy_specific_packet(struct rtw89_dev *rtwdev,
 				    enum btc_pkt_type pkt_type);
-void rtw89_btc_ntfy_eapol_packet_work(struct work_struct *work);
-void rtw89_btc_ntfy_arp_packet_work(struct work_struct *work);
-void rtw89_btc_ntfy_dhcp_packet_work(struct work_struct *work);
-void rtw89_btc_ntfy_icmp_packet_work(struct work_struct *work);
+void rtw89_btc_ntfy_eapol_packet_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_btc_ntfy_arp_packet_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_btc_ntfy_dhcp_packet_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_btc_ntfy_icmp_packet_work(struct wiphy *wiphy, struct wiphy_work *work);
 void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev,
 			      struct rtw89_vif_link *rtwvif_link,
 			      struct rtw89_sta_link *rtwsta_link,
@@ -282,14 +289,16 @@ void rtw89_btc_ntfy_wl_rfk(struct rtw89_dev *rtwdev, u8 phy_map,
 void rtw89_btc_ntfy_wl_sta(struct rtw89_dev *rtwdev);
 void rtw89_btc_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 			  u32 len, u8 class, u8 func);
-void rtw89_btc_dump_info(struct rtw89_dev *rtwdev, struct seq_file *m);
-void rtw89_coex_act1_work(struct work_struct *work);
-void rtw89_coex_bt_devinfo_work(struct work_struct *work);
-void rtw89_coex_rfk_chk_work(struct work_struct *work);
+ssize_t rtw89_btc_dump_info(struct rtw89_dev *rtwdev, char *buf, size_t bufsz);
+void rtw89_coex_act1_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_coex_bt_devinfo_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_coex_rfk_chk_work(struct wiphy *wiphy, struct wiphy_work *work);
 void rtw89_coex_power_on(struct rtw89_dev *rtwdev);
 void rtw89_btc_set_policy(struct rtw89_dev *rtwdev, u16 policy_type);
 void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type);
 void rtw89_coex_recognize_ver(struct rtw89_dev *rtwdev);
+void rtw89_btc_ntfy_preserve_bt_time(struct rtw89_dev *rtwdev, u32 ms);
+void rtw89_btc_ntfy_conn_rfk(struct rtw89_dev *rtwdev, bool state);
 
 static inline u8 rtw89_btc_phymap(struct rtw89_dev *rtwdev,
 				  enum rtw89_phy_idx phy_idx,
diff --git a/sys/contrib/dev/rtw89/core.c b/sys/contrib/dev/rtw89/core.c
index e002af84f1d1..1220378d08cf 100644
--- a/sys/contrib/dev/rtw89/core.c
+++ b/sys/contrib/dev/rtw89/core.c
@@ -223,6 +223,24 @@ static const struct ieee80211_iface_combination rtw89_iface_combs[] = {
 	},
 };
 
+static const u8 rtw89_ext_capa_sta[] = {
+	[0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
+	[2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
+	[7] = WLAN_EXT_CAPA8_OPMODE_NOTIF,
+};
+
+static const struct wiphy_iftype_ext_capab rtw89_iftypes_ext_capa[] = {
+	{
+		.iftype = NL80211_IFTYPE_STATION,
+		.extended_capabilities = rtw89_ext_capa_sta,
+		.extended_capabilities_mask = rtw89_ext_capa_sta,
+		.extended_capabilities_len = sizeof(rtw89_ext_capa_sta),
+		/* relevant only if EHT is supported */
+		.eml_capabilities = 0,
+		.mld_capa_and_ops = 0,
+	},
+};
+
 #define RTW89_6GHZ_SPAN_HEAD 6145
 #define RTW89_6GHZ_SPAN_IDX(center_freq) \
 	((((int)(center_freq) - RTW89_6GHZ_SPAN_HEAD) / 5) / 2)
@@ -231,6 +249,8 @@ static const struct ieee80211_iface_combination rtw89_iface_combs[] = {
 	[RTW89_6GHZ_SPAN_IDX(center_freq)] = { \
 		.sar_subband_low = RTW89_SAR_6GHZ_ ## subband_l, \
 		.sar_subband_high = RTW89_SAR_6GHZ_ ## subband_h, \
+		.acpi_sar_subband_low = RTW89_ACPI_SAR_6GHZ_ ## subband_l, \
+		.acpi_sar_subband_high = RTW89_ACPI_SAR_6GHZ_ ## subband_h, \
 		.ant_gain_subband_low = RTW89_ANT_GAIN_6GHZ_ ## subband_l, \
 		.ant_gain_subband_high = RTW89_ANT_GAIN_6GHZ_ ## subband_h, \
 	}
@@ -319,15 +339,25 @@ static const struct ieee80211_supported_band rtw89_sband_6ghz = {
 	.n_bitrates	= ARRAY_SIZE(rtw89_bitrates) - 4,
 };
 
+static void __rtw89_traffic_stats_accu(struct rtw89_traffic_stats *stats,
+				       struct sk_buff *skb, bool tx)
+{
+	if (tx) {
+		stats->tx_cnt++;
+		stats->tx_unicast += skb->len;
+	} else {
+		stats->rx_cnt++;
+		stats->rx_unicast += skb->len;
+	}
+}
+
 static void rtw89_traffic_stats_accu(struct rtw89_dev *rtwdev,
-				     struct rtw89_traffic_stats *stats,
-				     struct sk_buff *skb, bool tx)
+				     struct rtw89_vif *rtwvif,
+				     struct sk_buff *skb,
+				     bool accu_dev, bool tx)
 {
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 
-	if (tx && ieee80211_is_assoc_req(hdr->frame_control))
-		rtw89_wow_parse_akm(rtwdev, skb);
-
 	if (!ieee80211_is_data(hdr->frame_control))
 		return;
 
@@ -335,12 +365,12 @@ static void rtw89_traffic_stats_accu(struct rtw89_dev *rtwdev,
 	    is_multicast_ether_addr(hdr->addr1))
 		return;
 
-	if (tx) {
-		stats->tx_cnt++;
-		stats->tx_unicast += skb->len;
-	} else {
-		stats->rx_cnt++;
-		stats->rx_unicast += skb->len;
+	if (accu_dev)
+		__rtw89_traffic_stats_accu(&rtwdev->stats, skb, tx);
+
+	if (rtwvif) {
+		__rtw89_traffic_stats_accu(&rtwvif->stats, skb, tx);
+		__rtw89_traffic_stats_accu(&rtwvif->stats_ps, skb, tx);
 	}
 }
 
@@ -659,9 +689,17 @@ out:
 static u8 rtw89_core_tx_get_mac_id(struct rtw89_dev *rtwdev,
 				   struct rtw89_core_tx_request *tx_req)
 {
+	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
 	struct rtw89_vif_link *rtwvif_link = tx_req->rtwvif_link;
 	struct rtw89_sta_link *rtwsta_link = tx_req->rtwsta_link;
 
+	if (desc_info->mlo && !desc_info->sw_mld) {
+		if (rtwsta_link)
+			return rtw89_sta_get_main_macid(rtwsta_link->rtwsta);
+		else
+			return rtw89_vif_get_main_macid(rtwvif_link->rtwvif);
+	}
+
 	if (!rtwsta_link)
 		return rtwvif_link->mac_id;
 
@@ -691,7 +729,7 @@ rtw89_core_tx_update_mgmt_info(struct rtw89_dev *rtwdev,
 	struct sk_buff *skb = tx_req->skb;
 	u8 qsel, ch_dma;
 
-	qsel = desc_info->hiq ? RTW89_TX_QSEL_B0_HI : RTW89_TX_QSEL_B0_MGMT;
+	qsel = rtw89_core_get_qsel_mgmt(rtwdev, tx_req);
 	ch_dma = rtw89_core_get_ch_dma(rtwdev, qsel);
 
 	desc_info->qsel = qsel;
@@ -945,16 +983,17 @@ static enum btc_pkt_type
 rtw89_core_tx_btc_spec_pkt_notify(struct rtw89_dev *rtwdev,
 				  struct rtw89_core_tx_request *tx_req)
 {
+	struct wiphy *wiphy = rtwdev->hw->wiphy;
 	struct sk_buff *skb = tx_req->skb;
 	struct udphdr *udphdr;
 
 	if (IEEE80211_SKB_CB(skb)->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO) {
-		ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.eapol_notify_work);
+		wiphy_work_queue(wiphy, &rtwdev->btc.eapol_notify_work);
 		return PACKET_EAPOL;
 	}
 
 	if (skb->protocol == htons(ETH_P_ARP)) {
-		ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.arp_notify_work);
+		wiphy_work_queue(wiphy, &rtwdev->btc.arp_notify_work);
 		return PACKET_ARP;
 	}
 
@@ -964,14 +1003,14 @@ rtw89_core_tx_btc_spec_pkt_notify(struct rtw89_dev *rtwdev,
 		if (((udphdr->source == htons(67) && udphdr->dest == htons(68)) ||
 		     (udphdr->source == htons(68) && udphdr->dest == htons(67))) &&
 		    skb->len > 282) {
-			ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.dhcp_notify_work);
+			wiphy_work_queue(wiphy, &rtwdev->btc.dhcp_notify_work);
 			return PACKET_DHCP;
 		}
 	}
 
 	if (skb->protocol == htons(ETH_P_IP) &&
 	    ip_hdr(skb)->protocol == IPPROTO_ICMP) {
-		ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.icmp_notify_work);
+		wiphy_work_queue(wiphy, &rtwdev->btc.icmp_notify_work);
 		return PACKET_ICMP;
 	}
 
@@ -987,13 +1026,25 @@ rtw89_core_tx_wake(struct rtw89_dev *rtwdev,
 	if (!RTW89_CHK_FW_FEATURE(TX_WAKE, &rtwdev->fw))
 		return;
 
-	if (!test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags))
-		return;
+	switch (chip->chip_id) {
+	case RTL8852BT:
+		if (test_bit(RTW89_FLAG_LEISURE_PS, rtwdev->flags))
+			goto notify;
+		break;
+	case RTL8852C:
+		if (test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags))
+			goto notify;
+		break;
+	default:
+		if (test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags) &&
+		    tx_req->tx_type == RTW89_CORE_TX_TYPE_MGMT)
+			goto notify;
+		break;
+	}
 
-	if (chip->chip_id != RTL8852C &&
-	    tx_req->tx_type != RTW89_CORE_TX_TYPE_MGMT)
-		return;
+	return;
 
+notify:
 	rtw89_mac_notify_wake(rtwdev);
 }
 
@@ -1135,42 +1186,26 @@ int rtw89_h2c_tx(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
-int rtw89_core_tx_write(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
-			struct ieee80211_sta *sta, struct sk_buff *skb, int *qsel)
+static int rtw89_core_tx_write_link(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif_link *rtwvif_link,
+				    struct rtw89_sta_link *rtwsta_link,
+				    struct sk_buff *skb, int *qsel, bool sw_mld)
 {
-	struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta);
-	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
-	struct rtw89_core_tx_request tx_req = {0};
-	struct rtw89_sta_link *rtwsta_link = NULL;
-	struct rtw89_vif_link *rtwvif_link;
+	struct ieee80211_sta *sta = rtwsta_link_to_sta_safe(rtwsta_link);
+	struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link);
+	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
+	struct rtw89_core_tx_request tx_req = {};
 	int ret;
 
-	/* By default, driver writes tx via the link on HW-0. And then,
-	 * according to links' status, HW can change tx to another link.
-	 */
-
-	if (rtwsta) {
-		rtwsta_link = rtw89_sta_get_link_inst(rtwsta, 0);
-		if (unlikely(!rtwsta_link)) {
-			rtw89_err(rtwdev, "tx: find no sta link on HW-0\n");
-			return -ENOLINK;
-		}
-	}
-
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
-	if (unlikely(!rtwvif_link)) {
-		rtw89_err(rtwdev, "tx: find no vif link on HW-0\n");
-		return -ENOLINK;
-	}
-
 	tx_req.skb = skb;
 	tx_req.vif = vif;
 	tx_req.sta = sta;
 	tx_req.rtwvif_link = rtwvif_link;
 	tx_req.rtwsta_link = rtwsta_link;
+	tx_req.desc_info.sw_mld = sw_mld;
 
-	rtw89_traffic_stats_accu(rtwdev, &rtwdev->stats, skb, true);
-	rtw89_traffic_stats_accu(rtwdev, &rtwvif->stats, skb, true);
+	rtw89_traffic_stats_accu(rtwdev, rtwvif, skb, true, true);
+	rtw89_wow_parse_akm(rtwdev, skb);
 	rtw89_core_tx_update_desc_info(rtwdev, &tx_req);
 	rtw89_core_tx_wake(rtwdev, &tx_req);
 
@@ -1186,6 +1221,33 @@ int rtw89_core_tx_write(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 	return 0;
 }
 
+int rtw89_core_tx_write(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
+			struct ieee80211_sta *sta, struct sk_buff *skb, int *qsel)
+{
+	struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta);
+	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
+	struct rtw89_sta_link *rtwsta_link = NULL;
+	struct rtw89_vif_link *rtwvif_link;
+
+	if (rtwsta) {
+		rtwsta_link = rtw89_get_designated_link(rtwsta);
+		if (unlikely(!rtwsta_link)) {
+			rtw89_err(rtwdev, "tx: find no sta designated link\n");
+			return -ENOLINK;
+		}
+
+		rtwvif_link = rtwsta_link->rtwvif_link;
+	} else {
+		rtwvif_link = rtw89_get_designated_link(rtwvif);
+		if (unlikely(!rtwvif_link)) {
+			rtw89_err(rtwdev, "tx: find no vif designated link\n");
+			return -ENOLINK;
+		}
+	}
+
+	return rtw89_core_tx_write_link(rtwdev, rtwvif_link, rtwsta_link, skb, qsel, false);
+}
+
 static __le32 rtw89_build_txwd_body0(struct rtw89_tx_desc_info *desc_info)
 {
 	u32 dword = FIELD_PREP(RTW89_TXWD_BODY0_WP_OFFSET, desc_info->wp_offset) |
@@ -1413,7 +1475,9 @@ static __le32 rtw89_build_txwd_body2_v2(struct rtw89_tx_desc_info *desc_info)
 
 static __le32 rtw89_build_txwd_body3_v2(struct rtw89_tx_desc_info *desc_info)
 {
-	u32 dword = FIELD_PREP(BE_TXD_BODY3_WIFI_SEQ, desc_info->seq);
+	u32 dword = FIELD_PREP(BE_TXD_BODY3_WIFI_SEQ, desc_info->seq) |
+		    FIELD_PREP(BE_TXD_BODY3_MLO_FLAG, desc_info->mlo) |
+		    FIELD_PREP(BE_TXD_BODY3_IS_MLD_SW_EN, desc_info->sw_mld);
 
 	return cpu_to_le32(dword);
 }
@@ -1666,10 +1730,7 @@ static void rtw89_core_rx_process_phy_ppdu_iter(void *data,
 	u8 evm_pos = 0;
 	int i;
 
-	/* FIXME: For single link, taking link on HW-0 here is okay. But, when
-	 * enabling multiple active links, we should determine the right link.
-	 */
-	rtwsta_link = rtw89_sta_get_link_inst(rtwsta, 0);
+	rtwsta_link = rtw89_sta_get_link_inst(rtwsta, phy_ppdu->phy_idx);
 	if (unlikely(!rtwsta_link))
 		return;
 
@@ -1716,7 +1777,7 @@ static u16 rtw89_core_get_phy_status_ie_len(struct rtw89_dev *rtwdev,
 		},
 		[RTW89_CHIP_BE] = {
 			32, 40, 24, 24, 8, 8, 8, 8, VAR_LEN, 8, VAR_LEN, 176, VAR_LEN,
-			VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, 16, 24, VAR_LEN,
+			VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, 88, 56, VAR_LEN,
 			VAR_LEN, VAR_LEN, 0, 24, 24, 24, 24, 32, 32, 32, 32
 		},
 	};
@@ -1920,6 +1981,8 @@ static int rtw89_core_rx_parse_phy_sts(struct rtw89_dev *rtwdev,
 
 #if defined(__linux__)
 	pos = phy_ppdu->buf + PHY_STS_HDR_LEN;
+	if (phy_ppdu->hdr_2_en)
+		pos += PHY_STS_HDR_LEN;
 	end = phy_ppdu->buf + phy_ppdu->len;
 #elif defined(__FreeBSD__)
 	pos = (u8 *)phy_ppdu->buf + PHY_STS_HDR_LEN;
@@ -2106,10 +2169,21 @@ static void rtw89_stats_trigger_frame(struct rtw89_dev *rtwdev,
 			break;
 
 		if (aid == vif->cfg.aid) {
-			enum nl80211_he_ru_alloc rua = rtw89_he_rua_to_ru_alloc(tf_rua >> 1);
+			enum nl80211_he_ru_alloc rua;
 
 			rtwvif->stats.rx_tf_acc++;
 			rtwdev->stats.rx_tf_acc++;
+
+			/* The following only required for HE trigger frame, but we
+			 * cannot use UL HE-SIG-A2 reserved subfield to identify it
+			 * since some 11ax APs will fill it with all 0s, which will
+			 * be misunderstood as EHT trigger frame.
+			 */
+			if (bss_conf->eht_support)
+				break;
+
+			rua = rtw89_he_rua_to_ru_alloc(tf_rua >> 1);
+
 			if (tf_bw == IEEE80211_TRIGGER_ULBW_160_80P80MHZ &&
 			    rua <= NL80211_RATE_INFO_HE_RU_ALLOC_106)
 				rtwvif_link->pwr_diff_en = true;
@@ -2120,17 +2194,17 @@ static void rtw89_stats_trigger_frame(struct rtw89_dev *rtwdev,
 	}
 }
 
-static void rtw89_cancel_6ghz_probe_work(struct work_struct *work)
+static void rtw89_cancel_6ghz_probe_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						cancel_6ghz_probe_work);
 	struct list_head *pkt_list = rtwdev->scan_info.pkt_list;
 	struct rtw89_pktofld_info *info;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
 
 	if (!rtwdev->scanning)
-		goto out;
+		return;
 
 	list_for_each_entry(info, &pkt_list[NL80211_BAND_6GHZ], list) {
 		if (!info->cancel || !test_bit(info->id, rtwdev->pkt_offload))
@@ -2143,9 +2217,6 @@ static void rtw89_cancel_6ghz_probe_work(struct work_struct *work)
 		 * since if during scanning, pkt_list is accessed in bottom half.
 		 */
 	}
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static void rtw89_core_cancel_6ghz_probe_tx(struct rtw89_dev *rtwdev,
@@ -2161,6 +2232,11 @@ static void rtw89_core_cancel_6ghz_probe_tx(struct rtw89_dev *rtwdev,
 	if (rx_status->band != NL80211_BAND_6GHZ)
 		return;
 
+	if (unlikely(!(rtwdev->chip->support_bands & BIT(NL80211_BAND_6GHZ)))) {
+		rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "invalid rx on unsupported 6 GHz\n");
+		return;
+	}
+
 	ssid_ie = cfg80211_find_ie(WLAN_EID_SSID, ies, skb->len);
 
 	list_for_each_entry(info, &pkt_list[NL80211_BAND_6GHZ], list) {
@@ -2180,7 +2256,7 @@ static void rtw89_core_cancel_6ghz_probe_tx(struct rtw89_dev *rtwdev,
 	}
 
 	if (queue_work)
-		ieee80211_queue_work(rtwdev->hw, &rtwdev->cancel_6ghz_probe_work);
+		wiphy_work_queue(rtwdev->hw->wiphy, &rtwdev->cancel_6ghz_probe_work);
 }
 
 static void rtw89_vif_sync_bcn_tsf(struct rtw89_vif_link *rtwvif_link,
@@ -2203,8 +2279,10 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 	struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.cur_pkt_stat;
 	struct rtw89_rx_desc_info *desc_info = iter_data->desc_info;
 	struct sk_buff *skb = iter_data->skb;
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct rtw89_rx_phy_ppdu *phy_ppdu = iter_data->phy_ppdu;
+	bool is_mld = ieee80211_vif_is_mld(vif);
 	struct ieee80211_bss_conf *bss_conf;
 	struct rtw89_vif_link *rtwvif_link;
 	const u8 *bssid = iter_data->bssid;
@@ -2216,10 +2294,7 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 
 	rcu_read_lock();
 
-	/* FIXME: For single link, taking link on HW-0 here is okay. But, when
-	 * enabling multiple active links, we should determine the right link.
-	 */
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
+	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, desc_info->bb_sel);
 	if (unlikely(!rtwvif_link))
 		goto out;
 
@@ -2235,6 +2310,11 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 	if (!ether_addr_equal(bss_conf->bssid, bssid))
 		goto out;
 
+	if (is_mld) {
+		rx_status->link_valid = true;
+		rx_status->link_id = rtwvif_link->link_id;
+	}
+
 	if (ieee80211_is_beacon(hdr->frame_control)) {
 		if (vif->type == NL80211_IFTYPE_STATION &&
 		    !test_bit(RTW89_FLAG_WOWLAN, rtwdev->flags)) {
@@ -2243,8 +2323,11 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 		}
 		pkt_stat->beacon_nr++;
 
-		if (phy_ppdu)
+		if (phy_ppdu) {
 			ewma_rssi_add(&rtwdev->phystat.bcn_rssi, phy_ppdu->rssi_avg);
+			if (!test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags))
+				rtwvif_link->bcn_bw_idx = phy_ppdu->bw_idx;
+		}
 
 		pkt_stat->beacon_rate = desc_info->data_rate;
 	}
@@ -2255,7 +2338,7 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 	if (desc_info->data_rate < RTW89_HW_RATE_NR)
 		pkt_stat->rx_rate_cnt[desc_info->data_rate]++;
 
-	rtw89_traffic_stats_accu(rtwdev, &rtwvif->stats, skb, false);
+	rtw89_traffic_stats_accu(rtwdev, rtwvif, skb, false, false);
 
 out:
 	rcu_read_unlock();
@@ -2268,7 +2351,7 @@ static void rtw89_core_rx_stats(struct rtw89_dev *rtwdev,
 {
 	struct rtw89_vif_rx_stats_iter_data iter_data;
 
-	rtw89_traffic_stats_accu(rtwdev, &rtwdev->stats, skb, false);
+	rtw89_traffic_stats_accu(rtwdev, NULL, skb, true, false);
 
 	iter_data.rtwdev = rtwdev;
 	iter_data.phy_ppdu = phy_ppdu;
@@ -2434,6 +2517,84 @@ static void rtw89_core_validate_rx_signal(struct ieee80211_rx_status *rx_status)
 		rx_status->flag |= RX_FLAG_NO_SIGNAL_VAL;
 }
 
+static void rtw89_core_update_rx_freq_from_ie(struct rtw89_dev *rtwdev,
+					      struct sk_buff *skb,
+					      struct ieee80211_rx_status *rx_status)
+{
+	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
+	size_t hdr_len, ielen;
+	u8 *variable;
+	int chan;
+
+	if (!rtwdev->chip->rx_freq_frome_ie)
+		return;
+
+	if (!rtwdev->scanning)
+		return;
+
+	if (ieee80211_is_beacon(mgmt->frame_control)) {
+		variable = mgmt->u.beacon.variable;
+		hdr_len = offsetof(struct ieee80211_mgmt,
+				   u.beacon.variable);
+	} else if (ieee80211_is_probe_resp(mgmt->frame_control)) {
+		variable = mgmt->u.probe_resp.variable;
+		hdr_len = offsetof(struct ieee80211_mgmt,
+				   u.probe_resp.variable);
+	} else {
+		return;
+	}
+
+	if (skb->len > hdr_len)
+		ielen = skb->len - hdr_len;
+	else
+		return;
+
+	/* The parsing code for both 2GHz and 5GHz bands is the same in this
+	 * function.
+	 */
+	chan = cfg80211_get_ies_channel_number(variable, ielen, NL80211_BAND_2GHZ);
+	if (chan == -1)
+		return;
+
+	rx_status->band = chan > 14 ? RTW89_BAND_5G : RTW89_BAND_2G;
+	rx_status->freq = ieee80211_channel_to_frequency(chan, rx_status->band);
+}
+
+static void rtw89_core_correct_mcc_chan(struct rtw89_dev *rtwdev,
+					struct rtw89_rx_desc_info *desc_info,
+					struct ieee80211_rx_status *rx_status,
+					struct rtw89_rx_phy_ppdu *phy_ppdu)
+{
+	enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
+	struct rtw89_vif_link *rtwvif_link;
+	struct rtw89_sta_link *rtwsta_link;
+	const struct rtw89_chan *chan;
+	u8 mac_id = desc_info->mac_id;
+	enum rtw89_entity_mode mode;
+	enum nl80211_band band;
+
+	mode = rtw89_get_entity_mode(rtwdev);
+	if (likely(mode != RTW89_ENTITY_MODE_MCC))
+		return;
+
+	if (chip_gen == RTW89_CHIP_BE && phy_ppdu)
+		mac_id = phy_ppdu->mac_id;
+
+	rcu_read_lock();
+
+	rtwsta_link = rtw89_assoc_link_rcu_dereference(rtwdev, mac_id);
+	if (!rtwsta_link)
+		goto out;
+
+	rtwvif_link = rtwsta_link->rtwvif_link;
+	chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx);
+	band = rtw89_hw_to_nl80211_band(chan->band_type);
+	rx_status->freq = ieee80211_channel_to_frequency(chan->primary_channel, band);
+
+out:
+	rcu_read_unlock();
+}
+
 static void rtw89_core_rx_to_mac80211(struct rtw89_dev *rtwdev,
 				      struct rtw89_rx_phy_ppdu *phy_ppdu,
 				      struct rtw89_rx_desc_info *desc_info,
@@ -2451,6 +2612,8 @@ static void rtw89_core_rx_to_mac80211(struct rtw89_dev *rtwdev,
 	rtw89_core_update_rx_status_by_ppdu(rtwdev, rx_status, phy_ppdu);
 	rtw89_core_update_radiotap(rtwdev, skb_ppdu, rx_status);
 	rtw89_core_validate_rx_signal(rx_status);
+	rtw89_core_update_rx_freq_from_ie(rtwdev, skb_ppdu, rx_status);
+	rtw89_core_correct_mcc_chan(rtwdev, desc_info, rx_status, phy_ppdu);
 
 	/* In low power mode, it does RX in thread context. */
 	local_bh_disable();
@@ -2490,7 +2653,8 @@ static void rtw89_core_rx_process_ppdu_sts(struct rtw89_dev *rtwdev,
 					     .len = skb->len,
 					     .to_self = desc_info->addr1_match,
 					     .rate = desc_info->data_rate,
-					     .mac_id = desc_info->mac_id};
+					     .mac_id = desc_info->mac_id,
+					     .phy_idx = desc_info->bb_sel};
 	int ret;
 
 	if (desc_info->mac_info_valid) {
@@ -2601,6 +2765,7 @@ void rtw89_core_query_rxdesc_v2(struct rtw89_dev *rtwdev,
 	desc_info->shift = le32_get_bits(rxd_s->dword0, BE_RXD_SHIFT_MASK);
 	desc_info->long_rxdesc = le32_get_bits(rxd_s->dword0, BE_RXD_LONG_RXD);
 	desc_info->pkt_type = le32_get_bits(rxd_s->dword0, BE_RXD_RPKT_TYPE_MASK);
+	desc_info->bb_sel = le32_get_bits(rxd_s->dword0, BE_RXD_BB_SEL);
 	if (desc_info->pkt_type == RTW89_CORE_RX_TYPE_PPDU_STAT)
 		desc_info->mac_info_valid = true;
 
@@ -2673,10 +2838,7 @@ void rtw89_core_stats_sta_rx_status_iter(void *data, struct ieee80211_sta *sta)
 	struct rtw89_sta_link *rtwsta_link;
 	u8 mac_id = iter_data->mac_id;
 
-	/* FIXME: For single link, taking link on HW-0 here is okay. But, when
-	 * enabling multiple active links, we should determine the right link.
-	 */
-	rtwsta_link = rtw89_sta_get_link_inst(rtwsta, 0);
+	rtwsta_link = rtw89_sta_get_link_inst(rtwsta, desc_info->bb_sel);
 	if (unlikely(!rtwsta_link))
 		return;
 
@@ -2709,9 +2871,11 @@ static void rtw89_core_stats_sta_rx_status(struct rtw89_dev *rtwdev,
 }
 
 static void rtw89_core_update_rx_status(struct rtw89_dev *rtwdev,
+					struct sk_buff *skb,
 					struct rtw89_rx_desc_info *desc_info,
 					struct ieee80211_rx_status *rx_status)
 {
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	const struct cfg80211_chan_def *chandef =
 		rtw89_chandef_get(rtwdev, RTW89_CHANCTX_0);
 	u16 data_rate;
@@ -2723,6 +2887,10 @@ static void rtw89_core_update_rx_status(struct rtw89_dev *rtwdev,
 	rx_status->freq = chandef->chan->center_freq;
 	rx_status->band = chandef->chan->band;
 
+	if (ieee80211_is_beacon(hdr->frame_control) ||
+	    ieee80211_is_probe_resp(hdr->frame_control))
+		rx_status->boottime_ns = ktime_get_boottime_ns();
+
 	if (rtwdev->scanning &&
 	    RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw)) {
 		const struct rtw89_chan *cur = rtw89_scan_chan_get(rtwdev);
@@ -2791,6 +2959,9 @@ static enum rtw89_ps_mode rtw89_update_ps_mode(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 
+	if (rtwdev->hci.type != RTW89_HCI_TYPE_PCIE)
+		return RTW89_PS_MODE_NONE;
+
 	if (rtw89_disable_ps_mode || !chip->ps_mode_supported ||
 	    RTW89_CHK_FW_FEATURE(NO_DEEP_PS, &rtwdev->fw))
 		return RTW89_PS_MODE_NONE;
@@ -2879,7 +3050,7 @@ void rtw89_core_rx(struct rtw89_dev *rtwdev,
 
 	rx_status = IEEE80211_SKB_RXCB(skb);
 	memset(rx_status, 0, sizeof(*rx_status));
-	rtw89_core_update_rx_status(rtwdev, desc_info, rx_status);
+	rtw89_core_update_rx_status(rtwdev, skb, desc_info, rx_status);
 	rtw89_core_rx_pkt_hdl(rtwdev, skb, desc_info);
 	if (desc_info->long_rxdesc &&
 	    BIT(desc_info->frame_type) & PPDU_FILTER_BITMAP)
@@ -3125,9 +3296,9 @@ static bool rtw89_core_txq_agg_wait(struct rtw89_dev *rtwdev,
 	if (!rtwsta)
 		return false;
 
-	rtwsta_link = rtw89_sta_get_link_inst(rtwsta, 0);
+	rtwsta_link = rtw89_get_designated_link(rtwsta);
 	if (unlikely(!rtwsta_link)) {
-		rtw89_err(rtwdev, "agg wait: find no link on HW-0\n");
+		rtw89_err(rtwdev, "agg wait: find no designated link\n");
 		return false;
 	}
 
@@ -3196,13 +3367,14 @@ static void rtw89_core_txq_schedule(struct rtw89_dev *rtwdev, u8 ac, bool *reinv
 	ieee80211_txq_schedule_end(hw, ac);
 }
 
-static void rtw89_ips_work(struct work_struct *work)
+static void rtw89_ips_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						ips_work);
-	mutex_lock(&rtwdev->mutex);
+
+	lockdep_assert_wiphy(wiphy);
+
 	rtw89_enter_ips_by_hwflags(rtwdev);
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static void rtw89_core_txq_work(struct work_struct *w)
@@ -3286,13 +3458,15 @@ static void rtw89_core_handle_sta_pending_tx(struct rtw89_dev *rtwdev,
 					  rtwvif_link);
 }
 
-static int rtw89_core_send_nullfunc(struct rtw89_dev *rtwdev,
-				    struct rtw89_vif_link *rtwvif_link, bool qos, bool ps)
+int rtw89_core_send_nullfunc(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link,
+			     bool qos, bool ps, int timeout)
 {
 	struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link);
 	int link_id = ieee80211_vif_is_mld(vif) ? rtwvif_link->link_id : -1;
+	struct rtw89_sta_link *rtwsta_link;
 	struct ieee80211_sta *sta;
 	struct ieee80211_hdr *hdr;
+	struct rtw89_sta *rtwsta;
 	struct sk_buff *skb;
 	int ret, qsel;
 
@@ -3305,6 +3479,7 @@ static int rtw89_core_send_nullfunc(struct rtw89_dev *rtwdev,
 		ret = -EINVAL;
 		goto out;
 	}
+	rtwsta = sta_to_rtwsta(sta);
 
 	skb = ieee80211_nullfunc_get(rtwdev->hw, vif, link_id, qos);
 	if (!skb) {
@@ -3316,7 +3491,13 @@ static int rtw89_core_send_nullfunc(struct rtw89_dev *rtwdev,
 	if (ps)
 		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
 
-	ret = rtw89_core_tx_write(rtwdev, vif, sta, skb, &qsel);
+	rtwsta_link = rtwsta->links[rtwvif_link->link_id];
+	if (unlikely(!rtwsta_link)) {
+		ret = -ENOLINK;
+		goto out;
+	}
+
+	ret = rtw89_core_tx_write_link(rtwdev, rtwvif_link, rtwsta_link, skb, &qsel, true);
 	if (ret) {
 		rtw89_warn(rtwdev, "nullfunc transmit failed: %d\n", ret);
 		dev_kfree_skb_any(skb);
@@ -3326,7 +3507,7 @@ static int rtw89_core_send_nullfunc(struct rtw89_dev *rtwdev,
 	rcu_read_unlock();
 
 	return rtw89_core_tx_kick_off_and_wait(rtwdev, skb, qsel,
-					       RTW89_ROC_TX_TIMEOUT);
+					       timeout);
 out:
 	rcu_read_unlock();
 
@@ -3336,6 +3517,9 @@ out:
 void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
+	struct rtw89_chanctx_pause_parm pause_parm = {
+		.rsn = RTW89_CHANCTX_PAUSE_REASON_ROC,
+	};
 	struct ieee80211_hw *hw = rtwdev->hw;
 	struct rtw89_roc *roc = &rtwvif->roc;
 	struct rtw89_vif_link *rtwvif_link;
@@ -3344,21 +3528,24 @@ void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	u32 reg;
 	int ret;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	rtw89_leave_ips_by_hwflags(rtwdev);
 	rtw89_leave_lps(rtwdev);
 
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, RTW89_ROC_BY_LINK_INDEX);
+	rtwvif_link = rtw89_get_designated_link(rtwvif);
 	if (unlikely(!rtwvif_link)) {
-		rtw89_err(rtwdev, "roc start: find no link on HW-%u\n",
-			  RTW89_ROC_BY_LINK_INDEX);
+		rtw89_err(rtwdev, "roc start: find no designated link\n");
 		return;
 	}
 
-	rtw89_chanctx_pause(rtwdev, RTW89_CHANCTX_PAUSE_REASON_ROC);
+	roc->link_id = rtwvif_link->link_id;
+
+	pause_parm.trigger = rtwvif_link;
+	rtw89_chanctx_pause(rtwdev, &pause_parm);
 
-	ret = rtw89_core_send_nullfunc(rtwdev, rtwvif_link, true, true);
+	ret = rtw89_core_send_nullfunc(rtwdev, rtwvif_link, true, true,
+				       RTW89_ROC_TX_TIMEOUT);
 	if (ret)
 		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
 			    "roc send null-1 failed: %d\n", ret);
@@ -3376,16 +3563,16 @@ void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	}
 
 	cfg80211_chandef_create(&roc_chan, &roc->chan, NL80211_CHAN_NO_HT);
-	rtw89_config_roc_chandef(rtwdev, rtwvif_link->chanctx_idx, &roc_chan);
+	rtw89_config_roc_chandef(rtwdev, rtwvif_link, &roc_chan);
 	rtw89_set_channel(rtwdev);
 
 	reg = rtw89_mac_reg_by_idx(rtwdev, mac->rx_fltr, rtwvif_link->mac_idx);
 	rtw89_write32_clr(rtwdev, reg, B_AX_A_UC_CAM_MATCH | B_AX_A_BC_CAM_MATCH);
 
 	ieee80211_ready_on_channel(hw);
-	cancel_delayed_work(&rtwvif->roc.roc_work);
-	ieee80211_queue_delayed_work(hw, &rtwvif->roc.roc_work,
-				     msecs_to_jiffies(rtwvif->roc.duration));
+	wiphy_delayed_work_cancel(hw->wiphy, &rtwvif->roc.roc_work);
+	wiphy_delayed_work_queue(hw->wiphy, &rtwvif->roc.roc_work,
+				 msecs_to_jiffies(rtwvif->roc.duration));
 }
 
 void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
@@ -3398,17 +3585,17 @@ void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	u32 reg;
 	int ret;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	ieee80211_remain_on_channel_expired(hw);
 
 	rtw89_leave_ips_by_hwflags(rtwdev);
 	rtw89_leave_lps(rtwdev);
 
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, RTW89_ROC_BY_LINK_INDEX);
+	rtwvif_link = rtwvif->links[roc->link_id];
 	if (unlikely(!rtwvif_link)) {
-		rtw89_err(rtwdev, "roc end: find no link on HW-%u\n",
-			  RTW89_ROC_BY_LINK_INDEX);
+		rtw89_err(rtwdev, "roc end: find no link (link id %u)\n",
+			  roc->link_id);
 		return;
 	}
 
@@ -3416,9 +3603,10 @@ void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	rtw89_write32_mask(rtwdev, reg, B_AX_RX_FLTR_CFG_MASK, rtwdev->hal.rx_fltr);
 
 	roc->state = RTW89_ROC_IDLE;
-	rtw89_config_roc_chandef(rtwdev, rtwvif_link->chanctx_idx, NULL);
+	rtw89_config_roc_chandef(rtwdev, rtwvif_link, NULL);
 	rtw89_chanctx_proceed(rtwdev, NULL);
-	ret = rtw89_core_send_nullfunc(rtwdev, rtwvif_link, true, false);
+	ret = rtw89_core_send_nullfunc(rtwdev, rtwvif_link, true, false,
+				       RTW89_ROC_TX_TIMEOUT);
 	if (ret)
 		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
 			    "roc send null-0 failed: %d\n", ret);
@@ -3430,18 +3618,18 @@ void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	queue_work(rtwdev->txq_wq, &rtwdev->txq_work);
 
 	if (hw->conf.flags & IEEE80211_CONF_IDLE)
-		ieee80211_queue_delayed_work(hw, &roc->roc_work,
-					     msecs_to_jiffies(RTW89_ROC_IDLE_TIMEOUT));
+		wiphy_delayed_work_queue(hw->wiphy, &roc->roc_work,
+					 msecs_to_jiffies(RTW89_ROC_IDLE_TIMEOUT));
 }
 
-void rtw89_roc_work(struct work_struct *work)
+void rtw89_roc_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_vif *rtwvif = container_of(work, struct rtw89_vif,
 						roc.roc_work.work);
 	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
 	struct rtw89_roc *roc = &rtwvif->roc;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
 
 	switch (roc->state) {
 	case RTW89_ROC_IDLE:
@@ -3454,14 +3642,25 @@ void rtw89_roc_work(struct work_struct *work)
 	default:
 		break;
 	}
-
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static enum rtw89_tfc_lv rtw89_get_traffic_level(struct rtw89_dev *rtwdev,
-						 u32 throughput, u64 cnt)
+						 u32 throughput, u64 cnt,
+						 enum rtw89_tfc_interval interval)
 {
-	if (cnt < 100)
+	u64 cnt_level;
+
+	switch (interval) {
+	default:
+	case RTW89_TFC_INTERVAL_100MS:
+		cnt_level = 5;
+		break;
+	case RTW89_TFC_INTERVAL_2SEC:
+		cnt_level = 100;
+		break;
+	}
+
+	if (cnt < cnt_level)
 		return RTW89_TFC_IDLE;
 	if (throughput > 50)
 		return RTW89_TFC_HIGH;
@@ -3473,13 +3672,14 @@ static enum rtw89_tfc_lv rtw89_get_traffic_level(struct rtw89_dev *rtwdev,
 }
 
 static bool rtw89_traffic_stats_calc(struct rtw89_dev *rtwdev,
-				     struct rtw89_traffic_stats *stats)
+				     struct rtw89_traffic_stats *stats,
+				     enum rtw89_tfc_interval interval)
 {
 	enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
 	enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
 
-	stats->tx_throughput_raw = (u32)(stats->tx_unicast >> RTW89_TP_SHIFT);
-	stats->rx_throughput_raw = (u32)(stats->rx_unicast >> RTW89_TP_SHIFT);
+	stats->tx_throughput_raw = rtw89_bytes_to_mbps(stats->tx_unicast, interval);
+	stats->rx_throughput_raw = rtw89_bytes_to_mbps(stats->rx_unicast, interval);
 
 	ewma_tp_add(&stats->tx_ewma_tp, stats->tx_throughput_raw);
 	ewma_tp_add(&stats->rx_ewma_tp, stats->rx_throughput_raw);
@@ -3487,9 +3687,9 @@ static bool rtw89_traffic_stats_calc(struct rtw89_dev *rtwdev,
 	stats->tx_throughput = ewma_tp_read(&stats->tx_ewma_tp);
 	stats->rx_throughput = ewma_tp_read(&stats->rx_ewma_tp);
 	stats->tx_tfc_lv = rtw89_get_traffic_level(rtwdev, stats->tx_throughput,
-						   stats->tx_cnt);
+						   stats->tx_cnt, interval);
 	stats->rx_tfc_lv = rtw89_get_traffic_level(rtwdev, stats->rx_throughput,
-						   stats->rx_cnt);
+						   stats->rx_cnt, interval);
 	stats->tx_avg_len = stats->tx_cnt ?
 			    DIV_ROUND_DOWN_ULL(stats->tx_unicast, stats->tx_cnt) : 0;
 	stats->rx_avg_len = stats->rx_cnt ?
@@ -3515,10 +3715,12 @@ static bool rtw89_traffic_stats_track(struct rtw89_dev *rtwdev)
 	unsigned int link_id;
 	bool tfc_changed;
 
-	tfc_changed = rtw89_traffic_stats_calc(rtwdev, &rtwdev->stats);
+	tfc_changed = rtw89_traffic_stats_calc(rtwdev, &rtwdev->stats,
+					       RTW89_TFC_INTERVAL_2SEC);
 
 	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
-		rtw89_traffic_stats_calc(rtwdev, &rtwvif->stats);
+		rtw89_traffic_stats_calc(rtwdev, &rtwvif->stats,
+					 RTW89_TFC_INTERVAL_2SEC);
 
 		rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id)
 			rtw89_fw_h2c_tp_offload(rtwdev, rtwvif_link);
@@ -3538,8 +3740,8 @@ static void rtw89_enter_lps_track(struct rtw89_dev *rtwdev)
 		if (rtwvif->offchan)
 			continue;
 
-		if (rtwvif->stats.tx_tfc_lv != RTW89_TFC_IDLE ||
-		    rtwvif->stats.rx_tfc_lv != RTW89_TFC_IDLE)
+		if (rtwvif->stats_ps.tx_tfc_lv >= RTW89_TFC_MID ||
+		    rtwvif->stats_ps.rx_tfc_lv >= RTW89_TFC_MID)
 			continue;
 
 		vif = rtwvif_to_vif(rtwvif);
@@ -3586,26 +3788,146 @@ void rtw89_traffic_stats_init(struct rtw89_dev *rtwdev,
 	ewma_tp_init(&stats->rx_ewma_tp);
 }
 
-static void rtw89_track_work(struct work_struct *work)
+#define RTW89_MLSR_GOTO_2GHZ_THRESHOLD -53
+#define RTW89_MLSR_EXIT_2GHZ_THRESHOLD -38
+static void rtw89_core_mlsr_link_decision(struct rtw89_dev *rtwdev,
+					  struct rtw89_vif *rtwvif)
+{
+	unsigned int sel_link_id = IEEE80211_MLD_MAX_NUM_LINKS;
+	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
+	struct rtw89_vif_link *rtwvif_link;
+	const struct rtw89_chan *chan;
+	unsigned long usable_links;
+	unsigned int link_id;
+	u8 decided_bands;
+	u8 rssi;
+
+	rssi = ewma_rssi_read(&rtwdev->phystat.bcn_rssi);
+	if (unlikely(!rssi))
+		return;
+
+	if (RTW89_RSSI_RAW_TO_DBM(rssi) >= RTW89_MLSR_EXIT_2GHZ_THRESHOLD)
+		decided_bands = BIT(RTW89_BAND_5G) | BIT(RTW89_BAND_6G);
+	else if (RTW89_RSSI_RAW_TO_DBM(rssi) <= RTW89_MLSR_GOTO_2GHZ_THRESHOLD)
+		decided_bands = BIT(RTW89_BAND_2G);
+	else
+		return;
+
+	usable_links = ieee80211_vif_usable_links(vif);
+
+	rtwvif_link = rtw89_get_designated_link(rtwvif);
+	if (unlikely(!rtwvif_link))
+		goto select;
+
+	chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx);
+	if (decided_bands & BIT(chan->band_type))
+		return;
+
+	usable_links &= ~BIT(rtwvif_link->link_id);
+
+select:
+	rcu_read_lock();
+
+	for_each_set_bit(link_id, &usable_links, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link_conf;
+		struct ieee80211_channel *channel;
+		enum rtw89_band band;
+
+		link_conf = rcu_dereference(vif->link_conf[link_id]);
+		if (unlikely(!link_conf))
+			continue;
+
+		channel = link_conf->chanreq.oper.chan;
+		if (unlikely(!channel))
+			continue;
+
+		band = rtw89_nl80211_to_hw_band(channel->band);
+		if (decided_bands & BIT(band)) {
+			sel_link_id = link_id;
+			break;
+		}
+	}
+
+	rcu_read_unlock();
+
+	if (sel_link_id == IEEE80211_MLD_MAX_NUM_LINKS)
+		return;
+
+	rtw89_core_mlsr_switch(rtwdev, rtwvif, sel_link_id);
+}
+
+static void rtw89_core_mlo_track(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	struct ieee80211_vif *vif;
+	struct rtw89_vif *rtwvif;
+
+	if (hal->disabled_dm_bitmap & BIT(RTW89_DM_MLO))
+		return;
+
+	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
+		vif = rtwvif_to_vif(rtwvif);
+		if (!vif->cfg.assoc || !ieee80211_vif_is_mld(vif))
+			continue;
+
+		switch (rtwvif->mlo_mode) {
+		case RTW89_MLO_MODE_MLSR:
+			rtw89_core_mlsr_link_decision(rtwdev, rtwvif);
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+static void rtw89_track_ps_work(struct wiphy *wiphy, struct wiphy_work *work)
+{
+	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
+						track_ps_work.work);
+	struct rtw89_vif *rtwvif;
+
+	lockdep_assert_wiphy(wiphy);
+
+	if (test_bit(RTW89_FLAG_FORBIDDEN_TRACK_WORK, rtwdev->flags))
+		return;
+
+	if (!test_bit(RTW89_FLAG_RUNNING, rtwdev->flags))
+		return;
+
+	wiphy_delayed_work_queue(wiphy, &rtwdev->track_ps_work,
+				 RTW89_TRACK_PS_WORK_PERIOD);
+
+	rtw89_for_each_rtwvif(rtwdev, rtwvif)
+		rtw89_traffic_stats_calc(rtwdev, &rtwvif->stats_ps,
+					 RTW89_TFC_INTERVAL_100MS);
+
+	if (rtwdev->scanning)
+		return;
+
+	if (rtwdev->lps_enabled && !rtwdev->btc.lps)
+		rtw89_enter_lps_track(rtwdev);
+}
+
+static void rtw89_track_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						track_work.work);
 	bool tfc_changed;
 
-	if (test_bit(RTW89_FLAG_FORBIDDEN_TRACK_WROK, rtwdev->flags))
-		return;
+	lockdep_assert_wiphy(wiphy);
 
-	mutex_lock(&rtwdev->mutex);
+	if (test_bit(RTW89_FLAG_FORBIDDEN_TRACK_WORK, rtwdev->flags))
+		return;
 
 	if (!test_bit(RTW89_FLAG_RUNNING, rtwdev->flags))
-		goto out;
+		return;
 
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->track_work,
-				     RTW89_TRACK_WORK_PERIOD);
+	wiphy_delayed_work_queue(wiphy, &rtwdev->track_work,
+				 RTW89_TRACK_WORK_PERIOD);
 
 	tfc_changed = rtw89_traffic_stats_track(rtwdev);
 	if (rtwdev->scanning)
-		goto out;
+		return;
 
 	rtw89_leave_lps(rtwdev);
 
@@ -3624,15 +3946,13 @@ static void rtw89_track_work(struct work_struct *work)
 	rtw89_phy_antdiv_track(rtwdev);
 	rtw89_phy_ul_tb_ctrl_track(rtwdev);
 	rtw89_phy_edcca_track(rtwdev);
-	rtw89_tas_track(rtwdev);
+	rtw89_sar_track(rtwdev);
 	rtw89_chanctx_track(rtwdev);
 	rtw89_core_rfkill_poll(rtwdev, false);
+	rtw89_core_mlo_track(rtwdev);
 
 	if (rtwdev->lps_enabled && !rtwdev->btc.lps)
 		rtw89_enter_lps_track(rtwdev);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 u8 rtw89_core_acquire_bit_map(unsigned long *addr, unsigned long size)
@@ -3666,7 +3986,7 @@ int rtw89_core_acquire_sta_ba_entry(struct rtw89_dev *rtwdev,
 	u8 idx;
 	int i;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	idx = rtw89_core_acquire_bit_map(cam_info->ba_cam_map, chip->bacam_num);
 	if (idx == chip->bacam_num) {
@@ -3710,7 +4030,7 @@ int rtw89_core_release_sta_ba_entry(struct rtw89_dev *rtwdev,
 	struct rtw89_ba_cam_entry *entry = NULL, *tmp;
 	u8 idx;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	list_for_each_entry_safe(entry, tmp, &rtwsta_link->ba_cam_list, list) {
 		if (entry->tid != tid)
@@ -3821,6 +4141,13 @@ int rtw89_core_sta_link_add(struct rtw89_dev *rtwdev,
 		rtw89_btc_ntfy_role_info(rtwdev, rtwvif_link, rtwsta_link,
 					 BTC_ROLE_MSTS_STA_CONN_START);
 		rtw89_chip_rfk_channel(rtwdev, rtwvif_link);
+
+		if (vif->p2p) {
+			rtw89_mac_get_tx_retry_limit(rtwdev, rtwsta_link,
+						     &rtwsta_link->tx_retry);
+			rtw89_mac_set_tx_retry_limit(rtwdev, rtwsta_link, false, 60);
+		}
+		rtw89_phy_dig_suspend(rtwdev);
 	} else if (vif->type == NL80211_IFTYPE_AP || sta->tdls) {
 		ret = rtw89_mac_set_macid_pause(rtwdev, rtwsta_link->mac_id, false);
 		if (ret) {
@@ -3858,6 +4185,9 @@ int rtw89_core_sta_link_disassoc(struct rtw89_dev *rtwdev,
 	if (vif->type == NL80211_IFTYPE_STATION)
 		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, rtwvif_link, false);
 
+	if (rtwvif_link->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT)
+		rtw89_p2p_noa_once_deinit(rtwvif_link);
+
 	return 0;
 }
 
@@ -4002,6 +4332,11 @@ int rtw89_core_sta_link_assoc(struct rtw89_dev *rtwdev,
 		}
 
 		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, rtwvif_link, true);
+
+		if (vif->p2p)
+			rtw89_mac_set_tx_retry_limit(rtwdev, rtwsta_link, false,
+						     rtwsta_link->tx_retry);
+		rtw89_phy_dig_resume(rtwdev, false);
 	}
 
 	rtw89_assoc_link_set(rtwsta_link);
@@ -4020,6 +4355,10 @@ int rtw89_core_sta_link_remove(struct rtw89_dev *rtwdev,
 		rtw89_reg_6ghz_recalc(rtwdev, rtwvif_link, false);
 		rtw89_btc_ntfy_role_info(rtwdev, rtwvif_link, rtwsta_link,
 					 BTC_ROLE_MSTS_STA_DIS_CONN);
+
+		if (vif->p2p)
+			rtw89_mac_set_tx_retry_limit(rtwdev, rtwsta_link, false,
+						     rtwsta_link->tx_retry);
 	} else if (vif->type == NL80211_IFTYPE_AP || sta->tdls) {
 		ret = rtw89_fw_h2c_role_maintain(rtwdev, rtwvif_link, rtwsta_link,
 						 RTW89_ROLE_REMOVE);
@@ -4373,17 +4712,18 @@ static void rtw89_init_eht_cap(struct rtw89_dev *rtwdev,
 
 #define RTW89_SBAND_IFTYPES_NR 2
 
-static void rtw89_init_he_eht_cap(struct rtw89_dev *rtwdev,
-				  enum nl80211_band band,
-				  struct ieee80211_supported_band *sband)
+static int rtw89_init_he_eht_cap(struct rtw89_dev *rtwdev,
+				 enum nl80211_band band,
+				 struct ieee80211_supported_band *sband)
 {
 	struct ieee80211_sband_iftype_data *iftype_data;
 	enum nl80211_iftype iftype;
 	int idx = 0;
 
-	iftype_data = kcalloc(RTW89_SBAND_IFTYPES_NR, sizeof(*iftype_data), GFP_KERNEL);
+	iftype_data = devm_kcalloc(rtwdev->dev, RTW89_SBAND_IFTYPES_NR,
+				   sizeof(*iftype_data), GFP_KERNEL);
 	if (!iftype_data)
-		return;
+		return -ENOMEM;
 
 	for (iftype = 0; iftype < NUM_NL80211_IFTYPES; iftype++) {
 		switch (iftype) {
@@ -4408,118 +4748,162 @@ static void rtw89_init_he_eht_cap(struct rtw89_dev *rtwdev,
 	}
 
 	_ieee80211_set_sband_iftype_data(sband, iftype_data, idx);
+	return 0;
+}
+
+static struct ieee80211_supported_band *
+rtw89_core_sband_dup(struct rtw89_dev *rtwdev,
+		     const struct ieee80211_supported_band *sband)
+{
+	struct ieee80211_supported_band *dup;
+
+	dup = devm_kmemdup(rtwdev->dev, sband, sizeof(*sband), GFP_KERNEL);
+	if (!dup)
+		return NULL;
+
+	dup->channels = devm_kmemdup(rtwdev->dev, sband->channels,
+				     sizeof(*sband->channels) * sband->n_channels,
+				     GFP_KERNEL);
+	if (!dup->channels)
+		return NULL;
+
+	dup->bitrates = devm_kmemdup(rtwdev->dev, sband->bitrates,
+				     sizeof(*sband->bitrates) * sband->n_bitrates,
+				     GFP_KERNEL);
+	if (!dup->bitrates)
+		return NULL;
+
+	return dup;
 }
 
 static int rtw89_core_set_supported_band(struct rtw89_dev *rtwdev)
 {
 	struct ieee80211_hw *hw = rtwdev->hw;
-	struct ieee80211_supported_band *sband_2ghz = NULL, *sband_5ghz = NULL;
-	struct ieee80211_supported_band *sband_6ghz = NULL;
-	u32 size = sizeof(struct ieee80211_supported_band);
+	struct ieee80211_supported_band *sband;
 	u8 support_bands = rtwdev->chip->support_bands;
+	int ret;
 
 	if (support_bands & BIT(NL80211_BAND_2GHZ)) {
-		sband_2ghz = kmemdup(&rtw89_sband_2ghz, size, GFP_KERNEL);
-		if (!sband_2ghz)
-			goto err;
+		sband = rtw89_core_sband_dup(rtwdev, &rtw89_sband_2ghz);
+		if (!sband)
+			return -ENOMEM;
 #if defined(__FreeBSD__)
 		if (rtw_ht_support)
 #endif
-		rtw89_init_ht_cap(rtwdev, &sband_2ghz->ht_cap);
+		rtw89_init_ht_cap(rtwdev, &sband->ht_cap);
+#if defined(__FreeBSD__)
+		if (rtw_eht_support) {
+#endif
+		ret = rtw89_init_he_eht_cap(rtwdev, NL80211_BAND_2GHZ, sband);
+		if (ret)
+			return ret;
 #if defined(__FreeBSD__)
-		if (rtw_eht_support)
+		}
 #endif
-		rtw89_init_he_eht_cap(rtwdev, NL80211_BAND_2GHZ, sband_2ghz);
-		hw->wiphy->bands[NL80211_BAND_2GHZ] = sband_2ghz;
+		hw->wiphy->bands[NL80211_BAND_2GHZ] = sband;
 	}
 
 	if (support_bands & BIT(NL80211_BAND_5GHZ)) {
-		sband_5ghz = kmemdup(&rtw89_sband_5ghz, size, GFP_KERNEL);
-		if (!sband_5ghz)
-			goto err;
+		sband = rtw89_core_sband_dup(rtwdev, &rtw89_sband_5ghz);
+		if (!sband)
+			return -ENOMEM;
 #if defined(__FreeBSD__)
 		if (rtw_ht_support)
 #endif
-		rtw89_init_ht_cap(rtwdev, &sband_5ghz->ht_cap);
+		rtw89_init_ht_cap(rtwdev, &sband->ht_cap);
 #if defined(__FreeBSD__)
 		if (rtw_vht_support)
 #endif
-		rtw89_init_vht_cap(rtwdev, &sband_5ghz->vht_cap);
+		rtw89_init_vht_cap(rtwdev, &sband->vht_cap);
 #if defined(__FreeBSD__)
-		if (rtw_eht_support)
+		if (rtw_eht_support) {
 #endif
-		rtw89_init_he_eht_cap(rtwdev, NL80211_BAND_5GHZ, sband_5ghz);
-		hw->wiphy->bands[NL80211_BAND_5GHZ] = sband_5ghz;
+		ret = rtw89_init_he_eht_cap(rtwdev, NL80211_BAND_5GHZ, sband);
+		if (ret)
+			return ret;
+#if defined(__FreeBSD__)
+		}
+#endif
+		hw->wiphy->bands[NL80211_BAND_5GHZ] = sband;
 	}
 
+#if defined(__FreeBSD__)
+	if (rtw_eht_support)
+#endif
 	if (support_bands & BIT(NL80211_BAND_6GHZ)) {
-		sband_6ghz = kmemdup(&rtw89_sband_6ghz, size, GFP_KERNEL);
-		if (!sband_6ghz)
-			goto err;
-		rtw89_init_he_eht_cap(rtwdev, NL80211_BAND_6GHZ, sband_6ghz);
-		hw->wiphy->bands[NL80211_BAND_6GHZ] = sband_6ghz;
+		sband = rtw89_core_sband_dup(rtwdev, &rtw89_sband_6ghz);
+		if (!sband)
+			return -ENOMEM;
+		ret = rtw89_init_he_eht_cap(rtwdev, NL80211_BAND_6GHZ, sband);
+		if (ret)
+			return ret;
+		hw->wiphy->bands[NL80211_BAND_6GHZ] = sband;
 	}
 
 	return 0;
-
-err:
-	hw->wiphy->bands[NL80211_BAND_2GHZ] = NULL;
-	hw->wiphy->bands[NL80211_BAND_5GHZ] = NULL;
-	hw->wiphy->bands[NL80211_BAND_6GHZ] = NULL;
-	if (sband_2ghz)
-		kfree((__force void *)sband_2ghz->iftype_data);
-	if (sband_5ghz)
-		kfree((__force void *)sband_5ghz->iftype_data);
-	if (sband_6ghz)
-		kfree((__force void *)sband_6ghz->iftype_data);
-	kfree(sband_2ghz);
-	kfree(sband_5ghz);
-	kfree(sband_6ghz);
-	return -ENOMEM;
-}
-
-static void rtw89_core_clr_supported_band(struct rtw89_dev *rtwdev)
-{
-	struct ieee80211_hw *hw = rtwdev->hw;
-
-	if (hw->wiphy->bands[NL80211_BAND_2GHZ])
-		kfree((__force void *)hw->wiphy->bands[NL80211_BAND_2GHZ]->iftype_data);
-	if (hw->wiphy->bands[NL80211_BAND_5GHZ])
-		kfree((__force void *)hw->wiphy->bands[NL80211_BAND_5GHZ]->iftype_data);
-	if (hw->wiphy->bands[NL80211_BAND_6GHZ])
-		kfree((__force void *)hw->wiphy->bands[NL80211_BAND_6GHZ]->iftype_data);
-	kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]);
-	kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]);
-	kfree(hw->wiphy->bands[NL80211_BAND_6GHZ]);
-	hw->wiphy->bands[NL80211_BAND_2GHZ] = NULL;
-	hw->wiphy->bands[NL80211_BAND_5GHZ] = NULL;
-	hw->wiphy->bands[NL80211_BAND_6GHZ] = NULL;
 }
 
 static void rtw89_core_ppdu_sts_init(struct rtw89_dev *rtwdev)
 {
 	int i;
 
-	for (i = 0; i < RTW89_PHY_MAX; i++)
+	for (i = 0; i < RTW89_PHY_NUM; i++)
 		skb_queue_head_init(&rtwdev->ppdu_sts.rx_queue[i]);
-	for (i = 0; i < RTW89_PHY_MAX; i++)
+	for (i = 0; i < RTW89_PHY_NUM; i++)
 		rtwdev->ppdu_sts.curr_rx_ppdu_cnt[i] = U8_MAX;
 }
 
-void rtw89_core_update_beacon_work(struct work_struct *work)
+void rtw89_core_update_beacon_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev;
 	struct rtw89_vif_link *rtwvif_link = container_of(work, struct rtw89_vif_link,
 							  update_beacon_work);
 
+	lockdep_assert_wiphy(wiphy);
+
 	if (rtwvif_link->net_type != RTW89_NET_TYPE_AP_MODE)
 		return;
 
 	rtwdev = rtwvif_link->rtwvif->rtwdev;
 
-	mutex_lock(&rtwdev->mutex);
 	rtw89_chip_h2c_update_beacon(rtwdev, rtwvif_link);
-	mutex_unlock(&rtwdev->mutex);
+}
+
+void rtw89_core_csa_beacon_work(struct wiphy *wiphy, struct wiphy_work *work)
+{
+	struct rtw89_vif_link *rtwvif_link =
+		container_of(work, struct rtw89_vif_link, csa_beacon_work.work);
+	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
+	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
+	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
+	struct ieee80211_bss_conf *bss_conf;
+	unsigned int delay;
+
+	lockdep_assert_wiphy(wiphy);
+
+	if (rtwvif_link->net_type != RTW89_NET_TYPE_AP_MODE)
+		return;
+
+	rcu_read_lock();
+
+	bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
+	if (!bss_conf->csa_active) {
+		rcu_read_unlock();
+		return;
+	}
+
+	delay = ieee80211_tu_to_usec(bss_conf->beacon_int);
+
+	rcu_read_unlock();
+
+	if (!ieee80211_beacon_cntdwn_is_complete(vif, rtwvif_link->link_id)) {
+		rtw89_chip_h2c_update_beacon(rtwdev, rtwvif_link);
+
+		wiphy_delayed_work_queue(wiphy, &rtwvif_link->csa_beacon_work,
+					 usecs_to_jiffies(delay));
+	} else {
+		ieee80211_csa_finish(vif, rtwvif_link->link_id);
+	}
 }
 
 int rtw89_wait_for_cond(struct rtw89_wait_info *wait, unsigned int cond)
@@ -4628,16 +5012,16 @@ int rtw89_core_start(struct rtw89_dev *rtwdev)
 	rtw89_mac_cfg_phy_rpt_bands(rtwdev, true);
 	rtw89_mac_update_rts_threshold(rtwdev);
 
-	rtw89_tas_reset(rtwdev);
-
 	ret = rtw89_hci_start(rtwdev);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to start hci\n");
 		return ret;
 	}
 
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->track_work,
-				     RTW89_TRACK_WORK_PERIOD);
+	wiphy_delayed_work_queue(rtwdev->hw->wiphy, &rtwdev->track_work,
+				 RTW89_TRACK_WORK_PERIOD);
+	wiphy_delayed_work_queue(rtwdev->hw->wiphy, &rtwdev->track_ps_work,
+				 RTW89_TRACK_PS_WORK_PERIOD);
 
 	set_bit(RTW89_FLAG_RUNNING, rtwdev->flags);
 
@@ -4651,8 +5035,11 @@ int rtw89_core_start(struct rtw89_dev *rtwdev)
 
 void rtw89_core_stop(struct rtw89_dev *rtwdev)
 {
+	struct wiphy *wiphy = rtwdev->hw->wiphy;
 	struct rtw89_btc *btc = &rtwdev->btc;
 
+	lockdep_assert_wiphy(wiphy);
+
 	/* Prvent to stop twice; enter_ips and ops_stop */
 	if (!test_bit(RTW89_FLAG_RUNNING, rtwdev->flags))
 		return;
@@ -4661,25 +5048,23 @@ void rtw89_core_stop(struct rtw89_dev *rtwdev)
 
 	clear_bit(RTW89_FLAG_RUNNING, rtwdev->flags);
 
-	mutex_unlock(&rtwdev->mutex);
-
-	cancel_work_sync(&rtwdev->c2h_work);
-	cancel_work_sync(&rtwdev->cancel_6ghz_probe_work);
-	cancel_work_sync(&btc->eapol_notify_work);
-	cancel_work_sync(&btc->arp_notify_work);
-	cancel_work_sync(&btc->dhcp_notify_work);
-	cancel_work_sync(&btc->icmp_notify_work);
+	wiphy_work_cancel(wiphy, &rtwdev->c2h_work);
+	wiphy_work_cancel(wiphy, &rtwdev->cancel_6ghz_probe_work);
+	wiphy_work_cancel(wiphy, &btc->eapol_notify_work);
+	wiphy_work_cancel(wiphy, &btc->arp_notify_work);
+	wiphy_work_cancel(wiphy, &btc->dhcp_notify_work);
+	wiphy_work_cancel(wiphy, &btc->icmp_notify_work);
 	cancel_delayed_work_sync(&rtwdev->txq_reinvoke_work);
-	cancel_delayed_work_sync(&rtwdev->track_work);
-	cancel_delayed_work_sync(&rtwdev->chanctx_work);
-	cancel_delayed_work_sync(&rtwdev->coex_act1_work);
-	cancel_delayed_work_sync(&rtwdev->coex_bt_devinfo_work);
-	cancel_delayed_work_sync(&rtwdev->coex_rfk_chk_work);
-	cancel_delayed_work_sync(&rtwdev->cfo_track_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->track_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->track_ps_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->chanctx_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->coex_act1_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->coex_bt_devinfo_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->coex_rfk_chk_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->cfo_track_work);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->mcc_prepare_done_work);
 	cancel_delayed_work_sync(&rtwdev->forbid_ba_work);
-	cancel_delayed_work_sync(&rtwdev->antdiv_work);
-
-	mutex_lock(&rtwdev->mutex);
+	wiphy_delayed_work_cancel(wiphy, &rtwdev->antdiv_work);
 
 	rtw89_btc_ntfy_poweroff(rtwdev);
 	rtw89_hci_flush_queues(rtwdev, BIT(rtwdev->hw->queues) - 1, true);
@@ -4804,6 +5189,7 @@ struct rtw89_vif_link *rtw89_vif_set_link(struct rtw89_vif *rtwvif,
 
 	set_bit(index, rtwvif->links_inst_map);
 	rtwvif->links[link_id] = rtwvif_link;
+	list_add_tail(&rtwvif_link->dlink_schd, &rtwvif->dlink_pool);
 	return rtwvif_link;
 
 err:
@@ -4824,6 +5210,7 @@ void rtw89_vif_unset_link(struct rtw89_vif *rtwvif, unsigned int link_id)
 	index = rtw89_vif_link_inst_get_index(link);
 	clear_bit(index, rtwvif->links_inst_map);
 	*container = NULL;
+	list_del(&link->dlink_schd);
 }
 
 struct rtw89_sta_link *rtw89_sta_set_link(struct rtw89_sta *rtwsta,
@@ -4854,6 +5241,7 @@ struct rtw89_sta_link *rtw89_sta_set_link(struct rtw89_sta *rtwsta,
 
 	set_bit(index, rtwsta->links_inst_map);
 	rtwsta->links[link_id] = rtwsta_link;
+	list_add_tail(&rtwsta_link->dlink_schd, &rtwsta->dlink_pool);
 	return rtwsta_link;
 
 err:
@@ -4874,6 +5262,7 @@ void rtw89_sta_unset_link(struct rtw89_sta *rtwsta, unsigned int link_id)
 	index = rtw89_sta_link_inst_get_index(link);
 	clear_bit(index, rtwsta->links_inst_map);
 	*container = NULL;
+	list_del(&link->dlink_schd);
 }
 
 int rtw89_core_init(struct rtw89_dev *rtwdev)
@@ -4890,35 +5279,38 @@ int rtw89_core_init(struct rtw89_dev *rtwdev)
 			continue;
 		INIT_LIST_HEAD(&rtwdev->scan_info.pkt_list[band]);
 	}
+	INIT_LIST_HEAD(&rtwdev->scan_info.chan_list);
 	INIT_WORK(&rtwdev->ba_work, rtw89_core_ba_work);
 	INIT_WORK(&rtwdev->txq_work, rtw89_core_txq_work);
 	INIT_DELAYED_WORK(&rtwdev->txq_reinvoke_work, rtw89_core_txq_reinvoke_work);
-	INIT_DELAYED_WORK(&rtwdev->track_work, rtw89_track_work);
-	INIT_DELAYED_WORK(&rtwdev->chanctx_work, rtw89_chanctx_work);
-	INIT_DELAYED_WORK(&rtwdev->coex_act1_work, rtw89_coex_act1_work);
-	INIT_DELAYED_WORK(&rtwdev->coex_bt_devinfo_work, rtw89_coex_bt_devinfo_work);
-	INIT_DELAYED_WORK(&rtwdev->coex_rfk_chk_work, rtw89_coex_rfk_chk_work);
-	INIT_DELAYED_WORK(&rtwdev->cfo_track_work, rtw89_phy_cfo_track_work);
+	wiphy_delayed_work_init(&rtwdev->track_work, rtw89_track_work);
+	wiphy_delayed_work_init(&rtwdev->track_ps_work, rtw89_track_ps_work);
+	wiphy_delayed_work_init(&rtwdev->chanctx_work, rtw89_chanctx_work);
+	wiphy_delayed_work_init(&rtwdev->coex_act1_work, rtw89_coex_act1_work);
+	wiphy_delayed_work_init(&rtwdev->coex_bt_devinfo_work, rtw89_coex_bt_devinfo_work);
+	wiphy_delayed_work_init(&rtwdev->coex_rfk_chk_work, rtw89_coex_rfk_chk_work);
+	wiphy_delayed_work_init(&rtwdev->cfo_track_work, rtw89_phy_cfo_track_work);
+	wiphy_delayed_work_init(&rtwdev->mcc_prepare_done_work, rtw89_mcc_prepare_done_work);
 	INIT_DELAYED_WORK(&rtwdev->forbid_ba_work, rtw89_forbid_ba_work);
-	INIT_DELAYED_WORK(&rtwdev->antdiv_work, rtw89_phy_antdiv_work);
+	wiphy_delayed_work_init(&rtwdev->antdiv_work, rtw89_phy_antdiv_work);
 	rtwdev->txq_wq = alloc_workqueue("rtw89_tx_wq", WQ_UNBOUND | WQ_HIGHPRI, 0);
 	if (!rtwdev->txq_wq)
 		return -ENOMEM;
 	spin_lock_init(&rtwdev->ba_lock);
 	spin_lock_init(&rtwdev->rpwm_lock);
-	mutex_init(&rtwdev->mutex);
 	mutex_init(&rtwdev->rf_mutex);
 	rtwdev->total_sta_assoc = 0;
 
 	rtw89_init_wait(&rtwdev->mcc.wait);
+	rtw89_init_wait(&rtwdev->mlo.wait);
 	rtw89_init_wait(&rtwdev->mac.fw_ofld_wait);
 	rtw89_init_wait(&rtwdev->wow.wait);
 	rtw89_init_wait(&rtwdev->mac.ps_wait);
 
-	INIT_WORK(&rtwdev->c2h_work, rtw89_fw_c2h_work);
-	INIT_WORK(&rtwdev->ips_work, rtw89_ips_work);
+	wiphy_work_init(&rtwdev->c2h_work, rtw89_fw_c2h_work);
+	wiphy_work_init(&rtwdev->ips_work, rtw89_ips_work);
+	wiphy_work_init(&rtwdev->cancel_6ghz_probe_work, rtw89_cancel_6ghz_probe_work);
 	INIT_WORK(&rtwdev->load_firmware_work, rtw89_load_firmware_work);
-	INIT_WORK(&rtwdev->cancel_6ghz_probe_work, rtw89_cancel_6ghz_probe_work);
 
 	skb_queue_head_init(&rtwdev->c2h_queue);
 	rtw89_core_ppdu_sts_init(rtwdev);
@@ -4932,13 +5324,16 @@ int rtw89_core_init(struct rtw89_dev *rtwdev)
 	if (rtwdev->chip->chip_gen == RTW89_CHIP_BE) {
 		rtwdev->dbcc_en = true;
 		rtwdev->mac.qta_mode = RTW89_QTA_DBCC;
-		rtwdev->mlo_dbcc_mode = MLO_2_PLUS_0_1RF;
+		rtwdev->mlo_dbcc_mode = MLO_1_PLUS_1_1RF;
 	}
 
-	INIT_WORK(&btc->eapol_notify_work, rtw89_btc_ntfy_eapol_packet_work);
-	INIT_WORK(&btc->arp_notify_work, rtw89_btc_ntfy_arp_packet_work);
-	INIT_WORK(&btc->dhcp_notify_work, rtw89_btc_ntfy_dhcp_packet_work);
-	INIT_WORK(&btc->icmp_notify_work, rtw89_btc_ntfy_icmp_packet_work);
+	rtwdev->bbs[RTW89_PHY_0].phy_idx = RTW89_PHY_0;
+	rtwdev->bbs[RTW89_PHY_1].phy_idx = RTW89_PHY_1;
+
+	wiphy_work_init(&btc->eapol_notify_work, rtw89_btc_ntfy_eapol_packet_work);
+	wiphy_work_init(&btc->arp_notify_work, rtw89_btc_ntfy_arp_packet_work);
+	wiphy_work_init(&btc->dhcp_notify_work, rtw89_btc_ntfy_dhcp_packet_work);
+	wiphy_work_init(&btc->icmp_notify_work, rtw89_btc_ntfy_icmp_packet_work);
 
 	init_completion(&rtwdev->fw.req.completion);
 	init_completion(&rtwdev->rfk_wait.completion);
@@ -4947,7 +5342,7 @@ int rtw89_core_init(struct rtw89_dev *rtwdev)
 
 	rtw89_ser_init(rtwdev);
 	rtw89_entity_init(rtwdev);
-	rtw89_tas_init(rtwdev);
+	rtw89_sar_init(rtwdev);
 	rtw89_phy_ant_gain_init(rtwdev);
 
 	return 0;
@@ -4958,11 +5353,10 @@ void rtw89_core_deinit(struct rtw89_dev *rtwdev)
 {
 	rtw89_ser_deinit(rtwdev);
 	rtw89_unload_firmware(rtwdev);
-	rtw89_fw_free_all_early_h2c(rtwdev);
+	__rtw89_fw_free_all_early_h2c(rtwdev);
 
 	destroy_workqueue(rtwdev->txq_wq);
 	mutex_destroy(&rtwdev->rf_mutex);
-	mutex_destroy(&rtwdev->mutex);
 }
 EXPORT_SYMBOL(rtw89_core_deinit);
 
@@ -4971,17 +5365,16 @@ void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwv
 {
 	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev,
 						       rtwvif_link->chanctx_idx);
+	struct rtw89_bb_ctx *bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx);
 
 	rtwdev->scanning = true;
-	rtw89_leave_lps(rtwdev);
-	if (hw_scan)
-		rtw89_leave_ips_by_hwflags(rtwdev);
 
 	ether_addr_copy(rtwvif_link->mac_addr, mac_addr);
 	rtw89_btc_ntfy_scan_start(rtwdev, rtwvif_link->phy_idx, chan->band_type);
 	rtw89_chip_rfk_scan(rtwdev, rtwvif_link, true);
 	rtw89_hci_recalc_int_mit(rtwdev);
-	rtw89_phy_config_edcca(rtwdev, true);
+	rtw89_phy_config_edcca(rtwdev, bb, true);
+	rtw89_tas_scan(rtwdev, true);
 
 	rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, mac_addr);
 }
@@ -4990,6 +5383,8 @@ void rtw89_core_scan_complete(struct rtw89_dev *rtwdev,
 			      struct rtw89_vif_link *rtwvif_link, bool hw_scan)
 {
 	struct ieee80211_bss_conf *bss_conf;
+	struct rtw89_bb_ctx *bb;
+	int ret;
 
 	if (!rtwvif_link)
 		return;
@@ -5005,12 +5400,23 @@ void rtw89_core_scan_complete(struct rtw89_dev *rtwdev,
 
 	rtw89_chip_rfk_scan(rtwdev, rtwvif_link, false);
 	rtw89_btc_ntfy_scan_finish(rtwdev, rtwvif_link->phy_idx);
-	rtw89_phy_config_edcca(rtwdev, false);
+	bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx);
+	rtw89_phy_config_edcca(rtwdev, bb, false);
+	rtw89_tas_scan(rtwdev, false);
+
+	if (hw_scan) {
+		ret = rtw89_core_send_nullfunc(rtwdev, rtwvif_link, false, false,
+					       RTW89_SCAN_NULL_TIMEOUT);
+		if (ret)
+			rtw89_debug(rtwdev, RTW89_DBG_TXRX,
+				    "scan send null-0 failed: %d\n", ret);
+	}
 
 	rtwdev->scanning = false;
-	rtwdev->dig.bypass_dig = true;
+	rtw89_for_each_active_bb(rtwdev, bb)
+		bb->dig.bypass_dig = true;
 	if (hw_scan && (rtwdev->hw->conf.flags & IEEE80211_CONF_IDLE))
-		ieee80211_queue_work(rtwdev->hw, &rtwdev->ips_work);
+		wiphy_work_queue(rtwdev->hw->wiphy, &rtwdev->ips_work);
 }
 
 static void rtw89_read_chip_ver(struct rtw89_dev *rtwdev)
@@ -5085,6 +5491,77 @@ out:
 	rtw89_load_txpwr_table(rtwdev, rtwdev->rfe_parms->byr_tbl);
 }
 
+int rtw89_core_mlsr_switch(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+			   unsigned int link_id)
+{
+	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
+	u16 usable_links = ieee80211_vif_usable_links(vif);
+	u16 active_links = vif->active_links;
+	struct rtw89_vif_link *target, *cur;
+	int ret;
+
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	if (unlikely(!ieee80211_vif_is_mld(vif)))
+		return -EOPNOTSUPP;
+
+	if (unlikely(link_id >= IEEE80211_MLD_MAX_NUM_LINKS ||
+		     !(usable_links & BIT(link_id)))) {
+		rtw89_warn(rtwdev, "%s: link id %u is not usable\n", __func__,
+			   link_id);
+		return -ENOLINK;
+	}
+
+	if (active_links == BIT(link_id))
+		return 0;
+
+	rtw89_debug(rtwdev, RTW89_DBG_STATE, "%s: switch to link id %u MLSR\n",
+		    __func__, link_id);
+
+	rtw89_leave_lps(rtwdev);
+
+	ieee80211_stop_queues(rtwdev->hw);
+	flush_work(&rtwdev->txq_work);
+
+	cur = rtw89_get_designated_link(rtwvif);
+
+	ret = ieee80211_set_active_links(vif, active_links | BIT(link_id));
+	if (ret) {
+		rtw89_err(rtwdev, "%s: failed to activate link id %u\n",
+			  __func__, link_id);
+		goto wake_queue;
+	}
+
+	target = rtwvif->links[link_id];
+	if (unlikely(!target)) {
+		rtw89_err(rtwdev, "%s: failed to confirm link id %u\n",
+			  __func__, link_id);
+
+		ieee80211_set_active_links(vif, active_links);
+		ret = -EFAULT;
+		goto wake_queue;
+	}
+
+	if (likely(cur))
+		rtw89_fw_h2c_mlo_link_cfg(rtwdev, cur, false);
+
+	rtw89_fw_h2c_mlo_link_cfg(rtwdev, target, true);
+
+	ret = ieee80211_set_active_links(vif, BIT(link_id));
+	if (ret)
+		rtw89_err(rtwdev, "%s: failed to inactivate links 0x%x\n",
+			  __func__, active_links);
+
+	rtw89_chip_rfk_channel(rtwdev, target);
+
+	rtwvif->mlo_mode = RTW89_MLO_MODE_MLSR;
+
+wake_queue:
+	ieee80211_wake_queues(rtwdev->hw);
+
+	return ret;
+}
+
 static int rtw89_chip_efuse_info_setup(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
@@ -5110,8 +5587,6 @@ static int rtw89_chip_efuse_info_setup(struct rtw89_dev *rtwdev)
 
 	rtw89_hci_mac_pre_deinit(rtwdev);
 
-	rtw89_mac_pwr_off(rtwdev);
-
 	return 0;
 }
 
@@ -5192,36 +5667,45 @@ int rtw89_chip_info_setup(struct rtw89_dev *rtwdev)
 
 	rtw89_read_chip_ver(rtwdev);
 
+	ret = rtw89_mac_pwr_on(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to power on\n");
+		return ret;
+	}
+
 	ret = rtw89_wait_firmware_completion(rtwdev);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to wait firmware completion\n");
-		return ret;
+		goto out;
 	}
 
 	ret = rtw89_fw_recognize(rtwdev);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to recognize firmware\n");
-		return ret;
+		goto out;
 	}
 
 	ret = rtw89_chip_efuse_info_setup(rtwdev);
 	if (ret)
-		return ret;
+		goto out;
 
 	ret = rtw89_fw_recognize_elements(rtwdev);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to recognize firmware elements\n");
-		return ret;
+		goto out;
 	}
 
 	ret = rtw89_chip_board_info_setup(rtwdev);
 	if (ret)
-		return ret;
+		goto out;
 
 	rtw89_core_setup_rfe_parms(rtwdev);
 	rtwdev->ps_mode = rtw89_update_ps_mode(rtwdev);
 
-	return 0;
+out:
+	rtw89_mac_pwr_off(rtwdev);
+
+	return ret;
 }
 EXPORT_SYMBOL(rtw89_chip_info_setup);
 
@@ -5256,6 +5740,9 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	int ret;
 	int tx_headroom = IEEE80211_HT_CTL_LEN;
 
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_USB)
+		tx_headroom += chip->txwd_body_size + chip->txwd_info_size;
+
 	hw->vif_data_size = struct_size_t(struct rtw89_vif, links_inst, n);
 	hw->sta_data_size = struct_size_t(struct rtw89_sta, links_inst, n);
 	hw->txq_data_size = sizeof(struct rtw89_txq);
@@ -5287,6 +5774,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS);
 	ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID);
 	ieee80211_hw_set(hw, WANT_MONITOR_VIF);
+	ieee80211_hw_set(hw, CHANCTX_STA_CSA);
 
 	if (chip->support_bandwidths & BIT(NL80211_CHAN_WIDTH_160))
 		ieee80211_hw_set(hw, SUPPORTS_VHT_EXT_NSS_BW);
@@ -5313,6 +5801,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS |
 			    WIPHY_FLAG_TDLS_EXTERNAL_SETUP |
 			    WIPHY_FLAG_AP_UAPSD |
+			    WIPHY_FLAG_HAS_CHANNEL_SWITCH |
 			    WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK;
 
 	if (!chip->support_rnr)
@@ -5321,8 +5810,11 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	if (chip->chip_gen == RTW89_CHIP_BE)
 		hw->wiphy->flags |= WIPHY_FLAG_DISABLE_WEXT;
 
-	if (rtwdev->support_mlo)
+	if (rtwdev->support_mlo) {
 		hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_MLO;
+		hw->wiphy->iftype_ext_capab = rtw89_iftypes_ext_capa;
+		hw->wiphy->num_iftype_ext_capab = ARRAY_SIZE(rtw89_iftypes_ext_capa);
+	}
 
 	hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR;
 
@@ -5353,7 +5845,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	ret = rtw89_regd_setup(rtwdev);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to set up regd\n");
-		goto err_free_supported_band;
+		return ret;
 	}
 
 	hw->wiphy->sar_capa = &rtw89_sar_capa;
@@ -5361,10 +5853,10 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	ret = ieee80211_register_hw(hw);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to register hw\n");
-		goto err_free_supported_band;
+		return ret;
 	}
 
-	ret = rtw89_regd_init(rtwdev, rtw89_regd_notifier);
+	ret = rtw89_regd_init_hint(rtwdev);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to init regd\n");
 		goto err_unregister_hw;
@@ -5376,8 +5868,6 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 
 err_unregister_hw:
 	ieee80211_unregister_hw(hw);
-err_free_supported_band:
-	rtw89_core_clr_supported_band(rtwdev);
 
 	return ret;
 }
@@ -5388,7 +5878,6 @@ static void rtw89_core_unregister_hw(struct rtw89_dev *rtwdev)
 
 	rtw89_rfkill_polling_deinit(rtwdev);
 	ieee80211_unregister_hw(hw);
-	rtw89_core_clr_supported_band(rtwdev);
 }
 
 int rtw89_core_register(struct rtw89_dev *rtwdev)
@@ -5456,13 +5945,13 @@ struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device,
 	if (!hw)
 		goto err;
 
-	/* TODO: When driver MLO arch. is done, determine whether to support MLO
-	 * according to the following conditions.
-	 * 1. run with chanctx_ops
-	 * 2. chip->support_link_num != 0
-	 * 3. FW feature supports AP_LINK_PS
+	/* Currently, our AP_LINK_PS handling only works for non-MLD softap
+	 * or MLD-single-link softap. If RTW89_MLD_NON_STA_LINK_NUM enlarges,
+	 * please tweak entire AP_LINKS_PS handling before supporting MLO.
 	 */
-	support_mlo = false;
+	support_mlo = !no_chanctx && chip->support_link_num &&
+		      RTW89_CHK_FW_FEATURE(NOTIFY_AP_INFO, &early_fw) &&
+		      RTW89_MLD_NON_STA_LINK_NUM == 1;
 
 	hw->wiphy->iface_combinations = rtw89_iface_combs;
 
diff --git a/sys/contrib/dev/rtw89/core.h b/sys/contrib/dev/rtw89/core.h
index 1f1113fbfc75..1aead91341d1 100644
--- a/sys/contrib/dev/rtw89/core.h
+++ b/sys/contrib/dev/rtw89/core.h
@@ -23,11 +23,13 @@ struct rtw89_dev;
 struct rtw89_pci_info;
 struct rtw89_mac_gen_def;
 struct rtw89_phy_gen_def;
+struct rtw89_fw_blacklist;
 struct rtw89_efuse_block_cfg;
 struct rtw89_h2c_rf_tssi;
 struct rtw89_fw_txpwr_track_cfg;
 struct rtw89_phy_rfk_log_fmt;
 struct rtw89_debugfs;
+struct rtw89_regd_data;
 
 extern const struct ieee80211_ops rtw89_ops;
 
@@ -44,6 +46,7 @@ extern const struct ieee80211_ops rtw89_ops;
 #define BYPASS_CR_DATA 0xbabecafe
 
 #define RTW89_TRACK_WORK_PERIOD	round_jiffies_relative(HZ * 2)
+#define RTW89_TRACK_PS_WORK_PERIOD msecs_to_jiffies(100)
 #define RTW89_FORBID_BA_TIMER round_jiffies_relative(HZ * 4)
 #define CFO_TRACK_MAX_USER 64
 #define MAX_RSSI 110
@@ -134,6 +137,17 @@ enum rtw89_hci_type {
 	RTW89_HCI_TYPE_PCIE,
 	RTW89_HCI_TYPE_USB,
 	RTW89_HCI_TYPE_SDIO,
+
+	RTW89_HCI_TYPE_NUM,
+};
+
+enum rtw89_hci_dle_type {
+	RTW89_HCI_DLE_TYPE_PCIE,
+	RTW89_HCI_DLE_TYPE_USB2,
+	RTW89_HCI_DLE_TYPE_USB3,
+	RTW89_HCI_DLE_TYPE_SDIO,
+
+	RTW89_HCI_DLE_TYPE_NUM,
 };
 
 enum rtw89_core_chip_id {
@@ -724,6 +738,7 @@ enum rtw89_ofdma_type {
 	RTW89_OFDMA_NUM,
 };
 
+/* neither insert new in the middle, nor change any given definition */
 enum rtw89_regulation_type {
 	RTW89_WW	= 0,
 	RTW89_ETSI	= 1,
@@ -801,6 +816,7 @@ struct rtw89_rx_phy_ppdu {
 	u8 rssi[RF_PATH_MAX];
 	u8 mac_id;
 	u8 chan_idx;
+	u8 phy_idx;
 	u8 ie;
 	u16 rate;
 	u8 rpl_avg;
@@ -832,7 +848,7 @@ enum rtw89_mac_idx {
 enum rtw89_phy_idx {
 	RTW89_PHY_0 = 0,
 	RTW89_PHY_1 = 1,
-	RTW89_PHY_MAX
+	RTW89_PHY_NUM,
 };
 
 #define __RTW89_MLD_MAX_LINK_NUM 2
@@ -1177,6 +1193,7 @@ struct rtw89_tx_desc_info {
 	bool ldpc;
 	bool upd_wlan_hdr;
 	bool mlo;
+	bool sw_mld;
 };
 
 struct rtw89_core_tx_request {
@@ -1206,7 +1223,7 @@ struct rtw89_mac_ax_gnt {
 struct rtw89_mac_ax_wl_act {
 	u8 wlan_act_en;
 	u8 wlan_act;
-};
+} __packed;
 
 #define RTW89_MAC_AX_COEX_GNT_NR 2
 struct rtw89_mac_ax_coex_gnt {
@@ -1323,6 +1340,7 @@ enum rtw89_btc_bt_state_cnt {
 	BTC_BCNT_POLUT_NOW,
 	BTC_BCNT_POLUT_DIFF,
 	BTC_BCNT_RATECHG,
+	BTC_BCNT_BTTXPWR_UPDATE,
 	BTC_BCNT_NUM,
 };
 
@@ -1381,6 +1399,11 @@ struct rtw89_btc_wl_smap {
 	u32 emlsr: 1;
 };
 
+enum rtw89_tfc_interval {
+	RTW89_TFC_INTERVAL_100MS,
+	RTW89_TFC_INTERVAL_2SEC,
+};
+
 enum rtw89_tfc_lv {
 	RTW89_TFC_IDLE,
 	RTW89_TFC_ULTRA_LOW,
@@ -1389,7 +1412,6 @@ enum rtw89_tfc_lv {
 	RTW89_TFC_HIGH,
 };
 
-#define RTW89_TP_SHIFT 18 /* bytes/2s --> Mbps */
 DECLARE_EWMA(tp, 10, 2);
 
 struct rtw89_traffic_stats {
@@ -1546,16 +1568,35 @@ struct rtw89_btc_u8_sta_chg {
 };
 
 struct rtw89_btc_wl_scan_info {
-	u8 band[RTW89_PHY_MAX];
+	u8 band[RTW89_PHY_NUM];
 	u8 phy_map;
 	u8 rsvd;
 };
 
 struct rtw89_btc_wl_dbcc_info {
-	u8 op_band[RTW89_PHY_MAX]; /* op band in each phy */
-	u8 scan_band[RTW89_PHY_MAX]; /* scan band in  each phy */
-	u8 real_band[RTW89_PHY_MAX];
-	u8 role[RTW89_PHY_MAX]; /* role in each phy */
+	u8 op_band[RTW89_PHY_NUM]; /* op band in each phy */
+	u8 scan_band[RTW89_PHY_NUM]; /* scan band in  each phy */
+	u8 real_band[RTW89_PHY_NUM];
+	u8 role[RTW89_PHY_NUM]; /* role in each phy */
+};
+
+struct rtw89_btc_wl_mlo_info {
+	u8 wmode[RTW89_PHY_NUM]; /* enum phl_mr_wmode */
+	u8 ch_type[RTW89_PHY_NUM]; /* enum phl_mr_ch_type */
+	u8 hwb_rf_band[RTW89_PHY_NUM]; /* enum band_type, RF-band for HW-band */
+	u8 path_rf_band[RTW89_PHY_NUM]; /* enum band_type, RF-band for PHY0/1 */
+
+	u8 wtype; /* enum phl_mr_wtype */
+	u8 mrcx_mode;
+	u8 mrcx_act_hwb_map;
+	u8 mrcx_bt_slot_rsp;
+
+	u8 rf_combination; /* enum btc_mlo_rf_combin 0:2+0, 1:0+2, 2:1+1,3:2+2 */
+	u8 mlo_en; /* MLO enable */
+	u8 mlo_adie; /* a-die count */
+	u8 dual_hw_band_en; /* both 2 HW-band link exist */
+
+	u32 link_status; /* enum mlo_dbcc_mode_type */
 };
 
 struct rtw89_btc_wl_active_role {
@@ -1767,7 +1808,8 @@ struct rtw89_btc_wl_rfk_info {
 	u32 phy_map: 2;
 	u32 band: 2;
 	u32 type: 8;
-	u32 rsvd: 14;
+	u32 con_rfk: 1;
+	u32 rsvd: 13;
 
 	u32 start_time;
 	u32 proc_time;
@@ -1791,6 +1833,13 @@ union rtw89_btc_bt_state_map {
 #define BTC_BT_AFH_GROUP 12
 #define BTC_BT_AFH_LE_GROUP 5
 
+struct rtw89_btc_bt_txpwr_desc {
+	s8 br_dbm;
+	s8 le_dbm;
+	u8 br_gain_index;
+	u8 le_gain_index;
+};
+
 struct rtw89_btc_bt_link_info {
 	struct rtw89_btc_u8_sta_chg profile_cnt;
 	struct rtw89_btc_bool_sta_chg multi_link;
@@ -1800,6 +1849,7 @@ struct rtw89_btc_bt_link_info {
 	struct rtw89_btc_bt_a2dp_desc a2dp_desc;
 	struct rtw89_btc_bt_pan_desc pan_desc;
 	union rtw89_btc_bt_state_map status;
+	struct rtw89_btc_bt_txpwr_desc bt_txpwr_desc;
 
 	u8 sut_pwr_level[BTC_PROFILE_MAX];
 	u8 golden_rx_shift[BTC_PROFILE_MAX];
@@ -1895,6 +1945,7 @@ struct rtw89_btc_wl_info {
 	struct rtw89_btc_wl_role_info_v8 role_info_v8;
 	struct rtw89_btc_wl_scan_info scan_info;
 	struct rtw89_btc_wl_dbcc_info dbcc_info;
+	struct rtw89_btc_wl_mlo_info mlo_info;
 	struct rtw89_btc_rf_para rf_para;
 	struct rtw89_btc_wl_nhm nhm;
 	union rtw89_btc_wl_state_map status;
@@ -1904,15 +1955,19 @@ struct rtw89_btc_wl_info {
 	u8 cn_report;
 	u8 coex_mode;
 	u8 pta_req_mac;
-	u8 bt_polut_type[RTW89_PHY_MAX]; /* BT polluted WL-Tx type for phy0/1  */
+	u8 bt_polut_type[RTW89_PHY_NUM]; /* BT polluted WL-Tx type for phy0/1  */
 
 	bool is_5g_hi_channel;
+	bool go_client_exist;
+	bool noa_exist;
 	bool pta_reg_mac_chg;
 	bool bg_mode;
 	bool he_mode;
 	bool scbd_change;
 	bool fw_ver_mismatch;
 	bool client_cnt_inc_2g;
+	bool link_mode_chg;
+	bool dbcc_chg;
 	u32 scbd;
 };
 
@@ -2065,6 +2120,7 @@ struct rtw89_btc_bt_info {
 	union rtw89_btc_bt_rfk_info_map rfk_info;
 
 	u8 raw_info[BTC_BTINFO_MAX]; /* raw bt info from mailbox */
+	u8 txpwr_info[BTC_BTINFO_MAX];
 	u8 rssi_level;
 
 	u32 scbd;
@@ -2236,7 +2292,7 @@ struct rtw89_btc_fbtc_rpt_ctrl_v4 {
 	struct rtw89_btc_fbtc_rpt_ctrl_wl_fw_info wl_fw_info;
 	struct rtw89_btc_fbtc_rpt_ctrl_bt_mailbox bt_mbx_info;
 	__le32 bt_cnt[BTC_BCNT_STA_MAX];
-	struct rtw89_mac_ax_gnt gnt_val[RTW89_PHY_MAX];
+	struct rtw89_mac_ax_gnt gnt_val[RTW89_PHY_NUM];
 } __packed;
 
 struct rtw89_btc_fbtc_rpt_ctrl_v5 {
@@ -2244,7 +2300,7 @@ struct rtw89_btc_fbtc_rpt_ctrl_v5 {
 	u8 rsvd;
 	__le16 rsvd1;
 
-	u8 gnt_val[RTW89_PHY_MAX][4];
+	u8 gnt_val[RTW89_PHY_NUM][4];
 	__le16 bt_cnt[BTC_BCNT_STA_MAX];
 
 	struct rtw89_btc_fbtc_rpt_ctrl_info_v5 rpt_info;
@@ -2256,7 +2312,7 @@ struct rtw89_btc_fbtc_rpt_ctrl_v105 {
 	u8 rsvd;
 	__le16 rsvd1;
 
-	u8 gnt_val[RTW89_PHY_MAX][4];
+	u8 gnt_val[RTW89_PHY_NUM][4];
 	__le16 bt_cnt[BTC_BCNT_STA_MAX_V105];
 
 	struct rtw89_btc_fbtc_rpt_ctrl_info_v5 rpt_info;
@@ -2269,7 +2325,7 @@ struct rtw89_btc_fbtc_rpt_ctrl_v7 {
 	u8 rsvd1;
 	u8 rsvd2;
 
-	u8 gnt_val[RTW89_PHY_MAX][4];
+	u8 gnt_val[RTW89_PHY_NUM][4];
 	__le16 bt_cnt[BTC_BCNT_STA_MAX_V105];
 
 	struct rtw89_btc_fbtc_rpt_ctrl_info_v8 rpt_info;
@@ -2282,7 +2338,7 @@ struct rtw89_btc_fbtc_rpt_ctrl_v8 {
 	u8 rpt_len_max_l; /* BTC_RPT_MAX bit0~7 */
 	u8 rpt_len_max_h; /* BTC_RPT_MAX bit8~15 */
 
-	u8 gnt_val[RTW89_PHY_MAX][4];
+	u8 gnt_val[RTW89_PHY_NUM][4];
 	__le16 bt_cnt[BTC_BCNT_STA_MAX_V105];
 
 	struct rtw89_btc_fbtc_rpt_ctrl_info_v8 rpt_info;
@@ -2903,12 +2959,32 @@ struct rtw89_btc_trx_info {
 	u32 rx_err_ratio;
 };
 
+enum btc_rf_path {
+	BTC_RF_S0 = 0,
+	BTC_RF_S1 = 1,
+	BTC_RF_NUM,
+};
+
+struct rtw89_btc_fbtc_outsrc_set_info {
+	u8 rf_band[BTC_RF_NUM]; /* 0:2G, 1:non-2G */
+	u8 btg_rx[BTC_RF_NUM];
+	u8 nbtg_tx[BTC_RF_NUM];
+
+	struct rtw89_mac_ax_gnt gnt_set[BTC_RF_NUM]; /* refer to btc_gnt_ctrl */
+	struct rtw89_mac_ax_wl_act wlact_set[BTC_RF_NUM]; /* BT0/BT1 */
+
+	u8 pta_req_hw_band;
+	u8 rf_gbt_source;
+} __packed;
+
 union rtw89_btc_fbtc_slot_u {
 	struct rtw89_btc_fbtc_slot v1[CXST_MAX];
 	struct rtw89_btc_fbtc_slot_v7 v7[CXST_MAX];
 };
 
 struct rtw89_btc_dm {
+	struct rtw89_btc_fbtc_outsrc_set_info ost_info_last; /* outsrc API setup info */
+	struct rtw89_btc_fbtc_outsrc_set_info ost_info; /* outsrc API setup info */
 	union rtw89_btc_fbtc_slot_u slot;
 	union rtw89_btc_fbtc_slot_u slot_now;
 	struct rtw89_btc_fbtc_tdma tdma;
@@ -2998,6 +3074,7 @@ enum rtw89_btc_btf_fw_event {
 	BTF_EVNT_BT_LEAUDIO_INFO = 7, /* fwc2hfunc > 1 */
 	BTF_EVNT_BUF_OVERFLOW,
 	BTF_EVNT_C2H_LOOPBACK,
+	BTF_EVNT_BT_QUERY_TXPWR, /* fwc2hfunc > 3 */
 	BTF_EVNT_MAX,
 };
 
@@ -3046,6 +3123,7 @@ struct rtw89_btc_rpt_cmn_info {
 union rtw89_btc_fbtc_btafh_info {
 	struct rtw89_btc_fbtc_btafh v1;
 	struct rtw89_btc_fbtc_btafh_v2 v2;
+	struct rtw89_btc_fbtc_btafh_v7 v7;
 };
 
 struct rtw89_btc_report_ctrl_state {
@@ -3115,31 +3193,6 @@ enum rtw89_btc_btfre_type {
 	BTFRE_MAX,
 };
 
-struct rtw89_btc_btf_fwinfo {
-	u32 cnt_c2h;
-	u32 cnt_h2c;
-	u32 cnt_h2c_fail;
-	u32 event[BTF_EVNT_MAX];
-
-	u32 err[BTFRE_MAX];
-	u32 len_mismch;
-	u32 fver_mismch;
-	u32 rpt_en_map;
-
-	struct rtw89_btc_report_ctrl_state rpt_ctrl;
-	struct rtw89_btc_rpt_fbtc_tdma rpt_fbtc_tdma;
-	struct rtw89_btc_rpt_fbtc_slots rpt_fbtc_slots;
-	struct rtw89_btc_rpt_fbtc_cysta rpt_fbtc_cysta;
-	struct rtw89_btc_rpt_fbtc_step rpt_fbtc_step;
-	struct rtw89_btc_rpt_fbtc_nullsta rpt_fbtc_nullsta;
-	struct rtw89_btc_rpt_fbtc_mreg rpt_fbtc_mregval;
-	struct rtw89_btc_rpt_fbtc_gpio_dbg rpt_fbtc_gpio_dbg;
-	struct rtw89_btc_rpt_fbtc_btver rpt_fbtc_btver;
-	struct rtw89_btc_rpt_fbtc_btscan rpt_fbtc_btscan;
-	struct rtw89_btc_rpt_fbtc_btafh rpt_fbtc_btafh;
-	struct rtw89_btc_rpt_fbtc_btdev rpt_fbtc_btdev;
-};
-
 struct rtw89_btc_ver {
 	enum rtw89_core_chip_id chip_id;
 	u32 fw_ver_code;
@@ -3166,6 +3219,35 @@ struct rtw89_btc_ver {
 	u8 drvinfo_type;
 	u16 info_buf;
 	u8 max_role_num;
+	u8 fcxosi;
+	u8 fcxmlo;
+	u8 bt_desired;
+};
+
+struct rtw89_btc_btf_fwinfo {
+	u32 cnt_c2h;
+	u32 cnt_h2c;
+	u32 cnt_h2c_fail;
+	u32 event[BTF_EVNT_MAX];
+
+	u32 err[BTFRE_MAX];
+	u32 len_mismch;
+	u32 fver_mismch;
+	u32 rpt_en_map;
+
+	struct rtw89_btc_ver fw_subver;
+	struct rtw89_btc_report_ctrl_state rpt_ctrl;
+	struct rtw89_btc_rpt_fbtc_tdma rpt_fbtc_tdma;
+	struct rtw89_btc_rpt_fbtc_slots rpt_fbtc_slots;
+	struct rtw89_btc_rpt_fbtc_cysta rpt_fbtc_cysta;
+	struct rtw89_btc_rpt_fbtc_step rpt_fbtc_step;
+	struct rtw89_btc_rpt_fbtc_nullsta rpt_fbtc_nullsta;
+	struct rtw89_btc_rpt_fbtc_mreg rpt_fbtc_mregval;
+	struct rtw89_btc_rpt_fbtc_gpio_dbg rpt_fbtc_gpio_dbg;
+	struct rtw89_btc_rpt_fbtc_btver rpt_fbtc_btver;
+	struct rtw89_btc_rpt_fbtc_btscan rpt_fbtc_btscan;
+	struct rtw89_btc_rpt_fbtc_btafh rpt_fbtc_btafh;
+	struct rtw89_btc_rpt_fbtc_btdev rpt_fbtc_btdev;
 };
 
 #define RTW89_BTC_POLICY_MAXLEN 512
@@ -3180,10 +3262,10 @@ struct rtw89_btc {
 	struct rtw89_btc_btf_fwinfo fwinfo;
 	struct rtw89_btc_dbg dbg;
 
-	struct work_struct eapol_notify_work;
-	struct work_struct arp_notify_work;
-	struct work_struct dhcp_notify_work;
-	struct work_struct icmp_notify_work;
+	struct wiphy_work eapol_notify_work;
+	struct wiphy_work arp_notify_work;
+	struct wiphy_work dhcp_notify_work;
+	struct wiphy_work icmp_notify_work;
 
 	u32 bt_req_len;
 
@@ -3380,9 +3462,11 @@ struct rtw89_sec_cam_entry {
 
 struct rtw89_sta_link {
 	struct rtw89_sta *rtwsta;
+	struct list_head dlink_schd;
 	unsigned int link_id;
 
 	u8 mac_id;
+	u8 tx_retry;
 	bool er_cap;
 	struct rtw89_vif_link *rtwvif_link;
 	struct rtw89_ra_info ra;
@@ -3418,6 +3502,7 @@ struct rtw89_efuse {
 	u8 addr[ETH_ALEN];
 	u8 rfe_type;
 	char country_code[2];
+	u8 adc_td;
 };
 
 struct rtw89_phy_rate_pattern {
@@ -3438,6 +3523,8 @@ struct rtw89_tx_skb_data {
 	u8 hci_priv[];
 };
 
+#define RTW89_SCAN_NULL_TIMEOUT 30
+
 #define RTW89_ROC_IDLE_TIMEOUT 500
 #define RTW89_ROC_TX_TIMEOUT 30
 enum rtw89_roc_state {
@@ -3446,14 +3533,13 @@ enum rtw89_roc_state {
 	RTW89_ROC_MGMT,
 };
 
-#define RTW89_ROC_BY_LINK_INDEX 0
-
 struct rtw89_roc {
 	struct ieee80211_channel chan;
-	struct delayed_work roc_work;
+	struct wiphy_delayed_work roc_work;
 	enum ieee80211_roc_type type;
 	enum rtw89_roc_state state;
 	int duration;
+	unsigned int link_id;
 };
 
 #define RTW89_P2P_MAX_NOA_NUM 2
@@ -3484,8 +3570,17 @@ struct rtw89_p2p_noa_setter {
 	u8 noa_index;
 };
 
+struct rtw89_ps_noa_once_handler {
+	bool in_duration;
+	u64 tsf_begin;
+	u64 tsf_end;
+	struct wiphy_delayed_work set_work;
+	struct wiphy_delayed_work clr_work;
+};
+
 struct rtw89_vif_link {
 	struct rtw89_vif *rtwvif;
+	struct list_head dlink_schd;
 	unsigned int link_id;
 
 	bool chanctx_assigned; /* only valid when running with chanctx_ops */
@@ -3504,9 +3599,12 @@ struct rtw89_vif_link {
 	u8 self_role;
 	u8 wmm;
 	u8 bcn_hit_cond;
+	u8 bcn_bw_idx;
 	u8 hit_rule;
 	u8 last_noa_nr;
 	u64 sync_bcn_tsf;
+	u64 last_sync_bcn_tsf;
+	bool rand_tsf_done;
 	bool trigger;
 	bool lsig_txop;
 	u8 tgt_ind;
@@ -3520,13 +3618,17 @@ struct rtw89_vif_link {
 	bool pre_pwr_diff_en;
 	bool pwr_diff_en;
 	u8 def_tri_idx;
-	struct work_struct update_beacon_work;
+	struct wiphy_work update_beacon_work;
+	struct wiphy_delayed_work csa_beacon_work;
 	struct rtw89_addr_cam_entry addr_cam;
 	struct rtw89_bssid_cam_entry bssid_cam;
 	struct ieee80211_tx_queue_params tx_params[IEEE80211_NUM_ACS];
 	struct rtw89_phy_rate_pattern rate_pattern;
 	struct list_head general_pkt_list;
 	struct rtw89_p2p_noa_setter p2p_noa;
+	struct rtw89_ps_noa_once_handler noa_once;
+	struct wiphy_delayed_work mcc_gc_detect_beacon_work;
+	u8 detect_bcn_count;
 };
 
 enum rtw89_lv1_rcvy_step {
@@ -3583,6 +3685,7 @@ struct rtw89_hci_ops {
 struct rtw89_hci_info {
 	const struct rtw89_hci_ops *ops;
 	enum rtw89_hci_type type;
+	enum rtw89_hci_dle_type dle_type;
 	u32 rpwm_addr;
 	u32 cpwm_addr;
 	bool paused;
@@ -3632,6 +3735,8 @@ struct rtw89_chip_ops {
 			       enum rtw89_phy_idx phy_idx);
 	int (*init_txpwr_unit)(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx);
 	u8 (*get_thermal)(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path);
+	u32 (*chan_to_rf18_val)(struct rtw89_dev *rtwdev,
+				const struct rtw89_chan *chan);
 	void (*ctrl_btg_bt_rx)(struct rtw89_dev *rtwdev, bool en,
 			       enum rtw89_phy_idx phy_idx);
 	void (*query_ppdu)(struct rtw89_dev *rtwdev,
@@ -3678,6 +3783,11 @@ struct rtw89_chip_ops {
 	int (*h2c_ampdu_cmac_tbl)(struct rtw89_dev *rtwdev,
 				  struct rtw89_vif_link *rtwvif_link,
 				  struct rtw89_sta_link *rtwsta_link);
+	int (*h2c_txtime_cmac_tbl)(struct rtw89_dev *rtwdev,
+				   struct rtw89_sta_link *rtwsta_link);
+	int (*h2c_punctured_cmac_tbl)(struct rtw89_dev *rtwdev,
+				      struct rtw89_vif_link *rtwvif_link,
+				      u16 punctured);
 	int (*h2c_default_dmac_tbl)(struct rtw89_dev *rtwdev,
 				    struct rtw89_vif_link *rtwvif_link,
 				    struct rtw89_sta_link *rtwsta_link);
@@ -3762,7 +3872,7 @@ struct rtw89_scan_option {
 	u16 slow_pd;
 	u16 norm_cy;
 	u8 opch_end;
-	u16 delay;
+	u16 delay; /* in unit of ms */
 	u64 prohib_chan;
 	enum rtw89_phy_idx band;
 	enum rtw89_scan_be_operation operation;
@@ -3984,7 +4094,11 @@ struct rtw89_rfe_parms {
 	struct rtw89_txpwr_rule_2ghz rule_2ghz;
 	struct rtw89_txpwr_rule_5ghz rule_5ghz;
 	struct rtw89_txpwr_rule_6ghz rule_6ghz;
+	struct rtw89_txpwr_rule_2ghz rule_da_2ghz;
+	struct rtw89_txpwr_rule_5ghz rule_da_5ghz;
+	struct rtw89_txpwr_rule_6ghz rule_da_6ghz;
 	struct rtw89_tx_shape tx_shape;
+	bool has_da;
 };
 
 struct rtw89_rfe_parms_conf {
@@ -4089,9 +4203,15 @@ struct rtw89_rfe_data {
 	struct rtw89_txpwr_lmt_2ghz_data lmt_2ghz;
 	struct rtw89_txpwr_lmt_5ghz_data lmt_5ghz;
 	struct rtw89_txpwr_lmt_6ghz_data lmt_6ghz;
+	struct rtw89_txpwr_lmt_2ghz_data da_lmt_2ghz;
+	struct rtw89_txpwr_lmt_5ghz_data da_lmt_5ghz;
+	struct rtw89_txpwr_lmt_6ghz_data da_lmt_6ghz;
 	struct rtw89_txpwr_lmt_ru_2ghz_data lmt_ru_2ghz;
 	struct rtw89_txpwr_lmt_ru_5ghz_data lmt_ru_5ghz;
 	struct rtw89_txpwr_lmt_ru_6ghz_data lmt_ru_6ghz;
+	struct rtw89_txpwr_lmt_ru_2ghz_data da_lmt_ru_2ghz;
+	struct rtw89_txpwr_lmt_ru_5ghz_data da_lmt_ru_5ghz;
+	struct rtw89_txpwr_lmt_ru_6ghz_data da_lmt_ru_6ghz;
 	struct rtw89_tx_shape_lmt_data tx_shape_lmt;
 	struct rtw89_tx_shape_lmt_ru_data tx_shape_lmt_ru;
 	struct rtw89_rfe_parms rfe_parms;
@@ -4205,10 +4325,12 @@ struct rtw89_edcca_regs {
 	u32 edcca_p_mask;
 	u32 ppdu_level;
 	u32 ppdu_mask;
-	u32 rpt_a;
-	u32 rpt_b;
-	u32 rpt_sel;
-	u32 rpt_sel_mask;
+	struct rtw89_edcca_p_regs {
+		u32 rpt_a;
+		u32 rpt_b;
+		u32 rpt_sel;
+		u32 rpt_sel_mask;
+	} p[RTW89_PHY_NUM];
 	u32 rpt_sel_be;
 	u32 rpt_sel_be_mask;
 	u32 tx_collision_t2r_st;
@@ -4247,6 +4369,7 @@ enum rtw89_chanctx_state {
 enum rtw89_chanctx_callbacks {
 	RTW89_CHANCTX_CALLBACK_PLACEHOLDER,
 	RTW89_CHANCTX_CALLBACK_RFK,
+	RTW89_CHANCTX_CALLBACK_TAS,
 
 	NUM_OF_RTW89_CHANCTX_CALLBACKS,
 };
@@ -4267,14 +4390,15 @@ struct rtw89_chip_info {
 	bool try_ce_fw;
 	u8 bbmcu_nr;
 	u32 needed_fw_elms;
+	const struct rtw89_fw_blacklist *fw_blacklist;
 	u32 fifo_size;
 	bool small_fifo_size;
 	u32 dle_scc_rsvd_size;
 	u16 max_amsdu_limit;
 	bool dis_2g_40m_ul_ofdma;
 	u32 rsvd_ple_ofst;
-	const struct rtw89_hfc_param_ini *hfc_param_ini;
-	const struct rtw89_dle_mem *dle_mem;
+	const struct rtw89_hfc_param_ini *hfc_param_ini[RTW89_HCI_TYPE_NUM];
+	const struct rtw89_dle_mem *dle_mem[RTW89_HCI_DLE_TYPE_NUM];
 	u8 wde_qempty_acq_grpnum;
 	u8 wde_qempty_mgq_grpsel;
 	u32 rf_base_addr[2];
@@ -4287,10 +4411,15 @@ struct rtw89_chip_info {
 	bool support_unii4;
 	bool support_rnr;
 	bool support_ant_gain;
+	bool support_tas;
+	bool support_sar_by_ant;
 	bool ul_tb_waveform_ctrl;
 	bool ul_tb_pwr_diff;
+	bool rx_freq_frome_ie;
 	bool hw_sec_hdr;
 	bool hw_mgmt_tx_encrypt;
+	bool hw_tkip_crypto;
+	bool hw_mlo_bmc_crypto;
 	u8 rf_path_num;
 	u8 tx_nss;
 	u8 rx_nss;
@@ -4334,7 +4463,6 @@ struct rtw89_chip_info {
 
 	u32 para_ver;
 	u32 wlcx_desired;
-	u8 btcx_desired;
 	u8 scbd;
 	u8 mailbox;
 
@@ -4474,11 +4602,21 @@ enum rtw89_fw_type {
 	RTW89_FW_LOGFMT = 255,
 };
 
+#define RTW89_FW_FEATURE_GROUP(_grp, _features...) \
+	RTW89_FW_FEATURE_##_grp##_MIN, \
+	__RTW89_FW_FEATURE_##_grp##_S = RTW89_FW_FEATURE_##_grp##_MIN - 1, \
+	_features \
+	__RTW89_FW_FEATURE_##_grp##_E, \
+	RTW89_FW_FEATURE_##_grp##_MAX = __RTW89_FW_FEATURE_##_grp##_E - 1
+
 enum rtw89_fw_feature {
 	RTW89_FW_FEATURE_OLD_HT_RA_FORMAT,
 	RTW89_FW_FEATURE_SCAN_OFFLOAD,
 	RTW89_FW_FEATURE_TX_WAKE,
-	RTW89_FW_FEATURE_CRASH_TRIGGER,
+	RTW89_FW_FEATURE_GROUP(CRASH_TRIGGER,
+			       RTW89_FW_FEATURE_CRASH_TRIGGER_TYPE_0,
+			       RTW89_FW_FEATURE_CRASH_TRIGGER_TYPE_1,
+	),
 	RTW89_FW_FEATURE_NO_PACKET_DROP,
 	RTW89_FW_FEATURE_NO_DEEP_PS,
 	RTW89_FW_FEATURE_NO_LPS_PG,
@@ -4489,11 +4627,17 @@ enum rtw89_fw_feature {
 	RTW89_FW_FEATURE_RFK_PRE_NOTIFY_V0,
 	RTW89_FW_FEATURE_RFK_PRE_NOTIFY_V1,
 	RTW89_FW_FEATURE_RFK_RXDCK_V0,
+	RTW89_FW_FEATURE_RFK_IQK_V0,
 	RTW89_FW_FEATURE_NO_WOW_CPU_IO_RX,
 	RTW89_FW_FEATURE_NOTIFY_AP_INFO,
 	RTW89_FW_FEATURE_CH_INFO_BE_V0,
 	RTW89_FW_FEATURE_LPS_CH_INFO,
 	RTW89_FW_FEATURE_NO_PHYCAP_P1,
+	RTW89_FW_FEATURE_NO_POWER_DIFFERENCE,
+	RTW89_FW_FEATURE_BEACON_LOSS_COUNT_V1,
+	RTW89_FW_FEATURE_SCAN_OFFLOAD_EXTRA_OP,
+	RTW89_FW_FEATURE_RFK_NTFY_MCC_V0,
+	RTW89_FW_FEATURE_LPS_DACK_BY_C2H_REG,
 };
 
 struct rtw89_fw_suit {
@@ -4552,6 +4696,7 @@ struct rtw89_fw_elm_info {
 	struct rtw89_phy_table *rf_nctl;
 	struct rtw89_fw_txpwr_track_cfg *txpwr_trk;
 	struct rtw89_phy_rfk_log_fmt *rfk_log_fmt;
+	const struct rtw89_regd_data *regd;
 };
 
 enum rtw89_fw_mss_dev_type {
@@ -4590,6 +4735,10 @@ struct rtw89_fw_info {
 #define RTW89_CHK_FW_FEATURE(_feat, _fw) \
 	(!!((_fw)->feature_map & BIT(RTW89_FW_FEATURE_ ## _feat)))
 
+#define RTW89_CHK_FW_FEATURE_GROUP(_grp, _fw) \
+	(!!((_fw)->feature_map & GENMASK(RTW89_FW_FEATURE_ ## _grp ## _MAX, \
+					 RTW89_FW_FEATURE_ ## _grp ## _MIN)))
+
 #define RTW89_SET_FW_FEATURE(_fw_feature, _fw) \
 	((_fw)->feature_map |= BIT(_fw_feature))
 
@@ -4605,6 +4754,7 @@ struct rtw89_cam_info {
 enum rtw89_sar_sources {
 	RTW89_SAR_SOURCE_NONE,
 	RTW89_SAR_SOURCE_COMMON,
+	RTW89_SAR_SOURCE_ACPI,
 
 	RTW89_SAR_SOURCE_NR,
 };
@@ -4629,8 +4779,62 @@ struct rtw89_sar_cfg_common {
 	s32 cfg[RTW89_SAR_SUBBAND_NR];
 };
 
+enum rtw89_acpi_sar_subband {
+	RTW89_ACPI_SAR_2GHZ_SUBBAND,
+	RTW89_ACPI_SAR_5GHZ_SUBBAND_1,   /* U-NII-1 */
+	RTW89_ACPI_SAR_5GHZ_SUBBAND_2,   /* U-NII-2 */
+	RTW89_ACPI_SAR_5GHZ_SUBBAND_2E,  /* U-NII-2-Extended */
+	RTW89_ACPI_SAR_5GHZ_SUBBAND_3_4, /* U-NII-3 and U-NII-4 */
+	RTW89_ACPI_SAR_6GHZ_SUBBAND_5_L, /* U-NII-5 lower part */
+	RTW89_ACPI_SAR_6GHZ_SUBBAND_5_H, /* U-NII-5 higher part */
+	RTW89_ACPI_SAR_6GHZ_SUBBAND_6,   /* U-NII-6 */
+	RTW89_ACPI_SAR_6GHZ_SUBBAND_7_L, /* U-NII-7 lower part */
+	RTW89_ACPI_SAR_6GHZ_SUBBAND_7_H, /* U-NII-7 higher part */
+	RTW89_ACPI_SAR_6GHZ_SUBBAND_8,   /* U-NII-8 */
+
+	NUM_OF_RTW89_ACPI_SAR_SUBBAND,
+	RTW89_ACPI_SAR_SUBBAND_NR_LEGACY = RTW89_ACPI_SAR_5GHZ_SUBBAND_3_4 + 1,
+	RTW89_ACPI_SAR_SUBBAND_NR_HAS_6GHZ = RTW89_ACPI_SAR_6GHZ_SUBBAND_8 + 1,
+};
+
+#define TXPWR_FACTOR_OF_RTW89_ACPI_SAR 3 /* unit: 0.125 dBm */
+#define MAX_VAL_OF_RTW89_ACPI_SAR S16_MAX
+#define MIN_VAL_OF_RTW89_ACPI_SAR S16_MIN
+#define MAX_NUM_OF_RTW89_ACPI_SAR_TBL 6
+#define NUM_OF_RTW89_ACPI_SAR_RF_PATH (RF_PATH_B + 1)
+
+struct rtw89_sar_entry_from_acpi {
+	s16 v[NUM_OF_RTW89_ACPI_SAR_SUBBAND][NUM_OF_RTW89_ACPI_SAR_RF_PATH];
+};
+
+struct rtw89_sar_table_from_acpi {
+	/* If this table is active, must fill all fields according to either
+	 * configuration in BIOS or some default values for SAR to work well.
+	 */
+	struct rtw89_sar_entry_from_acpi entries[RTW89_REGD_NUM];
+};
+
+struct rtw89_sar_indicator_from_acpi {
+	bool enable_sync;
+	unsigned int fields;
+	u8 (*rfpath_to_antidx)(enum rtw89_rf_path rfpath);
+
+	/* Select among @tables of container, rtw89_sar_cfg_acpi, by path.
+	 * Not design with pointers since addresses will be invalid after
+	 * sync content with local container instance.
+	 */
+	u8 tblsel[NUM_OF_RTW89_ACPI_SAR_RF_PATH];
+};
+
+struct rtw89_sar_cfg_acpi {
+	u8 downgrade_2tx;
+	unsigned int valid_num;
+	struct rtw89_sar_table_from_acpi tables[MAX_NUM_OF_RTW89_ACPI_SAR_TBL];
+	struct rtw89_sar_indicator_from_acpi indicator;
+};
+
 struct rtw89_sar_info {
-	/* used to decide how to acces SAR cfg union */
+	/* used to decide how to access SAR cfg union */
 	enum rtw89_sar_sources src;
 
 	/* reserved for different knids of SAR cfg struct.
@@ -4638,6 +4842,7 @@ struct rtw89_sar_info {
 	 */
 	union {
 		struct rtw89_sar_cfg_common cfg_common;
+		struct rtw89_sar_cfg_acpi cfg_acpi;
 	};
 };
 
@@ -4667,33 +4872,49 @@ enum rtw89_ant_gain_domain_type {
 struct rtw89_ant_gain_info {
 	s8 offset[RTW89_ANT_GAIN_CHAIN_NUM][RTW89_ANT_GAIN_SUBBAND_NR];
 	u32 regd_enabled;
+	bool block_country;
 };
 
 struct rtw89_6ghz_span {
 	enum rtw89_sar_subband sar_subband_low;
 	enum rtw89_sar_subband sar_subband_high;
+	enum rtw89_acpi_sar_subband acpi_sar_subband_low;
+	enum rtw89_acpi_sar_subband acpi_sar_subband_high;
 	enum rtw89_ant_gain_subband ant_gain_subband_low;
 	enum rtw89_ant_gain_subband ant_gain_subband_high;
 };
 
 #define RTW89_SAR_SPAN_VALID(span) ((span)->sar_subband_high)
+#define RTW89_ACPI_SAR_SPAN_VALID(span) ((span)->acpi_sar_subband_high)
 #define RTW89_ANT_GAIN_SPAN_VALID(span) ((span)->ant_gain_subband_high)
 
 enum rtw89_tas_state {
 	RTW89_TAS_STATE_DPR_OFF,
 	RTW89_TAS_STATE_DPR_ON,
-	RTW89_TAS_STATE_DPR_FORBID,
+	RTW89_TAS_STATE_STATIC_SAR,
 };
 
-#define RTW89_TAS_MAX_WINDOW 50
+#define RTW89_TAS_TX_RATIO_WINDOW 6
+#define RTW89_TAS_TXPWR_WINDOW 180
 struct rtw89_tas_info {
-	s16 txpwr_history[RTW89_TAS_MAX_WINDOW];
-	s32 total_txpwr;
-	u8 cur_idx;
-	s8 dpr_gap;
-	s8 delta;
+	u16 tx_ratio_history[RTW89_TAS_TX_RATIO_WINDOW];
+	u64 txpwr_history[RTW89_TAS_TXPWR_WINDOW];
+	u8 enabled_countries;
+	u8 txpwr_head_idx;
+	u8 txpwr_tail_idx;
+	u8 tx_ratio_idx;
+	u16 total_tx_ratio;
+	u64 total_txpwr;
+	u64 instant_txpwr;
+	u32 window_size;
+	s8 dpr_on_threshold;
+	s8 dpr_off_threshold;
+	enum rtw89_tas_state backup_state;
 	enum rtw89_tas_state state;
+	bool keep_history;
+	bool block_regd;
 	bool enable;
+	bool pause;
 };
 
 struct rtw89_chanctx_cfg {
@@ -4751,6 +4972,8 @@ struct rtw89_edcca_bak {
 enum rtw89_dm_type {
 	RTW89_DM_DYNAMIC_EDCCA,
 	RTW89_DM_THERMAL_PROTECT,
+	RTW89_DM_TAS,
+	RTW89_DM_MLO,
 };
 
 #define RTW89_THERMAL_PROT_LV_MAX 5
@@ -4772,18 +4995,18 @@ struct rtw89_hal {
 	bool no_mcs_12_13;
 
 	atomic_t roc_chanctx_idx;
+	u8 roc_link_index;
 
 	DECLARE_BITMAP(changes, NUM_OF_RTW89_CHANCTX_CHANGES);
 	DECLARE_BITMAP(entity_map, NUM_OF_RTW89_CHANCTX);
 	struct rtw89_chanctx chanctx[NUM_OF_RTW89_CHANCTX];
 	struct cfg80211_chan_def roc_chandef;
 
-	bool entity_active[RTW89_PHY_MAX];
+	bool entity_active[RTW89_PHY_NUM];
 	bool entity_pause;
 	enum rtw89_entity_mode entity_mode;
 	struct rtw89_entity_mgnt entity_mgnt;
 
-	struct rtw89_edcca_bak edcca_bak;
 	u32 disabled_dm_bitmap; /* bitmap of enum rtw89_dm_type */
 
 	u8 thermal_prot_th;
@@ -4811,9 +5034,10 @@ enum rtw89_flags {
 	RTW89_FLAG_CRASH_SIMULATING,
 	RTW89_FLAG_SER_HANDLING,
 	RTW89_FLAG_WOWLAN,
-	RTW89_FLAG_FORBIDDEN_TRACK_WROK,
+	RTW89_FLAG_FORBIDDEN_TRACK_WORK,
 	RTW89_FLAG_CHANGING_INTERFACE,
 	RTW89_FLAG_HW_RFKILL_STATE,
+	RTW89_FLAG_UNPLUGGED,
 
 	NUM_OF_RTW89_FLAGS,
 };
@@ -4978,7 +5202,7 @@ struct rtw89_dpk_bkup_para {
 	enum rtw89_band band;
 	enum rtw89_bandwidth bw;
 	u8 ch;
-	bool path_ok;
+	u8 path_ok;
 	u8 mdpd_en;
 	u8 txagc_dpk;
 	u8 ther_dpk;
@@ -4989,7 +5213,7 @@ struct rtw89_dpk_bkup_para {
 struct rtw89_dpk_info {
 	bool is_dpk_enable;
 	bool is_dpk_reload_en;
-	u8 dpk_gs[RTW89_PHY_MAX];
+	u8 dpk_gs[RTW89_PHY_NUM];
 	u16 dc_i[RTW89_DPK_RF_PATH][RTW89_DPK_BKUP_NUM];
 	u16 dc_q[RTW89_DPK_RF_PATH][RTW89_DPK_BKUP_NUM];
 	u8 corr_val[RTW89_DPK_RF_PATH][RTW89_DPK_BKUP_NUM];
@@ -5056,8 +5280,10 @@ struct rtw89_dig_info {
 	s8 tia_gain_a[TIA_GAIN_NUM];
 	s8 tia_gain_g[TIA_GAIN_NUM];
 	s8 *tia_gain;
+	u32 bak_dig;
 	bool is_linked_pre;
 	bool bypass_dig;
+	bool pause_dig;
 };
 
 enum rtw89_multi_cfo_mode {
@@ -5151,7 +5377,7 @@ struct rtw89_tssi_info {
 	u32 alignment_backup_by_ch[RF_PATH_MAX][TSSI_MAX_CH_NUM][TSSI_ALIMK_VALUE_NUM];
 	u32 alignment_value[RF_PATH_MAX][TSSI_ALIMK_MAX][TSSI_ALIMK_VALUE_NUM];
 	bool alignment_done[RF_PATH_MAX][TSSI_ALIMK_MAX];
-	u32 tssi_alimk_time;
+	u64 tssi_alimk_time;
 };
 
 struct rtw89_power_trim_info {
@@ -5162,9 +5388,27 @@ struct rtw89_power_trim_info {
 	u8 pad_bias_trim[RF_PATH_MAX];
 };
 
+enum rtw89_regd_func {
+	RTW89_REGD_FUNC_TAS = 0, /* TAS (Time Average SAR) */
+	RTW89_REGD_FUNC_DAG = 1, /* DAG (Dynamic Antenna Gain) */
+
+	NUM_OF_RTW89_REGD_FUNC,
+};
+
 struct rtw89_regd {
 	char alpha2[3];
 	u8 txpwr_regd[RTW89_BAND_NUM];
+	DECLARE_BITMAP(func_bitmap, NUM_OF_RTW89_REGD_FUNC);
+};
+
+struct rtw89_regd_data {
+	unsigned int nr;
+	struct rtw89_regd map[] __counted_by(nr);
+};
+
+struct rtw89_regd_ctrl {
+	unsigned int nr;
+	const struct rtw89_regd *map;
 };
 
 #define RTW89_REGD_MAX_COUNTRY_NUM U8_MAX
@@ -5172,12 +5416,16 @@ struct rtw89_regd {
 #define RTW89_5GHZ_UNII4_START_INDEX 25
 
 struct rtw89_regulatory_info {
+	struct rtw89_regd_ctrl ctrl;
 	const struct rtw89_regd *regd;
 	enum rtw89_reg_6ghz_power reg_6ghz_power;
 	struct rtw89_reg_6ghz_tpe reg_6ghz_tpe;
+	bool txpwr_uk_follow_etsi;
+
 	DECLARE_BITMAP(block_unii4, RTW89_REGD_MAX_COUNTRY_NUM);
 	DECLARE_BITMAP(block_6ghz, RTW89_REGD_MAX_COUNTRY_NUM);
 	DECLARE_BITMAP(block_6ghz_sp, RTW89_REGD_MAX_COUNTRY_NUM);
+	DECLARE_BITMAP(block_6ghz_vlp, RTW89_REGD_MAX_COUNTRY_NUM);
 };
 
 enum rtw89_ifs_clm_application {
@@ -5315,8 +5563,8 @@ struct rtw89_lps_parm {
 };
 
 struct rtw89_ppdu_sts_info {
-	struct sk_buff_head rx_queue[RTW89_PHY_MAX];
-	u8 curr_rx_ppdu_cnt[RTW89_PHY_MAX];
+	struct sk_buff_head rx_queue[RTW89_PHY_NUM];
+	u8 curr_rx_ppdu_cnt[RTW89_PHY_NUM];
 };
 
 struct rtw89_early_h2c {
@@ -5325,12 +5573,24 @@ struct rtw89_early_h2c {
 	u16 h2c_len;
 };
 
+struct rtw89_hw_scan_extra_op {
+	bool set;
+	u8 macid;
+	u8 port;
+	struct rtw89_chan chan;
+	struct rtw89_vif_link *rtwvif_link;
+};
+
 struct rtw89_hw_scan_info {
 	struct rtw89_vif_link *scanning_vif;
 	struct list_head pkt_list[NUM_NL80211_BANDS];
+	struct list_head chan_list;
 	struct rtw89_chan op_chan;
+	struct rtw89_hw_scan_extra_op extra_op;
+	bool connected;
 	bool abort;
-	u32 last_chan_idx;
+	u16 delay; /* in unit of ms */
+	u8 seq: 2;
 };
 
 enum rtw89_phy_bb_gain_band {
@@ -5435,8 +5695,8 @@ struct rtw89_phy_efuse_gain {
 	bool offset_valid;
 	bool comp_valid;
 	s8 offset[RF_PATH_MAX][RTW89_GAIN_OFFSET_NR]; /* S(8, 0) */
-	s8 offset_base[RTW89_PHY_MAX]; /* S(8, 4) */
-	s8 rssi_base[RTW89_PHY_MAX]; /* S(8, 4) */
+	s8 offset_base[RTW89_PHY_NUM]; /* S(8, 4) */
+	s8 rssi_base[RTW89_PHY_NUM]; /* S(8, 4) */
 	s8 comp[RF_PATH_MAX][RTW89_SUBBAND_NR]; /* S(8, 0) */
 };
 
@@ -5542,30 +5802,37 @@ struct rtw89_mcc_role {
 	struct rtw89_mcc_policy policy;
 	struct rtw89_mcc_limit limit;
 
+	const struct rtw89_mcc_courtesy_cfg *crtz;
+
 	/* only valid when running with FW MRC mechanism */
 	u8 slot_idx;
 
 	/* byte-array in LE order for FW */
 	u8 macid_bitmap[BITS_TO_BYTES(RTW89_MAX_MAC_ID_NUM)];
+	u8 probe_count;
 
 	u16 duration; /* TU */
 	u16 beacon_interval; /* TU */
 	bool is_2ghz;
 	bool is_go;
 	bool is_gc;
+	bool ignore_bcn;
 };
 
 struct rtw89_mcc_bt_role {
 	u16 duration; /* TU */
 };
 
-struct rtw89_mcc_courtesy {
-	bool enable;
+struct rtw89_mcc_courtesy_cfg {
 	u8 slot_num;
-	u8 macid_src;
 	u8 macid_tgt;
 };
 
+struct rtw89_mcc_courtesy {
+	struct rtw89_mcc_courtesy_cfg ref;
+	struct rtw89_mcc_courtesy_cfg aux;
+};
+
 enum rtw89_mcc_plan {
 	RTW89_MCC_PLAN_TAIL_BT,
 	RTW89_MCC_PLAN_MID_BT,
@@ -5599,6 +5866,8 @@ struct rtw89_mcc_config {
 	struct rtw89_mcc_pattern pattern;
 	struct rtw89_mcc_sync sync;
 	u64 start_tsf;
+	u64 start_tsf_in_aux_domain;
+	u64 prepare_delay;
 	u16 mcc_interval; /* TU */
 	u16 beacon_offset; /* TU */
 };
@@ -5619,6 +5888,16 @@ struct rtw89_mcc_info {
 	struct rtw89_mcc_config config;
 };
 
+enum rtw89_mlo_mode {
+	RTW89_MLO_MODE_MLSR = 0,
+
+	NUM_OF_RTW89_MLO_MODE,
+};
+
+struct rtw89_mlo_info {
+	struct rtw89_wait_info wait;
+};
+
 struct rtw89_dev {
 	struct ieee80211_hw *hw;
 	struct device *dev;
@@ -5634,6 +5913,7 @@ struct rtw89_dev {
 	const struct rtw89_rfe_parms *rfe_parms;
 	struct rtw89_hal hal;
 	struct rtw89_mcc_info mcc;
+	struct rtw89_mlo_info mlo;
 	struct rtw89_mac_info mac;
 	struct rtw89_fw_info fw;
 	struct rtw89_hci_info hci;
@@ -5645,8 +5925,6 @@ struct rtw89_dev {
 	struct rtw89_sta_link __rcu *assoc_link_on_macid[RTW89_MAX_MAC_ID_NUM];
 	refcount_t refcount_ap_info;
 
-	/* ensures exclusive access from mac80211 callbacks */
-	struct mutex mutex;
 	struct list_head rtwvifs_list;
 	/* used to protect rf read write */
 	struct mutex rf_mutex;
@@ -5666,10 +5944,10 @@ struct rtw89_dev {
 	struct rtw89_cam_info cam_info;
 
 	struct sk_buff_head c2h_queue;
-	struct work_struct c2h_work;
-	struct work_struct ips_work;
+	struct wiphy_work c2h_work;
+	struct wiphy_work ips_work;
+	struct wiphy_work cancel_6ghz_probe_work;
 	struct work_struct load_firmware_work;
-	struct work_struct cancel_6ghz_probe_work;
 
 	struct list_head early_h2c_list;
 
@@ -5698,9 +5976,6 @@ struct rtw89_dev {
 	struct rtw89_power_trim_info pwr_trim;
 
 	struct rtw89_cfo_tracking_info cfo_tracking;
-	struct rtw89_env_monitor_info env_monitor;
-	struct rtw89_dig_info dig;
-	struct rtw89_phy_ch_info ch_info;
 	union {
 		struct rtw89_phy_bb_gain_info ax;
 		struct rtw89_phy_bb_gain_info_be be;
@@ -5709,15 +5984,24 @@ struct rtw89_dev {
 	struct rtw89_phy_ul_tb_info ul_tb_info;
 	struct rtw89_antdiv_info antdiv;
 
-	struct delayed_work track_work;
-	struct delayed_work chanctx_work;
-	struct delayed_work coex_act1_work;
-	struct delayed_work coex_bt_devinfo_work;
-	struct delayed_work coex_rfk_chk_work;
-	struct delayed_work cfo_track_work;
+	struct rtw89_bb_ctx {
+		enum rtw89_phy_idx phy_idx;
+		struct rtw89_env_monitor_info env_monitor;
+		struct rtw89_dig_info dig;
+		struct rtw89_phy_ch_info ch_info;
+		struct rtw89_edcca_bak edcca_bak;
+	} bbs[RTW89_PHY_NUM];
+
+	struct wiphy_delayed_work track_work;
+	struct wiphy_delayed_work track_ps_work;
+	struct wiphy_delayed_work chanctx_work;
+	struct wiphy_delayed_work coex_act1_work;
+	struct wiphy_delayed_work coex_bt_devinfo_work;
+	struct wiphy_delayed_work coex_rfk_chk_work;
+	struct wiphy_delayed_work cfo_track_work;
+	struct wiphy_delayed_work mcc_prepare_done_work;
 	struct delayed_work forbid_ba_work;
-	struct delayed_work roc_work;
-	struct delayed_work antdiv_work;
+	struct wiphy_delayed_work antdiv_work;
 	struct rtw89_ppdu_sts_info ppdu_sts;
 	u8 total_sta_assoc;
 	bool scanning;
@@ -5760,6 +6044,7 @@ struct rtw89_vif {
 	__be32 ip_addr;
 
 	struct rtw89_traffic_stats stats;
+	struct rtw89_traffic_stats stats_ps;
 	u32 tdls_peer;
 
 	struct ieee80211_scan_ies *scan_ies;
@@ -5768,6 +6053,9 @@ struct rtw89_vif {
 	struct rtw89_roc roc;
 	bool offchan;
 
+	enum rtw89_mlo_mode mlo_mode;
+
+	struct list_head dlink_pool;
 	u8 links_inst_valid_num;
 	DECLARE_BITMAP(links_inst_map, __RTW89_MLD_MAX_LINK_NUM);
 	struct rtw89_vif_link *links[IEEE80211_MLD_MAX_NUM_LINKS];
@@ -5807,6 +6095,7 @@ struct rtw89_sta {
 
 	DECLARE_BITMAP(pairwise_sec_cam_map, RTW89_MAX_SEC_CAM_NUM);
 
+	struct list_head dlink_pool;
 	u8 links_inst_valid_num;
 	DECLARE_BITMAP(links_inst_map, __RTW89_MLD_MAX_LINK_NUM);
 	struct rtw89_sta_link *links[IEEE80211_MLD_MAX_NUM_LINKS];
@@ -5902,6 +6191,12 @@ rtw89_assoc_link_rcu_dereference(struct rtw89_dev *rtwdev, u8 macid)
 	return rcu_dereference(rtwdev->assoc_link_on_macid[macid]);
 }
 
+#define rtw89_get_designated_link(links_holder) \
+({ \
+	typeof(links_holder) p = links_holder; \
+	list_first_entry_or_null(&p->dlink_pool, typeof(*p->links_inst), dlink_schd); \
+})
+
 static inline int rtw89_hci_tx_write(struct rtw89_dev *rtwdev,
 				     struct rtw89_core_tx_request *tx_req)
 {
@@ -6728,6 +7023,17 @@ static inline u8 rtw89_chip_get_thermal(struct rtw89_dev *rtwdev,
 	return chip->ops->get_thermal(rtwdev, rf_path);
 }
 
+static inline u32 rtw89_chip_chan_to_rf18_val(struct rtw89_dev *rtwdev,
+					      const struct rtw89_chan *chan)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	if (!chip->ops->chan_to_rf18_val)
+		return 0;
+
+	return chip->ops->chan_to_rf18_val(rtwdev, chan);
+}
+
 static inline void rtw89_chip_query_ppdu(struct rtw89_dev *rtwdev,
 					 struct rtw89_rx_phy_ppdu *phy_ppdu,
 					 struct ieee80211_rx_status *status)
@@ -6791,9 +7097,14 @@ static inline void rtw89_load_txpwr_table(struct rtw89_dev *rtwdev,
 
 static inline u8 rtw89_regd_get(struct rtw89_dev *rtwdev, u8 band)
 {
-	const struct rtw89_regd *regd = rtwdev->regulatory.regd;
+	const struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	const struct rtw89_regd *regd = regulatory->regd;
+	u8 txpwr_regd = regd->txpwr_regd[band];
+
+	if (regulatory->txpwr_uk_follow_etsi && txpwr_regd == RTW89_UK)
+		return RTW89_ETSI;
 
-	return regd->txpwr_regd[band];
+	return txpwr_regd;
 }
 
 static inline void rtw89_ctrl_btg_bt_rx(struct rtw89_dev *rtwdev, bool en,
@@ -6998,6 +7309,48 @@ static inline bool rtw89_is_mlo_1_1(struct rtw89_dev *rtwdev)
 	}
 }
 
+static inline u8 rtw89_get_active_phy_bitmap(struct rtw89_dev *rtwdev)
+{
+	if (!rtwdev->dbcc_en)
+		return BIT(RTW89_PHY_0);
+
+	switch (rtwdev->mlo_dbcc_mode) {
+	case MLO_0_PLUS_2_1RF:
+	case MLO_0_PLUS_2_2RF:
+		return BIT(RTW89_PHY_1);
+	case MLO_1_PLUS_1_1RF:
+	case MLO_1_PLUS_1_2RF:
+	case MLO_2_PLUS_2_2RF:
+	case DBCC_LEGACY:
+		return BIT(RTW89_PHY_0) | BIT(RTW89_PHY_1);
+	case MLO_2_PLUS_0_1RF:
+	case MLO_2_PLUS_0_2RF:
+	default:
+		return BIT(RTW89_PHY_0);
+	}
+}
+
+#define rtw89_for_each_active_bb(rtwdev, bb) \
+	for (u8 __active_bb_bitmap = rtw89_get_active_phy_bitmap(rtwdev), \
+	     __phy_idx = 0; __phy_idx < RTW89_PHY_NUM; __phy_idx++) \
+		if (__active_bb_bitmap & BIT(__phy_idx) && \
+		    (bb = &rtwdev->bbs[__phy_idx]))
+
+#define rtw89_for_each_capab_bb(rtwdev, bb) \
+	for (u8 __phy_idx_max = rtwdev->dbcc_en ? RTW89_PHY_1 : RTW89_PHY_0, \
+	     __phy_idx = 0; __phy_idx <= __phy_idx_max; __phy_idx++) \
+		if ((bb = &rtwdev->bbs[__phy_idx]))
+
+static inline
+struct rtw89_bb_ctx *rtw89_get_bb_ctx(struct rtw89_dev *rtwdev,
+				      enum rtw89_phy_idx phy_idx)
+{
+	if (phy_idx >= RTW89_PHY_NUM)
+		return &rtwdev->bbs[RTW89_PHY_0];
+
+	return &rtwdev->bbs[phy_idx];
+}
+
 static inline bool rtw89_is_rtl885xb(struct rtw89_dev *rtwdev)
 {
 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
@@ -7008,6 +7361,17 @@ static inline bool rtw89_is_rtl885xb(struct rtw89_dev *rtwdev)
 	return false;
 }
 
+static inline u32 rtw89_bytes_to_mbps(u64 bytes, enum rtw89_tfc_interval interval)
+{
+	switch (interval) {
+	default:
+	case RTW89_TFC_INTERVAL_2SEC:
+		return bytes >> 18; /* bytes/2s --> Mbps */;
+	case RTW89_TFC_INTERVAL_100MS:
+		return (bytes * 10) >> 17; /* bytes/100ms --> Mbps */
+	}
+}
+
 int rtw89_core_tx_write(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 			struct ieee80211_sta *sta, struct sk_buff *skb, int *qsel);
 int rtw89_h2c_tx(struct rtw89_dev *rtwdev,
@@ -7112,9 +7476,8 @@ void rtw89_chip_cfg_txpwr_ul_tb_offset(struct rtw89_dev *rtwdev,
 				       struct rtw89_vif_link *rtwvif_link);
 bool rtw89_ra_report_to_bitrate(struct rtw89_dev *rtwdev, u8 rpt_rate, u16 *bitrate);
 int rtw89_regd_setup(struct rtw89_dev *rtwdev);
-int rtw89_regd_init(struct rtw89_dev *rtwdev,
-		    void (*reg_notifier)(struct wiphy *wiphy, struct regulatory_request *request));
-void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request);
+int rtw89_regd_init_hint(struct rtw89_dev *rtwdev);
+const char *rtw89_regd_get_string(enum rtw89_regulation_type regd);
 void rtw89_traffic_stats_init(struct rtw89_dev *rtwdev,
 			      struct rtw89_traffic_stats *stats);
 int rtw89_wait_for_cond(struct rtw89_wait_info *wait, unsigned int cond);
@@ -7122,8 +7485,11 @@ void rtw89_complete_cond(struct rtw89_wait_info *wait, unsigned int cond,
 			 const struct rtw89_completion_data *data);
 int rtw89_core_start(struct rtw89_dev *rtwdev);
 void rtw89_core_stop(struct rtw89_dev *rtwdev);
-void rtw89_core_update_beacon_work(struct work_struct *work);
-void rtw89_roc_work(struct work_struct *work);
+void rtw89_core_update_beacon_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_core_csa_beacon_work(struct wiphy *wiphy, struct wiphy_work *work);
+int rtw89_core_send_nullfunc(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link,
+			     bool qos, bool ps, int timeout);
+void rtw89_roc_work(struct wiphy *wiphy, struct wiphy_work *work);
 void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
 void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
 void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link,
@@ -7136,6 +7502,8 @@ void rtw89_core_update_p2p_ps(struct rtw89_dev *rtwdev,
 			      struct rtw89_vif_link *rtwvif_link,
 			      struct ieee80211_bss_conf *bss_conf);
 void rtw89_core_ntfy_btc_event(struct rtw89_dev *rtwdev, enum rtw89_btc_hmsg event);
+int rtw89_core_mlsr_switch(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+			   unsigned int link_id);
 
 #if defined(__linux__)
 #define	rtw89_static_assert(_x)		static_assert(_x)
diff --git a/sys/contrib/dev/rtw89/debug.c b/sys/contrib/dev/rtw89/debug.c
index f7e451b0a032..d6624c2ed379 100644
--- a/sys/contrib/dev/rtw89/debug.c
+++ b/sys/contrib/dev/rtw89/debug.c
@@ -17,6 +17,7 @@
 #include "ps.h"
 #include "reg.h"
 #include "sar.h"
+#include "util.h"
 #if defined(__FreeBSD__)
 #ifdef CONFIG_RTW89_DEBUGFS
 #include <linux/debugfs.h>
@@ -31,11 +32,21 @@ MODULE_PARM_DESC(debug_mask, "Debugging mask");
 #endif
 
 #ifdef CONFIG_RTW89_DEBUGFS
+struct rtw89_debugfs_priv_opt {
+	bool rlock:1;
+	bool wlock:1;
+	size_t rsize;
+};
+
 struct rtw89_debugfs_priv {
 	struct rtw89_dev *rtwdev;
-	int (*cb_read)(struct seq_file *m, void *v);
-	ssize_t (*cb_write)(struct file *filp, const char __user *buffer,
-			    size_t count, loff_t *loff);
+	ssize_t (*cb_read)(struct rtw89_dev *rtwdev,
+			   struct rtw89_debugfs_priv *debugfs_priv,
+			   char *buf, size_t bufsz);
+	ssize_t (*cb_write)(struct rtw89_dev *rtwdev,
+			    struct rtw89_debugfs_priv *debugfs_priv,
+			    const char *buf, size_t count);
+	struct rtw89_debugfs_priv_opt opt;
 	union {
 		u32 cb_data;
 		struct {
@@ -60,6 +71,8 @@ struct rtw89_debugfs_priv {
 			u8 sel;
 		} mac_mem;
 	};
+	ssize_t rused;
+	char *rbuf;
 };
 
 struct rtw89_debugfs {
@@ -81,8 +94,31 @@ struct rtw89_debugfs {
 	struct rtw89_debugfs_priv phy_info;
 	struct rtw89_debugfs_priv stations;
 	struct rtw89_debugfs_priv disable_dm;
+	struct rtw89_debugfs_priv mlo_mode;
+};
+
+struct rtw89_debugfs_iter_data {
+	char *buf;
+	size_t bufsz;
+	int written_sz;
 };
 
+static void rtw89_debugfs_iter_data_setup(struct rtw89_debugfs_iter_data *iter_data,
+					  char *buf, size_t bufsz)
+{
+	iter_data->buf = buf;
+	iter_data->bufsz = bufsz;
+	iter_data->written_sz = 0;
+}
+
+static void rtw89_debugfs_iter_data_next(struct rtw89_debugfs_iter_data *iter_data,
+					 char *buf, size_t bufsz, int written_sz)
+{
+	iter_data->buf = buf;
+	iter_data->bufsz = bufsz;
+	iter_data->written_sz += written_sz;
+}
+
 static const u16 rtw89_rate_info_bw_to_mhz_map[] = {
 	[RATE_INFO_BW_20] = 20,
 	[RATE_INFO_BW_40] = 40,
@@ -99,84 +135,122 @@ static u16 rtw89_rate_info_bw_to_mhz(enum rate_info_bw bw)
 	return 0;
 }
 
-static int rtw89_debugfs_single_show(struct seq_file *m, void *v)
+static ssize_t rtw89_debugfs_file_read_helper(struct wiphy *wiphy, struct file *file,
+					      char *buf, size_t bufsz, void *data)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
+	struct rtw89_debugfs_priv *debugfs_priv = data;
+	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+	ssize_t n;
 
-	return debugfs_priv->cb_read(m, v);
+	n = debugfs_priv->cb_read(rtwdev, debugfs_priv, buf, bufsz);
+	rtw89_might_trailing_ellipsis(buf, bufsz, n);
+
+	return n;
 }
 
-static ssize_t rtw89_debugfs_single_write(struct file *filp,
-					  const char __user *buffer,
-					  size_t count, loff_t *loff)
+static ssize_t rtw89_debugfs_file_read(struct file *file, char __user *userbuf,
+				       size_t count, loff_t *ppos)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtw89_debugfs_priv *debugfs_priv = file->private_data;
+	struct rtw89_debugfs_priv_opt *opt = &debugfs_priv->opt;
+	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+	size_t bufsz = opt->rsize ? opt->rsize : PAGE_SIZE;
+	char *buf;
+	ssize_t n;
 
-	return debugfs_priv->cb_write(filp, buffer, count, loff);
-}
+	if (!debugfs_priv->rbuf)
+		debugfs_priv->rbuf = devm_kzalloc(rtwdev->dev, bufsz, GFP_KERNEL);
 
-static ssize_t rtw89_debugfs_seq_file_write(struct file *filp,
-					    const char __user *buffer,
-					    size_t count, loff_t *loff)
-{
-	struct seq_file *seqpriv = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = seqpriv->private;
+	buf = debugfs_priv->rbuf;
+	if (!buf)
+		return -ENOMEM;
+
+	if (*ppos) {
+		n = debugfs_priv->rused;
+		goto out;
+	}
 
-	return debugfs_priv->cb_write(filp, buffer, count, loff);
+	if (opt->rlock) {
+		n = wiphy_locked_debugfs_read(rtwdev->hw->wiphy, file, buf, bufsz,
+					      userbuf, count, ppos,
+					      rtw89_debugfs_file_read_helper,
+					      debugfs_priv);
+		debugfs_priv->rused = n;
+
+		return n;
+	}
+
+	n = rtw89_debugfs_file_read_helper(rtwdev->hw->wiphy, file, buf, bufsz,
+					   debugfs_priv);
+	debugfs_priv->rused = n;
+
+out:
+	return simple_read_from_buffer(userbuf, count, ppos, buf, n);
 }
 
-static int rtw89_debugfs_single_open(struct inode *inode, struct file *filp)
+static ssize_t rtw89_debugfs_file_write_helper(struct wiphy *wiphy, struct file *file,
+					       char *buf, size_t count, void *data)
 {
-	return single_open(filp, rtw89_debugfs_single_show, inode->i_private);
+	struct rtw89_debugfs_priv *debugfs_priv = data;
+	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+
+	return debugfs_priv->cb_write(rtwdev, debugfs_priv, buf, count);
 }
 
-static int rtw89_debugfs_close(struct inode *inode, struct file *filp)
+static ssize_t rtw89_debugfs_file_write(struct file *file,
+					const char __user *userbuf,
+					size_t count, loff_t *loff)
 {
-	return 0;
+	struct rtw89_debugfs_priv *debugfs_priv = file->private_data;
+	struct rtw89_debugfs_priv_opt *opt = &debugfs_priv->opt;
+	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+	char *buf __free(kfree) = kmalloc(count + 1, GFP_KERNEL);
+	ssize_t n;
+
+	if (!buf)
+		return -ENOMEM;
+
+	if (opt->wlock) {
+		n = wiphy_locked_debugfs_write(rtwdev->hw->wiphy,
+					       file, buf, count + 1,
+					       userbuf, count,
+					       rtw89_debugfs_file_write_helper,
+					       debugfs_priv);
+		return n;
+	}
+
+	if (copy_from_user(buf, userbuf, count))
+		return -EFAULT;
+
+	buf[count] = '\0';
+
+	return debugfs_priv->cb_write(rtwdev, debugfs_priv, buf, count);
 }
 
-static const struct file_operations file_ops_single_r = {
-	.owner = THIS_MODULE,
-	.open = rtw89_debugfs_single_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
+static const struct debugfs_short_fops file_ops_single_r = {
+	.read = rtw89_debugfs_file_read,
+	.llseek = generic_file_llseek,
 };
 
-static const struct file_operations file_ops_common_rw = {
-	.owner = THIS_MODULE,
-	.open = rtw89_debugfs_single_open,
-	.release = single_release,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.write = rtw89_debugfs_seq_file_write,
+static const struct debugfs_short_fops file_ops_common_rw = {
+	.read = rtw89_debugfs_file_read,
+	.write = rtw89_debugfs_file_write,
+	.llseek = generic_file_llseek,
 };
 
-static const struct file_operations file_ops_single_w = {
-	.owner = THIS_MODULE,
-	.write = rtw89_debugfs_single_write,
-	.open = simple_open,
-	.release = rtw89_debugfs_close,
+static const struct debugfs_short_fops file_ops_single_w = {
+	.write = rtw89_debugfs_file_write,
+	.llseek = generic_file_llseek,
 };
 
 static ssize_t
-rtw89_debug_priv_read_reg_select(struct file *filp,
-				 const char __user *user_buf,
-				 size_t count, loff_t *loff)
+rtw89_debug_priv_read_reg_select(struct rtw89_dev *rtwdev,
+				 struct rtw89_debugfs_priv *debugfs_priv,
+				 const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	char buf[32];
-	size_t buf_size;
 	u32 addr, len;
 	int num;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
 	num = sscanf(buf, "%x %x", &addr, &len);
 	if (num != 2) {
 		rtw89_info(rtwdev, "invalid format: <addr> <len>\n");
@@ -191,11 +265,13 @@ rtw89_debug_priv_read_reg_select(struct file *filp,
 	return count;
 }
 
-static int rtw89_debug_priv_read_reg_get(struct seq_file *m, void *v)
+static
+ssize_t rtw89_debug_priv_read_reg_get(struct rtw89_dev *rtwdev,
+				      struct rtw89_debugfs_priv *debugfs_priv,
+				      char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	u32 addr, end, data, k;
+	char *p = buf, *end = buf + bufsz;
+	u32 addr, addr_end, data, k;
 	u32 len;
 
 	len = debugfs_priv->read_reg.len;
@@ -219,41 +295,34 @@ static int rtw89_debug_priv_read_reg_get(struct seq_file *m, void *v)
 		return -EINVAL;
 	}
 
-	seq_printf(m, "get %d bytes at 0x%08x=0x%08x\n", len, addr, data);
+	p += scnprintf(p, end - p, "get %d bytes at 0x%08x=0x%08x\n", len,
+		       addr, data);
 
-	return 0;
+	return p - buf;
 
 ndata:
-	end = addr + len;
+	addr_end = addr + len;
 
-	for (; addr < end; addr += 16) {
-		seq_printf(m, "%08xh : ", 0x18600000 + addr);
+	for (; addr < addr_end; addr += 16) {
+		p += scnprintf(p, end - p, "%08xh : ", 0x18600000 + addr);
 		for (k = 0; k < 16; k += 4) {
 			data = rtw89_read32(rtwdev, addr + k);
-			seq_printf(m, "%08x ", data);
+			p += scnprintf(p, end - p, "%08x ", data);
 		}
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
 	}
 
-	return 0;
+	return p - buf;
 }
 
-static ssize_t rtw89_debug_priv_write_reg_set(struct file *filp,
-					      const char __user *user_buf,
-					      size_t count, loff_t *loff)
+static
+ssize_t rtw89_debug_priv_write_reg_set(struct rtw89_dev *rtwdev,
+				       struct rtw89_debugfs_priv *debugfs_priv,
+				       const char *buf, size_t count)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = filp->private_data;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	char buf[32];
-	size_t buf_size;
 	u32 addr, val, len;
 	int num;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
 	num = sscanf(buf, "%x %x %x", &addr, &val, &len);
 	if (num !=  3) {
 		rtw89_info(rtwdev, "invalid format: <addr> <val> <len>\n");
@@ -282,24 +351,14 @@ static ssize_t rtw89_debug_priv_write_reg_set(struct file *filp,
 }
 
 static ssize_t
-rtw89_debug_priv_read_rf_select(struct file *filp,
-				const char __user *user_buf,
-				size_t count, loff_t *loff)
+rtw89_debug_priv_read_rf_select(struct rtw89_dev *rtwdev,
+				struct rtw89_debugfs_priv *debugfs_priv,
+				const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	char buf[32];
-	size_t buf_size;
 	u32 addr, mask;
 	u8 path;
 	int num;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
 	num = sscanf(buf, "%hhd %x %x", &path, &addr, &mask);
 	if (num != 3) {
 		rtw89_info(rtwdev, "invalid format: <path> <addr> <mask>\n");
@@ -319,10 +378,12 @@ rtw89_debug_priv_read_rf_select(struct file *filp,
 	return count;
 }
 
-static int rtw89_debug_priv_read_rf_get(struct seq_file *m, void *v)
+static
+ssize_t rtw89_debug_priv_read_rf_get(struct rtw89_dev *rtwdev,
+				     struct rtw89_debugfs_priv *debugfs_priv,
+				     char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+	char *p = buf, *end = buf + bufsz;
 	u32 addr, data, mask;
 	u8 path;
 
@@ -332,28 +393,21 @@ static int rtw89_debug_priv_read_rf_get(struct seq_file *m, void *v)
 
 	data = rtw89_read_rf(rtwdev, path, addr, mask);
 
-	seq_printf(m, "path %d, rf register 0x%08x=0x%08x\n", path, addr, data);
+	p += scnprintf(p, end - p, "path %d, rf register 0x%08x=0x%08x\n",
+		       path, addr, data);
 
-	return 0;
+	return p - buf;
 }
 
-static ssize_t rtw89_debug_priv_write_rf_set(struct file *filp,
-					     const char __user *user_buf,
-					     size_t count, loff_t *loff)
+static
+ssize_t rtw89_debug_priv_write_rf_set(struct rtw89_dev *rtwdev,
+				      struct rtw89_debugfs_priv *debugfs_priv,
+				      const char *buf, size_t count)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = filp->private_data;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	char buf[32];
-	size_t buf_size;
 	u32 addr, val, mask;
 	u8 path;
 	int num;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
 	num = sscanf(buf, "%hhd %x %x %x", &path, &addr, &mask, &val);
 	if (num != 4) {
 		rtw89_info(rtwdev, "invalid format: <path> <addr> <mask> <val>\n");
@@ -372,29 +426,31 @@ static ssize_t rtw89_debug_priv_write_rf_set(struct file *filp,
 	return count;
 }
 
-static int rtw89_debug_priv_rf_reg_dump_get(struct seq_file *m, void *v)
+static
+ssize_t rtw89_debug_priv_rf_reg_dump_get(struct rtw89_dev *rtwdev,
+					 struct rtw89_debugfs_priv *debugfs_priv,
+					 char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	char *p = buf, *end = buf + bufsz;
 	u32 addr, offset, data;
 	u8 path;
 
 	for (path = 0; path < chip->rf_path_num; path++) {
-		seq_printf(m, "RF path %d:\n\n", path);
+		p += scnprintf(p, end - p, "RF path %d:\n\n", path);
 		for (addr = 0; addr < 0x100; addr += 4) {
-			seq_printf(m, "0x%08x: ", addr);
+			p += scnprintf(p, end - p, "0x%08x: ", addr);
 			for (offset = 0; offset < 4; offset++) {
 				data = rtw89_read_rf(rtwdev, path,
 						     addr + offset, RFREG_MASK);
-				seq_printf(m, "0x%05x  ", data);
+				p += scnprintf(p, end - p, "0x%05x  ", data);
 			}
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 		}
-		seq_puts(m, "\n");
+		p += scnprintf(p, end - p, "\n");
 	}
 
-	return 0;
+	return p - buf;
 }
 
 struct txpwr_ent {
@@ -725,56 +781,71 @@ static const struct txpwr_map __txpwr_map_lmt_ru_be = {
 };
 
 static unsigned int
-__print_txpwr_ent(struct seq_file *m, const struct txpwr_ent *ent,
-		  const s8 *buf, const unsigned int cur)
+__print_txpwr_ent(char *buf, size_t bufsz, const struct txpwr_ent *ent,
+		  const s8 *bufp, const unsigned int cur, unsigned int *ate)
 {
+	char *p = buf, *end = buf + bufsz;
 	unsigned int cnt, i;
+	unsigned int eaten;
 	char *fmt;
 
 	if (ent->nested) {
-		for (cnt = 0, i = 0; i < ent->len; i++)
-			cnt += __print_txpwr_ent(m, ent->ptr + i, buf,
-						 cur + cnt);
-		return cnt;
+		for (cnt = 0, i = 0; i < ent->len; i++, cnt += eaten)
+			p += __print_txpwr_ent(p, end - p, ent->ptr + i, bufp,
+					       cur + cnt, &eaten);
+		*ate = cnt;
+		goto out;
 	}
 
 	switch (ent->len) {
 	case 0:
-		seq_printf(m, "\t<< %s >>\n", ent->txt);
-		return 0;
+		p += scnprintf(p, end - p, "\t<< %s >>\n", ent->txt);
+		*ate = 0;
+		goto out;
 	case 2:
 		fmt = "%s\t| %3d, %3d,\t\tdBm\n";
-		seq_printf(m, fmt, ent->txt, buf[cur], buf[cur + 1]);
-		return 2;
+		p += scnprintf(p, end - p, fmt, ent->txt, bufp[cur],
+			       bufp[cur + 1]);
+		*ate = 2;
+		goto out;
 	case 4:
 		fmt = "%s\t| %3d, %3d, %3d, %3d,\tdBm\n";
-		seq_printf(m, fmt, ent->txt, buf[cur], buf[cur + 1],
-			   buf[cur + 2], buf[cur + 3]);
-		return 4;
+		p += scnprintf(p, end - p, fmt, ent->txt, bufp[cur],
+			       bufp[cur + 1],
+			       bufp[cur + 2], bufp[cur + 3]);
+		*ate = 4;
+		goto out;
 	case 8:
 		fmt = "%s\t| %3d, %3d, %3d, %3d, %3d, %3d, %3d, %3d,\tdBm\n";
-		seq_printf(m, fmt, ent->txt, buf[cur], buf[cur + 1],
-			   buf[cur + 2], buf[cur + 3], buf[cur + 4],
-			   buf[cur + 5], buf[cur + 6], buf[cur + 7]);
-		return 8;
+		p += scnprintf(p, end - p, fmt, ent->txt, bufp[cur],
+			       bufp[cur + 1],
+			       bufp[cur + 2], bufp[cur + 3], bufp[cur + 4],
+			       bufp[cur + 5], bufp[cur + 6], bufp[cur + 7]);
+		*ate = 8;
+		goto out;
 	default:
 		return 0;
 	}
+
+out:
+	return p - buf;
 }
 
-static int __print_txpwr_map(struct seq_file *m, struct rtw89_dev *rtwdev,
-			     const struct txpwr_map *map)
+static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t bufsz,
+				 const struct txpwr_map *map)
 {
 	u8 fct = rtwdev->chip->txpwr_factor_mac;
 	u8 path_num = rtwdev->chip->rf_path_num;
+	char *p = buf, *end = buf + bufsz;
 	unsigned int cur, i;
+	unsigned int eaten;
 	u32 max_valid_addr;
 	u32 val, addr;
-	s8 *buf, tmp;
+	s8 *bufp, tmp;
 	int ret;
 
-	buf = vzalloc(map->addr_to - map->addr_from + 4);
-	if (!buf)
+	bufp = vzalloc(map->addr_to - map->addr_from + 4);
+	if (!bufp)
 		return -ENOMEM;
 
 	if (path_num == 1)
@@ -794,52 +865,31 @@ static int __print_txpwr_map(struct seq_file *m, struct rtw89_dev *rtwdev,
 		for (i = 0; i < 4; i++, val >>= 8) {
 			/* signed 7 bits, and reserved BIT(7) */
 			tmp = sign_extend32(val, 6);
-			buf[cur + i] = tmp >> fct;
+			bufp[cur + i] = tmp >> fct;
 		}
 	}
 
-	for (cur = 0, i = 0; i < map->size; i++)
-		cur += __print_txpwr_ent(m, &map->ent[i], buf, cur);
+	for (cur = 0, i = 0; i < map->size; i++, cur += eaten)
+		p += __print_txpwr_ent(p, end - p, &map->ent[i], bufp, cur, &eaten);
 
-	vfree(buf);
-	return 0;
+	vfree(bufp);
+	return p - buf;
 }
 
-#define case_REGD(_regd) \
-	case RTW89_ ## _regd: \
-		seq_puts(m, #_regd "\n"); \
-		break
-
-static void __print_regd(struct seq_file *m, struct rtw89_dev *rtwdev,
-			 const struct rtw89_chan *chan)
+static int __print_regd(struct rtw89_dev *rtwdev, char *buf, size_t bufsz,
+			const struct rtw89_chan *chan)
 {
+	const struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	char *p = buf, *end = buf + bufsz;
 	u8 band = chan->band_type;
 	u8 regd = rtw89_regd_get(rtwdev, band);
 
-	switch (regd) {
-	default:
-		seq_printf(m, "UNKNOWN: %d\n", regd);
-		break;
-	case_REGD(WW);
-	case_REGD(ETSI);
-	case_REGD(FCC);
-	case_REGD(MKK);
-	case_REGD(NA);
-	case_REGD(IC);
-	case_REGD(KCC);
-	case_REGD(NCC);
-	case_REGD(CHILE);
-	case_REGD(ACMA);
-	case_REGD(MEXICO);
-	case_REGD(UKRAINE);
-	case_REGD(CN);
-	case_REGD(QATAR);
-	case_REGD(UK);
-	case_REGD(THAILAND);
-	}
-}
+	p += scnprintf(p, end - p, "%s\n", rtw89_regd_get_string(regd));
+	p += scnprintf(p, end - p, "\t(txpwr UK follow ETSI: %s)\n",
+		       str_yes_no(regulatory->txpwr_uk_follow_etsi));
 
-#undef case_REGD
+	return p - buf;
+}
 
 struct dbgfs_txpwr_table {
 	const struct txpwr_map *byr;
@@ -865,96 +915,95 @@ static const struct dbgfs_txpwr_table *dbgfs_txpwr_tables[RTW89_CHIP_GEN_NUM] =
 };
 
 static
-void rtw89_debug_priv_txpwr_table_get_regd(struct seq_file *m,
-					   struct rtw89_dev *rtwdev,
-					   const struct rtw89_chan *chan)
+int rtw89_debug_priv_txpwr_table_get_regd(struct rtw89_dev *rtwdev,
+					  char *buf, size_t bufsz,
+					  const struct rtw89_chan *chan)
 {
 	const struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
 	const struct rtw89_reg_6ghz_tpe *tpe6 = &regulatory->reg_6ghz_tpe;
+	char *p = buf, *end = buf + bufsz;
 
-	seq_printf(m, "[Chanctx] band %u, ch %u, bw %u\n",
-		   chan->band_type, chan->channel, chan->band_width);
+	p += scnprintf(p, end - p, "[Chanctx] band %u, ch %u, bw %u\n",
+		       chan->band_type, chan->channel, chan->band_width);
 
-	seq_puts(m, "[Regulatory] ");
-	__print_regd(m, rtwdev, chan);
+	p += scnprintf(p, end - p, "[Regulatory] ");
+	p += __print_regd(rtwdev, p, end - p, chan);
 
 	if (chan->band_type == RTW89_BAND_6G) {
-		seq_printf(m, "[reg6_pwr_type] %u\n", regulatory->reg_6ghz_power);
+		p += scnprintf(p, end - p, "[reg6_pwr_type] %u\n",
+			       regulatory->reg_6ghz_power);
 
 		if (tpe6->valid)
-			seq_printf(m, "[TPE] %d dBm\n", tpe6->constraint);
+			p += scnprintf(p, end - p, "[TPE] %d dBm\n",
+				       tpe6->constraint);
 	}
+
+	return p - buf;
 }
 
-static int rtw89_debug_priv_txpwr_table_get(struct seq_file *m, void *v)
+static
+ssize_t rtw89_debug_priv_txpwr_table_get(struct rtw89_dev *rtwdev,
+					 struct rtw89_debugfs_priv *debugfs_priv,
+					 char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
+	struct rtw89_sar_parm sar_parm = {};
 	const struct dbgfs_txpwr_table *tbl;
 	const struct rtw89_chan *chan;
-	int ret = 0;
+	char *p = buf, *end = buf + bufsz;
+	ssize_t n;
+
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
-	mutex_lock(&rtwdev->mutex);
 	rtw89_leave_ps_mode(rtwdev);
 	chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0);
+	sar_parm.center_freq = chan->freq;
 
-	rtw89_debug_priv_txpwr_table_get_regd(m, rtwdev, chan);
+	p += rtw89_debug_priv_txpwr_table_get_regd(rtwdev, p, end - p, chan);
 
-	seq_puts(m, "[SAR]\n");
-	rtw89_print_sar(m, rtwdev, chan->freq);
+	p += scnprintf(p, end - p, "[SAR]\n");
+	p += rtw89_print_sar(rtwdev, p, end - p, &sar_parm);
 
-	seq_puts(m, "[TAS]\n");
-	rtw89_print_tas(m, rtwdev);
+	p += scnprintf(p, end - p, "[TAS]\n");
+	p += rtw89_print_tas(rtwdev, p, end - p);
 
-	seq_puts(m, "[DAG]\n");
-	rtw89_print_ant_gain(m, rtwdev, chan);
+	p += scnprintf(p, end - p, "[DAG]\n");
+	p += rtw89_print_ant_gain(rtwdev, p, end - p, chan);
 
 	tbl = dbgfs_txpwr_tables[chip_gen];
-	if (!tbl) {
-		ret = -EOPNOTSUPP;
-		goto err;
-	}
-
-	seq_puts(m, "\n[TX power byrate]\n");
-	ret = __print_txpwr_map(m, rtwdev, tbl->byr);
-	if (ret)
-		goto err;
-
-	seq_puts(m, "\n[TX power limit]\n");
-	ret = __print_txpwr_map(m, rtwdev, tbl->lmt);
-	if (ret)
-		goto err;
-
-	seq_puts(m, "\n[TX power limit_ru]\n");
-	ret = __print_txpwr_map(m, rtwdev, tbl->lmt_ru);
-	if (ret)
-		goto err;
+	if (!tbl)
+		return -EOPNOTSUPP;
 
-err:
-	mutex_unlock(&rtwdev->mutex);
-	return ret;
+	p += scnprintf(p, end - p, "\n[TX power byrate]\n");
+	n = __print_txpwr_map(rtwdev, p, end - p, tbl->byr);
+	if (n < 0)
+		return n;
+	p += n;
+
+	p += scnprintf(p, end - p, "\n[TX power limit]\n");
+	n = __print_txpwr_map(rtwdev, p, end - p, tbl->lmt);
+	if (n < 0)
+		return n;
+	p += n;
+
+	p += scnprintf(p, end - p, "\n[TX power limit_ru]\n");
+	n = __print_txpwr_map(rtwdev, p, end - p, tbl->lmt_ru);
+	if (n < 0)
+		return n;
+	p += n;
+
+	return p - buf;
 }
 
 static ssize_t
-rtw89_debug_priv_mac_reg_dump_select(struct file *filp,
-				     const char __user *user_buf,
-				     size_t count, loff_t *loff)
+rtw89_debug_priv_mac_reg_dump_select(struct rtw89_dev *rtwdev,
+				     struct rtw89_debugfs_priv *debugfs_priv,
+				     const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	char buf[32];
-	size_t buf_size;
 	int sel;
 	int ret;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
 	ret = kstrtoint(buf, 0, &sel);
 	if (ret)
 		return ret;
@@ -978,99 +1027,91 @@ rtw89_debug_priv_mac_reg_dump_select(struct file *filp,
 
 #define RTW89_MAC_PAGE_SIZE		0x100
 
-static int rtw89_debug_priv_mac_reg_dump_get(struct seq_file *m, void *v)
+static
+ssize_t rtw89_debug_priv_mac_reg_dump_get(struct rtw89_dev *rtwdev,
+					  struct rtw89_debugfs_priv *debugfs_priv,
+					  char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	enum rtw89_debug_mac_reg_sel reg_sel = debugfs_priv->cb_data;
-	u32 start, end;
+	char *p = buf, *end = buf + bufsz;
+	u32 start, end_addr;
 	u32 i, j, k, page;
 	u32 val;
 
 	switch (reg_sel) {
 	case RTW89_DBG_SEL_MAC_00:
-		seq_puts(m, "Debug selected MAC page 0x00\n");
+		p += scnprintf(p, end - p, "Debug selected MAC page 0x00\n");
 		start = 0x000;
-		end = 0x014;
+		end_addr = 0x014;
 		break;
 	case RTW89_DBG_SEL_MAC_30:
-		seq_puts(m, "Debug selected MAC page 0x30\n");
+		p += scnprintf(p, end - p, "Debug selected MAC page 0x30\n");
 		start = 0x030;
-		end = 0x033;
+		end_addr = 0x033;
 		break;
 	case RTW89_DBG_SEL_MAC_40:
-		seq_puts(m, "Debug selected MAC page 0x40\n");
+		p += scnprintf(p, end - p, "Debug selected MAC page 0x40\n");
 		start = 0x040;
-		end = 0x07f;
+		end_addr = 0x07f;
 		break;
 	case RTW89_DBG_SEL_MAC_80:
-		seq_puts(m, "Debug selected MAC page 0x80\n");
+		p += scnprintf(p, end - p, "Debug selected MAC page 0x80\n");
 		start = 0x080;
-		end = 0x09f;
+		end_addr = 0x09f;
 		break;
 	case RTW89_DBG_SEL_MAC_C0:
-		seq_puts(m, "Debug selected MAC page 0xc0\n");
+		p += scnprintf(p, end - p, "Debug selected MAC page 0xc0\n");
 		start = 0x0c0;
-		end = 0x0df;
+		end_addr = 0x0df;
 		break;
 	case RTW89_DBG_SEL_MAC_E0:
-		seq_puts(m, "Debug selected MAC page 0xe0\n");
+		p += scnprintf(p, end - p, "Debug selected MAC page 0xe0\n");
 		start = 0x0e0;
-		end = 0x0ff;
+		end_addr = 0x0ff;
 		break;
 	case RTW89_DBG_SEL_BB:
-		seq_puts(m, "Debug selected BB register\n");
+		p += scnprintf(p, end - p, "Debug selected BB register\n");
 		start = 0x100;
-		end = 0x17f;
+		end_addr = 0x17f;
 		break;
 	case RTW89_DBG_SEL_IQK:
-		seq_puts(m, "Debug selected IQK register\n");
+		p += scnprintf(p, end - p, "Debug selected IQK register\n");
 		start = 0x180;
-		end = 0x1bf;
+		end_addr = 0x1bf;
 		break;
 	case RTW89_DBG_SEL_RFC:
-		seq_puts(m, "Debug selected RFC register\n");
+		p += scnprintf(p, end - p, "Debug selected RFC register\n");
 		start = 0x1c0;
-		end = 0x1ff;
+		end_addr = 0x1ff;
 		break;
 	default:
-		seq_puts(m, "Selected invalid register page\n");
+		p += scnprintf(p, end - p, "Selected invalid register page\n");
 		return -EINVAL;
 	}
 
-	for (i = start; i <= end; i++) {
+	for (i = start; i <= end_addr; i++) {
 		page = i << 8;
 		for (j = page; j < page + RTW89_MAC_PAGE_SIZE; j += 16) {
-			seq_printf(m, "%08xh : ", 0x18600000 + j);
+			p += scnprintf(p, end - p, "%08xh : ", 0x18600000 + j);
 			for (k = 0; k < 4; k++) {
 				val = rtw89_read32(rtwdev, j + (k << 2));
-				seq_printf(m, "%08x ", val);
+				p += scnprintf(p, end - p, "%08x ", val);
 			}
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 		}
 	}
 
-	return 0;
+	return p - buf;
 }
 
 static ssize_t
-rtw89_debug_priv_mac_mem_dump_select(struct file *filp,
-				     const char __user *user_buf,
-				     size_t count, loff_t *loff)
+rtw89_debug_priv_mac_mem_dump_select(struct rtw89_dev *rtwdev,
+				     struct rtw89_debugfs_priv *debugfs_priv,
+				     const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	char buf[32];
-	size_t buf_size;
 	u32 sel, start_addr, len;
 	int num;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
 	num = sscanf(buf, "%x %x %x", &sel, &start_addr, &len);
 	if (num != 3) {
 		rtw89_info(rtwdev, "invalid format: <sel> <start> <len>\n");
@@ -1087,51 +1128,58 @@ rtw89_debug_priv_mac_mem_dump_select(struct file *filp,
 	return count;
 }
 
-static void rtw89_debug_dump_mac_mem(struct seq_file *m,
-				     struct rtw89_dev *rtwdev,
-				     u8 sel, u32 start_addr, u32 len)
+static int rtw89_debug_dump_mac_mem(struct rtw89_dev *rtwdev,
+				    char *buf, size_t bufsz,
+				    u8 sel, u32 start_addr, u32 len)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
 	u32 filter_model_addr = mac->filter_model_addr;
 	u32 indir_access_addr = mac->indir_access_addr;
+	u32 mem_page_size = mac->mem_page_size;
 	u32 base_addr, start_page, residue;
-	u32 i, j, p, pages;
+	char *p = buf, *end = buf + bufsz;
+	u32 i, j, pp, pages;
 	u32 dump_len, remain;
 	u32 val;
 
 	remain = len;
-	pages = len / MAC_MEM_DUMP_PAGE_SIZE + 1;
-	start_page = start_addr / MAC_MEM_DUMP_PAGE_SIZE;
-	residue = start_addr % MAC_MEM_DUMP_PAGE_SIZE;
+	pages = len / mem_page_size + 1;
+	start_page = start_addr / mem_page_size;
+	residue = start_addr % mem_page_size;
 	base_addr = mac->mem_base_addrs[sel];
-	base_addr += start_page * MAC_MEM_DUMP_PAGE_SIZE;
+	base_addr += start_page * mem_page_size;
 
-	for (p = 0; p < pages; p++) {
-		dump_len = min_t(u32, remain, MAC_MEM_DUMP_PAGE_SIZE);
+	for (pp = 0; pp < pages; pp++) {
+		dump_len = min_t(u32, remain, mem_page_size);
 		rtw89_write32(rtwdev, filter_model_addr, base_addr);
 		for (i = indir_access_addr + residue;
 		     i < indir_access_addr + dump_len;) {
-			seq_printf(m, "%08xh:", i);
+			p += scnprintf(p, end - p, "%08xh:", i);
 			for (j = 0;
 			     j < 4 && i < indir_access_addr + dump_len;
 			     j++, i += 4) {
 				val = rtw89_read32(rtwdev, i);
-				seq_printf(m, "  %08x", val);
+				p += scnprintf(p, end - p, "  %08x", val);
 				remain -= 4;
 			}
-			seq_puts(m, "\n");
+			p += scnprintf(p, end - p, "\n");
 		}
-		base_addr += MAC_MEM_DUMP_PAGE_SIZE;
+		base_addr += mem_page_size;
 	}
+
+	return p - buf;
 }
 
-static int
-rtw89_debug_priv_mac_mem_dump_get(struct seq_file *m, void *v)
+static ssize_t
+rtw89_debug_priv_mac_mem_dump_get(struct rtw89_dev *rtwdev,
+				  struct rtw89_debugfs_priv *debugfs_priv,
+				  char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+	char *p = buf, *end = buf + bufsz;
 	bool grant_read = false;
 
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
 	if (debugfs_priv->mac_mem.sel >= RTW89_MAC_MEM_NUM)
 		return -ENOENT;
 
@@ -1148,40 +1196,28 @@ rtw89_debug_priv_mac_mem_dump_get(struct seq_file *m, void *v)
 		}
 	}
 
-	mutex_lock(&rtwdev->mutex);
 	rtw89_leave_ps_mode(rtwdev);
 	if (grant_read)
 		rtw89_write32_set(rtwdev, R_AX_TCR1, B_AX_TCR_FORCE_READ_TXDFIFO);
-	rtw89_debug_dump_mac_mem(m, rtwdev,
-				 debugfs_priv->mac_mem.sel,
-				 debugfs_priv->mac_mem.start,
-				 debugfs_priv->mac_mem.len);
+	p += rtw89_debug_dump_mac_mem(rtwdev, p, end - p,
+				      debugfs_priv->mac_mem.sel,
+				      debugfs_priv->mac_mem.start,
+				      debugfs_priv->mac_mem.len);
 	if (grant_read)
 		rtw89_write32_clr(rtwdev, R_AX_TCR1, B_AX_TCR_FORCE_READ_TXDFIFO);
-	mutex_unlock(&rtwdev->mutex);
 
-	return 0;
+	return p - buf;
 }
 
 static ssize_t
-rtw89_debug_priv_mac_dbg_port_dump_select(struct file *filp,
-					  const char __user *user_buf,
-					  size_t count, loff_t *loff)
+rtw89_debug_priv_mac_dbg_port_dump_select(struct rtw89_dev *rtwdev,
+					  struct rtw89_debugfs_priv *debugfs_priv,
+					  const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	char buf[32];
-	size_t buf_size;
 	int sel, set;
 	int num;
 	bool enable;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
 	num = sscanf(buf, "%d %d", &sel, &set);
 	if (num != 2) {
 		rtw89_info(rtwdev, "invalid format: <sel> <set>\n");
@@ -1217,13 +1253,13 @@ rtw89_debug_priv_mac_dbg_port_dump_select(struct file *filp,
 }
 
 static int rtw89_debug_mac_dump_ss_dbg(struct rtw89_dev *rtwdev,
-				       struct seq_file *m)
+				       char *buf, size_t bufsz)
 {
 	return 0;
 }
 
 static int rtw89_debug_mac_dump_dle_dbg(struct rtw89_dev *rtwdev,
-					struct seq_file *m)
+				       char *buf, size_t bufsz)
 {
 #define DLE_DFI_DUMP(__type, __target, __sel)				\
 ({									\
@@ -1252,7 +1288,7 @@ static int rtw89_debug_mac_dump_dle_dbg(struct rtw89_dev *rtwdev,
 	__data;								\
 })
 
-#define DLE_DFI_FREE_PAGE_DUMP(__m, __type)				\
+#define DLE_DFI_FREE_PAGE_DUMP(__p, __end, __type)			\
 ({									\
 	u32 __freepg, __pubpg;						\
 	u32 __freepg_head, __freepg_tail, __pubpg_num;			\
@@ -1262,24 +1298,25 @@ static int rtw89_debug_mac_dump_dle_dbg(struct rtw89_dev *rtwdev,
 	__freepg_head = FIELD_GET(B_AX_DLE_FREE_HEADPG, __freepg);	\
 	__freepg_tail = FIELD_GET(B_AX_DLE_FREE_TAILPG, __freepg);	\
 	__pubpg_num = FIELD_GET(B_AX_DLE_PUB_PGNUM, __pubpg);		\
-	seq_printf(__m, "[%s] freepg head: %d\n",			\
-		   #__type, __freepg_head);				\
-	seq_printf(__m, "[%s] freepg tail: %d\n",			\
-		   #__type, __freepg_tail);				\
-	seq_printf(__m, "[%s] pubpg num  : %d\n",			\
-		  #__type, __pubpg_num);				\
+	__p += scnprintf(__p, __end - __p, "[%s] freepg head: %d\n",	\
+			 #__type, __freepg_head);			\
+	__p += scnprintf(__p, __end - __p, "[%s] freepg tail: %d\n",	\
+			 #__type, __freepg_tail);			\
+	__p += scnprintf(__p, __end - __p, "[%s] pubpg num  : %d\n",	\
+			 #__type, __pubpg_num);				\
 })
 
-#define case_QUOTA(__m, __type, __id)					\
+#define case_QUOTA(__p, __end, __type, __id)				\
 	case __type##_QTAID_##__id:					\
-		val32 = DLE_DFI_DUMP(__type, QUOTA, __type##_QTAID_##__id);	\
+		val32 = DLE_DFI_DUMP(__type, QUOTA, __type##_QTAID_##__id); \
 		rsv_pgnum = FIELD_GET(B_AX_DLE_RSV_PGNUM, val32);	\
 		use_pgnum = FIELD_GET(B_AX_DLE_USE_PGNUM, val32);	\
-		seq_printf(__m, "[%s][%s] rsv_pgnum: %d\n",		\
-			   #__type, #__id, rsv_pgnum);			\
-		seq_printf(__m, "[%s][%s] use_pgnum: %d\n",		\
-			   #__type, #__id, use_pgnum);			\
+		__p += scnprintf(__p, __end - __p, "[%s][%s] rsv_pgnum: %d\n", \
+				 #__type, #__id, rsv_pgnum);		\
+		__p += scnprintf(__p, __end - __p, "[%s][%s] use_pgnum: %d\n", \
+				 #__type, #__id, use_pgnum);		\
 		break
+	char *p = buf, *end = buf + bufsz;
 	u32 quota_id;
 	u32 val32;
 	u16 rsv_pgnum, use_pgnum;
@@ -1287,38 +1324,39 @@ static int rtw89_debug_mac_dump_dle_dbg(struct rtw89_dev *rtwdev,
 
 	ret = rtw89_mac_check_mac_en(rtwdev, 0, RTW89_DMAC_SEL);
 	if (ret) {
-		seq_puts(m, "[DLE]  : DMAC not enabled\n");
-		return ret;
+		p += scnprintf(p, end - p, "[DLE]  : DMAC not enabled\n");
+		goto out;
 	}
 
-	DLE_DFI_FREE_PAGE_DUMP(m, WDE);
-	DLE_DFI_FREE_PAGE_DUMP(m, PLE);
+	DLE_DFI_FREE_PAGE_DUMP(p, end, WDE);
+	DLE_DFI_FREE_PAGE_DUMP(p, end, PLE);
 	for (quota_id = 0; quota_id <= WDE_QTAID_CPUIO; quota_id++) {
 		switch (quota_id) {
-		case_QUOTA(m, WDE, HOST_IF);
-		case_QUOTA(m, WDE, WLAN_CPU);
-		case_QUOTA(m, WDE, DATA_CPU);
-		case_QUOTA(m, WDE, PKTIN);
-		case_QUOTA(m, WDE, CPUIO);
+		case_QUOTA(p, end, WDE, HOST_IF);
+		case_QUOTA(p, end, WDE, WLAN_CPU);
+		case_QUOTA(p, end, WDE, DATA_CPU);
+		case_QUOTA(p, end, WDE, PKTIN);
+		case_QUOTA(p, end, WDE, CPUIO);
 		}
 	}
 	for (quota_id = 0; quota_id <= PLE_QTAID_CPUIO; quota_id++) {
 		switch (quota_id) {
-		case_QUOTA(m, PLE, B0_TXPL);
-		case_QUOTA(m, PLE, B1_TXPL);
-		case_QUOTA(m, PLE, C2H);
-		case_QUOTA(m, PLE, H2C);
-		case_QUOTA(m, PLE, WLAN_CPU);
-		case_QUOTA(m, PLE, MPDU);
-		case_QUOTA(m, PLE, CMAC0_RX);
-		case_QUOTA(m, PLE, CMAC1_RX);
-		case_QUOTA(m, PLE, CMAC1_BBRPT);
-		case_QUOTA(m, PLE, WDRLS);
-		case_QUOTA(m, PLE, CPUIO);
+		case_QUOTA(p, end, PLE, B0_TXPL);
+		case_QUOTA(p, end, PLE, B1_TXPL);
+		case_QUOTA(p, end, PLE, C2H);
+		case_QUOTA(p, end, PLE, H2C);
+		case_QUOTA(p, end, PLE, WLAN_CPU);
+		case_QUOTA(p, end, PLE, MPDU);
+		case_QUOTA(p, end, PLE, CMAC0_RX);
+		case_QUOTA(p, end, PLE, CMAC1_RX);
+		case_QUOTA(p, end, PLE, CMAC1_BBRPT);
+		case_QUOTA(p, end, PLE, WDRLS);
+		case_QUOTA(p, end, PLE, CPUIO);
 		}
 	}
 
-	return 0;
+out:
+	return p - buf;
 
 #undef case_QUOTA
 #undef DLE_DFI_DUMP
@@ -1326,73 +1364,88 @@ static int rtw89_debug_mac_dump_dle_dbg(struct rtw89_dev *rtwdev,
 }
 
 static int rtw89_debug_mac_dump_dmac_dbg(struct rtw89_dev *rtwdev,
-					 struct seq_file *m)
+					 char *buf, size_t bufsz)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	char *p = buf, *end = buf + bufsz;
 	u32 dmac_err;
 	int i, ret;
 
 	ret = rtw89_mac_check_mac_en(rtwdev, 0, RTW89_DMAC_SEL);
 	if (ret) {
-		seq_puts(m, "[DMAC] : DMAC not enabled\n");
-		return ret;
+		p += scnprintf(p, end - p, "[DMAC] : DMAC not enabled\n");
+		goto out;
 	}
 
 	dmac_err = rtw89_read32(rtwdev, R_AX_DMAC_ERR_ISR);
-	seq_printf(m, "R_AX_DMAC_ERR_ISR=0x%08x\n", dmac_err);
-	seq_printf(m, "R_AX_DMAC_ERR_IMR=0x%08x\n",
-		   rtw89_read32(rtwdev, R_AX_DMAC_ERR_IMR));
+	p += scnprintf(p, end - p, "R_AX_DMAC_ERR_ISR=0x%08x\n", dmac_err);
+	p += scnprintf(p, end - p, "R_AX_DMAC_ERR_IMR=0x%08x\n",
+		       rtw89_read32(rtwdev, R_AX_DMAC_ERR_IMR));
 
 	if (dmac_err) {
-		seq_printf(m, "R_AX_WDE_ERR_FLAG_CFG=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WDE_ERR_FLAG_CFG_NUM1));
-		seq_printf(m, "R_AX_PLE_ERR_FLAG_CFG=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PLE_ERR_FLAG_CFG_NUM1));
+		p += scnprintf(p, end - p, "R_AX_WDE_ERR_FLAG_CFG=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WDE_ERR_FLAG_CFG_NUM1));
+		p += scnprintf(p, end - p, "R_AX_PLE_ERR_FLAG_CFG=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PLE_ERR_FLAG_CFG_NUM1));
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_PLE_ERRFLAG_MSG=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_PLE_ERRFLAG_MSG));
-			seq_printf(m, "R_AX_WDE_ERRFLAG_MSG=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_WDE_ERRFLAG_MSG));
-			seq_printf(m, "R_AX_PLE_DBGERR_LOCKEN=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_PLE_DBGERR_LOCKEN));
-			seq_printf(m, "R_AX_PLE_DBGERR_STS=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_PLE_DBGERR_STS));
+			p += scnprintf(p, end - p,
+				       "R_AX_PLE_ERRFLAG_MSG=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_PLE_ERRFLAG_MSG));
+			p += scnprintf(p, end - p,
+				       "R_AX_WDE_ERRFLAG_MSG=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_WDE_ERRFLAG_MSG));
+			p += scnprintf(p, end - p,
+				       "R_AX_PLE_DBGERR_LOCKEN=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_PLE_DBGERR_LOCKEN));
+			p += scnprintf(p, end - p,
+				       "R_AX_PLE_DBGERR_STS=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_PLE_DBGERR_STS));
 		}
 	}
 
 	if (dmac_err & B_AX_WDRLS_ERR_FLAG) {
-		seq_printf(m, "R_AX_WDRLS_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WDRLS_ERR_IMR));
-		seq_printf(m, "R_AX_WDRLS_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WDRLS_ERR_ISR));
+		p += scnprintf(p, end - p, "R_AX_WDRLS_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WDRLS_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_WDRLS_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WDRLS_ERR_ISR));
 		if (chip->chip_id == RTL8852C)
-			seq_printf(m, "R_AX_RPQ_RXBD_IDX=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX_V1));
+			p += scnprintf(p, end - p,
+				       "R_AX_RPQ_RXBD_IDX=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX_V1));
 		else
-			seq_printf(m, "R_AX_RPQ_RXBD_IDX=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
+			p += scnprintf(p, end - p,
+				       "R_AX_RPQ_RXBD_IDX=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
 	}
 
 	if (dmac_err & B_AX_WSEC_ERR_FLAG) {
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_SEC_ERR_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_ERROR_FLAG_IMR));
-			seq_printf(m, "R_AX_SEC_ERR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_ERROR_FLAG));
-			seq_printf(m, "R_AX_SEC_ENG_CTRL=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_ENG_CTRL));
-			seq_printf(m, "R_AX_SEC_MPDU_PROC=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_MPDU_PROC));
-			seq_printf(m, "R_AX_SEC_CAM_ACCESS=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_CAM_ACCESS));
-			seq_printf(m, "R_AX_SEC_CAM_RDATA=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_CAM_RDATA));
-			seq_printf(m, "R_AX_SEC_DEBUG1=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_DEBUG1));
-			seq_printf(m, "R_AX_SEC_TX_DEBUG=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_TX_DEBUG));
-			seq_printf(m, "R_AX_SEC_RX_DEBUG=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_RX_DEBUG));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_ERR_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_ERROR_FLAG_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_ERR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_ERROR_FLAG));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_ENG_CTRL=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_ENG_CTRL));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_MPDU_PROC=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_MPDU_PROC));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_CAM_ACCESS=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_CAM_ACCESS));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_CAM_RDATA=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_CAM_RDATA));
+			p += scnprintf(p, end - p, "R_AX_SEC_DEBUG1=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_DEBUG1));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_TX_DEBUG=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_TX_DEBUG));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_RX_DEBUG=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_RX_DEBUG));
 
 			rtw89_write32_mask(rtwdev, R_AX_DBG_CTRL,
 					   B_AX_DBG_SEL0, 0x8B);
@@ -1403,187 +1456,229 @@ static int rtw89_debug_mac_dump_dmac_dbg(struct rtw89_dev *rtwdev,
 			for (i = 0; i < 0x10; i++) {
 				rtw89_write32_mask(rtwdev, R_AX_SEC_ENG_CTRL,
 						   B_AX_SEC_DBG_PORT_FIELD_MASK, i);
-				seq_printf(m, "sel=%x,R_AX_SEC_DEBUG2=0x%08x\n",
-					   i, rtw89_read32(rtwdev, R_AX_SEC_DEBUG2));
+				p += scnprintf(p, end - p,
+					       "sel=%x,R_AX_SEC_DEBUG2=0x%08x\n",
+					       i,
+					       rtw89_read32(rtwdev, R_AX_SEC_DEBUG2));
 			}
 		} else {
-			seq_printf(m, "R_AX_SEC_ERR_IMR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_DEBUG));
-			seq_printf(m, "R_AX_SEC_ENG_CTRL=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_ENG_CTRL));
-			seq_printf(m, "R_AX_SEC_MPDU_PROC=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_MPDU_PROC));
-			seq_printf(m, "R_AX_SEC_CAM_ACCESS=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_CAM_ACCESS));
-			seq_printf(m, "R_AX_SEC_CAM_RDATA=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_CAM_RDATA));
-			seq_printf(m, "R_AX_SEC_CAM_WDATA=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_CAM_WDATA));
-			seq_printf(m, "R_AX_SEC_TX_DEBUG=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_TX_DEBUG));
-			seq_printf(m, "R_AX_SEC_RX_DEBUG=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_RX_DEBUG));
-			seq_printf(m, "R_AX_SEC_TRX_PKT_CNT=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_TRX_PKT_CNT));
-			seq_printf(m, "R_AX_SEC_TRX_BLK_CNT=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_SEC_TRX_BLK_CNT));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_ERR_IMR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_DEBUG));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_ENG_CTRL=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_ENG_CTRL));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_MPDU_PROC=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_MPDU_PROC));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_CAM_ACCESS=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_CAM_ACCESS));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_CAM_RDATA=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_CAM_RDATA));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_CAM_WDATA=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_CAM_WDATA));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_TX_DEBUG=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_TX_DEBUG));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_RX_DEBUG=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_RX_DEBUG));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_TRX_PKT_CNT=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_TRX_PKT_CNT));
+			p += scnprintf(p, end - p,
+				       "R_AX_SEC_TRX_BLK_CNT=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_SEC_TRX_BLK_CNT));
 		}
 	}
 
 	if (dmac_err & B_AX_MPDU_ERR_FLAG) {
-		seq_printf(m, "R_AX_MPDU_TX_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_MPDU_TX_ERR_IMR));
-		seq_printf(m, "R_AX_MPDU_TX_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_MPDU_TX_ERR_ISR));
-		seq_printf(m, "R_AX_MPDU_RX_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_MPDU_RX_ERR_IMR));
-		seq_printf(m, "R_AX_MPDU_RX_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_MPDU_RX_ERR_ISR));
+		p += scnprintf(p, end - p, "R_AX_MPDU_TX_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_MPDU_TX_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_MPDU_TX_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_MPDU_TX_ERR_ISR));
+		p += scnprintf(p, end - p, "R_AX_MPDU_RX_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_MPDU_RX_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_MPDU_RX_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_MPDU_RX_ERR_ISR));
 	}
 
 	if (dmac_err & B_AX_STA_SCHEDULER_ERR_FLAG) {
-		seq_printf(m, "R_AX_STA_SCHEDULER_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_IMR));
-		seq_printf(m, "R_AX_STA_SCHEDULER_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_ISR));
+		p += scnprintf(p, end - p,
+			       "R_AX_STA_SCHEDULER_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_IMR));
+		p += scnprintf(p, end - p,
+			       "R_AX_STA_SCHEDULER_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_ISR));
 	}
 
 	if (dmac_err & B_AX_WDE_DLE_ERR_FLAG) {
-		seq_printf(m, "R_AX_WDE_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WDE_ERR_IMR));
-		seq_printf(m, "R_AX_WDE_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WDE_ERR_ISR));
-		seq_printf(m, "R_AX_PLE_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PLE_ERR_IMR));
-		seq_printf(m, "R_AX_PLE_ERR_FLAG_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PLE_ERR_FLAG_ISR));
+		p += scnprintf(p, end - p, "R_AX_WDE_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WDE_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_WDE_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WDE_ERR_ISR));
+		p += scnprintf(p, end - p, "R_AX_PLE_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PLE_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_PLE_ERR_FLAG_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PLE_ERR_FLAG_ISR));
 	}
 
 	if (dmac_err & B_AX_TXPKTCTRL_ERR_FLAG) {
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_TXPKTCTL_B0_ERRFLAG_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_TXPKTCTL_B0_ERRFLAG_IMR));
-			seq_printf(m, "R_AX_TXPKTCTL_B0_ERRFLAG_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_TXPKTCTL_B0_ERRFLAG_ISR));
-			seq_printf(m, "R_AX_TXPKTCTL_B1_ERRFLAG_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_TXPKTCTL_B1_ERRFLAG_IMR));
-			seq_printf(m, "R_AX_TXPKTCTL_B1_ERRFLAG_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_TXPKTCTL_B1_ERRFLAG_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_TXPKTCTL_B0_ERRFLAG_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_TXPKTCTL_B0_ERRFLAG_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_TXPKTCTL_B0_ERRFLAG_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_TXPKTCTL_B0_ERRFLAG_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_TXPKTCTL_B1_ERRFLAG_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_TXPKTCTL_B1_ERRFLAG_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_TXPKTCTL_B1_ERRFLAG_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_TXPKTCTL_B1_ERRFLAG_ISR));
 		} else {
-			seq_printf(m, "R_AX_TXPKTCTL_ERR_IMR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_TXPKTCTL_ERR_IMR_ISR));
-			seq_printf(m, "R_AX_TXPKTCTL_ERR_IMR_ISR_B1=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_TXPKTCTL_ERR_IMR_ISR_B1));
+			p += scnprintf(p, end - p,
+				       "R_AX_TXPKTCTL_ERR_IMR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_TXPKTCTL_ERR_IMR_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_TXPKTCTL_ERR_IMR_ISR_B1=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_TXPKTCTL_ERR_IMR_ISR_B1));
 		}
 	}
 
 	if (dmac_err & B_AX_PLE_DLE_ERR_FLAG) {
-		seq_printf(m, "R_AX_WDE_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WDE_ERR_IMR));
-		seq_printf(m, "R_AX_WDE_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WDE_ERR_ISR));
-		seq_printf(m, "R_AX_PLE_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PLE_ERR_IMR));
-		seq_printf(m, "R_AX_PLE_ERR_FLAG_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PLE_ERR_FLAG_ISR));
-		seq_printf(m, "R_AX_WD_CPUQ_OP_0=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_0));
-		seq_printf(m, "R_AX_WD_CPUQ_OP_1=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_1));
-		seq_printf(m, "R_AX_WD_CPUQ_OP_2=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_2));
-		seq_printf(m, "R_AX_WD_CPUQ_OP_STATUS=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_STATUS));
-		seq_printf(m, "R_AX_PL_CPUQ_OP_0=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_0));
-		seq_printf(m, "R_AX_PL_CPUQ_OP_1=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_1));
-		seq_printf(m, "R_AX_PL_CPUQ_OP_2=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_2));
-		seq_printf(m, "R_AX_PL_CPUQ_OP_STATUS=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_STATUS));
+		p += scnprintf(p, end - p, "R_AX_WDE_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WDE_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_WDE_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WDE_ERR_ISR));
+		p += scnprintf(p, end - p, "R_AX_PLE_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PLE_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_PLE_ERR_FLAG_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PLE_ERR_FLAG_ISR));
+		p += scnprintf(p, end - p, "R_AX_WD_CPUQ_OP_0=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_0));
+		p += scnprintf(p, end - p, "R_AX_WD_CPUQ_OP_1=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_1));
+		p += scnprintf(p, end - p, "R_AX_WD_CPUQ_OP_2=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_2));
+		p += scnprintf(p, end - p, "R_AX_WD_CPUQ_OP_STATUS=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_STATUS));
+		p += scnprintf(p, end - p, "R_AX_PL_CPUQ_OP_0=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_0));
+		p += scnprintf(p, end - p, "R_AX_PL_CPUQ_OP_1=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_1));
+		p += scnprintf(p, end - p, "R_AX_PL_CPUQ_OP_2=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_2));
+		p += scnprintf(p, end - p, "R_AX_PL_CPUQ_OP_STATUS=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_STATUS));
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_RX_CTRL0=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RX_CTRL0));
-			seq_printf(m, "R_AX_RX_CTRL1=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RX_CTRL1));
-			seq_printf(m, "R_AX_RX_CTRL2=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RX_CTRL2));
+			p += scnprintf(p, end - p, "R_AX_RX_CTRL0=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RX_CTRL0));
+			p += scnprintf(p, end - p, "R_AX_RX_CTRL1=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RX_CTRL1));
+			p += scnprintf(p, end - p, "R_AX_RX_CTRL2=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RX_CTRL2));
 		} else {
-			seq_printf(m, "R_AX_RXDMA_PKT_INFO_0=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_0));
-			seq_printf(m, "R_AX_RXDMA_PKT_INFO_1=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_1));
-			seq_printf(m, "R_AX_RXDMA_PKT_INFO_2=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_2));
+			p += scnprintf(p, end - p,
+				       "R_AX_RXDMA_PKT_INFO_0=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_0));
+			p += scnprintf(p, end - p,
+				       "R_AX_RXDMA_PKT_INFO_1=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_1));
+			p += scnprintf(p, end - p,
+				       "R_AX_RXDMA_PKT_INFO_2=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_2));
 		}
 	}
 
 	if (dmac_err & B_AX_PKTIN_ERR_FLAG) {
-		seq_printf(m, "R_AX_PKTIN_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PKTIN_ERR_IMR));
-		seq_printf(m, "R_AX_PKTIN_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_PKTIN_ERR_ISR));
+		p += scnprintf(p, end - p, "R_AX_PKTIN_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PKTIN_ERR_IMR));
+		p += scnprintf(p, end - p, "R_AX_PKTIN_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_PKTIN_ERR_ISR));
 	}
 
 	if (dmac_err & B_AX_DISPATCH_ERR_FLAG) {
-		seq_printf(m, "R_AX_HOST_DISPATCHER_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_IMR));
-		seq_printf(m, "R_AX_HOST_DISPATCHER_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_ISR));
-		seq_printf(m, "R_AX_CPU_DISPATCHER_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_IMR));
-		seq_printf(m, "R_AX_CPU_DISPATCHER_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_ISR));
-		seq_printf(m, "R_AX_OTHER_DISPATCHER_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_IMR));
-		seq_printf(m, "R_AX_OTHER_DISPATCHER_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_ISR));
+		p += scnprintf(p, end - p,
+			       "R_AX_HOST_DISPATCHER_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_IMR));
+		p += scnprintf(p, end - p,
+			       "R_AX_HOST_DISPATCHER_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_ISR));
+		p += scnprintf(p, end - p,
+			       "R_AX_CPU_DISPATCHER_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_IMR));
+		p += scnprintf(p, end - p,
+			       "R_AX_CPU_DISPATCHER_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_ISR));
+		p += scnprintf(p, end - p,
+			       "R_AX_OTHER_DISPATCHER_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_IMR));
+		p += scnprintf(p, end - p,
+			       "R_AX_OTHER_DISPATCHER_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_ISR));
 	}
 
 	if (dmac_err & B_AX_BBRPT_ERR_FLAG) {
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_BBRPT_COM_ERR_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_COM_ERR_IMR));
-			seq_printf(m, "R_AX_BBRPT_COM_ERR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_COM_ERR_ISR));
-			seq_printf(m, "R_AX_BBRPT_CHINFO_ERR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_ISR));
-			seq_printf(m, "R_AX_BBRPT_CHINFO_ERR_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_IMR));
-			seq_printf(m, "R_AX_BBRPT_DFS_ERR_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_IMR));
-			seq_printf(m, "R_AX_BBRPT_DFS_ERR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_COM_ERR_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_COM_ERR_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_COM_ERR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_COM_ERR_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_CHINFO_ERR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_CHINFO_ERR_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_DFS_ERR_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_DFS_ERR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_ISR));
 		} else {
-			seq_printf(m, "R_AX_BBRPT_COM_ERR_IMR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_COM_ERR_IMR_ISR));
-			seq_printf(m, "R_AX_BBRPT_CHINFO_ERR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_ISR));
-			seq_printf(m, "R_AX_BBRPT_CHINFO_ERR_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_IMR));
-			seq_printf(m, "R_AX_BBRPT_DFS_ERR_IMR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_IMR));
-			seq_printf(m, "R_AX_BBRPT_DFS_ERR_ISR=0x%08x\n",
-				   rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_COM_ERR_IMR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_COM_ERR_IMR_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_CHINFO_ERR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_ISR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_CHINFO_ERR_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_CHINFO_ERR_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_DFS_ERR_IMR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_IMR));
+			p += scnprintf(p, end - p,
+				       "R_AX_BBRPT_DFS_ERR_ISR=0x%08x\n",
+				       rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_ISR));
 		}
 	}
 
 	if (dmac_err & B_AX_HAXIDMA_ERR_FLAG && chip->chip_id == RTL8852C) {
-		seq_printf(m, "R_AX_HAXIDMA_ERR_IMR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_HAXI_IDCT_MSK));
-		seq_printf(m, "R_AX_HAXIDMA_ERR_ISR=0x%08x\n",
-			   rtw89_read32(rtwdev, R_AX_HAXI_IDCT));
+		p += scnprintf(p, end - p, "R_AX_HAXIDMA_ERR_IMR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_HAXI_IDCT_MSK));
+		p += scnprintf(p, end - p, "R_AX_HAXIDMA_ERR_ISR=0x%08x\n",
+			       rtw89_read32(rtwdev, R_AX_HAXI_IDCT));
 	}
 
-	return 0;
+out:
+	return p - buf;
 }
 
 static int rtw89_debug_mac_dump_cmac_err(struct rtw89_dev *rtwdev,
-					 struct seq_file *m,
+					 char *buf, size_t bufsz,
 					 enum rtw89_mac_idx band)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	char *p = buf, *end = buf + bufsz;
 	u32 offset = 0;
 	u32 cmac_err;
 	int ret;
@@ -1591,96 +1686,127 @@ static int rtw89_debug_mac_dump_cmac_err(struct rtw89_dev *rtwdev,
 	ret = rtw89_mac_check_mac_en(rtwdev, band, RTW89_CMAC_SEL);
 	if (ret) {
 		if (band)
-			seq_puts(m, "[CMAC] : CMAC1 not enabled\n");
+			p += scnprintf(p, end - p,
+				       "[CMAC] : CMAC1 not enabled\n");
 		else
-			seq_puts(m, "[CMAC] : CMAC0 not enabled\n");
-		return ret;
+			p += scnprintf(p, end - p,
+				       "[CMAC] : CMAC0 not enabled\n");
+		goto out;
 	}
 
 	if (band)
 		offset = RTW89_MAC_AX_BAND_REG_OFFSET;
 
 	cmac_err = rtw89_read32(rtwdev, R_AX_CMAC_ERR_ISR + offset);
-	seq_printf(m, "R_AX_CMAC_ERR_ISR [%d]=0x%08x\n", band,
-		   rtw89_read32(rtwdev, R_AX_CMAC_ERR_ISR + offset));
-	seq_printf(m, "R_AX_CMAC_FUNC_EN [%d]=0x%08x\n", band,
-		   rtw89_read32(rtwdev, R_AX_CMAC_FUNC_EN + offset));
-	seq_printf(m, "R_AX_CK_EN [%d]=0x%08x\n", band,
-		   rtw89_read32(rtwdev, R_AX_CK_EN + offset));
+	p += scnprintf(p, end - p, "R_AX_CMAC_ERR_ISR [%d]=0x%08x\n", band,
+		       rtw89_read32(rtwdev, R_AX_CMAC_ERR_ISR + offset));
+	p += scnprintf(p, end - p, "R_AX_CMAC_FUNC_EN [%d]=0x%08x\n", band,
+		       rtw89_read32(rtwdev, R_AX_CMAC_FUNC_EN + offset));
+	p += scnprintf(p, end - p, "R_AX_CK_EN [%d]=0x%08x\n", band,
+		       rtw89_read32(rtwdev, R_AX_CK_EN + offset));
 
 	if (cmac_err & B_AX_SCHEDULE_TOP_ERR_IND) {
-		seq_printf(m, "R_AX_SCHEDULE_ERR_IMR [%d]=0x%08x\n", band,
-			   rtw89_read32(rtwdev, R_AX_SCHEDULE_ERR_IMR + offset));
-		seq_printf(m, "R_AX_SCHEDULE_ERR_ISR [%d]=0x%08x\n", band,
-			   rtw89_read32(rtwdev, R_AX_SCHEDULE_ERR_ISR + offset));
+		p += scnprintf(p, end - p,
+			       "R_AX_SCHEDULE_ERR_IMR [%d]=0x%08x\n", band,
+			       rtw89_read32(rtwdev, R_AX_SCHEDULE_ERR_IMR + offset));
+		p += scnprintf(p, end - p,
+			       "R_AX_SCHEDULE_ERR_ISR [%d]=0x%08x\n", band,
+			       rtw89_read32(rtwdev, R_AX_SCHEDULE_ERR_ISR + offset));
 	}
 
 	if (cmac_err & B_AX_PTCL_TOP_ERR_IND) {
-		seq_printf(m, "R_AX_PTCL_IMR0 [%d]=0x%08x\n", band,
-			   rtw89_read32(rtwdev, R_AX_PTCL_IMR0 + offset));
-		seq_printf(m, "R_AX_PTCL_ISR0 [%d]=0x%08x\n", band,
-			   rtw89_read32(rtwdev, R_AX_PTCL_ISR0 + offset));
+		p += scnprintf(p, end - p, "R_AX_PTCL_IMR0 [%d]=0x%08x\n",
+			       band,
+			       rtw89_read32(rtwdev, R_AX_PTCL_IMR0 + offset));
+		p += scnprintf(p, end - p, "R_AX_PTCL_ISR0 [%d]=0x%08x\n",
+			       band,
+			       rtw89_read32(rtwdev, R_AX_PTCL_ISR0 + offset));
 	}
 
 	if (cmac_err & B_AX_DMA_TOP_ERR_IND) {
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_RX_ERR_FLAG [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_RX_ERR_FLAG + offset));
-			seq_printf(m, "R_AX_RX_ERR_FLAG_IMR [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_RX_ERR_FLAG_IMR + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_RX_ERR_FLAG [%d]=0x%08x\n", band,
+				       rtw89_read32(rtwdev, R_AX_RX_ERR_FLAG + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_RX_ERR_FLAG_IMR [%d]=0x%08x\n",
+				       band,
+				       rtw89_read32(rtwdev, R_AX_RX_ERR_FLAG_IMR + offset));
 		} else {
-			seq_printf(m, "R_AX_DLE_CTRL [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_DLE_CTRL + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_DLE_CTRL [%d]=0x%08x\n", band,
+				       rtw89_read32(rtwdev, R_AX_DLE_CTRL + offset));
 		}
 	}
 
 	if (cmac_err & B_AX_DMA_TOP_ERR_IND || cmac_err & B_AX_WMAC_RX_ERR_IND) {
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_PHYINFO_ERR_ISR [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_PHYINFO_ERR_ISR + offset));
-			seq_printf(m, "R_AX_PHYINFO_ERR_IMR [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_PHYINFO_ERR_IMR + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_PHYINFO_ERR_ISR [%d]=0x%08x\n",
+				       band,
+				       rtw89_read32(rtwdev, R_AX_PHYINFO_ERR_ISR + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_PHYINFO_ERR_IMR [%d]=0x%08x\n",
+				       band,
+				       rtw89_read32(rtwdev, R_AX_PHYINFO_ERR_IMR + offset));
 		} else {
-			seq_printf(m, "R_AX_PHYINFO_ERR_IMR [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_PHYINFO_ERR_IMR + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_PHYINFO_ERR_IMR [%d]=0x%08x\n",
+				       band,
+				       rtw89_read32(rtwdev, R_AX_PHYINFO_ERR_IMR + offset));
 		}
 	}
 
 	if (cmac_err & B_AX_TXPWR_CTRL_ERR_IND) {
-		seq_printf(m, "R_AX_TXPWR_IMR [%d]=0x%08x\n", band,
-			   rtw89_read32(rtwdev, R_AX_TXPWR_IMR + offset));
-		seq_printf(m, "R_AX_TXPWR_ISR [%d]=0x%08x\n", band,
-			   rtw89_read32(rtwdev, R_AX_TXPWR_ISR + offset));
+		p += scnprintf(p, end - p, "R_AX_TXPWR_IMR [%d]=0x%08x\n",
+			       band,
+			       rtw89_read32(rtwdev, R_AX_TXPWR_IMR + offset));
+		p += scnprintf(p, end - p, "R_AX_TXPWR_ISR [%d]=0x%08x\n",
+			       band,
+			       rtw89_read32(rtwdev, R_AX_TXPWR_ISR + offset));
 	}
 
 	if (cmac_err & B_AX_WMAC_TX_ERR_IND) {
 		if (chip->chip_id == RTL8852C) {
-			seq_printf(m, "R_AX_TRXPTCL_ERROR_INDICA [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_TRXPTCL_ERROR_INDICA + offset));
-			seq_printf(m, "R_AX_TRXPTCL_ERROR_INDICA_MASK [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_TRXPTCL_ERROR_INDICA_MASK + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_TRXPTCL_ERROR_INDICA [%d]=0x%08x\n",
+				       band,
+				       rtw89_read32(rtwdev,
+						    R_AX_TRXPTCL_ERROR_INDICA + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_TRXPTCL_ERROR_INDICA_MASK [%d]=0x%08x\n",
+				       band,
+				       rtw89_read32(rtwdev,
+						    R_AX_TRXPTCL_ERROR_INDICA_MASK + offset));
 		} else {
-			seq_printf(m, "R_AX_TMAC_ERR_IMR_ISR [%d]=0x%08x\n", band,
-				   rtw89_read32(rtwdev, R_AX_TMAC_ERR_IMR_ISR + offset));
+			p += scnprintf(p, end - p,
+				       "R_AX_TMAC_ERR_IMR_ISR [%d]=0x%08x\n",
+				       band,
+				       rtw89_read32(rtwdev,
+						    R_AX_TMAC_ERR_IMR_ISR + offset));
 		}
-		seq_printf(m, "R_AX_DBGSEL_TRXPTCL [%d]=0x%08x\n", band,
-			   rtw89_read32(rtwdev, R_AX_DBGSEL_TRXPTCL + offset));
+		p += scnprintf(p, end - p,
+			       "R_AX_DBGSEL_TRXPTCL [%d]=0x%08x\n", band,
+			       rtw89_read32(rtwdev, R_AX_DBGSEL_TRXPTCL + offset));
 	}
 
-	seq_printf(m, "R_AX_CMAC_ERR_IMR [%d]=0x%08x\n", band,
-		   rtw89_read32(rtwdev, R_AX_CMAC_ERR_IMR + offset));
+	p += scnprintf(p, end - p, "R_AX_CMAC_ERR_IMR [%d]=0x%08x\n", band,
+		       rtw89_read32(rtwdev, R_AX_CMAC_ERR_IMR + offset));
 
-	return 0;
+out:
+	return p - buf;
 }
 
 static int rtw89_debug_mac_dump_cmac_dbg(struct rtw89_dev *rtwdev,
-					 struct seq_file *m)
+					 char *buf, size_t bufsz)
 {
-	rtw89_debug_mac_dump_cmac_err(rtwdev, m, RTW89_MAC_0);
+	char *p = buf, *end = buf + bufsz;
+
+	p += rtw89_debug_mac_dump_cmac_err(rtwdev, p, end - p, RTW89_MAC_0);
 	if (rtwdev->dbcc_en)
-		rtw89_debug_mac_dump_cmac_err(rtwdev, m, RTW89_MAC_1);
+		p += rtw89_debug_mac_dump_cmac_err(rtwdev, p, end - p, RTW89_MAC_1);
 
-	return 0;
+	return p - buf;
 }
 
 static const struct rtw89_mac_dbg_port_info dbg_port_ptcl_c0 = {
@@ -2486,11 +2612,12 @@ static const struct rtw89_mac_dbg_port_info dbg_port_pcie_misc2 = {
 	.rd_msk = B_AX_DEBUG_ST_MASK
 };
 
-static const struct rtw89_mac_dbg_port_info *
-rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
-			     struct rtw89_dev *rtwdev, u32 sel)
+static int
+rtw89_debug_mac_dbg_port_sel(struct rtw89_dev *rtwdev, char *buf, size_t bufsz,
+			     u32 sel, const struct rtw89_mac_dbg_port_info **ppinfo)
 {
-	const struct rtw89_mac_dbg_port_info *info;
+	const struct rtw89_mac_dbg_port_info *info = NULL;
+	char *p = buf, *end = buf + bufsz;
 	u32 index;
 	u32 val32;
 	u16 val16;
@@ -2502,28 +2629,28 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val16 = rtw89_read16(rtwdev, R_AX_PTCL_DBG);
 		val16 |= B_AX_PTCL_DBG_EN;
 		rtw89_write16(rtwdev, R_AX_PTCL_DBG, val16);
-		seq_puts(m, "Enable PTCL C0 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable PTCL C0 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PTCL_C1:
 		info = &dbg_port_ptcl_c1;
 		val16 = rtw89_read16(rtwdev, R_AX_PTCL_DBG_C1);
 		val16 |= B_AX_PTCL_DBG_EN;
 		rtw89_write16(rtwdev, R_AX_PTCL_DBG_C1, val16);
-		seq_puts(m, "Enable PTCL C1 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable PTCL C1 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_SCH_C0:
 		info = &dbg_port_sch_c0;
 		val32 = rtw89_read32(rtwdev, R_AX_SCH_DBG_SEL);
 		val32 |= B_AX_SCH_DBG_EN;
 		rtw89_write32(rtwdev, R_AX_SCH_DBG_SEL, val32);
-		seq_puts(m, "Enable SCH C0 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable SCH C0 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_SCH_C1:
 		info = &dbg_port_sch_c1;
 		val32 = rtw89_read32(rtwdev, R_AX_SCH_DBG_SEL_C1);
 		val32 |= B_AX_SCH_DBG_EN;
 		rtw89_write32(rtwdev, R_AX_SCH_DBG_SEL_C1, val32);
-		seq_puts(m, "Enable SCH C1 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable SCH C1 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TMAC_C0:
 		info = &dbg_port_tmac_c0;
@@ -2540,7 +2667,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = rtw89_read32(rtwdev, R_AX_SYS_STATUS1);
 		val32 = u32_replace_bits(val32, MAC_DBG_SEL, B_AX_SEL_0XC0_MASK);
 		rtw89_write32(rtwdev, R_AX_SYS_STATUS1, val32);
-		seq_puts(m, "Enable TMAC C0 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable TMAC C0 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TMAC_C1:
 		info = &dbg_port_tmac_c1;
@@ -2557,7 +2684,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = rtw89_read32(rtwdev, R_AX_SYS_STATUS1);
 		val32 = u32_replace_bits(val32, MAC_DBG_SEL, B_AX_SEL_0XC0_MASK);
 		rtw89_write32(rtwdev, R_AX_SYS_STATUS1, val32);
-		seq_puts(m, "Enable TMAC C1 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable TMAC C1 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_RMAC_C0:
 		info = &dbg_port_rmac_c0;
@@ -2579,7 +2706,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val8 = u8_replace_bits(val8, RMAC_CMAC_DBG_SEL,
 				       B_AX_DBGSEL_TRXPTCL_MASK);
 		rtw89_write8(rtwdev, R_AX_DBGSEL_TRXPTCL, val8);
-		seq_puts(m, "Enable RMAC C0 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable RMAC C0 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_RMAC_C1:
 		info = &dbg_port_rmac_c1;
@@ -2601,23 +2728,23 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val8 = u8_replace_bits(val8, RMAC_CMAC_DBG_SEL,
 				       B_AX_DBGSEL_TRXPTCL_MASK);
 		rtw89_write8(rtwdev, R_AX_DBGSEL_TRXPTCL_C1, val8);
-		seq_puts(m, "Enable RMAC C1 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable RMAC C1 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_RMACST_C0:
 		info = &dbg_port_rmacst_c0;
-		seq_puts(m, "Enable RMAC state C0 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable RMAC state C0 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_RMACST_C1:
 		info = &dbg_port_rmacst_c1;
-		seq_puts(m, "Enable RMAC state C1 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable RMAC state C1 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_RMAC_PLCP_C0:
 		info = &dbg_port_rmac_plcp_c0;
-		seq_puts(m, "Enable RMAC PLCP C0 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable RMAC PLCP C0 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_RMAC_PLCP_C1:
 		info = &dbg_port_rmac_plcp_c1;
-		seq_puts(m, "Enable RMAC PLCP C1 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable RMAC PLCP C1 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TRXPTCL_C0:
 		info = &dbg_port_trxptcl_c0;
@@ -2629,7 +2756,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = rtw89_read32(rtwdev, R_AX_SYS_STATUS1);
 		val32 = u32_replace_bits(val32, MAC_DBG_SEL, B_AX_SEL_0XC0_MASK);
 		rtw89_write32(rtwdev, R_AX_SYS_STATUS1, val32);
-		seq_puts(m, "Enable TRXPTCL C0 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable TRXPTCL C0 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TRXPTCL_C1:
 		info = &dbg_port_trxptcl_c1;
@@ -2641,131 +2768,137 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = rtw89_read32(rtwdev, R_AX_SYS_STATUS1);
 		val32 = u32_replace_bits(val32, MAC_DBG_SEL, B_AX_SEL_0XC0_MASK);
 		rtw89_write32(rtwdev, R_AX_SYS_STATUS1, val32);
-		seq_puts(m, "Enable TRXPTCL C1 dbgport.\n");
+		p += scnprintf(p, end - p, "Enable TRXPTCL C1 dbgport.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TX_INFOL_C0:
 		info = &dbg_port_tx_infol_c0;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1, val32);
-		seq_puts(m, "Enable tx infol dump.\n");
+		p += scnprintf(p, end - p, "Enable tx infol dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TX_INFOH_C0:
 		info = &dbg_port_tx_infoh_c0;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1, val32);
-		seq_puts(m, "Enable tx infoh dump.\n");
+		p += scnprintf(p, end - p, "Enable tx infoh dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TX_INFOL_C1:
 		info = &dbg_port_tx_infol_c1;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1_C1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1_C1, val32);
-		seq_puts(m, "Enable tx infol dump.\n");
+		p += scnprintf(p, end - p, "Enable tx infol dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TX_INFOH_C1:
 		info = &dbg_port_tx_infoh_c1;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1_C1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1_C1, val32);
-		seq_puts(m, "Enable tx infoh dump.\n");
+		p += scnprintf(p, end - p, "Enable tx infoh dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TXTF_INFOL_C0:
 		info = &dbg_port_txtf_infol_c0;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1, val32);
-		seq_puts(m, "Enable tx tf infol dump.\n");
+		p += scnprintf(p, end - p, "Enable tx tf infol dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TXTF_INFOH_C0:
 		info = &dbg_port_txtf_infoh_c0;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1, val32);
-		seq_puts(m, "Enable tx tf infoh dump.\n");
+		p += scnprintf(p, end - p, "Enable tx tf infoh dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TXTF_INFOL_C1:
 		info = &dbg_port_txtf_infol_c1;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1_C1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1_C1, val32);
-		seq_puts(m, "Enable tx tf infol dump.\n");
+		p += scnprintf(p, end - p, "Enable tx tf infol dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_TXTF_INFOH_C1:
 		info = &dbg_port_txtf_infoh_c1;
 		val32 = rtw89_read32(rtwdev, R_AX_TCR1_C1);
 		val32 |= B_AX_TCR_FORCE_READ_TXDFIFO;
 		rtw89_write32(rtwdev, R_AX_TCR1_C1, val32);
-		seq_puts(m, "Enable tx tf infoh dump.\n");
+		p += scnprintf(p, end - p, "Enable tx tf infoh dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_BUFMGN_FREEPG:
 		info = &dbg_port_wde_bufmgn_freepg;
-		seq_puts(m, "Enable wde bufmgn freepg dump.\n");
+		p += scnprintf(p, end - p, "Enable wde bufmgn freepg dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_BUFMGN_QUOTA:
 		info = &dbg_port_wde_bufmgn_quota;
-		seq_puts(m, "Enable wde bufmgn quota dump.\n");
+		p += scnprintf(p, end - p, "Enable wde bufmgn quota dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_BUFMGN_PAGELLT:
 		info = &dbg_port_wde_bufmgn_pagellt;
-		seq_puts(m, "Enable wde bufmgn pagellt dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable wde bufmgn pagellt dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_BUFMGN_PKTINFO:
 		info = &dbg_port_wde_bufmgn_pktinfo;
-		seq_puts(m, "Enable wde bufmgn pktinfo dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable wde bufmgn pktinfo dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_QUEMGN_PREPKT:
 		info = &dbg_port_wde_quemgn_prepkt;
-		seq_puts(m, "Enable wde quemgn prepkt dump.\n");
+		p += scnprintf(p, end - p, "Enable wde quemgn prepkt dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_QUEMGN_NXTPKT:
 		info = &dbg_port_wde_quemgn_nxtpkt;
-		seq_puts(m, "Enable wde quemgn nxtpkt dump.\n");
+		p += scnprintf(p, end - p, "Enable wde quemgn nxtpkt dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_QUEMGN_QLNKTBL:
 		info = &dbg_port_wde_quemgn_qlnktbl;
-		seq_puts(m, "Enable wde quemgn qlnktbl dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable wde quemgn qlnktbl dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_WDE_QUEMGN_QEMPTY:
 		info = &dbg_port_wde_quemgn_qempty;
-		seq_puts(m, "Enable wde quemgn qempty dump.\n");
+		p += scnprintf(p, end - p, "Enable wde quemgn qempty dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_BUFMGN_FREEPG:
 		info = &dbg_port_ple_bufmgn_freepg;
-		seq_puts(m, "Enable ple bufmgn freepg dump.\n");
+		p += scnprintf(p, end - p, "Enable ple bufmgn freepg dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_BUFMGN_QUOTA:
 		info = &dbg_port_ple_bufmgn_quota;
-		seq_puts(m, "Enable ple bufmgn quota dump.\n");
+		p += scnprintf(p, end - p, "Enable ple bufmgn quota dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_BUFMGN_PAGELLT:
 		info = &dbg_port_ple_bufmgn_pagellt;
-		seq_puts(m, "Enable ple bufmgn pagellt dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable ple bufmgn pagellt dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_BUFMGN_PKTINFO:
 		info = &dbg_port_ple_bufmgn_pktinfo;
-		seq_puts(m, "Enable ple bufmgn pktinfo dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable ple bufmgn pktinfo dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_QUEMGN_PREPKT:
 		info = &dbg_port_ple_quemgn_prepkt;
-		seq_puts(m, "Enable ple quemgn prepkt dump.\n");
+		p += scnprintf(p, end - p, "Enable ple quemgn prepkt dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_QUEMGN_NXTPKT:
 		info = &dbg_port_ple_quemgn_nxtpkt;
-		seq_puts(m, "Enable ple quemgn nxtpkt dump.\n");
+		p += scnprintf(p, end - p, "Enable ple quemgn nxtpkt dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_QUEMGN_QLNKTBL:
 		info = &dbg_port_ple_quemgn_qlnktbl;
-		seq_puts(m, "Enable ple quemgn qlnktbl dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable ple quemgn qlnktbl dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PLE_QUEMGN_QEMPTY:
 		info = &dbg_port_ple_quemgn_qempty;
-		seq_puts(m, "Enable ple quemgn qempty dump.\n");
+		p += scnprintf(p, end - p, "Enable ple quemgn qempty dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PKTINFO:
 		info = &dbg_port_pktinfo;
-		seq_puts(m, "Enable pktinfo dump.\n");
+		p += scnprintf(p, end - p, "Enable pktinfo dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_TX0:
 		rtw89_write32_mask(rtwdev, R_AX_DBG_CTRL,
@@ -2784,7 +2917,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 0);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, index);
-		seq_printf(m, "Enable Dispatcher hdt tx%x dump.\n", index);
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt tx%x dump.\n", index);
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_TX6:
 		info = &dbg_port_dspt_hdt_tx6;
@@ -2792,7 +2926,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 0);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 6);
-		seq_puts(m, "Enable Dispatcher hdt tx6 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt tx6 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_TX7:
 		info = &dbg_port_dspt_hdt_tx7;
@@ -2800,7 +2935,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 0);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 7);
-		seq_puts(m, "Enable Dispatcher hdt tx7 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt tx7 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_TX8:
 		info = &dbg_port_dspt_hdt_tx8;
@@ -2808,7 +2944,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 0);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 8);
-		seq_puts(m, "Enable Dispatcher hdt tx8 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt tx8 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_TX9:
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_TXA:
@@ -2820,7 +2957,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 0);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, index);
-		seq_printf(m, "Enable Dispatcher hdt tx%x dump.\n", index);
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt tx%x dump.\n", index);
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_TXD:
 		info = &dbg_port_dspt_hdt_txD;
@@ -2828,7 +2966,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 0);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 0xD);
-		seq_puts(m, "Enable Dispatcher hdt txD dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt txD dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TX0:
 		info = &dbg_port_dspt_cdt_tx0;
@@ -2836,7 +2975,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 1);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 0);
-		seq_puts(m, "Enable Dispatcher cdt tx0 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt tx0 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TX1:
 		info = &dbg_port_dspt_cdt_tx1;
@@ -2844,7 +2984,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 1);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 1);
-		seq_puts(m, "Enable Dispatcher cdt tx1 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt tx1 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TX3:
 		info = &dbg_port_dspt_cdt_tx3;
@@ -2852,7 +2993,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 1);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 3);
-		seq_puts(m, "Enable Dispatcher cdt tx3 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt tx3 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TX4:
 		info = &dbg_port_dspt_cdt_tx4;
@@ -2860,7 +3002,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 1);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 4);
-		seq_puts(m, "Enable Dispatcher cdt tx4 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt tx4 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TX5:
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TX6:
@@ -2872,7 +3015,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 1);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, index);
-		seq_printf(m, "Enable Dispatcher cdt tx%x dump.\n", index);
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt tx%x dump.\n", index);
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TX9:
 		info = &dbg_port_dspt_cdt_tx9;
@@ -2880,7 +3024,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 1);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 9);
-		seq_puts(m, "Enable Dispatcher cdt tx9 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt tx9 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TXA:
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_TXB:
@@ -2891,7 +3036,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 1);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, index);
-		seq_printf(m, "Enable Dispatcher cdt tx%x dump.\n", index);
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt tx%x dump.\n", index);
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_RX0:
 		info = &dbg_port_dspt_hdt_rx0;
@@ -2899,7 +3045,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 2);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 0);
-		seq_puts(m, "Enable Dispatcher hdt rx0 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt rx0 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_RX1:
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_RX2:
@@ -2909,7 +3056,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 2);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, index);
-		seq_printf(m, "Enable Dispatcher hdt rx%x dump.\n", index);
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt rx%x dump.\n", index);
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_RX3:
 		info = &dbg_port_dspt_hdt_rx3;
@@ -2917,7 +3065,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 2);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 3);
-		seq_puts(m, "Enable Dispatcher hdt rx3 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt rx3 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_RX4:
 		info = &dbg_port_dspt_hdt_rx4;
@@ -2925,7 +3074,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 2);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 4);
-		seq_puts(m, "Enable Dispatcher hdt rx4 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt rx4 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_HDT_RX5:
 		info = &dbg_port_dspt_hdt_rx5;
@@ -2933,7 +3083,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 2);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 5);
-		seq_puts(m, "Enable Dispatcher hdt rx5 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher hdt rx5 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_RX_P0_0:
 		info = &dbg_port_dspt_cdt_rx_p0_0;
@@ -2941,7 +3092,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 3);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 0);
-		seq_puts(m, "Enable Dispatcher cdt rx part0 0 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt rx part0 0 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_RX_P0:
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_RX_P0_1:
@@ -2950,7 +3102,8 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 3);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 1);
-		seq_puts(m, "Enable Dispatcher cdt rx part0 1 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt rx part0 1 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_RX_P0_2:
 		info = &dbg_port_dspt_cdt_rx_p0_2;
@@ -2958,43 +3111,50 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 				   B_AX_DISPATCHER_INTN_SEL_MASK, 3);
 		rtw89_write16_mask(rtwdev, info->sel_addr,
 				   B_AX_DISPATCHER_CH_SEL_MASK, 2);
-		seq_puts(m, "Enable Dispatcher cdt rx part0 2 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt rx part0 2 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_CDT_RX_P1:
 		info = &dbg_port_dspt_cdt_rx_p1;
 		rtw89_write8_mask(rtwdev, info->sel_addr,
 				  B_AX_DISPATCHER_INTN_SEL_MASK, 3);
-		seq_puts(m, "Enable Dispatcher cdt rx part1 dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher cdt rx part1 dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_STF_CTRL:
 		info = &dbg_port_dspt_stf_ctrl;
 		rtw89_write8_mask(rtwdev, info->sel_addr,
 				  B_AX_DISPATCHER_INTN_SEL_MASK, 4);
-		seq_puts(m, "Enable Dispatcher stf control dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher stf control dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_ADDR_CTRL:
 		info = &dbg_port_dspt_addr_ctrl;
 		rtw89_write8_mask(rtwdev, info->sel_addr,
 				  B_AX_DISPATCHER_INTN_SEL_MASK, 5);
-		seq_puts(m, "Enable Dispatcher addr control dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher addr control dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_WDE_INTF:
 		info = &dbg_port_dspt_wde_intf;
 		rtw89_write8_mask(rtwdev, info->sel_addr,
 				  B_AX_DISPATCHER_INTN_SEL_MASK, 6);
-		seq_puts(m, "Enable Dispatcher wde interface dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher wde interface dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_PLE_INTF:
 		info = &dbg_port_dspt_ple_intf;
 		rtw89_write8_mask(rtwdev, info->sel_addr,
 				  B_AX_DISPATCHER_INTN_SEL_MASK, 7);
-		seq_puts(m, "Enable Dispatcher ple interface dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher ple interface dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_DSPT_FLOW_CTRL:
 		info = &dbg_port_dspt_flow_ctrl;
 		rtw89_write8_mask(rtwdev, info->sel_addr,
 				  B_AX_DISPATCHER_INTN_SEL_MASK, 8);
-		seq_puts(m, "Enable Dispatcher flow control dump.\n");
+		p += scnprintf(p, end - p,
+			       "Enable Dispatcher flow control dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PCIE_TXDMA:
 		info = &dbg_port_pcie_txdma;
@@ -3002,7 +3162,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = u32_replace_bits(val32, PCIE_TXDMA_DBG_SEL, B_AX_DBG_SEL0);
 		val32 = u32_replace_bits(val32, PCIE_TXDMA_DBG_SEL, B_AX_DBG_SEL1);
 		rtw89_write32(rtwdev, R_AX_DBG_CTRL, val32);
-		seq_puts(m, "Enable pcie txdma dump.\n");
+		p += scnprintf(p, end - p, "Enable pcie txdma dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PCIE_RXDMA:
 		info = &dbg_port_pcie_rxdma;
@@ -3010,7 +3170,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = u32_replace_bits(val32, PCIE_RXDMA_DBG_SEL, B_AX_DBG_SEL0);
 		val32 = u32_replace_bits(val32, PCIE_RXDMA_DBG_SEL, B_AX_DBG_SEL1);
 		rtw89_write32(rtwdev, R_AX_DBG_CTRL, val32);
-		seq_puts(m, "Enable pcie rxdma dump.\n");
+		p += scnprintf(p, end - p, "Enable pcie rxdma dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PCIE_CVT:
 		info = &dbg_port_pcie_cvt;
@@ -3018,7 +3178,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = u32_replace_bits(val32, PCIE_CVT_DBG_SEL, B_AX_DBG_SEL0);
 		val32 = u32_replace_bits(val32, PCIE_CVT_DBG_SEL, B_AX_DBG_SEL1);
 		rtw89_write32(rtwdev, R_AX_DBG_CTRL, val32);
-		seq_puts(m, "Enable pcie cvt dump.\n");
+		p += scnprintf(p, end - p, "Enable pcie cvt dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PCIE_CXPL:
 		info = &dbg_port_pcie_cxpl;
@@ -3026,7 +3186,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = u32_replace_bits(val32, PCIE_CXPL_DBG_SEL, B_AX_DBG_SEL0);
 		val32 = u32_replace_bits(val32, PCIE_CXPL_DBG_SEL, B_AX_DBG_SEL1);
 		rtw89_write32(rtwdev, R_AX_DBG_CTRL, val32);
-		seq_puts(m, "Enable pcie cxpl dump.\n");
+		p += scnprintf(p, end - p, "Enable pcie cxpl dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PCIE_IO:
 		info = &dbg_port_pcie_io;
@@ -3034,7 +3194,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = u32_replace_bits(val32, PCIE_IO_DBG_SEL, B_AX_DBG_SEL0);
 		val32 = u32_replace_bits(val32, PCIE_IO_DBG_SEL, B_AX_DBG_SEL1);
 		rtw89_write32(rtwdev, R_AX_DBG_CTRL, val32);
-		seq_puts(m, "Enable pcie io dump.\n");
+		p += scnprintf(p, end - p, "Enable pcie io dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PCIE_MISC:
 		info = &dbg_port_pcie_misc;
@@ -3042,7 +3202,7 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val32 = u32_replace_bits(val32, PCIE_MISC_DBG_SEL, B_AX_DBG_SEL0);
 		val32 = u32_replace_bits(val32, PCIE_MISC_DBG_SEL, B_AX_DBG_SEL1);
 		rtw89_write32(rtwdev, R_AX_DBG_CTRL, val32);
-		seq_puts(m, "Enable pcie misc dump.\n");
+		p += scnprintf(p, end - p, "Enable pcie misc dump.\n");
 		break;
 	case RTW89_DBG_PORT_SEL_PCIE_MISC2:
 		info = &dbg_port_pcie_misc2;
@@ -3050,14 +3210,16 @@ rtw89_debug_mac_dbg_port_sel(struct seq_file *m,
 		val16 = u16_replace_bits(val16, PCIE_MISC2_DBG_SEL,
 					 B_AX_PCIE_DBG_SEL_MASK);
 		rtw89_write16(rtwdev, R_AX_PCIE_DBG_CTRL, val16);
-		seq_puts(m, "Enable pcie misc2 dump.\n");
+		p += scnprintf(p, end - p, "Enable pcie misc2 dump.\n");
 		break;
 	default:
-		seq_puts(m, "Dbg port select err\n");
-		return NULL;
+		p += scnprintf(p, end - p, "Dbg port select err\n");
+		break;
 	}
 
-	return info;
+	*ppinfo = info;
+
+	return p - buf;
 }
 
 static bool is_dbg_port_valid(struct rtw89_dev *rtwdev, u32 sel)
@@ -3091,23 +3253,25 @@ static bool is_dbg_port_valid(struct rtw89_dev *rtwdev, u32 sel)
 }
 
 static int rtw89_debug_mac_dbg_port_dump(struct rtw89_dev *rtwdev,
-					 struct seq_file *m, u32 sel)
+					 char *buf, size_t bufsz, u32 sel)
 {
-	const struct rtw89_mac_dbg_port_info *info;
-	u8 val8;
-	u16 val16;
+	const struct rtw89_mac_dbg_port_info *info = NULL;
+	char *p = buf, *end = buf + bufsz;
 	u32 val32;
+	u16 val16;
+	u8 val8;
 	u32 i;
 
-	info = rtw89_debug_mac_dbg_port_sel(m, rtwdev, sel);
+	p += rtw89_debug_mac_dbg_port_sel(rtwdev, p, end - p, sel, &info);
+
 	if (!info) {
 		rtw89_err(rtwdev, "failed to select debug port %d\n", sel);
-		return -EINVAL;
+		goto out;
 	}
 
 #define case_DBG_SEL(__sel) \
 	case RTW89_DBG_PORT_SEL_##__sel: \
-		seq_puts(m, "Dump debug port " #__sel ":\n"); \
+		p += scnprintf(p, end - p, "Dump debug port " #__sel ":\n"); \
 		break
 
 	switch (sel) {
@@ -3203,8 +3367,8 @@ static int rtw89_debug_mac_dbg_port_dump(struct rtw89_dev *rtwdev,
 
 #undef case_DBG_SEL
 
-	seq_printf(m, "Sel addr = 0x%X\n", info->sel_addr);
-	seq_printf(m, "Read addr = 0x%X\n", info->rd_addr);
+	p += scnprintf(p, end - p, "Sel addr = 0x%X\n", info->sel_addr);
+	p += scnprintf(p, end - p, "Read addr = 0x%X\n", info->rd_addr);
 
 	for (i = info->srt; i <= info->end; i++) {
 		switch (info->sel_byte) {
@@ -3212,17 +3376,17 @@ static int rtw89_debug_mac_dbg_port_dump(struct rtw89_dev *rtwdev,
 		default:
 			rtw89_write8_mask(rtwdev, info->sel_addr,
 					  info->sel_msk, i);
-			seq_printf(m, "0x%02X: ", i);
+			p += scnprintf(p, end - p, "0x%02X: ", i);
 			break;
 		case 2:
 			rtw89_write16_mask(rtwdev, info->sel_addr,
 					   info->sel_msk, i);
-			seq_printf(m, "0x%04X: ", i);
+			p += scnprintf(p, end - p, "0x%04X: ", i);
 			break;
 		case 4:
 			rtw89_write32_mask(rtwdev, info->sel_addr,
 					   info->sel_msk, i);
-			seq_printf(m, "0x%04X: ", i);
+			p += scnprintf(p, end - p, "0x%04X: ", i);
 			break;
 		}
 
@@ -3233,77 +3397,75 @@ static int rtw89_debug_mac_dbg_port_dump(struct rtw89_dev *rtwdev,
 		default:
 			val8 = rtw89_read8_mask(rtwdev,
 						info->rd_addr, info->rd_msk);
-			seq_printf(m, "0x%02X\n", val8);
+			p += scnprintf(p, end - p, "0x%02X\n", val8);
 			break;
 		case 2:
 			val16 = rtw89_read16_mask(rtwdev,
 						  info->rd_addr, info->rd_msk);
-			seq_printf(m, "0x%04X\n", val16);
+			p += scnprintf(p, end - p, "0x%04X\n", val16);
 			break;
 		case 4:
 			val32 = rtw89_read32_mask(rtwdev,
 						  info->rd_addr, info->rd_msk);
-			seq_printf(m, "0x%08X\n", val32);
+			p += scnprintf(p, end - p, "0x%08X\n", val32);
 			break;
 		}
 	}
 
-	return 0;
+out:
+	return p - buf;
 }
 
 static int rtw89_debug_mac_dump_dbg_port(struct rtw89_dev *rtwdev,
-					 struct seq_file *m)
+					 char *buf, size_t bufsz)
 {
+	char *p = buf, *end = buf + bufsz;
+	ssize_t n;
 	u32 sel;
-	int ret = 0;
 
 	for (sel = RTW89_DBG_PORT_SEL_PTCL_C0;
 	     sel < RTW89_DBG_PORT_SEL_LAST; sel++) {
 		if (!is_dbg_port_valid(rtwdev, sel))
 			continue;
-		ret = rtw89_debug_mac_dbg_port_dump(rtwdev, m, sel);
-		if (ret) {
+		n = rtw89_debug_mac_dbg_port_dump(rtwdev, p, end - p, sel);
+		if (n < 0) {
 			rtw89_err(rtwdev,
 				  "failed to dump debug port %d\n", sel);
 			break;
 		}
+		p += n;
 	}
 
-	return ret;
+	return p - buf;
 }
 
-static int
-rtw89_debug_priv_mac_dbg_port_dump_get(struct seq_file *m, void *v)
+static ssize_t
+rtw89_debug_priv_mac_dbg_port_dump_get(struct rtw89_dev *rtwdev,
+				       struct rtw89_debugfs_priv *debugfs_priv,
+				       char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+	char *p = buf, *end = buf + bufsz;
 
 	if (debugfs_priv->dbgpkg_en.ss_dbg)
-		rtw89_debug_mac_dump_ss_dbg(rtwdev, m);
+		p += rtw89_debug_mac_dump_ss_dbg(rtwdev, p, end - p);
 	if (debugfs_priv->dbgpkg_en.dle_dbg)
-		rtw89_debug_mac_dump_dle_dbg(rtwdev, m);
+		p += rtw89_debug_mac_dump_dle_dbg(rtwdev, p, end - p);
 	if (debugfs_priv->dbgpkg_en.dmac_dbg)
-		rtw89_debug_mac_dump_dmac_dbg(rtwdev, m);
+		p += rtw89_debug_mac_dump_dmac_dbg(rtwdev, p, end - p);
 	if (debugfs_priv->dbgpkg_en.cmac_dbg)
-		rtw89_debug_mac_dump_cmac_dbg(rtwdev, m);
+		p += rtw89_debug_mac_dump_cmac_dbg(rtwdev, p, end - p);
 	if (debugfs_priv->dbgpkg_en.dbg_port)
-		rtw89_debug_mac_dump_dbg_port(rtwdev, m);
+		p += rtw89_debug_mac_dump_dbg_port(rtwdev, p, end - p);
 
-	return 0;
+	return p - buf;
 };
 
-static u8 *rtw89_hex2bin_user(struct rtw89_dev *rtwdev,
-			      const char __user *user_buf, size_t count)
+static u8 *rtw89_hex2bin(struct rtw89_dev *rtwdev, const char *buf, size_t count)
 {
-	char *buf;
 	u8 *bin;
 	int num;
 	int err = 0;
 
-	buf = memdup_user(user_buf, count);
-	if (IS_ERR(buf))
-		return buf;
-
 	num = count / 2;
 	bin = kmalloc(num, GFP_KERNEL);
 	if (!bin) {
@@ -3318,22 +3480,18 @@ static u8 *rtw89_hex2bin_user(struct rtw89_dev *rtwdev,
 	}
 
 out:
-	kfree(buf);
-
 	return err ? ERR_PTR(err) : bin;
 }
 
-static ssize_t rtw89_debug_priv_send_h2c_set(struct file *filp,
-					     const char __user *user_buf,
-					     size_t count, loff_t *loff)
+static ssize_t rtw89_debug_priv_send_h2c_set(struct rtw89_dev *rtwdev,
+					     struct rtw89_debugfs_priv *debugfs_priv,
+					     const char *buf, size_t count)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = filp->private_data;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	u8 *h2c;
 	int ret;
 	u16 h2c_len = count / 2;
 
-	h2c = rtw89_hex2bin_user(rtwdev, user_buf, count);
+	h2c = rtw89_hex2bin(rtwdev, buf, count);
 	if (IS_ERR(h2c))
 		return -EFAULT;
 
@@ -3344,34 +3502,36 @@ static ssize_t rtw89_debug_priv_send_h2c_set(struct file *filp,
 	return ret ? ret : count;
 }
 
-static int
-rtw89_debug_priv_early_h2c_get(struct seq_file *m, void *v)
+static ssize_t
+rtw89_debug_priv_early_h2c_get(struct rtw89_dev *rtwdev,
+			       struct rtw89_debugfs_priv *debugfs_priv,
+			       char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw89_early_h2c *early_h2c;
+	char *p = buf, *end = buf + bufsz;
 	int seq = 0;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
 	list_for_each_entry(early_h2c, &rtwdev->early_h2c_list, list)
-		seq_printf(m, "%d: %*ph\n", ++seq, early_h2c->h2c_len, early_h2c->h2c);
-	mutex_unlock(&rtwdev->mutex);
+		p += scnprintf(p, end - p, "%d: %*ph\n", ++seq,
+			       early_h2c->h2c_len, early_h2c->h2c);
 
-	return 0;
+	return p - buf;
 }
 
 static ssize_t
-rtw89_debug_priv_early_h2c_set(struct file *filp, const char __user *user_buf,
-			       size_t count, loff_t *loff)
+rtw89_debug_priv_early_h2c_set(struct rtw89_dev *rtwdev,
+			       struct rtw89_debugfs_priv *debugfs_priv,
+			       const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw89_early_h2c *early_h2c;
 	u8 *h2c;
 	u16 h2c_len = count / 2;
 
-	h2c = rtw89_hex2bin_user(rtwdev, user_buf, count);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	h2c = rtw89_hex2bin(rtwdev, buf, count);
 	if (IS_ERR(h2c))
 		return -EFAULT;
 
@@ -3390,9 +3550,7 @@ rtw89_debug_priv_early_h2c_set(struct file *filp, const char __user *user_buf,
 	early_h2c->h2c = h2c;
 	early_h2c->h2c_len = h2c_len;
 
-	mutex_lock(&rtwdev->mutex);
 	list_add_tail(&early_h2c->list, &rtwdev->early_h2c_list);
-	mutex_unlock(&rtwdev->mutex);
 
 out:
 	return count;
@@ -3425,15 +3583,16 @@ static int rtw89_dbg_trigger_ctrl_error(struct rtw89_dev *rtwdev)
 	return 0;
 }
 
-static int
-rtw89_debug_priv_fw_crash_get(struct seq_file *m, void *v)
+static ssize_t
+rtw89_debug_priv_fw_crash_get(struct rtw89_dev *rtwdev,
+			      struct rtw89_debugfs_priv *debugfs_priv,
+			      char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
+	char *p = buf, *end = buf + bufsz;
 
-	seq_printf(m, "%d\n",
-		   test_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags));
-	return 0;
+	p += scnprintf(p, end - p, "%d\n",
+		       test_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags));
+	return p - buf;
 }
 
 enum rtw89_dbg_crash_simulation_type {
@@ -3442,23 +3601,23 @@ enum rtw89_dbg_crash_simulation_type {
 };
 
 static ssize_t
-rtw89_debug_priv_fw_crash_set(struct file *filp, const char __user *user_buf,
-			      size_t count, loff_t *loff)
+rtw89_debug_priv_fw_crash_set(struct rtw89_dev *rtwdev,
+			      struct rtw89_debugfs_priv *debugfs_priv,
+			      const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	int (*sim)(struct rtw89_dev *rtwdev);
 	u8 crash_type;
 	int ret;
 
-	ret = kstrtou8_from_user(user_buf, count, 0, &crash_type);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	ret = kstrtou8(buf, 0, &crash_type);
 	if (ret)
 		return -EINVAL;
 
 	switch (crash_type) {
 	case RTW89_DBG_SIM_CPU_EXCEPTION:
-		if (!RTW89_CHK_FW_FEATURE(CRASH_TRIGGER, &rtwdev->fw))
+		if (!RTW89_CHK_FW_FEATURE_GROUP(CRASH_TRIGGER, &rtwdev->fw))
 			return -EOPNOTSUPP;
 		sim = rtw89_fw_h2c_trigger_cpu_exception;
 		break;
@@ -3469,10 +3628,8 @@ rtw89_debug_priv_fw_crash_set(struct file *filp, const char __user *user_buf,
 		return -EINVAL;
 	}
 
-	mutex_lock(&rtwdev->mutex);
 	set_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags);
 	ret = sim(rtwdev);
-	mutex_unlock(&rtwdev->mutex);
 
 	if (ret)
 		return ret;
@@ -3480,27 +3637,22 @@ rtw89_debug_priv_fw_crash_set(struct file *filp, const char __user *user_buf,
 	return count;
 }
 
-static int rtw89_debug_priv_btc_info_get(struct seq_file *m, void *v)
+static ssize_t rtw89_debug_priv_btc_info_get(struct rtw89_dev *rtwdev,
+					     struct rtw89_debugfs_priv *debugfs_priv,
+					     char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-
-	rtw89_btc_dump_info(rtwdev, m);
-
-	return 0;
+	return rtw89_btc_dump_info(rtwdev, buf, bufsz);
 }
 
-static ssize_t rtw89_debug_priv_btc_manual_set(struct file *filp,
-					       const char __user *user_buf,
-					       size_t count, loff_t *loff)
+static ssize_t rtw89_debug_priv_btc_manual_set(struct rtw89_dev *rtwdev,
+					       struct rtw89_debugfs_priv *debugfs_priv,
+					       const char *buf, size_t count)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = filp->private_data;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
 	int ret;
 
-	ret = kstrtobool_from_user(user_buf, count, &btc->manual_ctrl);
+	ret = kstrtobool(buf, &btc->manual_ctrl);
 	if (ret)
 		return ret;
 
@@ -3512,31 +3664,29 @@ static ssize_t rtw89_debug_priv_btc_manual_set(struct file *filp,
 	return count;
 }
 
-static ssize_t rtw89_debug_priv_fw_log_manual_set(struct file *filp,
-						  const char __user *user_buf,
-						  size_t count, loff_t *loff)
+static ssize_t rtw89_debug_priv_fw_log_manual_set(struct rtw89_dev *rtwdev,
+						  struct rtw89_debugfs_priv *debugfs_priv,
+						  const char *buf, size_t count)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = filp->private_data;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw89_fw_log *log = &rtwdev->fw.log;
 	bool fw_log_manual;
 
-	if (kstrtobool_from_user(user_buf, count, &fw_log_manual))
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	if (kstrtobool(buf, &fw_log_manual))
 		goto out;
 
-	mutex_lock(&rtwdev->mutex);
 	log->enable = fw_log_manual;
 	if (log->enable)
 		rtw89_fw_log_prepare(rtwdev);
 	rtw89_fw_h2c_fw_log(rtwdev, fw_log_manual);
-	mutex_unlock(&rtwdev->mutex);
 out:
 	return count;
 }
 
-static void rtw89_sta_link_info_get_iter(struct seq_file *m,
-					 struct rtw89_dev *rtwdev,
-					 struct rtw89_sta_link *rtwsta_link)
+static int rtw89_sta_link_info_get_iter(struct rtw89_dev *rtwdev,
+					char *buf, size_t bufsz,
+					struct rtw89_sta_link *rtwsta_link)
 {
 	static const char * const he_gi_str[] = {
 		[NL80211_RATE_INFO_HE_GI_0_8] = "0.8",
@@ -3554,6 +3704,7 @@ static void rtw89_sta_link_info_get_iter(struct seq_file *m,
 	u8 ant_num = hal->ant_diversity ? 2 : rtwdev->chip->rf_path_num;
 	bool ant_asterisk = hal->tx_path_diversity || hal->ant_diversity;
 	struct ieee80211_link_sta *link_sta;
+	char *p = buf, *end = buf + bufsz;
 	u8 evm_min, evm_max, evm_1ss;
 	u16 max_rc_amsdu_len;
 	u8 rssi;
@@ -3567,107 +3718,136 @@ static void rtw89_sta_link_info_get_iter(struct seq_file *m,
 
 	rcu_read_unlock();
 
-	seq_printf(m, "TX rate [%u, %u]: ", rtwsta_link->mac_id, rtwsta_link->link_id);
+	p += scnprintf(p, end - p, "TX rate [%u, %u]: ", rtwsta_link->mac_id,
+		       rtwsta_link->link_id);
 
 	if (rate->flags & RATE_INFO_FLAGS_MCS)
-		seq_printf(m, "HT MCS-%d%s", rate->mcs,
-			   rate->flags & RATE_INFO_FLAGS_SHORT_GI ? " SGI" : "");
+		p += scnprintf(p, end - p, "HT MCS-%d%s", rate->mcs,
+			       rate->flags & RATE_INFO_FLAGS_SHORT_GI ? " SGI" : "");
 	else if (rate->flags & RATE_INFO_FLAGS_VHT_MCS)
-		seq_printf(m, "VHT %dSS MCS-%d%s", rate->nss, rate->mcs,
-			   rate->flags & RATE_INFO_FLAGS_SHORT_GI ? " SGI" : "");
+		p += scnprintf(p, end - p, "VHT %dSS MCS-%d%s", rate->nss,
+			       rate->mcs,
+			       rate->flags & RATE_INFO_FLAGS_SHORT_GI ? " SGI" : "");
 	else if (rate->flags & RATE_INFO_FLAGS_HE_MCS)
-		seq_printf(m, "HE %dSS MCS-%d GI:%s", rate->nss, rate->mcs,
-			   rate->he_gi <= NL80211_RATE_INFO_HE_GI_3_2 ?
-			   he_gi_str[rate->he_gi] : "N/A");
+		p += scnprintf(p, end - p, "HE %dSS MCS-%d GI:%s", rate->nss,
+			       rate->mcs,
+			       rate->he_gi <= NL80211_RATE_INFO_HE_GI_3_2 ?
+			       he_gi_str[rate->he_gi] : "N/A");
 	else if (rate->flags & RATE_INFO_FLAGS_EHT_MCS)
-		seq_printf(m, "EHT %dSS MCS-%d GI:%s", rate->nss, rate->mcs,
-			   rate->eht_gi < ARRAY_SIZE(eht_gi_str) ?
-			   eht_gi_str[rate->eht_gi] : "N/A");
+		p += scnprintf(p, end - p, "EHT %dSS MCS-%d GI:%s", rate->nss,
+			       rate->mcs,
+			       rate->eht_gi < ARRAY_SIZE(eht_gi_str) ?
+			       eht_gi_str[rate->eht_gi] : "N/A");
 	else
-		seq_printf(m, "Legacy %d", rate->legacy);
-	seq_printf(m, "%s", rtwsta_link->ra_report.might_fallback_legacy ? " FB_G" : "");
-	seq_printf(m, " BW:%u", rtw89_rate_info_bw_to_mhz(rate->bw));
-	seq_printf(m, " (hw_rate=0x%x)", rtwsta_link->ra_report.hw_rate);
-	seq_printf(m, " ==> agg_wait=%d (%d)\n", rtwsta_link->max_agg_wait,
-		   max_rc_amsdu_len);
-
-	seq_printf(m, "RX rate [%u, %u]: ", rtwsta_link->mac_id, rtwsta_link->link_id);
+		p += scnprintf(p, end - p, "Legacy %d", rate->legacy);
+	p += scnprintf(p, end - p, "%s",
+		       rtwsta_link->ra_report.might_fallback_legacy ? " FB_G" : "");
+	p += scnprintf(p, end - p, " BW:%u",
+		       rtw89_rate_info_bw_to_mhz(rate->bw));
+	p += scnprintf(p, end - p, " (hw_rate=0x%x)",
+		       rtwsta_link->ra_report.hw_rate);
+	p += scnprintf(p, end - p, " ==> agg_wait=%d (%d)\n",
+		       rtwsta_link->max_agg_wait,
+		       max_rc_amsdu_len);
+
+	p += scnprintf(p, end - p, "RX rate [%u, %u]: ", rtwsta_link->mac_id,
+		       rtwsta_link->link_id);
 
 	switch (status->encoding) {
 	case RX_ENC_LEGACY:
-		seq_printf(m, "Legacy %d", status->rate_idx +
-			   (status->band != NL80211_BAND_2GHZ ? 4 : 0));
+		p += scnprintf(p, end - p, "Legacy %d", status->rate_idx +
+			       (status->band != NL80211_BAND_2GHZ ? 4 : 0));
 		break;
 	case RX_ENC_HT:
-		seq_printf(m, "HT MCS-%d%s", status->rate_idx,
-			   status->enc_flags & RX_ENC_FLAG_SHORT_GI ? " SGI" : "");
+		p += scnprintf(p, end - p, "HT MCS-%d%s", status->rate_idx,
+			       status->enc_flags & RX_ENC_FLAG_SHORT_GI ? " SGI" : "");
 		break;
 	case RX_ENC_VHT:
-		seq_printf(m, "VHT %dSS MCS-%d%s", status->nss, status->rate_idx,
-			   status->enc_flags & RX_ENC_FLAG_SHORT_GI ? " SGI" : "");
+		p += scnprintf(p, end - p, "VHT %dSS MCS-%d%s", status->nss,
+			       status->rate_idx,
+			       status->enc_flags & RX_ENC_FLAG_SHORT_GI ? " SGI" : "");
 		break;
 	case RX_ENC_HE:
-		seq_printf(m, "HE %dSS MCS-%d GI:%s", status->nss, status->rate_idx,
-			   status->he_gi <= NL80211_RATE_INFO_HE_GI_3_2 ?
-			   he_gi_str[status->he_gi] : "N/A");
+		p += scnprintf(p, end - p, "HE %dSS MCS-%d GI:%s",
+			       status->nss, status->rate_idx,
+			       status->he_gi <= NL80211_RATE_INFO_HE_GI_3_2 ?
+			       he_gi_str[status->he_gi] : "N/A");
 		break;
 	case RX_ENC_EHT:
-		seq_printf(m, "EHT %dSS MCS-%d GI:%s", status->nss, status->rate_idx,
-			   status->eht.gi < ARRAY_SIZE(eht_gi_str) ?
-			   eht_gi_str[status->eht.gi] : "N/A");
+		p += scnprintf(p, end - p, "EHT %dSS MCS-%d GI:%s",
+			       status->nss, status->rate_idx,
+			       status->eht.gi < ARRAY_SIZE(eht_gi_str) ?
+			       eht_gi_str[status->eht.gi] : "N/A");
 		break;
 	}
-	seq_printf(m, " BW:%u", rtw89_rate_info_bw_to_mhz(status->bw));
-	seq_printf(m, " (hw_rate=0x%x)\n", rtwsta_link->rx_hw_rate);
+	p += scnprintf(p, end - p, " BW:%u",
+		       rtw89_rate_info_bw_to_mhz(status->bw));
+	p += scnprintf(p, end - p, " (hw_rate=0x%x)\n",
+		       rtwsta_link->rx_hw_rate);
 
 	rssi = ewma_rssi_read(&rtwsta_link->avg_rssi);
-	seq_printf(m, "RSSI: %d dBm (raw=%d, prev=%d) [",
-		   RTW89_RSSI_RAW_TO_DBM(rssi), rssi, rtwsta_link->prev_rssi);
+	p += scnprintf(p, end - p, "RSSI: %d dBm (raw=%d, prev=%d) [",
+		       RTW89_RSSI_RAW_TO_DBM(rssi), rssi,
+		       rtwsta_link->prev_rssi);
 	for (i = 0; i < ant_num; i++) {
 		rssi = ewma_rssi_read(&rtwsta_link->rssi[i]);
-		seq_printf(m, "%d%s%s", RTW89_RSSI_RAW_TO_DBM(rssi),
-			   ant_asterisk && (hal->antenna_tx & BIT(i)) ? "*" : "",
-			   i + 1 == ant_num ? "" : ", ");
+		p += scnprintf(p, end - p, "%d%s%s",
+			       RTW89_RSSI_RAW_TO_DBM(rssi),
+			       ant_asterisk && (hal->antenna_tx & BIT(i)) ? "*" : "",
+			       i + 1 == ant_num ? "" : ", ");
 	}
-	seq_puts(m, "]\n");
+	p += scnprintf(p, end - p, "]\n");
 
 	evm_1ss = ewma_evm_read(&rtwsta_link->evm_1ss);
-	seq_printf(m, "EVM: [%2u.%02u, ", evm_1ss >> 2, (evm_1ss & 0x3) * 25);
+	p += scnprintf(p, end - p, "EVM: [%2u.%02u, ", evm_1ss >> 2,
+		       (evm_1ss & 0x3) * 25);
 	for (i = 0; i < (hal->ant_diversity ? 2 : 1); i++) {
 		evm_min = ewma_evm_read(&rtwsta_link->evm_min[i]);
 		evm_max = ewma_evm_read(&rtwsta_link->evm_max[i]);
 
-		seq_printf(m, "%s(%2u.%02u, %2u.%02u)", i == 0 ? "" : " ",
-			   evm_min >> 2, (evm_min & 0x3) * 25,
-			   evm_max >> 2, (evm_max & 0x3) * 25);
+		p += scnprintf(p, end - p, "%s(%2u.%02u, %2u.%02u)",
+			       i == 0 ? "" : " ",
+			       evm_min >> 2, (evm_min & 0x3) * 25,
+			       evm_max >> 2, (evm_max & 0x3) * 25);
 	}
-	seq_puts(m, "]\t");
+	p += scnprintf(p, end - p, "]\t");
 
 	snr = ewma_snr_read(&rtwsta_link->avg_snr);
-	seq_printf(m, "SNR: %u\n", snr);
+	p += scnprintf(p, end - p, "SNR: %u\n", snr);
+
+	return p - buf;
 }
 
 static void rtw89_sta_info_get_iter(void *data, struct ieee80211_sta *sta)
 {
-	struct seq_file *m = (struct seq_file *)data;
+	struct rtw89_debugfs_iter_data *iter_data =
+		(struct rtw89_debugfs_iter_data *)data;
 	struct rtw89_sta *rtwsta = sta_to_rtwsta(sta);
 	struct rtw89_dev *rtwdev = rtwsta->rtwdev;
 	struct rtw89_sta_link *rtwsta_link;
+	size_t bufsz = iter_data->bufsz;
+	char *buf = iter_data->buf;
+	char *p = buf, *end = buf + bufsz;
 	unsigned int link_id;
 
 	rtw89_sta_for_each_link(rtwsta, rtwsta_link, link_id)
-		rtw89_sta_link_info_get_iter(m, rtwdev, rtwsta_link);
+		p += rtw89_sta_link_info_get_iter(rtwdev, p, end - p, rtwsta_link);
+
+	rtw89_debugfs_iter_data_next(iter_data, p, end - p, p - buf);
 }
 
-static void
-rtw89_debug_append_rx_rate(struct seq_file *m, struct rtw89_pkt_stat *pkt_stat,
+static int
+rtw89_debug_append_rx_rate(char *buf, size_t bufsz, struct rtw89_pkt_stat *pkt_stat,
 			   enum rtw89_hw_rate first_rate, int len)
 {
+	char *p = buf, *end = buf + bufsz;
 	int i;
 
 	for (i = 0; i < len; i++)
-		seq_printf(m, "%s%u", i == 0 ? "" : ", ",
-			   pkt_stat->rx_rate_cnt[first_rate + i]);
+		p += scnprintf(p, end - p, "%s%u", i == 0 ? "" : ", ",
+			       pkt_stat->rx_rate_cnt[first_rate + i]);
+
+	return p - buf;
 }
 
 #define FIRST_RATE_SAME(rate) {RTW89_HW_RATE_ ## rate, RTW89_HW_RATE_ ## rate}
@@ -3692,34 +3872,40 @@ static const struct rtw89_rx_rate_cnt_info {
 	{FIRST_RATE_GEV1(EHT_NSS2_MCS0), 14, 0, "EHT 2SS:"},
 };
 
-static int rtw89_debug_priv_phy_info_get(struct seq_file *m, void *v)
+static ssize_t rtw89_debug_priv_phy_info_get(struct rtw89_dev *rtwdev,
+					     struct rtw89_debugfs_priv *debugfs_priv,
+					     char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw89_traffic_stats *stats = &rtwdev->stats;
 	struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.last_pkt_stat;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_debugfs_iter_data iter_data;
 	const struct rtw89_rx_rate_cnt_info *info;
 	struct rtw89_hal *hal = &rtwdev->hal;
+	char *p = buf, *end = buf + bufsz;
 	enum rtw89_hw_rate first_rate;
 	u8 rssi;
 	int i;
 
 	rssi = ewma_rssi_read(&rtwdev->phystat.bcn_rssi);
 
-	seq_printf(m, "TP TX: %u [%u] Mbps (lv: %d",
-		   stats->tx_throughput, stats->tx_throughput_raw, stats->tx_tfc_lv);
+	p += scnprintf(p, end - p, "TP TX: %u [%u] Mbps (lv: %d",
+		       stats->tx_throughput, stats->tx_throughput_raw,
+		       stats->tx_tfc_lv);
 	if (hal->thermal_prot_lv)
-		seq_printf(m, ", duty: %d%%",
-			   100 - hal->thermal_prot_lv * RTW89_THERMAL_PROT_STEP);
-	seq_printf(m, "), RX: %u [%u] Mbps (lv: %d)\n",
-		   stats->rx_throughput, stats->rx_throughput_raw, stats->rx_tfc_lv);
-	seq_printf(m, "Beacon: %u (%d dBm), TF: %u\n", pkt_stat->beacon_nr,
-		   RTW89_RSSI_RAW_TO_DBM(rssi), stats->rx_tf_periodic);
-	seq_printf(m, "Avg packet length: TX=%u, RX=%u\n", stats->tx_avg_len,
-		   stats->rx_avg_len);
-
-	seq_puts(m, "RX count:\n");
+		p += scnprintf(p, end - p, ", duty: %d%%",
+			       100 - hal->thermal_prot_lv * RTW89_THERMAL_PROT_STEP);
+	p += scnprintf(p, end - p, "), RX: %u [%u] Mbps (lv: %d)\n",
+		       stats->rx_throughput, stats->rx_throughput_raw,
+		       stats->rx_tfc_lv);
+	p += scnprintf(p, end - p, "Beacon: %u (%d dBm), TF: %u\n",
+		       pkt_stat->beacon_nr,
+		       RTW89_RSSI_RAW_TO_DBM(rssi), stats->rx_tf_periodic);
+	p += scnprintf(p, end - p, "Avg packet length: TX=%u, RX=%u\n",
+		       stats->tx_avg_len,
+		       stats->rx_avg_len);
+
+	p += scnprintf(p, end - p, "RX count:\n");
 
 	for (i = 0; i < ARRAY_SIZE(rtw89_rx_rate_cnt_infos); i++) {
 		info = &rtw89_rx_rate_cnt_infos[i];
@@ -3727,189 +3913,279 @@ static int rtw89_debug_priv_phy_info_get(struct seq_file *m, void *v)
 		if (first_rate >= RTW89_HW_RATE_NR)
 			continue;
 
-		seq_printf(m, "%10s [", info->rate_mode);
-		rtw89_debug_append_rx_rate(m, pkt_stat,
-					   first_rate, info->len);
+		p += scnprintf(p, end - p, "%10s [", info->rate_mode);
+		p += rtw89_debug_append_rx_rate(p, end - p, pkt_stat,
+						first_rate, info->len);
 		if (info->ext) {
-			seq_puts(m, "][");
-			rtw89_debug_append_rx_rate(m, pkt_stat,
-						   first_rate + info->len, info->ext);
+			p += scnprintf(p, end - p, "][");
+			p += rtw89_debug_append_rx_rate(p, end - p, pkt_stat,
+							first_rate + info->len, info->ext);
 		}
-		seq_puts(m, "]\n");
+		p += scnprintf(p, end - p, "]\n");
 	}
 
-	ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_sta_info_get_iter, m);
+	rtw89_debugfs_iter_data_setup(&iter_data, p, end - p);
+	ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_sta_info_get_iter, &iter_data);
+	p += iter_data.written_sz;
 
-	return 0;
+	return p - buf;
 }
 
-static void rtw89_dump_addr_cam(struct seq_file *m,
-				struct rtw89_dev *rtwdev,
-				struct rtw89_addr_cam_entry *addr_cam)
+static int rtw89_dump_addr_cam(struct rtw89_dev *rtwdev,
+			       char *buf, size_t bufsz,
+			       struct rtw89_addr_cam_entry *addr_cam)
 {
 	struct rtw89_cam_info *cam_info = &rtwdev->cam_info;
 	const struct rtw89_sec_cam_entry *sec_entry;
+	char *p = buf, *end = buf + bufsz;
 	u8 sec_cam_idx;
 	int i;
 
-	seq_printf(m, "\taddr_cam_idx=%u\n", addr_cam->addr_cam_idx);
-	seq_printf(m, "\t-> bssid_cam_idx=%u\n", addr_cam->bssid_cam_idx);
-	seq_printf(m, "\tsec_cam_bitmap=%*ph\n", (int)sizeof(addr_cam->sec_cam_map),
-		   addr_cam->sec_cam_map);
+	p += scnprintf(p, end - p, "\taddr_cam_idx=%u\n",
+		       addr_cam->addr_cam_idx);
+	p += scnprintf(p, end - p, "\t-> bssid_cam_idx=%u\n",
+		       addr_cam->bssid_cam_idx);
+	p += scnprintf(p, end - p, "\tsec_cam_bitmap=%*ph\n",
+		       (int)sizeof(addr_cam->sec_cam_map),
+		       addr_cam->sec_cam_map);
 	for_each_set_bit(i, addr_cam->sec_cam_map, RTW89_SEC_CAM_IN_ADDR_CAM) {
 		sec_cam_idx = addr_cam->sec_ent[i];
 		sec_entry = cam_info->sec_entries[sec_cam_idx];
 		if (!sec_entry)
 			continue;
-		seq_printf(m, "\tsec[%d]: sec_cam_idx %u", i, sec_entry->sec_cam_idx);
+		p += scnprintf(p, end - p, "\tsec[%d]: sec_cam_idx %u", i,
+			       sec_entry->sec_cam_idx);
 		if (sec_entry->ext_key)
-			seq_printf(m, ", %u", sec_entry->sec_cam_idx + 1);
-		seq_puts(m, "\n");
+			p += scnprintf(p, end - p, ", %u",
+				       sec_entry->sec_cam_idx + 1);
+		p += scnprintf(p, end - p, "\n");
 	}
+
+	return p - buf;
 }
 
-__printf(3, 4)
-static void rtw89_dump_pkt_offload(struct seq_file *m, struct list_head *pkt_list,
-				   const char *fmt, ...)
+__printf(4, 5)
+static int rtw89_dump_pkt_offload(char *buf, size_t bufsz, struct list_head *pkt_list,
+				  const char *fmt, ...)
 {
+	char *p = buf, *end = buf + bufsz;
 	struct rtw89_pktofld_info *info;
 	struct va_format vaf;
 	va_list args;
 
 	if (list_empty(pkt_list))
-		return;
+		return 0;
 
 	va_start(args, fmt);
 	vaf.va = &args;
 	vaf.fmt = fmt;
 
-	seq_printf(m, "%pV", &vaf);
+	p += scnprintf(p, end - p, "%pV", &vaf);
 
 	va_end(args);
 
 	list_for_each_entry(info, pkt_list, list)
-		seq_printf(m, "%d ", info->id);
+		p += scnprintf(p, end - p, "%d ", info->id);
+
+	p += scnprintf(p, end - p, "\n");
 
-	seq_puts(m, "\n");
+	return p - buf;
 }
 
-static void rtw89_vif_link_ids_get(struct seq_file *m, u8 *mac,
-				   struct rtw89_dev *rtwdev,
-				   struct rtw89_vif_link *rtwvif_link)
+static int rtw89_vif_link_ids_get(struct rtw89_dev *rtwdev,
+				  char *buf, size_t bufsz, u8 *mac,
+				  struct rtw89_vif_link *rtwvif_link,
+				  bool designated)
 {
 	struct rtw89_bssid_cam_entry *bssid_cam = &rtwvif_link->bssid_cam;
-
-	seq_printf(m, "    [%u] %pM\n", rtwvif_link->mac_id, rtwvif_link->mac_addr);
-	seq_printf(m, "\tlink_id=%u\n", rtwvif_link->link_id);
-	seq_printf(m, "\tbssid_cam_idx=%u\n", bssid_cam->bssid_cam_idx);
-	rtw89_dump_addr_cam(m, rtwdev, &rtwvif_link->addr_cam);
-	rtw89_dump_pkt_offload(m, &rtwvif_link->general_pkt_list,
-			       "\tpkt_ofld[GENERAL]: ");
+	char *p = buf, *end = buf + bufsz;
+
+	p += scnprintf(p, end - p, "    [%u] %pM\n", rtwvif_link->mac_id,
+		       rtwvif_link->mac_addr);
+	p += scnprintf(p, end - p, "\tlink_id=%u%s\n", rtwvif_link->link_id,
+		       designated ? " (*)" : "");
+	p += scnprintf(p, end - p, "\tbssid_cam_idx=%u\n",
+		       bssid_cam->bssid_cam_idx);
+	p += rtw89_dump_addr_cam(rtwdev, p, end - p, &rtwvif_link->addr_cam);
+	p += rtw89_dump_pkt_offload(p, end - p, &rtwvif_link->general_pkt_list,
+				    "\tpkt_ofld[GENERAL]: ");
+
+	return p - buf;
 }
 
 static
 void rtw89_vif_ids_get_iter(void *data, u8 *mac, struct ieee80211_vif *vif)
 {
-	struct seq_file *m = (struct seq_file *)data;
+	struct rtw89_debugfs_iter_data *iter_data =
+		(struct rtw89_debugfs_iter_data *)data;
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
+	struct rtw89_vif_link *designated_link;
 	struct rtw89_vif_link *rtwvif_link;
+	size_t bufsz = iter_data->bufsz;
+	char *buf = iter_data->buf;
+	char *p = buf, *end = buf + bufsz;
 	unsigned int link_id;
 
-	seq_printf(m, "VIF %pM\n", rtwvif->mac_addr);
+	designated_link = rtw89_get_designated_link(rtwvif);
+
+	p += scnprintf(p, end - p, "VIF %pM\n", rtwvif->mac_addr);
 	rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id)
-		rtw89_vif_link_ids_get(m, mac, rtwdev, rtwvif_link);
+		p += rtw89_vif_link_ids_get(rtwdev, p, end - p, mac, rtwvif_link,
+					    rtwvif_link == designated_link);
+
+	rtw89_debugfs_iter_data_next(iter_data, p, end - p, p - buf);
 }
 
-static void rtw89_dump_ba_cam(struct seq_file *m, struct rtw89_dev *rtwdev,
-			      struct rtw89_sta_link *rtwsta_link)
+static int rtw89_dump_ba_cam(struct rtw89_dev *rtwdev,
+			     char *buf, size_t bufsz,
+			     struct rtw89_sta_link *rtwsta_link)
 {
 	struct rtw89_ba_cam_entry *entry;
+	char *p = buf, *end = buf + bufsz;
 	bool first = true;
 
 	list_for_each_entry(entry, &rtwsta_link->ba_cam_list, list) {
 		if (first) {
-			seq_puts(m, "\tba_cam ");
+			p += scnprintf(p, end - p, "\tba_cam ");
 			first = false;
 		} else {
-			seq_puts(m, ", ");
+			p += scnprintf(p, end - p, ", ");
 		}
-		seq_printf(m, "tid[%u]=%d", entry->tid,
-			   (int)(entry - rtwdev->cam_info.ba_cam_entry));
+		p += scnprintf(p, end - p, "tid[%u]=%d", entry->tid,
+			       (int)(entry - rtwdev->cam_info.ba_cam_entry));
 	}
-	seq_puts(m, "\n");
+	p += scnprintf(p, end - p, "\n");
+
+	return p - buf;
 }
 
-static void rtw89_sta_link_ids_get(struct seq_file *m,
-				   struct rtw89_dev *rtwdev,
-				   struct rtw89_sta_link *rtwsta_link)
+static int rtw89_sta_link_ids_get(struct rtw89_dev *rtwdev,
+				  char *buf, size_t bufsz,
+				  struct rtw89_sta_link *rtwsta_link,
+				  bool designated)
 {
 	struct ieee80211_link_sta *link_sta;
+	char *p = buf, *end = buf + bufsz;
 
 	rcu_read_lock();
 
 	link_sta = rtw89_sta_rcu_dereference_link(rtwsta_link, true);
 
-	seq_printf(m, "    [%u] %pM\n", rtwsta_link->mac_id, link_sta->addr);
+	p += scnprintf(p, end - p, "    [%u] %pM\n", rtwsta_link->mac_id,
+		       link_sta->addr);
 
 	rcu_read_unlock();
 
-	seq_printf(m, "\tlink_id=%u\n", rtwsta_link->link_id);
-	rtw89_dump_addr_cam(m, rtwdev, &rtwsta_link->addr_cam);
-	rtw89_dump_ba_cam(m, rtwdev, rtwsta_link);
+	p += scnprintf(p, end - p, "\tlink_id=%u%s\n", rtwsta_link->link_id,
+		       designated ? " (*)" : "");
+	p += rtw89_dump_addr_cam(rtwdev, p, end - p, &rtwsta_link->addr_cam);
+	p += rtw89_dump_ba_cam(rtwdev, p, end - p, rtwsta_link);
+
+	return p - buf;
 }
 
 static void rtw89_sta_ids_get_iter(void *data, struct ieee80211_sta *sta)
 {
-	struct seq_file *m = (struct seq_file *)data;
+	struct rtw89_debugfs_iter_data *iter_data =
+		(struct rtw89_debugfs_iter_data *)data;
 	struct rtw89_sta *rtwsta = sta_to_rtwsta(sta);
 	struct rtw89_dev *rtwdev = rtwsta->rtwdev;
+	struct rtw89_sta_link *designated_link;
 	struct rtw89_sta_link *rtwsta_link;
+	size_t bufsz = iter_data->bufsz;
+	char *buf = iter_data->buf;
+	char *p = buf, *end = buf + bufsz;
 	unsigned int link_id;
 
-	seq_printf(m, "STA %pM %s\n", sta->addr, sta->tdls ? "(TDLS)" : "");
+	designated_link = rtw89_get_designated_link(rtwsta);
+
+	p += scnprintf(p, end - p, "STA %pM %s\n", sta->addr,
+		       sta->tdls ? "(TDLS)" : "");
 	rtw89_sta_for_each_link(rtwsta, rtwsta_link, link_id)
-		rtw89_sta_link_ids_get(m, rtwdev, rtwsta_link);
+		p += rtw89_sta_link_ids_get(rtwdev, p, end - p, rtwsta_link,
+					    rtwsta_link == designated_link);
+
+	rtw89_debugfs_iter_data_next(iter_data, p, end - p, p - buf);
 }
 
-static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v)
+static ssize_t rtw89_debug_priv_stations_get(struct rtw89_dev *rtwdev,
+					     struct rtw89_debugfs_priv *debugfs_priv,
+					     char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw89_cam_info *cam_info = &rtwdev->cam_info;
+	struct rtw89_debugfs_iter_data iter_data;
+	char *p = buf, *end = buf + bufsz;
 	u8 idx;
 
-	mutex_lock(&rtwdev->mutex);
-
-	seq_puts(m, "map:\n");
-	seq_printf(m, "\tmac_id:    %*ph\n", (int)sizeof(rtwdev->mac_id_map),
-		   rtwdev->mac_id_map);
-	seq_printf(m, "\taddr_cam:  %*ph\n", (int)sizeof(cam_info->addr_cam_map),
-		   cam_info->addr_cam_map);
-	seq_printf(m, "\tbssid_cam: %*ph\n", (int)sizeof(cam_info->bssid_cam_map),
-		   cam_info->bssid_cam_map);
-	seq_printf(m, "\tsec_cam:   %*ph\n", (int)sizeof(cam_info->sec_cam_map),
-		   cam_info->sec_cam_map);
-	seq_printf(m, "\tba_cam:    %*ph\n", (int)sizeof(cam_info->ba_cam_map),
-		   cam_info->ba_cam_map);
-	seq_printf(m, "\tpkt_ofld:  %*ph\n", (int)sizeof(rtwdev->pkt_offload),
-		   rtwdev->pkt_offload);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	p += scnprintf(p, end - p, "map:\n");
+	p += scnprintf(p, end - p, "\tmac_id:    %*ph\n",
+		       (int)sizeof(rtwdev->mac_id_map),
+		       rtwdev->mac_id_map);
+	p += scnprintf(p, end - p, "\taddr_cam:  %*ph\n",
+		       (int)sizeof(cam_info->addr_cam_map),
+		       cam_info->addr_cam_map);
+	p += scnprintf(p, end - p, "\tbssid_cam: %*ph\n",
+		       (int)sizeof(cam_info->bssid_cam_map),
+		       cam_info->bssid_cam_map);
+	p += scnprintf(p, end - p, "\tsec_cam:   %*ph\n",
+		       (int)sizeof(cam_info->sec_cam_map),
+		       cam_info->sec_cam_map);
+	p += scnprintf(p, end - p, "\tba_cam:    %*ph\n",
+		       (int)sizeof(cam_info->ba_cam_map),
+		       cam_info->ba_cam_map);
+	p += scnprintf(p, end - p, "\tpkt_ofld:  %*ph\n",
+		       (int)sizeof(rtwdev->pkt_offload),
+		       rtwdev->pkt_offload);
 
 	for (idx = NL80211_BAND_2GHZ; idx < NUM_NL80211_BANDS; idx++) {
 		if (!(rtwdev->chip->support_bands & BIT(idx)))
 			continue;
-		rtw89_dump_pkt_offload(m, &rtwdev->scan_info.pkt_list[idx],
-				       "\t\t[SCAN %u]: ", idx);
+		p += rtw89_dump_pkt_offload(p, end - p, &rtwdev->scan_info.pkt_list[idx],
+					    "\t\t[SCAN %u]: ", idx);
 	}
 
+	rtw89_debugfs_iter_data_setup(&iter_data, p, end - p);
 	ieee80211_iterate_active_interfaces_atomic(rtwdev->hw,
-		IEEE80211_IFACE_ITER_NORMAL, rtw89_vif_ids_get_iter, m);
+		IEEE80211_IFACE_ITER_NORMAL, rtw89_vif_ids_get_iter, &iter_data);
+	p += iter_data.written_sz;
+
+	rtw89_debugfs_iter_data_setup(&iter_data, p, end - p);
+	ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_sta_ids_get_iter, &iter_data);
+	p += iter_data.written_sz;
 
-	ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_sta_ids_get_iter, m);
+	return p - buf;
+}
 
-	mutex_unlock(&rtwdev->mutex);
+static void rtw89_debug_disable_dm_cfg_bmap(struct rtw89_dev *rtwdev, u32 new)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	u32 old = hal->disabled_dm_bitmap;
 
-	return 0;
+	if (new == old)
+		return;
+
+	hal->disabled_dm_bitmap = new;
+
+	rtw89_debug(rtwdev, RTW89_DBG_STATE, "Disable DM: 0x%x -> 0x%x\n", old, new);
+}
+
+static void rtw89_debug_disable_dm_set_flag(struct rtw89_dev *rtwdev, u8 flag)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	u32 cur = hal->disabled_dm_bitmap;
+
+	rtw89_debug_disable_dm_cfg_bmap(rtwdev, cur | BIT(flag));
+}
+
+static void rtw89_debug_disable_dm_clr_flag(struct rtw89_dev *rtwdev, u8 flag)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	u32 cur = hal->disabled_dm_bitmap;
+
+	rtw89_debug_disable_dm_cfg_bmap(rtwdev, cur & ~BIT(flag));
 }
 
 #define DM_INFO(type) {RTW89_DM_ ## type, #type}
@@ -3920,92 +4196,187 @@ static const struct rtw89_disabled_dm_info {
 } rtw89_disabled_dm_infos[] = {
 	DM_INFO(DYNAMIC_EDCCA),
 	DM_INFO(THERMAL_PROTECT),
+	DM_INFO(TAS),
+	DM_INFO(MLO),
 };
 
-static int
-rtw89_debug_priv_disable_dm_get(struct seq_file *m, void *v)
+static ssize_t
+rtw89_debug_priv_disable_dm_get(struct rtw89_dev *rtwdev,
+				struct rtw89_debugfs_priv *debugfs_priv,
+				char *buf, size_t bufsz)
 {
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	const struct rtw89_disabled_dm_info *info;
 	struct rtw89_hal *hal = &rtwdev->hal;
+	char *p = buf, *end = buf + bufsz;
 	u32 disabled;
 	int i;
 
-	seq_printf(m, "Disabled DM: 0x%x\n", hal->disabled_dm_bitmap);
+	p += scnprintf(p, end - p, "Disabled DM: 0x%x\n",
+		       hal->disabled_dm_bitmap);
 
 	for (i = 0; i < ARRAY_SIZE(rtw89_disabled_dm_infos); i++) {
 		info = &rtw89_disabled_dm_infos[i];
 		disabled = BIT(info->type) & hal->disabled_dm_bitmap;
 
-		seq_printf(m, "[%d] %s: %c\n", info->type, info->name,
-			   disabled ? 'X' : 'O');
+		p += scnprintf(p, end - p, "[%d] %s: %c\n", info->type,
+			       info->name,
+			       disabled ? 'X' : 'O');
 	}
 
-	return 0;
+	return p - buf;
 }
 
 static ssize_t
-rtw89_debug_priv_disable_dm_set(struct file *filp, const char __user *user_buf,
-				size_t count, loff_t *loff)
+rtw89_debug_priv_disable_dm_set(struct rtw89_dev *rtwdev,
+				struct rtw89_debugfs_priv *debugfs_priv,
+				const char *buf, size_t count)
 {
-	struct seq_file *m = (struct seq_file *)filp->private_data;
-	struct rtw89_debugfs_priv *debugfs_priv = m->private;
-	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	struct rtw89_hal *hal = &rtwdev->hal;
 	u32 conf;
 	int ret;
 
-	ret = kstrtou32_from_user(user_buf, count, 0, &conf);
+	ret = kstrtou32(buf, 0, &conf);
 	if (ret)
 		return -EINVAL;
 
-	hal->disabled_dm_bitmap = conf;
+	rtw89_debug_disable_dm_cfg_bmap(rtwdev, conf);
+
+	return count;
+}
+
+static void rtw89_debug_mlo_mode_set_mlsr(struct rtw89_dev *rtwdev,
+					  unsigned int link_id)
+{
+	struct ieee80211_vif *vif;
+	struct rtw89_vif *rtwvif;
+
+	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
+		vif = rtwvif_to_vif(rtwvif);
+		if (!ieee80211_vif_is_mld(vif))
+			continue;
+
+		rtw89_core_mlsr_switch(rtwdev, rtwvif, link_id);
+	}
+}
+
+static ssize_t
+rtw89_debug_priv_mlo_mode_get(struct rtw89_dev *rtwdev,
+			      struct rtw89_debugfs_priv *debugfs_priv,
+			      char *buf, size_t bufsz)
+{
+	bool mlo_dm_dis = rtwdev->hal.disabled_dm_bitmap & BIT(RTW89_DM_MLO);
+	char *p = buf, *end = buf + bufsz;
+	struct ieee80211_vif *vif;
+	struct rtw89_vif *rtwvif;
+	int count = 0;
+
+	p += scnprintf(p, end - p, "MLD(s) status: (MLO DM: %s)\n",
+		       str_disable_enable(mlo_dm_dis));
+
+	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
+		vif = rtwvif_to_vif(rtwvif);
+		if (!ieee80211_vif_is_mld(vif))
+			continue;
+
+		p += scnprintf(p, end - p,
+			       "\t#%u: MLO mode %x, valid 0x%x, active 0x%x\n",
+			       count++, rtwvif->mlo_mode, vif->valid_links,
+			       vif->active_links);
+	}
+
+	if (count == 0)
+		p += scnprintf(p, end - p, "\t(None)\n");
+
+	return p - buf;
+}
+
+static ssize_t
+rtw89_debug_priv_mlo_mode_set(struct rtw89_dev *rtwdev,
+			      struct rtw89_debugfs_priv *debugfs_priv,
+			      const char *buf, size_t count)
+{
+	u8 num, mlo_mode;
+	u32 argv;
+
+	num = sscanf(buf, "%hhx %u", &mlo_mode, &argv);
+	if (num != 2)
+		return -EINVAL;
+
+	rtw89_debug_disable_dm_set_flag(rtwdev, RTW89_DM_MLO);
+
+	rtw89_debug(rtwdev, RTW89_DBG_STATE, "Set MLO mode to %x\n", mlo_mode);
+
+	switch (mlo_mode) {
+	case RTW89_MLO_MODE_MLSR:
+		rtw89_debug_mlo_mode_set_mlsr(rtwdev, argv);
+		break;
+	default:
+		rtw89_debug(rtwdev, RTW89_DBG_STATE, "Unsupported MLO mode\n");
+		rtw89_debug_disable_dm_clr_flag(rtwdev, RTW89_DM_MLO);
+
+		return -EOPNOTSUPP;
+	}
 
 	return count;
 }
 
-#define rtw89_debug_priv_get(name)				\
+#define rtw89_debug_priv_get(name, opts...)			\
 {								\
 	.cb_read = rtw89_debug_priv_ ##name## _get,		\
+	.opt = { opts },					\
 }
 
-#define rtw89_debug_priv_set(name)				\
+#define rtw89_debug_priv_set(name, opts...)			\
 {								\
 	.cb_write = rtw89_debug_priv_ ##name## _set,		\
+	.opt = { opts },					\
 }
 
-#define rtw89_debug_priv_select_and_get(name)			\
+#define rtw89_debug_priv_select_and_get(name, opts...)		\
 {								\
 	.cb_write = rtw89_debug_priv_ ##name## _select,		\
 	.cb_read = rtw89_debug_priv_ ##name## _get,		\
+	.opt = { opts },					\
 }
 
-#define rtw89_debug_priv_set_and_get(name)			\
+#define rtw89_debug_priv_set_and_get(name, opts...)		\
 {								\
 	.cb_write = rtw89_debug_priv_ ##name## _set,		\
 	.cb_read = rtw89_debug_priv_ ##name## _get,		\
+	.opt = { opts },					\
 }
 
+#define RSIZE_8K .rsize = 0x2000
+#define RSIZE_12K .rsize = 0x3000
+#define RSIZE_16K .rsize = 0x4000
+#define RSIZE_20K .rsize = 0x5000
+#define RSIZE_32K .rsize = 0x8000
+#define RSIZE_64K .rsize = 0x10000
+#define RSIZE_128K .rsize = 0x20000
+#define RSIZE_1M .rsize = 0x100000
+#define RLOCK .rlock = 1
+#define WLOCK .wlock = 1
+#define RWLOCK RLOCK, WLOCK
+
 static const struct rtw89_debugfs rtw89_debugfs_templ = {
 	.read_reg = rtw89_debug_priv_select_and_get(read_reg),
 	.write_reg = rtw89_debug_priv_set(write_reg),
 	.read_rf = rtw89_debug_priv_select_and_get(read_rf),
 	.write_rf = rtw89_debug_priv_set(write_rf),
-	.rf_reg_dump = rtw89_debug_priv_get(rf_reg_dump),
-	.txpwr_table = rtw89_debug_priv_get(txpwr_table),
-	.mac_reg_dump = rtw89_debug_priv_select_and_get(mac_reg_dump),
-	.mac_mem_dump = rtw89_debug_priv_select_and_get(mac_mem_dump),
-	.mac_dbg_port_dump = rtw89_debug_priv_select_and_get(mac_dbg_port_dump),
+	.rf_reg_dump = rtw89_debug_priv_get(rf_reg_dump, RSIZE_8K),
+	.txpwr_table = rtw89_debug_priv_get(txpwr_table, RSIZE_20K, RLOCK),
+	.mac_reg_dump = rtw89_debug_priv_select_and_get(mac_reg_dump, RSIZE_128K),
+	.mac_mem_dump = rtw89_debug_priv_select_and_get(mac_mem_dump, RSIZE_16K, RLOCK),
+	.mac_dbg_port_dump = rtw89_debug_priv_select_and_get(mac_dbg_port_dump, RSIZE_1M),
 	.send_h2c = rtw89_debug_priv_set(send_h2c),
-	.early_h2c = rtw89_debug_priv_set_and_get(early_h2c),
-	.fw_crash = rtw89_debug_priv_set_and_get(fw_crash),
-	.btc_info = rtw89_debug_priv_get(btc_info),
+	.early_h2c = rtw89_debug_priv_set_and_get(early_h2c, RWLOCK),
+	.fw_crash = rtw89_debug_priv_set_and_get(fw_crash, WLOCK),
+	.btc_info = rtw89_debug_priv_get(btc_info, RSIZE_12K),
 	.btc_manual = rtw89_debug_priv_set(btc_manual),
-	.fw_log_manual = rtw89_debug_priv_set(fw_log_manual),
+	.fw_log_manual = rtw89_debug_priv_set(fw_log_manual, WLOCK),
 	.phy_info = rtw89_debug_priv_get(phy_info),
-	.stations = rtw89_debug_priv_get(stations),
-	.disable_dm = rtw89_debug_priv_set_and_get(disable_dm),
+	.stations = rtw89_debug_priv_get(stations, RLOCK),
+	.disable_dm = rtw89_debug_priv_set_and_get(disable_dm, RWLOCK),
+	.mlo_mode = rtw89_debug_priv_set_and_get(mlo_mode, RWLOCK),
 };
 
 #define rtw89_debugfs_add(name, mode, fopname, parent)				\
@@ -4050,6 +4421,7 @@ void rtw89_debugfs_add_sec1(struct rtw89_dev *rtwdev, struct dentry *debugfs_top
 	rtw89_debugfs_add_r(phy_info);
 	rtw89_debugfs_add_r(stations);
 	rtw89_debugfs_add_rw(disable_dm);
+	rtw89_debugfs_add_rw(mlo_mode);
 }
 
 void rtw89_debugfs_init(struct rtw89_dev *rtwdev)
diff --git a/sys/contrib/dev/rtw89/fw.c b/sys/contrib/dev/rtw89/fw.c
index e360f27c2ade..1685084b0140 100644
--- a/sys/contrib/dev/rtw89/fw.c
+++ b/sys/contrib/dev/rtw89/fw.c
@@ -15,6 +15,8 @@
 #include "util.h"
 #include "wow.h"
 
+static bool rtw89_is_any_vif_connected_or_connecting(struct rtw89_dev *rtwdev);
+
 struct rtw89_eapol_2_of_2 {
 	u8 gtkbody[14];
 	u8 key_des_ver;
@@ -38,6 +40,16 @@ struct rtw89_arp_rsp {
 
 static const u8 mss_signature[] = {0x4D, 0x53, 0x53, 0x4B, 0x50, 0x4F, 0x4F, 0x4C};
 
+const struct rtw89_fw_blacklist rtw89_fw_blacklist_default = {
+	.ver = 0x00,
+	.list = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+		 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+		 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+		 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+	},
+};
+EXPORT_SYMBOL(rtw89_fw_blacklist_default);
+
 union rtw89_fw_element_arg {
 	size_t offset;
 	enum rtw89_rf_path rf_path;
@@ -327,7 +339,7 @@ static int __parse_formatted_mssc(struct rtw89_dev *rtwdev,
 	if (!sec->secure_boot)
 		goto out;
 
-	sb_sel_ver = le32_to_cpu(section_content->sb_sel_ver.v);
+	sb_sel_ver = get_unaligned_le32(&section_content->sb_sel_ver.v);
 	if (sb_sel_ver && sb_sel_ver != sec->sb_sel_mgn)
 		goto ignore;
 
@@ -357,6 +369,46 @@ ignore:
 	return 0;
 }
 
+static int __check_secure_blacklist(struct rtw89_dev *rtwdev,
+				    struct rtw89_fw_bin_info *info,
+				    struct rtw89_fw_hdr_section_info *section_info,
+				    const void *content)
+{
+	const struct rtw89_fw_blacklist *chip_blacklist = rtwdev->chip->fw_blacklist;
+	const union rtw89_fw_section_mssc_content *section_content = content;
+	struct rtw89_fw_secure *sec = &rtwdev->fw.sec;
+	u8 byte_idx;
+	u8 bit_mask;
+
+	if (!sec->secure_boot)
+		return 0;
+
+	if (!info->secure_section_exist || section_info->ignore)
+		return 0;
+
+	if (!chip_blacklist) {
+		rtw89_warn(rtwdev, "chip no blacklist for secure firmware\n");
+		return -ENOENT;
+	}
+
+	byte_idx = section_content->blacklist.bit_in_chip_list >> 3;
+	bit_mask = BIT(section_content->blacklist.bit_in_chip_list & 0x7);
+
+	if (section_content->blacklist.ver > chip_blacklist->ver) {
+		rtw89_warn(rtwdev, "chip blacklist out of date (%u, %u)\n",
+			   section_content->blacklist.ver, chip_blacklist->ver);
+		return -EINVAL;
+	}
+
+	if (chip_blacklist->list[byte_idx] & bit_mask) {
+		rtw89_warn(rtwdev, "firmware %u in chip blacklist\n",
+			   section_content->blacklist.ver);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
 static int __parse_security_section(struct rtw89_dev *rtwdev,
 				    struct rtw89_fw_bin_info *info,
 				    struct rtw89_fw_hdr_section_info *section_info,
@@ -381,8 +433,11 @@ static int __parse_security_section(struct rtw89_dev *rtwdev,
 			*mssc_len += section_info->mssc * FWDL_SECURITY_CHKSUM_LEN;
 
 		if (sec->secure_boot) {
-			if (sec->mss_idx >= section_info->mssc)
+			if (sec->mss_idx >= section_info->mssc) {
+				rtw89_err(rtwdev, "unexpected MSS %d >= %d\n",
+					  sec->mss_idx, section_info->mssc);
 				return -EFAULT;
+			}
 			section_info->key_addr = content + section_info->len +
 						 sec->mss_idx * FWDL_SECURITY_SIGLEN;
 			section_info->key_len = FWDL_SECURITY_SIGLEN;
@@ -391,6 +446,9 @@ static int __parse_security_section(struct rtw89_dev *rtwdev,
 		info->secure_section_exist = true;
 	}
 
+	ret = __check_secure_blacklist(rtwdev, info, section_info, content);
+	WARN_ONCE(ret, "Current firmware in blacklist. Please update firmware.\n");
+
 	return 0;
 }
 
@@ -507,18 +565,69 @@ static int rtw89_fw_hdr_parser(struct rtw89_dev *rtwdev,
 }
 
 static
+const struct rtw89_mfw_hdr *rtw89_mfw_get_hdr_ptr(struct rtw89_dev *rtwdev,
+						  const struct firmware *firmware)
+{
+	const struct rtw89_mfw_hdr *mfw_hdr;
+
+	if (sizeof(*mfw_hdr) > firmware->size)
+		return NULL;
+
+	mfw_hdr = (const struct rtw89_mfw_hdr *)&firmware->data[0];
+
+	if (mfw_hdr->sig != RTW89_MFW_SIG)
+		return NULL;
+
+	return mfw_hdr;
+}
+
+static int rtw89_mfw_validate_hdr(struct rtw89_dev *rtwdev,
+				  const struct firmware *firmware,
+				  const struct rtw89_mfw_hdr *mfw_hdr)
+{
+#if defined(__linux__)
+	const void *mfw = firmware->data;
+#elif defined(__FreeBSD__)
+	const u8 *mfw = firmware->data;
+#endif
+	u32 mfw_len = firmware->size;
+	u8 fw_nr = mfw_hdr->fw_nr;
+	const void *ptr;
+
+	if (fw_nr == 0) {
+		rtw89_err(rtwdev, "mfw header has no fw entry\n");
+		return -ENOENT;
+	}
+
+	ptr = &mfw_hdr->info[fw_nr];
+
+#if defined(__linux__)
+	if (ptr > mfw + mfw_len) {
+#elif defined(__FreeBSD__)
+	if ((const u8 *)ptr > mfw + mfw_len) {
+#endif
+		rtw89_err(rtwdev, "mfw header out of address\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static
 int rtw89_mfw_recognize(struct rtw89_dev *rtwdev, enum rtw89_fw_type type,
 			struct rtw89_fw_suit *fw_suit, bool nowarn)
 {
 	struct rtw89_fw_info *fw_info = &rtwdev->fw;
 	const struct firmware *firmware = fw_info->req.firmware;
+	const struct rtw89_mfw_info *mfw_info = NULL, *tmp;
+	const struct rtw89_mfw_hdr *mfw_hdr;
 	const u8 *mfw = firmware->data;
 	u32 mfw_len = firmware->size;
-	const struct rtw89_mfw_hdr *mfw_hdr = (const struct rtw89_mfw_hdr *)mfw;
-	const struct rtw89_mfw_info *mfw_info = NULL, *tmp;
+	int ret;
 	int i;
 
-	if (mfw_hdr->sig != RTW89_MFW_SIG) {
+	mfw_hdr = rtw89_mfw_get_hdr_ptr(rtwdev, firmware);
+	if (!mfw_hdr) {
 		rtw89_debug(rtwdev, RTW89_DBG_FW, "use legacy firmware\n");
 		/* legacy firmware support normal type only */
 		if (type != RTW89_FW_NORMAL)
@@ -528,6 +637,10 @@ int rtw89_mfw_recognize(struct rtw89_dev *rtwdev, enum rtw89_fw_type type,
 		return 0;
 	}
 
+	ret = rtw89_mfw_validate_hdr(rtwdev, firmware, mfw_hdr);
+	if (ret)
+		return ret;
+
 	for (i = 0; i < mfw_hdr->fw_nr; i++) {
 		tmp = &mfw_hdr->info[i];
 		if (tmp->type != type)
@@ -557,6 +670,12 @@ int rtw89_mfw_recognize(struct rtw89_dev *rtwdev, enum rtw89_fw_type type,
 found:
 	fw_suit->data = mfw + le32_to_cpu(mfw_info->shift);
 	fw_suit->size = le32_to_cpu(mfw_info->size);
+
+	if (fw_suit->data + fw_suit->size > mfw + mfw_len) {
+		rtw89_err(rtwdev, "fw_suit %d out of address\n", type);
+		return -EFAULT;
+	}
+
 	return 0;
 }
 
@@ -564,16 +683,21 @@ static u32 rtw89_mfw_get_size(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_fw_info *fw_info = &rtwdev->fw;
 	const struct firmware *firmware = fw_info->req.firmware;
-	const struct rtw89_mfw_hdr *mfw_hdr =
-		(const struct rtw89_mfw_hdr *)firmware->data;
 	const struct rtw89_mfw_info *mfw_info;
+	const struct rtw89_mfw_hdr *mfw_hdr;
 	u32 size;
+	int ret;
 
-	if (mfw_hdr->sig != RTW89_MFW_SIG) {
+	mfw_hdr = rtw89_mfw_get_hdr_ptr(rtwdev, firmware);
+	if (!mfw_hdr) {
 		rtw89_warn(rtwdev, "not mfw format\n");
 		return 0;
 	}
 
+	ret = rtw89_mfw_validate_hdr(rtwdev, firmware, mfw_hdr);
+	if (ret)
+		return ret;
+
 	mfw_info = &mfw_hdr->info[mfw_hdr->fw_nr - 1];
 	size = le32_to_cpu(mfw_info->shift) + le32_to_cpu(mfw_info->size);
 
@@ -719,36 +843,45 @@ struct __fw_feat_cfg {
 static const struct __fw_feat_cfg fw_feat_tbl[] = {
 	__CFG_FW_FEAT(RTL8851B, ge, 0, 29, 37, 1, TX_WAKE),
 	__CFG_FW_FEAT(RTL8851B, ge, 0, 29, 37, 1, SCAN_OFFLOAD),
-	__CFG_FW_FEAT(RTL8851B, ge, 0, 29, 41, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8851B, ge, 0, 29, 41, 0, CRASH_TRIGGER_TYPE_0),
 	__CFG_FW_FEAT(RTL8852A, le, 0, 13, 29, 0, OLD_HT_RA_FORMAT),
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, TX_WAKE),
-	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 36, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 36, 0, CRASH_TRIGGER_TYPE_0),
 	__CFG_FW_FEAT(RTL8852A, lt, 0, 13, 37, 0, NO_WOW_CPU_IO_RX),
 	__CFG_FW_FEAT(RTL8852A, lt, 0, 13, 38, 0, NO_PACKET_DROP),
 	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 26, 0, NO_LPS_PG),
 	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 26, 0, TX_WAKE),
-	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 29, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 29, 0, CRASH_TRIGGER_TYPE_0),
 	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 29, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 29, 7, BEACON_FILTER),
 	__CFG_FW_FEAT(RTL8852B, lt, 0, 29, 30, 0, NO_WOW_CPU_IO_RX),
+	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 127, 0, LPS_DACK_BY_C2H_REG),
+	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 128, 0, CRASH_TRIGGER_TYPE_1),
+	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 128, 0, SCAN_OFFLOAD_EXTRA_OP),
 	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 74, 0, NO_LPS_PG),
 	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 74, 0, TX_WAKE),
-	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 90, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 90, 0, CRASH_TRIGGER_TYPE_0),
 	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 91, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 110, 0, BEACON_FILTER),
+	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 127, 0, SCAN_OFFLOAD_EXTRA_OP),
+	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 127, 0, LPS_DACK_BY_C2H_REG),
+	__CFG_FW_FEAT(RTL8852BT, ge, 0, 29, 127, 0, CRASH_TRIGGER_TYPE_1),
 	__CFG_FW_FEAT(RTL8852C, le, 0, 27, 33, 0, NO_DEEP_PS),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 0, 0, 0, RFK_NTFY_MCC_V0),
 	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 34, 0, TX_WAKE),
 	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 36, 0, SCAN_OFFLOAD),
-	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 40, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 40, 0, CRASH_TRIGGER_TYPE_0),
 	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 56, 10, BEACON_FILTER),
 	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 80, 0, WOW_REASON_V1),
-	__CFG_FW_FEAT(RTL8922A, ge, 0, 34, 30, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 128, 0, BEACON_LOSS_COUNT_V1),
+	__CFG_FW_FEAT(RTL8922A, ge, 0, 34, 30, 0, CRASH_TRIGGER_TYPE_0),
 	__CFG_FW_FEAT(RTL8922A, ge, 0, 34, 11, 0, MACID_PAUSE_SLEEP),
 	__CFG_FW_FEAT(RTL8922A, ge, 0, 34, 35, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 21, 0, SCAN_OFFLOAD_BE_V0),
 	__CFG_FW_FEAT(RTL8922A, ge, 0, 35, 12, 0, BEACON_FILTER),
 	__CFG_FW_FEAT(RTL8922A, ge, 0, 35, 22, 0, WOW_REASON_V1),
+	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 28, 0, RFK_IQK_V0),
 	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 31, 0, RFK_PRE_NOTIFY_V0),
 	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 31, 0, LPS_CH_INFO),
 	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 42, 0, RFK_RXDCK_V0),
@@ -756,6 +889,10 @@ static const struct __fw_feat_cfg fw_feat_tbl[] = {
 	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 47, 0, CH_INFO_BE_V0),
 	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 49, 0, RFK_PRE_NOTIFY_V1),
 	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 51, 0, NO_PHYCAP_P1),
+	__CFG_FW_FEAT(RTL8922A, lt, 0, 35, 64, 0, NO_POWER_DIFFERENCE),
+	__CFG_FW_FEAT(RTL8922A, ge, 0, 35, 71, 0, BEACON_LOSS_COUNT_V1),
+	__CFG_FW_FEAT(RTL8922A, ge, 0, 35, 76, 0, LPS_DACK_BY_C2H_REG),
+	__CFG_FW_FEAT(RTL8922A, ge, 0, 35, 79, 0, CRASH_TRIGGER_TYPE_1),
 };
 
 static void rtw89_fw_iterate_feature_cfg(struct rtw89_fw_info *fw,
@@ -908,11 +1045,7 @@ int rtw89_build_phy_tbl_from_elm(struct rtw89_dev *rtwdev,
 	case RTW89_FW_ELEMENT_ID_RADIO_B:
 	case RTW89_FW_ELEMENT_ID_RADIO_C:
 	case RTW89_FW_ELEMENT_ID_RADIO_D:
-#if defined(__linux__)
 		rf_path = arg.rf_path;
-#elif defined(__FreeBSD__)
-		rf_path = __DECONST(enum rtw89_rf_path, arg.rf_path);
-#endif
 		idx = elm->u.reg2.idx;
 
 		elm_info->rf_radio[idx] = tbl;
@@ -928,7 +1061,7 @@ int rtw89_build_phy_tbl_from_elm(struct rtw89_dev *rtwdev,
 	}
 
 	n_regs = le32_to_cpu(elm->size) / sizeof(tbl->regs[0]);
-	regs = kcalloc(n_regs, sizeof(tbl->regs[0]), GFP_KERNEL);
+	regs = kcalloc(n_regs, sizeof(*regs), GFP_KERNEL);
 	if (!regs)
 		goto out;
 
@@ -1017,7 +1150,7 @@ int rtw89_build_txpwr_trk_tbl_from_elm(struct rtw89_dev *rtwdev,
 	bitmap = le32_to_cpu(elm->u.txpwr_trk.bitmap);
 
 	if ((bitmap & needed_bitmap) != needed_bitmap) {
-		rtw89_warn(rtwdev, "needed txpwr trk bitmap %08x but %0x8x\n",
+		rtw89_warn(rtwdev, "needed txpwr trk bitmap %08x but %08x\n",
 			   needed_bitmap, bitmap);
 		return -ENOENT;
 	}
@@ -1085,6 +1218,110 @@ allocated:
 	return 0;
 }
 
+static bool rtw89_regd_entcpy(struct rtw89_regd *regd, const void *cursor,
+			      u8 cursor_size)
+{
+	/* fill default values if needed for backward compatibility */
+	struct rtw89_fw_regd_entry entry = {
+		.rule_2ghz = RTW89_NA,
+		.rule_5ghz = RTW89_NA,
+		.rule_6ghz = RTW89_NA,
+		.fmap = cpu_to_le32(0x0),
+	};
+	u8 valid_size = min_t(u8, sizeof(entry), cursor_size);
+	unsigned int i;
+	u32 fmap;
+
+	memcpy(&entry, cursor, valid_size);
+	memset(regd, 0, sizeof(*regd));
+
+	regd->alpha2[0] = entry.alpha2_0;
+	regd->alpha2[1] = entry.alpha2_1;
+	regd->alpha2[2] = '\0';
+
+	/* also need to consider forward compatibility */
+	regd->txpwr_regd[RTW89_BAND_2G] = entry.rule_2ghz < RTW89_REGD_NUM ?
+					  entry.rule_2ghz : RTW89_NA;
+	regd->txpwr_regd[RTW89_BAND_5G] = entry.rule_5ghz < RTW89_REGD_NUM ?
+					  entry.rule_5ghz : RTW89_NA;
+	regd->txpwr_regd[RTW89_BAND_6G] = entry.rule_6ghz < RTW89_REGD_NUM ?
+					  entry.rule_6ghz : RTW89_NA;
+
+	BUILD_BUG_ON(sizeof(fmap) != sizeof(entry.fmap));
+	BUILD_BUG_ON(sizeof(fmap) * 8 < NUM_OF_RTW89_REGD_FUNC);
+
+	fmap = le32_to_cpu(entry.fmap);
+	for (i = 0; i < NUM_OF_RTW89_REGD_FUNC; i++) {
+		if (fmap & BIT(i))
+			set_bit(i, regd->func_bitmap);
+	}
+
+	return true;
+}
+
+#if defined(__linux__)
+#define rtw89_for_each_in_regd_element(regd, element) \
+	for (const void *cursor = (element)->content, \
+	     *end = (element)->content + \
+		    le32_to_cpu((element)->num_ents) * (element)->ent_sz; \
+	     cursor < end; cursor += (element)->ent_sz) \
+		if (rtw89_regd_entcpy(regd, cursor, (element)->ent_sz))
+#elif defined(__FreeBSD__)
+#define rtw89_for_each_in_regd_element(regd, element) \
+	for (const u8 *cursor = (element)->content, \
+	     *end = (element)->content + \
+		    le32_to_cpu((element)->num_ents) * (element)->ent_sz; \
+	     cursor < end; cursor += (element)->ent_sz) \
+		if (rtw89_regd_entcpy(regd, cursor, (element)->ent_sz))
+#endif
+
+static
+int rtw89_recognize_regd_from_elm(struct rtw89_dev *rtwdev,
+				  const struct rtw89_fw_element_hdr *elm,
+				  const union rtw89_fw_element_arg arg)
+{
+	const struct __rtw89_fw_regd_element *regd_elm = &elm->u.regd;
+	struct rtw89_fw_elm_info *elm_info = &rtwdev->fw.elm_info;
+	u32 num_ents = le32_to_cpu(regd_elm->num_ents);
+	struct rtw89_regd_data *p;
+	struct rtw89_regd regd;
+	u32 i = 0;
+
+	if (num_ents > RTW89_REGD_MAX_COUNTRY_NUM) {
+		rtw89_warn(rtwdev,
+			   "regd element ents (%d) are over max num (%d)\n",
+			   num_ents, RTW89_REGD_MAX_COUNTRY_NUM);
+		rtw89_warn(rtwdev,
+			   "regd element ignore and take another/common\n");
+		return 1;
+	}
+
+	if (elm_info->regd) {
+		rtw89_debug(rtwdev, RTW89_DBG_REGD,
+			    "regd element take the latter\n");
+		devm_kfree(rtwdev->dev, elm_info->regd);
+		elm_info->regd = NULL;
+	}
+
+	p = devm_kzalloc(rtwdev->dev, struct_size(p, map, num_ents), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+
+	p->nr = num_ents;
+	rtw89_for_each_in_regd_element(&regd, regd_elm)
+		p->map[i++] = regd;
+
+	if (i != num_ents) {
+		rtw89_err(rtwdev, "regd element has %d invalid ents\n",
+			  num_ents - i);
+		devm_kfree(rtwdev->dev, p);
+		return -EINVAL;
+	}
+
+	elm_info->regd = p;
+	return 0;
+}
+
 static const struct rtw89_fw_element_handler __fw_element_handlers[] = {
 	[RTW89_FW_ELEMENT_ID_BBMCU0] = {__rtw89_fw_recognize_from_elm,
 					{ .fw_type = RTW89_FW_BBMCU0 }, NULL},
@@ -1117,6 +1354,18 @@ static const struct rtw89_fw_element_handler __fw_element_handlers[] = {
 		rtw89_fw_recognize_txpwr_from_elm,
 		{ .offset = offsetof(struct rtw89_rfe_data, lmt_6ghz.conf) }, NULL,
 	},
+	[RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_2GHZ] = {
+		rtw89_fw_recognize_txpwr_from_elm,
+		{ .offset = offsetof(struct rtw89_rfe_data, da_lmt_2ghz.conf) }, NULL,
+	},
+	[RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_5GHZ] = {
+		rtw89_fw_recognize_txpwr_from_elm,
+		{ .offset = offsetof(struct rtw89_rfe_data, da_lmt_5ghz.conf) }, NULL,
+	},
+	[RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_6GHZ] = {
+		rtw89_fw_recognize_txpwr_from_elm,
+		{ .offset = offsetof(struct rtw89_rfe_data, da_lmt_6ghz.conf) }, NULL,
+	},
 	[RTW89_FW_ELEMENT_ID_TXPWR_LMT_RU_2GHZ] = {
 		rtw89_fw_recognize_txpwr_from_elm,
 		{ .offset = offsetof(struct rtw89_rfe_data, lmt_ru_2ghz.conf) }, NULL,
@@ -1129,6 +1378,18 @@ static const struct rtw89_fw_element_handler __fw_element_handlers[] = {
 		rtw89_fw_recognize_txpwr_from_elm,
 		{ .offset = offsetof(struct rtw89_rfe_data, lmt_ru_6ghz.conf) }, NULL,
 	},
+	[RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_2GHZ] = {
+		rtw89_fw_recognize_txpwr_from_elm,
+		{ .offset = offsetof(struct rtw89_rfe_data, da_lmt_ru_2ghz.conf) }, NULL,
+	},
+	[RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_5GHZ] = {
+		rtw89_fw_recognize_txpwr_from_elm,
+		{ .offset = offsetof(struct rtw89_rfe_data, da_lmt_ru_5ghz.conf) }, NULL,
+	},
+	[RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_6GHZ] = {
+		rtw89_fw_recognize_txpwr_from_elm,
+		{ .offset = offsetof(struct rtw89_rfe_data, da_lmt_ru_6ghz.conf) }, NULL,
+	},
 	[RTW89_FW_ELEMENT_ID_TX_SHAPE_LMT] = {
 		rtw89_fw_recognize_txpwr_from_elm,
 		{ .offset = offsetof(struct rtw89_rfe_data, tx_shape_lmt.conf) }, NULL,
@@ -1143,6 +1404,9 @@ static const struct rtw89_fw_element_handler __fw_element_handlers[] = {
 	[RTW89_FW_ELEMENT_ID_RFKLOG_FMT] = {
 		rtw89_build_rfk_log_fmt_from_elm, {}, NULL,
 	},
+	[RTW89_FW_ELEMENT_ID_REGD] = {
+		rtw89_recognize_regd_from_elm, {}, "REGD",
+	},
 };
 
 int rtw89_fw_recognize_elements(struct rtw89_dev *rtwdev)
@@ -1351,7 +1615,6 @@ static int __rtw89_fw_download_hdr(struct rtw89_dev *rtwdev,
 	ret = rtw89_h2c_tx(rtwdev, skb, false);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
-		ret = -1;
 		goto fail;
 	}
 
@@ -1438,7 +1701,6 @@ static int __rtw89_fw_download_main(struct rtw89_dev *rtwdev,
 		ret = rtw89_h2c_tx(rtwdev, skb, true);
 		if (ret) {
 			rtw89_err(rtwdev, "failed to send h2c\n");
-			ret = -1;
 			goto fail;
 		}
 
@@ -2314,7 +2576,7 @@ int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable)
 	if (enable)
 		comp = BIT(RTW89_FW_LOG_COMP_INIT) | BIT(RTW89_FW_LOG_COMP_TASK) |
 		       BIT(RTW89_FW_LOG_COMP_PS) | BIT(RTW89_FW_LOG_COMP_ERROR) |
-		       BIT(RTW89_FW_LOG_COMP_SCAN);
+		       BIT(RTW89_FW_LOG_COMP_MLO) | BIT(RTW89_FW_LOG_COMP_SCAN);
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LOG_CFG_LEN);
 	if (!skb) {
@@ -2625,8 +2887,14 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev,
 			  struct rtw89_lps_parm *lps_param)
 {
 	struct sk_buff *skb;
+	bool done_ack;
 	int ret;
 
+	if (RTW89_CHK_FW_FEATURE(LPS_DACK_BY_C2H_REG, &rtwdev->fw))
+		done_ack = false;
+	else
+		done_ack = !lps_param->psmode;
+
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LPS_PARM_LEN);
 	if (!skb) {
 		rtw89_err(rtwdev, "failed to alloc skb for fw dl\n");
@@ -2648,7 +2916,7 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev,
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_MAC,
 			      H2C_CL_MAC_PS,
-			      H2C_FUNC_MAC_LPS_PARM, 0, !lps_param->psmode,
+			      H2C_FUNC_MAC_LPS_PARM, 0, done_ack,
 			      H2C_LPS_PARM_LEN);
 
 	ret = rtw89_h2c_tx(rtwdev, skb, false);
@@ -2739,7 +3007,9 @@ int rtw89_fw_h2c_lps_ml_cmn_info(struct rtw89_dev *rtwdev,
 {
 	const struct rtw89_phy_bb_gain_info_be *gain = &rtwdev->bb_gain.be;
 	struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.cur_pkt_stat;
+	static const u8 bcn_bw_ofst[] = {0, 0, 0, 3, 6, 9, 0, 12};
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_efuse *efuse = &rtwdev->efuse;
 	struct rtw89_h2c_lps_ml_cmn_info *h2c;
 	struct rtw89_vif_link *rtwvif_link;
 	const struct rtw89_chan *chan;
@@ -2747,6 +3017,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info(struct rtw89_dev *rtwdev,
 	u32 len = sizeof(*h2c);
 	unsigned int link_id;
 	struct sk_buff *skb;
+	u8 beacon_bw_ofst;
 	u8 gain_band;
 	u32 done;
 	u8 path;
@@ -2764,9 +3035,10 @@ int rtw89_fw_h2c_lps_ml_cmn_info(struct rtw89_dev *rtwdev,
 	skb_put(skb, len);
 	h2c = (struct rtw89_h2c_lps_ml_cmn_info *)skb->data;
 
-	h2c->fmt_id = 0x1;
+	h2c->fmt_id = 0x3;
 
 	h2c->mlo_dbcc_mode = cpu_to_le32(rtwdev->mlo_dbcc_mode);
+	h2c->rfe_type = efuse->rfe_type;
 
 	rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id) {
 		path = rtwvif_link->phy_idx == RTW89_PHY_1 ? RF_PATH_B : RF_PATH_A;
@@ -2787,9 +3059,21 @@ int rtw89_fw_h2c_lps_ml_cmn_info(struct rtw89_dev *rtwdev,
 			h2c->tia_gain[rtwvif_link->phy_idx][i] =
 				cpu_to_le16(gain->tia_gain[gain_band][bw_idx][path][i]);
 		}
+
+		if (rtwvif_link->bcn_bw_idx < ARRAY_SIZE(bcn_bw_ofst)) {
+			beacon_bw_ofst = bcn_bw_ofst[rtwvif_link->bcn_bw_idx];
+			h2c->dup_bcn_ofst[rtwvif_link->phy_idx] = beacon_bw_ofst;
+		}
+
 		memcpy(h2c->lna_gain[rtwvif_link->phy_idx],
 		       gain->lna_gain[gain_band][bw_idx][path],
 		       LNA_GAIN_NUM);
+		memcpy(h2c->tia_lna_op1db[rtwvif_link->phy_idx],
+		       gain->tia_lna_op1db[gain_band][bw_idx][path],
+		       LNA_GAIN_NUM + 1);
+		memcpy(h2c->lna_op1db[rtwvif_link->phy_idx],
+		       gain->lna_op1db[gain_band][bw_idx][path],
+		       LNA_GAIN_NUM);
 	}
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
@@ -2818,12 +3102,10 @@ fail:
 #define H2C_P2P_ACT_LEN 20
 int rtw89_fw_h2c_p2p_act(struct rtw89_dev *rtwdev,
 			 struct rtw89_vif_link *rtwvif_link,
-			 struct ieee80211_bss_conf *bss_conf,
 			 struct ieee80211_p2p_noa_desc *desc,
-			 u8 act, u8 noa_id)
+			 u8 act, u8 noa_id, u8 ctwindow_oppps)
 {
 	bool p2p_type_gc = rtwvif_link->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT;
-	u8 ctwindow_oppps = bss_conf->p2p_noa_attr.oppps_ctwindow;
 	struct sk_buff *skb;
 	u8 *cmd;
 	int ret;
@@ -2880,8 +3162,8 @@ static void __rtw89_fw_h2c_set_tx_path(struct rtw89_dev *rtwdev,
 		ntx_path = RF_A;
 		map_b = 0;
 	} else {
-		ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_B;
-		map_b = hal->antenna_tx == RF_AB ? 1 : 0;
+		ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_AB;
+		map_b = ntx_path == RF_AB ? 1 : 0;
 	}
 
 	SET_CMC_TBL_NTX_PATH_EN(skb->data, ntx_path);
@@ -3323,9 +3605,10 @@ int rtw89_fw_h2c_assoc_cmac_tbl_g7(struct rtw89_dev *rtwdev,
 			      CCTLINFO_G7_W5_NOMINAL_PKT_PADDING3 |
 			      CCTLINFO_G7_W5_NOMINAL_PKT_PADDING4);
 
-	h2c->w6 = le32_encode_bits(vif->type == NL80211_IFTYPE_STATION ? 1 : 0,
+	h2c->w6 = le32_encode_bits(vif->cfg.aid, CCTLINFO_G7_W6_AID12_PAID) |
+		  le32_encode_bits(vif->type == NL80211_IFTYPE_STATION ? 1 : 0,
 				   CCTLINFO_G7_W6_ULDL);
-	h2c->m6 = cpu_to_le32(CCTLINFO_G7_W6_ULDL);
+	h2c->m6 = cpu_to_le32(CCTLINFO_G7_W6_AID12_PAID | CCTLINFO_G7_W6_ULDL);
 
 	if (rtwsta_link) {
 		h2c->w8 = le32_encode_bits(link_sta->he_cap.has_he,
@@ -3461,6 +3744,104 @@ fail:
 
 	return ret;
 }
+EXPORT_SYMBOL(rtw89_fw_h2c_txtime_cmac_tbl);
+
+int rtw89_fw_h2c_txtime_cmac_tbl_g7(struct rtw89_dev *rtwdev,
+				    struct rtw89_sta_link *rtwsta_link)
+{
+	struct rtw89_h2c_cctlinfo_ud_g7 *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for txtime_cmac_g7\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_cctlinfo_ud_g7 *)skb->data;
+
+	h2c->c0 = le32_encode_bits(rtwsta_link->mac_id, CCTLINFO_G7_C0_MACID) |
+		  le32_encode_bits(1, CCTLINFO_G7_C0_OP);
+
+	if (rtwsta_link->cctl_tx_time) {
+		h2c->w3 |= le32_encode_bits(1, CCTLINFO_G7_W3_AMPDU_TIME_SEL);
+		h2c->m3 |= cpu_to_le32(CCTLINFO_G7_W3_AMPDU_TIME_SEL);
+
+		h2c->w2 |= le32_encode_bits(rtwsta_link->ampdu_max_time,
+					   CCTLINFO_G7_W2_AMPDU_MAX_TIME);
+		h2c->m2 |= cpu_to_le32(CCTLINFO_G7_W2_AMPDU_MAX_TIME);
+	}
+	if (rtwsta_link->cctl_tx_retry_limit) {
+		h2c->w2 |= le32_encode_bits(1, CCTLINFO_G7_W2_DATA_TXCNT_LMT_SEL) |
+			   le32_encode_bits(rtwsta_link->data_tx_cnt_lmt,
+					    CCTLINFO_G7_W2_DATA_TX_CNT_LMT);
+		h2c->m2 |= cpu_to_le32(CCTLINFO_G7_W2_DATA_TXCNT_LMT_SEL |
+				       CCTLINFO_G7_W2_DATA_TX_CNT_LMT);
+	}
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_FR_EXCHG,
+			      H2C_FUNC_MAC_CCTLINFO_UD_G7, 0, 1,
+			      len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+EXPORT_SYMBOL(rtw89_fw_h2c_txtime_cmac_tbl_g7);
+
+int rtw89_fw_h2c_punctured_cmac_tbl_g7(struct rtw89_dev *rtwdev,
+				       struct rtw89_vif_link *rtwvif_link,
+				       u16 punctured)
+{
+	struct rtw89_h2c_cctlinfo_ud_g7 *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for punctured cmac g7\n");
+		return -ENOMEM;
+	}
+
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_cctlinfo_ud_g7 *)skb->data;
+
+	h2c->c0 = le32_encode_bits(rtwvif_link->mac_id, CCTLINFO_G7_C0_MACID) |
+		  le32_encode_bits(1, CCTLINFO_G7_C0_OP);
+
+	h2c->w4 = le32_encode_bits(~punctured, CCTLINFO_G7_W4_ACT_SUBCH_CBW);
+	h2c->m4 = cpu_to_le32(CCTLINFO_G7_W4_ACT_SUBCH_CBW);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_FR_EXCHG,
+			      H2C_FUNC_MAC_CCTLINFO_UD_G7, 0, 1,
+			      len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+EXPORT_SYMBOL(rtw89_fw_h2c_punctured_cmac_tbl_g7);
 
 int rtw89_fw_h2c_txpath_cmac_tbl(struct rtw89_dev *rtwdev,
 				 struct rtw89_sta_link *rtwsta_link)
@@ -3664,14 +4045,15 @@ fail:
 }
 EXPORT_SYMBOL(rtw89_fw_h2c_update_beacon_be);
 
-#define H2C_ROLE_MAINTAIN_LEN 4
 int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev,
 			       struct rtw89_vif_link *rtwvif_link,
 			       struct rtw89_sta_link *rtwsta_link,
 			       enum rtw89_upd_mode upd_mode)
 {
-	struct sk_buff *skb;
 	u8 mac_id = rtwsta_link ? rtwsta_link->mac_id : rtwvif_link->mac_id;
+	struct rtw89_h2c_role_maintain *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
 	u8 self_role;
 	int ret;
 
@@ -3684,21 +4066,27 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev,
 		self_role = rtwvif_link->self_role;
 	}
 
-	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_ROLE_MAINTAIN_LEN);
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
 		rtw89_err(rtwdev, "failed to alloc skb for h2c join\n");
 		return -ENOMEM;
 	}
-	skb_put(skb, H2C_ROLE_MAINTAIN_LEN);
-	SET_FWROLE_MAINTAIN_MACID(skb->data, mac_id);
-	SET_FWROLE_MAINTAIN_SELF_ROLE(skb->data, self_role);
-	SET_FWROLE_MAINTAIN_UPD_MODE(skb->data, upd_mode);
-	SET_FWROLE_MAINTAIN_WIFI_ROLE(skb->data, rtwvif_link->wifi_role);
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_role_maintain *)skb->data;
+
+	h2c->w0 = le32_encode_bits(mac_id, RTW89_H2C_ROLE_MAINTAIN_W0_MACID) |
+		  le32_encode_bits(self_role, RTW89_H2C_ROLE_MAINTAIN_W0_SELF_ROLE) |
+		  le32_encode_bits(upd_mode, RTW89_H2C_ROLE_MAINTAIN_W0_UPD_MODE) |
+		  le32_encode_bits(rtwvif_link->wifi_role,
+				   RTW89_H2C_ROLE_MAINTAIN_W0_WIFI_ROLE) |
+		  le32_encode_bits(rtwvif_link->mac_idx,
+				   RTW89_H2C_ROLE_MAINTAIN_W0_BAND) |
+		  le32_encode_bits(rtwvif_link->port, RTW89_H2C_ROLE_MAINTAIN_W0_PORT);
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_MAC, H2C_CL_MAC_MEDIA_RPT,
 			      H2C_FUNC_MAC_FWROLE_MAINTAIN, 0, 1,
-			      H2C_ROLE_MAINTAIN_LEN);
+			      len);
 
 	ret = rtw89_h2c_tx(rtwdev, skb, false);
 	if (ret) {
@@ -3756,8 +4144,9 @@ out:
 int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link,
 			   struct rtw89_sta_link *rtwsta_link, bool dis_conn)
 {
-	struct sk_buff *skb;
 	u8 mac_id = rtwsta_link ? rtwsta_link->mac_id : rtwvif_link->mac_id;
+	struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link);
+	bool is_mld = ieee80211_vif_is_mld(vif);
 	u8 self_role = rtwvif_link->self_role;
 	enum rtw89_fw_sta_type sta_type;
 	u8 net_type = rtwvif_link->net_type;
@@ -3765,6 +4154,9 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwv
 	struct rtw89_h2c_join *h2c;
 	u32 len = sizeof(*h2c);
 	bool format_v1 = false;
+	struct sk_buff *skb;
+	u8 main_mac_id;
+	bool init_ps;
 	int ret;
 
 	if (rtwdev->chip->chip_gen == RTW89_CHIP_BE) {
@@ -3806,8 +4198,28 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwv
 	h2c_v1 = (struct rtw89_h2c_join_v1 *)skb->data;
 
 	sta_type = rtw89_fw_get_sta_type(rtwdev, rtwvif_link, rtwsta_link);
+	init_ps = rtwvif_link != rtw89_get_designated_link(rtwvif_link->rtwvif);
+
+	if (rtwsta_link)
+		main_mac_id = rtw89_sta_get_main_macid(rtwsta_link->rtwsta);
+	else
+		main_mac_id = rtw89_vif_get_main_macid(rtwvif_link->rtwvif);
+
+	h2c_v1->w1 = le32_encode_bits(sta_type, RTW89_H2C_JOININFO_W1_STA_TYPE) |
+		     le32_encode_bits(is_mld, RTW89_H2C_JOININFO_W1_IS_MLD) |
+		     le32_encode_bits(main_mac_id, RTW89_H2C_JOININFO_W1_MAIN_MACID) |
+		     le32_encode_bits(RTW89_H2C_JOININFO_MLO_MODE_MLSR,
+				      RTW89_H2C_JOININFO_W1_MLO_MODE) |
+		     le32_encode_bits(0, RTW89_H2C_JOININFO_W1_EMLSR_CAB) |
+		     le32_encode_bits(0, RTW89_H2C_JOININFO_W1_NSTR_EN) |
+		     le32_encode_bits(init_ps, RTW89_H2C_JOININFO_W1_INIT_PWR_STATE) |
+		     le32_encode_bits(IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US,
+				      RTW89_H2C_JOININFO_W1_EMLSR_PADDING) |
+		     le32_encode_bits(IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US,
+				      RTW89_H2C_JOININFO_W1_EMLSR_TRANS_DELAY) |
+		     le32_encode_bits(0, RTW89_H2C_JOININFO_W2_MACID_EXT) |
+		     le32_encode_bits(0, RTW89_H2C_JOININFO_W2_MAIN_MACID_EXT);
 
-	h2c_v1->w1 = le32_encode_bits(sta_type, RTW89_H2C_JOININFO_W1_STA_TYPE);
 	h2c_v1->w2 = 0;
 
 done:
@@ -4091,6 +4503,7 @@ int rtw89_fw_h2c_set_bcn_fltr_cfg(struct rtw89_dev *rtwdev,
 	struct rtw89_h2c_bcnfltr *h2c;
 	u32 len = sizeof(*h2c);
 	struct sk_buff *skb;
+	u8 max_cnt, cnt;
 	int ret;
 
 	if (!RTW89_CHK_FW_FEATURE(BEACON_FILTER, &rtwdev->fw))
@@ -4119,12 +4532,20 @@ int rtw89_fw_h2c_set_bcn_fltr_cfg(struct rtw89_dev *rtwdev,
 	skb_put(skb, len);
 	h2c = (struct rtw89_h2c_bcnfltr *)skb->data;
 
+	if (RTW89_CHK_FW_FEATURE(BEACON_LOSS_COUNT_V1, &rtwdev->fw))
+		max_cnt = BIT(7) - 1;
+	else
+		max_cnt = BIT(4) - 1;
+
+	cnt = min(RTW89_BCN_LOSS_CNT, max_cnt);
+
 	h2c->w0 = le32_encode_bits(connect, RTW89_H2C_BCNFLTR_W0_MON_RSSI) |
 		  le32_encode_bits(connect, RTW89_H2C_BCNFLTR_W0_MON_BCN) |
 		  le32_encode_bits(connect, RTW89_H2C_BCNFLTR_W0_MON_EN) |
 		  le32_encode_bits(RTW89_BCN_FLTR_OFFLOAD_MODE_DEFAULT,
 				   RTW89_H2C_BCNFLTR_W0_MODE) |
-		  le32_encode_bits(RTW89_BCN_LOSS_CNT, RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT) |
+		  le32_encode_bits(cnt >> 4, RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT_H3) |
+		  le32_encode_bits(cnt & 0xf, RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT_L4) |
 		  le32_encode_bits(hyst, RTW89_H2C_BCNFLTR_W0_RSSI_HYST) |
 		  le32_encode_bits(thold + MAX_RSSI,
 				   RTW89_H2C_BCNFLTR_W0_RSSI_THRESHOLD) |
@@ -4779,6 +5200,46 @@ fail:
 	return ret;
 }
 
+int rtw89_fw_h2c_cxdrv_osi_info(struct rtw89_dev *rtwdev, u8 type)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_fbtc_outsrc_set_info *osi = &btc->dm.ost_info;
+	struct rtw89_h2c_cxosi *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c cxdrv_osi\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_cxosi *)skb->data;
+
+	h2c->hdr.type = type;
+	h2c->hdr.ver = btc->ver->fcxosi;
+	h2c->hdr.len = len - H2C_LEN_CXDRVHDR_V7;
+	h2c->osi = *osi;
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_OUTSRC, BTFC_SET,
+			      SET_DRV_INFO, 0, 0,
+			      len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
 #define H2C_LEN_CXDRVINFO_CTRL (4 + H2C_LEN_CXDRVHDR)
 int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev, u8 type)
 {
@@ -5054,12 +5515,13 @@ int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id,
 }
 
 static
-int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int ch_num,
-				   struct list_head *chan_list)
+int rtw89_fw_h2c_scan_list_offload_ax(struct rtw89_dev *rtwdev, int ch_num,
+				      struct list_head *chan_list)
 {
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct rtw89_wait_info *wait = &rtwdev->mac.fw_ofld_wait;
 	struct rtw89_h2c_chinfo_elem *elem;
-	struct rtw89_mac_chinfo *ch_info;
+	struct rtw89_mac_chinfo_ax *ch_info;
 	struct rtw89_h2c_chinfo *h2c;
 	struct sk_buff *skb;
 	unsigned int cond;
@@ -5099,6 +5561,10 @@ int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int ch_num,
 			   le32_encode_bits(ch_info->tx_null, RTW89_H2C_CHINFO_W1_TX_NULL) |
 			   le32_encode_bits(ch_info->rand_seq_num, RTW89_H2C_CHINFO_W1_RANDOM);
 
+		if (scan_info->extra_op.set)
+			elem->w1 |= le32_encode_bits(ch_info->macid_tx,
+						     RTW89_H2C_CHINFO_W1_MACID_TX);
+
 		elem->w2 = le32_encode_bits(ch_info->pkt_id[0], RTW89_H2C_CHINFO_W2_PKT0) |
 			   le32_encode_bits(ch_info->pkt_id[1], RTW89_H2C_CHINFO_W2_PKT1) |
 			   le32_encode_bits(ch_info->pkt_id[2], RTW89_H2C_CHINFO_W2_PKT2) |
@@ -5233,12 +5699,12 @@ int rtw89_fw_h2c_scan_list_offload_be(struct rtw89_dev *rtwdev, int ch_num,
 	return 0;
 }
 
-#define RTW89_SCAN_DELAY_TSF_UNIT 104800
 int rtw89_fw_h2c_scan_offload_ax(struct rtw89_dev *rtwdev,
 				 struct rtw89_scan_option *option,
 				 struct rtw89_vif_link *rtwvif_link,
 				 bool wowlan)
 {
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct rtw89_wait_info *wait = &rtwdev->mac.fw_ofld_wait;
 	struct rtw89_chan *op = &rtwdev->scan_info.op_chan;
 	enum rtw89_scan_mode scan_mode = RTW89_SCAN_IMMEDIATE;
@@ -5264,7 +5730,7 @@ int rtw89_fw_h2c_scan_offload_ax(struct rtw89_dev *rtwdev,
 			scan_mode = RTW89_SCAN_IMMEDIATE;
 		} else {
 			scan_mode = RTW89_SCAN_DELAY;
-			tsf += (u64)option->delay * RTW89_SCAN_DELAY_TSF_UNIT;
+			tsf += (u64)option->delay * 1000;
 		}
 	}
 
@@ -5298,6 +5764,10 @@ int rtw89_fw_h2c_scan_offload_ax(struct rtw89_dev *rtwdev,
 	h2c->tsf_low = le32_encode_bits(lower_32_bits(tsf),
 					RTW89_H2C_SCANOFLD_W4_TSF_LOW);
 
+	if (scan_info->extra_op.set)
+		h2c->w6 = le32_encode_bits(scan_info->extra_op.macid,
+					   RTW89_H2C_SCANOFLD_W6_SECOND_MACID);
+
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD,
 			      H2C_FUNC_SCANOFLD, 1, 1,
@@ -5346,31 +5816,58 @@ int rtw89_fw_h2c_scan_offload_be(struct rtw89_dev *rtwdev,
 {
 	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
 	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	const struct rtw89_hw_scan_extra_op *ext = &scan_info->extra_op;
 	struct rtw89_wait_info *wait = &rtwdev->mac.fw_ofld_wait;
 	struct cfg80211_scan_request *req = rtwvif->scan_req;
 	struct rtw89_h2c_scanofld_be_macc_role *macc_role;
+	struct rtw89_hw_scan_extra_op scan_op[2] = {};
 	struct rtw89_chan *op = &scan_info->op_chan;
 	struct rtw89_h2c_scanofld_be_opch *opch;
 	struct rtw89_pktofld_info *pkt_info;
 	struct rtw89_h2c_scanofld_be *h2c;
+	struct ieee80211_vif *vif;
 	struct sk_buff *skb;
 	u8 macc_role_size = sizeof(*macc_role) * option->num_macc_role;
 	u8 opch_size = sizeof(*opch) * option->num_opch;
+	enum rtw89_scan_be_opmode opmode;
 	u8 probe_id[NUM_NL80211_BANDS];
+	u8 scan_offload_ver = U8_MAX;
 	u8 cfg_len = sizeof(*h2c);
 	unsigned int cond;
+	u8 ap_idx = U8_MAX;
 	u8 ver = U8_MAX;
+	u8 policy_val;
 #if defined(__linux__)
 	void *ptr;
 #elif defined(__FreeBSD__)
 	u8 *ptr;
 #endif
+	u8 txbcn;
 	int ret;
 	u32 len;
 	u8 i;
 
+	scan_op[0].macid = rtwvif_link->mac_id;
+	scan_op[0].port = rtwvif_link->port;
+	scan_op[0].chan = *op;
+	vif = rtwvif_to_vif(rtwvif_link->rtwvif);
+	if (vif->type == NL80211_IFTYPE_AP)
+		ap_idx = 0;
+
+	if (ext->set) {
+		scan_op[1] = *ext;
+		vif = rtwvif_to_vif(ext->rtwvif_link->rtwvif);
+		if (vif->type == NL80211_IFTYPE_AP)
+			ap_idx = 1;
+	}
+
 	rtw89_scan_get_6g_disabled_chan(rtwdev, option);
 
+	if (RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD_BE_V0, &rtwdev->fw)) {
+		cfg_len = offsetofend(typeof(*h2c), w8);
+		scan_offload_ver = 0;
+	}
+
 	len = cfg_len + macc_role_size + opch_size;
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
@@ -5424,7 +5921,7 @@ int rtw89_fw_h2c_scan_offload_be(struct rtw89_dev *rtwdev,
 				   RTW89_H2C_SCANOFLD_BE_W4_PROBE_5G) |
 		  le32_encode_bits(probe_id[NL80211_BAND_6GHZ],
 				   RTW89_H2C_SCANOFLD_BE_W4_PROBE_6G) |
-		  le32_encode_bits(option->delay, RTW89_H2C_SCANOFLD_BE_W4_DELAY_START);
+		  le32_encode_bits(option->delay / 1000, RTW89_H2C_SCANOFLD_BE_W4_DELAY_START);
 
 	h2c->w5 = le32_encode_bits(option->mlo_mode, RTW89_H2C_SCANOFLD_BE_W5_MLO_MODE);
 
@@ -5442,10 +5939,8 @@ int rtw89_fw_h2c_scan_offload_be(struct rtw89_dev *rtwdev,
 					   RTW89_H2C_SCANOFLD_BE_W8_PROBE_RATE_6GHZ);
 	}
 
-	if (RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD_BE_V0, &rtwdev->fw)) {
-		cfg_len = offsetofend(typeof(*h2c), w8);
+	if (scan_offload_ver == 0)
 		goto flex_member;
-	}
 
 	h2c->w9 = le32_encode_bits(sizeof(*h2c) / sizeof(h2c->w0),
 				   RTW89_H2C_SCANOFLD_BE_W9_SIZE_CFG) |
@@ -5472,41 +5967,49 @@ flex_member:
 	}
 
 	for (i = 0; i < option->num_opch; i++) {
+		bool is_ap_idx = i == ap_idx;
+
+		opmode = is_ap_idx ? RTW89_SCAN_OPMODE_TBTT : RTW89_SCAN_OPMODE_INTV;
+		policy_val = is_ap_idx ? 2 : RTW89_OFF_CHAN_TIME / 10;
+		txbcn = is_ap_idx ? 1 : 0;
+
 #if defined(__linux__)
 		opch = ptr;
 #elif defined(__FreeBSD__)
 		opch = (void *)ptr;
 #endif
-		opch->w0 = le32_encode_bits(rtwvif_link->mac_id,
+		opch->w0 = le32_encode_bits(scan_op[i].macid,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W0_MACID) |
 			   le32_encode_bits(option->band,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W0_BAND) |
-			   le32_encode_bits(rtwvif_link->port,
+			   le32_encode_bits(scan_op[i].port,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W0_PORT) |
-			   le32_encode_bits(RTW89_SCAN_OPMODE_INTV,
+			   le32_encode_bits(opmode,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W0_POLICY) |
 			   le32_encode_bits(true,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W0_TXNULL) |
-			   le32_encode_bits(RTW89_OFF_CHAN_TIME / 10,
+			   le32_encode_bits(policy_val,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W0_POLICY_VAL);
 
-		opch->w1 = le32_encode_bits(op->band_type,
+		opch->w1 = le32_encode_bits(scan_op[i].chan.band_type,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W1_CH_BAND) |
-			   le32_encode_bits(op->band_width,
+			   le32_encode_bits(scan_op[i].chan.band_width,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W1_BW) |
 			   le32_encode_bits(0x3,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W1_NOTIFY) |
-			   le32_encode_bits(op->primary_channel,
+			   le32_encode_bits(scan_op[i].chan.primary_channel,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W1_PRI_CH) |
-			   le32_encode_bits(op->channel,
+			   le32_encode_bits(scan_op[i].chan.channel,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W1_CENTRAL_CH);
 
 		opch->w2 = le32_encode_bits(0,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W2_PKTS_CTRL) |
 			   le32_encode_bits(0,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W2_SW_DEF) |
-			   le32_encode_bits(2,
-					    RTW89_H2C_SCANOFLD_BE_OPCH_W2_SS);
+			   le32_encode_bits(rtw89_is_mlo_1_1(rtwdev) ? 1 : 2,
+					    RTW89_H2C_SCANOFLD_BE_OPCH_W2_SS) |
+			   le32_encode_bits(txbcn,
+					    RTW89_H2C_SCANOFLD_BE_OPCH_W2_TXBCN);
 
 		opch->w3 = le32_encode_bits(RTW89_SCANOFLD_PKT_NONE,
 					    RTW89_H2C_SCANOFLD_BE_OPCH_W3_PKT0) |
@@ -5581,31 +6084,48 @@ fail:
 int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
+	struct rtw89_fw_h2c_rf_get_mccch_v0 *mccch_v0;
 	struct rtw89_fw_h2c_rf_get_mccch *mccch;
+	u32 len = sizeof(*mccch);
 	struct sk_buff *skb;
+	u8 ver = U8_MAX;
 	int ret;
 	u8 idx;
 
-	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, sizeof(*mccch));
+	if (RTW89_CHK_FW_FEATURE(RFK_NTFY_MCC_V0, &rtwdev->fw)) {
+		len = sizeof(*mccch_v0);
+		ver = 0;
+	}
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
 		rtw89_err(rtwdev, "failed to alloc skb for h2c cxdrv_ctrl\n");
 		return -ENOMEM;
 	}
-	skb_put(skb, sizeof(*mccch));
-	mccch = (struct rtw89_fw_h2c_rf_get_mccch *)skb->data;
+	skb_put(skb, len);
 
 	idx = rfk_mcc->table_idx;
-	mccch->ch_0 = cpu_to_le32(rfk_mcc->ch[0]);
-	mccch->ch_1 = cpu_to_le32(rfk_mcc->ch[1]);
-	mccch->band_0 = cpu_to_le32(rfk_mcc->band[0]);
-	mccch->band_1 = cpu_to_le32(rfk_mcc->band[1]);
-	mccch->current_channel = cpu_to_le32(rfk_mcc->ch[idx]);
-	mccch->current_band_type = cpu_to_le32(rfk_mcc->band[idx]);
+	if (ver == 0) {
+		mccch_v0 = (struct rtw89_fw_h2c_rf_get_mccch_v0 *)skb->data;
+		mccch_v0->ch_0 = cpu_to_le32(rfk_mcc->ch[0]);
+		mccch_v0->ch_1 = cpu_to_le32(rfk_mcc->ch[1]);
+		mccch_v0->band_0 = cpu_to_le32(rfk_mcc->band[0]);
+		mccch_v0->band_1 = cpu_to_le32(rfk_mcc->band[1]);
+		mccch_v0->current_band_type = cpu_to_le32(rfk_mcc->band[idx]);
+		mccch_v0->current_channel = cpu_to_le32(rfk_mcc->ch[idx]);
+	} else {
+		mccch = (struct rtw89_fw_h2c_rf_get_mccch *)skb->data;
+		mccch->ch_0_0 = cpu_to_le32(rfk_mcc->ch[0]);
+		mccch->ch_0_1 = cpu_to_le32(rfk_mcc->ch[0]);
+		mccch->ch_1_0 = cpu_to_le32(rfk_mcc->ch[1]);
+		mccch->ch_1_1 = cpu_to_le32(rfk_mcc->ch[1]);
+		mccch->current_channel = cpu_to_le32(rfk_mcc->ch[idx]);
+	}
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_OUTSRC, H2C_CL_OUTSRC_RF_FW_NOTIFY,
 			      H2C_FUNC_OUTSRC_RF_GET_MCCCH, 0, 0,
-			      sizeof(*mccch));
+			      len);
 
 	ret = rtw89_h2c_tx(rtwdev, skb, false);
 	if (ret) {
@@ -5621,6 +6141,118 @@ fail:
 }
 EXPORT_SYMBOL(rtw89_fw_h2c_rf_ntfy_mcc);
 
+int rtw89_fw_h2c_mcc_dig(struct rtw89_dev *rtwdev,
+			 enum rtw89_chanctx_idx chanctx_idx,
+			 u8 mcc_role_idx, u8 pd_val, bool en)
+{
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, chanctx_idx);
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
+	struct rtw89_h2c_mcc_dig *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c mcc_dig\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_mcc_dig *)skb->data;
+
+	h2c->w0 = le32_encode_bits(1, RTW89_H2C_MCC_DIG_W0_REG_CNT) |
+		  le32_encode_bits(en, RTW89_H2C_MCC_DIG_W0_DM_EN) |
+		  le32_encode_bits(mcc_role_idx, RTW89_H2C_MCC_DIG_W0_IDX) |
+		  le32_encode_bits(1, RTW89_H2C_MCC_DIG_W0_SET) |
+		  le32_encode_bits(1, RTW89_H2C_MCC_DIG_W0_PHY0_EN) |
+		  le32_encode_bits(chan->channel, RTW89_H2C_MCC_DIG_W0_CENTER_CH) |
+		  le32_encode_bits(chan->band_type, RTW89_H2C_MCC_DIG_W0_BAND_TYPE);
+	h2c->w1 = le32_encode_bits(dig_regs->seg0_pd_reg,
+				   RTW89_H2C_MCC_DIG_W1_ADDR_LSB) |
+		  le32_encode_bits(dig_regs->seg0_pd_reg >> 8,
+				   RTW89_H2C_MCC_DIG_W1_ADDR_MSB) |
+		  le32_encode_bits(dig_regs->pd_lower_bound_mask,
+				   RTW89_H2C_MCC_DIG_W1_BMASK_LSB) |
+		  le32_encode_bits(dig_regs->pd_lower_bound_mask >> 8,
+				   RTW89_H2C_MCC_DIG_W1_BMASK_MSB);
+	h2c->w2 = le32_encode_bits(pd_val, RTW89_H2C_MCC_DIG_W2_VAL_LSB);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_OUTSRC, H2C_CL_OUTSRC_DM,
+			      H2C_FUNC_FW_MCC_DIG, 0, 0, len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+int rtw89_fw_h2c_rf_ps_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_vif_link *rtwvif_link;
+	struct rtw89_h2c_rf_ps_info *h2c;
+	const struct rtw89_chan *chan;
+	u32 len = sizeof(*h2c);
+	unsigned int link_id;
+	struct sk_buff *skb;
+	int ret;
+	u8 path;
+	u32 val;
+
+	if (chip->chip_gen != RTW89_CHIP_BE)
+		return 0;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c rf ps info\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_rf_ps_info *)skb->data;
+	h2c->mlo_mode = cpu_to_le32(rtwdev->mlo_dbcc_mode);
+
+	rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id) {
+		chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx);
+		path = rtw89_phy_get_syn_sel(rtwdev, rtwvif_link->phy_idx);
+		val = rtw89_chip_chan_to_rf18_val(rtwdev, chan);
+
+		if (path >= chip->rf_path_num || path >= NUM_OF_RTW89_FW_RFK_PATH) {
+			rtw89_err(rtwdev, "unsupported rf path (%d)\n", path);
+			ret = -ENOENT;
+			goto fail;
+		}
+
+		h2c->rf18[path] = cpu_to_le32(val);
+		h2c->pri_ch[path] = chan->primary_channel;
+	}
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_OUTSRC, H2C_CL_OUTSRC_RF_FW_NOTIFY,
+			      H2C_FUNC_OUTSRC_RF_PS_INFO, 0, 0,
+			      sizeof(*h2c));
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+EXPORT_SYMBOL(rtw89_fw_h2c_rf_ps_info);
+
 int rtw89_fw_h2c_rf_pre_ntfy(struct rtw89_dev *rtwdev,
 			     enum rtw89_phy_idx phy_idx)
 {
@@ -5735,6 +6367,7 @@ fail:
 int rtw89_fw_h2c_rf_tssi(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx,
 			 const struct rtw89_chan *chan, enum rtw89_tssi_mode tssi_mode)
 {
+	struct rtw89_efuse *efuse = &rtwdev->efuse;
 	struct rtw89_hal *hal = &rtwdev->hal;
 	struct rtw89_h2c_rf_tssi *h2c;
 	u32 len = sizeof(*h2c);
@@ -5757,6 +6390,7 @@ int rtw89_fw_h2c_rf_tssi(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx,
 	h2c->hwtx_en = true;
 	h2c->cv = hal->cv;
 	h2c->tssi_mode = tssi_mode;
+	h2c->rfe_type = efuse->rfe_type;
 
 	rtw89_phy_rfk_tssi_fill_fwcmd_efuse_to_de(rtwdev, phy_idx, chan, h2c);
 	rtw89_phy_rfk_tssi_fill_fwcmd_tmeter_tbl(rtwdev, phy_idx, chan, h2c);
@@ -5781,22 +6415,47 @@ fail:
 int rtw89_fw_h2c_rf_iqk(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx,
 			const struct rtw89_chan *chan)
 {
+	struct rtw89_hal *hal = &rtwdev->hal;
+	struct rtw89_h2c_rf_iqk_v0 *h2c_v0;
 	struct rtw89_h2c_rf_iqk *h2c;
 	u32 len = sizeof(*h2c);
 	struct sk_buff *skb;
+	u8 ver = U8_MAX;
 	int ret;
 
+	if (RTW89_CHK_FW_FEATURE(RFK_IQK_V0, &rtwdev->fw)) {
+		len = sizeof(*h2c_v0);
+		ver = 0;
+	}
+
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
 		rtw89_err(rtwdev, "failed to alloc skb for h2c RF IQK\n");
 		return -ENOMEM;
 	}
 	skb_put(skb, len);
+
+	if (ver == 0) {
+		h2c_v0 = (struct rtw89_h2c_rf_iqk_v0 *)skb->data;
+
+		h2c_v0->phy_idx = cpu_to_le32(phy_idx);
+		h2c_v0->dbcc = cpu_to_le32(rtwdev->dbcc_en);
+
+		goto done;
+	}
+
 	h2c = (struct rtw89_h2c_rf_iqk *)skb->data;
 
-	h2c->phy_idx = cpu_to_le32(phy_idx);
-	h2c->dbcc = cpu_to_le32(rtwdev->dbcc_en);
+	h2c->len = sizeof(*h2c);
+	h2c->ktype = 0;
+	h2c->phy = phy_idx;
+	h2c->kpath = rtw89_phy_get_kpath(rtwdev, phy_idx);
+	h2c->band = chan->band_type;
+	h2c->bw = chan->band_width;
+	h2c->ch = chan->channel;
+	h2c->cv = hal->cv;
 
+done:
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_OUTSRC, H2C_CL_OUTSRC_RF_FW_RFK,
 			      H2C_FUNC_RFK_IQK_OFFLOAD, 0, 0, len);
@@ -6052,24 +6711,29 @@ void rtw89_fw_send_all_early_h2c(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_early_h2c *early_h2c;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	list_for_each_entry(early_h2c, &rtwdev->early_h2c_list, list) {
 		rtw89_fw_h2c_raw(rtwdev, early_h2c->h2c, early_h2c->h2c_len);
 	}
 }
 
-void rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev)
+void __rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_early_h2c *early_h2c, *tmp;
 
-	mutex_lock(&rtwdev->mutex);
 	list_for_each_entry_safe(early_h2c, tmp, &rtwdev->early_h2c_list, list) {
 		list_del(&early_h2c->list);
 		kfree(early_h2c->h2c);
 		kfree(early_h2c);
 	}
-	mutex_unlock(&rtwdev->mutex);
+}
+
+void rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev)
+{
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	__rtw89_fw_free_all_early_h2c(rtwdev);
 }
 
 static void rtw89_fw_c2h_parse_attr(struct sk_buff *c2h)
@@ -6113,7 +6777,7 @@ void rtw89_fw_c2h_irqsafe(struct rtw89_dev *rtwdev, struct sk_buff *c2h)
 
 enqueue:
 	skb_queue_tail(&rtwdev->c2h_queue, c2h);
-	ieee80211_queue_work(rtwdev->hw, &rtwdev->c2h_work);
+	wiphy_work_queue(rtwdev->hw->wiphy, &rtwdev->c2h_work);
 }
 
 static void rtw89_fw_c2h_cmd_handle(struct rtw89_dev *rtwdev,
@@ -6151,17 +6815,45 @@ static void rtw89_fw_c2h_cmd_handle(struct rtw89_dev *rtwdev,
 		rtw89_hex_dump(rtwdev, RTW89_DBG_FW, "C2H: ", skb->data, skb->len);
 }
 
-void rtw89_fw_c2h_work(struct work_struct *work)
+void rtw89_fw_c2h_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						c2h_work);
 	struct sk_buff *skb, *tmp;
 
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
 	skb_queue_walk_safe(&rtwdev->c2h_queue, skb, tmp) {
 		skb_unlink(skb, &rtwdev->c2h_queue);
-		mutex_lock(&rtwdev->mutex);
 		rtw89_fw_c2h_cmd_handle(rtwdev, skb);
-		mutex_unlock(&rtwdev->mutex);
+		dev_kfree_skb_any(skb);
+	}
+}
+
+void rtw89_fw_c2h_purge_obsoleted_scan_events(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct sk_buff *skb, *tmp;
+	int limit;
+
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	limit = skb_queue_len(&rtwdev->c2h_queue);
+
+	skb_queue_walk_safe(&rtwdev->c2h_queue, skb, tmp) {
+		struct rtw89_fw_c2h_attr *attr = RTW89_SKB_C2H_CB(skb);
+
+		if (--limit < 0)
+			return;
+
+		if (!attr->is_scan_event || attr->scan_seq == scan_info->seq)
+			continue;
+
+		rtw89_debug(rtwdev, RTW89_DBG_HW_SCAN,
+			    "purge obsoleted scan event with seq=%d (cur=%d)\n",
+			    attr->scan_seq, scan_info->seq);
+
+		skb_unlink(skb, &rtwdev->c2h_queue);
 		dev_kfree_skb_any(skb);
 	}
 }
@@ -6205,13 +6897,18 @@ static int rtw89_fw_read_c2h_reg(struct rtw89_dev *rtwdev,
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_fw_info *fw_info = &rtwdev->fw;
 	const u32 *c2h_reg = chip->c2h_regs;
-	u32 ret;
+	u32 ret, timeout;
 	u8 i, val;
 
 	info->id = RTW89_FWCMD_C2HREG_FUNC_NULL;
 
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_USB)
+		timeout = RTW89_C2H_TIMEOUT_USB;
+	else
+		timeout = RTW89_C2H_TIMEOUT;
+
 	ret = read_poll_timeout_atomic(rtw89_read8, val, val, 1,
-				       RTW89_C2H_TIMEOUT, false, rtwdev,
+				       timeout, false, rtwdev,
 				       chip->c2h_ctrl_reg);
 	if (ret) {
 		rtw89_warn(rtwdev, "c2h reg timeout\n");
@@ -6242,7 +6939,7 @@ int rtw89_fw_msg_reg(struct rtw89_dev *rtwdev,
 	u32 ret;
 
 	if (h2c_info && h2c_info->id != RTW89_FWCMD_H2CREG_FUNC_GET_FEATURE)
-		lockdep_assert_held(&rtwdev->mutex);
+		lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (!h2c_info && !c2h_info)
 		return -EINVAL;
@@ -6284,7 +6981,7 @@ void rtw89_fw_st_dbg_dump(struct rtw89_dev *rtwdev)
 	rtw89_fw_prog_cnt_dump(rtwdev);
 }
 
-static void rtw89_release_pkt_list(struct rtw89_dev *rtwdev)
+static void rtw89_hw_scan_release_pkt_list(struct rtw89_dev *rtwdev)
 {
 	struct list_head *pkt_list = rtwdev->scan_info.pkt_list;
 	struct rtw89_pktofld_info *info, *tmp;
@@ -6303,6 +7000,24 @@ static void rtw89_release_pkt_list(struct rtw89_dev *rtwdev)
 	}
 }
 
+static void rtw89_hw_scan_cleanup(struct rtw89_dev *rtwdev,
+				  struct rtw89_vif_link *rtwvif_link)
+{
+	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
+
+	mac->free_chan_list(rtwdev);
+	rtw89_hw_scan_release_pkt_list(rtwdev);
+
+	rtwvif->scan_req = NULL;
+	rtwvif->scan_ies = NULL;
+	scan_info->scanning_vif = NULL;
+	scan_info->abort = false;
+	scan_info->connected = false;
+	scan_info->delay = 0;
+}
+
 static bool rtw89_is_6ghz_wildcard_probe_req(struct rtw89_dev *rtwdev,
 					     struct cfg80211_scan_request *req,
 					     struct rtw89_pktofld_info *info,
@@ -6371,7 +7086,8 @@ out:
 }
 
 static int rtw89_hw_scan_update_probe_req(struct rtw89_dev *rtwdev,
-					  struct rtw89_vif_link *rtwvif_link)
+					  struct rtw89_vif_link *rtwvif_link,
+					  const u8 *mac_addr)
 {
 	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
 	struct cfg80211_scan_request *req = rtwvif->scan_req;
@@ -6380,7 +7096,7 @@ static int rtw89_hw_scan_update_probe_req(struct rtw89_dev *rtwdev,
 	int ret;
 
 	for (i = 0; i < num; i++) {
-		skb = ieee80211_probereq_get(rtwdev->hw, rtwvif_link->mac_addr,
+		skb = ieee80211_probereq_get(rtwdev->hw, mac_addr,
 					     req->ssids[i].ssid,
 					     req->ssids[i].ssid_len,
 					     req->ie_len);
@@ -6397,10 +7113,10 @@ static int rtw89_hw_scan_update_probe_req(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
-static int rtw89_update_6ghz_rnr_chan(struct rtw89_dev *rtwdev,
-				      struct ieee80211_scan_ies *ies,
-				      struct cfg80211_scan_request *req,
-				      struct rtw89_mac_chinfo *ch_info)
+static int rtw89_update_6ghz_rnr_chan_ax(struct rtw89_dev *rtwdev,
+					 struct ieee80211_scan_ies *ies,
+					 struct cfg80211_scan_request *req,
+					 struct rtw89_mac_chinfo_ax *ch_info)
 {
 	struct rtw89_vif_link *rtwvif_link = rtwdev->scan_info.scanning_vif;
 	struct list_head *pkt_list = rtwdev->scan_info.pkt_list;
@@ -6472,7 +7188,7 @@ out:
 
 static void rtw89_pno_scan_add_chan_ax(struct rtw89_dev *rtwdev,
 				       int chan_type, int ssid_num,
-				       struct rtw89_mac_chinfo *ch_info)
+				       struct rtw89_mac_chinfo_ax *ch_info)
 {
 	struct rtw89_wow_param *rtw_wow = &rtwdev->wow;
 	struct rtw89_pktofld_info *info;
@@ -6520,12 +7236,13 @@ static void rtw89_pno_scan_add_chan_ax(struct rtw89_dev *rtwdev,
 	}
 }
 
-static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
-				   int ssid_num,
-				   struct rtw89_mac_chinfo *ch_info)
+static void rtw89_hw_scan_add_chan_ax(struct rtw89_dev *rtwdev, int chan_type,
+				      int ssid_num,
+				      struct rtw89_mac_chinfo_ax *ch_info)
 {
 	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct rtw89_vif_link *rtwvif_link = rtwdev->scan_info.scanning_vif;
+	const struct rtw89_hw_scan_extra_op *ext = &scan_info->extra_op;
 	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
 	struct ieee80211_scan_ies *ies = rtwvif->scan_ies;
 	struct cfg80211_scan_request *req = rtwvif->scan_req;
@@ -6553,7 +7270,7 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
 		}
 	}
 
-	ret = rtw89_update_6ghz_rnr_chan(rtwdev, ies, req, ch_info);
+	ret = rtw89_update_6ghz_rnr_chan_ax(rtwdev, ies, req, ch_info);
 	if (ret)
 		rtw89_warn(rtwdev, "RNR fails: %d\n", ret);
 
@@ -6596,6 +7313,15 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
 	case RTW89_CHAN_ACTIVE:
 		ch_info->pause_data = true;
 		break;
+	case RTW89_CHAN_EXTRA_OP:
+		ch_info->central_ch = ext->chan.channel;
+		ch_info->pri_ch = ext->chan.primary_channel;
+		ch_info->ch_band = ext->chan.band_type;
+		ch_info->bw = ext->chan.band_width;
+		ch_info->tx_null = true;
+		ch_info->num_pkt = 0;
+		ch_info->macid_tx = true;
+		break;
 	default:
 		rtw89_err(rtwdev, "Channel type out of bound\n");
 	}
@@ -6709,7 +7435,7 @@ int rtw89_pno_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
 {
 	struct rtw89_wow_param *rtw_wow = &rtwdev->wow;
 	struct cfg80211_sched_scan_request *nd_config = rtw_wow->nd_config;
-	struct rtw89_mac_chinfo	*ch_info, *tmp;
+	struct rtw89_mac_chinfo_ax *ch_info, *tmp;
 	struct ieee80211_channel *channel;
 	struct list_head chan_list;
 	int list_len;
@@ -6743,7 +7469,7 @@ int rtw89_pno_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
 		rtw89_pno_scan_add_chan_ax(rtwdev, type, nd_config->n_match_sets, ch_info);
 		list_add_tail(&ch_info->list, &chan_list);
 	}
-	ret = rtw89_fw_h2c_scan_list_offload(rtwdev, list_len, &chan_list);
+	ret = rtw89_fw_h2c_scan_list_offload_ax(rtwdev, list_len, &chan_list);
 
 out:
 	list_for_each_entry_safe(ch_info, tmp, &chan_list, list) {
@@ -6754,24 +7480,59 @@ out:
 	return ret;
 }
 
-int rtw89_hw_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
-				   struct rtw89_vif_link *rtwvif_link, bool connected)
+static int rtw89_hw_scan_add_op_types_ax(struct rtw89_dev *rtwdev,
+					 enum rtw89_chan_type type,
+					 struct list_head *chan_list,
+					 struct cfg80211_scan_request *req,
+					 int *off_chan_time)
+{
+	struct rtw89_mac_chinfo_ax *tmp;
+
+	tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp)
+		return -ENOMEM;
+
+	switch (type) {
+	case RTW89_CHAN_OPERATE:
+		tmp->period = req->duration_mandatory ?
+			      req->duration : RTW89_CHANNEL_TIME;
+		*off_chan_time = 0;
+		break;
+	case RTW89_CHAN_EXTRA_OP:
+		tmp->period = RTW89_CHANNEL_TIME_EXTRA_OP;
+		/* still calc @off_chan_time for scan op */
+		*off_chan_time += tmp->period;
+		break;
+	default:
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	rtw89_hw_scan_add_chan_ax(rtwdev, type, 0, tmp);
+	list_add_tail(&tmp->list, chan_list);
+
+	return 0;
+}
+
+int rtw89_hw_scan_prep_chan_list_ax(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif_link *rtwvif_link)
 {
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	const struct rtw89_hw_scan_extra_op *ext = &scan_info->extra_op;
 	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
 	struct cfg80211_scan_request *req = rtwvif->scan_req;
-	struct rtw89_mac_chinfo	*ch_info, *tmp;
+	struct rtw89_mac_chinfo_ax *ch_info, *tmp;
 	struct ieee80211_channel *channel;
 	struct list_head chan_list;
 	bool random_seq = req->flags & NL80211_SCAN_FLAG_RANDOM_SN;
-	int list_len, off_chan_time = 0;
 	enum rtw89_chan_type type;
-	int ret = 0;
+	int off_chan_time = 0;
+	int ret;
 	u32 idx;
 
 	INIT_LIST_HEAD(&chan_list);
-	for (idx = rtwdev->scan_info.last_chan_idx, list_len = 0;
-	     idx < req->n_channels && list_len < RTW89_SCAN_LIST_LIMIT_AX;
-	     idx++, list_len++) {
+
+	for (idx = 0; idx < req->n_channels; idx++) {
 		channel = req->channels[idx];
 		ch_info = kzalloc(sizeof(*ch_info), GFP_KERNEL);
 		if (!ch_info) {
@@ -6784,6 +7545,8 @@ int rtw89_hw_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
 		else if (channel->band == NL80211_BAND_6GHZ)
 			ch_info->period = RTW89_CHANNEL_TIME_6G +
 					  RTW89_DWELL_TIME_6G;
+		else if (rtwvif_link->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT)
+			ch_info->period = RTW89_P2P_CHAN_TIME;
 		else
 			ch_info->period = RTW89_CHANNEL_TIME;
 
@@ -6798,30 +7561,36 @@ int rtw89_hw_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
 			type = RTW89_CHAN_DFS;
 		else
 			type = RTW89_CHAN_ACTIVE;
-		rtw89_hw_scan_add_chan(rtwdev, type, req->n_ssids, ch_info);
-
-		if (connected &&
-		    off_chan_time + ch_info->period > RTW89_OFF_CHAN_TIME) {
-			tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
-			if (!tmp) {
-				ret = -ENOMEM;
-				kfree(ch_info);
-				goto out;
-			}
+		rtw89_hw_scan_add_chan_ax(rtwdev, type, req->n_ssids, ch_info);
 
-			type = RTW89_CHAN_OPERATE;
-			tmp->period = req->duration_mandatory ?
-				      req->duration : RTW89_CHANNEL_TIME;
-			rtw89_hw_scan_add_chan(rtwdev, type, 0, tmp);
-			list_add_tail(&tmp->list, &chan_list);
-			off_chan_time = 0;
-			list_len++;
+		if (!(scan_info->connected &&
+		      off_chan_time + ch_info->period > RTW89_OFF_CHAN_TIME))
+			goto next;
+
+		ret = rtw89_hw_scan_add_op_types_ax(rtwdev, RTW89_CHAN_OPERATE,
+						    &chan_list, req, &off_chan_time);
+		if (ret) {
+			kfree(ch_info);
+			goto out;
 		}
+
+		if (!ext->set)
+			goto next;
+
+		ret = rtw89_hw_scan_add_op_types_ax(rtwdev, RTW89_CHAN_EXTRA_OP,
+						    &chan_list, req, &off_chan_time);
+		if (ret) {
+			kfree(ch_info);
+			goto out;
+		}
+
+next:
 		list_add_tail(&ch_info->list, &chan_list);
 		off_chan_time += ch_info->period;
 	}
-	rtwdev->scan_info.last_chan_idx = idx;
-	ret = rtw89_fw_h2c_scan_list_offload(rtwdev, list_len, &chan_list);
+
+	list_splice_tail(&chan_list, &scan_info->chan_list);
+	return 0;
 
 out:
 	list_for_each_entry_safe(ch_info, tmp, &chan_list, list) {
@@ -6832,6 +7601,46 @@ out:
 	return ret;
 }
 
+void rtw89_hw_scan_free_chan_list_ax(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_mac_chinfo_ax *ch_info, *tmp;
+
+	list_for_each_entry_safe(ch_info, tmp, &scan_info->chan_list, list) {
+		list_del(&ch_info->list);
+		kfree(ch_info);
+	}
+}
+
+int rtw89_hw_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
+				   struct rtw89_vif_link *rtwvif_link)
+{
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_mac_chinfo_ax *ch_info, *tmp;
+	unsigned int list_len = 0;
+	struct list_head list;
+	int ret;
+
+	INIT_LIST_HEAD(&list);
+
+	list_for_each_entry_safe(ch_info, tmp, &scan_info->chan_list, list) {
+		list_move_tail(&ch_info->list, &list);
+
+		list_len++;
+		if (list_len == RTW89_SCAN_LIST_LIMIT_AX)
+			break;
+	}
+
+	ret = rtw89_fw_h2c_scan_list_offload_ax(rtwdev, list_len, &list);
+
+	list_for_each_entry_safe(ch_info, tmp, &list, list) {
+		list_del(&ch_info->list);
+		kfree(ch_info);
+	}
+
+	return ret;
+}
+
 int rtw89_pno_scan_add_chan_list_be(struct rtw89_dev *rtwdev,
 				    struct rtw89_vif_link *rtwvif_link)
 {
@@ -6885,25 +7694,24 @@ out:
 	return ret;
 }
 
-int rtw89_hw_scan_add_chan_list_be(struct rtw89_dev *rtwdev,
-				   struct rtw89_vif_link *rtwvif_link, bool connected)
+int rtw89_hw_scan_prep_chan_list_be(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif_link *rtwvif_link)
 {
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
 	struct cfg80211_scan_request *req = rtwvif->scan_req;
 	struct rtw89_mac_chinfo_be *ch_info, *tmp;
 	struct ieee80211_channel *channel;
 	struct list_head chan_list;
 	enum rtw89_chan_type type;
-	int list_len, ret;
 	bool random_seq;
+	int ret;
 	u32 idx;
 
 	random_seq = !!(req->flags & NL80211_SCAN_FLAG_RANDOM_SN);
 	INIT_LIST_HEAD(&chan_list);
 
-	for (idx = rtwdev->scan_info.last_chan_idx, list_len = 0;
-	     idx < req->n_channels && list_len < RTW89_SCAN_LIST_LIMIT_BE;
-	     idx++, list_len++) {
+	for (idx = 0; idx < req->n_channels; idx++) {
 		channel = req->channels[idx];
 		ch_info = kzalloc(sizeof(*ch_info), GFP_KERNEL);
 		if (!ch_info) {
@@ -6915,6 +7723,8 @@ int rtw89_hw_scan_add_chan_list_be(struct rtw89_dev *rtwdev,
 			ch_info->period = req->duration;
 		else if (channel->band == NL80211_BAND_6GHZ)
 			ch_info->period = RTW89_CHANNEL_TIME_6G + RTW89_DWELL_TIME_6G;
+		else if (rtwvif_link->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT)
+			ch_info->period = RTW89_P2P_CHAN_TIME;
 		else
 			ch_info->period = RTW89_CHANNEL_TIME;
 
@@ -6933,9 +7743,8 @@ int rtw89_hw_scan_add_chan_list_be(struct rtw89_dev *rtwdev,
 		list_add_tail(&ch_info->list, &chan_list);
 	}
 
-	rtwdev->scan_info.last_chan_idx = idx;
-	ret = rtw89_fw_h2c_scan_list_offload_be(rtwdev, list_len, &chan_list,
-						rtwvif_link);
+	list_splice_tail(&chan_list, &scan_info->chan_list);
+	return 0;
 
 out:
 	list_for_each_entry_safe(ch_info, tmp, &chan_list, list) {
@@ -6946,13 +7755,55 @@ out:
 	return ret;
 }
 
+void rtw89_hw_scan_free_chan_list_be(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_mac_chinfo_be *ch_info, *tmp;
+
+	list_for_each_entry_safe(ch_info, tmp, &scan_info->chan_list, list) {
+		list_del(&ch_info->list);
+		kfree(ch_info);
+	}
+}
+
+int rtw89_hw_scan_add_chan_list_be(struct rtw89_dev *rtwdev,
+				   struct rtw89_vif_link *rtwvif_link)
+{
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_mac_chinfo_be *ch_info, *tmp;
+	unsigned int list_len = 0;
+	struct list_head list;
+	int ret;
+
+	INIT_LIST_HEAD(&list);
+
+	list_for_each_entry_safe(ch_info, tmp, &scan_info->chan_list, list) {
+		list_move_tail(&ch_info->list, &list);
+
+		list_len++;
+		if (list_len == RTW89_SCAN_LIST_LIMIT_BE)
+			break;
+	}
+
+	ret = rtw89_fw_h2c_scan_list_offload_be(rtwdev, list_len, &list,
+						rtwvif_link);
+
+	list_for_each_entry_safe(ch_info, tmp, &list, list) {
+		list_del(&ch_info->list);
+		kfree(ch_info);
+	}
+
+	return ret;
+}
+
 static int rtw89_hw_scan_prehandle(struct rtw89_dev *rtwdev,
-				   struct rtw89_vif_link *rtwvif_link, bool connected)
+				   struct rtw89_vif_link *rtwvif_link,
+				   const u8 *mac_addr)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
 	int ret;
 
-	ret = rtw89_hw_scan_update_probe_req(rtwdev, rtwvif_link);
+	ret = rtw89_hw_scan_update_probe_req(rtwdev, rtwvif_link, mac_addr);
 	if (ret) {
 #if defined(__linux__)
 		rtw89_err(rtwdev, "Update probe request failed\n");
@@ -6961,40 +7812,204 @@ static int rtw89_hw_scan_prehandle(struct rtw89_dev *rtwdev,
 #endif
 		goto out;
 	}
-	ret = mac->add_chan_list(rtwdev, rtwvif_link, connected);
+	ret = mac->prep_chan_list(rtwdev, rtwvif_link);
 out:
 	return ret;
 }
 
-void rtw89_hw_scan_start(struct rtw89_dev *rtwdev,
-			 struct rtw89_vif_link *rtwvif_link,
-			 struct ieee80211_scan_request *scan_req)
+static void rtw89_hw_scan_update_link_beacon_noa(struct rtw89_dev *rtwdev,
+						 struct rtw89_vif_link *rtwvif_link,
+						 u16 tu, bool scan)
+{
+	struct ieee80211_p2p_noa_desc noa_desc = {};
+	struct ieee80211_bss_conf *bss_conf;
+	u16 beacon_int;
+	u64 tsf;
+	int ret;
+
+	rcu_read_lock();
+
+	bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
+	beacon_int = bss_conf->beacon_int;
+
+	rcu_read_unlock();
+
+	tu += beacon_int * 3;
+	if (rtwdev->chip->chip_gen == RTW89_CHIP_AX)
+		rtwdev->scan_info.delay = ieee80211_tu_to_usec(beacon_int * 3) / 1000;
+
+	ret = rtw89_mac_port_get_tsf(rtwdev, rtwvif_link, &tsf);
+	if (ret) {
+		rtw89_warn(rtwdev, "%s: failed to get tsf\n", __func__);
+		return;
+	}
+
+	noa_desc.start_time = cpu_to_le32(tsf);
+	if (rtwdev->chip->chip_gen == RTW89_CHIP_AX) {
+		noa_desc.interval = cpu_to_le32(ieee80211_tu_to_usec(tu));
+		noa_desc.duration = cpu_to_le32(ieee80211_tu_to_usec(tu));
+		noa_desc.count = 1;
+	} else {
+		noa_desc.duration = cpu_to_le32(ieee80211_tu_to_usec(20000));
+		noa_desc.interval = cpu_to_le32(ieee80211_tu_to_usec(20000));
+		noa_desc.count = 255;
+	}
+
+	rtw89_p2p_noa_renew(rtwvif_link);
+	if (scan)
+		rtw89_p2p_noa_append(rtwvif_link, &noa_desc);
+
+	rtw89_chip_h2c_update_beacon(rtwdev, rtwvif_link);
+}
+
+static void rtw89_hw_scan_update_beacon_noa(struct rtw89_dev *rtwdev, bool scan)
+{
+	const struct rtw89_entity_mgnt *mgnt = &rtwdev->hal.entity_mgnt;
+	const struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_mac_chinfo_ax *chinfo_ax;
+	struct rtw89_mac_chinfo_be *chinfo_be;
+	struct rtw89_vif_link *rtwvif_link;
+	struct list_head *pos, *tmp;
+	struct ieee80211_vif *vif;
+	struct rtw89_vif *rtwvif;
+	u16 tu = 0;
+
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	if (!scan)
+		goto update;
+
+	list_for_each_safe(pos, tmp, &scan_info->chan_list) {
+		switch (chip->chip_gen) {
+		case RTW89_CHIP_AX:
+			chinfo_ax = list_entry(pos, typeof(*chinfo_ax), list);
+			tu += chinfo_ax->period;
+			break;
+		case RTW89_CHIP_BE:
+			chinfo_be = list_entry(pos, typeof(*chinfo_be), list);
+			tu += chinfo_be->period;
+			break;
+		default:
+			rtw89_warn(rtwdev, "%s: invalid chip gen %d\n",
+				   __func__, chip->chip_gen);
+			return;
+		}
+	}
+
+	if (unlikely(tu == 0)) {
+		rtw89_debug(rtwdev, RTW89_DBG_HW_SCAN,
+			    "%s: cannot estimate needed TU\n", __func__);
+		return;
+	}
+
+update:
+	list_for_each_entry(rtwvif, &mgnt->active_list, mgnt_entry) {
+		unsigned int link_id;
+
+		vif = rtwvif_to_vif(rtwvif);
+		if (vif->type != NL80211_IFTYPE_AP || !vif->p2p)
+			continue;
+
+		rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id)
+			rtw89_hw_scan_update_link_beacon_noa(rtwdev, rtwvif_link,
+							     tu, scan);
+	}
+}
+
+static void rtw89_hw_scan_set_extra_op_info(struct rtw89_dev *rtwdev,
+					    struct rtw89_vif *scan_rtwvif,
+					    const struct rtw89_chan *scan_op)
+{
+	struct rtw89_entity_mgnt *mgnt = &rtwdev->hal.entity_mgnt;
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_hw_scan_extra_op *ext = &scan_info->extra_op;
+	struct rtw89_vif *tmp;
+
+	ext->set = false;
+	if (!RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD_EXTRA_OP, &rtwdev->fw))
+		return;
+
+	list_for_each_entry(tmp, &mgnt->active_list, mgnt_entry) {
+		const struct rtw89_chan *tmp_chan;
+		struct rtw89_vif_link *tmp_link;
+
+		if (tmp == scan_rtwvif)
+			continue;
+
+		tmp_link = rtw89_vif_get_link_inst(tmp, 0);
+		if (unlikely(!tmp_link)) {
+			rtw89_debug(rtwdev, RTW89_DBG_HW_SCAN,
+				    "hw scan: no HW-0 link for extra op\n");
+			continue;
+		}
+
+		tmp_chan = rtw89_chan_get(rtwdev, tmp_link->chanctx_idx);
+		*ext = (struct rtw89_hw_scan_extra_op){
+			.set = true,
+			.macid = tmp_link->mac_id,
+			.port = tmp_link->port,
+			.chan = *tmp_chan,
+			.rtwvif_link = tmp_link,
+		};
+
+		rtw89_debug(rtwdev, RTW89_DBG_HW_SCAN,
+			    "hw scan: extra op: center %d primary %d\n",
+			    ext->chan.channel, ext->chan.primary_channel);
+		break;
+	}
+}
+
+int rtw89_hw_scan_start(struct rtw89_dev *rtwdev,
+			struct rtw89_vif_link *rtwvif_link,
+			struct ieee80211_scan_request *scan_req)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
+	enum rtw89_entity_mode mode = rtw89_get_entity_mode(rtwdev);
 	struct cfg80211_scan_request *req = &scan_req->req;
 	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev,
 						       rtwvif_link->chanctx_idx);
 	struct rtw89_vif *rtwvif = rtwvif_link->rtwvif;
+	struct rtw89_chanctx_pause_parm pause_parm = {
+		.rsn = RTW89_CHANCTX_PAUSE_REASON_HW_SCAN,
+		.trigger = rtwvif_link,
+	};
 	u32 rx_fltr = rtwdev->hal.rx_fltr;
 	u8 mac_addr[ETH_ALEN];
 	u32 reg;
+	int ret;
 
 	/* clone op and keep it during scan */
 	rtwdev->scan_info.op_chan = *chan;
 
+	rtw89_debug(rtwdev, RTW89_DBG_HW_SCAN,
+		    "hw scan: op: center %d primary %d\n",
+		    chan->channel, chan->primary_channel);
+
+	rtw89_hw_scan_set_extra_op_info(rtwdev, rtwvif, chan);
+
+	rtwdev->scan_info.connected = rtw89_is_any_vif_connected_or_connecting(rtwdev);
 	rtwdev->scan_info.scanning_vif = rtwvif_link;
-	rtwdev->scan_info.last_chan_idx = 0;
 	rtwdev->scan_info.abort = false;
+	rtwdev->scan_info.delay = 0;
 	rtwvif->scan_ies = &scan_req->ies;
 	rtwvif->scan_req = req;
-	ieee80211_stop_queues(rtwdev->hw);
-	rtw89_mac_port_cfg_rx_sync(rtwdev, rtwvif_link, false);
 
 	if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR)
 		get_random_mask_addr(mac_addr, req->mac_addr,
 				     req->mac_addr_mask);
 	else
 		ether_addr_copy(mac_addr, rtwvif_link->mac_addr);
+
+	ret = rtw89_hw_scan_prehandle(rtwdev, rtwvif_link, mac_addr);
+	if (ret) {
+		rtw89_hw_scan_cleanup(rtwdev, rtwvif_link);
+		return ret;
+	}
+
+	ieee80211_stop_queues(rtwdev->hw);
+	rtw89_mac_port_cfg_rx_sync(rtwdev, rtwvif_link, false);
+
 	rtw89_core_scan_start(rtwdev, rtwvif_link, mac_addr, true);
 
 	rx_fltr &= ~B_AX_A_BCN_CHK_EN;
@@ -7004,7 +8019,13 @@ void rtw89_hw_scan_start(struct rtw89_dev *rtwdev,
 	reg = rtw89_mac_reg_by_idx(rtwdev, mac->rx_fltr, rtwvif_link->mac_idx);
 	rtw89_write32_mask(rtwdev, reg, B_AX_RX_FLTR_CFG_MASK, rx_fltr);
 
-	rtw89_chanctx_pause(rtwdev, RTW89_CHANCTX_PAUSE_REASON_HW_SCAN);
+	rtw89_chanctx_pause(rtwdev, &pause_parm);
+	rtw89_phy_dig_suspend(rtwdev);
+
+	if (mode == RTW89_ENTITY_MODE_MCC)
+		rtw89_hw_scan_update_beacon_noa(rtwdev, true);
+
+	return 0;
 }
 
 struct rtw89_hw_scan_complete_cb_data {
@@ -7015,20 +8036,17 @@ struct rtw89_hw_scan_complete_cb_data {
 static int rtw89_hw_scan_complete_cb(struct rtw89_dev *rtwdev, void *data)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
-	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	enum rtw89_entity_mode mode = rtw89_get_entity_mode(rtwdev);
 	struct rtw89_hw_scan_complete_cb_data *cb_data = data;
 	struct rtw89_vif_link *rtwvif_link = cb_data->rtwvif_link;
 	struct cfg80211_scan_info info = {
 		.aborted = cb_data->aborted,
 	};
-	struct rtw89_vif *rtwvif;
 	u32 reg;
 
 	if (!rtwvif_link)
 		return -EINVAL;
 
-	rtwvif = rtwvif_link->rtwvif;
-
 	reg = rtw89_mac_reg_by_idx(rtwdev, mac->rx_fltr, rtwvif_link->mac_idx);
 	rtw89_write32_mask(rtwdev, reg, B_AX_RX_FLTR_CFG_MASK, rtwdev->hal.rx_fltr);
 
@@ -7037,13 +8055,12 @@ static int rtw89_hw_scan_complete_cb(struct rtw89_dev *rtwdev, void *data)
 	ieee80211_wake_queues(rtwdev->hw);
 	rtw89_mac_port_cfg_rx_sync(rtwdev, rtwvif_link, true);
 	rtw89_mac_enable_beacon_for_ap_vifs(rtwdev, true);
+	rtw89_phy_dig_resume(rtwdev, true);
 
-	rtw89_release_pkt_list(rtwdev);
-	rtwvif->scan_req = NULL;
-	rtwvif->scan_ies = NULL;
-	scan_info->last_chan_idx = 0;
-	scan_info->scanning_vif = NULL;
-	scan_info->abort = false;
+	rtw89_hw_scan_cleanup(rtwdev, rtwvif_link);
+
+	if (mode == RTW89_ENTITY_MODE_MCC)
+		rtw89_hw_scan_update_beacon_noa(rtwdev, false);
 
 	return 0;
 }
@@ -7111,6 +8128,8 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev,
 			  bool enable)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	const struct rtw89_hw_scan_extra_op *ext = &scan_info->extra_op;
 	struct rtw89_scan_option opt = {0};
 	bool connected;
 	int ret = 0;
@@ -7118,11 +8137,12 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev,
 	if (!rtwvif_link)
 		return -EINVAL;
 
-	connected = rtw89_is_any_vif_connected_or_connecting(rtwdev);
+	connected = rtwdev->scan_info.connected;
 	opt.enable = enable;
 	opt.target_ch_mode = connected;
+	opt.delay = rtwdev->scan_info.delay;
 	if (enable) {
-		ret = rtw89_hw_scan_prehandle(rtwdev, rtwvif_link, connected);
+		ret = mac->add_chan_list(rtwdev, rtwvif_link);
 		if (ret)
 			goto out;
 	}
@@ -7134,49 +8154,62 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev,
 		opt.num_macc_role = 0;
 		opt.mlo_mode = rtwdev->mlo_dbcc_mode;
 		opt.num_opch = connected ? 1 : 0;
+		if (connected && ext->set)
+			opt.num_opch++;
+
 		opt.opch_end = connected ? 0 : RTW89_CHAN_INVALID;
 	}
 
-	ret = mac->scan_offload(rtwdev, &opt, rtwvif_link, false);
+	ret = rtw89_mac_scan_offload(rtwdev, &opt, rtwvif_link, false);
+
 out:
 	return ret;
 }
 
-#define H2C_FW_CPU_EXCEPTION_LEN 4
-#define H2C_FW_CPU_EXCEPTION_TYPE_DEF 0x5566
+#define H2C_FW_CPU_EXCEPTION_TYPE_0 0x5566
+#define H2C_FW_CPU_EXCEPTION_TYPE_1 0x0
 int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev)
 {
+	struct rtw89_h2c_trig_cpu_except *h2c;
+	u32 cpu_exception_type_def;
+	u32 len = sizeof(*h2c);
 	struct sk_buff *skb;
 	int ret;
 
-	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_FW_CPU_EXCEPTION_LEN);
+	if (RTW89_CHK_FW_FEATURE(CRASH_TRIGGER_TYPE_1, &rtwdev->fw))
+		cpu_exception_type_def = H2C_FW_CPU_EXCEPTION_TYPE_1;
+	else if (RTW89_CHK_FW_FEATURE(CRASH_TRIGGER_TYPE_0, &rtwdev->fw))
+		cpu_exception_type_def = H2C_FW_CPU_EXCEPTION_TYPE_0;
+	else
+		return -EOPNOTSUPP;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
 		rtw89_err(rtwdev,
 			  "failed to alloc skb for fw cpu exception\n");
 		return -ENOMEM;
 	}
 
-	skb_put(skb, H2C_FW_CPU_EXCEPTION_LEN);
-	RTW89_SET_FWCMD_CPU_EXCEPTION_TYPE(skb->data,
-					   H2C_FW_CPU_EXCEPTION_TYPE_DEF);
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_trig_cpu_except *)skb->data;
+
+	h2c->w0 = le32_encode_bits(cpu_exception_type_def,
+				   RTW89_H2C_CPU_EXCEPTION_TYPE);
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_TEST,
 			      H2C_CL_FW_STATUS_TEST,
 			      H2C_FUNC_CPU_EXCEPTION, 0, 0,
-			      H2C_FW_CPU_EXCEPTION_LEN);
+			      len);
 
 	ret = rtw89_h2c_tx(rtwdev, skb, false);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
-		goto fail;
+		dev_kfree_skb_any(skb);
+		return ret;
 	}
 
 	return 0;
-
-fail:
-	dev_kfree_skb_any(skb);
-	return ret;
 }
 
 #define H2C_PKT_DROP_LEN 24
@@ -8494,6 +9527,47 @@ int rtw89_fw_h2c_ap_info_refcount(struct rtw89_dev *rtwdev, bool en)
 	return 0;
 }
 
+int rtw89_fw_h2c_mlo_link_cfg(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link,
+			      bool enable)
+{
+	struct rtw89_wait_info *wait = &rtwdev->mlo.wait;
+	struct rtw89_h2c_mlo_link_cfg *h2c;
+	u8 mac_id = rtwvif_link->mac_id;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	unsigned int cond;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for mlo link cfg\n");
+		return -ENOMEM;
+	}
+
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_mlo_link_cfg *)skb->data;
+
+	h2c->w0 = le32_encode_bits(mac_id, RTW89_H2C_MLO_LINK_CFG_W0_MACID) |
+		  le32_encode_bits(enable, RTW89_H2C_MLO_LINK_CFG_W0_OPTION);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC,
+			      H2C_CL_MLO,
+			      H2C_FUNC_MLO_LINK_CFG, 0, 0,
+			      len);
+
+	cond = RTW89_MLO_WAIT_COND(mac_id, H2C_FUNC_MLO_LINK_CFG);
+
+	ret = rtw89_h2c_tx_and_wait(rtwdev, skb, wait, cond);
+	if (ret) {
+		rtw89_err(rtwdev, "mlo link cfg (%s link id %u) failed: %d\n",
+			  str_enable_disable(enable), rtwvif_link->link_id, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 static bool __fw_txpwr_entry_zero_ext(const void *ext_ptr, u8 ext_len)
 {
 	static const u8 zeros[U8_MAX] = {};
@@ -8931,6 +10005,26 @@ void rtw89_fw_load_tx_shape_lmt_ru(struct rtw89_tx_shape_lmt_ru_data *data)
 	}
 }
 
+static bool rtw89_fw_has_da_txpwr_table(struct rtw89_dev *rtwdev,
+					const struct rtw89_rfe_parms *parms)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	if (chip->support_bands & BIT(NL80211_BAND_2GHZ) &&
+	    !(parms->rule_da_2ghz.lmt && parms->rule_da_2ghz.lmt_ru))
+		return false;
+
+	if (chip->support_bands & BIT(NL80211_BAND_5GHZ) &&
+	    !(parms->rule_da_5ghz.lmt && parms->rule_da_5ghz.lmt_ru))
+		return false;
+
+	if (chip->support_bands & BIT(NL80211_BAND_6GHZ) &&
+	    !(parms->rule_da_6ghz.lmt && parms->rule_da_6ghz.lmt_ru))
+		return false;
+
+	return true;
+}
+
 const struct rtw89_rfe_parms *
 rtw89_load_rfe_data_from_fw(struct rtw89_dev *rtwdev,
 			    const struct rtw89_rfe_parms *init)
@@ -8967,6 +10061,21 @@ rtw89_load_rfe_data_from_fw(struct rtw89_dev *rtwdev,
 		parms->rule_6ghz.lmt = &rfe_data->lmt_6ghz.v;
 	}
 
+	if (rtw89_txpwr_conf_valid(&rfe_data->da_lmt_2ghz.conf)) {
+		rtw89_fw_load_txpwr_lmt_2ghz(&rfe_data->da_lmt_2ghz);
+		parms->rule_da_2ghz.lmt = &rfe_data->da_lmt_2ghz.v;
+	}
+
+	if (rtw89_txpwr_conf_valid(&rfe_data->da_lmt_5ghz.conf)) {
+		rtw89_fw_load_txpwr_lmt_5ghz(&rfe_data->da_lmt_5ghz);
+		parms->rule_da_5ghz.lmt = &rfe_data->da_lmt_5ghz.v;
+	}
+
+	if (rtw89_txpwr_conf_valid(&rfe_data->da_lmt_6ghz.conf)) {
+		rtw89_fw_load_txpwr_lmt_6ghz(&rfe_data->da_lmt_6ghz);
+		parms->rule_da_6ghz.lmt = &rfe_data->da_lmt_6ghz.v;
+	}
+
 	if (rtw89_txpwr_conf_valid(&rfe_data->lmt_ru_2ghz.conf)) {
 		rtw89_fw_load_txpwr_lmt_ru_2ghz(&rfe_data->lmt_ru_2ghz);
 		parms->rule_2ghz.lmt_ru = &rfe_data->lmt_ru_2ghz.v;
@@ -8982,6 +10091,21 @@ rtw89_load_rfe_data_from_fw(struct rtw89_dev *rtwdev,
 		parms->rule_6ghz.lmt_ru = &rfe_data->lmt_ru_6ghz.v;
 	}
 
+	if (rtw89_txpwr_conf_valid(&rfe_data->da_lmt_ru_2ghz.conf)) {
+		rtw89_fw_load_txpwr_lmt_ru_2ghz(&rfe_data->da_lmt_ru_2ghz);
+		parms->rule_da_2ghz.lmt_ru = &rfe_data->da_lmt_ru_2ghz.v;
+	}
+
+	if (rtw89_txpwr_conf_valid(&rfe_data->da_lmt_ru_5ghz.conf)) {
+		rtw89_fw_load_txpwr_lmt_ru_5ghz(&rfe_data->da_lmt_ru_5ghz);
+		parms->rule_da_5ghz.lmt_ru = &rfe_data->da_lmt_ru_5ghz.v;
+	}
+
+	if (rtw89_txpwr_conf_valid(&rfe_data->da_lmt_ru_6ghz.conf)) {
+		rtw89_fw_load_txpwr_lmt_ru_6ghz(&rfe_data->da_lmt_ru_6ghz);
+		parms->rule_da_6ghz.lmt_ru = &rfe_data->da_lmt_ru_6ghz.v;
+	}
+
 	if (rtw89_txpwr_conf_valid(&rfe_data->tx_shape_lmt.conf)) {
 		rtw89_fw_load_tx_shape_lmt(&rfe_data->tx_shape_lmt);
 		parms->tx_shape.lmt = &rfe_data->tx_shape_lmt.v;
@@ -8992,5 +10116,7 @@ rtw89_load_rfe_data_from_fw(struct rtw89_dev *rtwdev,
 		parms->tx_shape.lmt_ru = &rfe_data->tx_shape_lmt_ru.v;
 	}
 
+	parms->has_da = rtw89_fw_has_da_txpwr_table(rtwdev, parms);
+
 	return parms;
 }
diff --git a/sys/contrib/dev/rtw89/fw.h b/sys/contrib/dev/rtw89/fw.h
index de0330bd6f59..4bd7d1e94e6a 100644
--- a/sys/contrib/dev/rtw89/fw.h
+++ b/sys/contrib/dev/rtw89/fw.h
@@ -87,6 +87,9 @@ struct rtw89_c2hreg_phycap {
 #define RTW89_C2HREG_AOAC_RPT_2_W3_IGTK_IPN_IV_6 GENMASK(7, 0)
 #define RTW89_C2HREG_AOAC_RPT_2_W3_IGTK_IPN_IV_7 GENMASK(15, 8)
 
+#define RTW89_C2HREG_PS_LEAVE_ACK_RET GENMASK(7, 0)
+#define RTW89_C2HREG_PS_LEAVE_ACK_MACID GENMASK(31, 16)
+
 struct rtw89_h2creg_hdr {
 	u32 w0;
 };
@@ -112,6 +115,8 @@ struct rtw89_h2creg_sch_tx_en {
 #define RTW89_C2HREG_HDR_LEN 2
 #define RTW89_H2CREG_HDR_LEN 2
 #define RTW89_C2H_TIMEOUT 1000000
+#define RTW89_C2H_TIMEOUT_USB 4000
+
 struct rtw89_mac_c2h_info {
 	u8 id;
 	u8 content_len;
@@ -154,6 +159,7 @@ enum rtw89_mac_c2h_type {
 	RTW89_FWCMD_C2HREG_FUNC_TX_PAUSE_RPT,
 	RTW89_FWCMD_C2HREG_FUNC_WOW_CPUIO_RX_ACK = 0xA,
 	RTW89_FWCMD_C2HREG_FUNC_PHY_CAP_PART1 = 0xC,
+	RTW89_FWCMD_C2HREG_FUNC_PS_LEAVE_ACK = 0xD,
 	RTW89_FWCMD_C2HREG_FUNC_NULL = 0xFF,
 };
 
@@ -199,6 +205,7 @@ enum rtw89_fw_log_comp {
 	RTW89_FW_LOG_COMP_TWT,
 	RTW89_FW_LOG_COMP_RF,
 	RTW89_FW_LOG_COMP_MCC = 20,
+	RTW89_FW_LOG_COMP_MLO = 26,
 	RTW89_FW_LOG_COMP_SCAN = 28,
 };
 
@@ -236,6 +243,7 @@ enum rtw89_chan_type {
 	RTW89_CHAN_OPERATE = 0,
 	RTW89_CHAN_ACTIVE,
 	RTW89_CHAN_DFS,
+	RTW89_CHAN_EXTRA_OP,
 };
 
 enum rtw89_p2pps_action {
@@ -315,8 +323,10 @@ struct rtw89_fw_macid_pause_sleep_grp {
 #define RTW89_H2C_MAX_SIZE 2048
 #define RTW89_CHANNEL_TIME 45
 #define RTW89_CHANNEL_TIME_6G 20
+#define RTW89_CHANNEL_TIME_EXTRA_OP 30
 #define RTW89_DFS_CHAN_TIME 105
 #define RTW89_OFF_CHAN_TIME 100
+#define RTW89_P2P_CHAN_TIME 105
 #define RTW89_DWELL_TIME 20
 #define RTW89_DWELL_TIME_6G 10
 #define RTW89_SCAN_WIDTH 0
@@ -333,9 +343,9 @@ struct rtw89_fw_macid_pause_sleep_grp {
 #define RTW89_SCAN_LIST_LIMIT_AX RTW89_SCAN_LIST_LIMIT(RTW89_MAC_CHINFO_SIZE)
 #define RTW89_SCAN_LIST_LIMIT_BE RTW89_SCAN_LIST_LIMIT(RTW89_MAC_CHINFO_SIZE_BE)
 
-#define RTW89_BCN_LOSS_CNT 10
+#define RTW89_BCN_LOSS_CNT 60
 
-struct rtw89_mac_chinfo {
+struct rtw89_mac_chinfo_ax {
 	u8 period;
 	u8 dwell_time;
 	u8 central_ch;
@@ -351,7 +361,8 @@ struct rtw89_mac_chinfo {
 	u8 tx_null:1;
 	u8 rand_seq_num:1;
 	u8 cfg_tx_pwr:1;
-	u8 rsvd0: 4;
+	u8 macid_tx: 1;
+	u8 rsvd0: 3;
 	u8 pkt_id[RTW89_SCANOFLD_MAX_SSID];
 	u16 tx_pwr_idx;
 	u8 rsvd1;
@@ -664,6 +675,11 @@ struct rtw89_fw_mss_pool_hdr {
 
 union rtw89_fw_section_mssc_content {
 	struct {
+		u8 pad[0x20];
+		u8 bit_in_chip_list;
+		u8 ver;
+	} __packed blacklist;
+	struct {
 		u8 pad[58];
 		__le32 v;
 	} __packed sb_sel_ver;
@@ -673,6 +689,13 @@ union rtw89_fw_section_mssc_content {
 	} __packed key_sign_len;
 } __packed;
 
+struct rtw89_fw_blacklist {
+	u8 ver;
+	u8 list[32];
+};
+
+extern const struct rtw89_fw_blacklist rtw89_fw_blacklist_default;
+
 static inline void SET_CTRL_INFO_MACID(void *table, u32 val)
 {
 	le32p_replace_bits((__le32 *)(table) + 0, val, GENMASK(6, 0));
@@ -1579,25 +1602,17 @@ struct rtw89_h2c_bcn_upd_be {
 #define RTW89_H2C_BCN_UPD_BE_W7_ECSA_OFST GENMASK(30, 16)
 #define RTW89_H2C_BCN_UPD_BE_W7_PROTECTION_KEY_ID BIT(31)
 
-static inline void SET_FWROLE_MAINTAIN_MACID(void *h2c, u32 val)
-{
-	le32p_replace_bits((__le32 *)h2c, val, GENMASK(7, 0));
-}
-
-static inline void SET_FWROLE_MAINTAIN_SELF_ROLE(void *h2c, u32 val)
-{
-	le32p_replace_bits((__le32 *)h2c, val, GENMASK(9, 8));
-}
-
-static inline void SET_FWROLE_MAINTAIN_UPD_MODE(void *h2c, u32 val)
-{
-	le32p_replace_bits((__le32 *)h2c, val, GENMASK(12, 10));
-}
+struct rtw89_h2c_role_maintain {
+	__le32 w0;
+};
 
-static inline void SET_FWROLE_MAINTAIN_WIFI_ROLE(void *h2c, u32 val)
-{
-	le32p_replace_bits((__le32 *)h2c, val, GENMASK(16, 13));
-}
+#define RTW89_H2C_ROLE_MAINTAIN_W0_MACID GENMASK(7, 0)
+#define RTW89_H2C_ROLE_MAINTAIN_W0_SELF_ROLE GENMASK(9, 8)
+#define RTW89_H2C_ROLE_MAINTAIN_W0_UPD_MODE GENMASK(12, 10)
+#define RTW89_H2C_ROLE_MAINTAIN_W0_WIFI_ROLE GENMASK(16, 13)
+#define RTW89_H2C_ROLE_MAINTAIN_W0_BAND GENMASK(18, 17)
+#define RTW89_H2C_ROLE_MAINTAIN_W0_PORT GENMASK(21, 19)
+#define RTW89_H2C_ROLE_MAINTAIN_W0_MACID_EXT GENMASK(31, 24)
 
 enum rtw89_fw_sta_type { /* value of RTW89_H2C_JOININFO_W1_STA_TYPE */
 	RTW89_FW_N_AC_STA = 0,
@@ -1632,6 +1647,8 @@ struct rtw89_h2c_join_v1 {
 #define RTW89_H2C_JOININFO_W1_IS_MLD BIT(3)
 #define RTW89_H2C_JOININFO_W1_MAIN_MACID GENMASK(11, 4)
 #define RTW89_H2C_JOININFO_W1_MLO_MODE BIT(12)
+#define RTW89_H2C_JOININFO_MLO_MODE_MLMR 0
+#define RTW89_H2C_JOININFO_MLO_MODE_MLSR 1
 #define RTW89_H2C_JOININFO_W1_EMLSR_CAB BIT(13)
 #define RTW89_H2C_JOININFO_W1_NSTR_EN BIT(14)
 #define RTW89_H2C_JOININFO_W1_INIT_PWR_STATE BIT(15)
@@ -1801,23 +1818,28 @@ struct rtw89_h2c_lps_ch_info {
 
 struct rtw89_h2c_lps_ml_cmn_info {
 	u8 fmt_id;
-	u8 rsvd0[3];
+	u8 rfe_type;
+	u8 rsvd0[2];
 	__le32 mlo_dbcc_mode;
-	u8 central_ch[RTW89_PHY_MAX];
-	u8 pri_ch[RTW89_PHY_MAX];
-	u8 bw[RTW89_PHY_MAX];
-	u8 band[RTW89_PHY_MAX];
-	u8 bcn_rate_type[RTW89_PHY_MAX];
+	u8 central_ch[RTW89_PHY_NUM];
+	u8 pri_ch[RTW89_PHY_NUM];
+	u8 bw[RTW89_PHY_NUM];
+	u8 band[RTW89_PHY_NUM];
+	u8 bcn_rate_type[RTW89_PHY_NUM];
 	u8 rsvd1[2];
-	__le16 tia_gain[RTW89_PHY_MAX][TIA_GAIN_NUM];
-	u8 lna_gain[RTW89_PHY_MAX][LNA_GAIN_NUM];
+	__le16 tia_gain[RTW89_PHY_NUM][TIA_GAIN_NUM];
+	u8 lna_gain[RTW89_PHY_NUM][LNA_GAIN_NUM];
 	u8 rsvd2[2];
+	u8 tia_lna_op1db[RTW89_PHY_NUM][LNA_GAIN_NUM + 1];
+	u8 lna_op1db[RTW89_PHY_NUM][LNA_GAIN_NUM];
+	u8 dup_bcn_ofst[RTW89_PHY_NUM];
 } __packed;
 
-static inline void RTW89_SET_FWCMD_CPU_EXCEPTION_TYPE(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 0));
-}
+struct rtw89_h2c_trig_cpu_except {
+	__le32 w0;
+} __packed;
+
+#define RTW89_H2C_CPU_EXCEPTION_TYPE GENMASK(31, 0)
 
 static inline void RTW89_SET_FWCMD_PKT_DROP_SEL(void *cmd, u32 val)
 {
@@ -2236,6 +2258,11 @@ struct rtw89_h2c_cxrole_v8 {
 	struct rtw89_btc_wl_role_info_v8_u32 _u32;
 } __packed;
 
+struct rtw89_h2c_cxosi {
+	struct rtw89_h2c_cxhdr_v7 hdr;
+	struct rtw89_btc_fbtc_outsrc_set_info osi;
+} __packed;
+
 struct rtw89_h2c_cxinit {
 	struct rtw89_h2c_cxhdr hdr;
 	u8 ant_type;
@@ -2663,6 +2690,7 @@ struct rtw89_h2c_chinfo_elem {
 #define RTW89_H2C_CHINFO_W1_TX_NULL BIT(25)
 #define RTW89_H2C_CHINFO_W1_RANDOM BIT(26)
 #define RTW89_H2C_CHINFO_W1_CFG_TX BIT(27)
+#define RTW89_H2C_CHINFO_W1_MACID_TX BIT(29)
 #define RTW89_H2C_CHINFO_W2_PKT0 GENMASK(7, 0)
 #define RTW89_H2C_CHINFO_W2_PKT1 GENMASK(15, 8)
 #define RTW89_H2C_CHINFO_W2_PKT2 GENMASK(23, 16)
@@ -2762,6 +2790,7 @@ struct rtw89_h2c_scanofld {
 #define RTW89_H2C_SCANOFLD_W2_SLOW_PD GENMASK(23, 16)
 #define RTW89_H2C_SCANOFLD_W3_TSF_HIGH GENMASK(31, 0)
 #define RTW89_H2C_SCANOFLD_W4_TSF_LOW GENMASK(31, 0)
+#define RTW89_H2C_SCANOFLD_W6_SECOND_MACID GENMASK(31, 24)
 
 struct rtw89_h2c_scanofld_be_macc_role {
 	__le32 w0;
@@ -2795,6 +2824,7 @@ struct rtw89_h2c_scanofld_be_opch {
 #define RTW89_H2C_SCANOFLD_BE_OPCH_W2_PKTS_CTRL GENMASK(7, 0)
 #define RTW89_H2C_SCANOFLD_BE_OPCH_W2_SW_DEF GENMASK(15, 8)
 #define RTW89_H2C_SCANOFLD_BE_OPCH_W2_SS GENMASK(18, 16)
+#define RTW89_H2C_SCANOFLD_BE_OPCH_W2_TXBCN BIT(19)
 #define RTW89_H2C_SCANOFLD_BE_OPCH_W3_PKT0 GENMASK(7, 0)
 #define RTW89_H2C_SCANOFLD_BE_OPCH_W3_PKT1 GENMASK(15, 8)
 #define RTW89_H2C_SCANOFLD_BE_OPCH_W3_PKT2 GENMASK(23, 16)
@@ -2855,6 +2885,13 @@ struct rtw89_h2c_fwips {
 #define RTW89_H2C_FW_IPS_W0_MACID GENMASK(7, 0)
 #define RTW89_H2C_FW_IPS_W0_ENABLE BIT(8)
 
+struct rtw89_h2c_mlo_link_cfg {
+	__le32 w0;
+};
+
+#define RTW89_H2C_MLO_LINK_CFG_W0_MACID GENMASK(15, 0)
+#define RTW89_H2C_MLO_LINK_CFG_W0_OPTION GENMASK(19, 16)
+
 static inline void RTW89_SET_FWCMD_P2P_MACID(void *cmd, u32 val)
 {
 	le32p_replace_bits((__le32 *)cmd, val, GENMASK(7, 0));
@@ -3535,6 +3572,8 @@ struct rtw89_fw_c2h_attr {
 	u8 class;
 	u8 func;
 	u16 len;
+	u8 is_scan_event: 1;
+	u8 scan_seq: 2;
 };
 
 static inline struct rtw89_fw_c2h_attr *RTW89_SKB_C2H_CB(struct sk_buff *skb)
@@ -3558,6 +3597,7 @@ struct rtw89_c2h_done_ack {
 #define RTW89_C2H_DONE_ACK_W2_CLASS GENMASK(7, 2)
 #define RTW89_C2H_DONE_ACK_W2_FUNC GENMASK(15, 8)
 #define RTW89_C2H_DONE_ACK_W2_H2C_RETURN GENMASK(23, 16)
+#define	RTW89_C2H_SCAN_DONE_ACK_RETURN GENMASK(5, 0)
 #define RTW89_C2H_DONE_ACK_W2_H2C_SEQ GENMASK(31, 24)
 
 #define RTW89_GET_MAC_C2H_REV_ACK_CAT(c2h) \
@@ -3616,6 +3656,19 @@ struct rtw89_c2h_ra_rpt {
 #define RTW89_C2H_RA_RPT_W3_MD_SEL_B2 BIT(15)
 #define RTW89_C2H_RA_RPT_W3_BW_B2 BIT(16)
 
+struct rtw89_c2h_fw_scan_rpt {
+	struct rtw89_c2h_hdr hdr;
+	u8 phy_idx;
+	u8 band;
+	u8 center_ch;
+	u8 ofdm_pd_idx; /* in unit of 2 dBm */
+#define PD_LOWER_BOUND_BASE 102
+	s8 cck_pd_idx;
+	u8 rsvd0;
+	u8 rsvd1;
+	u8 rsvd2;
+} __packed;
+
 /* For WiFi 6 chips:
  *   VHT, HE, HT-old: [6:4]: NSS, [3:0]: MCS
  *   HT-new: [6:5]: NA, [4:0]: MCS
@@ -3717,6 +3770,25 @@ rtw89_static_assert(sizeof(struct rtw89_mac_mcc_tsf_rpt) <= RTW89_COMPLETION_BUF
 #define RTW89_GET_MAC_C2H_MCC_STATUS_RPT_TSF_HIGH(c2h) \
 	le32_get_bits(*((const __le32 *)(c2h) + 4), GENMASK(31, 0))
 
+struct rtw89_c2h_mlo_link_cfg_rpt {
+	struct rtw89_c2h_hdr hdr;
+	__le32 w2;
+} __packed;
+
+#define RTW89_C2H_MLO_LINK_CFG_RPT_W2_MACID GENMASK(15, 0)
+#define RTW89_C2H_MLO_LINK_CFG_RPT_W2_STATUS GENMASK(19, 16)
+
+enum rtw89_c2h_mlo_link_status {
+	RTW89_C2H_MLO_LINK_CFG_IDLE = 0,
+	RTW89_C2H_MLO_LINK_CFG_DONE = 1,
+	RTW89_C2H_MLO_LINK_CFG_ISSUE_NULL_FAIL = 2,
+	RTW89_C2H_MLO_LINK_CFG_TX_NULL_FAIL = 3,
+	RTW89_C2H_MLO_LINK_CFG_ROLE_NOT_EXIST = 4,
+	RTW89_C2H_MLO_LINK_CFG_NULL_1_TIMEOUT = 5,
+	RTW89_C2H_MLO_LINK_CFG_NULL_0_TIMEOUT = 6,
+	RTW89_C2H_MLO_LINK_CFG_RUNNING = 0xff,
+};
+
 struct rtw89_mac_mrc_tsf_rpt {
 	unsigned int num;
 	u64 tsfs[RTW89_MAC_MRC_MAX_REQ_TSF_NUM];
@@ -3813,7 +3885,8 @@ struct rtw89_h2c_bcnfltr {
 #define RTW89_H2C_BCNFLTR_W0_MON_BCN BIT(1)
 #define RTW89_H2C_BCNFLTR_W0_MON_EN BIT(2)
 #define RTW89_H2C_BCNFLTR_W0_MODE GENMASK(4, 3)
-#define RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT GENMASK(11, 8)
+#define RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT_H3 GENMASK(7, 5)
+#define RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT_L4 GENMASK(11, 8)
 #define RTW89_H2C_BCNFLTR_W0_RSSI_HYST GENMASK(15, 12)
 #define RTW89_H2C_BCNFLTR_W0_RSSI_THRESHOLD GENMASK(23, 16)
 #define RTW89_H2C_BCNFLTR_W0_MAC_ID GENMASK(31, 24)
@@ -3890,6 +3963,13 @@ enum rtw89_fw_element_id {
 	RTW89_FW_ELEMENT_ID_TX_SHAPE_LMT_RU = 17,
 	RTW89_FW_ELEMENT_ID_TXPWR_TRK = 18,
 	RTW89_FW_ELEMENT_ID_RFKLOG_FMT = 19,
+	RTW89_FW_ELEMENT_ID_REGD = 20,
+	RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_2GHZ = 21,
+	RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_5GHZ = 22,
+	RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_6GHZ = 23,
+	RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_2GHZ = 24,
+	RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_5GHZ = 25,
+	RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_6GHZ = 26,
 
 	RTW89_FW_ELEMENT_ID_NUM,
 };
@@ -3933,6 +4013,15 @@ struct __rtw89_fw_txpwr_element {
 	u8 content[];
 } __packed;
 
+struct __rtw89_fw_regd_element {
+	u8 rsvd0;
+	u8 rsvd1;
+	u8 rsvd2;
+	u8 ent_sz;
+	__le32 num_ents;
+	u8 content[];
+} __packed;
+
 enum rtw89_fw_txpwr_trk_type {
 	__RTW89_FW_TXPWR_TRK_TYPE_6GHZ_START = 0,
 	RTW89_FW_TXPWR_TRK_TYPE_6GB_N = 0,
@@ -4024,6 +4113,7 @@ struct rtw89_fw_element_hdr {
 			__le16 offset[];
 		} __packed rfk_log_fmt;
 		struct __rtw89_fw_txpwr_element txpwr;
+		struct __rtw89_fw_regd_element regd;
 	} __packed u;
 } __packed;
 
@@ -4218,6 +4308,26 @@ enum rtw89_mcc_h2c_func {
 #define RTW89_MCC_WAIT_COND(group, func) \
 	((group) * NUM_OF_RTW89_MCC_H2C_FUNC + (func))
 
+/* CLASS 20 - MLO */
+#define H2C_CL_MLO                     0x14
+enum rtw89_mlo_h2c_func {
+	H2C_FUNC_MLO_TBL_CFG		= 0x0,
+	H2C_FUNC_MLO_STA_CFG		= 0x1,
+	H2C_FUNC_MLO_TTLM		= 0x2,
+	H2C_FUNC_MLO_DM_CFG		= 0x3,
+	H2C_FUNC_MLO_EMLSR_STA_CFG	= 0x4,
+	H2C_FUNC_MLO_MCMLO_RELINK_DROP	= 0x5,
+	H2C_FUNC_MLO_MCMLO_SN_SYNC	= 0x6,
+	H2C_FUNC_MLO_RELINK		= 0x7,
+	H2C_FUNC_MLO_LINK_CFG		= 0x8,
+	H2C_FUNC_MLO_DM_DBG		= 0x9,
+
+	NUM_OF_RTW89_MLO_H2C_FUNC,
+};
+
+#define RTW89_MLO_WAIT_COND(macid, func) \
+	((macid) * NUM_OF_RTW89_MLO_H2C_FUNC + (func))
+
 /* CLASS 24 - MRC */
 #define H2C_CL_MRC			0x18
 enum rtw89_mrc_h2c_func {
@@ -4249,6 +4359,7 @@ enum rtw89_mrc_h2c_func {
 #define H2C_FUNC_OUTSRC_RA_MACIDCFG	0x0
 
 #define H2C_CL_OUTSRC_DM		0x2
+#define H2C_FUNC_FW_MCC_DIG		0x6
 #define H2C_FUNC_FW_LPS_CH_INFO		0xb
 #define H2C_FUNC_FW_LPS_ML_CMN_INFO	0xe
 
@@ -4256,6 +4367,7 @@ enum rtw89_mrc_h2c_func {
 #define H2C_CL_OUTSRC_RF_REG_B		0x9
 #define H2C_CL_OUTSRC_RF_FW_NOTIFY	0xa
 #define H2C_FUNC_OUTSRC_RF_GET_MCCCH	0x2
+#define H2C_FUNC_OUTSRC_RF_PS_INFO	0x10
 #define H2C_CL_OUTSRC_RF_FW_RFK		0xb
 
 enum rtw89_rfk_offload_h2c_func {
@@ -4269,6 +4381,14 @@ enum rtw89_rfk_offload_h2c_func {
 };
 
 struct rtw89_fw_h2c_rf_get_mccch {
+	__le32 ch_0_0;
+	__le32 ch_0_1;
+	__le32 ch_1_0;
+	__le32 ch_1_1;
+	__le32 current_channel;
+} __packed;
+
+struct rtw89_fw_h2c_rf_get_mccch_v0 {
 	__le32 ch_0;
 	__le32 ch_1;
 	__le32 band_0;
@@ -4277,9 +4397,36 @@ struct rtw89_fw_h2c_rf_get_mccch {
 	__le32 current_band_type;
 } __packed;
 
+struct rtw89_h2c_mcc_dig {
+	__le32 w0;
+	__le32 w1;
+	__le32 w2;
+} __packed;
+
+#define RTW89_H2C_MCC_DIG_W0_REG_CNT GENMASK(7, 0)
+#define RTW89_H2C_MCC_DIG_W0_DM_EN BIT(8)
+#define RTW89_H2C_MCC_DIG_W0_IDX GENMASK(10, 9)
+#define RTW89_H2C_MCC_DIG_W0_SET BIT(11)
+#define RTW89_H2C_MCC_DIG_W0_PHY0_EN BIT(12)
+#define RTW89_H2C_MCC_DIG_W0_PHY1_EN BIT(13)
+#define RTW89_H2C_MCC_DIG_W0_CENTER_CH GENMASK(23, 16)
+#define RTW89_H2C_MCC_DIG_W0_BAND_TYPE GENMASK(31, 24)
+#define RTW89_H2C_MCC_DIG_W1_ADDR_LSB GENMASK(7, 0)
+#define RTW89_H2C_MCC_DIG_W1_ADDR_MSB GENMASK(15, 8)
+#define RTW89_H2C_MCC_DIG_W1_BMASK_LSB GENMASK(23, 16)
+#define RTW89_H2C_MCC_DIG_W1_BMASK_MSB GENMASK(31, 24)
+#define RTW89_H2C_MCC_DIG_W2_VAL_LSB GENMASK(7, 0)
+#define RTW89_H2C_MCC_DIG_W2_VAL_MSB GENMASK(15, 8)
+
 #define NUM_OF_RTW89_FW_RFK_PATH 2
 #define NUM_OF_RTW89_FW_RFK_TBL 3
 
+struct rtw89_h2c_rf_ps_info {
+	__le32 rf18[NUM_OF_RTW89_FW_RFK_PATH];
+	__le32 mlo_mode;
+	u8 pri_ch[NUM_OF_RTW89_FW_RFK_PATH];
+} __packed;
+
 struct rtw89_fw_h2c_rfk_pre_info_common {
 	struct {
 		__le32 ch[NUM_OF_RTW89_FW_RFK_PATH][NUM_OF_RTW89_FW_RFK_TBL];
@@ -4354,13 +4501,25 @@ struct rtw89_h2c_rf_tssi {
 	u8 pg_thermal[2];
 	u8 ftable[2][128];
 	u8 tssi_mode;
+	u8 rfe_type;
 } __packed;
 
-struct rtw89_h2c_rf_iqk {
+struct rtw89_h2c_rf_iqk_v0 {
 	__le32 phy_idx;
 	__le32 dbcc;
 } __packed;
 
+struct rtw89_h2c_rf_iqk {
+	u8 len;
+	u8 ktype;
+	u8 phy;
+	u8 kpath;
+	u8 band;
+	u8 bw;
+	u8 ch;
+	u8 cv;
+} __packed;
+
 struct rtw89_h2c_rf_dpk {
 	u8 len;
 	u8 phy;
@@ -4532,6 +4691,12 @@ struct rtw89_c2h_rfk_report {
 	u8 version;
 } __packed;
 
+struct rtw89_c2h_rf_tas_info {
+	struct rtw89_c2h_hdr hdr;
+	__le32 cur_idx;
+	__le16 txpwr_history[20];
+} __packed;
+
 #define RTW89_FW_RSVD_PLE_SIZE 0x800
 
 #define RTW89_FW_BACKTRACE_INFO_SIZE 8
@@ -4542,6 +4707,7 @@ struct rtw89_c2h_rfk_report {
 #define RTW89_FW_BACKTRACE_KEY 0xBACEBACE
 
 #define FWDL_WAIT_CNT 400000
+#define FWDL_WAIT_CNT_USB 3200
 
 int rtw89_fw_check_rdy(struct rtw89_dev *rtwdev, enum rtw89_fwdl_check_type type);
 int rtw89_fw_recognize(struct rtw89_dev *rtwdev);
@@ -4581,6 +4747,11 @@ int rtw89_fw_h2c_ampdu_cmac_tbl_g7(struct rtw89_dev *rtwdev,
 				   struct rtw89_sta_link *rtwsta_link);
 int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev,
 				 struct rtw89_sta_link *rtwsta_link);
+int rtw89_fw_h2c_txtime_cmac_tbl_g7(struct rtw89_dev *rtwdev,
+				    struct rtw89_sta_link *rtwsta_link);
+int rtw89_fw_h2c_punctured_cmac_tbl_g7(struct rtw89_dev *rtwdev,
+				       struct rtw89_vif_link *rtwvif_link,
+				       u16 punctured);
 int rtw89_fw_h2c_txpath_cmac_tbl(struct rtw89_dev *rtwdev,
 				 struct rtw89_sta_link *rtwsta_link);
 int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev,
@@ -4596,7 +4767,8 @@ int rtw89_fw_h2c_dctl_sec_cam_v2(struct rtw89_dev *rtwdev,
 				 struct rtw89_vif_link *rtwvif_link,
 				 struct rtw89_sta_link *rtwsta_link);
 void rtw89_fw_c2h_irqsafe(struct rtw89_dev *rtwdev, struct sk_buff *c2h);
-void rtw89_fw_c2h_work(struct work_struct *work);
+void rtw89_fw_c2h_work(struct wiphy *wiphy, struct wiphy_work *work);
+void rtw89_fw_c2h_purge_obsoleted_scan_events(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev,
 			       struct rtw89_vif_link *rtwvif_link,
 			       struct rtw89_sta_link *rtwsta_link,
@@ -4624,6 +4796,7 @@ int rtw89_fw_h2c_cxdrv_role_v1(struct rtw89_dev *rtwdev, u8 type);
 int rtw89_fw_h2c_cxdrv_role_v2(struct rtw89_dev *rtwdev, u8 type);
 int rtw89_fw_h2c_cxdrv_role_v7(struct rtw89_dev *rtwdev, u8 type);
 int rtw89_fw_h2c_cxdrv_role_v8(struct rtw89_dev *rtwdev, u8 type);
+int rtw89_fw_h2c_cxdrv_osi_info(struct rtw89_dev *rtwdev, u8 type);
 int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev, u8 type);
 int rtw89_fw_h2c_cxdrv_ctrl_v7(struct rtw89_dev *rtwdev, u8 type);
 int rtw89_fw_h2c_cxdrv_trx(struct rtw89_dev *rtwdev, u8 type);
@@ -4643,8 +4816,12 @@ int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev,
 			struct rtw89_fw_h2c_rf_reg_info *info,
 			u16 len, u8 page);
 int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev);
+int rtw89_fw_h2c_rf_ps_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
 int rtw89_fw_h2c_rf_pre_ntfy(struct rtw89_dev *rtwdev,
 			     enum rtw89_phy_idx phy_idx);
+int rtw89_fw_h2c_mcc_dig(struct rtw89_dev *rtwdev,
+			 enum rtw89_chanctx_idx chanctx_idx,
+			 u8 mcc_role_idx, u8 pd_val, bool en);
 int rtw89_fw_h2c_rf_tssi(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx,
 			 const struct rtw89_chan *chan, enum rtw89_tssi_mode tssi_mode);
 int rtw89_fw_h2c_rf_iqk(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx,
@@ -4662,6 +4839,7 @@ int rtw89_fw_h2c_raw_with_hdr(struct rtw89_dev *rtwdev,
 			      bool rack, bool dack);
 int rtw89_fw_h2c_raw(struct rtw89_dev *rtwdev, const u8 *buf, u16 len);
 void rtw89_fw_send_all_early_h2c(struct rtw89_dev *rtwdev);
+void __rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev);
 void rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link,
 			     u8 macid);
@@ -4695,9 +4873,9 @@ int rtw89_fw_msg_reg(struct rtw89_dev *rtwdev,
 		     struct rtw89_mac_c2h_info *c2h_info);
 int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable);
 void rtw89_fw_st_dbg_dump(struct rtw89_dev *rtwdev);
-void rtw89_hw_scan_start(struct rtw89_dev *rtwdev,
-			 struct rtw89_vif_link *rtwvif_link,
-			 struct ieee80211_scan_request *scan_req);
+int rtw89_hw_scan_start(struct rtw89_dev *rtwdev,
+			struct rtw89_vif_link *rtwvif_link,
+			struct ieee80211_scan_request *scan_req);
 void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev,
 			    struct rtw89_vif_link *rtwvif_link,
 			    bool aborted);
@@ -4706,12 +4884,18 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev,
 			  bool enable);
 void rtw89_hw_scan_abort(struct rtw89_dev *rtwdev,
 			 struct rtw89_vif_link *rtwvif_link);
+int rtw89_hw_scan_prep_chan_list_ax(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif_link *rtwvif_link);
+void rtw89_hw_scan_free_chan_list_ax(struct rtw89_dev *rtwdev);
 int rtw89_hw_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
-				   struct rtw89_vif_link *rtwvif_link, bool connected);
+				   struct rtw89_vif_link *rtwvif_link);
 int rtw89_pno_scan_add_chan_list_ax(struct rtw89_dev *rtwdev,
 				    struct rtw89_vif_link *rtwvif_link);
+int rtw89_hw_scan_prep_chan_list_be(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif_link *rtwvif_link);
+void rtw89_hw_scan_free_chan_list_be(struct rtw89_dev *rtwdev);
 int rtw89_hw_scan_add_chan_list_be(struct rtw89_dev *rtwdev,
-				   struct rtw89_vif_link *rtwvif_link, bool connected);
+				   struct rtw89_vif_link *rtwvif_link);
 int rtw89_pno_scan_add_chan_list_be(struct rtw89_dev *rtwdev,
 				    struct rtw89_vif_link *rtwvif_link);
 int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev);
@@ -4719,9 +4903,8 @@ int rtw89_fw_h2c_pkt_drop(struct rtw89_dev *rtwdev,
 			  const struct rtw89_pkt_drop_params *params);
 int rtw89_fw_h2c_p2p_act(struct rtw89_dev *rtwdev,
 			 struct rtw89_vif_link *rtwvif_link,
-			 struct ieee80211_bss_conf *bss_conf,
 			 struct ieee80211_p2p_noa_desc *desc,
-			 u8 act, u8 noa_id);
+			 u8 act, u8 noa_id, u8 ctwindow_oppps);
 int rtw89_fw_h2c_tsf32_toggle(struct rtw89_dev *rtwdev,
 			      struct rtw89_vif_link *rtwvif_link,
 			      bool en);
@@ -4780,6 +4963,8 @@ int rtw89_fw_h2c_mrc_sync(struct rtw89_dev *rtwdev,
 int rtw89_fw_h2c_mrc_upd_duration(struct rtw89_dev *rtwdev,
 				  const struct rtw89_fw_mrc_upd_duration_arg *arg);
 int rtw89_fw_h2c_ap_info_refcount(struct rtw89_dev *rtwdev, bool en);
+int rtw89_fw_h2c_mlo_link_cfg(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link,
+			      bool enable);
 
 static inline void rtw89_fw_h2c_init_ba_cam(struct rtw89_dev *rtwdev)
 {
@@ -4862,6 +5047,28 @@ static inline int rtw89_chip_h2c_ampdu_cmac_tbl(struct rtw89_dev *rtwdev,
 }
 
 static inline
+int rtw89_chip_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev,
+				   struct rtw89_sta_link *rtwsta_link)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	return chip->ops->h2c_txtime_cmac_tbl(rtwdev, rtwsta_link);
+}
+
+static inline
+int rtw89_chip_h2c_punctured_cmac_tbl(struct rtw89_dev *rtwdev,
+				      struct rtw89_vif_link *rtwvif_link,
+				      u16 punctured)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	if (!chip->ops->h2c_punctured_cmac_tbl)
+		return 0;
+
+	return chip->ops->h2c_punctured_cmac_tbl(rtwdev, rtwvif_link, punctured);
+}
+
+static inline
 int rtw89_chip_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
 			  bool valid, struct ieee80211_ampdu_params *params)
 {
@@ -4882,6 +5089,18 @@ int rtw89_chip_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
 	return 0;
 }
 
+/* Must consider compatibility; don't insert new in the mid.
+ * Fill each field's default value in rtw89_regd_entcpy().
+ */
+struct rtw89_fw_regd_entry {
+	u8 alpha2_0;
+	u8 alpha2_1;
+	u8 rule_2ghz;
+	u8 rule_5ghz;
+	u8 rule_6ghz;
+	__le32 fmap;
+} __packed;
+
 /* must consider compatibility; don't insert new in the mid */
 struct rtw89_fw_txpwr_byrate_entry {
 	u8 band;
diff --git a/sys/contrib/dev/rtw89/mac.c b/sys/contrib/dev/rtw89/mac.c
index 0f025e785834..1d8eeb6e9997 100644
--- a/sys/contrib/dev/rtw89/mac.c
+++ b/sys/contrib/dev/rtw89/mac.c
@@ -88,7 +88,7 @@ int rtw89_mac_write_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 val)
 
 	ret = read_poll_timeout(rtw89_read8, lte_ctrl, (lte_ctrl & BIT(5)) != 0,
 				50, 50000, false, rtwdev, R_AX_LTE_CTRL + 3);
-	if (ret)
+	if (ret && !test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
 		rtw89_err(rtwdev, "[ERR]lte not ready(W)\n");
 
 	rtw89_write32(rtwdev, R_AX_LTE_WDATA, val);
@@ -104,7 +104,7 @@ int rtw89_mac_read_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 *val)
 
 	ret = read_poll_timeout(rtw89_read8, lte_ctrl, (lte_ctrl & BIT(5)) != 0,
 				50, 50000, false, rtwdev, R_AX_LTE_CTRL + 3);
-	if (ret)
+	if (ret && !test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
 		rtw89_err(rtwdev, "[ERR]lte not ready(W)\n");
 
 	rtw89_write32(rtwdev, R_AX_LTE_CTRL, 0x800F0000 | offset);
@@ -875,31 +875,30 @@ EXPORT_SYMBOL(rtw89_mac_set_err_status);
 
 static int hfc_reset_param(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_hfc_param_ini *param_ini, *param_inis;
 	struct rtw89_hfc_param *param = &rtwdev->mac.hfc_param;
-	struct rtw89_hfc_param_ini param_ini = {NULL};
 	u8 qta_mode = rtwdev->mac.dle_info.qta_mode;
 
-	switch (rtwdev->hci.type) {
-	case RTW89_HCI_TYPE_PCIE:
-		param_ini = rtwdev->chip->hfc_param_ini[qta_mode];
-		param->en = 0;
-		break;
-	default:
+	param_inis = rtwdev->chip->hfc_param_ini[rtwdev->hci.type];
+	if (!param_inis)
 		return -EINVAL;
-	}
 
-	if (param_ini.pub_cfg)
-		param->pub_cfg = *param_ini.pub_cfg;
+	param_ini = &param_inis[qta_mode];
+
+	param->en = 0;
+
+	if (param_ini->pub_cfg)
+		param->pub_cfg = *param_ini->pub_cfg;
 
-	if (param_ini.prec_cfg)
-		param->prec_cfg = *param_ini.prec_cfg;
+	if (param_ini->prec_cfg)
+		param->prec_cfg = *param_ini->prec_cfg;
 
-	if (param_ini.ch_cfg)
-		param->ch_cfg = param_ini.ch_cfg;
+	if (param_ini->ch_cfg)
+		param->ch_cfg = param_ini->ch_cfg;
 
 	memset(&param->ch_info, 0, sizeof(param->ch_info));
 	memset(&param->pub_info, 0, sizeof(param->pub_info));
-	param->mode = param_ini.mode;
+	param->mode = param_ini->mode;
 
 	return 0;
 }
@@ -1441,6 +1440,23 @@ void rtw89_mac_notify_wake(struct rtw89_dev *rtwdev)
 	rtw89_mac_send_rpwm(rtwdev, state, true);
 }
 
+static void rtw89_mac_power_switch_boot_mode(struct rtw89_dev *rtwdev)
+{
+	u32 boot_mode;
+
+	if (rtwdev->hci.type != RTW89_HCI_TYPE_USB)
+		return;
+
+	boot_mode = rtw89_read32_mask(rtwdev, R_AX_GPIO_MUXCFG, B_AX_BOOT_MODE);
+	if (!boot_mode)
+		return;
+
+	rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_APFN_ONMAC);
+	rtw89_write32_clr(rtwdev, R_AX_SYS_STATUS1, B_AX_AUTO_WLPON);
+	rtw89_write32_clr(rtwdev, R_AX_GPIO_MUXCFG, B_AX_BOOT_MODE);
+	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
+}
+
 static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on)
 {
 #define PWR_ACT 1
@@ -1451,6 +1467,8 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on)
 	int ret;
 	u8 val;
 
+	rtw89_mac_power_switch_boot_mode(rtwdev);
+
 	if (on) {
 		cfg_seq = chip->pwr_on_seq;
 		cfg_func = chip->ops->pwr_on_func;
@@ -1495,6 +1513,21 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on)
 #undef PWR_ACT
 }
 
+int rtw89_mac_pwr_on(struct rtw89_dev *rtwdev)
+{
+	int ret;
+
+	ret = rtw89_mac_power_switch(rtwdev, true);
+	if (ret) {
+		rtw89_mac_power_switch(rtwdev, false);
+		ret = rtw89_mac_power_switch(rtwdev, true);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 void rtw89_mac_pwr_off(struct rtw89_dev *rtwdev)
 {
 	rtw89_mac_power_switch(rtwdev, false);
@@ -1631,6 +1664,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	/* 8852C PCIE SCC */
 	.wde_size19 = {RTW89_WDE_PG_64, 3328, 0,},
 	.wde_size23 = {RTW89_WDE_PG_64, 1022, 2,},
+	/* 8852B USB2.0/USB3.0 SCC */
+	.wde_size25 = {RTW89_WDE_PG_64, 162, 94,},
 	/* PCIE */
 	.ple_size0 = {RTW89_PLE_PG_128, 1520, 16,},
 	.ple_size0_v1 = {RTW89_PLE_PG_128, 2688, 240, 212992,},
@@ -1646,6 +1681,10 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	.ple_size18 = {RTW89_PLE_PG_128, 2544, 16,},
 	/* 8852C PCIE SCC */
 	.ple_size19 = {RTW89_PLE_PG_128, 1904, 16,},
+	/* 8852B USB2.0 SCC */
+	.ple_size32 = {RTW89_PLE_PG_128, 620, 20,},
+	/* 8852B USB3.0 SCC */
+	.ple_size33 = {RTW89_PLE_PG_128, 632, 8,},
 	/* PCIE 64 */
 	.wde_qt0 = {3792, 196, 0, 107,},
 	.wde_qt0_v1 = {3302, 6, 0, 20,},
@@ -1660,6 +1699,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	/* 8852C PCIE SCC */
 	.wde_qt18 = {3228, 60, 0, 40,},
 	.wde_qt23 = {958, 48, 0, 16,},
+	/* 8852B USB2.0/USB3.0 SCC */
+	.wde_qt25 = {152, 2, 0, 8,},
 	.ple_qt0 = {320, 320, 32, 16, 13, 13, 292, 292, 64, 18, 1, 4, 0,},
 	.ple_qt1 = {320, 320, 32, 16, 1316, 1316, 1595, 1595, 1367, 1321, 1, 1307, 0,},
 	/* PCIE SCC */
@@ -1683,6 +1724,13 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	/* PCIE 64 */
 	.ple_qt58 = {147, 0, 16, 20, 157, 13, 229, 0, 172, 14, 24, 0,},
 	.ple_qt59 = {147, 0, 32, 20, 1860, 13, 2025, 0, 1879, 14, 24, 0,},
+	/* USB2.0 52B SCC */
+	.ple_qt72 = {130, 0, 16, 48, 4, 13, 322, 0, 32, 14, 8, 0, 0,},
+	/* USB2.0 52B 92K */
+	.ple_qt73 = {130, 0, 32, 48, 37, 13, 355, 0, 65, 14, 24, 0, 0,},
+	/* USB3.0 52B 92K */
+	.ple_qt74 = {286, 0, 16, 48, 4, 13, 178, 0, 32, 14, 8, 0, 0,},
+	.ple_qt75 = {286, 0, 32, 48, 37, 13, 211, 0, 65, 14, 24, 0, 0,},
 	/* 8852A PCIE WOW */
 	.ple_qt_52a_wow = {264, 0, 32, 20, 64, 13, 1005, 0, 64, 128, 120,},
 	/* 8852B PCIE WOW */
@@ -1702,12 +1750,13 @@ static const struct rtw89_dle_mem *get_dle_mem_cfg(struct rtw89_dev *rtwdev,
 						   enum rtw89_qta_mode mode)
 {
 	struct rtw89_mac_info *mac = &rtwdev->mac;
-	const struct rtw89_dle_mem *cfg;
+	const struct rtw89_dle_mem *cfg, *cfgs;
 
-	cfg = &rtwdev->chip->dle_mem[mode];
-	if (!cfg)
+	cfgs = rtwdev->chip->dle_mem[rtwdev->hci.dle_type];
+	if (!cfgs)
 		return NULL;
 
+	cfg = &cfgs[mode];
 	if (cfg->mode != mode) {
 		rtw89_warn(rtwdev, "qta mode unmatch!\n");
 		return NULL;
@@ -3996,14 +4045,6 @@ int rtw89_mac_partial_init(struct rtw89_dev *rtwdev, bool include_bb)
 {
 	int ret;
 
-	ret = rtw89_mac_power_switch(rtwdev, true);
-	if (ret) {
-		rtw89_mac_power_switch(rtwdev, false);
-		ret = rtw89_mac_power_switch(rtwdev, true);
-		if (ret)
-			return ret;
-	}
-
 	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
 
 	if (include_bb) {
@@ -4036,6 +4077,10 @@ int rtw89_mac_init(struct rtw89_dev *rtwdev)
 	bool include_bb = !!chip->bbmcu_nr;
 	int ret;
 
+	ret = rtw89_mac_pwr_on(rtwdev);
+	if (ret)
+		return ret;
+
 	ret = rtw89_mac_partial_init(rtwdev, include_bb);
 	if (ret)
 		goto fail;
@@ -4067,7 +4112,7 @@ int rtw89_mac_init(struct rtw89_dev *rtwdev)
 
 	return ret;
 fail:
-	rtw89_mac_power_switch(rtwdev, false);
+	rtw89_mac_pwr_off(rtwdev);
 
 	return ret;
 }
@@ -4377,7 +4422,33 @@ static void rtw89_mac_port_cfg_tx_sw_by_nettype(struct rtw89_dev *rtwdev,
 	rtw89_mac_port_cfg_tx_sw(rtwdev, rtwvif_link, en);
 }
 
-void rtw89_mac_enable_beacon_for_ap_vifs(struct rtw89_dev *rtwdev, bool en)
+static void rtw89_mac_enable_ap_bcn_by_chan(struct rtw89_dev *rtwdev,
+					    struct rtw89_vif_link *rtwvif_link,
+					    const struct rtw89_chan *to_match,
+					    bool en)
+{
+	const struct rtw89_chan *chan;
+
+	if (rtwvif_link->net_type != RTW89_NET_TYPE_AP_MODE)
+		return;
+
+	if (!to_match)
+		goto doit;
+
+	/* @to_match may not be in the same domain as return of calling
+	 * rtw89_chan_get(). So, cannot compare their addresses directly.
+	 */
+	chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx);
+	if (chan->channel != to_match->channel)
+		return;
+
+doit:
+	rtw89_mac_port_cfg_tx_sw(rtwdev, rtwvif_link, en);
+}
+
+static void rtw89_mac_enable_aps_bcn_by_chan(struct rtw89_dev *rtwdev,
+					     const struct rtw89_chan *to_match,
+					     bool en)
 {
 	struct rtw89_vif_link *rtwvif_link;
 	struct rtw89_vif *rtwvif;
@@ -4385,8 +4456,13 @@ void rtw89_mac_enable_beacon_for_ap_vifs(struct rtw89_dev *rtwdev, bool en)
 
 	rtw89_for_each_rtwvif(rtwdev, rtwvif)
 		rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id)
-			if (rtwvif_link->net_type == RTW89_NET_TYPE_AP_MODE)
-				rtw89_mac_port_cfg_tx_sw(rtwdev, rtwvif_link, en);
+			rtw89_mac_enable_ap_bcn_by_chan(rtwdev, rtwvif_link,
+							to_match, en);
+}
+
+void rtw89_mac_enable_beacon_for_ap_vifs(struct rtw89_dev *rtwdev, bool en)
+{
+	rtw89_mac_enable_aps_bcn_by_chan(rtwdev, NULL, en);
 }
 
 static void rtw89_mac_port_cfg_bcn_intv(struct rtw89_dev *rtwdev,
@@ -4620,11 +4696,17 @@ static void rtw89_mac_port_tsf_sync_rand(struct rtw89_dev *rtwdev,
 	if (rtwvif_link->net_type != RTW89_NET_TYPE_AP_MODE || rtwvif_link == rtwvif_src)
 		return;
 
+	if (rtwvif_link->rand_tsf_done)
+		goto out;
+
 	/* adjust offset randomly to avoid beacon conflict */
 	offset = offset - offset / 4 + get_random_u32() % (offset / 2);
 	rtw89_mac_port_tsf_sync(rtwdev, rtwvif_link, rtwvif_src,
 				(*n_offset) * offset);
 
+	rtwvif_link->rand_tsf_done = true;
+
+out:
 	(*n_offset)++;
 }
 
@@ -4826,9 +4908,37 @@ void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev,
 		rtw89_write32_set(rtwdev, reg, mac->narrow_bw_ru_dis.mask);
 }
 
+void rtw89_mac_set_he_tb(struct rtw89_dev *rtwdev,
+			 struct rtw89_vif_link *rtwvif_link)
+{
+	struct ieee80211_bss_conf *bss_conf;
+	bool set;
+	u32 reg;
+
+	if (rtwdev->chip->chip_gen != RTW89_CHIP_BE)
+		return;
+
+	rcu_read_lock();
+
+	bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
+	set = bss_conf->he_support && !bss_conf->eht_support;
+
+	rcu_read_unlock();
+
+	reg = rtw89_mac_reg_by_idx(rtwdev, R_BE_CLIENT_OM_CTRL,
+				   rtwvif_link->mac_idx);
+
+	if (set)
+		rtw89_write32_set(rtwdev, reg, B_BE_TRIG_DIS_EHTTB);
+	else
+		rtw89_write32_clr(rtwdev, reg, B_BE_TRIG_DIS_EHTTB);
+}
+
 void rtw89_mac_stop_ap(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link)
 {
 	rtw89_mac_port_cfg_func_sw(rtwdev, rtwvif_link);
+
+	rtwvif_link->rand_tsf_done = false;
 }
 
 int rtw89_mac_add_vif(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link)
@@ -4846,11 +4956,22 @@ rtw89_mac_c2h_macid_pause(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len
 {
 }
 
-static bool rtw89_is_op_chan(struct rtw89_dev *rtwdev, u8 band, u8 channel)
+static const struct rtw89_chan *
+rtw89_hw_scan_search_op_chan(struct rtw89_dev *rtwdev, u8 band, u8 channel)
 {
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	const struct rtw89_chan *op = &rtwdev->scan_info.op_chan;
 
-	return band == op->band_type && channel == op->primary_channel;
+	if (band == op->band_type && channel == op->primary_channel)
+		return op;
+
+	if (scan_info->extra_op.set) {
+		op = &scan_info->extra_op.chan;
+		if (band == op->band_type && channel == op->primary_channel)
+			return op;
+	}
+
+	return NULL;
 }
 
 static void
@@ -4860,13 +4981,14 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 	const struct rtw89_c2h_scanofld *c2h =
 		(const struct rtw89_c2h_scanofld *)skb->data;
 	struct rtw89_vif_link *rtwvif_link = rtwdev->scan_info.scanning_vif;
+	const struct rtw89_chan *op_chan;
 	struct rtw89_vif *rtwvif;
 	struct rtw89_chan new;
-	u32 last_chan = rtwdev->scan_info.last_chan_idx, report_tsf;
 	u16 actual_period, expect_period;
 	u8 reason, status, tx_fail, band;
 	u8 mac_idx, sw_def, fw_def;
 	u8 ver = U8_MAX;
+	u32 report_tsf;
 	u16 chan;
 	int ret;
 
@@ -4915,8 +5037,9 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 	switch (reason) {
 	case RTW89_SCAN_LEAVE_OP_NOTIFY:
 	case RTW89_SCAN_LEAVE_CH_NOTIFY:
-		if (rtw89_is_op_chan(rtwdev, band, chan)) {
-			rtw89_mac_enable_beacon_for_ap_vifs(rtwdev, false);
+		op_chan = rtw89_hw_scan_search_op_chan(rtwdev, band, chan);
+		if (op_chan) {
+			rtw89_mac_enable_aps_bcn_by_chan(rtwdev, op_chan, false);
 			ieee80211_stop_queues(rtwdev->hw);
 		}
 		return;
@@ -4925,7 +5048,8 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 			return;
 
 		if (rtwvif_link && rtwvif->scan_req &&
-		    last_chan < rtwvif->scan_req->n_channels) {
+		    !list_empty(&rtwdev->scan_info.chan_list)) {
+			rtwdev->scan_info.delay = 0;
 			ret = rtw89_hw_scan_offload(rtwdev, rtwvif_link, true);
 			if (ret) {
 				rtw89_hw_scan_abort(rtwdev, rtwvif_link);
@@ -4937,10 +5061,10 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 		break;
 	case RTW89_SCAN_ENTER_OP_NOTIFY:
 	case RTW89_SCAN_ENTER_CH_NOTIFY:
-		if (rtw89_is_op_chan(rtwdev, band, chan)) {
-			rtw89_assign_entity_chan(rtwdev, rtwvif_link->chanctx_idx,
-						 &rtwdev->scan_info.op_chan);
-			rtw89_mac_enable_beacon_for_ap_vifs(rtwdev, true);
+		op_chan = rtw89_hw_scan_search_op_chan(rtwdev, band, chan);
+		if (op_chan) {
+			rtw89_assign_entity_chan(rtwdev, rtwvif_link->chanctx_idx, op_chan);
+			rtw89_mac_enable_aps_bcn_by_chan(rtwdev, op_chan, true);
 			ieee80211_wake_queues(rtwdev->hw);
 		} else {
 			rtw89_chan_create(&new, chan, chan, band,
@@ -4964,6 +5088,7 @@ rtw89_mac_bcn_fltr_rpt(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_l
 	const struct rtw89_c2h_mac_bcnfltr_rpt *c2h =
 		(const struct rtw89_c2h_mac_bcnfltr_rpt *)skb->data;
 	u8 type, event, mac_id;
+	bool start_detect;
 	s8 sig;
 
 	type = le32_get_bits(c2h->w2, RTW89_C2H_MAC_BCNFLTR_RPT_W2_TYPE);
@@ -4980,10 +5105,16 @@ rtw89_mac_bcn_fltr_rpt(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_l
 
 	switch (type) {
 	case RTW89_BCN_FLTR_BEACON_LOSS:
-		if (!rtwdev->scanning && !rtwvif->offchan)
+		if (!rtwdev->scanning && !rtwvif->offchan &&
+		    !rtwvif_link->noa_once.in_duration) {
+			start_detect = rtw89_mcc_detect_go_bcn(rtwdev, rtwvif_link);
+			if (start_detect)
+				return;
+
 			ieee80211_connection_loss(vif);
-		else
+		} else {
 			rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, rtwvif_link, true);
+		}
 		return;
 	case RTW89_BCN_FLTR_NOTIFY:
 		nl_event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH;
@@ -5034,6 +5165,7 @@ rtw89_mac_c2h_done_ack(struct rtw89_dev *rtwdev, struct sk_buff *skb_c2h, u32 le
 {
 	/* N.B. This will run in interrupt context. */
 	struct rtw89_wait_info *fw_ofld_wait = &rtwdev->mac.fw_ofld_wait;
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct rtw89_wait_info *ps_wait = &rtwdev->mac.ps_wait;
 	const struct rtw89_c2h_done_ack *c2h =
 		(const struct rtw89_c2h_done_ack *)skb_c2h->data;
@@ -5073,12 +5205,16 @@ rtw89_mac_c2h_done_ack(struct rtw89_dev *rtwdev, struct sk_buff *skb_c2h, u32 le
 			return;
 		case H2C_FUNC_ADD_SCANOFLD_CH:
 			cond = RTW89_SCANOFLD_WAIT_COND_ADD_CH;
+			h2c_return &= RTW89_C2H_SCAN_DONE_ACK_RETURN;
 			break;
 		case H2C_FUNC_SCANOFLD:
+			scan_info->seq++;
 			cond = RTW89_SCANOFLD_WAIT_COND_START;
 			break;
 		case H2C_FUNC_SCANOFLD_BE:
+			scan_info->seq++;
 			cond = RTW89_SCANOFLD_BE_WAIT_COND_START;
+			h2c_return &= RTW89_C2H_SCAN_DONE_ACK_RETURN;
 			break;
 		}
 
@@ -5378,6 +5514,27 @@ rtw89_mac_c2h_wow_aoac_rpt(struct rtw89_dev *rtwdev, struct sk_buff *skb, u32 le
 }
 
 static void
+rtw89_mac_c2h_mlo_link_cfg_stat(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len)
+{
+	const struct rtw89_c2h_mlo_link_cfg_rpt *c2h_rpt;
+	struct rtw89_wait_info *wait = &rtwdev->mlo.wait;
+	struct rtw89_completion_data data = {};
+	unsigned int cond;
+	u16 mac_id;
+	u8 status;
+
+	c2h_rpt = (const struct rtw89_c2h_mlo_link_cfg_rpt *)c2h->data;
+
+	mac_id = le32_get_bits(c2h_rpt->w2, RTW89_C2H_MLO_LINK_CFG_RPT_W2_MACID);
+	status = le32_get_bits(c2h_rpt->w2, RTW89_C2H_MLO_LINK_CFG_RPT_W2_STATUS);
+
+	data.err = status == RTW89_C2H_MLO_LINK_CFG_ROLE_NOT_EXIST ||
+		   status == RTW89_C2H_MLO_LINK_CFG_RUNNING;
+	cond = RTW89_MLO_WAIT_COND(mac_id, H2C_FUNC_MLO_LINK_CFG);
+	rtw89_complete_cond(wait, cond, &data);
+}
+
+static void
 rtw89_mac_c2h_mrc_status_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len)
 {
 	struct rtw89_wait_info *wait = &rtwdev->mcc.wait;
@@ -5537,6 +5694,18 @@ void (* const rtw89_mac_c2h_mcc_handler[])(struct rtw89_dev *rtwdev,
 };
 
 static
+void (* const rtw89_mac_c2h_mlo_handler[])(struct rtw89_dev *rtwdev,
+					   struct sk_buff *c2h, u32 len) = {
+	[RTW89_MAC_C2H_FUNC_MLO_GET_TBL] = NULL,
+	[RTW89_MAC_C2H_FUNC_MLO_EMLSR_TRANS_DONE] = NULL,
+	[RTW89_MAC_C2H_FUNC_MLO_EMLSR_STA_CFG_DONE] = NULL,
+	[RTW89_MAC_C2H_FUNC_MCMLO_RELINK_RPT] = NULL,
+	[RTW89_MAC_C2H_FUNC_MCMLO_SN_SYNC_RPT] = NULL,
+	[RTW89_MAC_C2H_FUNC_MLO_LINK_CFG_STAT] = rtw89_mac_c2h_mlo_link_cfg_stat,
+	[RTW89_MAC_C2H_FUNC_MLO_DM_DBG_DUMP] = NULL,
+};
+
+static
 void (* const rtw89_mac_c2h_mrc_handler[])(struct rtw89_dev *rtwdev,
 					   struct sk_buff *c2h, u32 len) = {
 	[RTW89_MAC_C2H_FUNC_MRC_TSF_RPT] = rtw89_mac_c2h_mrc_tsf_rpt,
@@ -5561,10 +5730,15 @@ static void rtw89_mac_c2h_scanofld_rsp_atomic(struct rtw89_dev *rtwdev,
 	const struct rtw89_c2h_scanofld *c2h =
 		(const struct rtw89_c2h_scanofld *)skb->data;
 	struct rtw89_wait_info *fw_ofld_wait = &rtwdev->mac.fw_ofld_wait;
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_fw_c2h_attr *attr = RTW89_SKB_C2H_CB(skb);
 	struct rtw89_completion_data data = {};
 	unsigned int cond;
 	u8 status, reason;
 
+	attr->is_scan_event = 1;
+	attr->scan_seq = scan_info->seq;
+
 	status = le32_get_bits(c2h->w2, RTW89_C2H_SCANOFLD_W2_STATUS);
 	reason = le32_get_bits(c2h->w2, RTW89_C2H_SCANOFLD_W2_RSN);
 	data.err = status != RTW89_SCAN_STATUS_SUCCESS;
@@ -5605,6 +5779,8 @@ bool rtw89_mac_c2h_chk_atomic(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
 		}
 	case RTW89_MAC_C2H_CLASS_MCC:
 		return true;
+	case RTW89_MAC_C2H_CLASS_MLO:
+		return true;
 	case RTW89_MAC_C2H_CLASS_MRC:
 		return true;
 	case RTW89_MAC_C2H_CLASS_WOW:
@@ -5638,6 +5814,10 @@ void rtw89_mac_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 		if (func < NUM_OF_RTW89_MAC_C2H_FUNC_MCC)
 			handler = rtw89_mac_c2h_mcc_handler[func];
 		break;
+	case RTW89_MAC_C2H_CLASS_MLO:
+		if (func < NUM_OF_RTW89_MAC_C2H_FUNC_MLO)
+			handler = rtw89_mac_c2h_mlo_handler[func];
+		break;
 	case RTW89_MAC_C2H_CLASS_MRC:
 		if (func < NUM_OF_RTW89_MAC_C2H_FUNC_MRC)
 			handler = rtw89_mac_c2h_mrc_handler[func];
@@ -5651,6 +5831,7 @@ void rtw89_mac_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 			handler = rtw89_mac_c2h_ap_handler[func];
 		break;
 	case RTW89_MAC_C2H_CLASS_FWDBG:
+	case RTW89_MAC_C2H_CLASS_ROLE:
 		return;
 	default:
 		rtw89_info(rtwdev, "MAC c2h class %d not support\n", class);
@@ -5713,7 +5894,7 @@ int rtw89_mac_cfg_ppdu_status_ax(struct rtw89_dev *rtwdev, u8 mac_idx, bool enab
 
 	rtw89_write32(rtwdev, reg, B_AX_PPDU_STAT_RPT_EN |
 				   B_AX_APP_MAC_INFO_RPT |
-				   B_AX_APP_RX_CNT_RPT | B_AX_APP_PLCP_HDR_RPT |
+				   B_AX_APP_PLCP_HDR_RPT |
 				   B_AX_PPDU_STAT_RPT_CRC32);
 	rtw89_write32_mask(rtwdev, R_AX_HW_RPT_FWD, B_AX_FWD_PPDU_STAT_MASK,
 			   RTW89_PRPT_DEST_HOST);
@@ -5796,13 +5977,15 @@ int rtw89_mac_coex_init(struct rtw89_dev *rtwdev, const struct rtw89_mac_ax_coex
 
 	ret = rtw89_mac_read_lte(rtwdev, R_AX_LTE_SW_CFG_2, &val32);
 	if (ret) {
-		rtw89_err(rtwdev, "Read R_AX_LTE_SW_CFG_2 fail!\n");
+		if (!test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
+			rtw89_err(rtwdev, "Read R_AX_LTE_SW_CFG_2 fail!\n");
 		return ret;
 	}
 	val32 = val32 & B_AX_WL_RX_CTRL;
 	ret = rtw89_mac_write_lte(rtwdev, R_AX_LTE_SW_CFG_2, val32);
 	if (ret) {
-		rtw89_err(rtwdev, "Write R_AX_LTE_SW_CFG_2 fail!\n");
+		if (!test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
+			rtw89_err(rtwdev, "Write R_AX_LTE_SW_CFG_2 fail!\n");
 		return ret;
 	}
 
@@ -5926,7 +6109,8 @@ int rtw89_mac_cfg_gnt(struct rtw89_dev *rtwdev,
 
 	ret = rtw89_mac_write_lte(rtwdev, R_AX_LTE_SW_CFG_1, val);
 	if (ret) {
-		rtw89_err(rtwdev, "Write LTE fail!\n");
+		if (!test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
+			rtw89_err(rtwdev, "Write LTE fail!\n");
 		return ret;
 	}
 
@@ -6052,7 +6236,7 @@ int rtw89_mac_cfg_ctrl_path_v1(struct rtw89_dev *rtwdev, bool wl)
 	if (wl)
 		return 0;
 
-	for (i = 0; i < RTW89_PHY_MAX; i++) {
+	for (i = 0; i < RTW89_PHY_NUM; i++) {
 		g[i].gnt_bt_sw_en = 1;
 		g[i].gnt_bt = 1;
 		g[i].gnt_wl_sw_en = 1;
@@ -6443,6 +6627,7 @@ __rtw89_mac_set_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta_link *rtwsta_
 			u32 tx_time)
 {
 #define MAC_AX_DFLT_TX_TIME 5280
+	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
 	u8 mac_idx = rtwsta_link->rtwvif_link->mac_idx;
 	u32 max_tx_time = tx_time == 0 ? MAC_AX_DFLT_TX_TIME : tx_time;
 	u32 reg;
@@ -6450,7 +6635,7 @@ __rtw89_mac_set_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta_link *rtwsta_
 
 	if (rtwsta_link->cctl_tx_time) {
 		rtwsta_link->ampdu_max_time = (max_tx_time - 512) >> 9;
-		ret = rtw89_fw_h2c_txtime_cmac_tbl(rtwdev, rtwsta_link);
+		ret = rtw89_chip_h2c_txtime_cmac_tbl(rtwdev, rtwsta_link);
 	} else {
 		ret = rtw89_mac_check_mac_en(rtwdev, mac_idx, RTW89_CMAC_SEL);
 		if (ret) {
@@ -6458,8 +6643,8 @@ __rtw89_mac_set_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta_link *rtwsta_
 			return ret;
 		}
 
-		reg = rtw89_mac_reg_by_idx(rtwdev, R_AX_AMPDU_AGG_LIMIT, mac_idx);
-		rtw89_write32_mask(rtwdev, reg, B_AX_AMPDU_MAX_TIME_MASK,
+		reg = rtw89_mac_reg_by_idx(rtwdev, mac->agg_limit.addr, mac_idx);
+		rtw89_write32_mask(rtwdev, reg, mac->agg_limit.mask,
 				   max_tx_time >> 5);
 	}
 
@@ -6485,6 +6670,7 @@ int rtw89_mac_set_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta_link *rtwst
 int rtw89_mac_get_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta_link *rtwsta_link,
 			  u32 *tx_time)
 {
+	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
 	u8 mac_idx = rtwsta_link->rtwvif_link->mac_idx;
 	u32 reg;
 	int ret = 0;
@@ -6498,8 +6684,8 @@ int rtw89_mac_get_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta_link *rtwst
 			return ret;
 		}
 
-		reg = rtw89_mac_reg_by_idx(rtwdev, R_AX_AMPDU_AGG_LIMIT, mac_idx);
-		*tx_time = rtw89_read32_mask(rtwdev, reg, B_AX_AMPDU_MAX_TIME_MASK) << 5;
+		reg = rtw89_mac_reg_by_idx(rtwdev, mac->agg_limit.addr, mac_idx);
+		*tx_time = rtw89_read32_mask(rtwdev, reg, mac->agg_limit.mask) << 5;
 	}
 
 	return ret;
@@ -6515,9 +6701,9 @@ int rtw89_mac_set_tx_retry_limit(struct rtw89_dev *rtwdev,
 
 	if (!resume) {
 		rtwsta_link->cctl_tx_retry_limit = true;
-		ret = rtw89_fw_h2c_txtime_cmac_tbl(rtwdev, rtwsta_link);
+		ret = rtw89_chip_h2c_txtime_cmac_tbl(rtwdev, rtwsta_link);
 	} else {
-		ret = rtw89_fw_h2c_txtime_cmac_tbl(rtwdev, rtwsta_link);
+		ret = rtw89_chip_h2c_txtime_cmac_tbl(rtwdev, rtwsta_link);
 		rtwsta_link->cctl_tx_retry_limit = false;
 	}
 
@@ -6527,6 +6713,7 @@ int rtw89_mac_set_tx_retry_limit(struct rtw89_dev *rtwdev,
 int rtw89_mac_get_tx_retry_limit(struct rtw89_dev *rtwdev,
 				 struct rtw89_sta_link *rtwsta_link, u8 *tx_retry)
 {
+	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
 	u8 mac_idx = rtwsta_link->rtwvif_link->mac_idx;
 	u32 reg;
 	int ret = 0;
@@ -6540,8 +6727,8 @@ int rtw89_mac_get_tx_retry_limit(struct rtw89_dev *rtwdev,
 			return ret;
 		}
 
-		reg = rtw89_mac_reg_by_idx(rtwdev, R_AX_TXCNT, mac_idx);
-		*tx_retry = rtw89_read32_mask(rtwdev, reg, B_AX_L_TXCNT_LMT_MASK);
+		reg = rtw89_mac_reg_by_idx(rtwdev, mac->txcnt_limit.addr, mac_idx);
+		*tx_retry = rtw89_read32_mask(rtwdev, reg, mac->txcnt_limit.mask);
 	}
 
 	return ret;
@@ -6774,10 +6961,16 @@ int rtw89_fwdl_check_path_ready_ax(struct rtw89_dev *rtwdev,
 				   bool h2c_or_fwdl)
 {
 	u8 check = h2c_or_fwdl ? B_AX_H2C_PATH_RDY : B_AX_FWDL_PATH_RDY;
+	u32 timeout;
 	u8 val;
 
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_USB)
+		timeout = FWDL_WAIT_CNT_USB;
+	else
+		timeout = FWDL_WAIT_CNT;
+
 	return read_poll_timeout_atomic(rtw89_read8, val, val & check,
-					1, FWDL_WAIT_CNT, false,
+					1, timeout, false,
 					rtwdev, R_AX_WCPU_FW_CTRL);
 }
 
@@ -6800,6 +6993,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = {
 	.filter_model_addr = R_AX_FILTER_MODEL_ADDR,
 	.indir_access_addr = R_AX_INDIR_ACCESS_ENTRY,
 	.mem_base_addrs = rtw89_mac_mem_base_addrs_ax,
+	.mem_page_size = MAC_MEM_DUMP_PAGE_SIZE_AX,
 	.rx_fltr = R_AX_RX_FLTR_OPT,
 	.port_base = &rtw89_port_base_ax,
 	.agg_len_ht = R_AX_AGG_LEN_HT_0,
@@ -6819,6 +7013,8 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = {
 		.mask = B_AX_RXTRIG_RU26_DIS,
 	},
 	.wow_ctrl = {.addr = R_AX_WOW_CTRL, .mask = B_AX_WOW_WOWEN,},
+	.agg_limit = {.addr = R_AX_AMPDU_AGG_LIMIT, .mask = B_AX_AMPDU_MAX_TIME_MASK,},
+	.txcnt_limit = {.addr = R_AX_TXCNT, .mask = B_AX_L_TXCNT_LMT_MASK,},
 
 	.check_mac_en = rtw89_mac_check_mac_en_ax,
 	.sys_init = sys_init_ax,
@@ -6868,6 +7064,8 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = {
 
 	.is_txq_empty = mac_is_txq_empty_ax,
 
+	.prep_chan_list = rtw89_hw_scan_prep_chan_list_ax,
+	.free_chan_list = rtw89_hw_scan_free_chan_list_ax,
 	.add_chan_list = rtw89_hw_scan_add_chan_list_ax,
 	.add_chan_list_pno = rtw89_pno_scan_add_chan_list_ax,
 	.scan_offload = rtw89_fw_h2c_scan_offload_ax,
diff --git a/sys/contrib/dev/rtw89/mac.h b/sys/contrib/dev/rtw89/mac.h
index 8edea96d037f..241e89983c4a 100644
--- a/sys/contrib/dev/rtw89/mac.h
+++ b/sys/contrib/dev/rtw89/mac.h
@@ -6,9 +6,12 @@
 #define __RTW89_MAC_H__
 
 #include "core.h"
+#include "fw.h"
 #include "reg.h"
 
-#define MAC_MEM_DUMP_PAGE_SIZE 0x40000
+#define MAC_MEM_DUMP_PAGE_SIZE_AX 0x40000
+#define MAC_MEM_DUMP_PAGE_SIZE_BE 0x80000
+
 #define ADDR_CAM_ENT_SIZE  0x40
 #define ADDR_CAM_ENT_SHORT_SIZE 0x20
 #define BSSID_CAM_ENT_SIZE 0x08
@@ -370,6 +373,7 @@ enum rtw89_mac_mem_sel {
 	RTW89_MAC_MEM_TXD_FIFO_0_V1,
 	RTW89_MAC_MEM_TXD_FIFO_1_V1,
 	RTW89_MAC_MEM_WD_PAGE,
+	RTW89_MAC_MEM_MLD_TBL,
 
 	/* keep last */
 	RTW89_MAC_MEM_NUM,
@@ -427,6 +431,18 @@ enum rtw89_mac_c2h_mcc_func {
 	NUM_OF_RTW89_MAC_C2H_FUNC_MCC,
 };
 
+enum rtw89_mac_c2h_mlo_func {
+	RTW89_MAC_C2H_FUNC_MLO_GET_TBL			= 0x0,
+	RTW89_MAC_C2H_FUNC_MLO_EMLSR_TRANS_DONE		= 0x1,
+	RTW89_MAC_C2H_FUNC_MLO_EMLSR_STA_CFG_DONE	= 0x2,
+	RTW89_MAC_C2H_FUNC_MCMLO_RELINK_RPT		= 0x3,
+	RTW89_MAC_C2H_FUNC_MCMLO_SN_SYNC_RPT		= 0x4,
+	RTW89_MAC_C2H_FUNC_MLO_LINK_CFG_STAT		= 0x5,
+	RTW89_MAC_C2H_FUNC_MLO_DM_DBG_DUMP		= 0x6,
+
+	NUM_OF_RTW89_MAC_C2H_FUNC_MLO,
+};
+
 enum rtw89_mac_c2h_mrc_func {
 	RTW89_MAC_C2H_FUNC_MRC_TSF_RPT = 0,
 	RTW89_MAC_C2H_FUNC_MRC_STATUS_RPT = 1,
@@ -453,8 +469,10 @@ enum rtw89_mac_c2h_class {
 	RTW89_MAC_C2H_CLASS_WOW = 0x3,
 	RTW89_MAC_C2H_CLASS_MCC = 0x4,
 	RTW89_MAC_C2H_CLASS_FWDBG = 0x5,
+	RTW89_MAC_C2H_CLASS_MLO = 0xc,
 	RTW89_MAC_C2H_CLASS_MRC = 0xe,
 	RTW89_MAC_C2H_CLASS_AP = 0x18,
+	RTW89_MAC_C2H_CLASS_ROLE = 0x1b,
 	RTW89_MAC_C2H_CLASS_MAX,
 };
 
@@ -907,6 +925,7 @@ struct rtw89_mac_size_set {
 	const struct rtw89_dle_size wde_size18;
 	const struct rtw89_dle_size wde_size19;
 	const struct rtw89_dle_size wde_size23;
+	const struct rtw89_dle_size wde_size25;
 	const struct rtw89_dle_size ple_size0;
 	const struct rtw89_dle_size ple_size0_v1;
 	const struct rtw89_dle_size ple_size3_v1;
@@ -916,6 +935,8 @@ struct rtw89_mac_size_set {
 	const struct rtw89_dle_size ple_size9;
 	const struct rtw89_dle_size ple_size18;
 	const struct rtw89_dle_size ple_size19;
+	const struct rtw89_dle_size ple_size32;
+	const struct rtw89_dle_size ple_size33;
 	const struct rtw89_wde_quota wde_qt0;
 	const struct rtw89_wde_quota wde_qt0_v1;
 	const struct rtw89_wde_quota wde_qt4;
@@ -924,6 +945,7 @@ struct rtw89_mac_size_set {
 	const struct rtw89_wde_quota wde_qt17;
 	const struct rtw89_wde_quota wde_qt18;
 	const struct rtw89_wde_quota wde_qt23;
+	const struct rtw89_wde_quota wde_qt25;
 	const struct rtw89_ple_quota ple_qt0;
 	const struct rtw89_ple_quota ple_qt1;
 	const struct rtw89_ple_quota ple_qt4;
@@ -938,6 +960,10 @@ struct rtw89_mac_size_set {
 	const struct rtw89_ple_quota ple_qt57;
 	const struct rtw89_ple_quota ple_qt58;
 	const struct rtw89_ple_quota ple_qt59;
+	const struct rtw89_ple_quota ple_qt72;
+	const struct rtw89_ple_quota ple_qt73;
+	const struct rtw89_ple_quota ple_qt74;
+	const struct rtw89_ple_quota ple_qt75;
 	const struct rtw89_ple_quota ple_qt_52a_wow;
 	const struct rtw89_ple_quota ple_qt_52b_wow;
 	const struct rtw89_ple_quota ple_qt_52bt_wow;
@@ -955,6 +981,7 @@ struct rtw89_mac_gen_def {
 	u32 filter_model_addr;
 	u32 indir_access_addr;
 	const u32 *mem_base_addrs;
+	u32 mem_page_size;
 	u32 rx_fltr;
 	const struct rtw89_port_reg *port_base;
 	u32 agg_len_ht;
@@ -964,6 +991,8 @@ struct rtw89_mac_gen_def {
 	struct rtw89_reg_def bfee_ctrl;
 	struct rtw89_reg_def narrow_bw_ru_dis;
 	struct rtw89_reg_def wow_ctrl;
+	struct rtw89_reg_def agg_limit;
+	struct rtw89_reg_def txcnt_limit;
 
 	int (*check_mac_en)(struct rtw89_dev *rtwdev, u8 band,
 			    enum rtw89_mac_hwmod_sel sel);
@@ -1029,8 +1058,11 @@ struct rtw89_mac_gen_def {
 
 	bool (*is_txq_empty)(struct rtw89_dev *rtwdev);
 
+	int (*prep_chan_list)(struct rtw89_dev *rtwdev,
+			      struct rtw89_vif_link *rtwvif_link);
+	void (*free_chan_list)(struct rtw89_dev *rtwdev);
 	int (*add_chan_list)(struct rtw89_dev *rtwdev,
-			     struct rtw89_vif_link *rtwvif_link, bool connected);
+			     struct rtw89_vif_link *rtwvif_link);
 	int (*add_chan_list_pno)(struct rtw89_dev *rtwdev,
 				 struct rtw89_vif_link *rtwvif_link);
 	int (*scan_offload)(struct rtw89_dev *rtwdev,
@@ -1145,6 +1177,7 @@ rtw89_write32_port_set(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_l
 	rtw89_write32_set(rtwdev, reg, bit);
 }
 
+int rtw89_mac_pwr_on(struct rtw89_dev *rtwdev);
 void rtw89_mac_pwr_off(struct rtw89_dev *rtwdev);
 int rtw89_mac_partial_init(struct rtw89_dev *rtwdev, bool include_bb);
 int rtw89_mac_init(struct rtw89_dev *rtwdev);
@@ -1185,6 +1218,8 @@ void rtw89_mac_port_cfg_rx_sync(struct rtw89_dev *rtwdev,
 				struct rtw89_vif_link *rtwvif_link, bool en);
 void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev,
 					struct rtw89_vif_link *rtwvif_link);
+void rtw89_mac_set_he_tb(struct rtw89_dev *rtwdev,
+			 struct rtw89_vif_link *rtwvif_link);
 void rtw89_mac_stop_ap(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link);
 void rtw89_mac_enable_beacon_for_ap_vifs(struct rtw89_dev *rtwdev, bool en);
 int rtw89_mac_remove_vif(struct rtw89_dev *rtwdev, struct rtw89_vif_link *vif);
@@ -1549,4 +1584,28 @@ void rtw89_fwdl_secure_idmem_share_mode(struct rtw89_dev *rtwdev, u8 mode)
 
 	return mac->fwdl_secure_idmem_share_mode(rtwdev, mode);
 }
+
+static inline
+int rtw89_mac_scan_offload(struct rtw89_dev *rtwdev,
+			   struct rtw89_scan_option *option,
+			   struct rtw89_vif_link *rtwvif_link,
+			   bool wowlan)
+{
+	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
+	int ret;
+
+	ret = mac->scan_offload(rtwdev, option, rtwvif_link, wowlan);
+
+	if (option->enable) {
+		/*
+		 * At this point, new scan request is acknowledged by firmware,
+		 * so scan events of previous scan request become obsoleted.
+		 * Purge the queued scan events to prevent interference to
+		 * current new request.
+		 */
+		rtw89_fw_c2h_purge_obsoleted_scan_events(rtwdev);
+	}
+
+	return ret;
+}
 #endif
diff --git a/sys/contrib/dev/rtw89/mac80211.c b/sys/contrib/dev/rtw89/mac80211.c
index 3eb074c591cf..c1ca6d741b32 100644
--- a/sys/contrib/dev/rtw89/mac80211.c
+++ b/sys/contrib/dev/rtw89/mac80211.c
@@ -16,10 +16,6 @@
 #include "util.h"
 #include "wow.h"
 
-#if defined(__FreeBSD__)
-DEFINE_GUARD(mutex, struct mutex *, mutex_lock(_T), mutex_unlock(_T))
-#endif
-
 static void rtw89_ops_tx(struct ieee80211_hw *hw,
 			 struct ieee80211_tx_control *control,
 			 struct sk_buff *skb)
@@ -61,32 +57,30 @@ static void rtw89_ops_wake_tx_queue(struct ieee80211_hw *hw,
 static int rtw89_ops_start(struct ieee80211_hw *hw)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
-	int ret;
 
-	mutex_lock(&rtwdev->mutex);
-	ret = rtw89_core_start(rtwdev);
-	mutex_unlock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
-	return ret;
+	return rtw89_core_start(rtwdev);
 }
 
 static void rtw89_ops_stop(struct ieee80211_hw *hw, bool suspend)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_core_stop(rtwdev);
-	mutex_unlock(&rtwdev->mutex);
 }
 
-static int rtw89_ops_config(struct ieee80211_hw *hw, u32 changed)
+static int rtw89_ops_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	/* let previous ips work finish to ensure we don't leave ips twice */
-	cancel_work_sync(&rtwdev->ips_work);
+	wiphy_work_cancel(hw->wiphy, &rtwdev->ips_work);
 
-	mutex_lock(&rtwdev->mutex);
 	rtw89_leave_ps_mode(rtwdev);
 
 	if ((changed & IEEE80211_CONF_CHANGE_IDLE) &&
@@ -104,8 +98,6 @@ static int rtw89_ops_config(struct ieee80211_hw *hw, u32 changed)
 	    !rtwdev->scanning)
 		rtw89_enter_ips(rtwdev);
 
-	mutex_unlock(&rtwdev->mutex);
-
 	return 0;
 }
 
@@ -119,14 +111,22 @@ static int __rtw89_ops_add_iface_link(struct rtw89_dev *rtwdev,
 
 	rtw89_vif_type_mapping(rtwvif_link, false);
 
-	INIT_WORK(&rtwvif_link->update_beacon_work, rtw89_core_update_beacon_work);
+	wiphy_work_init(&rtwvif_link->update_beacon_work, rtw89_core_update_beacon_work);
+	wiphy_delayed_work_init(&rtwvif_link->csa_beacon_work, rtw89_core_csa_beacon_work);
+	wiphy_delayed_work_init(&rtwvif_link->mcc_gc_detect_beacon_work,
+				rtw89_mcc_gc_detect_beacon_work);
+
 	INIT_LIST_HEAD(&rtwvif_link->general_pkt_list);
 
+	rtw89_p2p_noa_once_init(rtwvif_link);
+
 	rtwvif_link->hit_rule = 0;
 	rtwvif_link->bcn_hit_cond = 0;
 	rtwvif_link->chanctx_assigned = false;
 	rtwvif_link->chanctx_idx = RTW89_CHANCTX_0;
 	rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT;
+	rtwvif_link->rand_tsf_done = false;
+	rtwvif_link->detect_bcn_count = 0;
 
 	rcu_read_lock();
 
@@ -146,9 +146,14 @@ static int __rtw89_ops_add_iface_link(struct rtw89_dev *rtwdev,
 static void __rtw89_ops_remove_iface_link(struct rtw89_dev *rtwdev,
 					  struct rtw89_vif_link *rtwvif_link)
 {
-	mutex_unlock(&rtwdev->mutex);
-	cancel_work_sync(&rtwvif_link->update_beacon_work);
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	wiphy_work_cancel(rtwdev->hw->wiphy, &rtwvif_link->update_beacon_work);
+	wiphy_delayed_work_cancel(rtwdev->hw->wiphy, &rtwvif_link->csa_beacon_work);
+	wiphy_delayed_work_cancel(rtwdev->hw->wiphy,
+				  &rtwvif_link->mcc_gc_detect_beacon_work);
+
+	rtw89_p2p_noa_once_deinit(rtwvif_link);
 
 	rtw89_leave_ps_mode(rtwdev);
 
@@ -166,11 +171,11 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw,
 	u8 mac_id, port;
 	int ret = 0;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_debug(rtwdev, RTW89_DBG_STATE, "add vif %pM type %d, p2p %d\n",
 		    vif->addr, vif->type, vif->p2p);
 
-	mutex_lock(&rtwdev->mutex);
-
 	rtw89_leave_ips_by_hwflags(rtwdev);
 
 	if (RTW89_CHK_FW_FEATURE(BEACON_FILTER, &rtwdev->fw))
@@ -178,10 +183,8 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw,
 				     IEEE80211_VIF_SUPPORTS_CQM_RSSI;
 
 	mac_id = rtw89_acquire_mac_id(rtwdev);
-	if (mac_id == RTW89_MAX_MAC_ID_NUM) {
-		ret = -ENOSPC;
-		goto err;
-	}
+	if (mac_id == RTW89_MAX_MAC_ID_NUM)
+		return -ENOSPC;
 
 	port = rtw89_core_acquire_bit_map(rtwdev->hw_port, RTW89_PORT_NUM);
 	if (port == RTW89_PORT_NUM) {
@@ -196,13 +199,14 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw,
 	if (!rtw89_rtwvif_in_list(rtwdev, rtwvif)) {
 		list_add_tail(&rtwvif->list, &rtwdev->rtwvifs_list);
 		INIT_LIST_HEAD(&rtwvif->mgnt_entry);
+		INIT_LIST_HEAD(&rtwvif->dlink_pool);
 	}
 
 	ether_addr_copy(rtwvif->mac_addr, vif->addr);
 
 	rtwvif->offchan = false;
 	rtwvif->roc.state = RTW89_ROC_IDLE;
-	INIT_DELAYED_WORK(&rtwvif->roc.roc_work, rtw89_roc_work);
+	wiphy_delayed_work_init(&rtwvif->roc.roc_work, rtw89_roc_work);
 
 	rtw89_traffic_stats_init(rtwdev, &rtwvif->stats);
 
@@ -217,8 +221,6 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw,
 		goto unset_link;
 
 	rtw89_recalc_lps(rtwdev);
-
-	mutex_unlock(&rtwdev->mutex);
 	return 0;
 
 unset_link:
@@ -228,8 +230,6 @@ release_port:
 	rtw89_core_release_bit_map(rtwdev->hw_port, port);
 release_macid:
 	rtw89_release_mac_id(rtwdev, mac_id);
-err:
-	mutex_unlock(&rtwdev->mutex);
 
 	return ret;
 }
@@ -243,12 +243,12 @@ static void rtw89_ops_remove_interface(struct ieee80211_hw *hw,
 	u8 port = rtw89_vif_get_main_port(rtwvif);
 	struct rtw89_vif_link *rtwvif_link;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_debug(rtwdev, RTW89_DBG_STATE, "remove vif %pM type %d p2p %d\n",
 		    vif->addr, vif->type, vif->p2p);
 
-	cancel_delayed_work_sync(&rtwvif->roc.roc_work);
-
-	mutex_lock(&rtwdev->mutex);
+	wiphy_delayed_work_cancel(hw->wiphy, &rtwvif->roc.roc_work);
 
 	rtwvif_link = rtwvif->links[RTW89_VIF_IDLE_LINK_ID];
 	if (unlikely(!rtwvif_link)) {
@@ -269,8 +269,6 @@ bottom:
 
 	rtw89_recalc_lps(rtwdev);
 	rtw89_enter_ips_by_hwflags(rtwdev);
-
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static int rtw89_ops_change_interface(struct ieee80211_hw *hw,
@@ -308,7 +306,8 @@ static void rtw89_ops_configure_filter(struct ieee80211_hw *hw,
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
 	u32 rx_fltr;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 
 	*new_flags &= FIF_ALLMULTI | FIF_OTHER_BSS | FIF_FCSFAIL |
@@ -371,14 +370,11 @@ static void rtw89_ops_configure_filter(struct ieee80211_hw *hw,
 			   B_AX_RX_FLTR_CFG_MASK,
 			   rx_fltr);
 	if (!rtwdev->dbcc_en)
-		goto out;
+		return;
 	rtw89_write32_mask(rtwdev,
 			   rtw89_mac_reg_by_idx(rtwdev, mac->rx_fltr, RTW89_MAC_1),
 			   B_AX_RX_FLTR_CFG_MASK,
 			   rx_fltr);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static const u8 ac_to_fw_idx[IEEE80211_NUM_ACS] = {
@@ -497,6 +493,8 @@ static int __rtw89_ops_sta_add(struct rtw89_dev *rtwdev,
 	int i;
 
 	if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) {
+		rtwvif->mlo_mode = RTW89_MLO_MODE_MLSR;
+
 		/* for station mode, assign the mac_id from itself */
 		macid = rtw89_vif_get_main_macid(rtwvif);
 	} else {
@@ -512,6 +510,8 @@ static int __rtw89_ops_sta_add(struct rtw89_dev *rtwdev,
 	for (i = 0; i < ARRAY_SIZE(sta->txq); i++)
 		rtw89_core_txq_init(rtwdev, sta->txq[i]);
 
+	INIT_LIST_HEAD(&rtwsta->dlink_pool);
+
 	skb_queue_head_init(&rtwsta->roc_queue);
 	bitmap_zero(rtwsta->pairwise_sec_cam_map, RTW89_MAX_SEC_CAM_NUM);
 
@@ -674,6 +674,7 @@ static void __rtw89_ops_bss_link_assoc(struct rtw89_dev *rtwdev,
 	rtw89_chip_cfg_txpwr_ul_tb_offset(rtwdev, rtwvif_link);
 	rtw89_mac_port_update(rtwdev, rtwvif_link);
 	rtw89_mac_set_he_obss_narrow_bw_ru(rtwdev, rtwvif_link);
+	rtw89_mac_set_he_tb(rtwdev, rtwvif_link);
 }
 
 static void __rtw89_ops_bss_assoc(struct rtw89_dev *rtwdev,
@@ -693,7 +694,8 @@ static void rtw89_ops_vif_cfg_changed(struct ieee80211_hw *hw,
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 
 	if (changed & BSS_CHANGED_ASSOC) {
@@ -716,8 +718,6 @@ static void rtw89_ops_vif_cfg_changed(struct ieee80211_hw *hw,
 
 	if (changed & BSS_CHANGED_ARP_FILTER)
 		rtwvif->ip_addr = vif->cfg.arp_addr_list[0];
-
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static void rtw89_ops_link_info_changed(struct ieee80211_hw *hw,
@@ -729,7 +729,8 @@ static void rtw89_ops_link_info_changed(struct ieee80211_hw *hw,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 
 	rtwvif_link = rtwvif->links[conf->link_id];
@@ -737,7 +738,7 @@ static void rtw89_ops_link_info_changed(struct ieee80211_hw *hw,
 		rtw89_err(rtwdev,
 			  "%s: rtwvif link (link_id %u) is not active\n",
 			  __func__, conf->link_id);
-		goto out;
+		return;
 	}
 
 	if (changed & BSS_CHANGED_BSSID) {
@@ -767,9 +768,6 @@ static void rtw89_ops_link_info_changed(struct ieee80211_hw *hw,
 
 	if (changed & BSS_CHANGED_TPE)
 		rtw89_reg_6ghz_recalc(rtwdev, rtwvif_link, true);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static int rtw89_ops_start_ap(struct ieee80211_hw *hw,
@@ -782,22 +780,19 @@ static int rtw89_ops_start_ap(struct ieee80211_hw *hw,
 	const struct rtw89_chan *chan;
 	int ret = 0;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	rtwvif_link = rtwvif->links[link_conf->link_id];
 	if (unlikely(!rtwvif_link)) {
 		rtw89_err(rtwdev,
 			  "%s: rtwvif link (link_id %u) is not active\n",
 			  __func__, link_conf->link_id);
-		ret = -ENOLINK;
-		goto out;
+		return -ENOLINK;
 	}
 
 	chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx);
-	if (chan->band_type == RTW89_BAND_6G) {
-		mutex_unlock(&rtwdev->mutex);
+	if (chan->band_type == RTW89_BAND_6G)
 		return -EOPNOTSUPP;
-	}
 
 	if (rtwdev->scanning)
 		rtw89_hw_scan_abort(rtwdev, rtwdev->scan_info.scanning_vif);
@@ -814,15 +809,12 @@ static int rtw89_ops_start_ap(struct ieee80211_hw *hw,
 	if (RTW89_CHK_FW_FEATURE(NOTIFY_AP_INFO, &rtwdev->fw)) {
 		ret = rtw89_fw_h2c_ap_info_refcount(rtwdev, true);
 		if (ret)
-			goto out;
+			return ret;
 	}
 
 	rtw89_queue_chanctx_work(rtwdev);
 
-out:
-	mutex_unlock(&rtwdev->mutex);
-
-	return ret;
+	return 0;
 }
 
 static
@@ -833,14 +825,14 @@ void rtw89_ops_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	rtwvif_link = rtwvif->links[link_conf->link_id];
 	if (unlikely(!rtwvif_link)) {
 		rtw89_err(rtwdev,
 			  "%s: rtwvif link (link_id %u) is not active\n",
 			  __func__, link_conf->link_id);
-		goto out;
+		return;
 	}
 
 	if (RTW89_CHK_FW_FEATURE(NOTIFY_AP_INFO, &rtwdev->fw))
@@ -849,22 +841,18 @@ void rtw89_ops_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	rtw89_mac_stop_ap(rtwdev, rtwvif_link);
 	rtw89_chip_h2c_assoc_cmac_tbl(rtwdev, rtwvif_link, NULL);
 	rtw89_fw_h2c_join_info(rtwdev, rtwvif_link, NULL, true);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static int rtw89_ops_set_tim(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
 			     bool set)
 {
-	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_sta *rtwsta = sta_to_rtwsta(sta);
 	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
 	struct rtw89_vif_link *rtwvif_link;
 	unsigned int link_id;
 
 	rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id)
-		ieee80211_queue_work(rtwdev->hw, &rtwvif_link->update_beacon_work);
+		wiphy_work_queue(hw->wiphy, &rtwvif_link->update_beacon_work);
 
 	return 0;
 }
@@ -877,9 +865,9 @@ static int rtw89_ops_conf_tx(struct ieee80211_hw *hw,
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
-	int ret = 0;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 
 	rtwvif_link = rtwvif->links[link_id];
@@ -887,17 +875,13 @@ static int rtw89_ops_conf_tx(struct ieee80211_hw *hw,
 		rtw89_err(rtwdev,
 			  "%s: rtwvif link (link_id %u) is not active\n",
 			  __func__, link_id);
-		ret = -ENOLINK;
-		goto out;
+		return -ENOLINK;
 	}
 
 	rtwvif_link->tx_params[ac] = *params;
 	__rtw89_conf_tx(rtwdev, rtwvif_link, ac);
 
-out:
-	mutex_unlock(&rtwdev->mutex);
-
-	return ret;
+	return 0;
 }
 
 static int __rtw89_ops_sta_state(struct ieee80211_hw *hw,
@@ -941,14 +925,11 @@ static int rtw89_ops_sta_state(struct ieee80211_hw *hw,
 			       enum ieee80211_sta_state new_state)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
-	int ret;
 
-	mutex_lock(&rtwdev->mutex);
-	rtw89_leave_ps_mode(rtwdev);
-	ret = __rtw89_ops_sta_state(hw, vif, sta, old_state, new_state);
-	mutex_unlock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
-	return ret;
+	rtw89_leave_ps_mode(rtwdev);
+	return __rtw89_ops_sta_state(hw, vif, sta, old_state, new_state);
 }
 
 static int rtw89_ops_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
@@ -957,9 +938,10 @@ static int rtw89_ops_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 			     struct ieee80211_key_conf *key)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
-	int ret = 0;
+	int ret;
+
+	lockdep_assert_wiphy(hw->wiphy);
 
-	mutex_lock(&rtwdev->mutex);
 	rtw89_leave_ps_mode(rtwdev);
 
 	switch (cmd) {
@@ -968,7 +950,7 @@ static int rtw89_ops_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		ret = rtw89_cam_sec_key_add(rtwdev, vif, sta, key);
 		if (ret && ret != -EOPNOTSUPP) {
 			rtw89_err(rtwdev, "failed to add key to sec cam\n");
-			goto out;
+			return ret;
 		}
 		break;
 	case DISABLE_KEY:
@@ -978,14 +960,11 @@ static int rtw89_ops_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		ret = rtw89_cam_sec_key_del(rtwdev, vif, sta, key, true);
 		if (ret) {
 			rtw89_err(rtwdev, "failed to remove key from sec cam\n");
-			goto out;
+			return ret;
 		}
 		break;
 	}
 
-out:
-	mutex_unlock(&rtwdev->mutex);
-
 	return ret;
 }
 
@@ -1001,38 +980,32 @@ static int rtw89_ops_ampdu_action(struct ieee80211_hw *hw,
 	struct ieee80211_txq *txq = sta->txq[tid];
 	struct rtw89_txq *rtwtxq = (struct rtw89_txq *)txq->drv_priv;
 
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
 	switch (params->action) {
 	case IEEE80211_AMPDU_TX_START:
 		return IEEE80211_AMPDU_TX_START_IMMEDIATE;
 	case IEEE80211_AMPDU_TX_STOP_CONT:
 	case IEEE80211_AMPDU_TX_STOP_FLUSH:
 	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
-		mutex_lock(&rtwdev->mutex);
 		clear_bit(RTW89_TXQ_F_AMPDU, &rtwtxq->flags);
 		clear_bit(tid, rtwsta->ampdu_map);
 		rtw89_chip_h2c_ampdu_cmac_tbl(rtwdev, rtwvif, rtwsta);
-		mutex_unlock(&rtwdev->mutex);
 		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		break;
 	case IEEE80211_AMPDU_TX_OPERATIONAL:
-		mutex_lock(&rtwdev->mutex);
 		set_bit(RTW89_TXQ_F_AMPDU, &rtwtxq->flags);
 		rtwsta->ampdu_params[tid].agg_num = params->buf_size;
 		rtwsta->ampdu_params[tid].amsdu = params->amsdu;
 		set_bit(tid, rtwsta->ampdu_map);
 		rtw89_leave_ps_mode(rtwdev);
 		rtw89_chip_h2c_ampdu_cmac_tbl(rtwdev, rtwvif, rtwsta);
-		mutex_unlock(&rtwdev->mutex);
 		break;
 	case IEEE80211_AMPDU_RX_START:
-		mutex_lock(&rtwdev->mutex);
 		rtw89_chip_h2c_ba_cam(rtwdev, rtwsta, true, params);
-		mutex_unlock(&rtwdev->mutex);
 		break;
 	case IEEE80211_AMPDU_RX_STOP:
-		mutex_lock(&rtwdev->mutex);
 		rtw89_chip_h2c_ba_cam(rtwdev, rtwsta, false, params);
-		mutex_unlock(&rtwdev->mutex);
 		break;
 	default:
 		WARN_ON(1);
@@ -1042,15 +1015,16 @@ static int rtw89_ops_ampdu_action(struct ieee80211_hw *hw,
 	return 0;
 }
 
-static int rtw89_ops_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
+static int rtw89_ops_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
+				       u32 value)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_leave_ps_mode(rtwdev);
 	if (test_bit(RTW89_FLAG_POWERON, rtwdev->flags))
 		rtw89_mac_update_rts_threshold(rtwdev);
-	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
@@ -1063,7 +1037,7 @@ static void rtw89_ops_sta_statistics(struct ieee80211_hw *hw,
 	struct rtw89_sta *rtwsta = sta_to_rtwsta(sta);
 	struct rtw89_sta_link *rtwsta_link;
 
-	rtwsta_link = rtw89_sta_get_link_inst(rtwsta, 0);
+	rtwsta_link = rtw89_get_designated_link(rtwsta);
 	if (unlikely(!rtwsta_link))
 		return;
 
@@ -1090,7 +1064,8 @@ static void rtw89_ops_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_leave_lps(rtwdev);
 	rtw89_hci_flush_queues(rtwdev, queues, drop);
 
@@ -1098,8 +1073,6 @@ static void rtw89_ops_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		__rtw89_drop_packets(rtwdev, vif);
 	else
 		rtw89_mac_flush_txq(rtwdev, queues, drop);
-
-	mutex_unlock(&rtwdev->mutex);
 }
 
 struct rtw89_iter_bitrate_mask_data {
@@ -1146,20 +1119,22 @@ static int rtw89_ops_set_bitrate_mask(struct ieee80211_hw *hw,
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_phy_rate_pattern_vif(rtwdev, vif, mask);
 	rtw89_ra_mask_info_update(rtwdev, vif, mask);
-	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
 
 static
-int rtw89_ops_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
+int rtw89_ops_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_ant)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_hal *hal = &rtwdev->hal;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	if (hal->ant_diversity) {
 		if (tx_ant != rx_ant || hweight32(tx_ant) != 1)
 			return -EINVAL;
@@ -1167,18 +1142,17 @@ int rtw89_ops_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
 		return -EINVAL;
 	}
 
-	mutex_lock(&rtwdev->mutex);
 	hal->antenna_tx = tx_ant;
 	hal->antenna_rx = rx_ant;
 	hal->tx_path_diversity = false;
 	hal->ant_diversity_fixed = true;
-	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
 
 static
-int rtw89_ops_get_antenna(struct ieee80211_hw *hw,  u32 *tx_ant, u32 *rx_ant)
+int rtw89_ops_get_antenna(struct ieee80211_hw *hw, int radio_idx, u32 *tx_ant,
+			  u32 *rx_ant)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_hal *hal = &rtwdev->hal;
@@ -1197,18 +1171,17 @@ static void rtw89_ops_sw_scan_start(struct ieee80211_hw *hw,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
+	rtwvif_link = rtw89_get_designated_link(rtwvif);
 	if (unlikely(!rtwvif_link)) {
-		rtw89_err(rtwdev, "sw scan start: find no link on HW-0\n");
-		goto out;
+		rtw89_err(rtwdev, "sw scan start: find no designated link\n");
+		return;
 	}
 
-	rtw89_core_scan_start(rtwdev, rtwvif_link, mac_addr, false);
+	rtw89_leave_lps(rtwdev);
 
-out:
-	mutex_unlock(&rtwdev->mutex);
+	rtw89_core_scan_start(rtwdev, rtwvif_link, mac_addr, false);
 }
 
 static void rtw89_ops_sw_scan_complete(struct ieee80211_hw *hw,
@@ -1218,18 +1191,15 @@ static void rtw89_ops_sw_scan_complete(struct ieee80211_hw *hw,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
+	rtwvif_link = rtw89_get_designated_link(rtwvif);
 	if (unlikely(!rtwvif_link)) {
-		rtw89_err(rtwdev, "sw scan complete: find no link on HW-0\n");
-		goto out;
+		rtw89_err(rtwdev, "sw scan complete: find no designated link\n");
+		return;
 	}
 
 	rtw89_core_scan_complete(rtwdev, rtwvif_link, false);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static void rtw89_ops_reconfig_complete(struct ieee80211_hw *hw,
@@ -1249,33 +1219,33 @@ static int rtw89_ops_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	struct rtw89_vif_link *rtwvif_link;
 	int ret;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	if (!RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw))
 		return 1;
 
-	mutex_lock(&rtwdev->mutex);
-
-	if (rtwdev->scanning || rtwvif->offchan) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if (rtwdev->scanning || rtwvif->offchan)
+		return -EBUSY;
 
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
+	rtwvif_link = rtw89_get_designated_link(rtwvif);
 	if (unlikely(!rtwvif_link)) {
-		rtw89_err(rtwdev, "hw scan: find no link on HW-0\n");
-		ret = -ENOLINK;
-		goto out;
+		rtw89_err(rtwdev, "hw scan: find no designated link\n");
+		return -ENOLINK;
 	}
 
-	rtw89_hw_scan_start(rtwdev, rtwvif_link, req);
+	rtw89_leave_lps(rtwdev);
+	rtw89_leave_ips_by_hwflags(rtwdev);
+
+	ret = rtw89_hw_scan_start(rtwdev, rtwvif_link, req);
+	if (ret)
+		return ret;
+
 	ret = rtw89_hw_scan_offload(rtwdev, rtwvif_link, true);
 	if (ret) {
 		rtw89_hw_scan_abort(rtwdev, rtwvif_link);
 		rtw89_err(rtwdev, "HW scan failed with status: %d\n", ret);
 	}
 
-out:
-	mutex_unlock(&rtwdev->mutex);
-
 	return ret;
 }
 
@@ -1286,24 +1256,21 @@ static void rtw89_ops_cancel_hw_scan(struct ieee80211_hw *hw,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	if (!RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw))
 		return;
 
-	mutex_lock(&rtwdev->mutex);
-
 	if (!rtwdev->scanning)
-		goto out;
+		return;
 
-	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
+	rtwvif_link = rtw89_get_designated_link(rtwvif);
 	if (unlikely(!rtwvif_link)) {
-		rtw89_err(rtwdev, "cancel hw scan: find no link on HW-0\n");
-		goto out;
+		rtw89_err(rtwdev, "cancel hw scan: find no designated link\n");
+		return;
 	}
 
 	rtw89_hw_scan_abort(rtwdev, rtwvif_link);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static void rtw89_ops_sta_rc_update(struct ieee80211_hw *hw,
@@ -1326,13 +1293,10 @@ static int rtw89_ops_add_chanctx(struct ieee80211_hw *hw,
 				 struct ieee80211_chanctx_conf *ctx)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
-	int ret;
 
-	mutex_lock(&rtwdev->mutex);
-	ret = rtw89_chanctx_ops_add(rtwdev, ctx);
-	mutex_unlock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
-	return ret;
+	return rtw89_chanctx_ops_add(rtwdev, ctx);
 }
 
 static void rtw89_ops_remove_chanctx(struct ieee80211_hw *hw,
@@ -1340,9 +1304,9 @@ static void rtw89_ops_remove_chanctx(struct ieee80211_hw *hw,
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_chanctx_ops_remove(rtwdev, ctx);
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static void rtw89_ops_change_chanctx(struct ieee80211_hw *hw,
@@ -1351,9 +1315,9 @@ static void rtw89_ops_change_chanctx(struct ieee80211_hw *hw,
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	rtw89_chanctx_ops_change(rtwdev, ctx, changed);
-	mutex_unlock(&rtwdev->mutex);
 }
 
 static int rtw89_ops_assign_vif_chanctx(struct ieee80211_hw *hw,
@@ -1364,25 +1328,18 @@ static int rtw89_ops_assign_vif_chanctx(struct ieee80211_hw *hw,
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
-	int ret;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	rtwvif_link = rtwvif->links[link_conf->link_id];
 	if (unlikely(!rtwvif_link)) {
 		rtw89_err(rtwdev,
 			  "%s: rtwvif link (link_id %u) is not active\n",
 			  __func__, link_conf->link_id);
-		ret = -ENOLINK;
-		goto out;
+		return -ENOLINK;
 	}
 
-	ret = rtw89_chanctx_ops_assign_vif(rtwdev, rtwvif_link, ctx);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
-
-	return ret;
+	return rtw89_chanctx_ops_assign_vif(rtwdev, rtwvif_link, ctx);
 }
 
 static void rtw89_ops_unassign_vif_chanctx(struct ieee80211_hw *hw,
@@ -1394,11 +1351,10 @@ static void rtw89_ops_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_vif_link *rtwvif_link;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	rtwvif_link = rtwvif->links[link_conf->link_id];
 	if (unlikely(!rtwvif_link)) {
-		mutex_unlock(&rtwdev->mutex);
 		rtw89_err(rtwdev,
 			  "%s: rtwvif link (link_id %u) is not active\n",
 			  __func__, link_conf->link_id);
@@ -1406,7 +1362,73 @@ static void rtw89_ops_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	}
 
 	rtw89_chanctx_ops_unassign_vif(rtwdev, rtwvif_link, ctx);
-	mutex_unlock(&rtwdev->mutex);
+}
+
+static
+int rtw89_ops_switch_vif_chanctx(struct ieee80211_hw *hw,
+				 struct ieee80211_vif_chanctx_switch *vifs,
+				 int n_vifs,
+				 enum ieee80211_chanctx_switch_mode mode)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+	bool replace;
+	int ret;
+	int i;
+
+	lockdep_assert_wiphy(hw->wiphy);
+
+	switch (mode) {
+	case CHANCTX_SWMODE_REASSIGN_VIF:
+		replace = false;
+		break;
+	case CHANCTX_SWMODE_SWAP_CONTEXTS:
+		replace = true;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	for (i = 0; i < n_vifs; i++) {
+		struct ieee80211_vif_chanctx_switch *p = &vifs[i];
+		struct ieee80211_bss_conf *link_conf = p->link_conf;
+		struct rtw89_vif *rtwvif = vif_to_rtwvif(p->vif);
+		struct rtw89_vif_link *rtwvif_link;
+
+		rtwvif_link = rtwvif->links[link_conf->link_id];
+		if (unlikely(!rtwvif_link)) {
+			rtw89_err(rtwdev,
+				  "%s: rtwvif link (link_id %u) is not active\n",
+				  __func__, link_conf->link_id);
+			return -ENOLINK;
+		}
+
+		ret = rtw89_chanctx_ops_reassign_vif(rtwdev, rtwvif_link,
+						     p->old_ctx, p->new_ctx,
+						     replace);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void rtw89_ops_channel_switch_beacon(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif,
+					    struct cfg80211_chan_def *chandef)
+{
+	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
+	struct rtw89_dev *rtwdev = hw->priv;
+	struct rtw89_vif_link *rtwvif_link;
+
+	BUILD_BUG_ON(RTW89_MLD_NON_STA_LINK_NUM != 1);
+
+	rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
+	if (unlikely(!rtwvif_link)) {
+		rtw89_err(rtwdev, "chsw bcn: find no link on HW-0\n");
+		return;
+	}
+
+	wiphy_delayed_work_queue(hw->wiphy, &rtwvif_link->csa_beacon_work, 0);
 }
 
 static int rtw89_ops_remain_on_channel(struct ieee80211_hw *hw,
@@ -1419,13 +1441,12 @@ static int rtw89_ops_remain_on_channel(struct ieee80211_hw *hw,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
 	struct rtw89_roc *roc = &rtwvif->roc;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	if (!rtwvif)
 		return -EINVAL;
 
-	mutex_lock(&rtwdev->mutex);
-
 	if (roc->state != RTW89_ROC_IDLE) {
-		mutex_unlock(&rtwdev->mutex);
 		return -EBUSY;
 	}
 
@@ -1443,8 +1464,6 @@ static int rtw89_ops_remain_on_channel(struct ieee80211_hw *hw,
 
 	rtw89_roc_start(rtwdev, rtwvif);
 
-	mutex_unlock(&rtwdev->mutex);
-
 	return 0;
 }
 
@@ -1454,14 +1473,14 @@ static int rtw89_ops_cancel_remain_on_channel(struct ieee80211_hw *hw,
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	if (!rtwvif)
 		return -EINVAL;
 
-	cancel_delayed_work_sync(&rtwvif->roc.roc_work);
+	wiphy_delayed_work_cancel(hw->wiphy, &rtwvif->roc.roc_work);
 
-	mutex_lock(&rtwdev->mutex);
 	rtw89_roc_end(rtwdev, rtwvif);
-	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
@@ -1482,14 +1501,14 @@ static int rtw89_ops_set_tid_config(struct ieee80211_hw *hw,
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	if (sta)
 		rtw89_core_set_tid_config(rtwdev, sta, tid_config);
 	else
 		ieee80211_iterate_stations_atomic(rtwdev->hw,
 						  rtw89_set_tid_config_iter,
 						  tid_config);
-	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
@@ -1513,7 +1532,7 @@ static bool rtw89_ops_can_activate_links(struct ieee80211_hw *hw,
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	guard(mutex)(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	return rtw89_can_work_on_links(rtwdev, vif, active_links);
 }
@@ -1575,7 +1594,7 @@ int rtw89_ops_change_vif_links(struct ieee80211_hw *hw,
 	int ret = 0;
 	int i;
 
-	guard(mutex)(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	rtw89_debug(rtwdev, RTW89_DBG_STATE,
 		    "%s: old_links (0x%08x) -> new_links (0x%08x)\n",
@@ -1724,7 +1743,7 @@ int rtw89_ops_change_sta_links(struct ieee80211_hw *hw,
 	unsigned long set_links = new_links & ~old_links;
 	int ret = 0;
 
-	guard(mutex)(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	rtw89_debug(rtwdev, RTW89_DBG_STATE,
 		    "%s: old_links (0x%08x) -> new_links (0x%08x)\n",
@@ -1754,16 +1773,16 @@ static int rtw89_ops_suspend(struct ieee80211_hw *hw,
 	struct rtw89_dev *rtwdev = hw->priv;
 	int ret;
 
-	set_bit(RTW89_FLAG_FORBIDDEN_TRACK_WROK, rtwdev->flags);
-	cancel_delayed_work_sync(&rtwdev->track_work);
+	lockdep_assert_wiphy(hw->wiphy);
 
-	mutex_lock(&rtwdev->mutex);
-	ret = rtw89_wow_suspend(rtwdev, wowlan);
-	mutex_unlock(&rtwdev->mutex);
+	set_bit(RTW89_FLAG_FORBIDDEN_TRACK_WORK, rtwdev->flags);
+	wiphy_delayed_work_cancel(hw->wiphy, &rtwdev->track_work);
+	wiphy_delayed_work_cancel(hw->wiphy, &rtwdev->track_ps_work);
 
+	ret = rtw89_wow_suspend(rtwdev, wowlan);
 	if (ret) {
 		rtw89_warn(rtwdev, "failed to suspend for wow %d\n", ret);
-		clear_bit(RTW89_FLAG_FORBIDDEN_TRACK_WROK, rtwdev->flags);
+		clear_bit(RTW89_FLAG_FORBIDDEN_TRACK_WORK, rtwdev->flags);
 		return 1;
 	}
 
@@ -1775,15 +1794,17 @@ static int rtw89_ops_resume(struct ieee80211_hw *hw)
 	struct rtw89_dev *rtwdev = hw->priv;
 	int ret;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
+
 	ret = rtw89_wow_resume(rtwdev);
 	if (ret)
 		rtw89_warn(rtwdev, "failed to resume for wow %d\n", ret);
-	mutex_unlock(&rtwdev->mutex);
 
-	clear_bit(RTW89_FLAG_FORBIDDEN_TRACK_WROK, rtwdev->flags);
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->track_work,
-				     RTW89_TRACK_WORK_PERIOD);
+	clear_bit(RTW89_FLAG_FORBIDDEN_TRACK_WORK, rtwdev->flags);
+	wiphy_delayed_work_queue(hw->wiphy, &rtwdev->track_work,
+				 RTW89_TRACK_WORK_PERIOD);
+	wiphy_delayed_work_queue(hw->wiphy, &rtwdev->track_ps_work,
+				 RTW89_TRACK_PS_WORK_PERIOD);
 
 	return ret ? 1 : 0;
 }
@@ -1803,18 +1824,16 @@ static void rtw89_set_rekey_data(struct ieee80211_hw *hw,
 	struct rtw89_wow_param *rtw_wow = &rtwdev->wow;
 	struct rtw89_wow_gtk_info *gtk_info = &rtw_wow->gtk_info;
 
+	lockdep_assert_wiphy(hw->wiphy);
+
 	if (data->kek_len > sizeof(gtk_info->kek) ||
 	    data->kck_len > sizeof(gtk_info->kck)) {
 		rtw89_warn(rtwdev, "kek or kck length over fw limit\n");
 		return;
 	}
 
-	mutex_lock(&rtwdev->mutex);
-
 	memcpy(gtk_info->kek, data->kek, data->kek_len);
 	memcpy(gtk_info->kck, data->kck, data->kck_len);
-
-	mutex_unlock(&rtwdev->mutex);
 }
 #endif
 
@@ -1822,16 +1841,13 @@ static void rtw89_ops_rfkill_poll(struct ieee80211_hw *hw)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(hw->wiphy);
 
 	/* wl_disable GPIO get floating when entering LPS */
 	if (test_bit(RTW89_FLAG_RUNNING, rtwdev->flags))
-		goto out;
+		return;
 
 	rtw89_core_rfkill_poll(rtwdev, false);
-
-out:
-	mutex_unlock(&rtwdev->mutex);
 }
 
 const struct ieee80211_ops rtw89_ops = {
@@ -1869,6 +1885,8 @@ const struct ieee80211_ops rtw89_ops = {
 	.change_chanctx		= rtw89_ops_change_chanctx,
 	.assign_vif_chanctx	= rtw89_ops_assign_vif_chanctx,
 	.unassign_vif_chanctx	= rtw89_ops_unassign_vif_chanctx,
+	.switch_vif_chanctx	= rtw89_ops_switch_vif_chanctx,
+	.channel_switch_beacon	= rtw89_ops_channel_switch_beacon,
 	.remain_on_channel		= rtw89_ops_remain_on_channel,
 	.cancel_remain_on_channel	= rtw89_ops_cancel_remain_on_channel,
 	.set_sar_specs		= rtw89_ops_set_sar_specs,
diff --git a/sys/contrib/dev/rtw89/mac_be.c b/sys/contrib/dev/rtw89/mac_be.c
index 2dbdeae904ad..0078080b3999 100644
--- a/sys/contrib/dev/rtw89/mac_be.c
+++ b/sys/contrib/dev/rtw89/mac_be.c
@@ -29,6 +29,7 @@ static const u32 rtw89_mac_mem_base_addrs_be[RTW89_MAC_MEM_NUM] = {
 	[RTW89_MAC_MEM_CPU_LOCAL]	= CPU_LOCAL_BASE_ADDR_BE,
 	[RTW89_MAC_MEM_BSSID_CAM]	= BSSID_CAM_BASE_ADDR_BE,
 	[RTW89_MAC_MEM_WD_PAGE]		= WD_PAGE_BASE_ADDR_BE,
+	[RTW89_MAC_MEM_MLD_TBL]		= MLD_TBL_BASE_ADDR_BE,
 };
 
 static const struct rtw89_port_reg rtw89_port_base_be = {
@@ -708,8 +709,8 @@ static int sec_eng_init_be(struct rtw89_dev *rtwdev)
 	val32 |= B_BE_CLK_EN_CGCMP | B_BE_CLK_EN_WAPI | B_BE_CLK_EN_WEP_TKIP |
 		 B_BE_SEC_TX_ENC | B_BE_SEC_RX_DEC |
 		 B_BE_MC_DEC | B_BE_BC_DEC |
-		 B_BE_BMC_MGNT_DEC | B_BE_UC_MGNT_DEC;
-	val32 &= ~B_BE_SEC_PRE_ENQUE_TX;
+		 B_BE_BMC_MGNT_DEC | B_BE_UC_MGNT_DEC |
+		 B_BE_SEC_PRE_ENQUE_TX;
 	rtw89_write32(rtwdev, R_BE_SEC_ENG_CTRL, val32);
 
 	rtw89_write32_set(rtwdev, R_BE_SEC_MPDU_PROC, B_BE_APPEND_ICV | B_BE_APPEND_MIC);
@@ -1865,7 +1866,7 @@ int rtw89_mac_cfg_ctrl_path_v2(struct rtw89_dev *rtwdev, bool wl)
 	if (wl)
 		return 0;
 
-	for (i = 0; i < RTW89_PHY_MAX; i++) {
+	for (i = 0; i < RTW89_PHY_NUM; i++) {
 		g[i].gnt_bt_sw_en = 1;
 		g[i].gnt_bt = 1;
 		g[i].gnt_wl_sw_en = 1;
@@ -2566,6 +2567,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = {
 	.filter_model_addr = R_BE_FILTER_MODEL_ADDR,
 	.indir_access_addr = R_BE_INDIR_ACCESS_ENTRY,
 	.mem_base_addrs = rtw89_mac_mem_base_addrs_be,
+	.mem_page_size = MAC_MEM_DUMP_PAGE_SIZE_BE,
 	.rx_fltr = R_BE_RX_FLTR_OPT,
 	.port_base = &rtw89_port_base_be,
 	.agg_len_ht = R_BE_AGG_LEN_HT_0,
@@ -2585,6 +2587,8 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = {
 		.mask = B_BE_RXTRIG_RU26_DIS,
 	},
 	.wow_ctrl = {.addr = R_BE_WOW_CTRL, .mask = B_BE_WOW_WOWEN,},
+	.agg_limit = {.addr = R_BE_AMPDU_AGG_LIMIT, .mask = B_BE_AMPDU_MAX_TIME_MASK,},
+	.txcnt_limit = {.addr = R_BE_TXCNT, .mask = B_BE_L_TXCNT_LMT_MASK,},
 
 	.check_mac_en = rtw89_mac_check_mac_en_be,
 	.sys_init = sys_init_be,
@@ -2634,6 +2638,8 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = {
 
 	.is_txq_empty = mac_is_txq_empty_be,
 
+	.prep_chan_list = rtw89_hw_scan_prep_chan_list_be,
+	.free_chan_list = rtw89_hw_scan_free_chan_list_be,
 	.add_chan_list = rtw89_hw_scan_add_chan_list_be,
 	.add_chan_list_pno = rtw89_pno_scan_add_chan_list_be,
 	.scan_offload = rtw89_fw_h2c_scan_offload_be,
diff --git a/sys/contrib/dev/rtw89/pci.c b/sys/contrib/dev/rtw89/pci.c
index 2b2f1c7f09b4..fd61ddf3841c 100644
--- a/sys/contrib/dev/rtw89/pci.c
+++ b/sys/contrib/dev/rtw89/pci.c
@@ -235,7 +235,7 @@ int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev,
 						       struct sk_buff *skb)
 {
 	struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
-	int rx_tag_retry = 100;
+	int rx_tag_retry = 1000;
 	int ret;
 
 	do {
@@ -2710,6 +2710,10 @@ static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
 	rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
 			  B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
 
+	rtw89_write32_mask(rtwdev, R_AX_PCIE_EXP_CTRL,
+			   B_AX_EN_STUCK_DBG | B_AX_ASFF_FULL_NO_STK,
+			   B_AX_EN_STUCK_DBG);
+
 	if (rtwdev->chip->chip_id == RTL8852A)
 		rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
 				  B_AX_EN_CHKDSC_NO_RX_STUCK);
@@ -3177,17 +3181,26 @@ static bool rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev *rtwdev)
 	return false;
 }
 
-static void rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev)
+static int rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev, bool force)
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	struct pci_dev *pdev = rtwpci->pdev;
+	int ret;
+	u8 val;
 
-	if (!rtwpci->enable_dac)
-		return;
+	if (!rtwpci->enable_dac && !force)
+		return 0;
 
 	if (!rtw89_pci_chip_is_manual_dac(rtwdev))
-		return;
+		return 0;
+
+	/* Configure DAC only via PCI config API, not DBI interfaces */
+	ret = pci_read_config_byte(pdev, RTW89_PCIE_L1_CTRL, &val);
+	if (ret)
+		return ret;
 
-	rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, RTW89_PCIE_BIT_EN_64BITS);
+	val |= RTW89_PCIE_BIT_EN_64BITS;
+	return pci_write_config_byte(pdev, RTW89_PCIE_L1_CTRL, val);
 }
 
 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
@@ -3205,13 +3218,16 @@ static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
 	}
 
 	if (!rtw89_pci_is_dac_compatible_bridge(rtwdev))
-		goto no_dac;
+		goto try_dac_done;
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36));
 	if (!ret) {
-		rtwpci->enable_dac = true;
-		rtw89_pci_cfg_dac(rtwdev);
-	} else {
+		ret = rtw89_pci_cfg_dac(rtwdev, true);
+		if (!ret) {
+			rtwpci->enable_dac = true;
+			goto try_dac_done;
+		}
+
 		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 		if (ret) {
 			rtw89_err(rtwdev,
@@ -3219,7 +3235,7 @@ static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
 			goto err_release_regions;
 		}
 	}
-no_dac:
+try_dac_done:
 
 #if defined(__FreeBSD__)
 	linuxkpi_pcim_want_to_use_bus_functions(pdev);
@@ -4385,7 +4401,7 @@ static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
 void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume)
 {
 	if (resume)
-		rtw89_pci_cfg_dac(rtwdev);
+		rtw89_pci_cfg_dac(rtwdev, false);
 
 	rtw89_pci_disable_eq(rtwdev);
 	rtw89_pci_filter_out(rtwdev);
@@ -4424,6 +4440,43 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev)
 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
 EXPORT_SYMBOL(rtw89_pm_ops);
 
+static pci_ers_result_t rtw89_pci_io_error_detected(struct pci_dev *pdev,
+						    pci_channel_state_t state)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+
+	netif_device_detach(netdev);
+
+	return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static pci_ers_result_t rtw89_pci_io_slot_reset(struct pci_dev *pdev)
+{
+	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
+	struct rtw89_dev *rtwdev = hw->priv;
+
+	rtw89_ser_notify(rtwdev, MAC_AX_ERR_ASSERTION);
+
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+static void rtw89_pci_io_resume(struct pci_dev *pdev)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+
+	/* ack any pending wake events, disable PME */
+	pci_enable_wake(pdev, PCI_D0, 0);
+
+	netif_device_attach(netdev);
+}
+
+const struct pci_error_handlers rtw89_pci_err_handler = {
+	.error_detected = rtw89_pci_io_error_detected,
+	.slot_reset = rtw89_pci_io_slot_reset,
+	.resume = rtw89_pci_io_resume,
+};
+EXPORT_SYMBOL(rtw89_pci_err_handler);
+
 const struct rtw89_pci_gen_def rtw89_pci_gen_ax = {
 	.isr_rdu = B_AX_RDU_INT,
 	.isr_halt_c2h = B_AX_HALT_C2H_INT_EN,
@@ -4520,6 +4573,7 @@ int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	rtwdev->pci_info = info->bus.pci;
 	rtwdev->hci.ops = &rtw89_pci_ops;
 	rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
+	rtwdev->hci.dle_type = RTW89_HCI_DLE_TYPE_PCIE;
 	rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
 	rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
 
diff --git a/sys/contrib/dev/rtw89/pci.h b/sys/contrib/dev/rtw89/pci.h
index 4d11c3dd60a5..52f527069da6 100644
--- a/sys/contrib/dev/rtw89/pci.h
+++ b/sys/contrib/dev/rtw89/pci.h
@@ -455,34 +455,36 @@
 #define B_BE_RX0DMA_INT_EN BIT(0)
 
 #define R_BE_HAXI_HISR00 0xB0B4
-#define B_BE_RDU_CH6_INT BIT(28)
-#define B_BE_RDU_CH5_INT BIT(27)
-#define B_BE_RDU_CH4_INT BIT(26)
-#define B_BE_RDU_CH2_INT BIT(25)
-#define B_BE_RDU_CH1_INT BIT(24)
-#define B_BE_RDU_CH0_INT BIT(23)
-#define B_BE_RXDMA_STUCK_INT BIT(22)
-#define B_BE_TXDMA_STUCK_INT BIT(21)
-#define B_BE_TXDMA_CH14_INT BIT(20)
-#define B_BE_TXDMA_CH13_INT BIT(19)
-#define B_BE_TXDMA_CH12_INT BIT(18)
-#define B_BE_TXDMA_CH11_INT BIT(17)
-#define B_BE_TXDMA_CH10_INT BIT(16)
-#define B_BE_TXDMA_CH9_INT BIT(15)
-#define B_BE_TXDMA_CH8_INT BIT(14)
-#define B_BE_TXDMA_CH7_INT BIT(13)
-#define B_BE_TXDMA_CH6_INT BIT(12)
-#define B_BE_TXDMA_CH5_INT BIT(11)
-#define B_BE_TXDMA_CH4_INT BIT(10)
-#define B_BE_TXDMA_CH3_INT BIT(9)
-#define B_BE_TXDMA_CH2_INT BIT(8)
-#define B_BE_TXDMA_CH1_INT BIT(7)
-#define B_BE_TXDMA_CH0_INT BIT(6)
-#define B_BE_RPQ1DMA_INT BIT(5)
-#define B_BE_RX1P1DMA_INT BIT(4)
+#define B_BE_RDU_CH5_INT_V1 BIT(30)
+#define B_BE_RDU_CH4_INT_V1 BIT(29)
+#define B_BE_RDU_CH3_INT_V1 BIT(28)
+#define B_BE_RDU_CH2_INT_V1 BIT(27)
+#define B_BE_RDU_CH1_INT_V1 BIT(26)
+#define B_BE_RDU_CH0_INT_V1 BIT(25)
+#define B_BE_RXDMA_STUCK_INT_V1 BIT(24)
+#define B_BE_TXDMA_STUCK_INT_V1 BIT(23)
+#define B_BE_TXDMA_CH14_INT_V1 BIT(22)
+#define B_BE_TXDMA_CH13_INT_V1 BIT(21)
+#define B_BE_TXDMA_CH12_INT_V1 BIT(20)
+#define B_BE_TXDMA_CH11_INT_V1 BIT(19)
+#define B_BE_TXDMA_CH10_INT_V1 BIT(18)
+#define B_BE_TXDMA_CH9_INT_V1 BIT(17)
+#define B_BE_TXDMA_CH8_INT_V1 BIT(16)
+#define B_BE_TXDMA_CH7_INT_V1 BIT(15)
+#define B_BE_TXDMA_CH6_INT_V1 BIT(14)
+#define B_BE_TXDMA_CH5_INT_V1 BIT(13)
+#define B_BE_TXDMA_CH4_INT_V1 BIT(12)
+#define B_BE_TXDMA_CH3_INT_V1 BIT(11)
+#define B_BE_TXDMA_CH2_INT_V1 BIT(10)
+#define B_BE_TXDMA_CH1_INT_V1 BIT(9)
+#define B_BE_TXDMA_CH0_INT_V1 BIT(8)
+#define B_BE_RX1P1DMA_INT_V1 BIT(7)
+#define B_BE_RX0P1DMA_INT_V1 BIT(6)
+#define B_BE_RO1DMA_INT BIT(5)
+#define B_BE_RP1DMA_INT BIT(4)
 #define B_BE_RX1DMA_INT BIT(3)
-#define B_BE_RPQ0DMA_INT BIT(2)
-#define B_BE_RX0P1DMA_INT BIT(1)
+#define B_BE_RO0DMA_INT BIT(2)
+#define B_BE_RP0DMA_INT BIT(1)
 #define B_BE_RX0DMA_INT BIT(0)
 
 /* TX/RX */
@@ -1620,6 +1622,7 @@ static inline bool rtw89_pci_ltr_is_err_reg_val(u32 val)
 
 extern const struct dev_pm_ops rtw89_pm_ops;
 extern const struct dev_pm_ops rtw89_pm_ops_be;
+extern const struct pci_error_handlers rtw89_pci_err_handler;
 extern const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set;
 extern const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1;
 extern const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be;
diff --git a/sys/contrib/dev/rtw89/pci_be.c b/sys/contrib/dev/rtw89/pci_be.c
index cd39eebe8186..12e6a0cbb889 100644
--- a/sys/contrib/dev/rtw89/pci_be.c
+++ b/sys/contrib/dev/rtw89/pci_be.c
@@ -666,7 +666,7 @@ SIMPLE_DEV_PM_OPS(rtw89_pm_ops_be, rtw89_pci_suspend_be, rtw89_pci_resume_be);
 EXPORT_SYMBOL(rtw89_pm_ops_be);
 
 const struct rtw89_pci_gen_def rtw89_pci_gen_be = {
-	.isr_rdu = B_BE_RDU_CH1_INT | B_BE_RDU_CH0_INT,
+	.isr_rdu = B_BE_RDU_CH1_INT_V1 | B_BE_RDU_CH0_INT_V1,
 	.isr_halt_c2h = B_BE_HALT_C2H_INT,
 	.isr_wdt_timeout = B_BE_WDT_TIMEOUT_INT,
 	.isr_clear_rpq = {R_BE_PCIE_DMA_ISR, B_BE_PCIE_RX_RPQ0_ISR_V1},
diff --git a/sys/contrib/dev/rtw89/phy.c b/sys/contrib/dev/rtw89/phy.c
index 3aea05eaa73f..5b48051e62d1 100644
--- a/sys/contrib/dev/rtw89/phy.c
+++ b/sys/contrib/dev/rtw89/phy.c
@@ -119,10 +119,12 @@ static u64 get_eht_mcs_ra_mask(u8 *max_nss, u8 start_mcs, u8 n_nss)
 	return mask;
 }
 
-static u64 get_eht_ra_mask(struct ieee80211_link_sta *link_sta)
+static u64 get_eht_ra_mask(struct rtw89_vif_link *rtwvif_link,
+			   struct ieee80211_link_sta *link_sta)
 {
-	struct ieee80211_sta_eht_cap *eht_cap = &link_sta->eht_cap;
+	struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link);
 	struct ieee80211_eht_mcs_nss_supp_20mhz_only *mcs_nss_20mhz;
+	struct ieee80211_sta_eht_cap *eht_cap = &link_sta->eht_cap;
 	struct ieee80211_eht_mcs_nss_supp_bw *mcs_nss;
 	u8 *he_phy_cap = link_sta->he_cap.he_cap_elem.phy_cap_info;
 
@@ -136,8 +138,8 @@ static u64 get_eht_ra_mask(struct ieee80211_link_sta *link_sta)
 		/* MCS 9, 11, 13 */
 		return get_eht_mcs_ra_mask(mcs_nss->rx_tx_max_nss, 9, 3);
 	case IEEE80211_STA_RX_BW_20:
-		if (!(he_phy_cap[0] &
-		      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
+		if (vif->type == NL80211_IFTYPE_AP &&
+		    !(he_phy_cap[0] & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
 			mcs_nss_20mhz = &eht_cap->eht_mcs_nss_supp.only_20mhz;
 			/* MCS 7, 9, 11, 13 */
 			return get_eht_mcs_ra_mask(mcs_nss_20mhz->rx_tx_max_nss, 7, 4);
@@ -332,7 +334,7 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 	/* Set the ra mask from sta's capability */
 	if (link_sta->eht_cap.has_eht) {
 		mode |= RTW89_RA_MODE_EHT;
-		ra_mask |= get_eht_ra_mask(link_sta);
+		ra_mask |= get_eht_ra_mask(rtwvif_link, link_sta);
 
 		if (rtwdev->hal.no_mcs_12_13)
 			high_rate_masks = rtw89_ra_mask_eht_mcs0_11;
@@ -902,7 +904,8 @@ static u32 rtw89_phy_read_rf_a(struct rtw89_dev *rtwdev,
 				       30, false, rtwdev, R_SWSI_V1,
 				       B_SWSI_R_DATA_DONE_V1);
 	if (ret) {
-		rtw89_err(rtwdev, "read swsi busy\n");
+		if (!test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
+			rtw89_err(rtwdev, "read swsi busy\n");
 		return INV_RF_DATA;
 	}
 
@@ -2045,33 +2048,47 @@ static s8 rtw89_phy_ant_gain_query(struct rtw89_dev *rtwdev,
 		   ant_gain->offset[path][subband_h]);
 }
 
-static s8 rtw89_phy_ant_gain_offset(struct rtw89_dev *rtwdev, u8 band, u32 center_freq)
+static s8 rtw89_phy_ant_gain_offset(struct rtw89_dev *rtwdev, u32 center_freq)
+{
+	s8 offset_patha, offset_pathb;
+
+	offset_patha = rtw89_phy_ant_gain_query(rtwdev, RF_PATH_A, center_freq);
+	offset_pathb = rtw89_phy_ant_gain_query(rtwdev, RF_PATH_B, center_freq);
+
+	if (RTW89_CHK_FW_FEATURE(NO_POWER_DIFFERENCE, &rtwdev->fw))
+		return min(offset_patha, offset_pathb);
+
+	return max(offset_patha, offset_pathb);
+}
+
+static bool rtw89_can_apply_ant_gain(struct rtw89_dev *rtwdev, u8 band)
 {
+	const struct rtw89_rfe_parms *rfe_parms = rtwdev->rfe_parms;
 	struct rtw89_ant_gain_info *ant_gain = &rtwdev->ant_gain;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	u8 regd = rtw89_regd_get(rtwdev, band);
-	s8 offset_patha, offset_pathb;
 
 	if (!chip->support_ant_gain)
-		return 0;
+		return false;
 
-	if (!(ant_gain->regd_enabled & BIT(regd)))
-		return 0;
+	if (ant_gain->block_country || !(ant_gain->regd_enabled & BIT(regd)))
+		return false;
 
-	offset_patha = rtw89_phy_ant_gain_query(rtwdev, RF_PATH_A, center_freq);
-	offset_pathb = rtw89_phy_ant_gain_query(rtwdev, RF_PATH_B, center_freq);
+	if (!rfe_parms->has_da)
+		return false;
 
-	return max(offset_patha, offset_pathb);
+	return true;
 }
 
 s16 rtw89_phy_ant_gain_pwr_offset(struct rtw89_dev *rtwdev,
 				  const struct rtw89_chan *chan)
 {
-	struct rtw89_ant_gain_info *ant_gain = &rtwdev->ant_gain;
-	u8 regd = rtw89_regd_get(rtwdev, chan->band_type);
 	s8 offset_patha, offset_pathb;
 
-	if (!(ant_gain->regd_enabled & BIT(regd)))
+	if (!rtw89_can_apply_ant_gain(rtwdev, chan->band_type))
+		return 0;
+
+	if (RTW89_CHK_FW_FEATURE(NO_POWER_DIFFERENCE, &rtwdev->fw))
 		return 0;
 
 	offset_patha = rtw89_phy_ant_gain_query(rtwdev, RF_PATH_A, chan->freq);
@@ -2081,24 +2098,25 @@ s16 rtw89_phy_ant_gain_pwr_offset(struct rtw89_dev *rtwdev,
 }
 EXPORT_SYMBOL(rtw89_phy_ant_gain_pwr_offset);
 
-void rtw89_print_ant_gain(struct seq_file *m, struct rtw89_dev *rtwdev,
-			  const struct rtw89_chan *chan)
+int rtw89_print_ant_gain(struct rtw89_dev *rtwdev, char *buf, size_t bufsz,
+			 const struct rtw89_chan *chan)
 {
-	struct rtw89_ant_gain_info *ant_gain = &rtwdev->ant_gain;
-	const struct rtw89_chip_info *chip = rtwdev->chip;
-	u8 regd = rtw89_regd_get(rtwdev, chan->band_type);
+	char *p = buf, *end = buf + bufsz;
 	s8 offset_patha, offset_pathb;
 
-	if (!chip->support_ant_gain || !(ant_gain->regd_enabled & BIT(regd))) {
-		seq_puts(m, "no DAG is applied\n");
-		return;
+	if (!rtw89_can_apply_ant_gain(rtwdev, chan->band_type)) {
+		p += scnprintf(p, end - p, "no DAG is applied\n");
+		goto out;
 	}
 
 	offset_patha = rtw89_phy_ant_gain_query(rtwdev, RF_PATH_A, chan->freq);
 	offset_pathb = rtw89_phy_ant_gain_query(rtwdev, RF_PATH_B, chan->freq);
 
-	seq_printf(m, "ChainA offset: %d dBm\n", offset_patha);
-	seq_printf(m, "ChainB offset: %d dBm\n", offset_pathb);
+	p += scnprintf(p, end - p, "ChainA offset: %d dBm\n", offset_patha);
+	p += scnprintf(p, end - p, "ChainB offset: %d dBm\n", offset_pathb);
+
+out:
+	return p - buf;
 }
 
 static const u8 rtw89_rs_idx_num_ax[] = {
@@ -2251,20 +2269,31 @@ s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
 			      u8 bw, u8 ntx, u8 rs, u8 bf, u8 ch)
 {
 	const struct rtw89_rfe_parms *rfe_parms = rtwdev->rfe_parms;
+	const struct rtw89_txpwr_rule_2ghz *rule_da_2ghz = &rfe_parms->rule_da_2ghz;
+	const struct rtw89_txpwr_rule_5ghz *rule_da_5ghz = &rfe_parms->rule_da_5ghz;
+	const struct rtw89_txpwr_rule_6ghz *rule_da_6ghz = &rfe_parms->rule_da_6ghz;
 	const struct rtw89_txpwr_rule_2ghz *rule_2ghz = &rfe_parms->rule_2ghz;
 	const struct rtw89_txpwr_rule_5ghz *rule_5ghz = &rfe_parms->rule_5ghz;
 	const struct rtw89_txpwr_rule_6ghz *rule_6ghz = &rfe_parms->rule_6ghz;
 	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
 	enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band);
+	bool has_ant_gain = rtw89_can_apply_ant_gain(rtwdev, band);
 	u32 freq = ieee80211_channel_to_frequency(ch, nl_band);
 	u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch);
+	s8 lmt = 0, da_lmt = S8_MAX, sar, offset = 0;
 	u8 regd = rtw89_regd_get(rtwdev, band);
 	u8 reg6 = regulatory->reg_6ghz_power;
-	s8 lmt = 0, sar, offset;
+	struct rtw89_sar_parm sar_parm = {
+		.center_freq = freq,
+		.ntx = ntx,
+	};
 	s8 cstr;
 
 	switch (band) {
 	case RTW89_BAND_2G:
+		if (has_ant_gain)
+			da_lmt = (*rule_da_2ghz->lmt)[bw][ntx][rs][bf][regd][ch_idx];
+
 		lmt = (*rule_2ghz->lmt)[bw][ntx][rs][bf][regd][ch_idx];
 		if (lmt)
 			break;
@@ -2272,6 +2301,9 @@ s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
 		lmt = (*rule_2ghz->lmt)[bw][ntx][rs][bf][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_5G:
+		if (has_ant_gain)
+			da_lmt = (*rule_da_5ghz->lmt)[bw][ntx][rs][bf][regd][ch_idx];
+
 		lmt = (*rule_5ghz->lmt)[bw][ntx][rs][bf][regd][ch_idx];
 		if (lmt)
 			break;
@@ -2279,6 +2311,9 @@ s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
 		lmt = (*rule_5ghz->lmt)[bw][ntx][rs][bf][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_6G:
+		if (has_ant_gain)
+			da_lmt = (*rule_da_6ghz->lmt)[bw][ntx][rs][bf][regd][reg6][ch_idx];
+
 		lmt = (*rule_6ghz->lmt)[bw][ntx][rs][bf][regd][reg6][ch_idx];
 		if (lmt)
 			break;
@@ -2292,9 +2327,12 @@ s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
 		return 0;
 	}
 
-	offset = rtw89_phy_ant_gain_offset(rtwdev, band, freq);
-	lmt = rtw89_phy_txpwr_rf_to_mac(rtwdev, lmt + offset);
-	sar = rtw89_query_sar(rtwdev, freq);
+	da_lmt = da_lmt ?: S8_MAX;
+	if (da_lmt != S8_MAX)
+		offset = rtw89_phy_ant_gain_offset(rtwdev, freq);
+
+	lmt = rtw89_phy_txpwr_rf_to_mac(rtwdev, min(lmt + offset, da_lmt));
+	sar = rtw89_query_sar(rtwdev, &sar_parm);
 	cstr = rtw89_phy_get_tpe_constraint(rtwdev, band);
 
 	return min3(lmt, sar, cstr);
@@ -2511,20 +2549,31 @@ s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, u8 band,
 				 u8 ru, u8 ntx, u8 ch)
 {
 	const struct rtw89_rfe_parms *rfe_parms = rtwdev->rfe_parms;
+	const struct rtw89_txpwr_rule_2ghz *rule_da_2ghz = &rfe_parms->rule_da_2ghz;
+	const struct rtw89_txpwr_rule_5ghz *rule_da_5ghz = &rfe_parms->rule_da_5ghz;
+	const struct rtw89_txpwr_rule_6ghz *rule_da_6ghz = &rfe_parms->rule_da_6ghz;
 	const struct rtw89_txpwr_rule_2ghz *rule_2ghz = &rfe_parms->rule_2ghz;
 	const struct rtw89_txpwr_rule_5ghz *rule_5ghz = &rfe_parms->rule_5ghz;
 	const struct rtw89_txpwr_rule_6ghz *rule_6ghz = &rfe_parms->rule_6ghz;
 	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
 	enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band);
+	bool has_ant_gain = rtw89_can_apply_ant_gain(rtwdev, band);
 	u32 freq = ieee80211_channel_to_frequency(ch, nl_band);
 	u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch);
+	s8 lmt_ru = 0, da_lmt_ru = S8_MAX, sar, offset = 0;
 	u8 regd = rtw89_regd_get(rtwdev, band);
 	u8 reg6 = regulatory->reg_6ghz_power;
-	s8 lmt_ru = 0, sar, offset;
+	struct rtw89_sar_parm sar_parm = {
+		.center_freq = freq,
+		.ntx = ntx,
+	};
 	s8 cstr;
 
 	switch (band) {
 	case RTW89_BAND_2G:
+		if (has_ant_gain)
+			da_lmt_ru = (*rule_da_2ghz->lmt_ru)[ru][ntx][regd][ch_idx];
+
 		lmt_ru = (*rule_2ghz->lmt_ru)[ru][ntx][regd][ch_idx];
 		if (lmt_ru)
 			break;
@@ -2532,6 +2581,9 @@ s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, u8 band,
 		lmt_ru = (*rule_2ghz->lmt_ru)[ru][ntx][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_5G:
+		if (has_ant_gain)
+			da_lmt_ru = (*rule_da_5ghz->lmt_ru)[ru][ntx][regd][ch_idx];
+
 		lmt_ru = (*rule_5ghz->lmt_ru)[ru][ntx][regd][ch_idx];
 		if (lmt_ru)
 			break;
@@ -2539,6 +2591,9 @@ s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, u8 band,
 		lmt_ru = (*rule_5ghz->lmt_ru)[ru][ntx][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_6G:
+		if (has_ant_gain)
+			da_lmt_ru = (*rule_da_6ghz->lmt_ru)[ru][ntx][regd][reg6][ch_idx];
+
 		lmt_ru = (*rule_6ghz->lmt_ru)[ru][ntx][regd][reg6][ch_idx];
 		if (lmt_ru)
 			break;
@@ -2552,9 +2607,12 @@ s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, u8 band,
 		return 0;
 	}
 
-	offset = rtw89_phy_ant_gain_offset(rtwdev, band, freq);
-	lmt_ru = rtw89_phy_txpwr_rf_to_mac(rtwdev, lmt_ru + offset);
-	sar = rtw89_query_sar(rtwdev, freq);
+	da_lmt_ru = da_lmt_ru ?: S8_MAX;
+	if (da_lmt_ru != S8_MAX)
+		offset = rtw89_phy_ant_gain_offset(rtwdev, freq);
+
+	lmt_ru = rtw89_phy_txpwr_rf_to_mac(rtwdev, min(lmt_ru + offset, da_lmt_ru));
+	sar = rtw89_query_sar(rtwdev, &sar_parm);
 	cstr = rtw89_phy_get_tpe_constraint(rtwdev, band);
 
 	return min3(lmt_ru, sar, cstr);
@@ -3015,6 +3073,35 @@ void (* const rtw89_phy_c2h_ra_handler[])(struct rtw89_dev *rtwdev,
 	[RTW89_PHY_C2H_FUNC_TXSTS] = NULL,
 };
 
+static void
+rtw89_phy_c2h_lowrt_rty(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len)
+{
+}
+
+static void
+rtw89_phy_c2h_fw_scan_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len)
+{
+	const struct rtw89_c2h_fw_scan_rpt *c2h_rpt =
+		(const struct rtw89_c2h_fw_scan_rpt *)c2h->data;
+
+	rtw89_debug(rtwdev, RTW89_DBG_DIG,
+		    "%s: band: %u, op_chan: %u, PD_low_bd(ofdm, cck): (-%d, %d), phy_idx: %u\n",
+		    __func__, c2h_rpt->band, c2h_rpt->center_ch,
+		    PD_LOWER_BOUND_BASE - (c2h_rpt->ofdm_pd_idx << 1),
+		    c2h_rpt->cck_pd_idx, c2h_rpt->phy_idx);
+}
+
+static
+void (* const rtw89_phy_c2h_dm_handler[])(struct rtw89_dev *rtwdev,
+					  struct sk_buff *c2h, u32 len) = {
+	[RTW89_PHY_C2H_DM_FUNC_FW_TEST] = NULL,
+	[RTW89_PHY_C2H_DM_FUNC_FW_TRIG_TX_RPT] = NULL,
+	[RTW89_PHY_C2H_DM_FUNC_SIGB] = NULL,
+	[RTW89_PHY_C2H_DM_FUNC_LOWRT_RTY] = rtw89_phy_c2h_lowrt_rty,
+	[RTW89_PHY_C2H_DM_FUNC_MCC_DIG] = NULL,
+	[RTW89_PHY_C2H_DM_FUNC_FW_SCAN] = rtw89_phy_c2h_fw_scan_rpt,
+};
+
 static void rtw89_phy_c2h_rfk_rpt_log(struct rtw89_dev *rtwdev,
 				      enum rtw89_phy_c2h_rfk_log_func func,
 				      void *content, u16 len)
@@ -3483,6 +3570,30 @@ rtw89_phy_c2h_rfk_report_state(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u3
 static void
 rtw89_phy_c2h_rfk_log_tas_pwr(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len)
 {
+	const struct rtw89_c2h_rf_tas_info *rf_tas =
+		(const struct rtw89_c2h_rf_tas_info *)c2h->data;
+	const enum rtw89_sar_sources src = rtwdev->sar.src;
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	u64 linear = 0;
+	u32 i, cur_idx;
+	s16 txpwr;
+
+	if (!tas->enable || src == RTW89_SAR_SOURCE_NONE)
+		return;
+
+	cur_idx = le32_to_cpu(rf_tas->cur_idx);
+	for (i = 0; i < cur_idx; i++) {
+		txpwr = (s16)le16_to_cpu(rf_tas->txpwr_history[i]);
+		linear += rtw89_db_quarter_to_linear(txpwr);
+
+		rtw89_debug(rtwdev, RTW89_DBG_SAR,
+			    "tas: index: %u, txpwr: %d\n", i, txpwr);
+	}
+
+	if (cur_idx == 0)
+		tas->instant_txpwr = rtw89_db_to_linear(0);
+	else
+		tas->instant_txpwr = DIV_ROUND_DOWN_ULL(linear, cur_idx);
 }
 
 static
@@ -3539,9 +3650,9 @@ void rtw89_phy_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb,
 			handler = rtw89_phy_c2h_rfk_report_handler[func];
 		break;
 	case RTW89_PHY_C2H_CLASS_DM:
-		if (func == RTW89_PHY_C2H_DM_FUNC_LOWRT_RTY)
-			return;
-		fallthrough;
+		if (func < ARRAY_SIZE(rtw89_phy_c2h_dm_handler))
+			handler = rtw89_phy_c2h_dm_handler[func];
+		break;
 	default:
 		rtw89_info(rtwdev, "PHY c2h class %d not support\n", class);
 		return;
@@ -4628,7 +4739,7 @@ static void rtw89_phy_cfo_dm(struct rtw89_dev *rtwdev)
 	cfo->dcfo_avg = 0;
 	rtw89_debug(rtwdev, RTW89_DBG_CFO, "CFO:total_sta_assoc=%d\n",
 		    rtwdev->total_sta_assoc);
-	if (rtwdev->total_sta_assoc == 0) {
+	if (rtwdev->total_sta_assoc == 0 || rtw89_is_mlo_1_1(rtwdev)) {
 		rtw89_phy_cfo_reset(rtwdev);
 		return;
 	}
@@ -4679,29 +4790,28 @@ static void rtw89_phy_cfo_dm(struct rtw89_dev *rtwdev)
 	rtw89_phy_cfo_statistics_reset(rtwdev);
 }
 
-void rtw89_phy_cfo_track_work(struct work_struct *work)
+void rtw89_phy_cfo_track_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						cfo_track_work.work);
 	struct rtw89_cfo_tracking_info *cfo = &rtwdev->cfo_tracking;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
+
 	if (!cfo->cfo_trig_by_timer_en)
-		goto out;
+		return;
 	rtw89_leave_ps_mode(rtwdev);
 	rtw89_phy_cfo_dm(rtwdev);
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->cfo_track_work,
-				     msecs_to_jiffies(cfo->cfo_timer_ms));
-out:
-	mutex_unlock(&rtwdev->mutex);
+	wiphy_delayed_work_queue(wiphy, &rtwdev->cfo_track_work,
+				 msecs_to_jiffies(cfo->cfo_timer_ms));
 }
 
 static void rtw89_phy_cfo_start_work(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_cfo_tracking_info *cfo = &rtwdev->cfo_tracking;
 
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->cfo_track_work,
-				     msecs_to_jiffies(cfo->cfo_timer_ms));
+	wiphy_delayed_work_queue(rtwdev->hw->wiphy, &rtwdev->cfo_track_work,
+				 msecs_to_jiffies(cfo->cfo_timer_ms));
 }
 
 void rtw89_phy_cfo_track(struct rtw89_dev *rtwdev)
@@ -5143,7 +5253,6 @@ static void rtw89_phy_stat_thermal_update(struct rtw89_dev *rtwdev)
 
 struct rtw89_phy_iter_rssi_data {
 	struct rtw89_dev *rtwdev;
-	struct rtw89_phy_ch_info *ch_info;
 	bool rssi_changed;
 };
 
@@ -5151,10 +5260,15 @@ static
 void __rtw89_phy_stat_rssi_update_iter(struct rtw89_sta_link *rtwsta_link,
 				       struct rtw89_phy_iter_rssi_data *rssi_data)
 {
-	struct rtw89_phy_ch_info *ch_info = rssi_data->ch_info;
+	struct rtw89_vif_link *rtwvif_link = rtwsta_link->rtwvif_link;
+	struct rtw89_dev *rtwdev = rssi_data->rtwdev;
+	struct rtw89_phy_ch_info *ch_info;
+	struct rtw89_bb_ctx *bb;
 	unsigned long rssi_curr;
 
 	rssi_curr = ewma_rssi_read(&rtwsta_link->avg_rssi);
+	bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx);
+	ch_info = &bb->ch_info;
 
 	if (rssi_curr < ch_info->rssi_min) {
 		ch_info->rssi_min = rssi_curr;
@@ -5185,11 +5299,13 @@ static void rtw89_phy_stat_rssi_update_iter(void *data,
 
 static void rtw89_phy_stat_rssi_update(struct rtw89_dev *rtwdev)
 {
-	struct rtw89_phy_iter_rssi_data rssi_data = {0};
+	struct rtw89_phy_iter_rssi_data rssi_data = {};
+	struct rtw89_bb_ctx *bb;
 
 	rssi_data.rtwdev = rtwdev;
-	rssi_data.ch_info = &rtwdev->ch_info;
-	rssi_data.ch_info->rssi_min = U8_MAX;
+	rtw89_for_each_active_bb(rtwdev, bb)
+		bb->ch_info.rssi_min = U8_MAX;
+
 	ieee80211_iterate_stations_atomic(rtwdev->hw,
 					  rtw89_phy_stat_rssi_update_iter,
 					  &rssi_data);
@@ -5227,24 +5343,27 @@ void rtw89_phy_stat_track(struct rtw89_dev *rtwdev)
 	memset(&phystat->cur_pkt_stat, 0, sizeof(phystat->cur_pkt_stat));
 }
 
-static u16 rtw89_phy_ccx_us_to_idx(struct rtw89_dev *rtwdev, u32 time_us)
+static u16 rtw89_phy_ccx_us_to_idx(struct rtw89_dev *rtwdev,
+				   struct rtw89_bb_ctx *bb, u32 time_us)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 
 	return time_us >> (ilog2(CCX_US_BASE_RATIO) + env->ccx_unit_idx);
 }
 
-static u32 rtw89_phy_ccx_idx_to_us(struct rtw89_dev *rtwdev, u16 idx)
+static u32 rtw89_phy_ccx_idx_to_us(struct rtw89_dev *rtwdev,
+				   struct rtw89_bb_ctx *bb, u16 idx)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 
 	return idx << (ilog2(CCX_US_BASE_RATIO) + env->ccx_unit_idx);
 }
 
-static void rtw89_phy_ccx_top_setting_init(struct rtw89_dev *rtwdev)
+static void rtw89_phy_ccx_top_setting_init(struct rtw89_dev *rtwdev,
+					   struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_phy_gen_def *phy = rtwdev->chip->phy_def;
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	const struct rtw89_ccx_regs *ccx = phy->ccx;
 
 	env->ccx_manual_ctrl = false;
@@ -5253,17 +5372,20 @@ static void rtw89_phy_ccx_top_setting_init(struct rtw89_dev *rtwdev)
 	env->ccx_period = 0;
 	env->ccx_unit_idx = RTW89_CCX_32_US;
 
-	rtw89_phy_set_phy_regs(rtwdev, ccx->setting_addr, ccx->en_mask, 1);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->setting_addr, ccx->trig_opt_mask, 1);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->setting_addr, ccx->measurement_trig_mask, 1);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->setting_addr, ccx->edcca_opt_mask,
-			       RTW89_CCX_EDCCA_BW20_0);
+	rtw89_phy_write32_idx(rtwdev, ccx->setting_addr, ccx->en_mask, 1, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->setting_addr, ccx->trig_opt_mask, 1,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->setting_addr, ccx->measurement_trig_mask, 1,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->setting_addr, ccx->edcca_opt_mask,
+			      RTW89_CCX_EDCCA_BW20_0, bb->phy_idx);
 }
 
-static u16 rtw89_phy_ccx_get_report(struct rtw89_dev *rtwdev, u16 report,
-				    u16 score)
+static u16 rtw89_phy_ccx_get_report(struct rtw89_dev *rtwdev,
+				    struct rtw89_bb_ctx *bb,
+				    u16 report, u16 score)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	u32 numer = 0;
 	u16 ret = 0;
 
@@ -5303,9 +5425,10 @@ static void rtw89_phy_ccx_ms_to_period_unit(struct rtw89_dev *rtwdev,
 		    *period, *unit_idx);
 }
 
-static void rtw89_phy_ccx_racing_release(struct rtw89_dev *rtwdev)
+static void rtw89_phy_ccx_racing_release(struct rtw89_dev *rtwdev,
+					 struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 
 	rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK,
 		    "lv:(%d)->(0)\n", env->ccx_rac_lv);
@@ -5316,9 +5439,10 @@ static void rtw89_phy_ccx_racing_release(struct rtw89_dev *rtwdev)
 }
 
 static bool rtw89_phy_ifs_clm_th_update_check(struct rtw89_dev *rtwdev,
+					      struct rtw89_bb_ctx *bb,
 					      struct rtw89_ccx_para_info *para)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	bool is_update = env->ifs_clm_app != para->ifs_clm_app;
 	u8 i = 0;
 	u16 *ifs_th_l = env->ifs_clm_th_l;
@@ -5353,12 +5477,12 @@ static bool rtw89_phy_ifs_clm_th_update_check(struct rtw89_dev *rtwdev,
 	 */
 	ifs_th_l[IFS_CLM_TH_START_IDX] = 0;
 	ifs_th_h_us[IFS_CLM_TH_START_IDX] = ifs_th0_us;
-	ifs_th_h[IFS_CLM_TH_START_IDX] = rtw89_phy_ccx_us_to_idx(rtwdev,
+	ifs_th_h[IFS_CLM_TH_START_IDX] = rtw89_phy_ccx_us_to_idx(rtwdev, bb,
 								 ifs_th0_us);
 	for (i = 1; i < RTW89_IFS_CLM_NUM; i++) {
 		ifs_th_l[i] = ifs_th_h[i - 1] + 1;
 		ifs_th_h_us[i] = ifs_th_h_us[i - 1] * ifs_th_times;
-		ifs_th_h[i] = rtw89_phy_ccx_us_to_idx(rtwdev, ifs_th_h_us[i]);
+		ifs_th_h[i] = rtw89_phy_ccx_us_to_idx(rtwdev, bb, ifs_th_h_us[i]);
 	}
 
 ifs_update_finished:
@@ -5369,30 +5493,31 @@ ifs_update_finished:
 	return is_update;
 }
 
-static void rtw89_phy_ifs_clm_set_th_reg(struct rtw89_dev *rtwdev)
+static void rtw89_phy_ifs_clm_set_th_reg(struct rtw89_dev *rtwdev,
+					 struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_phy_gen_def *phy = rtwdev->chip->phy_def;
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	const struct rtw89_ccx_regs *ccx = phy->ccx;
 	u8 i = 0;
 
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t1_addr, ccx->ifs_t1_th_l_mask,
-			       env->ifs_clm_th_l[0]);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t2_addr, ccx->ifs_t2_th_l_mask,
-			       env->ifs_clm_th_l[1]);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t3_addr, ccx->ifs_t3_th_l_mask,
-			       env->ifs_clm_th_l[2]);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t4_addr, ccx->ifs_t4_th_l_mask,
-			       env->ifs_clm_th_l[3]);
-
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t1_addr, ccx->ifs_t1_th_h_mask,
-			       env->ifs_clm_th_h[0]);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t2_addr, ccx->ifs_t2_th_h_mask,
-			       env->ifs_clm_th_h[1]);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t3_addr, ccx->ifs_t3_th_h_mask,
-			       env->ifs_clm_th_h[2]);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t4_addr, ccx->ifs_t4_th_h_mask,
-			       env->ifs_clm_th_h[3]);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t1_addr, ccx->ifs_t1_th_l_mask,
+			      env->ifs_clm_th_l[0], bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t2_addr, ccx->ifs_t2_th_l_mask,
+			      env->ifs_clm_th_l[1], bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t3_addr, ccx->ifs_t3_th_l_mask,
+			      env->ifs_clm_th_l[2], bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t4_addr, ccx->ifs_t4_th_l_mask,
+			      env->ifs_clm_th_l[3], bb->phy_idx);
+
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t1_addr, ccx->ifs_t1_th_h_mask,
+			      env->ifs_clm_th_h[0], bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t2_addr, ccx->ifs_t2_th_h_mask,
+			      env->ifs_clm_th_h[1], bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t3_addr, ccx->ifs_t3_th_h_mask,
+			      env->ifs_clm_th_h[2], bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t4_addr, ccx->ifs_t4_th_h_mask,
+			      env->ifs_clm_th_h[3], bb->phy_idx);
 
 	for (i = 0; i < RTW89_IFS_CLM_NUM; i++)
 		rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK,
@@ -5400,31 +5525,38 @@ static void rtw89_phy_ifs_clm_set_th_reg(struct rtw89_dev *rtwdev)
 			    i + 1, env->ifs_clm_th_l[i], env->ifs_clm_th_h[i]);
 }
 
-static void rtw89_phy_ifs_clm_setting_init(struct rtw89_dev *rtwdev)
+static void rtw89_phy_ifs_clm_setting_init(struct rtw89_dev *rtwdev,
+					   struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_phy_gen_def *phy = rtwdev->chip->phy_def;
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	const struct rtw89_ccx_regs *ccx = phy->ccx;
-	struct rtw89_ccx_para_info para = {0};
+	struct rtw89_ccx_para_info para = {};
 
 	env->ifs_clm_app = RTW89_IFS_CLM_BACKGROUND;
 	env->ifs_clm_mntr_time = 0;
 
 	para.ifs_clm_app = RTW89_IFS_CLM_INIT;
-	if (rtw89_phy_ifs_clm_th_update_check(rtwdev, &para))
-		rtw89_phy_ifs_clm_set_th_reg(rtwdev);
+	if (rtw89_phy_ifs_clm_th_update_check(rtwdev, bb, &para))
+		rtw89_phy_ifs_clm_set_th_reg(rtwdev, bb);
 
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_cnt_addr, ccx->ifs_collect_en_mask, true);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t1_addr, ccx->ifs_t1_en_mask, true);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t2_addr, ccx->ifs_t2_en_mask, true);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t3_addr, ccx->ifs_t3_en_mask, true);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_t4_addr, ccx->ifs_t4_en_mask, true);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_cnt_addr, ccx->ifs_collect_en_mask, true,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t1_addr, ccx->ifs_t1_en_mask, true,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t2_addr, ccx->ifs_t2_en_mask, true,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t3_addr, ccx->ifs_t3_en_mask, true,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_t4_addr, ccx->ifs_t4_en_mask, true,
+			      bb->phy_idx);
 }
 
 static int rtw89_phy_ccx_racing_ctrl(struct rtw89_dev *rtwdev,
+				     struct rtw89_bb_ctx *bb,
 				     enum rtw89_env_racing_lv level)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	int ret = 0;
 
 	if (level >= RTW89_RAC_MAX_NUM) {
@@ -5453,56 +5585,62 @@ static int rtw89_phy_ccx_racing_ctrl(struct rtw89_dev *rtwdev,
 	return ret;
 }
 
-static void rtw89_phy_ccx_trigger(struct rtw89_dev *rtwdev)
+static void rtw89_phy_ccx_trigger(struct rtw89_dev *rtwdev,
+				  struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_phy_gen_def *phy = rtwdev->chip->phy_def;
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	const struct rtw89_ccx_regs *ccx = phy->ccx;
 
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_cnt_addr, ccx->ifs_clm_cnt_clear_mask, 0);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->setting_addr, ccx->measurement_trig_mask, 0);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_cnt_addr, ccx->ifs_clm_cnt_clear_mask, 1);
-	rtw89_phy_set_phy_regs(rtwdev, ccx->setting_addr, ccx->measurement_trig_mask, 1);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_cnt_addr, ccx->ifs_clm_cnt_clear_mask, 0,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->setting_addr, ccx->measurement_trig_mask, 0,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->ifs_cnt_addr, ccx->ifs_clm_cnt_clear_mask, 1,
+			      bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, ccx->setting_addr, ccx->measurement_trig_mask, 1,
+			      bb->phy_idx);
 
 	env->ccx_ongoing = true;
 }
 
-static void rtw89_phy_ifs_clm_get_utility(struct rtw89_dev *rtwdev)
+static void rtw89_phy_ifs_clm_get_utility(struct rtw89_dev *rtwdev,
+					  struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	u8 i = 0;
 	u32 res = 0;
 
 	env->ifs_clm_tx_ratio =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_tx, PERCENT);
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_tx, PERCENT);
 	env->ifs_clm_edcca_excl_cca_ratio =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_edcca_excl_cca,
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_edcca_excl_cca,
 					 PERCENT);
 	env->ifs_clm_cck_fa_ratio =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_cckfa, PERCENT);
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckfa, PERCENT);
 	env->ifs_clm_ofdm_fa_ratio =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_ofdmfa, PERCENT);
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmfa, PERCENT);
 	env->ifs_clm_cck_cca_excl_fa_ratio =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_cckcca_excl_fa,
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckcca_excl_fa,
 					 PERCENT);
 	env->ifs_clm_ofdm_cca_excl_fa_ratio =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_ofdmcca_excl_fa,
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmcca_excl_fa,
 					 PERCENT);
 	env->ifs_clm_cck_fa_permil =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_cckfa, PERMIL);
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckfa, PERMIL);
 	env->ifs_clm_ofdm_fa_permil =
-		rtw89_phy_ccx_get_report(rtwdev, env->ifs_clm_ofdmfa, PERMIL);
+		rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmfa, PERMIL);
 
 	for (i = 0; i < RTW89_IFS_CLM_NUM; i++) {
 		if (env->ifs_clm_his[i] > ENV_MNTR_IFSCLM_HIS_MAX) {
 			env->ifs_clm_ifs_avg[i] = ENV_MNTR_FAIL_DWORD;
 		} else {
 			env->ifs_clm_ifs_avg[i] =
-				rtw89_phy_ccx_idx_to_us(rtwdev,
+				rtw89_phy_ccx_idx_to_us(rtwdev, bb,
 							env->ifs_clm_avg[i]);
 		}
 
-		res = rtw89_phy_ccx_idx_to_us(rtwdev, env->ifs_clm_cca[i]);
+		res = rtw89_phy_ccx_idx_to_us(rtwdev, bb, env->ifs_clm_cca[i]);
 		res += env->ifs_clm_his[i] >> 1;
 		if (env->ifs_clm_his[i])
 			res /= env->ifs_clm_his[i];
@@ -5532,81 +5670,82 @@ static void rtw89_phy_ifs_clm_get_utility(struct rtw89_dev *rtwdev)
 			    env->ifs_clm_cca_avg[i]);
 }
 
-static bool rtw89_phy_ifs_clm_get_result(struct rtw89_dev *rtwdev)
+static bool rtw89_phy_ifs_clm_get_result(struct rtw89_dev *rtwdev,
+					 struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_phy_gen_def *phy = rtwdev->chip->phy_def;
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	const struct rtw89_ccx_regs *ccx = phy->ccx;
 	u8 i = 0;
 
-	if (rtw89_phy_read32_mask(rtwdev, ccx->ifs_total_addr,
-				  ccx->ifs_cnt_done_mask) == 0) {
+	if (rtw89_phy_read32_idx(rtwdev, ccx->ifs_total_addr,
+				 ccx->ifs_cnt_done_mask, bb->phy_idx) == 0) {
 		rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK,
 			    "Get IFS_CLM report Fail\n");
 		return false;
 	}
 
 	env->ifs_clm_tx =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_clm_tx_cnt_addr,
-				      ccx->ifs_clm_tx_cnt_msk);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_clm_tx_cnt_addr,
+				     ccx->ifs_clm_tx_cnt_msk, bb->phy_idx);
 	env->ifs_clm_edcca_excl_cca =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_clm_tx_cnt_addr,
-				      ccx->ifs_clm_edcca_excl_cca_fa_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_clm_tx_cnt_addr,
+				     ccx->ifs_clm_edcca_excl_cca_fa_mask, bb->phy_idx);
 	env->ifs_clm_cckcca_excl_fa =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_clm_cca_addr,
-				      ccx->ifs_clm_cckcca_excl_fa_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_clm_cca_addr,
+				     ccx->ifs_clm_cckcca_excl_fa_mask, bb->phy_idx);
 	env->ifs_clm_ofdmcca_excl_fa =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_clm_cca_addr,
-				      ccx->ifs_clm_ofdmcca_excl_fa_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_clm_cca_addr,
+				     ccx->ifs_clm_ofdmcca_excl_fa_mask, bb->phy_idx);
 	env->ifs_clm_cckfa =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_clm_fa_addr,
-				      ccx->ifs_clm_cck_fa_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_clm_fa_addr,
+				     ccx->ifs_clm_cck_fa_mask, bb->phy_idx);
 	env->ifs_clm_ofdmfa =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_clm_fa_addr,
-				      ccx->ifs_clm_ofdm_fa_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_clm_fa_addr,
+				     ccx->ifs_clm_ofdm_fa_mask, bb->phy_idx);
 
 	env->ifs_clm_his[0] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_his_addr,
-				      ccx->ifs_t1_his_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_his_addr,
+				     ccx->ifs_t1_his_mask, bb->phy_idx);
 	env->ifs_clm_his[1] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_his_addr,
-				      ccx->ifs_t2_his_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_his_addr,
+				     ccx->ifs_t2_his_mask, bb->phy_idx);
 	env->ifs_clm_his[2] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_his_addr,
-				      ccx->ifs_t3_his_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_his_addr,
+				     ccx->ifs_t3_his_mask, bb->phy_idx);
 	env->ifs_clm_his[3] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_his_addr,
-				      ccx->ifs_t4_his_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_his_addr,
+				     ccx->ifs_t4_his_mask, bb->phy_idx);
 
 	env->ifs_clm_avg[0] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_avg_l_addr,
-				      ccx->ifs_t1_avg_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_avg_l_addr,
+				     ccx->ifs_t1_avg_mask, bb->phy_idx);
 	env->ifs_clm_avg[1] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_avg_l_addr,
-				      ccx->ifs_t2_avg_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_avg_l_addr,
+				     ccx->ifs_t2_avg_mask, bb->phy_idx);
 	env->ifs_clm_avg[2] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_avg_h_addr,
-				      ccx->ifs_t3_avg_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_avg_h_addr,
+				     ccx->ifs_t3_avg_mask, bb->phy_idx);
 	env->ifs_clm_avg[3] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_avg_h_addr,
-				      ccx->ifs_t4_avg_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_avg_h_addr,
+				     ccx->ifs_t4_avg_mask, bb->phy_idx);
 
 	env->ifs_clm_cca[0] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_cca_l_addr,
-				      ccx->ifs_t1_cca_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_cca_l_addr,
+				     ccx->ifs_t1_cca_mask, bb->phy_idx);
 	env->ifs_clm_cca[1] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_cca_l_addr,
-				      ccx->ifs_t2_cca_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_cca_l_addr,
+				     ccx->ifs_t2_cca_mask, bb->phy_idx);
 	env->ifs_clm_cca[2] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_cca_h_addr,
-				      ccx->ifs_t3_cca_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_cca_h_addr,
+				     ccx->ifs_t3_cca_mask, bb->phy_idx);
 	env->ifs_clm_cca[3] =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_cca_h_addr,
-				      ccx->ifs_t4_cca_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_cca_h_addr,
+				     ccx->ifs_t4_cca_mask, bb->phy_idx);
 
 	env->ifs_clm_total_ifs =
-		rtw89_phy_read32_mask(rtwdev, ccx->ifs_total_addr,
-				      ccx->ifs_total_mask);
+		rtw89_phy_read32_idx(rtwdev, ccx->ifs_total_addr,
+				     ccx->ifs_total_mask, bb->phy_idx);
 
 	rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK, "IFS-CLM total_ifs = %d\n",
 		    env->ifs_clm_total_ifs);
@@ -5626,16 +5765,17 @@ static bool rtw89_phy_ifs_clm_get_result(struct rtw89_dev *rtwdev)
 			    "T%d:[%d, %d, %d]\n", i + 1, env->ifs_clm_his[i],
 			    env->ifs_clm_avg[i], env->ifs_clm_cca[i]);
 
-	rtw89_phy_ifs_clm_get_utility(rtwdev);
+	rtw89_phy_ifs_clm_get_utility(rtwdev, bb);
 
 	return true;
 }
 
 static int rtw89_phy_ifs_clm_set(struct rtw89_dev *rtwdev,
+				 struct rtw89_bb_ctx *bb,
 				 struct rtw89_ccx_para_info *para)
 {
 	const struct rtw89_phy_gen_def *phy = rtwdev->chip->phy_def;
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	const struct rtw89_ccx_regs *ccx = phy->ccx;
 	u32 period = 0;
 	u32 unit_idx = 0;
@@ -5646,17 +5786,17 @@ static int rtw89_phy_ifs_clm_set(struct rtw89_dev *rtwdev,
 		return -EINVAL;
 	}
 
-	if (rtw89_phy_ccx_racing_ctrl(rtwdev, para->rac_lv))
+	if (rtw89_phy_ccx_racing_ctrl(rtwdev, bb, para->rac_lv))
 		return -EINVAL;
 
 	if (para->mntr_time != env->ifs_clm_mntr_time) {
 		rtw89_phy_ccx_ms_to_period_unit(rtwdev, para->mntr_time,
 						&period, &unit_idx);
-		rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_cnt_addr,
-				       ccx->ifs_clm_period_mask, period);
-		rtw89_phy_set_phy_regs(rtwdev, ccx->ifs_cnt_addr,
-				       ccx->ifs_clm_cnt_unit_mask,
-				       unit_idx);
+		rtw89_phy_write32_idx(rtwdev, ccx->ifs_cnt_addr,
+				      ccx->ifs_clm_period_mask, period, bb->phy_idx);
+		rtw89_phy_write32_idx(rtwdev, ccx->ifs_cnt_addr,
+				      ccx->ifs_clm_cnt_unit_mask,
+				      unit_idx, bb->phy_idx);
 
 		rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK,
 			    "Update IFS-CLM time ((%d)) -> ((%d))\n",
@@ -5667,18 +5807,19 @@ static int rtw89_phy_ifs_clm_set(struct rtw89_dev *rtwdev,
 		env->ccx_unit_idx = (u8)unit_idx;
 	}
 
-	if (rtw89_phy_ifs_clm_th_update_check(rtwdev, para)) {
+	if (rtw89_phy_ifs_clm_th_update_check(rtwdev, bb, para)) {
 		env->ifs_clm_app = para->ifs_clm_app;
-		rtw89_phy_ifs_clm_set_th_reg(rtwdev);
+		rtw89_phy_ifs_clm_set_th_reg(rtwdev, bb);
 	}
 
 	return 0;
 }
 
-void rtw89_phy_env_monitor_track(struct rtw89_dev *rtwdev)
+static void __rtw89_phy_env_monitor_track(struct rtw89_dev *rtwdev,
+					  struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
-	struct rtw89_ccx_para_info para = {0};
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
+	struct rtw89_ccx_para_info para = {};
 	u8 chk_result = RTW89_PHY_ENV_MON_CCX_FAIL;
 
 	env->ccx_watchdog_result = RTW89_PHY_ENV_MON_CCX_FAIL;
@@ -5688,33 +5829,50 @@ void rtw89_phy_env_monitor_track(struct rtw89_dev *rtwdev)
 		return;
 	}
 
+	rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK,
+		    "BB-%d env_monitor track\n", bb->phy_idx);
+
 	/* only ifs_clm for now */
-	if (rtw89_phy_ifs_clm_get_result(rtwdev))
+	if (rtw89_phy_ifs_clm_get_result(rtwdev, bb))
 		env->ccx_watchdog_result |= RTW89_PHY_ENV_MON_IFS_CLM;
 
-	rtw89_phy_ccx_racing_release(rtwdev);
+	rtw89_phy_ccx_racing_release(rtwdev, bb);
 	para.mntr_time = 1900;
 	para.rac_lv = RTW89_RAC_LV_1;
 	para.ifs_clm_app = RTW89_IFS_CLM_BACKGROUND;
 
-	if (rtw89_phy_ifs_clm_set(rtwdev, &para) == 0)
+	if (rtw89_phy_ifs_clm_set(rtwdev, bb, &para) == 0)
 		chk_result |= RTW89_PHY_ENV_MON_IFS_CLM;
 	if (chk_result)
-		rtw89_phy_ccx_trigger(rtwdev);
+		rtw89_phy_ccx_trigger(rtwdev, bb);
 
 	rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK,
 		    "get_result=0x%x, chk_result:0x%x\n",
 		    env->ccx_watchdog_result, chk_result);
 }
 
-static bool rtw89_physts_ie_page_valid(enum rtw89_phy_status_bitmap *ie_page)
+void rtw89_phy_env_monitor_track(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_bb_ctx *bb;
+
+	rtw89_for_each_active_bb(rtwdev, bb)
+		__rtw89_phy_env_monitor_track(rtwdev, bb);
+}
+
+static bool rtw89_physts_ie_page_valid(struct rtw89_dev *rtwdev,
+				       enum rtw89_phy_status_bitmap *ie_page)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
 	if (*ie_page >= RTW89_PHYSTS_BITMAP_NUM ||
 	    *ie_page == RTW89_RSVD_9)
 		return false;
-	else if (*ie_page > RTW89_RSVD_9)
+	else if (*ie_page > RTW89_RSVD_9 && *ie_page < RTW89_EHT_PKT)
 		*ie_page -= 1;
 
+	if (*ie_page == RTW89_EHT_PKT && chip->chip_gen == RTW89_CHIP_AX)
+		return false;
+
 	return true;
 }
 
@@ -5722,6 +5880,9 @@ static u32 rtw89_phy_get_ie_bitmap_addr(enum rtw89_phy_status_bitmap ie_page)
 {
 	static const u8 ie_page_shift = 2;
 
+	if (ie_page == RTW89_EHT_PKT)
+		return R_PHY_STS_BITMAP_EHT;
+
 	return R_PHY_STS_BITMAP_ADDR_START + (ie_page << ie_page_shift);
 }
 
@@ -5731,7 +5892,7 @@ static u32 rtw89_physts_get_ie_bitmap(struct rtw89_dev *rtwdev,
 {
 	u32 addr;
 
-	if (!rtw89_physts_ie_page_valid(&ie_page))
+	if (!rtw89_physts_ie_page_valid(rtwdev, &ie_page))
 		return 0;
 
 	addr = rtw89_phy_get_ie_bitmap_addr(ie_page);
@@ -5746,7 +5907,7 @@ static void rtw89_physts_set_ie_bitmap(struct rtw89_dev *rtwdev,
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	u32 addr;
 
-	if (!rtw89_physts_ie_page_valid(&ie_page))
+	if (!rtw89_physts_ie_page_valid(rtwdev, &ie_page))
 		return;
 
 	if (chip->chip_id == RTL8852A)
@@ -5756,21 +5917,6 @@ static void rtw89_physts_set_ie_bitmap(struct rtw89_dev *rtwdev,
 	rtw89_phy_write32_idx(rtwdev, addr, MASKDWORD, val, phy_idx);
 }
 
-static void rtw89_physts_enable_ie_bitmap(struct rtw89_dev *rtwdev,
-					  enum rtw89_phy_status_bitmap bitmap,
-					  enum rtw89_phy_status_ie_type ie,
-					  bool enable, enum rtw89_phy_idx phy_idx)
-{
-	u32 val = rtw89_physts_get_ie_bitmap(rtwdev, bitmap, phy_idx);
-
-	if (enable)
-		val |= BIT(ie);
-	else
-		val &= ~BIT(ie);
-
-	rtw89_physts_set_ie_bitmap(rtwdev, bitmap, val, phy_idx);
-}
-
 static void rtw89_physts_enable_fail_report(struct rtw89_dev *rtwdev,
 					    bool enable,
 					    enum rtw89_phy_idx phy_idx)
@@ -5794,30 +5940,37 @@ static void rtw89_physts_enable_fail_report(struct rtw89_dev *rtwdev,
 static void __rtw89_physts_parsing_init(struct rtw89_dev *rtwdev,
 					enum rtw89_phy_idx phy_idx)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	u32 val;
 	u8 i;
 
 	rtw89_physts_enable_fail_report(rtwdev, false, phy_idx);
 
 	for (i = 0; i < RTW89_PHYSTS_BITMAP_NUM; i++) {
-		if (i >= RTW89_CCK_PKT)
-			rtw89_physts_enable_ie_bitmap(rtwdev, i,
-						      RTW89_PHYSTS_IE09_FTR_0,
-						      true, phy_idx);
-		if ((i >= RTW89_CCK_BRK && i <= RTW89_VHT_MU) ||
-		    (i >= RTW89_RSVD_9 && i <= RTW89_CCK_PKT))
+		if (i == RTW89_RSVD_9 ||
+		    (i == RTW89_EHT_PKT && chip->chip_gen == RTW89_CHIP_AX))
 			continue;
-		rtw89_physts_enable_ie_bitmap(rtwdev, i,
-					      RTW89_PHYSTS_IE24_OFDM_TD_PATH_A,
-					      true, phy_idx);
-	}
-	rtw89_physts_enable_ie_bitmap(rtwdev, RTW89_VHT_PKT,
-				      RTW89_PHYSTS_IE13_DL_MU_DEF, true, phy_idx);
-	rtw89_physts_enable_ie_bitmap(rtwdev, RTW89_HE_PKT,
-				      RTW89_PHYSTS_IE13_DL_MU_DEF, true, phy_idx);
 
-	/* force IE01 for channel index, only channel field is valid */
-	rtw89_physts_enable_ie_bitmap(rtwdev, RTW89_CCK_PKT,
-				      RTW89_PHYSTS_IE01_CMN_OFDM, true, phy_idx);
+		val = rtw89_physts_get_ie_bitmap(rtwdev, i, phy_idx);
+		if (i == RTW89_HE_MU || i == RTW89_VHT_MU) {
+			val |= BIT(RTW89_PHYSTS_IE13_DL_MU_DEF);
+		} else if (i == RTW89_TRIG_BASE_PPDU) {
+			val |= BIT(RTW89_PHYSTS_IE13_DL_MU_DEF) |
+			       BIT(RTW89_PHYSTS_IE01_CMN_OFDM);
+		} else if (i >= RTW89_CCK_PKT) {
+			val |= BIT(RTW89_PHYSTS_IE09_FTR_0);
+
+			val &= ~(GENMASK(RTW89_PHYSTS_IE07_CMN_EXT_PATH_D,
+					 RTW89_PHYSTS_IE04_CMN_EXT_PATH_A));
+
+			if (i == RTW89_CCK_PKT)
+				val |= BIT(RTW89_PHYSTS_IE01_CMN_OFDM);
+			else if (i >= RTW89_HT_PKT)
+				val |= BIT(RTW89_PHYSTS_IE20_DBG_OFDM_FD_USER_SEG_0);
+		}
+
+		rtw89_physts_set_ie_bitmap(rtwdev, i, val, phy_idx);
+	}
 }
 
 static void rtw89_physts_parsing_init(struct rtw89_dev *rtwdev)
@@ -5827,11 +5980,12 @@ static void rtw89_physts_parsing_init(struct rtw89_dev *rtwdev)
 		__rtw89_physts_parsing_init(rtwdev, RTW89_PHY_1);
 }
 
-static void rtw89_phy_dig_read_gain_table(struct rtw89_dev *rtwdev, int type)
+static void rtw89_phy_dig_read_gain_table(struct rtw89_dev *rtwdev,
+					  struct rtw89_bb_ctx *bb, int type)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	struct rtw89_dig_info *dig = &rtwdev->dig;
 	const struct rtw89_phy_dig_gain_cfg *cfg;
+	struct rtw89_dig_info *dig = &bb->dig;
 	const char *msg;
 	u8 i;
 	s8 gain_base;
@@ -5868,8 +6022,8 @@ static void rtw89_phy_dig_read_gain_table(struct rtw89_dev *rtwdev, int type)
 	}
 
 	for (i = 0; i < cfg->size; i++) {
-		tmp = rtw89_phy_read32_mask(rtwdev, cfg->table[i].addr,
-					    cfg->table[i].mask);
+		tmp = rtw89_phy_read32_idx(rtwdev, cfg->table[i].addr,
+					   cfg->table[i].mask, bb->phy_idx);
 		tmp >>= DIG_GAIN_SHIFT;
 		gain_arr[i] = sign_extend32(tmp, U4_MAX_BIT) + gain_base;
 		gain_base += DIG_GAIN;
@@ -5879,25 +6033,26 @@ static void rtw89_phy_dig_read_gain_table(struct rtw89_dev *rtwdev, int type)
 	}
 }
 
-static void rtw89_phy_dig_update_gain_para(struct rtw89_dev *rtwdev)
+static void rtw89_phy_dig_update_gain_para(struct rtw89_dev *rtwdev,
+					   struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_dig_info *dig = &bb->dig;
 	u32 tmp;
 	u8 i;
 
 	if (!rtwdev->hal.support_igi)
 		return;
 
-	tmp = rtw89_phy_read32_mask(rtwdev, R_PATH0_IB_PKPW,
-				    B_PATH0_IB_PKPW_MSK);
+	tmp = rtw89_phy_read32_idx(rtwdev, R_PATH0_IB_PKPW,
+				   B_PATH0_IB_PKPW_MSK, bb->phy_idx);
 	dig->ib_pkpwr = sign_extend32(tmp >> DIG_GAIN_SHIFT, U8_MAX_BIT);
-	dig->ib_pbk = rtw89_phy_read32_mask(rtwdev, R_PATH0_IB_PBK,
-					    B_PATH0_IB_PBK_MSK);
+	dig->ib_pbk = rtw89_phy_read32_idx(rtwdev, R_PATH0_IB_PBK,
+					   B_PATH0_IB_PBK_MSK, bb->phy_idx);
 	rtw89_debug(rtwdev, RTW89_DBG_DIG, "ib_pkpwr=%d, ib_pbk=%d\n",
 		    dig->ib_pkpwr, dig->ib_pbk);
 
 	for (i = RTW89_DIG_GAIN_LNA_G; i < RTW89_DIG_GAIN_MAX; i++)
-		rtw89_phy_dig_read_gain_table(rtwdev, i);
+		rtw89_phy_dig_read_gain_table(rtwdev, bb, i);
 }
 
 static const u8 rssi_nolink = 22;
@@ -5906,10 +6061,11 @@ static const u16 fa_th_2g[FA_TH_NUM] = {22, 44, 66, 88};
 static const u16 fa_th_5g[FA_TH_NUM] = {4, 8, 12, 16};
 static const u16 fa_th_nolink[FA_TH_NUM] = {196, 352, 440, 528};
 
-static void rtw89_phy_dig_update_rssi_info(struct rtw89_dev *rtwdev)
+static void rtw89_phy_dig_update_rssi_info(struct rtw89_dev *rtwdev,
+					   struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_phy_ch_info *ch_info = &rtwdev->ch_info;
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_phy_ch_info *ch_info = &bb->ch_info;
+	struct rtw89_dig_info *dig = &bb->dig;
 	bool is_linked = rtwdev->total_sta_assoc > 0;
 
 	if (is_linked) {
@@ -5920,10 +6076,11 @@ static void rtw89_phy_dig_update_rssi_info(struct rtw89_dev *rtwdev)
 	}
 }
 
-static void rtw89_phy_dig_update_para(struct rtw89_dev *rtwdev)
+static void rtw89_phy_dig_update_para(struct rtw89_dev *rtwdev,
+				      struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
-	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0);
+	const struct rtw89_chan *chan = rtw89_mgnt_chan_get(rtwdev, bb->phy_idx);
+	struct rtw89_dig_info *dig = &bb->dig;
 	bool is_linked = rtwdev->total_sta_assoc > 0;
 	const u16 *fa_th_src = NULL;
 
@@ -5952,9 +6109,10 @@ static const u8 pd_low_th_offset = 16, dynamic_igi_min = 0x20;
 static const u8 igi_max_performance_mode = 0x5a;
 static const u8 dynamic_pd_threshold_max;
 
-static void rtw89_phy_dig_para_reset(struct rtw89_dev *rtwdev)
+static void rtw89_phy_dig_para_reset(struct rtw89_dev *rtwdev,
+				     struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_dig_info *dig = &bb->dig;
 
 	dig->cur_gaincode.lna_idx = LNA_IDX_MAX;
 	dig->cur_gaincode.tia_idx = TIA_IDX_MAX;
@@ -5970,15 +6128,27 @@ static void rtw89_phy_dig_para_reset(struct rtw89_dev *rtwdev)
 	dig->is_linked_pre = false;
 }
 
+static void __rtw89_phy_dig_init(struct rtw89_dev *rtwdev,
+				 struct rtw89_bb_ctx *bb)
+{
+	rtw89_debug(rtwdev, RTW89_DBG_DIG, "BB-%d dig_init\n", bb->phy_idx);
+
+	rtw89_phy_dig_update_gain_para(rtwdev, bb);
+	rtw89_phy_dig_reset(rtwdev, bb);
+}
+
 static void rtw89_phy_dig_init(struct rtw89_dev *rtwdev)
 {
-	rtw89_phy_dig_update_gain_para(rtwdev);
-	rtw89_phy_dig_reset(rtwdev);
+	struct rtw89_bb_ctx *bb;
+
+	rtw89_for_each_capab_bb(rtwdev, bb)
+		__rtw89_phy_dig_init(rtwdev, bb);
 }
 
-static u8 rtw89_phy_dig_lna_idx_by_rssi(struct rtw89_dev *rtwdev, u8 rssi)
+static u8 rtw89_phy_dig_lna_idx_by_rssi(struct rtw89_dev *rtwdev,
+					struct rtw89_bb_ctx *bb, u8 rssi)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_dig_info *dig = &bb->dig;
 	u8 lna_idx;
 
 	if (rssi < dig->igi_rssi_th[0])
@@ -5997,9 +6167,10 @@ static u8 rtw89_phy_dig_lna_idx_by_rssi(struct rtw89_dev *rtwdev, u8 rssi)
 	return lna_idx;
 }
 
-static u8 rtw89_phy_dig_tia_idx_by_rssi(struct rtw89_dev *rtwdev, u8 rssi)
+static u8 rtw89_phy_dig_tia_idx_by_rssi(struct rtw89_dev *rtwdev,
+					struct rtw89_bb_ctx *bb, u8 rssi)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_dig_info *dig = &bb->dig;
 	u8 tia_idx;
 
 	if (rssi < dig->igi_rssi_th[0])
@@ -6012,10 +6183,11 @@ static u8 rtw89_phy_dig_tia_idx_by_rssi(struct rtw89_dev *rtwdev, u8 rssi)
 
 #define IB_PBK_BASE 110
 #define WB_RSSI_BASE 10
-static u8 rtw89_phy_dig_rxb_idx_by_rssi(struct rtw89_dev *rtwdev, u8 rssi,
+static u8 rtw89_phy_dig_rxb_idx_by_rssi(struct rtw89_dev *rtwdev,
+					struct rtw89_bb_ctx *bb, u8 rssi,
 					struct rtw89_agc_gaincode_set *set)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_dig_info *dig = &bb->dig;
 	s8 lna_gain = dig->lna_gain[set->lna_idx];
 	s8 tia_gain = dig->tia_gain[set->tia_idx];
 	s32 wb_rssi = rssi + lna_gain + tia_gain;
@@ -6031,12 +6203,13 @@ static u8 rtw89_phy_dig_rxb_idx_by_rssi(struct rtw89_dev *rtwdev, u8 rssi,
 	return rxb_idx;
 }
 
-static void rtw89_phy_dig_gaincode_by_rssi(struct rtw89_dev *rtwdev, u8 rssi,
+static void rtw89_phy_dig_gaincode_by_rssi(struct rtw89_dev *rtwdev,
+					   struct rtw89_bb_ctx *bb, u8 rssi,
 					   struct rtw89_agc_gaincode_set *set)
 {
-	set->lna_idx = rtw89_phy_dig_lna_idx_by_rssi(rtwdev, rssi);
-	set->tia_idx = rtw89_phy_dig_tia_idx_by_rssi(rtwdev, rssi);
-	set->rxb_idx = rtw89_phy_dig_rxb_idx_by_rssi(rtwdev, rssi, set);
+	set->lna_idx = rtw89_phy_dig_lna_idx_by_rssi(rtwdev, bb, rssi);
+	set->tia_idx = rtw89_phy_dig_tia_idx_by_rssi(rtwdev, bb, rssi);
+	set->rxb_idx = rtw89_phy_dig_rxb_idx_by_rssi(rtwdev, bb, rssi, set);
 
 	rtw89_debug(rtwdev, RTW89_DBG_DIG,
 		    "final_rssi=%03d, (lna,tia,rab)=(%d,%d,%02d)\n",
@@ -6045,10 +6218,11 @@ static void rtw89_phy_dig_gaincode_by_rssi(struct rtw89_dev *rtwdev, u8 rssi,
 
 #define IGI_OFFSET_MAX 25
 #define IGI_OFFSET_MUL 2
-static void rtw89_phy_dig_igi_offset_by_env(struct rtw89_dev *rtwdev)
+static void rtw89_phy_dig_igi_offset_by_env(struct rtw89_dev *rtwdev,
+					    struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
+	struct rtw89_dig_info *dig = &bb->dig;
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
 	enum rtw89_dig_noisy_level noisy_lv;
 	u8 igi_offset = dig->fa_rssi_ofst;
 	u16 fa_ratio = 0;
@@ -6085,96 +6259,98 @@ static void rtw89_phy_dig_igi_offset_by_env(struct rtw89_dev *rtwdev)
 		    noisy_lv, igi_offset);
 }
 
-static void rtw89_phy_dig_set_lna_idx(struct rtw89_dev *rtwdev, u8 lna_idx)
+static void rtw89_phy_dig_set_lna_idx(struct rtw89_dev *rtwdev,
+				      struct rtw89_bb_ctx *bb, u8 lna_idx)
 {
 	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
 
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_lna_init.addr,
-			       dig_regs->p0_lna_init.mask, lna_idx);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_lna_init.addr,
-			       dig_regs->p1_lna_init.mask, lna_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p0_lna_init.addr,
+			      dig_regs->p0_lna_init.mask, lna_idx, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p1_lna_init.addr,
+			      dig_regs->p1_lna_init.mask, lna_idx, bb->phy_idx);
 }
 
-static void rtw89_phy_dig_set_tia_idx(struct rtw89_dev *rtwdev, u8 tia_idx)
+static void rtw89_phy_dig_set_tia_idx(struct rtw89_dev *rtwdev,
+				      struct rtw89_bb_ctx *bb, u8 tia_idx)
 {
 	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
 
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_tia_init.addr,
-			       dig_regs->p0_tia_init.mask, tia_idx);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_tia_init.addr,
-			       dig_regs->p1_tia_init.mask, tia_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p0_tia_init.addr,
+			      dig_regs->p0_tia_init.mask, tia_idx, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p1_tia_init.addr,
+			      dig_regs->p1_tia_init.mask, tia_idx, bb->phy_idx);
 }
 
-static void rtw89_phy_dig_set_rxb_idx(struct rtw89_dev *rtwdev, u8 rxb_idx)
+static void rtw89_phy_dig_set_rxb_idx(struct rtw89_dev *rtwdev,
+				      struct rtw89_bb_ctx *bb, u8 rxb_idx)
 {
 	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
 
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_rxb_init.addr,
-			       dig_regs->p0_rxb_init.mask, rxb_idx);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_rxb_init.addr,
-			       dig_regs->p1_rxb_init.mask, rxb_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p0_rxb_init.addr,
+			      dig_regs->p0_rxb_init.mask, rxb_idx, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p1_rxb_init.addr,
+			      dig_regs->p1_rxb_init.mask, rxb_idx, bb->phy_idx);
 }
 
 static void rtw89_phy_dig_set_igi_cr(struct rtw89_dev *rtwdev,
+				     struct rtw89_bb_ctx *bb,
 				     const struct rtw89_agc_gaincode_set set)
 {
 	if (!rtwdev->hal.support_igi)
 		return;
 
-	rtw89_phy_dig_set_lna_idx(rtwdev, set.lna_idx);
-	rtw89_phy_dig_set_tia_idx(rtwdev, set.tia_idx);
-	rtw89_phy_dig_set_rxb_idx(rtwdev, set.rxb_idx);
+	rtw89_phy_dig_set_lna_idx(rtwdev, bb, set.lna_idx);
+	rtw89_phy_dig_set_tia_idx(rtwdev, bb, set.tia_idx);
+	rtw89_phy_dig_set_rxb_idx(rtwdev, bb, set.rxb_idx);
 
 	rtw89_debug(rtwdev, RTW89_DBG_DIG, "Set (lna,tia,rxb)=((%d,%d,%02d))\n",
 		    set.lna_idx, set.tia_idx, set.rxb_idx);
 }
 
 static void rtw89_phy_dig_sdagc_follow_pagc_config(struct rtw89_dev *rtwdev,
+						   struct rtw89_bb_ctx *bb,
 						   bool enable)
 {
 	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
 
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_p20_pagcugc_en.addr,
-			       dig_regs->p0_p20_pagcugc_en.mask, enable);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_s20_pagcugc_en.addr,
-			       dig_regs->p0_s20_pagcugc_en.mask, enable);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_p20_pagcugc_en.addr,
-			       dig_regs->p1_p20_pagcugc_en.mask, enable);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_s20_pagcugc_en.addr,
-			       dig_regs->p1_s20_pagcugc_en.mask, enable);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p0_p20_pagcugc_en.addr,
+			      dig_regs->p0_p20_pagcugc_en.mask, enable, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p0_s20_pagcugc_en.addr,
+			      dig_regs->p0_s20_pagcugc_en.mask, enable, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p1_p20_pagcugc_en.addr,
+			      dig_regs->p1_p20_pagcugc_en.mask, enable, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->p1_s20_pagcugc_en.addr,
+			      dig_regs->p1_s20_pagcugc_en.mask, enable, bb->phy_idx);
 
 	rtw89_debug(rtwdev, RTW89_DBG_DIG, "sdagc_follow_pagc=%d\n", enable);
 }
 
-static void rtw89_phy_dig_config_igi(struct rtw89_dev *rtwdev)
+static void rtw89_phy_dig_config_igi(struct rtw89_dev *rtwdev,
+				     struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_dig_info *dig = &bb->dig;
 
 	if (!rtwdev->hal.support_igi)
 		return;
 
 	if (dig->force_gaincode_idx_en) {
-		rtw89_phy_dig_set_igi_cr(rtwdev, dig->force_gaincode);
+		rtw89_phy_dig_set_igi_cr(rtwdev, bb, dig->force_gaincode);
 		rtw89_debug(rtwdev, RTW89_DBG_DIG,
 			    "Force gaincode index enabled.\n");
 	} else {
-		rtw89_phy_dig_gaincode_by_rssi(rtwdev, dig->igi_fa_rssi,
+		rtw89_phy_dig_gaincode_by_rssi(rtwdev, bb, dig->igi_fa_rssi,
 					       &dig->cur_gaincode);
-		rtw89_phy_dig_set_igi_cr(rtwdev, dig->cur_gaincode);
+		rtw89_phy_dig_set_igi_cr(rtwdev, bb, dig->cur_gaincode);
 	}
 }
 
-static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi,
-				    bool enable)
+static u8 rtw89_phy_dig_cal_under_region(struct rtw89_dev *rtwdev,
+					 struct rtw89_bb_ctx *bb,
+					 const struct rtw89_chan *chan)
 {
-	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0);
-	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
 	enum rtw89_bandwidth cbw = chan->band_width;
-	struct rtw89_dig_info *dig = &rtwdev->dig;
-	u8 final_rssi = 0, under_region = dig->pd_low_th_ofst;
-	u8 ofdm_cca_th;
-	s8 cck_cca_th;
-	u32 pd_val = 0;
+	struct rtw89_dig_info *dig = &bb->dig;
+	u8 under_region = dig->pd_low_th_ofst;
 
 	if (rtwdev->chip->chip_gen == RTW89_CHIP_AX)
 		under_region += PD_TH_SB_FLTR_CMP_VAL;
@@ -6196,6 +6372,20 @@ static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi,
 		break;
 	}
 
+	return under_region;
+}
+
+static u32 __rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev,
+				     struct rtw89_bb_ctx *bb,
+				     u8 rssi, bool enable,
+				     const struct rtw89_chan *chan)
+{
+	struct rtw89_dig_info *dig = &bb->dig;
+	u8 ofdm_cca_th, under_region;
+	u8 final_rssi;
+	u32 pd_val;
+
+	under_region = rtw89_phy_dig_cal_under_region(rtwdev, bb, chan);
 	dig->dyn_pd_th_max = dig->igi_rssi;
 
 	final_rssi = min_t(u8, rssi, dig->igi_rssi);
@@ -6208,18 +6398,38 @@ static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi,
 			    "igi=%d, ofdm_ccaTH=%d, backoff=%d, PD_low=%d\n",
 			    final_rssi, ofdm_cca_th, under_region, pd_val);
 	} else {
+		pd_val = 0;
 		rtw89_debug(rtwdev, RTW89_DBG_DIG,
 			    "Dynamic PD th disabled, Set PD_low_bd=0\n");
 	}
 
-	rtw89_phy_write32_mask(rtwdev, dig_regs->seg0_pd_reg,
-			       dig_regs->pd_lower_bound_mask, pd_val);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->seg0_pd_reg,
-			       dig_regs->pd_spatial_reuse_en, enable);
+	return pd_val;
+}
+
+static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev,
+				    struct rtw89_bb_ctx *bb,
+				    u8 rssi, bool enable)
+{
+	const struct rtw89_chan *chan = rtw89_mgnt_chan_get(rtwdev, bb->phy_idx);
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
+	struct rtw89_dig_info *dig = &bb->dig;
+	u8 final_rssi, under_region = dig->pd_low_th_ofst;
+	s8 cck_cca_th;
+	u32 pd_val;
+
+	pd_val = __rtw89_phy_dig_dyn_pd_th(rtwdev, bb, rssi, enable, chan);
+	dig->bak_dig = pd_val;
+
+	rtw89_phy_write32_idx(rtwdev, dig_regs->seg0_pd_reg,
+			      dig_regs->pd_lower_bound_mask, pd_val, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->seg0_pd_reg,
+			      dig_regs->pd_spatial_reuse_en, enable, bb->phy_idx);
 
 	if (!rtwdev->hal.support_cckpd)
 		return;
 
+	final_rssi = min_t(u8, rssi, dig->igi_rssi);
+	under_region = rtw89_phy_dig_cal_under_region(rtwdev, bb, chan);
 	cck_cca_th = max_t(s8, final_rssi - under_region, CCKPD_TH_MIN_RSSI);
 	pd_val = (u32)(cck_cca_th - IGI_RSSI_MAX);
 
@@ -6227,77 +6437,234 @@ static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi,
 		    "igi=%d, cck_ccaTH=%d, backoff=%d, cck_PD_low=((%d))dB\n",
 		    final_rssi, cck_cca_th, under_region, pd_val);
 
-	rtw89_phy_write32_mask(rtwdev, dig_regs->bmode_pd_reg,
-			       dig_regs->bmode_cca_rssi_limit_en, enable);
-	rtw89_phy_write32_mask(rtwdev, dig_regs->bmode_pd_lower_bound_reg,
-			       dig_regs->bmode_rssi_nocca_low_th_mask, pd_val);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->bmode_pd_reg,
+			      dig_regs->bmode_cca_rssi_limit_en, enable, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->bmode_pd_lower_bound_reg,
+			      dig_regs->bmode_rssi_nocca_low_th_mask, pd_val, bb->phy_idx);
 }
 
-void rtw89_phy_dig_reset(struct rtw89_dev *rtwdev)
+void rtw89_phy_dig_reset(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
+	struct rtw89_dig_info *dig = &bb->dig;
 
 	dig->bypass_dig = false;
-	rtw89_phy_dig_para_reset(rtwdev);
-	rtw89_phy_dig_set_igi_cr(rtwdev, dig->force_gaincode);
-	rtw89_phy_dig_dyn_pd_th(rtwdev, rssi_nolink, false);
-	rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, false);
-	rtw89_phy_dig_update_para(rtwdev);
+	rtw89_phy_dig_para_reset(rtwdev, bb);
+	rtw89_phy_dig_set_igi_cr(rtwdev, bb, dig->force_gaincode);
+	rtw89_phy_dig_dyn_pd_th(rtwdev, bb, rssi_nolink, false);
+	rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, bb, false);
+	rtw89_phy_dig_update_para(rtwdev, bb);
 }
 
 #define IGI_RSSI_MIN 10
 #define ABS_IGI_MIN 0xc
-void rtw89_phy_dig(struct rtw89_dev *rtwdev)
+static
+void rtw89_phy_cal_igi_fa_rssi(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_dig_info *dig = &rtwdev->dig;
-	bool is_linked = rtwdev->total_sta_assoc > 0;
+	struct rtw89_dig_info *dig = &bb->dig;
 	u8 igi_min;
 
+	rtw89_phy_dig_igi_offset_by_env(rtwdev, bb);
+
+	igi_min = max_t(int, dig->igi_rssi - IGI_RSSI_MIN, 0);
+	dig->dyn_igi_max = min(igi_min + IGI_OFFSET_MAX, igi_max_performance_mode);
+	dig->dyn_igi_min = max(igi_min, ABS_IGI_MIN);
+
+	if (dig->dyn_igi_max >= dig->dyn_igi_min) {
+		dig->igi_fa_rssi += dig->fa_rssi_ofst;
+		dig->igi_fa_rssi = clamp(dig->igi_fa_rssi, dig->dyn_igi_min,
+					 dig->dyn_igi_max);
+	} else {
+		dig->igi_fa_rssi = dig->dyn_igi_max;
+	}
+}
+
+struct rtw89_phy_iter_mcc_dig {
+	struct rtw89_vif_link *rtwvif_link;
+	bool has_sta;
+	u8 rssi_min;
+};
+
+static void rtw89_phy_set_mcc_dig(struct rtw89_dev *rtwdev,
+				  struct rtw89_vif_link *rtwvif_link,
+				  struct rtw89_bb_ctx *bb,
+				  u8 rssi_min, u8 mcc_role_idx,
+				  bool is_linked)
+{
+	struct rtw89_dig_info *dig = &bb->dig;
+	const struct rtw89_chan *chan;
+	u8 pd_val;
+
+	if (is_linked) {
+		dig->igi_rssi = rssi_min >> 1;
+		dig->igi_fa_rssi = dig->igi_rssi;
+	} else {
+		rtw89_debug(rtwdev, RTW89_DBG_DIG, "RSSI update : NO Link\n");
+		dig->igi_rssi = rssi_nolink;
+		dig->igi_fa_rssi = dig->igi_rssi;
+	}
+
+	chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx);
+	rtw89_phy_cal_igi_fa_rssi(rtwdev, bb);
+	pd_val = __rtw89_phy_dig_dyn_pd_th(rtwdev, bb, dig->igi_fa_rssi,
+					   is_linked, chan);
+	rtw89_fw_h2c_mcc_dig(rtwdev, rtwvif_link->chanctx_idx,
+			     mcc_role_idx, pd_val, true);
+
+	rtw89_debug(rtwdev, RTW89_DBG_DIG,
+		    "MCC chanctx_idx %d chan %d rssi %d pd_val %d",
+		    rtwvif_link->chanctx_idx, chan->primary_channel,
+		    dig->igi_rssi, pd_val);
+}
+
+static void rtw89_phy_set_mcc_dig_iter(void *data, struct ieee80211_sta *sta)
+{
+	struct rtw89_phy_iter_mcc_dig *mcc_dig = (struct rtw89_phy_iter_mcc_dig *)data;
+	unsigned int link_id = mcc_dig->rtwvif_link->link_id;
+	struct rtw89_sta *rtwsta = sta_to_rtwsta(sta);
+	struct rtw89_sta_link *rtwsta_link;
+
+	if (rtwsta->rtwvif != mcc_dig->rtwvif_link->rtwvif)
+		return;
+
+	rtwsta_link = rtwsta->links[link_id];
+	if (!rtwsta_link)
+		return;
+
+	mcc_dig->has_sta = true;
+	if (ewma_rssi_read(&rtwsta_link->avg_rssi) < mcc_dig->rssi_min)
+		mcc_dig->rssi_min = ewma_rssi_read(&rtwsta_link->avg_rssi);
+}
+
+static void rtw89_phy_dig_mcc(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb)
+{
+	struct rtw89_phy_iter_mcc_dig mcc_dig;
+	struct rtw89_vif_link *rtwvif_link;
+	struct rtw89_mcc_links_info info;
+	int i;
+
+	rtw89_mcc_get_links(rtwdev, &info);
+	for (i = 0; i < ARRAY_SIZE(info.links); i++) {
+		rtwvif_link = info.links[i];
+		if (!rtwvif_link)
+			continue;
+
+		memset(&mcc_dig, 0, sizeof(mcc_dig));
+		mcc_dig.rtwvif_link = rtwvif_link;
+		mcc_dig.has_sta = false;
+		mcc_dig.rssi_min = U8_MAX;
+		ieee80211_iterate_stations_atomic(rtwdev->hw,
+						  rtw89_phy_set_mcc_dig_iter,
+						  &mcc_dig);
+
+		rtw89_phy_set_mcc_dig(rtwdev, rtwvif_link, bb,
+				      mcc_dig.rssi_min, i, mcc_dig.has_sta);
+	}
+}
+
+static void rtw89_phy_dig_ctrl(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb,
+			       bool pause_dig, bool restore)
+{
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
+	struct rtw89_dig_info *dig = &bb->dig;
+	bool en_dig;
+	u32 pd_val;
+
+	if (dig->pause_dig == pause_dig)
+		return;
+
+	if (pause_dig) {
+		en_dig = false;
+		pd_val = 0;
+	} else {
+		en_dig = rtwdev->total_sta_assoc > 0;
+		pd_val = restore ? dig->bak_dig : 0;
+	}
+
+	rtw89_debug(rtwdev, RTW89_DBG_DIG, "%s <%s> PD_low=%d", __func__,
+		    pause_dig ? "suspend" : "resume", pd_val);
+
+	rtw89_phy_write32_idx(rtwdev, dig_regs->seg0_pd_reg,
+			      dig_regs->pd_lower_bound_mask, pd_val, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, dig_regs->seg0_pd_reg,
+			      dig_regs->pd_spatial_reuse_en, en_dig, bb->phy_idx);
+
+	dig->pause_dig = pause_dig;
+}
+
+void rtw89_phy_dig_suspend(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_bb_ctx *bb;
+
+	rtw89_for_each_active_bb(rtwdev, bb)
+		rtw89_phy_dig_ctrl(rtwdev, bb, true, false);
+}
+
+void rtw89_phy_dig_resume(struct rtw89_dev *rtwdev, bool restore)
+{
+	struct rtw89_bb_ctx *bb;
+
+	rtw89_for_each_active_bb(rtwdev, bb)
+		rtw89_phy_dig_ctrl(rtwdev, bb, false, restore);
+}
+
+static void __rtw89_phy_dig(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb)
+{
+	struct rtw89_dig_info *dig = &bb->dig;
+	bool is_linked = rtwdev->total_sta_assoc > 0;
+	enum rtw89_entity_mode mode;
+
 	if (unlikely(dig->bypass_dig)) {
 		dig->bypass_dig = false;
 		return;
 	}
 
-	rtw89_phy_dig_update_rssi_info(rtwdev);
+	rtw89_debug(rtwdev, RTW89_DBG_DIG, "BB-%d dig track\n", bb->phy_idx);
+
+	rtw89_phy_dig_update_rssi_info(rtwdev, bb);
+
+	mode = rtw89_get_entity_mode(rtwdev);
+	if (mode == RTW89_ENTITY_MODE_MCC) {
+		rtw89_phy_dig_mcc(rtwdev, bb);
+		return;
+	}
+
+	if (unlikely(dig->pause_dig))
+		return;
 
 	if (!dig->is_linked_pre && is_linked) {
 		rtw89_debug(rtwdev, RTW89_DBG_DIG, "First connected\n");
-		rtw89_phy_dig_update_para(rtwdev);
+		rtw89_phy_dig_update_para(rtwdev, bb);
 		dig->igi_fa_rssi = dig->igi_rssi;
 	} else if (dig->is_linked_pre && !is_linked) {
 		rtw89_debug(rtwdev, RTW89_DBG_DIG, "First disconnected\n");
-		rtw89_phy_dig_update_para(rtwdev);
+		rtw89_phy_dig_update_para(rtwdev, bb);
 		dig->igi_fa_rssi = dig->igi_rssi;
 	}
 	dig->is_linked_pre = is_linked;
 
-	rtw89_phy_dig_igi_offset_by_env(rtwdev);
-
-	igi_min = max_t(int, dig->igi_rssi - IGI_RSSI_MIN, 0);
-	dig->dyn_igi_max = min(igi_min + IGI_OFFSET_MAX, igi_max_performance_mode);
-	dig->dyn_igi_min = max(igi_min, ABS_IGI_MIN);
-
-	if (dig->dyn_igi_max >= dig->dyn_igi_min) {
-		dig->igi_fa_rssi += dig->fa_rssi_ofst;
-		dig->igi_fa_rssi = clamp(dig->igi_fa_rssi, dig->dyn_igi_min,
-					 dig->dyn_igi_max);
-	} else {
-		dig->igi_fa_rssi = dig->dyn_igi_max;
-	}
+	rtw89_phy_cal_igi_fa_rssi(rtwdev, bb);
 
 	rtw89_debug(rtwdev, RTW89_DBG_DIG,
 		    "rssi=%03d, dyn_joint(max,min)=(%d,%d), final_rssi=%d\n",
 		    dig->igi_rssi, dig->dyn_igi_max, dig->dyn_igi_min,
 		    dig->igi_fa_rssi);
 
-	rtw89_phy_dig_config_igi(rtwdev);
+	rtw89_phy_dig_config_igi(rtwdev, bb);
 
-	rtw89_phy_dig_dyn_pd_th(rtwdev, dig->igi_fa_rssi, dig->dyn_pd_th_en);
+	rtw89_phy_dig_dyn_pd_th(rtwdev, bb, dig->igi_fa_rssi, dig->dyn_pd_th_en);
 
 	if (dig->dyn_pd_th_en && dig->igi_fa_rssi > dig->dyn_pd_th_max)
-		rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, true);
+		rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, bb, true);
 	else
-		rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, false);
+		rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, bb, false);
+}
+
+void rtw89_phy_dig(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_bb_ctx *bb;
+
+	rtw89_for_each_active_bb(rtwdev, bb)
+		__rtw89_phy_dig(rtwdev, bb);
 }
 
 static void __rtw89_phy_tx_path_div_sta_iter(struct rtw89_dev *rtwdev,
@@ -6471,17 +6838,17 @@ static void rtw89_phy_antdiv_training_state(struct rtw89_dev *rtwdev)
 	}
 
 	antdiv->training_count++;
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->antdiv_work,
-				     state_period);
+	wiphy_delayed_work_queue(rtwdev->hw->wiphy, &rtwdev->antdiv_work,
+				 state_period);
 }
 
-void rtw89_phy_antdiv_work(struct work_struct *work)
+void rtw89_phy_antdiv_work(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						antdiv_work.work);
 	struct rtw89_antdiv_info *antdiv = &rtwdev->antdiv;
 
-	mutex_lock(&rtwdev->mutex);
+	lockdep_assert_wiphy(wiphy);
 
 	if (antdiv->training_count <= ANTDIV_TRAINNING_CNT) {
 		rtw89_phy_antdiv_training_state(rtwdev);
@@ -6489,8 +6856,6 @@ void rtw89_phy_antdiv_work(struct work_struct *work)
 		rtw89_phy_antdiv_decision_state(rtwdev);
 		rtw89_phy_antdiv_set_ant(rtwdev);
 	}
-
-	mutex_unlock(&rtwdev->mutex);
 }
 
 void rtw89_phy_antdiv_track(struct rtw89_dev *rtwdev)
@@ -6511,19 +6876,34 @@ void rtw89_phy_antdiv_track(struct rtw89_dev *rtwdev)
 		return;
 
 	antdiv->training_count = 0;
-	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->antdiv_work, 0);
+	wiphy_delayed_work_queue(rtwdev->hw->wiphy, &rtwdev->antdiv_work, 0);
+}
+
+static void __rtw89_phy_env_monitor_init(struct rtw89_dev *rtwdev,
+					 struct rtw89_bb_ctx *bb)
+{
+	rtw89_debug(rtwdev, RTW89_DBG_PHY_TRACK,
+		    "BB-%d env_monitor init\n", bb->phy_idx);
+
+	rtw89_phy_ccx_top_setting_init(rtwdev, bb);
+	rtw89_phy_ifs_clm_setting_init(rtwdev, bb);
 }
 
 static void rtw89_phy_env_monitor_init(struct rtw89_dev *rtwdev)
 {
-	rtw89_phy_ccx_top_setting_init(rtwdev);
-	rtw89_phy_ifs_clm_setting_init(rtwdev);
+	struct rtw89_bb_ctx *bb;
+
+	rtw89_for_each_capab_bb(rtwdev, bb)
+		__rtw89_phy_env_monitor_init(rtwdev, bb);
 }
 
-static void rtw89_phy_edcca_init(struct rtw89_dev *rtwdev)
+static void __rtw89_phy_edcca_init(struct rtw89_dev *rtwdev,
+				   struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs;
-	struct rtw89_edcca_bak *edcca_bak = &rtwdev->hal.edcca_bak;
+	struct rtw89_edcca_bak *edcca_bak = &bb->edcca_bak;
+
+	rtw89_debug(rtwdev, RTW89_DBG_EDCCA, "BB-%d edcca init\n", bb->phy_idx);
 
 	memset(edcca_bak, 0, sizeof(*edcca_bak));
 
@@ -6539,8 +6919,16 @@ static void rtw89_phy_edcca_init(struct rtw89_dev *rtwdev)
 		rtw89_phy_set_phy_regs(rtwdev, R_DFS_FFT_CG, B_DFS_FFT_EN, 1);
 	}
 
-	rtw89_phy_write32_mask(rtwdev, edcca_regs->tx_collision_t2r_st,
-			       edcca_regs->tx_collision_t2r_st_mask, 0x29);
+	rtw89_phy_write32_idx(rtwdev, edcca_regs->tx_collision_t2r_st,
+			      edcca_regs->tx_collision_t2r_st_mask, 0x29, bb->phy_idx);
+}
+
+static void rtw89_phy_edcca_init(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_bb_ctx *bb;
+
+	rtw89_for_each_capab_bb(rtwdev, bb)
+		__rtw89_phy_edcca_init(rtwdev, bb);
 }
 
 void rtw89_phy_dm_init(struct rtw89_dev *rtwdev)
@@ -6907,60 +7295,67 @@ void rtw89_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx,
 }
 EXPORT_SYMBOL(rtw89_decode_chan_idx);
 
-void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev, bool scan)
+void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev,
+			    struct rtw89_bb_ctx *bb, bool scan)
 {
 	const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs;
-	struct rtw89_edcca_bak *edcca_bak = &rtwdev->hal.edcca_bak;
+	struct rtw89_edcca_bak *edcca_bak = &bb->edcca_bak;
 
 	if (scan) {
 		edcca_bak->a =
-			rtw89_phy_read32_mask(rtwdev, edcca_regs->edcca_level,
-					      edcca_regs->edcca_mask);
+			rtw89_phy_read32_idx(rtwdev, edcca_regs->edcca_level,
+					     edcca_regs->edcca_mask, bb->phy_idx);
 		edcca_bak->p =
-			rtw89_phy_read32_mask(rtwdev, edcca_regs->edcca_level,
-					      edcca_regs->edcca_p_mask);
+			rtw89_phy_read32_idx(rtwdev, edcca_regs->edcca_level,
+					     edcca_regs->edcca_p_mask, bb->phy_idx);
 		edcca_bak->ppdu =
-			rtw89_phy_read32_mask(rtwdev, edcca_regs->ppdu_level,
-					      edcca_regs->ppdu_mask);
-
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level,
-				       edcca_regs->edcca_mask, EDCCA_MAX);
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level,
-				       edcca_regs->edcca_p_mask, EDCCA_MAX);
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->ppdu_level,
-				       edcca_regs->ppdu_mask, EDCCA_MAX);
+			rtw89_phy_read32_idx(rtwdev, edcca_regs->ppdu_level,
+					     edcca_regs->ppdu_mask, bb->phy_idx);
+
+		rtw89_phy_write32_idx(rtwdev, edcca_regs->edcca_level,
+				      edcca_regs->edcca_mask, EDCCA_MAX, bb->phy_idx);
+		rtw89_phy_write32_idx(rtwdev, edcca_regs->edcca_level,
+				      edcca_regs->edcca_p_mask, EDCCA_MAX, bb->phy_idx);
+		rtw89_phy_write32_idx(rtwdev, edcca_regs->ppdu_level,
+				      edcca_regs->ppdu_mask, EDCCA_MAX, bb->phy_idx);
 	} else {
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level,
-				       edcca_regs->edcca_mask,
-				       edcca_bak->a);
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level,
-				       edcca_regs->edcca_p_mask,
-				       edcca_bak->p);
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->ppdu_level,
-				       edcca_regs->ppdu_mask,
-				       edcca_bak->ppdu);
+		rtw89_phy_write32_idx(rtwdev, edcca_regs->edcca_level,
+				      edcca_regs->edcca_mask,
+				      edcca_bak->a, bb->phy_idx);
+		rtw89_phy_write32_idx(rtwdev, edcca_regs->edcca_level,
+				      edcca_regs->edcca_p_mask,
+				      edcca_bak->p, bb->phy_idx);
+		rtw89_phy_write32_idx(rtwdev, edcca_regs->ppdu_level,
+				      edcca_regs->ppdu_mask,
+				      edcca_bak->ppdu, bb->phy_idx);
 	}
 }
 
-static void rtw89_phy_edcca_log(struct rtw89_dev *rtwdev)
+static void rtw89_phy_edcca_log(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs;
+	const struct rtw89_edcca_p_regs *edcca_p_regs;
 	bool flag_fb, flag_p20, flag_s20, flag_s40, flag_s80;
 	s8 pwdb_fb, pwdb_p20, pwdb_s20, pwdb_s40, pwdb_s80;
-	u8 path, per20_bitmap;
+	u8 path, per20_bitmap = 0;
 	u8 pwdb[8];
 	u32 tmp;
 
 	if (!rtw89_debug_is_enabled(rtwdev, RTW89_DBG_EDCCA))
 		return;
 
+	if (bb->phy_idx == RTW89_PHY_1)
+		edcca_p_regs = &edcca_regs->p[RTW89_PHY_1];
+	else
+		edcca_p_regs = &edcca_regs->p[RTW89_PHY_0];
+
 	if (rtwdev->chip->chip_id == RTL8922A)
 		rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel_be,
 				       edcca_regs->rpt_sel_be_mask, 0);
 
-	rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel,
-			       edcca_regs->rpt_sel_mask, 0);
-	tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b);
+	rtw89_phy_write32_mask(rtwdev, edcca_p_regs->rpt_sel,
+			       edcca_p_regs->rpt_sel_mask, 0);
+	tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_b);
 	path = u32_get_bits(tmp, B_EDCCA_RPT_B_PATH_MASK);
 	flag_s80 = u32_get_bits(tmp, B_EDCCA_RPT_B_S80);
 	flag_s40 = u32_get_bits(tmp, B_EDCCA_RPT_B_S40);
@@ -6971,53 +7366,52 @@ static void rtw89_phy_edcca_log(struct rtw89_dev *rtwdev)
 	pwdb_p20 = u32_get_bits(tmp, MASKBYTE2);
 	pwdb_fb = u32_get_bits(tmp, MASKBYTE3);
 
-	rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel,
-			       edcca_regs->rpt_sel_mask, 4);
-	tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b);
+	rtw89_phy_write32_mask(rtwdev, edcca_p_regs->rpt_sel,
+			       edcca_p_regs->rpt_sel_mask, 5);
+	tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_b);
 	pwdb_s80 = u32_get_bits(tmp, MASKBYTE1);
 	pwdb_s40 = u32_get_bits(tmp, MASKBYTE2);
 
-	per20_bitmap = rtw89_phy_read32_mask(rtwdev, edcca_regs->rpt_a,
-					     MASKBYTE0);
-
 	if (rtwdev->chip->chip_id == RTL8922A) {
 		rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel_be,
 				       edcca_regs->rpt_sel_be_mask, 4);
-		tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b);
+		tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_b);
 		pwdb[0] = u32_get_bits(tmp, MASKBYTE3);
 		pwdb[1] = u32_get_bits(tmp, MASKBYTE2);
 		pwdb[2] = u32_get_bits(tmp, MASKBYTE1);
 		pwdb[3] = u32_get_bits(tmp, MASKBYTE0);
+		per20_bitmap = rtw89_phy_read32_mask(rtwdev, edcca_p_regs->rpt_a,
+						     MASKBYTE0);
 
 		rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel_be,
 				       edcca_regs->rpt_sel_be_mask, 5);
-		tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b);
+		tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_b);
 		pwdb[4] = u32_get_bits(tmp, MASKBYTE3);
 		pwdb[5] = u32_get_bits(tmp, MASKBYTE2);
 		pwdb[6] = u32_get_bits(tmp, MASKBYTE1);
 		pwdb[7] = u32_get_bits(tmp, MASKBYTE0);
 	} else {
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel,
-				       edcca_regs->rpt_sel_mask, 0);
-		tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a);
+		rtw89_phy_write32_mask(rtwdev, edcca_p_regs->rpt_sel,
+				       edcca_p_regs->rpt_sel_mask, 0);
+		tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_a);
 		pwdb[0] = u32_get_bits(tmp, MASKBYTE3);
 		pwdb[1] = u32_get_bits(tmp, MASKBYTE2);
 
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel,
-				       edcca_regs->rpt_sel_mask, 1);
-		tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a);
+		rtw89_phy_write32_mask(rtwdev, edcca_p_regs->rpt_sel,
+				       edcca_p_regs->rpt_sel_mask, 5);
+		tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_a);
 		pwdb[2] = u32_get_bits(tmp, MASKBYTE3);
 		pwdb[3] = u32_get_bits(tmp, MASKBYTE2);
 
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel,
-				       edcca_regs->rpt_sel_mask, 2);
-		tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a);
+		rtw89_phy_write32_mask(rtwdev, edcca_p_regs->rpt_sel,
+				       edcca_p_regs->rpt_sel_mask, 2);
+		tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_a);
 		pwdb[4] = u32_get_bits(tmp, MASKBYTE3);
 		pwdb[5] = u32_get_bits(tmp, MASKBYTE2);
 
-		rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel,
-				       edcca_regs->rpt_sel_mask, 3);
-		tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a);
+		rtw89_phy_write32_mask(rtwdev, edcca_p_regs->rpt_sel,
+				       edcca_p_regs->rpt_sel_mask, 3);
+		tmp = rtw89_phy_read32(rtwdev, edcca_p_regs->rpt_a);
 		pwdb[6] = u32_get_bits(tmp, MASKBYTE3);
 		pwdb[7] = u32_get_bits(tmp, MASKBYTE2);
 	}
@@ -7039,9 +7433,10 @@ static void rtw89_phy_edcca_log(struct rtw89_dev *rtwdev)
 		    pwdb_fb, pwdb_p20, pwdb_s20, pwdb_s40, pwdb_s80);
 }
 
-static u8 rtw89_phy_edcca_get_thre_by_rssi(struct rtw89_dev *rtwdev)
+static u8 rtw89_phy_edcca_get_thre_by_rssi(struct rtw89_dev *rtwdev,
+					   struct rtw89_bb_ctx *bb)
 {
-	struct rtw89_phy_ch_info *ch_info = &rtwdev->ch_info;
+	struct rtw89_phy_ch_info *ch_info = &bb->ch_info;
 	bool is_linked = rtwdev->total_sta_assoc > 0;
 	u8 rssi_min = ch_info->rssi_min >> 1;
 	u8 edcca_thre;
@@ -7057,13 +7452,13 @@ static u8 rtw89_phy_edcca_get_thre_by_rssi(struct rtw89_dev *rtwdev)
 	return edcca_thre;
 }
 
-void rtw89_phy_edcca_thre_calc(struct rtw89_dev *rtwdev)
+void rtw89_phy_edcca_thre_calc(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb)
 {
 	const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs;
-	struct rtw89_edcca_bak *edcca_bak = &rtwdev->hal.edcca_bak;
+	struct rtw89_edcca_bak *edcca_bak = &bb->edcca_bak;
 	u8 th;
 
-	th = rtw89_phy_edcca_get_thre_by_rssi(rtwdev);
+	th = rtw89_phy_edcca_get_thre_by_rssi(rtwdev, bb);
 	if (th == edcca_bak->th_old)
 		return;
 
@@ -7072,23 +7467,33 @@ void rtw89_phy_edcca_thre_calc(struct rtw89_dev *rtwdev)
 	rtw89_debug(rtwdev, RTW89_DBG_EDCCA,
 		    "[EDCCA]: Normal Mode, EDCCA_th = %d\n", th);
 
-	rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level,
-			       edcca_regs->edcca_mask, th);
-	rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level,
-			       edcca_regs->edcca_p_mask, th);
-	rtw89_phy_write32_mask(rtwdev, edcca_regs->ppdu_level,
-			       edcca_regs->ppdu_mask, th);
+	rtw89_phy_write32_idx(rtwdev, edcca_regs->edcca_level,
+			      edcca_regs->edcca_mask, th, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, edcca_regs->edcca_level,
+			      edcca_regs->edcca_p_mask, th, bb->phy_idx);
+	rtw89_phy_write32_idx(rtwdev, edcca_regs->ppdu_level,
+			      edcca_regs->ppdu_mask, th, bb->phy_idx);
+}
+
+static
+void __rtw89_phy_edcca_track(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb)
+{
+	rtw89_debug(rtwdev, RTW89_DBG_EDCCA, "BB-%d edcca track\n", bb->phy_idx);
+
+	rtw89_phy_edcca_thre_calc(rtwdev, bb);
+	rtw89_phy_edcca_log(rtwdev, bb);
 }
 
 void rtw89_phy_edcca_track(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_hal *hal = &rtwdev->hal;
+	struct rtw89_bb_ctx *bb;
 
 	if (hal->disabled_dm_bitmap & BIT(RTW89_DM_DYNAMIC_EDCCA))
 		return;
 
-	rtw89_phy_edcca_thre_calc(rtwdev);
-	rtw89_phy_edcca_log(rtwdev);
+	rtw89_for_each_active_bb(rtwdev, bb)
+		__rtw89_phy_edcca_track(rtwdev, bb);
 }
 
 enum rtw89_rf_path_bit rtw89_phy_get_kpath(struct rtw89_dev *rtwdev,
diff --git a/sys/contrib/dev/rtw89/phy.h b/sys/contrib/dev/rtw89/phy.h
index 08b635c93ac3..dc156376d951 100644
--- a/sys/contrib/dev/rtw89/phy.h
+++ b/sys/contrib/dev/rtw89/phy.h
@@ -164,6 +164,7 @@ enum rtw89_phy_c2h_dm_func {
 	RTW89_PHY_C2H_DM_FUNC_SIGB,
 	RTW89_PHY_C2H_DM_FUNC_LOWRT_RTY,
 	RTW89_PHY_C2H_DM_FUNC_MCC_DIG,
+	RTW89_PHY_C2H_DM_FUNC_FW_SCAN = 0xc,
 	RTW89_PHY_C2H_DM_FUNC_NUM,
 };
 
@@ -251,6 +252,7 @@ enum rtw89_phy_status_bitmap {
 	RTW89_HT_PKT          = 13,
 	RTW89_VHT_PKT         = 14,
 	RTW89_HE_PKT          = 15,
+	RTW89_EHT_PKT         = 16,
 
 	RTW89_PHYSTS_BITMAP_NUM
 };
@@ -835,8 +837,8 @@ s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev, u8 band, u8 bw,
 void rtw89_phy_ant_gain_init(struct rtw89_dev *rtwdev);
 s16 rtw89_phy_ant_gain_pwr_offset(struct rtw89_dev *rtwdev,
 				  const struct rtw89_chan *chan);
-void rtw89_print_ant_gain(struct seq_file *m, struct rtw89_dev *rtwdev,
-			  const struct rtw89_chan *chan);
+int rtw89_print_ant_gain(struct rtw89_dev *rtwdev, char *buf, size_t bufsz,
+			 const struct rtw89_chan *chan);
 void rtw89_phy_load_txpwr_byrate(struct rtw89_dev *rtwdev,
 				 const struct rtw89_txpwr_table *tbl);
 s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
@@ -914,6 +916,13 @@ static inline s8 rtw89_phy_txpwr_rf_to_bb(struct rtw89_dev *rtwdev, s8 txpwr_rf)
 	return txpwr_rf << (chip->txpwr_factor_bb - chip->txpwr_factor_rf);
 }
 
+static inline s8 rtw89_phy_txpwr_bb_to_rf(struct rtw89_dev *rtwdev, s8 txpwr_bb)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	return txpwr_bb >> (chip->txpwr_factor_bb - chip->txpwr_factor_rf);
+}
+
 static inline s8 rtw89_phy_txpwr_rf_to_mac(struct rtw89_dev *rtwdev, s8 txpwr_rf)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
@@ -928,6 +937,20 @@ static inline s8 rtw89_phy_txpwr_dbm_to_mac(struct rtw89_dev *rtwdev, s8 dbm)
 	return clamp_t(s16, dbm << chip->txpwr_factor_mac, -64, 63);
 }
 
+static inline s16 rtw89_phy_txpwr_mac_to_rf(struct rtw89_dev *rtwdev, s8 txpwr_mac)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	return txpwr_mac << (chip->txpwr_factor_rf - chip->txpwr_factor_mac);
+}
+
+static inline s16 rtw89_phy_txpwr_mac_to_bb(struct rtw89_dev *rtwdev, s8 txpwr_mac)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	return txpwr_mac << (chip->txpwr_factor_bb - chip->txpwr_factor_mac);
+}
+
 void rtw89_phy_ra_assoc(struct rtw89_dev *rtwdev, struct rtw89_sta_link *rtwsta_link);
 void rtw89_phy_ra_update(struct rtw89_dev *rtwdev);
 void rtw89_phy_ra_update_sta(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta,
@@ -978,20 +1001,22 @@ void rtw89_phy_rfk_tssi_fill_fwcmd_tmeter_tbl(struct rtw89_dev *rtwdev,
 					      const struct rtw89_chan *chan,
 					      struct rtw89_h2c_rf_tssi *h2c);
 void rtw89_phy_cfo_track(struct rtw89_dev *rtwdev);
-void rtw89_phy_cfo_track_work(struct work_struct *work);
+void rtw89_phy_cfo_track_work(struct wiphy *wiphy, struct wiphy_work *work);
 void rtw89_phy_cfo_parse(struct rtw89_dev *rtwdev, s16 cfo_val,
 			 struct rtw89_rx_phy_ppdu *phy_ppdu);
 void rtw89_phy_stat_track(struct rtw89_dev *rtwdev);
 void rtw89_phy_env_monitor_track(struct rtw89_dev *rtwdev);
 void rtw89_phy_set_phy_regs(struct rtw89_dev *rtwdev, u32 addr, u32 mask,
 			    u32 val);
-void rtw89_phy_dig_reset(struct rtw89_dev *rtwdev);
+void rtw89_phy_dig_reset(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb);
 void rtw89_phy_dig(struct rtw89_dev *rtwdev);
+void rtw89_phy_dig_suspend(struct rtw89_dev *rtwdev);
+void rtw89_phy_dig_resume(struct rtw89_dev *rtwdev, bool restore);
 void rtw89_phy_tx_path_div_track(struct rtw89_dev *rtwdev);
 void rtw89_phy_antdiv_parse(struct rtw89_dev *rtwdev,
 			    struct rtw89_rx_phy_ppdu *phy_ppdu);
 void rtw89_phy_antdiv_track(struct rtw89_dev *rtwdev);
-void rtw89_phy_antdiv_work(struct work_struct *work);
+void rtw89_phy_antdiv_work(struct wiphy *wiphy, struct wiphy_work *work);
 void rtw89_phy_set_bss_color(struct rtw89_dev *rtwdev,
 			     struct rtw89_vif_link *rtwvif_link);
 void rtw89_phy_tssi_ctrl_set_bandedge_cfg(struct rtw89_dev *rtwdev,
@@ -1002,9 +1027,10 @@ void rtw89_phy_ul_tb_ctrl_track(struct rtw89_dev *rtwdev);
 u8 rtw89_encode_chan_idx(struct rtw89_dev *rtwdev, u8 central_ch, u8 band);
 void rtw89_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx,
 			   u8 *ch, enum nl80211_band *band);
-void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev, bool scan);
+void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev,
+			    struct rtw89_bb_ctx *bb, bool scan);
 void rtw89_phy_edcca_track(struct rtw89_dev *rtwdev);
-void rtw89_phy_edcca_thre_calc(struct rtw89_dev *rtwdev);
+void rtw89_phy_edcca_thre_calc(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb);
 enum rtw89_rf_path_bit rtw89_phy_get_kpath(struct rtw89_dev *rtwdev,
 					   enum rtw89_phy_idx phy_idx);
 enum rtw89_rf_path rtw89_phy_get_syn_sel(struct rtw89_dev *rtwdev,
diff --git a/sys/contrib/dev/rtw89/phy_be.c b/sys/contrib/dev/rtw89/phy_be.c
index 37d8f254ae32..d321cf1fc485 100644
--- a/sys/contrib/dev/rtw89/phy_be.c
+++ b/sys/contrib/dev/rtw89/phy_be.c
@@ -362,7 +362,7 @@ static void rtw89_phy_bb_wrap_force_cr_init(struct rtw89_dev *rtwdev,
 	rtw89_write32_mask(rtwdev, addr, B_BE_PWR_FORCE_RU_ENON, 0);
 	rtw89_write32_mask(rtwdev, addr, B_BE_PWR_FORCE_RU_ON, 0);
 	addr = rtw89_mac_reg_by_idx(rtwdev, R_BE_PWR_FORCE_MACID, mac_idx);
-	rtw89_write32_mask(rtwdev, addr, B_BE_PWR_FORCE_MACID_ON, 0);
+	rtw89_write32_mask(rtwdev, addr, B_BE_PWR_FORCE_MACID_ALL, 0);
 	addr = rtw89_mac_reg_by_idx(rtwdev, R_BE_PWR_COEX_CTRL, mac_idx);
 	rtw89_write32_mask(rtwdev, addr, B_BE_PWR_FORCE_COEX_ON, 0);
 	addr = rtw89_mac_reg_by_idx(rtwdev, R_BE_PWR_RATE_CTRL, mac_idx);
diff --git a/sys/contrib/dev/rtw89/ps.c b/sys/contrib/dev/rtw89/ps.c
index a47c6e7145d3..098726f91db7 100644
--- a/sys/contrib/dev/rtw89/ps.c
+++ b/sys/contrib/dev/rtw89/ps.c
@@ -13,6 +13,31 @@
 #include "reg.h"
 #include "util.h"
 
+static int rtw89_fw_receive_lps_h2c_check(struct rtw89_dev *rtwdev, u8 macid)
+{
+	struct rtw89_mac_c2h_info c2h_info = {};
+	u16 c2hreg_macid;
+	u32 c2hreg_ret;
+	int ret;
+
+	if (!RTW89_CHK_FW_FEATURE(LPS_DACK_BY_C2H_REG, &rtwdev->fw))
+		return 0;
+
+	c2h_info.id = RTW89_FWCMD_C2HREG_FUNC_PS_LEAVE_ACK;
+	ret = rtw89_fw_msg_reg(rtwdev, NULL, &c2h_info);
+	if (ret)
+		return ret;
+
+	c2hreg_macid = u32_get_bits(c2h_info.u.c2hreg[0],
+				    RTW89_C2HREG_PS_LEAVE_ACK_MACID);
+	c2hreg_ret = u32_get_bits(c2h_info.u.c2hreg[1], RTW89_C2HREG_PS_LEAVE_ACK_RET);
+
+	if (macid != c2hreg_macid || c2hreg_ret)
+		rtw89_warn(rtwdev, "rtw89: check lps h2c received by firmware fail\n");
+
+	return 0;
+}
+
 static int rtw89_fw_leave_lps_check(struct rtw89_dev *rtwdev, u8 macid)
 {
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
@@ -106,14 +131,15 @@ static void __rtw89_leave_lps(struct rtw89_dev *rtwdev,
 	};
 
 	rtw89_fw_h2c_lps_parm(rtwdev, &lps_param);
-	rtw89_fw_leave_lps_check(rtwdev, 0);
+	rtw89_fw_receive_lps_h2c_check(rtwdev, rtwvif_link->mac_id);
+	rtw89_fw_leave_lps_check(rtwdev, rtwvif_link->mac_id);
 	rtw89_btc_ntfy_radio_state(rtwdev, BTC_RFCTRL_WL_ON);
 	rtw89_chip_digital_pwr_comp(rtwdev, rtwvif_link->phy_idx);
 }
 
 void rtw89_leave_ps_mode(struct rtw89_dev *rtwdev)
 {
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	__rtw89_leave_ps_mode(rtwdev);
 }
@@ -125,7 +151,7 @@ void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 	bool can_ps_mode = true;
 	unsigned int link_id;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (test_and_set_bit(RTW89_FLAG_LEISURE_PS, rtwdev->flags))
 		return;
@@ -137,6 +163,8 @@ void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			can_ps_mode = false;
 	}
 
+	rtw89_fw_h2c_rf_ps_info(rtwdev, rtwvif);
+
 	if (RTW89_CHK_FW_FEATURE(LPS_CH_INFO, &rtwdev->fw))
 		rtw89_fw_h2c_lps_ch_info(rtwdev, rtwvif);
 	else
@@ -162,7 +190,7 @@ void rtw89_leave_lps(struct rtw89_dev *rtwdev)
 	struct rtw89_vif *rtwvif;
 	unsigned int link_id;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (!test_and_clear_bit(RTW89_FLAG_LEISURE_PS, rtwdev->flags))
 		return;
@@ -236,13 +264,23 @@ static void rtw89_tsf32_toggle(struct rtw89_dev *rtwdev,
 		rtw89_fw_h2c_tsf32_toggle(rtwdev, rtwvif_link, false);
 }
 
-static void rtw89_p2p_disable_all_noa(struct rtw89_dev *rtwdev,
-				      struct rtw89_vif_link *rtwvif_link,
-				      struct ieee80211_bss_conf *bss_conf)
+void rtw89_p2p_disable_all_noa(struct rtw89_dev *rtwdev,
+			       struct rtw89_vif_link *rtwvif_link,
+			       struct ieee80211_bss_conf *bss_conf)
 {
 	enum rtw89_p2pps_action act;
+	u8 oppps_ctwindow;
 	u8 noa_id;
 
+	rcu_read_lock();
+
+	if (!bss_conf)
+		bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
+
+	oppps_ctwindow = bss_conf->p2p_noa_attr.oppps_ctwindow;
+
+	rcu_read_unlock();
+
 	if (rtwvif_link->last_noa_nr == 0)
 		return;
 
@@ -252,8 +290,8 @@ static void rtw89_p2p_disable_all_noa(struct rtw89_dev *rtwdev,
 		else
 			act = RTW89_P2P_ACT_REMOVE;
 		rtw89_tsf32_toggle(rtwdev, rtwvif_link, act);
-		rtw89_fw_h2c_p2p_act(rtwdev, rtwvif_link, bss_conf,
-				     NULL, act, noa_id);
+		rtw89_fw_h2c_p2p_act(rtwdev, rtwvif_link, NULL,
+				     act, noa_id, oppps_ctwindow);
 	}
 }
 
@@ -275,8 +313,8 @@ static void rtw89_p2p_update_noa(struct rtw89_dev *rtwdev,
 		else
 			act = RTW89_P2P_ACT_UPDATE;
 		rtw89_tsf32_toggle(rtwdev, rtwvif_link, act);
-		rtw89_fw_h2c_p2p_act(rtwdev, rtwvif_link, bss_conf,
-				     desc, act, noa_id);
+		rtw89_fw_h2c_p2p_act(rtwdev, rtwvif_link, desc, act, noa_id,
+				     bss_conf->p2p_noa_attr.oppps_ctwindow);
 	}
 	rtwvif_link->last_noa_nr = noa_id;
 }
@@ -391,3 +429,150 @@ u8 rtw89_p2p_noa_fetch(struct rtw89_vif_link *rtwvif_link, void **data)
 	return tail - (u8 *)*data;
 #endif
 }
+
+static void rtw89_ps_noa_once_set_work(struct wiphy *wiphy, struct wiphy_work *work)
+{
+	struct rtw89_ps_noa_once_handler *noa_once =
+		container_of(work, struct rtw89_ps_noa_once_handler, set_work.work);
+
+	lockdep_assert_wiphy(wiphy);
+
+	noa_once->in_duration = true;
+}
+
+static void rtw89_ps_noa_once_clr_work(struct wiphy *wiphy, struct wiphy_work *work)
+{
+	struct rtw89_ps_noa_once_handler *noa_once =
+		container_of(work, struct rtw89_ps_noa_once_handler, clr_work.work);
+	struct rtw89_vif_link *rtwvif_link =
+		container_of(noa_once, struct rtw89_vif_link, noa_once);
+	struct rtw89_dev *rtwdev = rtwvif_link->rtwvif->rtwdev;
+
+	lockdep_assert_wiphy(wiphy);
+
+	rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, rtwvif_link, true);
+	noa_once->in_duration = false;
+}
+
+void rtw89_p2p_noa_once_init(struct rtw89_vif_link *rtwvif_link)
+{
+	struct rtw89_ps_noa_once_handler *noa_once = &rtwvif_link->noa_once;
+
+	noa_once->in_duration = false;
+	noa_once->tsf_begin = 0;
+	noa_once->tsf_end = 0;
+
+	wiphy_delayed_work_init(&noa_once->set_work, rtw89_ps_noa_once_set_work);
+	wiphy_delayed_work_init(&noa_once->clr_work, rtw89_ps_noa_once_clr_work);
+}
+
+static void rtw89_p2p_noa_once_cancel(struct rtw89_vif_link *rtwvif_link)
+{
+	struct rtw89_ps_noa_once_handler *noa_once = &rtwvif_link->noa_once;
+	struct rtw89_dev *rtwdev = rtwvif_link->rtwvif->rtwdev;
+	struct wiphy *wiphy = rtwdev->hw->wiphy;
+
+	wiphy_delayed_work_cancel(wiphy, &noa_once->set_work);
+	wiphy_delayed_work_cancel(wiphy, &noa_once->clr_work);
+}
+
+void rtw89_p2p_noa_once_deinit(struct rtw89_vif_link *rtwvif_link)
+{
+	rtw89_p2p_noa_once_cancel(rtwvif_link);
+	rtw89_p2p_noa_once_init(rtwvif_link);
+}
+
+void rtw89_p2p_noa_once_recalc(struct rtw89_vif_link *rtwvif_link)
+{
+	struct rtw89_ps_noa_once_handler *noa_once = &rtwvif_link->noa_once;
+	struct rtw89_dev *rtwdev = rtwvif_link->rtwvif->rtwdev;
+	const struct ieee80211_p2p_noa_desc *noa_desc;
+	struct wiphy *wiphy = rtwdev->hw->wiphy;
+	struct ieee80211_bss_conf *bss_conf;
+	u64 tsf_begin = U64_MAX, tsf_end;
+	u64 set_delay_us = 0;
+	u64 clr_delay_us = 0;
+	u32 start_time;
+	u32 interval;
+	u32 duration;
+	u64 tsf;
+	int ret;
+	int i;
+
+	lockdep_assert_wiphy(wiphy);
+
+	ret = rtw89_mac_port_get_tsf(rtwdev, rtwvif_link, &tsf);
+	if (ret) {
+		rtw89_warn(rtwdev, "%s: failed to get tsf\n", __func__);
+		return;
+	}
+
+	rcu_read_lock();
+
+	bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true);
+
+	for (i = 0; i < ARRAY_SIZE(bss_conf->p2p_noa_attr.desc); i++) {
+		bool first = tsf_begin == U64_MAX;
+		u64 tmp;
+
+		noa_desc = &bss_conf->p2p_noa_attr.desc[i];
+		if (noa_desc->count == 0 || noa_desc->count == 255)
+			continue;
+
+		start_time = le32_to_cpu(noa_desc->start_time);
+		interval = le32_to_cpu(noa_desc->interval);
+		duration = le32_to_cpu(noa_desc->duration);
+
+		if (unlikely(duration == 0 ||
+			     (noa_desc->count > 1 && interval == 0)))
+			continue;
+
+		tmp = start_time + interval * (noa_desc->count - 1) + duration;
+		tmp = (tsf & GENMASK_ULL(63, 32)) + tmp;
+		if (unlikely(tmp <= tsf))
+			continue;
+		tsf_end = first ? tmp : max(tsf_end, tmp);
+
+		tmp = (tsf & GENMASK_ULL(63, 32)) | start_time;
+		tsf_begin = first ? tmp : min(tsf_begin, tmp);
+	}
+
+	rcu_read_unlock();
+
+	if (tsf_begin == U64_MAX)
+		return;
+
+	rtw89_p2p_noa_once_cancel(rtwvif_link);
+
+	if (noa_once->tsf_end > tsf) {
+		tsf_begin = min(tsf_begin, noa_once->tsf_begin);
+		tsf_end = max(tsf_end, noa_once->tsf_end);
+	}
+
+	clr_delay_us = min_t(u64, tsf_end - tsf, UINT_MAX);
+
+	if (tsf_begin <= tsf) {
+		noa_once->in_duration = true;
+		goto out;
+	}
+
+	set_delay_us = tsf_begin - tsf;
+	if (unlikely(set_delay_us > UINT_MAX)) {
+		rtw89_warn(rtwdev, "%s: unhandled begin\n", __func__);
+		set_delay_us = 0;
+		clr_delay_us = 0;
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, rtwvif_link, true);
+		noa_once->in_duration = false;
+	}
+
+out:
+	if (set_delay_us)
+		wiphy_delayed_work_queue(wiphy, &noa_once->set_work,
+					 usecs_to_jiffies(set_delay_us));
+	if (clr_delay_us)
+		wiphy_delayed_work_queue(wiphy, &noa_once->clr_work,
+					 usecs_to_jiffies(clr_delay_us));
+
+	noa_once->tsf_begin = tsf_begin;
+	noa_once->tsf_end = tsf_end;
+}
diff --git a/sys/contrib/dev/rtw89/ps.h b/sys/contrib/dev/rtw89/ps.h
index 2b88f254a32d..729477153de6 100644
--- a/sys/contrib/dev/rtw89/ps.h
+++ b/sys/contrib/dev/rtw89/ps.h
@@ -22,6 +22,12 @@ void rtw89_p2p_noa_renew(struct rtw89_vif_link *rtwvif_link);
 void rtw89_p2p_noa_append(struct rtw89_vif_link *rtwvif_link,
 			  const struct ieee80211_p2p_noa_desc *desc);
 u8 rtw89_p2p_noa_fetch(struct rtw89_vif_link *rtwvif_link, void **data);
+void rtw89_p2p_noa_once_init(struct rtw89_vif_link *rtwvif_link);
+void rtw89_p2p_noa_once_deinit(struct rtw89_vif_link *rtwvif_link);
+void rtw89_p2p_noa_once_recalc(struct rtw89_vif_link *rtwvif_link);
+void rtw89_p2p_disable_all_noa(struct rtw89_dev *rtwdev,
+			       struct rtw89_vif_link *rtwvif_link,
+			       struct ieee80211_bss_conf *bss_conf);
 
 static inline void rtw89_leave_ips_by_hwflags(struct rtw89_dev *rtwdev)
 {
diff --git a/sys/contrib/dev/rtw89/reg.h b/sys/contrib/dev/rtw89/reg.h
index 10d0efa7a58e..de81103a072f 100644
--- a/sys/contrib/dev/rtw89/reg.h
+++ b/sys/contrib/dev/rtw89/reg.h
@@ -21,6 +21,7 @@
 #define R_AX_SYS_PW_CTRL 0x0004
 #define B_AX_SOP_ASWRM BIT(31)
 #define B_AX_SOP_PWMM_DSWR BIT(29)
+#define B_AX_SOP_EDSWR BIT(28)
 #define B_AX_XTAL_OFF_A_DIE BIT(22)
 #define B_AX_DIS_WLBT_PDNSUSEN_SOPC BIT(18)
 #define B_AX_RDY_SYSPWR BIT(17)
@@ -182,6 +183,7 @@
 
 #define R_AX_SYS_STATUS1 0x00F4
 #define B_AX_SEL_0XC0_MASK GENMASK(17, 16)
+#define B_AX_AUTO_WLPON BIT(10)
 #define B_AX_PAD_HCI_SEL_V2_MASK GENMASK(5, 3)
 #define MAC_AX_HCI_SEL_SDIO_UART 0
 #define MAC_AX_HCI_SEL_MULTI_USB 1
@@ -380,6 +382,18 @@
 #define B_AX_ACH1_BUSY BIT(9)
 #define B_AX_ACH0_BUSY BIT(8)
 
+#define R_AX_USB_ENDPOINT_0 0x1060
+#define B_AX_EP_IDX GENMASK(3, 0)
+#define R_AX_USB_ENDPOINT_2 0x1068
+#define NUMP 0x1
+#define R_AX_USB_HOST_REQUEST_2 0x1078
+#define B_AX_R_USBIO_MODE BIT(4)
+#define R_AX_USB3_MAC_NPI_CONFIG_INTF_0 0x1114
+#define B_AX_SSPHY_LFPS_FILTER BIT(31)
+#define R_AX_USB_WLAN0_1 0x1174
+#define B_AX_USBRX_RST BIT(9)
+#define B_AX_USBTX_RST BIT(8)
+
 #define R_AX_PCIE_DBG_CTRL 0x11C0
 #define B_AX_DBG_DUMMY_MASK GENMASK(23, 16)
 #define B_AX_PCIE_DBG_SEL_MASK GENMASK(15, 13)
@@ -459,6 +473,17 @@
 #define R_AX_WP_PAGE_CTRL2_V1 0x17A4
 #define R_AX_WP_PAGE_INFO1_V1 0x17A8
 
+#define R_AX_USB_ENDPOINT_0_V1 0x5060
+#define B_AX_EP_IDX_V1 GENMASK(3, 0)
+#define R_AX_USB_ENDPOINT_2_V1 0x5068
+#define R_AX_USB_HOST_REQUEST_2_V1 0x5078
+#define B_AX_R_USBIO_MODE_V1 BIT(4)
+#define R_AX_USB3_MAC_NPI_CONFIG_INTF_0_V1 0x5114
+#define B_AX_SSPHY_LFPS_FILTER_V1 BIT(31)
+#define R_AX_USB_WLAN0_1_V1 0x5174
+#define B_AX_USBRX_RST_V1 BIT(9)
+#define B_AX_USBTX_RST_V1 BIT(8)
+
 #define R_AX_H2CREG_DATA0_V1 0x7140
 #define R_AX_H2CREG_DATA1_V1 0x7144
 #define R_AX_H2CREG_DATA2_V1 0x7148
@@ -1025,6 +1050,12 @@
 #define B_AX_DISPATCHER_INTN_SEL_MASK GENMASK(7, 4)
 #define B_AX_DISPATCHER_CH_SEL_MASK GENMASK(3, 0)
 
+#define R_AX_RXDMA_SETTING 0x8908
+#define B_AX_BULK_SIZE GENMASK(1, 0)
+#define USB11_BULKSIZE 0x2
+#define USB2_BULKSIZE 0x1
+#define USB3_BULKSIZE 0x0
+
 #define R_AX_RX_FUNCTION_STOP 0x8920
 #define B_AX_HDR_RX_STOP BIT(0)
 
@@ -6070,6 +6101,7 @@
 #define B_BE_MACID_ACQ_GRP0_CLR_P BIT(2)
 #define B_BE_R_MACID_ACQ_CHK_EN BIT(0)
 
+#define R_BE_BTC_CFG 0x0E300
 #define R_BE_BT_BREAK_TABLE 0x0E344
 
 #define R_BE_GNT_SW_CTRL 0x0E348
@@ -6618,6 +6650,13 @@
 #define B_BE_RTS_LIMIT_IN_OFDM6 BIT(1)
 #define B_BE_CHECK_CCK_EN BIT(0)
 
+#define R_BE_TXCNT 0x1082C
+#define R_BE_TXCNT_C1 0x1482C
+#define B_BE_ADD_TXCNT_BY BIT(31)
+#define B_BE_TOTAL_TC_OPT BIT(30)
+#define B_BE_S_TXCNT_LMT_MASK GENMASK(29, 24)
+#define B_BE_L_TXCNT_LMT_MASK GENMASK(21, 16)
+
 #define R_BE_MBSSID_DROP_0 0x1083C
 #define R_BE_MBSSID_DROP_0_C1 0x1483C
 #define B_BE_GI_LTF_FB_SEL BIT(30)
@@ -7095,6 +7134,10 @@
 #define B_BE_MACLBK_RDY_NUM_MASK GENMASK(7, 3)
 #define B_BE_MACLBK_EN BIT(0)
 
+#define R_BE_CLIENT_OM_CTRL 0x11040
+#define R_BE_CLIENT_OM_CTRL_C1 0x15040
+#define B_BE_TRIG_DIS_EHTTB BIT(24)
+
 #define R_BE_WMAC_NAV_CTL 0x11080
 #define R_BE_WMAC_NAV_CTL_C1 0x15080
 #define B_BE_WMAC_NAV_UPPER_EN BIT(26)
@@ -7590,7 +7633,15 @@
 #define B_BE_PWR_FORCE_RU_ON BIT(18)
 #define B_BE_PWR_FORCE_RU_ENON BIT(28)
 #define R_BE_PWR_FORCE_MACID 0x11A48
-#define B_BE_PWR_FORCE_MACID_ON BIT(9)
+#define B_BE_PWR_FORCE_MACID_DBM_ON BIT(9)
+#define B_BE_PWR_FORCE_MACID_DBM_VAL GENMASK(17, 10)
+#define B_BE_PWR_FORCE_MACID_EN_VAL BIT(18)
+#define B_BE_PWR_FORCE_MACID_EN_ON BIT(19)
+#define B_BE_PWR_FORCE_MACID_ALL \
+	(B_BE_PWR_FORCE_MACID_DBM_ON | \
+	 B_BE_PWR_FORCE_MACID_DBM_VAL | \
+	 B_BE_PWR_FORCE_MACID_EN_VAL | \
+	 B_BE_PWR_FORCE_MACID_EN_ON)
 
 #define R_BE_PWR_REG_CTRL 0x11A50
 #define B_BE_PWR_BT_EN BIT(23)
@@ -8005,6 +8056,7 @@
 #define R_PHY_STS_BITMAP_HT 0x076C
 #define R_PHY_STS_BITMAP_VHT 0x0770
 #define R_PHY_STS_BITMAP_HE 0x0774
+#define R_PHY_STS_BITMAP_EHT 0x0788
 #define R_EDCCA_RPTREG_SEL_BE 0x078C
 #define B_EDCCA_RPTREG_SEL_BE_MSK GENMASK(22, 20)
 #define R_PMAC_GNT 0x0980
@@ -8157,6 +8209,8 @@
 #define B_EDCCA_RPT_B_S40 BIT(4)
 #define B_EDCCA_RPT_B_S80 BIT(3)
 #define B_EDCCA_RPT_B_PATH_MASK GENMASK(2, 1)
+#define R_EDCCA_RPT_P1_A 0x1740
+#define R_EDCCA_RPT_P1_B 0x1744
 #define R_SWSI_V1 0x174C
 #define B_SWSI_W_BUSY_V1 BIT(24)
 #define B_SWSI_R_BUSY_V1 BIT(25)
@@ -8222,6 +8276,7 @@
 #define B_TXCKEN_FORCE_ALL GENMASK(24, 0)
 #define R_EDCCA_RPT_SEL 0x20CC
 #define B_EDCCA_RPT_SEL_MSK GENMASK(2, 0)
+#define B_EDCCA_RPT_SEL_P1_MSK GENMASK(5, 3)
 #define R_ADC_FIFO 0x20fc
 #define B_ADC_FIFO_RST GENMASK(31, 24)
 #define B_ADC_FIFO_RXK GENMASK(31, 16)
@@ -8291,6 +8346,8 @@
 #define B_P1_EN_SOUND_WO_NDP BIT(1)
 #define R_EDCCA_RPT_A_BE 0x2E38
 #define R_EDCCA_RPT_B_BE 0x2E3C
+#define R_EDCCA_RPT_P1_A_BE 0x2E40
+#define R_EDCCA_RPT_P1_B_BE 0x2E44
 #define R_S1_HW_SI_DIS 0x3200
 #define B_S1_HW_SI_DIS_W_R_TRIG GENMASK(30, 28)
 #define R_P1_RXCK 0x32A0
@@ -8721,8 +8778,10 @@
 #define B_DPD_GDIS BIT(13)
 #define B_IQK_RFC_ON BIT(1)
 #define R_TXPWRB 0x56CC
+#define R_P1_TXPWRB 0x76CC
 #define B_TXPWRB_ON BIT(28)
 #define B_TXPWRB_VAL GENMASK(27, 19)
+#define B_TXPWRB_MAX GENMASK(8, 0)
 #define R_DPD_OFT_EN 0x5800
 #define B_DPD_OFT_EN BIT(28)
 #define B_DPD_TSSI_CW GENMASK(26, 18)
@@ -8747,6 +8806,8 @@
 #define B_P0_TSSI_RFC GENMASK(28, 27)
 #define B_P0_TSSI_OFT_EN BIT(28)
 #define B_P0_TSSI_OFT GENMASK(7, 0)
+#define R_P0_TSSI_SLOPE_CAL 0x581c
+#define B_P0_TSSI_SLOPE_CAL_EN BIT(20)
 #define R_P0_TSSI_AVG 0x5820
 #define B_P0_TSSI_EN BIT(31)
 #define B_P0_TSSI_AVG GENMASK(15, 12)
@@ -9169,6 +9230,16 @@
 #define B_IQKINF2_FCNT GENMASK(23, 16)
 #define B_IQKINF2_KCNT GENMASK(15, 8)
 #define B_IQKINF2_NCTLV GENMASK(7, 0)
+#define R_TXAGC_REF_DBM_RF1_P0 0xBC04
+#define B_TXAGC_OFDM_REF_DBM_RF1_P0 GENMASK(10, 2)
+#define B_TXAGC_CCK_REF_DBM_RF1_P0 GENMASK(19, 11)
+#define R_TSSI_K_RF1_P0 0xBC28
+#define B_TSSI_K_OFDM_RF1_P0 GENMASK(9, 0)
+#define R_TXAGC_REF_DBM_RF1_P1 0xBD04
+#define B_TXAGC_OFDM_REF_DBM_RF1_P1 GENMASK(10, 2)
+#define B_TXAGC_CCK_REF_DBM_RF1_P1 GENMASK(19, 11)
+#define R_TSSI_K_RF1_P1 0xBD28
+#define B_TSSI_K_OFDM_RF1_P1 GENMASK(9, 0)
 #define R_RFK_ST 0xBFF8
 #define R_DCOF0 0xC000
 #define B_DCOF0_RST BIT(17)
@@ -9230,6 +9301,7 @@
 #define B_WDADC_SEL GENMASK(5, 4)
 #define R_ADCMOD 0xC0E8
 #define B_ADCMOD_LP GENMASK(31, 16)
+#define B_ADCMOD_AUTO_RST BIT(6)
 #define R_DCIM 0xC0EC
 #define B_DCIM_RC GENMASK(23, 16)
 #define B_DCIM_FR GENMASK(14, 13)
@@ -9334,20 +9406,25 @@
 #define R_TSSI_PWR_P0 0xE610
 #define R_TSSI_PWR_P1 0xE710
 #define B_TSSI_CONT_EN BIT(3)
+#define R_P0_TXPWRB_BE 0xE61C
+#define R_P1_TXPWRB_BE 0xE71C
+#define B_TXPWRB_MAX_BE GENMASK(20, 12)
 #define R_TSSI_MAP_OFST_P0 0xE620
 #define R_TSSI_MAP_OFST_P1 0xE720
 #define B_TSSI_MAP_OFST_OFDM GENMASK(17, 9)
 #define B_TSSI_MAP_OFST_CCK GENMASK(26, 18)
-#define R_TXAGC_REF0_P0 0xE628
-#define R_TXAGC_REF0_P1 0xE728
-#define B_TXAGC_REF0_OFDM_DBM GENMASK(8, 0)
-#define B_TXAGC_REF0_CCK_DBM GENMASK(17, 9)
-#define B_TXAGC_REF0_OFDM_CW GENMASK(26, 18)
-#define R_TXAGC_REF1_P0 0xE62C
-#define R_TXAGC_REF1_P1 0xE72C
-#define B_TXAGC_REF1_CCK_CW GENMASK(8, 0)
+#define R_TXAGC_REF_DBM_P0 0xE628
+#define B_TXAGC_OFDM_REF_DBM_P0 GENMASK(8, 0)
+#define B_TXAGC_CCK_REF_DBM_P0 GENMASK(17, 9)
+#define R_TSSI_K_P0 0xE6A0
+#define B_TSSI_K_OFDM_P0 GENMASK(29, 20)
 #define R_TXPWR_RSTB 0xE70C
 #define B_TXPWR_RSTB BIT(16)
+#define R_TXAGC_REF_DBM_P1 0xE728
+#define B_TXAGC_OFDM_REF_DBM_P1 GENMASK(8, 0)
+#define B_TXAGC_CCK_REF_DBM_P1 GENMASK(17, 9)
+#define R_TSSI_K_P1 0xE7A0
+#define B_TSSI_K_OFDM_P1 GENMASK(29, 20)
 
 /* WiFi CPU local domain */
 #define R_AX_WDT_CTRL 0x0040
diff --git a/sys/contrib/dev/rtw89/regd.c b/sys/contrib/dev/rtw89/regd.c
index 4b71d5a68c45..e0a79612ebfd 100644
--- a/sys/contrib/dev/rtw89/regd.c
+++ b/sys/contrib/dev/rtw89/regd.c
@@ -7,257 +7,266 @@
 #include "ps.h"
 #include "util.h"
 
-#define COUNTRY_REGD(_alpha2, _txpwr_regd...) \
-	{.alpha2 = (_alpha2), \
-	 .txpwr_regd = {_txpwr_regd}, \
+static
+void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request);
+
+#define COUNTRY_REGD(_alpha2, _rule_2ghz, _rule_5ghz, _rule_6ghz, _fmap) \
+	{							\
+		.alpha2 = _alpha2,				\
+		.txpwr_regd[RTW89_BAND_2G] = _rule_2ghz,	\
+		.txpwr_regd[RTW89_BAND_5G] = _rule_5ghz,	\
+		.txpwr_regd[RTW89_BAND_6G] = _rule_6ghz,	\
+		.func_bitmap = { _fmap, },	\
 	}
 
+static_assert(BITS_PER_TYPE(unsigned long) >= NUM_OF_RTW89_REGD_FUNC);
+
 static const struct rtw89_regd rtw89_ww_regd =
-	COUNTRY_REGD("00", RTW89_WW, RTW89_WW, RTW89_WW);
+	COUNTRY_REGD("00", RTW89_WW, RTW89_WW, RTW89_WW, 0x0);
 
 static const struct rtw89_regd rtw89_regd_map[] = {
-	COUNTRY_REGD("AR", RTW89_MEXICO, RTW89_MEXICO, RTW89_FCC),
-	COUNTRY_REGD("BO", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("BR", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("CL", RTW89_CHILE, RTW89_CHILE, RTW89_CHILE),
-	COUNTRY_REGD("CO", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("CR", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("EC", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("SV", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("GT", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("HN", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("MX", RTW89_MEXICO, RTW89_MEXICO, RTW89_FCC),
-	COUNTRY_REGD("NI", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("PA", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("PY", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("PE", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("US", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("UY", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("VE", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("PR", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("DO", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("AT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("BE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CY", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("DK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("EE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("FI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("FR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("DE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("GR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("HU", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("IS", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("IE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("IT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LV", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LU", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MC", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("NL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("NO", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("PL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("PT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("SK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("SI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("ES", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("SE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("GB", RTW89_UK, RTW89_UK, RTW89_UK),
-	COUNTRY_REGD("AL", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("AZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("BH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("BA", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("BG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("HR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("EG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("GH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("IQ", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("IL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("JO", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("KZ", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("KE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("KW", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("KG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LB", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LS", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MK", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("MA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("NA", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("NG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("OM", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("QA", RTW89_QATAR, RTW89_QATAR, RTW89_QATAR),
-	COUNTRY_REGD("RO", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("RU", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("SA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("SN", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("RS", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("ME", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("ZA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("TR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("UA", RTW89_UKRAINE, RTW89_UKRAINE, RTW89_UKRAINE),
-	COUNTRY_REGD("AE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("YE", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("ZW", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("BD", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("KH", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("CN", RTW89_CN, RTW89_CN, RTW89_CN),
-	COUNTRY_REGD("HK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("IN", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("ID", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("KR", RTW89_KCC, RTW89_KCC, RTW89_KCC),
-	COUNTRY_REGD("MY", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("PK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("PH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("SG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LK", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("TW", RTW89_FCC, RTW89_FCC, RTW89_ETSI),
-	COUNTRY_REGD("TH", RTW89_THAILAND, RTW89_THAILAND, RTW89_THAILAND),
-	COUNTRY_REGD("VN", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("AU", RTW89_ACMA, RTW89_ACMA, RTW89_ACMA),
-	COUNTRY_REGD("NZ", RTW89_ACMA, RTW89_ACMA, RTW89_ACMA),
-	COUNTRY_REGD("PG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CA", RTW89_IC, RTW89_IC, RTW89_IC),
-	COUNTRY_REGD("JP", RTW89_MKK, RTW89_MKK, RTW89_MKK),
-	COUNTRY_REGD("JM", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("AN", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("TT", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("TN", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("AF", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("DZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("AS", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("AD", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("AO", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("AI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("AQ", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("AG", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("AM", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("AW", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("BS", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("BB", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("BY", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("BZ", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("BJ", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("BM", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("BT", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("BW", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("BV", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("IO", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("VG", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("BN", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("BF", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("BI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("CV", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("KY", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("CF", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("TD", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CX", RTW89_ACMA, RTW89_ACMA, RTW89_NA),
-	COUNTRY_REGD("CC", RTW89_ACMA, RTW89_ACMA, RTW89_NA),
-	COUNTRY_REGD("KM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("CG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CD", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("CK", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("CI", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("DJ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("DM", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("GQ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("ER", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("ET", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("FK", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("FO", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("FJ", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("GF", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("PF", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("TF", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("GA", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("GM", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("GE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("GI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("GL", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("GD", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("GP", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("GU", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("GG", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("GN", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("GW", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("GY", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("HT", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("HM", RTW89_ACMA, RTW89_ACMA, RTW89_NA),
-	COUNTRY_REGD("VA", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("IM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("JE", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("KI", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("XK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("LY", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("MO", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("MG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MW", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("MV", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("ML", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("MH", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("MQ", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("MR", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("MU", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("YT", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("FM", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("MD", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MN", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("MS", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("NR", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("NP", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("NC", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("NE", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("NU", RTW89_ACMA, RTW89_ACMA, RTW89_NA),
-	COUNTRY_REGD("NF", RTW89_ACMA, RTW89_ACMA, RTW89_NA),
-	COUNTRY_REGD("MP", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("PW", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("RE", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("RW", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("SH", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("KN", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("LC", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("MF", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("SX", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("PM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("VC", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("WS", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("SM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("ST", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("SC", RTW89_FCC, RTW89_FCC, RTW89_NA),
-	COUNTRY_REGD("SL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("SB", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("SO", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("GS", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("SR", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("SJ", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("SZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("TJ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("TZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("TG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("TK", RTW89_ACMA, RTW89_ACMA, RTW89_NA),
-	COUNTRY_REGD("TO", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("TM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("TC", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("TV", RTW89_ETSI, RTW89_NA, RTW89_NA),
-	COUNTRY_REGD("UG", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("VI", RTW89_FCC, RTW89_FCC, RTW89_FCC),
-	COUNTRY_REGD("UZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI),
-	COUNTRY_REGD("VU", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("WF", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("EH", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("ZM", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("CU", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("IR", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("SY", RTW89_ETSI, RTW89_NA, RTW89_NA),
-	COUNTRY_REGD("SD", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
-	COUNTRY_REGD("PS", RTW89_ETSI, RTW89_ETSI, RTW89_NA),
+	COUNTRY_REGD("AR", RTW89_MEXICO, RTW89_MEXICO, RTW89_FCC, 0x0),
+	COUNTRY_REGD("BO", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("BR", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("CL", RTW89_CHILE, RTW89_CHILE, RTW89_CHILE, 0x0),
+	COUNTRY_REGD("CO", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("CR", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("EC", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("SV", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("GT", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("HN", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("MX", RTW89_MEXICO, RTW89_MEXICO, RTW89_FCC, 0x0),
+	COUNTRY_REGD("NI", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("PA", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("PY", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("PE", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("US", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x1),
+	COUNTRY_REGD("UY", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("VE", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("PR", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("DO", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("AT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("BE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("CY", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("CZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("DK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("EE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("FI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("FR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("DE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("GR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("HU", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("IS", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("IE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("IT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("LV", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("LI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("LT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("LU", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("MT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("MC", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("NL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("NO", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("PL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("PT", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("SK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("SI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("ES", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("SE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("CH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("GB", RTW89_UK, RTW89_UK, RTW89_UK, 0x0),
+	COUNTRY_REGD("AL", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("AZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("BH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("BA", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("BG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("HR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("EG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("GH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("IQ", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("IL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("JO", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("KZ", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("KE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("KW", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("KG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("LB", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("LS", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("MK", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("MA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("MZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("NA", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("NG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("OM", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("QA", RTW89_QATAR, RTW89_QATAR, RTW89_QATAR, 0x0),
+	COUNTRY_REGD("RO", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x2),
+	COUNTRY_REGD("RU", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("SA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("SN", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("RS", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("ME", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("ZA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("TR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("UA", RTW89_UKRAINE, RTW89_UKRAINE, RTW89_UKRAINE, 0x0),
+	COUNTRY_REGD("AE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("YE", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("ZW", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("BD", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("KH", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("CN", RTW89_CN, RTW89_CN, RTW89_CN, 0x0),
+	COUNTRY_REGD("HK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("IN", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("ID", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("KR", RTW89_KCC, RTW89_KCC, RTW89_KCC, 0x1),
+	COUNTRY_REGD("MY", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("PK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("PH", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("SG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("LK", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("TW", RTW89_FCC, RTW89_FCC, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("TH", RTW89_THAILAND, RTW89_THAILAND, RTW89_THAILAND, 0x0),
+	COUNTRY_REGD("VN", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("AU", RTW89_ACMA, RTW89_ACMA, RTW89_ACMA, 0x0),
+	COUNTRY_REGD("NZ", RTW89_ACMA, RTW89_ACMA, RTW89_ACMA, 0x0),
+	COUNTRY_REGD("PG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("CA", RTW89_IC, RTW89_IC, RTW89_IC, 0x1),
+	COUNTRY_REGD("JP", RTW89_MKK, RTW89_MKK, RTW89_MKK, 0x0),
+	COUNTRY_REGD("JM", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("AN", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("TT", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("TN", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("AF", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("DZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("AS", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("AD", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("AO", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("AI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("AQ", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("AG", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("AM", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("AW", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("BS", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("BB", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("BY", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("BZ", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("BJ", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("BM", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("BT", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("BW", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("BV", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("IO", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("VG", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("BN", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("BF", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("MM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("BI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("CM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("CV", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("KY", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("CF", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("TD", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("CX", RTW89_ACMA, RTW89_ACMA, RTW89_NA, 0x0),
+	COUNTRY_REGD("CC", RTW89_ACMA, RTW89_ACMA, RTW89_NA, 0x0),
+	COUNTRY_REGD("KM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("CG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("CD", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("CK", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("CI", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("DJ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("DM", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("GQ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("ER", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("ET", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("FK", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("FO", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("FJ", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("GF", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("PF", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("TF", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("GA", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("GM", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("GE", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("GI", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("GL", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("GD", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("GP", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("GU", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("GG", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("GN", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("GW", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("GY", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("HT", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("HM", RTW89_ACMA, RTW89_ACMA, RTW89_NA, 0x0),
+	COUNTRY_REGD("VA", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("IM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("JE", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("KI", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("XK", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("LA", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("LR", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("LY", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("MO", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("MG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("MW", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("MV", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("ML", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("MH", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("MQ", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("MR", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("MU", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("YT", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("FM", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("MD", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("MN", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("MS", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("NR", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("NP", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("NC", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("NE", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("NU", RTW89_ACMA, RTW89_ACMA, RTW89_NA, 0x0),
+	COUNTRY_REGD("NF", RTW89_ACMA, RTW89_ACMA, RTW89_NA, 0x0),
+	COUNTRY_REGD("MP", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("PW", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("RE", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("RW", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("SH", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("KN", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("LC", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("MF", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("SX", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("PM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("VC", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("WS", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("SM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("ST", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("SC", RTW89_FCC, RTW89_FCC, RTW89_NA, 0x0),
+	COUNTRY_REGD("SL", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("SB", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("SO", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("GS", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("SR", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("SJ", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("SZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("TJ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("TZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("TG", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("TK", RTW89_ACMA, RTW89_ACMA, RTW89_NA, 0x0),
+	COUNTRY_REGD("TO", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("TM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("TC", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("TV", RTW89_ETSI, RTW89_NA, RTW89_NA, 0x0),
+	COUNTRY_REGD("UG", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("VI", RTW89_FCC, RTW89_FCC, RTW89_FCC, 0x0),
+	COUNTRY_REGD("UZ", RTW89_ETSI, RTW89_ETSI, RTW89_ETSI, 0x0),
+	COUNTRY_REGD("VU", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("WF", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("EH", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("ZM", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("CU", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("IR", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("SY", RTW89_ETSI, RTW89_NA, RTW89_NA, 0x0),
+	COUNTRY_REGD("SD", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
+	COUNTRY_REGD("PS", RTW89_ETSI, RTW89_ETSI, RTW89_NA, 0x0),
 };
 
 static const char rtw89_alpha2_list_eu[][3] = {
@@ -295,13 +304,16 @@ static const char rtw89_alpha2_list_eu[][3] = {
 	"RO",
 };
 
-static const struct rtw89_regd *rtw89_regd_find_reg_by_name(const char *alpha2)
+static const struct rtw89_regd *rtw89_regd_find_reg_by_name(struct rtw89_dev *rtwdev,
+							    const char *alpha2)
 {
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	const struct rtw89_regd_ctrl *regd_ctrl = &regulatory->ctrl;
 	u32 i;
 
-	for (i = 0; i < ARRAY_SIZE(rtw89_regd_map); i++) {
-		if (!memcmp(rtw89_regd_map[i].alpha2, alpha2, 2))
-			return &rtw89_regd_map[i];
+	for (i = 0; i < regd_ctrl->nr; i++) {
+		if (!memcmp(regd_ctrl->map[i].alpha2, alpha2, 2))
+			return &regd_ctrl->map[i];
 	}
 
 	return &rtw89_ww_regd;
@@ -312,22 +324,25 @@ static bool rtw89_regd_is_ww(const struct rtw89_regd *regd)
 	return regd == &rtw89_ww_regd;
 }
 
-static u8 rtw89_regd_get_index(const struct rtw89_regd *regd)
+static u8 rtw89_regd_get_index(struct rtw89_dev *rtwdev, const struct rtw89_regd *regd)
 {
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	const struct rtw89_regd_ctrl *regd_ctrl = &regulatory->ctrl;
+
 	BUILD_BUG_ON(ARRAY_SIZE(rtw89_regd_map) > RTW89_REGD_MAX_COUNTRY_NUM);
 
 	if (rtw89_regd_is_ww(regd))
 		return RTW89_REGD_MAX_COUNTRY_NUM;
 
-	return regd - rtw89_regd_map;
+	return regd - regd_ctrl->map;
 }
 
-static u8 rtw89_regd_get_index_by_name(const char *alpha2)
+static u8 rtw89_regd_get_index_by_name(struct rtw89_dev *rtwdev, const char *alpha2)
 {
 	const struct rtw89_regd *regd;
 
-	regd = rtw89_regd_find_reg_by_name(alpha2);
-	return rtw89_regd_get_index(regd);
+	regd = rtw89_regd_find_reg_by_name(rtwdev, alpha2);
+	return rtw89_regd_get_index(rtwdev, regd);
 }
 
 #define rtw89_debug_regd(_dev, _regd, _desc, _argv...) \
@@ -348,11 +363,10 @@ static void rtw89_regd_setup_unii4(struct rtw89_dev *rtwdev,
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct ieee80211_supported_band *sband;
 	struct rtw89_acpi_dsm_result res = {};
-	bool enable_by_fcc;
-	bool enable_by_ic;
+	bool enable;
+	u8 index;
 	int ret;
 	u8 val;
-	int i;
 
 	sband = wiphy->bands[NL80211_BAND_5GHZ];
 	if (!sband)
@@ -369,35 +383,25 @@ static void rtw89_regd_setup_unii4(struct rtw89_dev *rtwdev,
 	if (ret) {
 		rtw89_debug(rtwdev, RTW89_DBG_REGD,
 			    "acpi: cannot eval unii 4: %d\n", ret);
-		enable_by_fcc = true;
-		enable_by_ic = false;
+		val = u8_encode_bits(1, RTW89_ACPI_CONF_UNII4_US);
 		goto bottom;
 	}
 
 	val = res.u.value;
-	enable_by_fcc = u8_get_bits(val, RTW89_ACPI_CONF_UNII4_FCC);
-	enable_by_ic = u8_get_bits(val, RTW89_ACPI_CONF_UNII4_IC);
 
 	rtw89_debug(rtwdev, RTW89_DBG_REGD,
 		    "acpi: eval if allow unii-4: 0x%x\n", val);
 
 bottom:
-	for (i = 0; i < ARRAY_SIZE(rtw89_regd_map); i++) {
-		const struct rtw89_regd *regd = &rtw89_regd_map[i];
-
-		switch (regd->txpwr_regd[RTW89_BAND_5G]) {
-		case RTW89_FCC:
-			if (enable_by_fcc)
-				clear_bit(i, regulatory->block_unii4);
-			break;
-		case RTW89_IC:
-			if (enable_by_ic)
-				clear_bit(i, regulatory->block_unii4);
-			break;
-		default:
-			break;
-		}
-	}
+	index = rtw89_regd_get_index_by_name(rtwdev, "US");
+	enable = u8_get_bits(val, RTW89_ACPI_CONF_UNII4_US);
+	if (enable && index != RTW89_REGD_MAX_COUNTRY_NUM)
+		clear_bit(index, regulatory->block_unii4);
+
+	index = rtw89_regd_get_index_by_name(rtwdev, "CA");
+	enable = u8_get_bits(val, RTW89_ACPI_CONF_UNII4_CA);
+	if (enable && index != RTW89_REGD_MAX_COUNTRY_NUM)
+		clear_bit(index, regulatory->block_unii4);
 }
 
 static void __rtw89_regd_setup_policy_6ghz(struct rtw89_dev *rtwdev, bool block,
@@ -406,7 +410,7 @@ static void __rtw89_regd_setup_policy_6ghz(struct rtw89_dev *rtwdev, bool block,
 	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
 	u8 index;
 
-	index = rtw89_regd_get_index_by_name(alpha2);
+	index = rtw89_regd_get_index_by_name(rtwdev, alpha2);
 	if (index == RTW89_REGD_MAX_COUNTRY_NUM) {
 		rtw89_debug(rtwdev, RTW89_DBG_REGD, "%s: unknown alpha2 %c%c\n",
 			    __func__, alpha2[0], alpha2[1]);
@@ -476,9 +480,9 @@ static void rtw89_regd_setup_policy_6ghz_sp(struct rtw89_dev *rtwdev)
 	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
 	const struct rtw89_acpi_policy_6ghz_sp *ptr;
 	struct rtw89_acpi_dsm_result res = {};
-	bool enable_by_us;
+	bool enable;
+	u8 index;
 	int ret;
-	int i;
 
 	ret = rtw89_acpi_evaluate_dsm(rtwdev, RTW89_ACPI_DSM_FUNC_6GHZ_SP_SUP, &res);
 	if (ret) {
@@ -503,16 +507,66 @@ static void rtw89_regd_setup_policy_6ghz_sp(struct rtw89_dev *rtwdev)
 
 	bitmap_fill(regulatory->block_6ghz_sp, RTW89_REGD_MAX_COUNTRY_NUM);
 
-	enable_by_us = u8_get_bits(ptr->conf, RTW89_ACPI_CONF_6GHZ_SP_US);
+	index = rtw89_regd_get_index_by_name(rtwdev, "US");
+	enable = u8_get_bits(ptr->conf, RTW89_ACPI_CONF_6GHZ_SP_US);
+	if (enable && index != RTW89_REGD_MAX_COUNTRY_NUM)
+		clear_bit(index, regulatory->block_6ghz_sp);
+
+	index = rtw89_regd_get_index_by_name(rtwdev, "CA");
+	enable = u8_get_bits(ptr->conf, RTW89_ACPI_CONF_6GHZ_SP_CA);
+	if (enable && index != RTW89_REGD_MAX_COUNTRY_NUM)
+		clear_bit(index, regulatory->block_6ghz_sp);
+
+out:
+	kfree(ptr);
+}
+
+static void rtw89_regd_setup_policy_6ghz_vlp(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	const struct rtw89_acpi_policy_6ghz_vlp *ptr = NULL;
+	struct rtw89_acpi_dsm_result res = {};
+	bool enable;
+	u8 index;
+	int ret;
+	u8 val;
 
-	for (i = 0; i < ARRAY_SIZE(rtw89_regd_map); i++) {
-		const struct rtw89_regd *tmp = &rtw89_regd_map[i];
+	/* By default, allow 6 GHz VLP on all countries except US and CA. */
+	val = ~(RTW89_ACPI_CONF_6GHZ_VLP_US | RTW89_ACPI_CONF_6GHZ_VLP_CA);
 
-		if (enable_by_us && memcmp(tmp->alpha2, "US", 2) == 0)
-			clear_bit(i, regulatory->block_6ghz_sp);
+	ret = rtw89_acpi_evaluate_dsm(rtwdev, RTW89_ACPI_DSM_FUNC_6GHZ_VLP_SUP, &res);
+	if (ret) {
+		rtw89_debug(rtwdev, RTW89_DBG_REGD,
+			    "acpi: cannot eval policy 6ghz-vlp: %d\n", ret);
+		goto bottom;
 	}
 
-out:
+	ptr = res.u.policy_6ghz_vlp;
+
+	switch (ptr->override) {
+	default:
+		rtw89_debug(rtwdev, RTW89_DBG_REGD,
+			    "%s: unknown override case: %d\n", __func__,
+			    ptr->override);
+		fallthrough;
+	case 0:
+		break;
+	case 1:
+		val = ptr->conf;
+		break;
+	}
+
+bottom:
+	index = rtw89_regd_get_index_by_name(rtwdev, "US");
+	enable = u8_get_bits(val, RTW89_ACPI_CONF_6GHZ_VLP_US);
+	if (!enable && index != RTW89_REGD_MAX_COUNTRY_NUM)
+		set_bit(index, regulatory->block_6ghz_vlp);
+
+	index = rtw89_regd_get_index_by_name(rtwdev, "CA");
+	enable = u8_get_bits(val, RTW89_ACPI_CONF_6GHZ_VLP_CA);
+	if (!enable && index != RTW89_REGD_MAX_COUNTRY_NUM)
+		set_bit(index, regulatory->block_6ghz_vlp);
+
 	kfree(ptr);
 }
 
@@ -559,6 +613,7 @@ bottom:
 	if (regd_allow_6ghz) {
 		rtw89_regd_setup_policy_6ghz(rtwdev);
 		rtw89_regd_setup_policy_6ghz_sp(rtwdev);
+		rtw89_regd_setup_policy_6ghz_vlp(rtwdev);
 		return;
 	}
 
@@ -571,10 +626,81 @@ bottom:
 	kfree(sband);
 }
 
+#define RTW89_DEF_REGD_STR(regd) \
+	[RTW89_ ## regd] = #regd
+
+static const char * const rtw89_regd_string[] = {
+	RTW89_DEF_REGD_STR(WW),
+	RTW89_DEF_REGD_STR(ETSI),
+	RTW89_DEF_REGD_STR(FCC),
+	RTW89_DEF_REGD_STR(MKK),
+	RTW89_DEF_REGD_STR(NA),
+	RTW89_DEF_REGD_STR(IC),
+	RTW89_DEF_REGD_STR(KCC),
+	RTW89_DEF_REGD_STR(ACMA),
+	RTW89_DEF_REGD_STR(NCC),
+	RTW89_DEF_REGD_STR(MEXICO),
+	RTW89_DEF_REGD_STR(CHILE),
+	RTW89_DEF_REGD_STR(UKRAINE),
+	RTW89_DEF_REGD_STR(CN),
+	RTW89_DEF_REGD_STR(QATAR),
+	RTW89_DEF_REGD_STR(UK),
+	RTW89_DEF_REGD_STR(THAILAND),
+};
+
+static_assert(ARRAY_SIZE(rtw89_regd_string) == RTW89_REGD_NUM);
+
+const char *rtw89_regd_get_string(enum rtw89_regulation_type regd)
+{
+	if (regd < 0 || regd >= RTW89_REGD_NUM)
+		return "(unknown)";
+
+	return rtw89_regd_string[regd];
+}
+
+static void rtw89_regd_setup_reg_rules(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	const struct rtw89_acpi_policy_reg_rules *ptr;
+	struct rtw89_acpi_dsm_result res = {};
+	int ret;
+
+	regulatory->txpwr_uk_follow_etsi = true;
+
+	ret = rtw89_acpi_evaluate_dsm(rtwdev, RTW89_ACPI_DSM_FUNC_REG_RULES_EN, &res);
+	if (ret) {
+		rtw89_debug(rtwdev, RTW89_DBG_REGD,
+			    "acpi: cannot eval policy reg-rules: %d\n", ret);
+		return;
+	}
+
+	ptr = res.u.policy_reg_rules;
+
+	regulatory->txpwr_uk_follow_etsi =
+		!u8_get_bits(ptr->conf, RTW89_ACPI_CONF_REG_RULE_REGD_UK);
+
+	kfree(ptr);
+}
+
 int rtw89_regd_setup(struct rtw89_dev *rtwdev)
 {
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	struct rtw89_fw_elm_info *elm_info = &rtwdev->fw.elm_info;
+	const struct rtw89_regd_data *regd_data = elm_info->regd;
 	struct wiphy *wiphy = rtwdev->hw->wiphy;
 
+	if (regd_data) {
+		regulatory->ctrl.nr = regd_data->nr;
+		regulatory->ctrl.map = regd_data->map;
+	} else {
+		regulatory->ctrl.nr = ARRAY_SIZE(rtw89_regd_map);
+		regulatory->ctrl.map = rtw89_regd_map;
+	}
+
+	regulatory->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT;
+
+	rtw89_regd_setup_reg_rules(rtwdev);
+
 	if (!wiphy)
 		return -EINVAL;
 
@@ -585,21 +711,16 @@ int rtw89_regd_setup(struct rtw89_dev *rtwdev)
 	return 0;
 }
 
-int rtw89_regd_init(struct rtw89_dev *rtwdev,
-		    void (*reg_notifier)(struct wiphy *wiphy,
-					 struct regulatory_request *request))
+int rtw89_regd_init_hint(struct rtw89_dev *rtwdev)
 {
-	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
 	const struct rtw89_regd *chip_regd;
 	struct wiphy *wiphy = rtwdev->hw->wiphy;
 	int ret;
 
-	regulatory->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT;
-
 	if (!wiphy)
 		return -EINVAL;
 
-	chip_regd = rtw89_regd_find_reg_by_name(rtwdev->efuse.country_code);
+	chip_regd = rtw89_regd_find_reg_by_name(rtwdev, rtwdev->efuse.country_code);
 #if defined(__FreeBSD__)
 	rtwdev->regulatory.regd = chip_regd;
 #endif
@@ -642,7 +763,7 @@ static void rtw89_regd_apply_policy_unii4(struct rtw89_dev *rtwdev,
 	if (!chip->support_unii4)
 		return;
 
-	index = rtw89_regd_get_index(regd);
+	index = rtw89_regd_get_index(rtwdev, regd);
 	if (index != RTW89_REGD_MAX_COUNTRY_NUM &&
 	    !test_bit(index, regulatory->block_unii4))
 		return;
@@ -660,7 +781,7 @@ static bool regd_is_6ghz_blocked(struct rtw89_dev *rtwdev)
 	const struct rtw89_regd *regd = regulatory->regd;
 	u8 index;
 
-	index = rtw89_regd_get_index(regd);
+	index = rtw89_regd_get_index(rtwdev, regd);
 	if (index != RTW89_REGD_MAX_COUNTRY_NUM &&
 	    !test_bit(index, regulatory->block_6ghz))
 		return false;
@@ -701,11 +822,47 @@ static void rtw89_regd_apply_policy_6ghz(struct rtw89_dev *rtwdev,
 		sband->channels[i].flags |= IEEE80211_CHAN_DISABLED;
 }
 
+static void rtw89_regd_apply_policy_tas(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	const struct rtw89_regd *regd = regulatory->regd;
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	u8 tas_country;
+
+	if (!tas->enable)
+		return;
+
+	if (memcmp("US", regd->alpha2, 2) == 0)
+		tas_country = RTW89_ACPI_CONF_TAS_US;
+	else if (memcmp("CA", regd->alpha2, 2) == 0)
+		tas_country = RTW89_ACPI_CONF_TAS_CA;
+	else if (memcmp("KR", regd->alpha2, 2) == 0)
+		tas_country = RTW89_ACPI_CONF_TAS_KR;
+	else
+		tas_country = RTW89_ACPI_CONF_TAS_OTHERS;
+
+	tas->block_regd = !(tas->enabled_countries & tas_country &&
+			    test_bit(RTW89_REGD_FUNC_TAS, regd->func_bitmap));
+}
+
+static void rtw89_regd_apply_policy_ant_gain(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	struct rtw89_ant_gain_info *ant_gain = &rtwdev->ant_gain;
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_regd *regd = regulatory->regd;
+
+	if (!chip->support_ant_gain)
+		return;
+
+	ant_gain->block_country = !test_bit(RTW89_REGD_FUNC_DAG, regd->func_bitmap);
+}
+
 static void rtw89_regd_notifier_apply(struct rtw89_dev *rtwdev,
 				      struct wiphy *wiphy,
 				      struct regulatory_request *request)
 {
-	rtwdev->regulatory.regd = rtw89_regd_find_reg_by_name(request->alpha2);
+	rtwdev->regulatory.regd = rtw89_regd_find_reg_by_name(rtwdev, request->alpha2);
 	/* This notification might be set from the system of distros,
 	 * and it does not expect the regulatory will be modified by
 	 * connecting to an AP (i.e. country ie).
@@ -718,14 +875,17 @@ static void rtw89_regd_notifier_apply(struct rtw89_dev *rtwdev,
 
 	rtw89_regd_apply_policy_unii4(rtwdev, wiphy);
 	rtw89_regd_apply_policy_6ghz(rtwdev, wiphy);
+	rtw89_regd_apply_policy_tas(rtwdev);
+	rtw89_regd_apply_policy_ant_gain(rtwdev);
 }
 
+static
 void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request)
 {
 	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
 	struct rtw89_dev *rtwdev = hw->priv;
 
-	mutex_lock(&rtwdev->mutex);
+	wiphy_lock(wiphy);
 	rtw89_leave_ps_mode(rtwdev);
 
 	if (wiphy->regd) {
@@ -741,7 +901,7 @@ void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request
 	rtw89_core_set_chip_txpwr(rtwdev);
 
 exit:
-	mutex_unlock(&rtwdev->mutex);
+	wiphy_unlock(wiphy);
 }
 
 /* Maximum Transmit Power field (@raw) can be EIRP or PSD.
@@ -930,7 +1090,7 @@ static bool __rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev)
 		sel = RTW89_REG_6GHZ_POWER_DFLT;
 
 	if (sel == RTW89_REG_6GHZ_POWER_STD) {
-		index = rtw89_regd_get_index(regd);
+		index = rtw89_regd_get_index(rtwdev, regd);
 		if (index == RTW89_REGD_MAX_COUNTRY_NUM ||
 		    test_bit(index, regulatory->block_6ghz_sp)) {
 			rtw89_debug(rtwdev, RTW89_DBG_REGD,
@@ -954,7 +1114,16 @@ static int rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev,
 				       struct rtw89_vif_link *rtwvif_link, bool active,
 				       unsigned int *changed)
 {
+	struct rtw89_regulatory_info *regulatory = &rtwdev->regulatory;
+	const struct rtw89_regd *regd = regulatory->regd;
+	bool blocked[NUM_OF_RTW89_REG_6GHZ_POWER] = {};
+	u8 index = rtw89_regd_get_index(rtwdev, regd);
 	struct ieee80211_bss_conf *bss_conf;
+	bool dflt = false;
+
+	if (index == RTW89_REGD_MAX_COUNTRY_NUM ||
+	    test_bit(index, regulatory->block_6ghz_vlp))
+		blocked[RTW89_REG_6GHZ_POWER_VLP] = true;
 
 	rcu_read_lock();
 
@@ -973,6 +1142,7 @@ static int rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev,
 			break;
 		default:
 			rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT;
+			dflt = true;
 			break;
 		}
 	} else {
@@ -981,6 +1151,14 @@ static int rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev,
 
 	rcu_read_unlock();
 
+	if (!dflt && blocked[rtwvif_link->reg_6ghz_power]) {
+		rtw89_debug(rtwdev, RTW89_DBG_REGD,
+			    "%c%c 6 GHz power type-%u is blocked by policy\n",
+			    regd->alpha2[0], regd->alpha2[1],
+			    rtwvif_link->reg_6ghz_power);
+		return -EINVAL;
+	}
+
 	*changed += __rtw89_reg_6ghz_power_recalc(rtwdev);
 	return 0;
 }
@@ -991,7 +1169,7 @@ int rtw89_reg_6ghz_recalc(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvi
 	unsigned int changed = 0;
 	int ret;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	/* The result of reg_6ghz_tpe may depend on reg_6ghz_power type,
 	 * so must do reg_6ghz_tpe_recalc() after reg_6ghz_power_recalc().
diff --git a/sys/contrib/dev/rtw89/rtw8851b.c b/sys/contrib/dev/rtw89/rtw8851b.c
index 6f7a068b30a5..b2d75de83602 100644
--- a/sys/contrib/dev/rtw89/rtw8851b.c
+++ b/sys/contrib/dev/rtw89/rtw8851b.c
@@ -51,6 +51,48 @@ static const struct rtw89_hfc_param_ini rtw8851b_hfc_param_ini_pcie[] = {
 	[RTW89_QTA_INVALID] = {NULL},
 };
 
+static const struct rtw89_hfc_ch_cfg rtw8851b_hfc_chcfg_usb[] = {
+	{18, 152, grp_0}, /* ACH 0 */
+	{18, 152, grp_0}, /* ACH 1 */
+	{18, 152, grp_0}, /* ACH 2 */
+	{18, 152, grp_0}, /* ACH 3 */
+	{0, 0, grp_0}, /* ACH 4 */
+	{0, 0, grp_0}, /* ACH 5 */
+	{0, 0, grp_0}, /* ACH 6 */
+	{0, 0, grp_0}, /* ACH 7 */
+	{18, 152, grp_0}, /* B0MGQ */
+	{18, 152, grp_0}, /* B0HIQ */
+	{0, 0, grp_0}, /* B1MGQ */
+	{0, 0, grp_0}, /* B1HIQ */
+	{0, 0, 0} /* FWCMDQ */
+};
+
+static const struct rtw89_hfc_pub_cfg rtw8851b_hfc_pubcfg_usb = {
+	152, /* Group 0 */
+	0, /* Group 1 */
+	152, /* Public Max */
+	0 /* WP threshold */
+};
+
+static const struct rtw89_hfc_prec_cfg rtw8851b_hfc_preccfg_usb = {
+	9, /* CH 0-11 pre-cost */
+	32, /* H2C pre-cost */
+	64, /* WP CH 0-7 pre-cost */
+	24, /* WP CH 8-11 pre-cost */
+	1, /* CH 0-11 full condition */
+	1, /* H2C full condition */
+	1, /* WP CH 0-7 full condition */
+	1, /* WP CH 8-11 full condition */
+};
+
+static const struct rtw89_hfc_param_ini rtw8851b_hfc_param_ini_usb[] = {
+	[RTW89_QTA_SCC] = {rtw8851b_hfc_chcfg_usb, &rtw8851b_hfc_pubcfg_usb,
+			   &rtw8851b_hfc_preccfg_usb, RTW89_HCIFC_STF},
+	[RTW89_QTA_DLFW] = {NULL, NULL,
+			   &rtw8851b_hfc_preccfg_usb, RTW89_HCIFC_STF},
+	[RTW89_QTA_INVALID] = {NULL},
+};
+
 static const struct rtw89_dle_mem rtw8851b_dle_mem_pcie[] = {
 	[RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size6,
 			   &rtw89_mac_size.ple_size6, &rtw89_mac_size.wde_qt6,
@@ -68,6 +110,32 @@ static const struct rtw89_dle_mem rtw8851b_dle_mem_pcie[] = {
 			       NULL},
 };
 
+static const struct rtw89_dle_mem rtw8851b_dle_mem_usb2[] = {
+	[RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size25,
+			   &rtw89_mac_size.ple_size32, &rtw89_mac_size.wde_qt25,
+			   &rtw89_mac_size.wde_qt25, &rtw89_mac_size.ple_qt72,
+			   &rtw89_mac_size.ple_qt73},
+	[RTW89_QTA_DLFW] = {RTW89_QTA_DLFW, &rtw89_mac_size.wde_size9,
+			    &rtw89_mac_size.ple_size8, &rtw89_mac_size.wde_qt4,
+			    &rtw89_mac_size.wde_qt4, &rtw89_mac_size.ple_qt13,
+			    &rtw89_mac_size.ple_qt13},
+	[RTW89_QTA_INVALID] = {RTW89_QTA_INVALID, NULL, NULL, NULL, NULL, NULL,
+			       NULL},
+};
+
+static const struct rtw89_dle_mem rtw8851b_dle_mem_usb3[] = {
+	[RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size25,
+			   &rtw89_mac_size.ple_size33, &rtw89_mac_size.wde_qt25,
+			   &rtw89_mac_size.wde_qt25, &rtw89_mac_size.ple_qt74,
+			   &rtw89_mac_size.ple_qt75},
+	[RTW89_QTA_DLFW] = {RTW89_QTA_DLFW, &rtw89_mac_size.wde_size9,
+			    &rtw89_mac_size.ple_size8, &rtw89_mac_size.wde_qt4,
+			    &rtw89_mac_size.wde_qt4, &rtw89_mac_size.ple_qt13,
+			    &rtw89_mac_size.ple_qt13},
+	[RTW89_QTA_INVALID] = {RTW89_QTA_INVALID, NULL, NULL, NULL, NULL, NULL,
+			       NULL},
+};
+
 static const struct rtw89_reg3_def rtw8851b_btc_preagc_en_defs[] = {
 	{0x46D0, GENMASK(1, 0), 0x3},
 	{0x4AD4, GENMASK(31, 0), 0xf},
@@ -224,10 +292,17 @@ static const struct rtw89_edcca_regs rtw8851b_edcca_regs = {
 	.edcca_p_mask			= B_EDCCA_LVL_MSK1,
 	.ppdu_level			= R_SEG0R_EDCCA_LVL_V1,
 	.ppdu_mask			= B_EDCCA_LVL_MSK3,
-	.rpt_a				= R_EDCCA_RPT_A,
-	.rpt_b				= R_EDCCA_RPT_B,
-	.rpt_sel			= R_EDCCA_RPT_SEL,
-	.rpt_sel_mask			= B_EDCCA_RPT_SEL_MSK,
+	.p = {{
+		.rpt_a			= R_EDCCA_RPT_A,
+		.rpt_b			= R_EDCCA_RPT_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_MSK,
+	}, {
+		.rpt_a			= R_EDCCA_RPT_P1_A,
+		.rpt_b			= R_EDCCA_RPT_P1_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_P1_MSK,
+	}},
 	.tx_collision_t2r_st		= R_TX_COLLISION_T2R_ST,
 	.tx_collision_t2r_st_mask	= B_TX_COLLISION_T2R_ST_M,
 };
@@ -310,7 +385,8 @@ static int rtw8851b_pwr_on_func(struct rtw89_dev *rtwdev)
 	rtw89_write8_clr(rtwdev, R_AX_PLATFORM_ENABLE, B_AX_PLATFORM_EN);
 
 	rtw89_write8_set(rtwdev, R_AX_PLATFORM_ENABLE, B_AX_PLATFORM_EN);
-	rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_CALIB_EN_V1);
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE)
+		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_CALIB_EN_V1);
 
 	ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, XTAL_SI_OFF_WEI,
 				      XTAL_SI_OFF_WEI);
@@ -355,8 +431,9 @@ static int rtw8851b_pwr_on_func(struct rtw89_dev *rtwdev)
 
 	rtw89_write32_clr(rtwdev, R_AX_SYS_ISO_CTRL, B_AX_PWC_EV2EF_B14);
 	rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2, B_AX_SYSON_DIS_PMCR_AX_WRMSK);
-	rtw89_write32_set(rtwdev, R_AX_GPIO0_16_EECS_EESK_LED1_PULL_LOW_EN,
-			  B_AX_GPIO10_PULL_LOW_EN | B_AX_GPIO16_PULL_LOW_EN_V1);
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE)
+		rtw89_write32_set(rtwdev, R_AX_GPIO0_16_EECS_EESK_LED1_PULL_LOW_EN,
+				  B_AX_GPIO10_PULL_LOW_EN | B_AX_GPIO16_PULL_LOW_EN_V1);
 
 	if (rtwdev->hal.cv == CHIP_CAV) {
 		ret = rtw89_read_efuse_ver(rtwdev, &val8);
@@ -440,7 +517,10 @@ static int rtw8851b_pwr_off_func(struct rtw89_dev *rtwdev)
 	if (ret)
 		return ret;
 
-	rtw89_write32(rtwdev, R_AX_WLLPS_CTRL, SW_LPS_OPTION);
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE)
+		rtw89_write32(rtwdev, R_AX_WLLPS_CTRL, SW_LPS_OPTION);
+	else if (rtwdev->hci.type == RTW89_HCI_TYPE_USB)
+		rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_SOP_EDSWR);
 
 	if (rtwdev->hal.cv == CHIP_CAV) {
 		rtw8851b_patch_swr_pfm2pwm(rtwdev);
@@ -449,19 +529,18 @@ static int rtw8851b_pwr_off_func(struct rtw89_dev *rtwdev)
 		rtw89_write32_set(rtwdev, R_AX_SPSANA_ON_CTRL1, B_AX_FPWMDELAY);
 	}
 
-	rtw89_write32_set(rtwdev, R_AX_SYS_PW_CTRL, B_AX_APFM_SWLPS);
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE) {
+		rtw89_write32_set(rtwdev, R_AX_SYS_PW_CTRL, B_AX_APFM_SWLPS);
+	} else if (rtwdev->hci.type == RTW89_HCI_TYPE_USB) {
+		val32 = rtw89_read32(rtwdev, R_AX_SYS_PW_CTRL);
+		val32 &= ~B_AX_AFSM_PCIE_SUS_EN;
+		val32 |= B_AX_AFSM_WLSUS_EN;
+		rtw89_write32(rtwdev, R_AX_SYS_PW_CTRL, val32);
+	}
 
 	return 0;
 }
 
-static void rtw8851b_efuse_parsing(struct rtw89_efuse *efuse,
-				   struct rtw8851b_efuse *map)
-{
-	ether_addr_copy(efuse->addr, map->e.mac_addr);
-	efuse->rfe_type = map->rfe_type;
-	efuse->xtal_cap = map->xtal_k;
-}
-
 static void rtw8851b_efuse_parsing_tssi(struct rtw89_dev *rtwdev,
 					struct rtw8851b_efuse *map)
 {
@@ -542,12 +621,18 @@ static int rtw8851b_read_efuse(struct rtw89_dev *rtwdev, u8 *log_map,
 
 	switch (rtwdev->hci.type) {
 	case RTW89_HCI_TYPE_PCIE:
-		rtw8851b_efuse_parsing(efuse, map);
+		ether_addr_copy(efuse->addr, map->e.mac_addr);
+		break;
+	case RTW89_HCI_TYPE_USB:
+		ether_addr_copy(efuse->addr, map->u.mac_addr);
 		break;
 	default:
 		return -EOPNOTSUPP;
 	}
 
+	efuse->rfe_type = map->rfe_type;
+	efuse->xtal_cap = map->xtal_k;
+
 	rtw89_info(rtwdev, "chip rfe_type is %d\n", efuse->rfe_type);
 
 	return 0;
@@ -705,12 +790,22 @@ static void rtw8851b_phycap_parsing_gain_comp(struct rtw89_dev *rtwdev, u8 *phyc
 	gain->comp_valid = valid;
 }
 
+static void rtw8851b_phycap_parsing_adc_td(struct rtw89_dev *rtwdev, u8 *phycap_map)
+{
+	u32 phycap_addr = rtwdev->chip->phycap_addr;
+	struct rtw89_efuse *efuse = &rtwdev->efuse;
+	const u32 addr_adc_td = 0x5AF;
+
+	efuse->adc_td = phycap_map[addr_adc_td - phycap_addr] & GENMASK(4, 0);
+}
+
 static int rtw8851b_read_phycap(struct rtw89_dev *rtwdev, u8 *phycap_map)
 {
 	rtw8851b_phycap_parsing_tssi(rtwdev, phycap_map);
 	rtw8851b_phycap_parsing_thermal_trim(rtwdev, phycap_map);
 	rtw8851b_phycap_parsing_pa_bias_trim(rtwdev, phycap_map);
 	rtw8851b_phycap_parsing_gain_comp(rtwdev, phycap_map);
+	rtw8851b_phycap_parsing_adc_td(rtwdev, phycap_map);
 
 	return 0;
 }
@@ -1076,39 +1171,72 @@ static void rtw8851b_ctrl_ch(struct rtw89_dev *rtwdev,
 
 static void rtw8851b_bw_setting(struct rtw89_dev *rtwdev, u8 bw)
 {
-	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_CTL, 0x8);
-	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_EN, 0x2);
-	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_BW0, 0x2);
-	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW1, B_P0_CFCH_BW1, 0x4);
+	struct rtw89_efuse *efuse = &rtwdev->efuse;
+	u8 adc_bw_sel;
+
+	switch (efuse->adc_td) {
+	default:
+	case 0x19:
+		adc_bw_sel = 0x4;
+		break;
+	case 0x11:
+		adc_bw_sel = 0x5;
+		break;
+	case 0x9:
+		adc_bw_sel = 0x3;
+		break;
+	}
+
+	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW1, B_P0_CFCH_BW1, adc_bw_sel);
 	rtw89_phy_write32_mask(rtwdev, R_DRCK, B_DRCK_MUL, 0xf);
 	rtw89_phy_write32_mask(rtwdev, R_ADCMOD, B_ADCMOD_LP, 0xa);
-	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK, B_P0_RXCK_ADJ, 0x92);
+	rtw89_phy_write32_mask(rtwdev, R_DCIM, B_DCIM_RC, 0x3);
 
 	switch (bw) {
 	case RTW89_CHANNEL_WIDTH_5:
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_CTL, 0x8);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_EN, 0x2);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_BW0, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_DCIM, B_DCIM_FR, 0x1);
 		rtw89_phy_write32_mask(rtwdev, R_WDADC, B_WDADC_SEL, 0x0);
 		rtw89_phy_write32_mask(rtwdev, R_ADDCK0D, B_ADDCK_DS, 0x1);
+		rtw89_phy_write32_mask(rtwdev, R_P0_RXCK, B_P0_RXCK_ADJ, 0x92);
 		break;
 	case RTW89_CHANNEL_WIDTH_10:
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_CTL, 0x8);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_EN, 0x2);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_BW0, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_DCIM, B_DCIM_FR, 0x1);
 		rtw89_phy_write32_mask(rtwdev, R_WDADC, B_WDADC_SEL, 0x1);
 		rtw89_phy_write32_mask(rtwdev, R_ADDCK0D, B_ADDCK_DS, 0x0);
+		rtw89_phy_write32_mask(rtwdev, R_P0_RXCK, B_P0_RXCK_ADJ, 0x92);
 		break;
 	case RTW89_CHANNEL_WIDTH_20:
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_CTL, 0x8);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_EN, 0x2);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_BW0, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_DCIM, B_DCIM_FR, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_WDADC, B_WDADC_SEL, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_ADDCK0D, B_ADDCK_DS, 0x0);
+		rtw89_phy_write32_mask(rtwdev, R_P0_RXCK, B_P0_RXCK_ADJ, 0x92);
 		break;
 	case RTW89_CHANNEL_WIDTH_40:
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_CTL, 0x8);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_EN, 0x2);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_BW0, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_DCIM, B_DCIM_FR, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_WDADC, B_WDADC_SEL, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_ADDCK0D, B_ADDCK_DS, 0x0);
+		rtw89_phy_write32_mask(rtwdev, R_P0_RXCK, B_P0_RXCK_ADJ, 0x92);
 		break;
 	case RTW89_CHANNEL_WIDTH_80:
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_CTL, 0x8);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_EN, 0x2);
+		rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0, B_P0_CFCH_BW0, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_DCIM, B_DCIM_FR, 0x0);
 		rtw89_phy_write32_mask(rtwdev, R_WDADC, B_WDADC_SEL, 0x2);
 		rtw89_phy_write32_mask(rtwdev, R_ADDCK0D, B_ADDCK_DS, 0x0);
+		rtw89_phy_write32_mask(rtwdev, R_P0_RXCK, B_P0_RXCK_ADJ, 0x92);
 		break;
 	default:
 		rtw89_warn(rtwdev, "Fail to set ADC\n");
@@ -1596,10 +1724,16 @@ static void rtw8851b_rfk_channel(struct rtw89_dev *rtwdev,
 	enum rtw89_chanctx_idx chanctx_idx = rtwvif_link->chanctx_idx;
 	enum rtw89_phy_idx phy_idx = rtwvif_link->phy_idx;
 
+	rtw89_btc_ntfy_conn_rfk(rtwdev, true);
+
 	rtw8851b_rx_dck(rtwdev, phy_idx, chanctx_idx);
 	rtw8851b_iqk(rtwdev, phy_idx, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8851b_tssi(rtwdev, phy_idx, true, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8851b_dpk(rtwdev, phy_idx, chanctx_idx);
+
+	rtw89_btc_ntfy_conn_rfk(rtwdev, false);
 }
 
 static void rtw8851b_rfk_band_changed(struct rtw89_dev *rtwdev,
@@ -2395,6 +2529,7 @@ static const struct rtw89_chip_ops rtw8851b_chip_ops = {
 	.set_txpwr_ctrl		= rtw8851b_set_txpwr_ctrl,
 	.init_txpwr_unit	= rtw8851b_init_txpwr_unit,
 	.get_thermal		= rtw8851b_get_thermal,
+	.chan_to_rf18_val	= NULL,
 	.ctrl_btg_bt_rx		= rtw8851b_ctrl_btg_bt_rx,
 	.query_ppdu		= rtw8851b_query_ppdu,
 	.convert_rpl_to_rssi	= NULL,
@@ -2416,6 +2551,8 @@ static const struct rtw89_chip_ops rtw8851b_chip_ops = {
 	.h2c_default_cmac_tbl	= rtw89_fw_h2c_default_cmac_tbl,
 	.h2c_assoc_cmac_tbl	= rtw89_fw_h2c_assoc_cmac_tbl,
 	.h2c_ampdu_cmac_tbl	= NULL,
+	.h2c_txtime_cmac_tbl	= rtw89_fw_h2c_txtime_cmac_tbl,
+	.h2c_punctured_cmac_tbl	= NULL,
 	.h2c_default_dmac_tbl	= NULL,
 	.h2c_update_beacon	= rtw89_fw_h2c_update_beacon,
 	.h2c_ba_cam		= rtw89_fw_h2c_ba_cam,
@@ -2451,14 +2588,20 @@ const struct rtw89_chip_info rtw8851b_chip_info = {
 	.try_ce_fw		= true,
 	.bbmcu_nr		= 0,
 	.needed_fw_elms		= 0,
+	.fw_blacklist		= NULL,
 	.fifo_size		= 196608,
 	.small_fifo_size	= true,
 	.dle_scc_rsvd_size	= 98304,
 	.max_amsdu_limit	= 3500,
 	.dis_2g_40m_ul_ofdma	= true,
 	.rsvd_ple_ofst		= 0x2f800,
-	.hfc_param_ini		= rtw8851b_hfc_param_ini_pcie,
-	.dle_mem		= rtw8851b_dle_mem_pcie,
+	.hfc_param_ini		= {rtw8851b_hfc_param_ini_pcie,
+				   rtw8851b_hfc_param_ini_usb,
+				   NULL},
+	.dle_mem		= {rtw8851b_dle_mem_pcie,
+				   rtw8851b_dle_mem_usb2,
+				   rtw8851b_dle_mem_usb3,
+				   NULL},
 	.wde_qempty_acq_grpnum	= 4,
 	.wde_qempty_mgq_grpsel	= 4,
 	.rf_base_addr		= {0xe000},
@@ -2489,10 +2632,15 @@ const struct rtw89_chip_info rtw8851b_chip_info = {
 				  BIT(NL80211_CHAN_WIDTH_80),
 	.support_unii4		= true,
 	.support_ant_gain	= false,
+	.support_tas		= false,
+	.support_sar_by_ant	= false,
 	.ul_tb_waveform_ctrl	= true,
 	.ul_tb_pwr_diff		= false,
+	.rx_freq_frome_ie	= true,
 	.hw_sec_hdr		= false,
 	.hw_mgmt_tx_encrypt	= false,
+	.hw_tkip_crypto		= false,
+	.hw_mlo_bmc_crypto	= false,
 	.rf_path_num		= 1,
 	.tx_nss			= 1,
 	.rx_nss			= 1,
@@ -2514,7 +2662,6 @@ const struct rtw89_chip_info rtw8851b_chip_info = {
 	.phycap_size		= 128,
 	.para_ver		= 0,
 	.wlcx_desired		= 0x06000000,
-	.btcx_desired		= 0x7,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
 
diff --git a/sys/contrib/dev/rtw89/rtw8851b_rfk.c b/sys/contrib/dev/rtw89/rtw8851b_rfk.c
index f72b3ac6f149..7a319a6c838a 100644
--- a/sys/contrib/dev/rtw89/rtw8851b_rfk.c
+++ b/sys/contrib/dev/rtw89/rtw8851b_rfk.c
@@ -12,14 +12,14 @@
 #include "rtw8851b_rfk_table.h"
 #include "rtw8851b_table.h"
 
-#define DPK_VER_8851B 0x5
-#define DPK_KIP_REG_NUM_8851B 7
+#define DPK_VER_8851B 0x11
+#define DPK_KIP_REG_NUM_8851B 8
 #define DPK_RF_REG_NUM_8851B 4
 #define DPK_KSET_NUM 4
 #define RTW8851B_RXK_GROUP_NR 4
 #define RTW8851B_RXK_GROUP_IDX_NR 2
 #define RTW8851B_TXK_GROUP_NR 1
-#define RTW8851B_IQK_VER 0x2a
+#define RTW8851B_IQK_VER 0x14
 #define RTW8851B_IQK_SS 1
 #define RTW8851B_LOK_GRAM 10
 #define RTW8851B_TSSI_PATH_NR 1
@@ -85,6 +85,24 @@ enum rf_mode {
 	RF_RXK2 = 0x7,
 };
 
+enum adc_ck {
+	ADC_NA = 0,
+	ADC_480M = 1,
+	ADC_960M = 2,
+	ADC_1920M = 3,
+};
+
+enum dac_ck {
+	DAC_40M = 0,
+	DAC_80M = 1,
+	DAC_120M = 2,
+	DAC_160M = 3,
+	DAC_240M = 4,
+	DAC_320M = 5,
+	DAC_480M = 6,
+	DAC_960M = 7,
+};
+
 static const u32 _tssi_de_cck_long[RF_PATH_NUM_8851B] = {0x5858};
 static const u32 _tssi_de_cck_short[RF_PATH_NUM_8851B] = {0x5860};
 static const u32 _tssi_de_mcs_20m[RF_PATH_NUM_8851B] = {0x5838};
@@ -116,7 +134,7 @@ static const u32 rtw8851b_backup_rf_regs[] = {
 #define BACKUP_RF_REGS_NR ARRAY_SIZE(rtw8851b_backup_rf_regs)
 
 static const u32 dpk_kip_reg[DPK_KIP_REG_NUM_8851B] = {
-	0x813c, 0x8124, 0xc0ec, 0xc0e8, 0xc0c4, 0xc0d4, 0xc0d8};
+	0x813c, 0x8124, 0xc0ec, 0xc0e8, 0xc0c4, 0xc0d4, 0xc0d8, 0x12a0};
 static const u32 dpk_rf_reg[DPK_RF_REG_NUM_8851B] = {0xde, 0x8f, 0x5, 0x10005};
 
 static void _set_ch(struct rtw89_dev *rtwdev, u32 val);
@@ -163,6 +181,51 @@ static void _rfk_drf_direct_cntrl(struct rtw89_dev *rtwdev,
 		rtw89_write_rf(rtwdev, path, RR_BBDC, RR_BBDC_SEL, 0x0);
 }
 
+static void _txck_force(struct rtw89_dev *rtwdev, enum rtw89_rf_path path,
+			bool force, enum dac_ck ck)
+{
+	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK | (path << 13), B_P0_TXCK_ON, 0x0);
+
+	if (!force)
+		return;
+
+	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK | (path << 13), B_P0_TXCK_VAL, ck);
+	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK | (path << 13), B_P0_TXCK_ON, 0x1);
+}
+
+static void _rxck_force(struct rtw89_dev *rtwdev, enum rtw89_rf_path path,
+			bool force, enum adc_ck ck)
+{
+	static const u32 ck960_8851b[] = {0x8, 0x2, 0x2, 0x4, 0xf, 0xa, 0x93};
+	static const u32 ck1920_8851b[] = {0x9, 0x0, 0x0, 0x3, 0xf, 0xa, 0x49};
+	const u32 *data;
+
+	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK | (path << 13), B_P0_RXCK_ON, 0x0);
+	if (!force)
+		return;
+
+	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK | (path << 13), B_P0_RXCK_VAL, ck);
+	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK | (path << 13), B_P0_RXCK_ON, 0x1);
+
+	switch (ck) {
+	case ADC_960M:
+		data = ck960_8851b;
+		break;
+	case ADC_1920M:
+	default:
+		data = ck1920_8851b;
+		break;
+	}
+
+	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0 | (path << 8), B_P0_CFCH_CTL, data[0]);
+	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0 | (path << 8), B_P0_CFCH_EN, data[1]);
+	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW0 | (path << 8), B_P0_CFCH_BW0, data[2]);
+	rtw89_phy_write32_mask(rtwdev, R_P0_CFCH_BW1 | (path << 8), B_P0_CFCH_BW1, data[3]);
+	rtw89_phy_write32_mask(rtwdev, R_DRCK | (path << 8), B_DRCK_MUL, data[4]);
+	rtw89_phy_write32_mask(rtwdev, R_ADCMOD | (path << 8), B_ADCMOD_LP, data[5]);
+	rtw89_phy_write32_mask(rtwdev, R_P0_RXCK | (path << 8), B_P0_RXCK_ADJ, data[6]);
+}
+
 static void _wait_rx_mode(struct rtw89_dev *rtwdev, u8 kpath)
 {
 	u32 rf_mode;
@@ -1044,10 +1107,43 @@ static void _iqk_rxclk_setting(struct rtw89_dev *rtwdev, u8 path)
 
 	rtw89_write_rf(rtwdev, path, RR_RXBB2, RR_RXBB2_CKT, 0x1);
 
-	if (iqk_info->iqk_bw[path] == RTW89_CHANNEL_WIDTH_80)
+	if (iqk_info->iqk_bw[path] == RTW89_CHANNEL_WIDTH_80) {
+		rtw89_phy_write32_mask(rtwdev, R_ADC_FIFO, B_ADC_FIFO_RXK, 0x0101);
+		rtw89_phy_write32_mask(rtwdev, R_UPD_CLK, B_DPD_GDIS, 0x1);
+
+		_rxck_force(rtwdev, path, true, ADC_960M);
+
 		rtw89_rfk_parser(rtwdev, &rtw8851b_iqk_rxclk_80_defs_tbl);
-	else
+	} else {
+		rtw89_phy_write32_mask(rtwdev, R_ADC_FIFO, B_ADC_FIFO_RXK, 0x0101);
+		rtw89_phy_write32_mask(rtwdev, R_UPD_CLK, B_DPD_GDIS, 0x1);
+
+		_rxck_force(rtwdev, path, true, ADC_960M);
+
 		rtw89_rfk_parser(rtwdev, &rtw8851b_iqk_rxclk_others_defs_tbl);
+	}
+
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, (2)before RXK IQK\n", path);
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[07:10] = 0x%x\n", path,
+		    0xc0d4, rtw89_phy_read32_mask(rtwdev, 0xc0d4, GENMASK(10, 7)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[11:14] = 0x%x\n", path,
+		    0xc0d4, rtw89_phy_read32_mask(rtwdev, 0xc0d4, GENMASK(14, 11)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[26:27] = 0x%x\n", path,
+		    0xc0d4, rtw89_phy_read32_mask(rtwdev, 0xc0d4, GENMASK(27, 26)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[05:08] = 0x%x\n", path,
+		    0xc0d8, rtw89_phy_read32_mask(rtwdev, 0xc0d8, GENMASK(8, 5)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[17:21] = 0x%x\n", path,
+		    0xc0c4, rtw89_phy_read32_mask(rtwdev, 0xc0c4, GENMASK(21, 17)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[16:31] = 0x%x\n", path,
+		    0xc0e8, rtw89_phy_read32_mask(rtwdev, 0xc0e8, GENMASK(31, 16)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[04:05] = 0x%x\n", path,
+		    0xc0e4, rtw89_phy_read32_mask(rtwdev, 0xc0e4, GENMASK(5, 4)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[23:31]  = 0x%x\n", path,
+		    0x12a0, rtw89_phy_read32_mask(rtwdev, 0x12a0, GENMASK(31, 23)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[13:14] = 0x%x\n", path,
+		    0xc0ec, rtw89_phy_read32_mask(rtwdev, 0xc0ec, GENMASK(14, 13)));
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]S%x, 0x%x[16:23] = 0x%x\n", path,
+		    0xc0ec, rtw89_phy_read32_mask(rtwdev, 0xc0ec, GENMASK(23, 16)));
 }
 
 static bool _txk_5g_group_sel(struct rtw89_dev *rtwdev,
@@ -1553,6 +1649,14 @@ static void _iqk_macbb_setting(struct rtw89_dev *rtwdev,
 	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]===>%s\n", __func__);
 
 	rtw89_rfk_parser(rtwdev, &rtw8851b_iqk_macbb_defs_tbl);
+
+	_txck_force(rtwdev, path, true, DAC_960M);
+
+	rtw89_phy_write32_mask(rtwdev, R_UPD_CLK, B_DPD_GDIS, 0x1);
+
+	_rxck_force(rtwdev, path, true, ADC_1920M);
+
+	rtw89_rfk_parser(rtwdev, &rtw8851b_iqk_macbb_bh_defs_tbl);
 }
 
 static void _iqk_init(struct rtw89_dev *rtwdev)
@@ -1794,7 +1898,21 @@ static void _dpk_bb_afe_setting(struct rtw89_dev *rtwdev, enum rtw89_rf_path pat
 	rtw89_phy_write32_mask(rtwdev, R_ADC_FIFO, BIT(28 + path), 0x0);
 	rtw89_phy_write32_mask(rtwdev, R_UPD_CLK + (path << 13), MASKDWORD, 0xd801dffd);
 
-	rtw89_rfk_parser(rtwdev, &rtw8851b_iqk_bb_afe_defs_tbl);
+	_txck_force(rtwdev, path, true, DAC_960M);
+	_rxck_force(rtwdev, path, true, ADC_1920M);
+
+	rtw89_phy_write32_mask(rtwdev, R_DCIM, B_DCIM_FR, 0x0);
+	rtw89_phy_write32_mask(rtwdev, R_ADCMOD, B_ADCMOD_AUTO_RST, 0x1);
+	rtw89_phy_write32_mask(rtwdev, R_P0_NRBW, B_P0_NRBW_DBG, 0x1);
+	udelay(1);
+	rtw89_phy_write32_mask(rtwdev, R_ANAPAR_PW15, B_ANAPAR_PW15, 0x1f);
+	udelay(10);
+	rtw89_phy_write32_mask(rtwdev, R_ANAPAR_PW15, B_ANAPAR_PW15, 0x13);
+	udelay(2);
+	rtw89_phy_write32_mask(rtwdev, R_ANAPAR, B_ANAPAR_15, 0x0001);
+	udelay(2);
+	rtw89_phy_write32_mask(rtwdev, R_ANAPAR, B_ANAPAR_15, 0x0041);
+	udelay(10);
 
 	rtw89_phy_write32_mask(rtwdev, R_ADC_FIFO, BIT(20 + path), 0x1);
 	rtw89_phy_write32_mask(rtwdev, R_ADC_FIFO, BIT(28 + path), 0x1);
@@ -1827,6 +1945,17 @@ static void _dpk_tssi_pause(struct rtw89_dev *rtwdev, enum rtw89_rf_path path,
 		    is_pause ? "pause" : "resume");
 }
 
+static
+void _dpk_tssi_slope_k_onoff(struct rtw89_dev *rtwdev, enum rtw89_rf_path path,
+			     bool is_on)
+{
+	rtw89_phy_write32_mask(rtwdev, R_P0_TSSI_SLOPE_CAL + (path << 13),
+			       B_P0_TSSI_SLOPE_CAL_EN, is_on);
+
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] S%d TSSI slpoe_k %s\n", path,
+		    str_on_off(is_on));
+}
+
 static void _dpk_tpg_sel(struct rtw89_dev *rtwdev, enum rtw89_rf_path path, u8 kidx)
 {
 	struct rtw89_dpk_info *dpk = &rtwdev->dpk;
@@ -1874,9 +2003,6 @@ static void _dpk_kip_control_rfc(struct rtw89_dev *rtwdev,
 {
 	rtw89_phy_write32_mask(rtwdev, R_UPD_CLK + (path << 13),
 			       B_IQK_RFC_ON, ctrl_by_kip);
-
-	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] RFC is controlled by %s\n",
-		    ctrl_by_kip ? "KIP" : "BB");
 }
 
 static void _dpk_kip_preset(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
@@ -2279,7 +2405,7 @@ static void _dpk_set_mdpd_para(struct rtw89_dev *rtwdev, u8 order)
 	case 0: /* (5,3,1) */
 		rtw89_phy_write32_mask(rtwdev, R_LDL_NORM, B_LDL_NORM_OP, 0x0);
 		rtw89_phy_write32_mask(rtwdev, R_DPK_IDL, B_DPK_IDL_SEL, 0x2);
-		rtw89_phy_write32_mask(rtwdev, R_LDL_NORM, B_LDL_NORM_PN, 0x4);
+		rtw89_phy_write32_mask(rtwdev, R_LDL_NORM, B_LDL_NORM_PN, 0x3);
 		rtw89_phy_write32_mask(rtwdev, R_MDPK_SYNC, B_MDPK_SYNC_DMAN, 0x1);
 		break;
 	case 1: /* (5,3,0) */
@@ -2315,8 +2441,6 @@ static void _dpk_set_mdpd_para(struct rtw89_dev *rtwdev, u8 order)
 static void _dpk_idl_mpa(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			 enum rtw89_rf_path path, u8 kidx)
 {
-	rtw89_phy_write32_mask(rtwdev, R_LDL_NORM, B_LDL_NORM_MA, 0x1);
-
 	if (rtw89_phy_read32_mask(rtwdev, R_IDL_MPA, B_IDL_MD500) == 0x1)
 		_dpk_set_mdpd_para(rtwdev, 0x2);
 	else if (rtw89_phy_read32_mask(rtwdev, R_IDL_MPA, B_IDL_MD530) == 0x1)
@@ -2419,9 +2543,6 @@ static bool _dpk_main(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 	u8 init_xdbm = 17;
 	bool is_fail;
 
-	if (dpk->bp[path][kidx].band != RTW89_BAND_2G)
-		init_xdbm = 15;
-
 	_dpk_kip_control_rfc(rtwdev, path, false);
 	_rfk_rf_direct_cntrl(rtwdev, path, false);
 	rtw89_write_rf(rtwdev, path, RR_BBDC, RFREG_MASK, 0x03ffd);
@@ -2485,6 +2606,7 @@ static void _dpk_cal_select(struct rtw89_dev *rtwdev, bool force,
 			    "[DPK] ========= S%d[%d] DPK Start =========\n",
 			    path, dpk->cur_idx[path]);
 
+		_dpk_tssi_slope_k_onoff(rtwdev, path, false);
 		_dpk_rxagc_onoff(rtwdev, path, false);
 		_rfk_drf_direct_cntrl(rtwdev, path, false);
 		_dpk_bb_afe_setting(rtwdev, path);
@@ -2502,7 +2624,7 @@ static void _dpk_cal_select(struct rtw89_dev *rtwdev, bool force,
 		_dpk_reload_rf(rtwdev, dpk_rf_reg, rf_bkup, path);
 		_dpk_bb_afe_restore(rtwdev, path);
 		_dpk_rxagc_onoff(rtwdev, path, true);
-
+		_dpk_tssi_slope_k_onoff(rtwdev, path, true);
 		if (rtwdev->is_tssi_mode[path])
 			_dpk_tssi_pause(rtwdev, path, false);
 	}
diff --git a/sys/contrib/dev/rtw89/rtw8851b_rfk_table.c b/sys/contrib/dev/rtw89/rtw8851b_rfk_table.c
index 0abf7978ccab..c5f70c045692 100644
--- a/sys/contrib/dev/rtw89/rtw8851b_rfk_table.c
+++ b/sys/contrib/dev/rtw89/rtw8851b_rfk_table.c
@@ -63,16 +63,7 @@ static const struct rtw89_reg5_def rtw8851b_dack_manual_off_defs[] = {
 RTW89_DECLARE_RFK_TBL(rtw8851b_dack_manual_off_defs);
 
 static const struct rtw89_reg5_def rtw8851b_iqk_rxclk_80_defs[] = {
-	RTW89_DECL_RFK_WM(0x20fc, 0xffff0000, 0x0101),
-	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x2),
 	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x1),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x8),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x2),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x2),
-	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x5),
-	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xf),
 	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x0),
 	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
 	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x0f),
@@ -85,16 +76,7 @@ static const struct rtw89_reg5_def rtw8851b_iqk_rxclk_80_defs[] = {
 RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_rxclk_80_defs);
 
 static const struct rtw89_reg5_def rtw8851b_iqk_rxclk_others_defs[] = {
-	RTW89_DECL_RFK_WM(0x20fc, 0xffff0000, 0x0101),
-	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x2),
 	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x0),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x8),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x2),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x2),
-	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x5),
-	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xf),
 	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x2),
 	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
 	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x0f),
@@ -157,55 +139,38 @@ static const struct rtw89_reg5_def rtw8851b_iqk_macbb_defs[] = {
 	RTW89_DECL_RFK_WM(0x20fc, 0x10000000, 0x0),
 	RTW89_DECL_RFK_WM(0x5670, MASKDWORD, 0xf801fffd),
 	RTW89_DECL_RFK_WM(0x5670, 0x00004000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00008000, 0x1),
 	RTW89_DECL_RFK_WM(0x5670, 0x80000000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00007000, 0x7),
-	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x3),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_macbb_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_macbb_bh_defs[] = {
 	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x2),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x9),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x1),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x0),
-	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x3),
-	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xa),
-	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x0),
-	RTW89_DECL_RFK_WM(0xc0e8, 0x00000040, 0x1),
 	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
+	RTW89_DECL_RFK_DELAY(2),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x1f),
+	RTW89_DECL_RFK_DELAY(10),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x13),
+	RTW89_DECL_RFK_DELAY(2),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0001),
+	RTW89_DECL_RFK_DELAY(2),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0041),
+	RTW89_DECL_RFK_DELAY(10),
+	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
+	RTW89_DECL_RFK_DELAY(2),
 	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x1f),
+	RTW89_DECL_RFK_DELAY(10),
 	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x13),
+	RTW89_DECL_RFK_DELAY(2),
 	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0001),
+	RTW89_DECL_RFK_DELAY(2),
 	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0041),
+	RTW89_DECL_RFK_DELAY(10),
 	RTW89_DECL_RFK_WM(0x20fc, 0x00100000, 0x1),
 	RTW89_DECL_RFK_WM(0x20fc, 0x10000000, 0x1),
 };
 
-RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_macbb_defs);
-
-static const struct rtw89_reg5_def rtw8851b_iqk_bb_afe_defs[] = {
-	RTW89_DECL_RFK_WM(0x5670, 0x00004000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00008000, 0x1),
-	RTW89_DECL_RFK_WM(0x5670, 0x80000000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00007000, 0x7),
-	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
-	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x3),
-	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x2),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x9),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x1),
-	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x0),
-	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x3),
-	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xa),
-	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x0),
-	RTW89_DECL_RFK_WM(0xc0e8, 0x00000040, 0x1),
-	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
-	RTW89_DECL_RFK_WM(0x030c, MASKBYTE3, 0x1f),
-	RTW89_DECL_RFK_WM(0x030c, MASKBYTE3, 0x13),
-	RTW89_DECL_RFK_WM(0x032c, MASKHWORD, 0x0001),
-	RTW89_DECL_RFK_WM(0x032c, MASKHWORD, 0x0041),
-};
-
-RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_bb_afe_defs);
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_macbb_bh_defs);
 
 static const struct rtw89_reg5_def rtw8851b_tssi_sys_defs[] = {
 	RTW89_DECL_RFK_WM(0x12bc, 0x000ffff0, 0xb5b5),
diff --git a/sys/contrib/dev/rtw89/rtw8851b_rfk_table.h b/sys/contrib/dev/rtw89/rtw8851b_rfk_table.h
index febfbecb691c..3f1547f57505 100644
--- a/sys/contrib/dev/rtw89/rtw8851b_rfk_table.h
+++ b/sys/contrib/dev/rtw89/rtw8851b_rfk_table.h
@@ -17,8 +17,8 @@ extern const struct rtw89_rfk_tbl rtw8851b_iqk_rxclk_others_defs_tbl;
 extern const struct rtw89_rfk_tbl rtw8851b_iqk_txk_2ghz_defs_tbl;
 extern const struct rtw89_rfk_tbl rtw8851b_iqk_txk_5ghz_defs_tbl;
 extern const struct rtw89_rfk_tbl rtw8851b_iqk_afebb_restore_defs_tbl;
-extern const struct rtw89_rfk_tbl rtw8851b_iqk_bb_afe_defs_tbl;
 extern const struct rtw89_rfk_tbl rtw8851b_iqk_macbb_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_macbb_bh_defs_tbl;
 extern const struct rtw89_rfk_tbl rtw8851b_tssi_sys_defs_tbl;
 extern const struct rtw89_rfk_tbl rtw8851b_tssi_sys_a_defs_2g_tbl;
 extern const struct rtw89_rfk_tbl rtw8851b_tssi_sys_a_defs_5g_tbl;
diff --git a/sys/contrib/dev/rtw89/rtw8851b_table.c b/sys/contrib/dev/rtw89/rtw8851b_table.c
index 522883c8dfb9..a9c309c245c3 100644
--- a/sys/contrib/dev/rtw89/rtw8851b_table.c
+++ b/sys/contrib/dev/rtw89/rtw8851b_table.c
@@ -2320,9 +2320,9 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x813c, 0x40000000},
 	{0x8140, 0x00000000},
 	{0x8144, 0x0b040b03},
-	{0x8148, 0x07020b04},
-	{0x814c, 0x07020b04},
-	{0x8150, 0xa0a00000},
+	{0x8148, 0x07020a04},
+	{0x814c, 0x07020a04},
+	{0x8150, 0xe4e40000},
 	{0x8158, 0xffffffff},
 	{0x815c, 0xffffffff},
 	{0x8160, 0xffffffff},
@@ -2577,14 +2577,14 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8534, 0x400042fe},
 	{0x8538, 0x50554200},
 	{0x853c, 0xb4183000},
-	{0x8540, 0xe537a50f},
+	{0x8540, 0xe535a50f},
 	{0x8544, 0xf12bf02b},
 	{0x8548, 0xf32bf22b},
 	{0x854c, 0xf62bf42b},
 	{0x8550, 0xf82bf72b},
 	{0x8554, 0xfa2bf92b},
 	{0x8558, 0xfd2bfc2b},
-	{0x855c, 0xe537fe2b},
+	{0x855c, 0xe535fe2b},
 	{0x8560, 0xf12af02a},
 	{0x8564, 0xf32af22a},
 	{0x8568, 0xf52af42a},
@@ -2653,7 +2653,7 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8664, 0x9700140f},
 	{0x8668, 0x00017430},
 	{0x866c, 0xe39ce35e},
-	{0x8670, 0xe52a0bbd},
+	{0x8670, 0xe5280bbd},
 	{0x8674, 0xe36a0001},
 	{0x8678, 0x0001e3c4},
 	{0x867c, 0x55005b30},
@@ -2664,93 +2664,93 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8690, 0x46100005},
 	{0x8694, 0x00010004},
 	{0x8698, 0x30f8e35e},
-	{0x869c, 0xe52a0023},
+	{0x869c, 0xe5280023},
 	{0x86a0, 0x54ed0002},
 	{0x86a4, 0x00230baa},
-	{0x86a8, 0x0002e52a},
+	{0x86a8, 0x0002e528},
 	{0x86ac, 0xe356e3e4},
 	{0x86b0, 0xe35e0001},
 	{0x86b4, 0x002230f3},
-	{0x86b8, 0x0002e52a},
+	{0x86b8, 0x0002e528},
 	{0x86bc, 0x0baa54ec},
-	{0x86c0, 0xe52a0022},
+	{0x86c0, 0xe5280022},
 	{0x86c4, 0xe3e40002},
 	{0x86c8, 0x0001e356},
 	{0x86cc, 0x0baae35e},
 	{0x86d0, 0xe3e430ec},
 	{0x86d4, 0x0001e356},
 	{0x86d8, 0x6d0f6c67},
-	{0x86dc, 0xe52ae39c},
+	{0x86dc, 0xe528e39c},
 	{0x86e0, 0xe39c6c8b},
-	{0x86e4, 0x0bace52a},
+	{0x86e4, 0x0bace528},
 	{0x86e8, 0x6d0f6cb3},
-	{0x86ec, 0xe52ae39c},
+	{0x86ec, 0xe528e39c},
 	{0x86f0, 0x6cdb0bad},
 	{0x86f4, 0xe39c6d0f},
-	{0x86f8, 0x6cf5e52a},
+	{0x86f8, 0x6cf5e528},
 	{0x86fc, 0xe39c6d0f},
-	{0x8700, 0x6c0be52a},
+	{0x8700, 0x6c0be528},
 	{0x8704, 0xe39c6d00},
-	{0x8708, 0x6c25e52a},
-	{0x870c, 0xe52ae39c},
+	{0x8708, 0x6c25e528},
+	{0x870c, 0xe528e39c},
 	{0x8710, 0x6c4df8c6},
-	{0x8714, 0xe52ae39c},
+	{0x8714, 0xe528e39c},
 	{0x8718, 0x6c75f9cf},
-	{0x871c, 0xe52ae39c},
+	{0x871c, 0xe528e39c},
 	{0x8720, 0xe39c6c99},
-	{0x8724, 0xfad6e52a},
+	{0x8724, 0xfad6e528},
 	{0x8728, 0x21e87410},
 	{0x872c, 0x6e670009},
 	{0x8730, 0xe3c46f0f},
-	{0x8734, 0x7410e52f},
+	{0x8734, 0x7410e52d},
 	{0x8738, 0x000b21e8},
 	{0x873c, 0xe3c46e8b},
-	{0x8740, 0x7410e52f},
+	{0x8740, 0x7410e52d},
 	{0x8744, 0x000d21e8},
 	{0x8748, 0x6f0f6eb3},
-	{0x874c, 0xe52fe3c4},
+	{0x874c, 0xe52de3c4},
 	{0x8750, 0xfe07ff08},
 	{0x8754, 0x21e87410},
 	{0x8758, 0x6ec7000e},
-	{0x875c, 0xe52fe3c4},
+	{0x875c, 0xe52de3c4},
 	{0x8760, 0x21e87410},
 	{0x8764, 0x6edb000f},
 	{0x8768, 0xe3c46f0f},
-	{0x876c, 0x7410e52f},
+	{0x876c, 0x7410e52d},
 	{0x8770, 0x001021e8},
 	{0x8774, 0xe3c46eef},
-	{0x8778, 0xff03e52f},
-	{0x877c, 0xe52ffe02},
+	{0x8778, 0xff03e52d},
+	{0x877c, 0xe52dfe02},
 	{0x8780, 0x21e87410},
 	{0x8784, 0x6e110013},
 	{0x8788, 0xe3c46f00},
-	{0x878c, 0xff03e52f},
-	{0x8790, 0xe52ffe02},
+	{0x878c, 0xff03e52d},
+	{0x8790, 0xe52dfe02},
 	{0x8794, 0x21e87410},
 	{0x8798, 0x6e250014},
-	{0x879c, 0xe52fe3c4},
+	{0x879c, 0xe52de3c4},
 	{0x87a0, 0xff08fc24},
 	{0x87a4, 0x7410fe07},
 	{0x87a8, 0x001521e8},
 	{0x87ac, 0xe3c46e39},
-	{0x87b0, 0x7410e52f},
+	{0x87b0, 0x7410e52d},
 	{0x87b4, 0x001621e8},
 	{0x87b8, 0xe3c46e4d},
-	{0x87bc, 0xfd27e52f},
+	{0x87bc, 0xfd27e52d},
 	{0x87c0, 0x21e87410},
 	{0x87c4, 0x6e750018},
-	{0x87c8, 0xe52fe3c4},
+	{0x87c8, 0xe52de3c4},
 	{0x87cc, 0x21e87410},
 	{0x87d0, 0x6e99001a},
-	{0x87d4, 0xe52fe3c4},
+	{0x87d4, 0xe52de3c4},
 	{0x87d8, 0xe36afe24},
 	{0x87dc, 0x63404380},
 	{0x87e0, 0x43006880},
 	{0x87e4, 0x31300bac},
-	{0x87e8, 0xe52f0022},
+	{0x87e8, 0xe52d0022},
 	{0x87ec, 0x54ec0002},
 	{0x87f0, 0x00220baa},
-	{0x87f4, 0x0002e52f},
+	{0x87f4, 0x0002e52d},
 	{0x87f8, 0xe362e3e4},
 	{0x87fc, 0xe36a0001},
 	{0x8800, 0x63404380},
@@ -2770,7 +2770,7 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8838, 0x55010004},
 	{0x883c, 0x66055b40},
 	{0x8840, 0x62000007},
-	{0x8844, 0xe40e6300},
+	{0x8844, 0xe40c6300},
 	{0x8848, 0x09000004},
 	{0x884c, 0x0b400a01},
 	{0x8850, 0x0e010d00},
@@ -2782,13 +2782,13 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8868, 0x00044d01},
 	{0x886c, 0x00074300},
 	{0x8870, 0x05a30562},
-	{0x8874, 0xe40e961f},
+	{0x8874, 0xe40c961f},
 	{0x8878, 0xe37e0004},
 	{0x887c, 0x06a20007},
-	{0x8880, 0xe40e07a3},
+	{0x8880, 0xe40c07a3},
 	{0x8884, 0xe37e0004},
-	{0x8888, 0x0002e3fe},
-	{0x888c, 0x4380e406},
+	{0x8888, 0x0002e3fc},
+	{0x888c, 0x4380e404},
 	{0x8890, 0x4d000007},
 	{0x8894, 0x43000004},
 	{0x8898, 0x000742fe},
@@ -2815,13 +2815,13 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x88ec, 0x42000004},
 	{0x88f0, 0x00070001},
 	{0x88f4, 0x62006220},
-	{0x88f8, 0x0001e406},
+	{0x88f8, 0x0001e404},
 	{0x88fc, 0x63000007},
 	{0x8900, 0x09000004},
 	{0x8904, 0x0e010a00},
 	{0x8908, 0x00070032},
-	{0x890c, 0xe40e06a2},
-	{0x8910, 0x0002e41a},
+	{0x890c, 0xe40c06a2},
+	{0x8910, 0x0002e418},
 	{0x8914, 0x000742fe},
 	{0x8918, 0x00044d00},
 	{0x891c, 0x00014200},
@@ -2839,50 +2839,50 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x894c, 0x00014300},
 	{0x8950, 0x6c060005},
 	{0x8954, 0xe2aae298},
-	{0x8958, 0xe42ae285},
-	{0x895c, 0xe432e2f3},
+	{0x8958, 0xe428e285},
+	{0x895c, 0xe430e2f3},
 	{0x8960, 0x0001e30c},
 	{0x8964, 0x0005e285},
 	{0x8968, 0xe2986c06},
-	{0x896c, 0xe42ae4a9},
-	{0x8970, 0xe432e2f3},
+	{0x896c, 0xe428e4a7},
+	{0x8970, 0xe430e2f3},
 	{0x8974, 0x0001e30c},
 	{0x8978, 0x6c000005},
 	{0x897c, 0xe2aae298},
-	{0x8980, 0xe445e285},
-	{0x8984, 0xe44de2f3},
+	{0x8980, 0xe443e285},
+	{0x8984, 0xe44be2f3},
 	{0x8988, 0x0001e30c},
 	{0x898c, 0x0005e285},
 	{0x8990, 0xe2986c00},
-	{0x8994, 0xe445e4a9},
-	{0x8998, 0xe44de2f3},
+	{0x8994, 0xe443e4a7},
+	{0x8998, 0xe44be2f3},
 	{0x899c, 0x0001e30c},
 	{0x89a0, 0x6c040005},
 	{0x89a4, 0xe2aae298},
-	{0x89a8, 0xe460e285},
-	{0x89ac, 0xe468e2f3},
+	{0x89a8, 0xe45ee285},
+	{0x89ac, 0xe466e2f3},
 	{0x89b0, 0x0001e30c},
 	{0x89b4, 0x0005e285},
 	{0x89b8, 0xe2986c04},
-	{0x89bc, 0xe460e4a9},
-	{0x89c0, 0xe468e2f3},
+	{0x89bc, 0xe45ee4a7},
+	{0x89c0, 0xe466e2f3},
 	{0x89c4, 0x0001e30c},
 	{0x89c8, 0x6c020005},
 	{0x89cc, 0xe2aae298},
-	{0x89d0, 0xe47be285},
-	{0x89d4, 0xe483e2f3},
+	{0x89d0, 0xe479e285},
+	{0x89d4, 0xe481e2f3},
 	{0x89d8, 0x0001e30c},
 	{0x89dc, 0x0005e285},
 	{0x89e0, 0xe2986c02},
-	{0x89e4, 0xe47be4a9},
-	{0x89e8, 0xe483e2f3},
+	{0x89e4, 0xe479e4a7},
+	{0x89e8, 0xe481e2f3},
 	{0x89ec, 0x0001e30c},
 	{0x89f0, 0x43800004},
 	{0x89f4, 0x610a6008},
 	{0x89f8, 0x63ce6200},
 	{0x89fc, 0x60800006},
 	{0x8a00, 0x00047f00},
-	{0x8a04, 0xe4e04300},
+	{0x8a04, 0xe4de4300},
 	{0x8a08, 0x00070001},
 	{0x8a0c, 0x4d015500},
 	{0x8a10, 0x74200004},
@@ -2895,22 +2895,22 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8a2c, 0x00014300},
 	{0x8a30, 0x74200004},
 	{0x8a34, 0x77000005},
-	{0x8a38, 0x73887e07},
+	{0x8a38, 0x73887e06},
 	{0x8a3c, 0x8f007380},
 	{0x8a40, 0x0004140f},
 	{0x8a44, 0x00057430},
 	{0x8a48, 0x00017300},
-	{0x8a4c, 0x0005e496},
+	{0x8a4c, 0x0005e494},
 	{0x8a50, 0x00017300},
 	{0x8a54, 0x43800004},
 	{0x8a58, 0x0006b103},
 	{0x8a5c, 0x91037cdb},
 	{0x8a60, 0x40db0007},
 	{0x8a64, 0x43000004},
-	{0x8a68, 0x0005e496},
+	{0x8a68, 0x0005e494},
 	{0x8a6c, 0x00067380},
 	{0x8a70, 0x60025d01},
-	{0x8a74, 0xe4ba6200},
+	{0x8a74, 0xe4b86200},
 	{0x8a78, 0x73000005},
 	{0x8a7c, 0x76080007},
 	{0x8a80, 0x00047578},
@@ -2943,8 +2943,8 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8aec, 0x7cdb0006},
 	{0x8af0, 0x00079103},
 	{0x8af4, 0x000440db},
-	{0x8af8, 0xe4964300},
-	{0x8afc, 0xe4ba7e03},
+	{0x8af8, 0xe4944300},
+	{0x8afc, 0xe4b87e03},
 	{0x8b00, 0x43800004},
 	{0x8b04, 0x0006b103},
 	{0x8b08, 0x91037c5b},
@@ -2976,14 +2976,14 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8b70, 0x43000004},
 	{0x8b74, 0x73000005},
 	{0x8b78, 0x76000007},
-	{0x8b7c, 0xe4c30001},
+	{0x8b7c, 0xe4c10001},
 	{0x8b80, 0x00040001},
 	{0x8b84, 0x60004380},
 	{0x8b88, 0x62016100},
 	{0x8b8c, 0x00066310},
 	{0x8b90, 0x00046000},
 	{0x8b94, 0x00014300},
-	{0x8b98, 0x0001e4e0},
+	{0x8b98, 0x0001e4de},
 	{0x8b9c, 0x4e004f02},
 	{0x8ba0, 0x52015302},
 	{0x8ba4, 0x140f0001},
@@ -3030,7 +3030,7 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8c48, 0xbf1ae3ac},
 	{0x8c4c, 0xe36e300b},
 	{0x8c50, 0xe390e377},
-	{0x8c54, 0x0001e523},
+	{0x8c54, 0x0001e521},
 	{0x8c58, 0x54c054bf},
 	{0x8c5c, 0x54c154a3},
 	{0x8c60, 0x4c1854a4},
@@ -3039,7 +3039,7 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8c6c, 0xbf051402},
 	{0x8c70, 0x54a354c1},
 	{0x8c74, 0xbf011402},
-	{0x8c78, 0x54dfe534},
+	{0x8c78, 0x54dfe532},
 	{0x8c7c, 0x54bf0001},
 	{0x8c80, 0x050a54e5},
 	{0x8c84, 0x000154df},
@@ -3060,186 +3060,185 @@ static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
 	{0x8cc0, 0x56005610},
 	{0x8cc4, 0x00018c00},
 	{0x8cc8, 0x57005704},
-	{0x8ccc, 0xa7038e00},
-	{0x8cd0, 0x33f0aff7},
-	{0x8cd4, 0xaf034019},
-	{0x8cd8, 0x33f0402b},
-	{0x8cdc, 0x33df402b},
-	{0x8ce0, 0x57005708},
-	{0x8ce4, 0x57818e00},
-	{0x8ce8, 0x8e005780},
-	{0x8cec, 0x00074380},
-	{0x8cf0, 0x5c005c01},
-	{0x8cf4, 0x00041403},
-	{0x8cf8, 0x00014300},
-	{0x8cfc, 0x0007427f},
-	{0x8d00, 0x62006280},
-	{0x8d04, 0x00049200},
-	{0x8d08, 0x00014200},
-	{0x8d0c, 0x0007427f},
-	{0x8d10, 0x63146394},
-	{0x8d14, 0x00049200},
-	{0x8d18, 0x00014200},
-	{0x8d1c, 0x42fe0004},
-	{0x8d20, 0x4d010007},
-	{0x8d24, 0x42000004},
-	{0x8d28, 0x140f7420},
-	{0x8d2c, 0x57005710},
-	{0x8d30, 0x0001141f},
-	{0x8d34, 0x42fe0004},
-	{0x8d38, 0x4d010007},
-	{0x8d3c, 0x42000004},
-	{0x8d40, 0x140f7420},
-	{0x8d44, 0x000742bf},
-	{0x8d48, 0x62006240},
-	{0x8d4c, 0x0004141f},
-	{0x8d50, 0x00014200},
-	{0x8d54, 0x5d060006},
-	{0x8d58, 0x61046003},
-	{0x8d5c, 0x00056201},
-	{0x8d60, 0x00017310},
-	{0x8d64, 0x43800004},
-	{0x8d68, 0x5e010007},
-	{0x8d6c, 0x140a5e00},
-	{0x8d70, 0x0006b103},
-	{0x8d74, 0x91037f07},
-	{0x8d78, 0x43070007},
-	{0x8d7c, 0x5c000006},
-	{0x8d80, 0x5e035d02},
-	{0x8d84, 0x43000004},
-	{0x8d88, 0x00060001},
-	{0x8d8c, 0x60005d04},
-	{0x8d90, 0x62016104},
-	{0x8d94, 0x73100005},
-	{0x8d98, 0x00040001},
-	{0x8d9c, 0x00074380},
-	{0x8da0, 0x5e005e01},
-	{0x8da4, 0xb103140a},
-	{0x8da8, 0x7fc60006},
-	{0x8dac, 0x00079103},
-	{0x8db0, 0x000643c6},
-	{0x8db4, 0x5d025c00},
-	{0x8db8, 0x00045e03},
-	{0x8dbc, 0x00014300},
-	{0x8dc0, 0x5d040006},
-	{0x8dc4, 0x61046000},
-	{0x8dc8, 0x00056201},
-	{0x8dcc, 0x00017310},
-	{0x8dd0, 0x43800004},
-	{0x8dd4, 0x5e010007},
-	{0x8dd8, 0x140a5e00},
-	{0x8ddc, 0x0006b103},
-	{0x8de0, 0x91037fc6},
-	{0x8de4, 0x43c60007},
-	{0x8de8, 0x5c000006},
-	{0x8dec, 0x5e035d02},
-	{0x8df0, 0x43000004},
-	{0x8df4, 0x00060001},
-	{0x8df8, 0x60025d00},
-	{0x8dfc, 0x62016100},
-	{0x8e00, 0x73000005},
-	{0x8e04, 0x00040001},
-	{0x8e08, 0x00074380},
-	{0x8e0c, 0x5e005e01},
-	{0x8e10, 0xb103140a},
-	{0x8e14, 0x7fc00006},
-	{0x8e18, 0x00079103},
-	{0x8e1c, 0x000643c0},
-	{0x8e20, 0x5d025c00},
-	{0x8e24, 0x00045e03},
-	{0x8e28, 0x00014300},
-	{0x8e2c, 0x7e020005},
-	{0x8e30, 0x42f70004},
-	{0x8e34, 0x6c080005},
-	{0x8e38, 0x42700004},
-	{0x8e3c, 0x73810005},
-	{0x8e40, 0x93007380},
-	{0x8e44, 0x42f70004},
-	{0x8e48, 0x6c000005},
-	{0x8e4c, 0x42000004},
-	{0x8e50, 0x00040001},
-	{0x8e54, 0x00074380},
-	{0x8e58, 0x73007304},
-	{0x8e5c, 0x72401405},
-	{0x8e60, 0x43000004},
-	{0x8e64, 0x74040006},
-	{0x8e68, 0x40010007},
-	{0x8e6c, 0xab004000},
-	{0x8e70, 0x0001140f},
-	{0x8e74, 0x140ae517},
-	{0x8e78, 0x140ae4c3},
-	{0x8e7c, 0x0001e51e},
-	{0x8e80, 0xe4c3e517},
-	{0x8e84, 0x00040001},
-	{0x8e88, 0x00047410},
-	{0x8e8c, 0x42f04380},
-	{0x8e90, 0x62080007},
-	{0x8e94, 0x24206301},
-	{0x8e98, 0x14c80000},
-	{0x8e9c, 0x00002428},
-	{0x8ea0, 0x1a4215f4},
-	{0x8ea4, 0x6300000b},
-	{0x8ea8, 0x42000004},
-	{0x8eac, 0x74304300},
-	{0x8eb0, 0x4380140f},
-	{0x8eb4, 0x73080007},
-	{0x8eb8, 0x00047300},
-	{0x8ebc, 0x00014300},
-	{0x8ec0, 0x4bf00007},
-	{0x8ec4, 0x490b4a8f},
-	{0x8ec8, 0x4a8e48f1},
-	{0x8ecc, 0x48a5490a},
-	{0x8ed0, 0x49094a8d},
-	{0x8ed4, 0x4a8c487d},
-	{0x8ed8, 0x48754908},
-	{0x8edc, 0x49074a8b},
-	{0x8ee0, 0x4a8a4889},
-	{0x8ee4, 0x48b74906},
-	{0x8ee8, 0x49054a89},
-	{0x8eec, 0x4a8848fc},
-	{0x8ef0, 0x48564905},
-	{0x8ef4, 0x49044a87},
-	{0x8ef8, 0x4a8648c1},
-	{0x8efc, 0x483d4904},
-	{0x8f00, 0x49034a85},
-	{0x8f04, 0x4a8448c7},
-	{0x8f08, 0x485e4903},
-	{0x8f0c, 0x49024a83},
-	{0x8f10, 0x4a8248ac},
-	{0x8f14, 0x48624902},
-	{0x8f18, 0x49024a81},
-	{0x8f1c, 0x4a804820},
-	{0x8f20, 0x48004900},
-	{0x8f24, 0x49014a90},
-	{0x8f28, 0x4a10481f},
-	{0x8f2c, 0x00060001},
-	{0x8f30, 0x5f005f80},
-	{0x8f34, 0x00059900},
-	{0x8f38, 0x00017300},
-	{0x8f3c, 0x63800006},
-	{0x8f40, 0x98006300},
-	{0x8f44, 0x549f0001},
-	{0x8f48, 0x5c015400},
-	{0x8f4c, 0x540054df},
-	{0x8f50, 0x00015c02},
-	{0x8f54, 0x07145c01},
-	{0x8f58, 0x5c025400},
-	{0x8f5c, 0x5c020001},
-	{0x8f60, 0x54000714},
-	{0x8f64, 0x00015c01},
-	{0x8f68, 0x4c184c98},
-	{0x8f6c, 0x00080001},
-	{0x8f70, 0x5c020004},
-	{0x8f74, 0x09017430},
-	{0x8f78, 0x0ba60c01},
-	{0x8f7c, 0x77800005},
-	{0x8f80, 0x52200007},
-	{0x8f84, 0x43800004},
-	{0x8f88, 0x610a6008},
-	{0x8f8c, 0x63c26200},
-	{0x8f90, 0x5c000007},
-	{0x8f94, 0x43000004},
-	{0x8f98, 0x00000001},
+	{0x8ccc, 0x33ee8e00},
+	{0x8cd0, 0xaf034019},
+	{0x8cd4, 0x33ee402b},
+	{0x8cd8, 0x33df402b},
+	{0x8cdc, 0x57005708},
+	{0x8ce0, 0x57818e00},
+	{0x8ce4, 0x8e005780},
+	{0x8ce8, 0x00074380},
+	{0x8cec, 0x5c005c01},
+	{0x8cf0, 0x00041403},
+	{0x8cf4, 0x00014300},
+	{0x8cf8, 0x0007427f},
+	{0x8cfc, 0x62006280},
+	{0x8d00, 0x00049200},
+	{0x8d04, 0x00014200},
+	{0x8d08, 0x0007427f},
+	{0x8d0c, 0x63146394},
+	{0x8d10, 0x00049200},
+	{0x8d14, 0x00014200},
+	{0x8d18, 0x42fe0004},
+	{0x8d1c, 0x4d010007},
+	{0x8d20, 0x42000004},
+	{0x8d24, 0x140f7420},
+	{0x8d28, 0x57005710},
+	{0x8d2c, 0x0001141f},
+	{0x8d30, 0x42fe0004},
+	{0x8d34, 0x4d010007},
+	{0x8d38, 0x42000004},
+	{0x8d3c, 0x140f7420},
+	{0x8d40, 0x000742bf},
+	{0x8d44, 0x62006240},
+	{0x8d48, 0x0004141f},
+	{0x8d4c, 0x00014200},
+	{0x8d50, 0x5d060006},
+	{0x8d54, 0x61046003},
+	{0x8d58, 0x00056200},
+	{0x8d5c, 0x00017310},
+	{0x8d60, 0x43800004},
+	{0x8d64, 0x5e010007},
+	{0x8d68, 0x140a5e00},
+	{0x8d6c, 0x0006b103},
+	{0x8d70, 0x91037f07},
+	{0x8d74, 0x43070007},
+	{0x8d78, 0x5c000006},
+	{0x8d7c, 0x5e035d02},
+	{0x8d80, 0x43000004},
+	{0x8d84, 0x00060001},
+	{0x8d88, 0x60005d04},
+	{0x8d8c, 0x62006104},
+	{0x8d90, 0x73100005},
+	{0x8d94, 0x00040001},
+	{0x8d98, 0x00074380},
+	{0x8d9c, 0x5e005e01},
+	{0x8da0, 0xb103140a},
+	{0x8da4, 0x7fc60006},
+	{0x8da8, 0x00079103},
+	{0x8dac, 0x000643c6},
+	{0x8db0, 0x5d025c00},
+	{0x8db4, 0x00045e03},
+	{0x8db8, 0x00014300},
+	{0x8dbc, 0x5d040006},
+	{0x8dc0, 0x61046000},
+	{0x8dc4, 0x00056200},
+	{0x8dc8, 0x00017310},
+	{0x8dcc, 0x43800004},
+	{0x8dd0, 0x5e010007},
+	{0x8dd4, 0x140a5e00},
+	{0x8dd8, 0x0006b103},
+	{0x8ddc, 0x91037fc6},
+	{0x8de0, 0x43c60007},
+	{0x8de4, 0x5c000006},
+	{0x8de8, 0x5e035d02},
+	{0x8dec, 0x43000004},
+	{0x8df0, 0x00060001},
+	{0x8df4, 0x60025d00},
+	{0x8df8, 0x62006100},
+	{0x8dfc, 0x73000005},
+	{0x8e00, 0x00040001},
+	{0x8e04, 0x00074380},
+	{0x8e08, 0x5e005e01},
+	{0x8e0c, 0xb103140a},
+	{0x8e10, 0x7fc00006},
+	{0x8e14, 0x00079103},
+	{0x8e18, 0x000643c0},
+	{0x8e1c, 0x5d025c00},
+	{0x8e20, 0x00045e03},
+	{0x8e24, 0x00014300},
+	{0x8e28, 0x7e020005},
+	{0x8e2c, 0x42f70004},
+	{0x8e30, 0x6c080005},
+	{0x8e34, 0x42700004},
+	{0x8e38, 0x73810005},
+	{0x8e3c, 0x93007380},
+	{0x8e40, 0x42f70004},
+	{0x8e44, 0x6c000005},
+	{0x8e48, 0x42000004},
+	{0x8e4c, 0x00040001},
+	{0x8e50, 0x00074380},
+	{0x8e54, 0x73007304},
+	{0x8e58, 0x72401405},
+	{0x8e5c, 0x43000004},
+	{0x8e60, 0x74040006},
+	{0x8e64, 0x40010007},
+	{0x8e68, 0xab004000},
+	{0x8e6c, 0x0001140f},
+	{0x8e70, 0x140ae515},
+	{0x8e74, 0x140ae4c1},
+	{0x8e78, 0x0001e51c},
+	{0x8e7c, 0xe4c1e515},
+	{0x8e80, 0x00040001},
+	{0x8e84, 0x00047410},
+	{0x8e88, 0x42f04380},
+	{0x8e8c, 0x62080007},
+	{0x8e90, 0x24206301},
+	{0x8e94, 0x14c80000},
+	{0x8e98, 0x00002428},
+	{0x8e9c, 0x1a4215f4},
+	{0x8ea0, 0x6300000b},
+	{0x8ea4, 0x42000004},
+	{0x8ea8, 0x74304300},
+	{0x8eac, 0x4380140f},
+	{0x8eb0, 0x73080007},
+	{0x8eb4, 0x00047300},
+	{0x8eb8, 0x00014300},
+	{0x8ebc, 0x4bf00007},
+	{0x8ec0, 0x490b4a8f},
+	{0x8ec4, 0x4a8e48f1},
+	{0x8ec8, 0x48a5490a},
+	{0x8ecc, 0x49094a8d},
+	{0x8ed0, 0x4a8c487d},
+	{0x8ed4, 0x48754908},
+	{0x8ed8, 0x49074a8b},
+	{0x8edc, 0x4a8a4889},
+	{0x8ee0, 0x48b74906},
+	{0x8ee4, 0x49054a89},
+	{0x8ee8, 0x4a8848fc},
+	{0x8eec, 0x48564905},
+	{0x8ef0, 0x49044a87},
+	{0x8ef4, 0x4a8648c1},
+	{0x8ef8, 0x483d4904},
+	{0x8efc, 0x49034a85},
+	{0x8f00, 0x4a8448c7},
+	{0x8f04, 0x485e4903},
+	{0x8f08, 0x49024a83},
+	{0x8f0c, 0x4a8248ac},
+	{0x8f10, 0x48624902},
+	{0x8f14, 0x49024a81},
+	{0x8f18, 0x4a804820},
+	{0x8f1c, 0x48004900},
+	{0x8f20, 0x49014a90},
+	{0x8f24, 0x4a10481f},
+	{0x8f28, 0x00060001},
+	{0x8f2c, 0x5f005f80},
+	{0x8f30, 0x00059900},
+	{0x8f34, 0x00017300},
+	{0x8f38, 0x63800006},
+	{0x8f3c, 0x98006300},
+	{0x8f40, 0x549f0001},
+	{0x8f44, 0x5c015400},
+	{0x8f48, 0x540054df},
+	{0x8f4c, 0x00015c02},
+	{0x8f50, 0x07145c01},
+	{0x8f54, 0x5c025400},
+	{0x8f58, 0x5c020001},
+	{0x8f5c, 0x54000714},
+	{0x8f60, 0x00015c01},
+	{0x8f64, 0x4c184c98},
+	{0x8f68, 0x00080001},
+	{0x8f6c, 0x5c020004},
+	{0x8f70, 0x09017430},
+	{0x8f74, 0x0ba60c01},
+	{0x8f78, 0x77800005},
+	{0x8f7c, 0x52200007},
+	{0x8f80, 0x43800004},
+	{0x8f84, 0x610a6008},
+	{0x8f88, 0x63c26200},
+	{0x8f8c, 0x5c000007},
+	{0x8f90, 0x43000004},
+	{0x8f94, 0x00000001},
 	{0x8080, 0x00000004},
 	{0x8080, 0x00000000},
 	{0x8088, 0x00000000},
diff --git a/sys/contrib/dev/rtw89/rtw8851be.c b/sys/contrib/dev/rtw89/rtw8851be.c
index cbf200f0068e..b0ad4cb4953b 100644
--- a/sys/contrib/dev/rtw89/rtw8851be.c
+++ b/sys/contrib/dev/rtw89/rtw8851be.c
@@ -89,6 +89,7 @@ static struct pci_driver rtw89_8851be_driver = {
 	.probe		= rtw89_pci_probe,
 	.remove		= rtw89_pci_remove,
 	.driver.pm	= &rtw89_pm_ops,
+	.err_handler    = &rtw89_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name	= KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw89/rtw8851bu.c b/sys/contrib/dev/rtw89/rtw8851bu.c
new file mode 100644
index 000000000000..c3722547c6b0
--- /dev/null
+++ b/sys/contrib/dev/rtw89/rtw8851bu.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/usb.h>
+#include "rtw8851b.h"
+#include "usb.h"
+
+static const struct rtw89_driver_info rtw89_8851bu_info = {
+	.chip = &rtw8851b_chip_info,
+	.variant = NULL,
+	.quirks = NULL,
+};
+
+static const struct usb_device_id rtw_8851bu_id_table[] = {
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0bda, 0xb851, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8851bu_info },
+	/* TP-Link Archer TX10UB Nano */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x3625, 0x010b, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8851bu_info },
+	/* Edimax EW-7611UXB */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x7392, 0xe611, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8851bu_info },
+	{},
+};
+MODULE_DEVICE_TABLE(usb, rtw_8851bu_id_table);
+
+static struct usb_driver rtw_8851bu_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = rtw_8851bu_id_table,
+	.probe = rtw89_usb_probe,
+	.disconnect = rtw89_usb_disconnect,
+};
+module_usb_driver(rtw_8851bu_driver);
+
+MODULE_AUTHOR("Bitterblue Smith <rtl8821cerfe2@gmail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ax wireless 8851BU driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sys/contrib/dev/rtw89/rtw8852a.c b/sys/contrib/dev/rtw89/rtw8852a.c
index 9bd2842c27d5..3bbe2a808844 100644
--- a/sys/contrib/dev/rtw89/rtw8852a.c
+++ b/sys/contrib/dev/rtw89/rtw8852a.c
@@ -522,10 +522,17 @@ static const struct rtw89_edcca_regs rtw8852a_edcca_regs = {
 	.edcca_p_mask			= B_EDCCA_LVL_MSK1,
 	.ppdu_level			= R_SEG0R_EDCCA_LVL,
 	.ppdu_mask			= B_EDCCA_LVL_MSK3,
-	.rpt_a				= R_EDCCA_RPT_A,
-	.rpt_b				= R_EDCCA_RPT_B,
-	.rpt_sel			= R_EDCCA_RPT_SEL,
-	.rpt_sel_mask			= B_EDCCA_RPT_SEL_MSK,
+	.p = {{
+		.rpt_a			= R_EDCCA_RPT_A,
+		.rpt_b			= R_EDCCA_RPT_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_MSK,
+	}, {
+		.rpt_a			= R_EDCCA_RPT_P1_A,
+		.rpt_b			= R_EDCCA_RPT_P1_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_P1_MSK,
+	}},
 	.tx_collision_t2r_st		= R_TX_COLLISION_T2R_ST,
 	.tx_collision_t2r_st_mask	= B_TX_COLLISION_T2R_ST_M,
 };
@@ -1356,10 +1363,16 @@ static void rtw8852a_rfk_channel(struct rtw89_dev *rtwdev,
 	enum rtw89_chanctx_idx chanctx_idx = rtwvif_link->chanctx_idx;
 	enum rtw89_phy_idx phy_idx = rtwvif_link->phy_idx;
 
+	rtw89_btc_ntfy_conn_rfk(rtwdev, true);
+
 	rtw8852a_rx_dck(rtwdev, phy_idx, true, chanctx_idx);
 	rtw8852a_iqk(rtwdev, phy_idx, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852a_tssi(rtwdev, phy_idx, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852a_dpk(rtwdev, phy_idx, chanctx_idx);
+
+	rtw89_btc_ntfy_conn_rfk(rtwdev, false);
 }
 
 static void rtw8852a_rfk_band_changed(struct rtw89_dev *rtwdev,
@@ -2115,6 +2128,7 @@ static const struct rtw89_chip_ops rtw8852a_chip_ops = {
 	.set_txpwr_ctrl		= rtw8852a_set_txpwr_ctrl,
 	.init_txpwr_unit	= rtw8852a_init_txpwr_unit,
 	.get_thermal		= rtw8852a_get_thermal,
+	.chan_to_rf18_val	= NULL,
 	.ctrl_btg_bt_rx		= rtw8852a_ctrl_btg_bt_rx,
 	.query_ppdu		= rtw8852a_query_ppdu,
 	.convert_rpl_to_rssi	= NULL,
@@ -2136,6 +2150,8 @@ static const struct rtw89_chip_ops rtw8852a_chip_ops = {
 	.h2c_default_cmac_tbl	= rtw89_fw_h2c_default_cmac_tbl,
 	.h2c_assoc_cmac_tbl	= rtw89_fw_h2c_assoc_cmac_tbl,
 	.h2c_ampdu_cmac_tbl	= NULL,
+	.h2c_txtime_cmac_tbl	= rtw89_fw_h2c_txtime_cmac_tbl,
+	.h2c_punctured_cmac_tbl	= NULL,
 	.h2c_default_dmac_tbl	= NULL,
 	.h2c_update_beacon	= rtw89_fw_h2c_update_beacon,
 	.h2c_ba_cam		= rtw89_fw_h2c_ba_cam,
@@ -2162,14 +2178,15 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.try_ce_fw		= false,
 	.bbmcu_nr		= 0,
 	.needed_fw_elms		= 0,
+	.fw_blacklist		= NULL,
 	.fifo_size		= 458752,
 	.small_fifo_size	= false,
 	.dle_scc_rsvd_size	= 0,
 	.max_amsdu_limit	= 3500,
 	.dis_2g_40m_ul_ofdma	= true,
 	.rsvd_ple_ofst		= 0x6f800,
-	.hfc_param_ini		= rtw8852a_hfc_param_ini_pcie,
-	.dle_mem		= rtw8852a_dle_mem_pcie,
+	.hfc_param_ini		= {rtw8852a_hfc_param_ini_pcie, NULL, NULL},
+	.dle_mem		= {rtw8852a_dle_mem_pcie, NULL, NULL, NULL},
 	.wde_qempty_acq_grpnum	= 16,
 	.wde_qempty_mgq_grpsel	= 16,
 	.rf_base_addr		= {0xc000, 0xd000},
@@ -2201,10 +2218,15 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 				  BIT(NL80211_CHAN_WIDTH_80),
 	.support_unii4		= false,
 	.support_ant_gain	= false,
+	.support_tas		= false,
+	.support_sar_by_ant	= false,
 	.ul_tb_waveform_ctrl	= false,
 	.ul_tb_pwr_diff		= false,
+	.rx_freq_frome_ie	= true,
 	.hw_sec_hdr		= false,
 	.hw_mgmt_tx_encrypt	= false,
+	.hw_tkip_crypto		= false,
+	.hw_mlo_bmc_crypto	= false,
 	.rf_path_num		= 2,
 	.tx_nss			= 2,
 	.rx_nss			= 2,
@@ -2226,7 +2248,6 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.phycap_size		= 128,
 	.para_ver		= 0x0,
 	.wlcx_desired		= 0x06000000,
-	.btcx_desired		= 0x7,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
 
diff --git a/sys/contrib/dev/rtw89/rtw8852ae.c b/sys/contrib/dev/rtw89/rtw8852ae.c
index 79b9450432e7..cc6ae04e7bab 100644
--- a/sys/contrib/dev/rtw89/rtw8852ae.c
+++ b/sys/contrib/dev/rtw89/rtw8852ae.c
@@ -91,6 +91,7 @@ static struct pci_driver rtw89_8852ae_driver = {
 	.probe		= rtw89_pci_probe,
 	.remove		= rtw89_pci_remove,
 	.driver.pm	= &rtw89_pm_ops,
+	.err_handler    = &rtw89_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name	= KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw89/rtw8852b.c b/sys/contrib/dev/rtw89/rtw8852b.c
index dfb2bf61b0b8..7ede07f7b1eb 100644
--- a/sys/contrib/dev/rtw89/rtw8852b.c
+++ b/sys/contrib/dev/rtw89/rtw8852b.c
@@ -49,6 +49,48 @@ static const struct rtw89_hfc_param_ini rtw8852b_hfc_param_ini_pcie[] = {
 	[RTW89_QTA_INVALID] = {NULL},
 };
 
+static const struct rtw89_hfc_ch_cfg rtw8852b_hfc_chcfg_usb[] = {
+	{18, 152, grp_0}, /* ACH 0 */
+	{18, 152, grp_0}, /* ACH 1 */
+	{18, 152, grp_0}, /* ACH 2 */
+	{18, 152, grp_0}, /* ACH 3 */
+	{0, 0, grp_0}, /* ACH 4 */
+	{0, 0, grp_0}, /* ACH 5 */
+	{0, 0, grp_0}, /* ACH 6 */
+	{0, 0, grp_0}, /* ACH 7 */
+	{18, 152, grp_0}, /* B0MGQ */
+	{18, 152, grp_0}, /* B0HIQ */
+	{0, 0, grp_0}, /* B1MGQ */
+	{0, 0, grp_0}, /* B1HIQ */
+	{0, 0, 0} /* FWCMDQ */
+};
+
+static const struct rtw89_hfc_pub_cfg rtw8852b_hfc_pubcfg_usb = {
+	152, /* Group 0 */
+	0, /* Group 1 */
+	152, /* Public Max */
+	0 /* WP threshold */
+};
+
+static const struct rtw89_hfc_prec_cfg rtw8852b_hfc_preccfg_usb = {
+	9, /* CH 0-11 pre-cost */
+	32, /* H2C pre-cost */
+	64, /* WP CH 0-7 pre-cost */
+	24, /* WP CH 8-11 pre-cost */
+	1, /* CH 0-11 full condition */
+	1, /* H2C full condition */
+	1, /* WP CH 0-7 full condition */
+	1, /* WP CH 8-11 full condition */
+};
+
+static const struct rtw89_hfc_param_ini rtw8852b_hfc_param_ini_usb[] = {
+	[RTW89_QTA_SCC] = {rtw8852b_hfc_chcfg_usb, &rtw8852b_hfc_pubcfg_usb,
+			   &rtw8852b_hfc_preccfg_usb, RTW89_HCIFC_STF},
+	[RTW89_QTA_DLFW] = {NULL, NULL,
+			    &rtw8852b_hfc_preccfg_usb, RTW89_HCIFC_STF},
+	[RTW89_QTA_INVALID] = {NULL},
+};
+
 static const struct rtw89_dle_mem rtw8852b_dle_mem_pcie[] = {
 	[RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size7,
 			   &rtw89_mac_size.ple_size6, &rtw89_mac_size.wde_qt7,
@@ -66,6 +108,19 @@ static const struct rtw89_dle_mem rtw8852b_dle_mem_pcie[] = {
 			       NULL},
 };
 
+static const struct rtw89_dle_mem rtw8852b_dle_mem_usb3[] = {
+	[RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size25,
+			   &rtw89_mac_size.ple_size33, &rtw89_mac_size.wde_qt25,
+			   &rtw89_mac_size.wde_qt25, &rtw89_mac_size.ple_qt74,
+			   &rtw89_mac_size.ple_qt75},
+	[RTW89_QTA_DLFW] = {RTW89_QTA_DLFW, &rtw89_mac_size.wde_size9,
+			    &rtw89_mac_size.ple_size8, &rtw89_mac_size.wde_qt4,
+			    &rtw89_mac_size.wde_qt4, &rtw89_mac_size.ple_qt13,
+			    &rtw89_mac_size.ple_qt13},
+	[RTW89_QTA_INVALID] = {RTW89_QTA_INVALID, NULL, NULL, NULL, NULL, NULL,
+			       NULL},
+};
+
 static const u32 rtw8852b_h2c_regs[RTW89_H2CREG_MAX] = {
 	R_AX_H2CREG_DATA0, R_AX_H2CREG_DATA1,  R_AX_H2CREG_DATA2,
 	R_AX_H2CREG_DATA3
@@ -189,10 +244,17 @@ static const struct rtw89_edcca_regs rtw8852b_edcca_regs = {
 	.edcca_p_mask			= B_EDCCA_LVL_MSK1,
 	.ppdu_level			= R_SEG0R_EDCCA_LVL_V1,
 	.ppdu_mask			= B_EDCCA_LVL_MSK3,
-	.rpt_a				= R_EDCCA_RPT_A,
-	.rpt_b				= R_EDCCA_RPT_B,
-	.rpt_sel			= R_EDCCA_RPT_SEL,
-	.rpt_sel_mask			= B_EDCCA_RPT_SEL_MSK,
+	.p = {{
+		.rpt_a			= R_EDCCA_RPT_A,
+		.rpt_b			= R_EDCCA_RPT_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_MSK,
+	}, {
+		.rpt_a			= R_EDCCA_RPT_P1_A,
+		.rpt_b			= R_EDCCA_RPT_P1_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_P1_MSK,
+	}},
 	.tx_collision_t2r_st		= R_TX_COLLISION_T2R_ST,
 	.tx_collision_t2r_st_mask	= B_TX_COLLISION_T2R_ST_M,
 };
@@ -292,7 +354,8 @@ static int rtw8852b_pwr_on_func(struct rtw89_dev *rtwdev)
 	rtw89_write8_clr(rtwdev, R_AX_PLATFORM_ENABLE, B_AX_PLATFORM_EN);
 
 	rtw89_write8_set(rtwdev, R_AX_PLATFORM_ENABLE, B_AX_PLATFORM_EN);
-	rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_CALIB_EN_V1);
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE)
+		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_CALIB_EN_V1);
 
 	rtw89_write32_set(rtwdev, R_AX_SYS_ADIE_PAD_PWR_CTRL, B_AX_SYM_PADPDN_WL_PTA_1P3);
 
@@ -354,7 +417,7 @@ static int rtw8852b_pwr_on_func(struct rtw89_dev *rtwdev)
 	rtw89_write32_mask(rtwdev, R_AX_SPS_DIG_ON_CTRL0, B_AX_VOL_L1_MASK, 0x9);
 	rtw89_write32_mask(rtwdev, R_AX_SPS_DIG_ON_CTRL0, B_AX_VREFPFM_L_MASK, 0xA);
 
-	if (rtwdev->hal.cv == CHIP_CBV) {
+	if (rtwdev->hal.cv == CHIP_CBV && rtwdev->hci.type == RTW89_HCI_TYPE_PCIE) {
 		rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2, B_AX_SYSON_DIS_PMCR_AX_WRMSK);
 		rtw89_write16_mask(rtwdev, R_AX_HCI_LDO_CTRL, B_AX_R_AX_VADJ_MASK, 0xA);
 		rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2, B_AX_SYSON_DIS_PMCR_AX_WRMSK);
@@ -436,10 +499,22 @@ static int rtw8852b_pwr_off_func(struct rtw89_dev *rtwdev)
 	if (ret)
 		return ret;
 
-	rtw89_write32(rtwdev, R_AX_WLLPS_CTRL, SW_LPS_OPTION);
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE)
+		rtw89_write32(rtwdev, R_AX_WLLPS_CTRL, SW_LPS_OPTION);
+	else if (rtwdev->hci.type == RTW89_HCI_TYPE_USB)
+		rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_SOP_EDSWR);
+
 	rtw89_write32_set(rtwdev, R_AX_SYS_SWR_CTRL1, B_AX_SYM_CTRL_SPS_PWMFREQ);
 	rtw89_write32_mask(rtwdev, R_AX_SPS_DIG_ON_CTRL0, B_AX_REG_ZCDC_H_MASK, 0x3);
-	rtw89_write32_set(rtwdev, R_AX_SYS_PW_CTRL, B_AX_APFM_SWLPS);
+
+	if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE) {
+		rtw89_write32_set(rtwdev, R_AX_SYS_PW_CTRL, B_AX_APFM_SWLPS);
+	} else if (rtwdev->hci.type == RTW89_HCI_TYPE_USB) {
+		val32 = rtw89_read32(rtwdev, R_AX_SYS_PW_CTRL);
+		val32 &= ~B_AX_AFSM_PCIE_SUS_EN;
+		val32 |= B_AX_AFSM_WLSUS_EN;
+		rtw89_write32(rtwdev, R_AX_SYS_PW_CTRL, val32);
+	}
 
 	return 0;
 }
@@ -553,8 +628,11 @@ static void rtw8852b_set_channel_help(struct rtw89_dev *rtwdev, bool enter,
 
 static void rtw8852b_rfk_init(struct rtw89_dev *rtwdev)
 {
+	struct rtw89_rfk_mcc_info *rfk_mcc = &rtwdev->rfk_mcc;
+
 	rtwdev->is_tssi_mode[RF_PATH_A] = false;
 	rtwdev->is_tssi_mode[RF_PATH_B] = false;
+	memset(rfk_mcc, 0, sizeof(*rfk_mcc));
 
 	rtw8852b_dpk_init(rtwdev);
 	rtw8852b_rck(rtwdev);
@@ -568,10 +646,18 @@ static void rtw8852b_rfk_channel(struct rtw89_dev *rtwdev,
 	enum rtw89_chanctx_idx chanctx_idx = rtwvif_link->chanctx_idx;
 	enum rtw89_phy_idx phy_idx = rtwvif_link->phy_idx;
 
+	rtw8852b_mcc_get_ch_info(rtwdev, phy_idx);
+	rtw89_btc_ntfy_conn_rfk(rtwdev, true);
+
 	rtw8852b_rx_dck(rtwdev, phy_idx, chanctx_idx);
 	rtw8852b_iqk(rtwdev, phy_idx, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852b_tssi(rtwdev, phy_idx, true, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852b_dpk(rtwdev, phy_idx, chanctx_idx);
+
+	rtw89_btc_ntfy_conn_rfk(rtwdev, false);
+	rtw89_fw_h2c_rf_ntfy_mcc(rtwdev);
 }
 
 static void rtw8852b_rfk_band_changed(struct rtw89_dev *rtwdev,
@@ -742,6 +828,7 @@ static const struct rtw89_chip_ops rtw8852b_chip_ops = {
 	.set_txpwr_ctrl		= rtw8852bx_set_txpwr_ctrl,
 	.init_txpwr_unit	= rtw8852bx_init_txpwr_unit,
 	.get_thermal		= rtw8852bx_get_thermal,
+	.chan_to_rf18_val	= NULL,
 	.ctrl_btg_bt_rx		= rtw8852bx_ctrl_btg_bt_rx,
 	.query_ppdu		= rtw8852bx_query_ppdu,
 	.convert_rpl_to_rssi	= rtw8852bx_convert_rpl_to_rssi,
@@ -763,6 +850,8 @@ static const struct rtw89_chip_ops rtw8852b_chip_ops = {
 	.h2c_default_cmac_tbl	= rtw89_fw_h2c_default_cmac_tbl,
 	.h2c_assoc_cmac_tbl	= rtw89_fw_h2c_assoc_cmac_tbl,
 	.h2c_ampdu_cmac_tbl	= NULL,
+	.h2c_txtime_cmac_tbl	= rtw89_fw_h2c_txtime_cmac_tbl,
+	.h2c_punctured_cmac_tbl	= NULL,
 	.h2c_default_dmac_tbl	= NULL,
 	.h2c_update_beacon	= rtw89_fw_h2c_update_beacon,
 	.h2c_ba_cam		= rtw89_fw_h2c_ba_cam,
@@ -778,6 +867,10 @@ static const struct rtw89_chip_ops rtw8852b_chip_ops = {
 	.btc_set_policy		= rtw89_btc_set_policy_v1,
 };
 
+static const struct rtw89_chanctx_listener rtw8852b_chanctx_listener = {
+	.callbacks[RTW89_CHANCTX_CALLBACK_RFK] = rtw8852b_rfk_chanctx_cb,
+};
+
 #ifdef CONFIG_PM
 static const struct wiphy_wowlan_support rtw_wowlan_stub_8852b = {
 	.flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT,
@@ -798,14 +891,20 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 	.try_ce_fw		= true,
 	.bbmcu_nr		= 0,
 	.needed_fw_elms		= 0,
+	.fw_blacklist		= &rtw89_fw_blacklist_default,
 	.fifo_size		= 196608,
 	.small_fifo_size	= true,
 	.dle_scc_rsvd_size	= 98304,
 	.max_amsdu_limit	= 5000,
 	.dis_2g_40m_ul_ofdma	= true,
 	.rsvd_ple_ofst		= 0x2f800,
-	.hfc_param_ini		= rtw8852b_hfc_param_ini_pcie,
-	.dle_mem		= rtw8852b_dle_mem_pcie,
+	.hfc_param_ini		= {rtw8852b_hfc_param_ini_pcie,
+				   rtw8852b_hfc_param_ini_usb,
+				   NULL},
+	.dle_mem		= {rtw8852b_dle_mem_pcie,
+				   rtw8852b_dle_mem_usb3,
+				   rtw8852b_dle_mem_usb3,
+				   NULL},
 	.wde_qempty_acq_grpnum	= 4,
 	.wde_qempty_mgq_grpsel	= 4,
 	.rf_base_addr		= {0xe000, 0xf000},
@@ -820,6 +919,7 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 	.nctl_post_table	= NULL,
 	.dflt_parms		= &rtw89_8852b_dflt_parms,
 	.rfe_parms_conf		= NULL,
+	.chanctx_listener	= &rtw8852b_chanctx_listener,
 	.txpwr_factor_bb	= 3,
 	.txpwr_factor_rf	= 2,
 	.txpwr_factor_mac	= 1,
@@ -828,7 +928,7 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 	.tssi_dbw_table		= NULL,
 	.support_macid_num	= RTW89_MAX_MAC_ID_NUM,
 	.support_link_num	= 0,
-	.support_chanctx_num	= 0,
+	.support_chanctx_num	= 2,
 	.support_rnr		= false,
 	.support_bands		= BIT(NL80211_BAND_2GHZ) |
 				  BIT(NL80211_BAND_5GHZ),
@@ -837,10 +937,15 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 				  BIT(NL80211_CHAN_WIDTH_80),
 	.support_unii4		= true,
 	.support_ant_gain	= true,
+	.support_tas		= false,
+	.support_sar_by_ant	= true,
 	.ul_tb_waveform_ctrl	= true,
 	.ul_tb_pwr_diff		= false,
+	.rx_freq_frome_ie	= true,
 	.hw_sec_hdr		= false,
 	.hw_mgmt_tx_encrypt	= false,
+	.hw_tkip_crypto		= false,
+	.hw_mlo_bmc_crypto	= false,
 	.rf_path_num		= 2,
 	.tx_nss			= 2,
 	.rx_nss			= 2,
@@ -862,7 +967,6 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 	.phycap_size		= 128,
 	.para_ver		= 0,
 	.wlcx_desired		= 0x05050000,
-	.btcx_desired		= 0x5,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
 
diff --git a/sys/contrib/dev/rtw89/rtw8852b_common.c b/sys/contrib/dev/rtw89/rtw8852b_common.c
index 0e094ce9c9b0..3fb2972ae6f6 100644
--- a/sys/contrib/dev/rtw89/rtw8852b_common.c
+++ b/sys/contrib/dev/rtw89/rtw8852b_common.c
@@ -8,6 +8,7 @@
 #include "phy.h"
 #include "reg.h"
 #include "rtw8852b_common.h"
+#include "sar.h"
 #include "util.h"
 
 static const struct rtw89_reg3_def rtw8852bx_pmac_ht20_mcs7_tbl[] = {
@@ -171,14 +172,6 @@ static const struct rtw89_reg3_def rtw8852bx_btc_preagc_dis_defs[] = {
 
 static DECLARE_PHY_REG3_TBL(rtw8852bx_btc_preagc_dis_defs);
 
-static void rtw8852be_efuse_parsing(struct rtw89_efuse *efuse,
-				    struct rtw8852bx_efuse *map)
-{
-	ether_addr_copy(efuse->addr, map->e.mac_addr);
-	efuse->rfe_type = map->rfe_type;
-	efuse->xtal_cap = map->xtal_k;
-}
-
 static void rtw8852bx_efuse_parsing_tssi(struct rtw89_dev *rtwdev,
 					 struct rtw8852bx_efuse *map)
 {
@@ -260,12 +253,18 @@ static int __rtw8852bx_read_efuse(struct rtw89_dev *rtwdev, u8 *log_map,
 
 	switch (rtwdev->hci.type) {
 	case RTW89_HCI_TYPE_PCIE:
-		rtw8852be_efuse_parsing(efuse, map);
+		ether_addr_copy(efuse->addr, map->e.mac_addr);
+		break;
+	case RTW89_HCI_TYPE_USB:
+		ether_addr_copy(efuse->addr, map->u.mac_addr);
 		break;
 	default:
 		return -EOPNOTSUPP;
 	}
 
+	efuse->rfe_type = map->rfe_type;
+	efuse->xtal_cap = map->xtal_k;
+
 	rtw89_info(rtwdev, "chip rfe_type is %d\n", efuse->rfe_type);
 
 	return 0;
@@ -621,9 +620,9 @@ static void rtw8852bt_ext_loss_avg_update(struct rtw89_dev *rtwdev,
 	if (ext_loss_a == ext_loss_b) {
 		ext_loss_avg = ext_loss_a;
 	} else {
-		linear = rtw89_db_2_linear(abs(ext_loss_a - ext_loss_b)) + 1;
-		linear = DIV_ROUND_CLOSEST_ULL(linear / 2, 1 << RTW89_LINEAR_FRAC_BITS);
-		ext_loss_avg = rtw89_linear_2_db(linear);
+		linear = rtw89_db_to_linear(abs(ext_loss_a - ext_loss_b)) + 1;
+		linear /= 2;
+		ext_loss_avg = rtw89_linear_to_db(linear);
 		ext_loss_avg += min(ext_loss_a, ext_loss_b);
 	}
 
@@ -1234,6 +1233,7 @@ static u32 rtw8852bx_bb_cal_txpwr_ref(struct rtw89_dev *rtwdev,
 	       u32_encode_bits(ref, B_DPD_REF);
 }
 
+/* @pwr_ofst (unit: 1/8 dBm): power of path A minus power of path B */
 static void rtw8852bx_set_txpwr_ref(struct rtw89_dev *rtwdev,
 				    enum rtw89_phy_idx phy_idx, s16 pwr_ofst)
 {
@@ -1336,6 +1336,27 @@ static void rtw8852bx_set_tx_shape(struct rtw89_dev *rtwdev,
 			       tx_shape_ofdm);
 }
 
+static s16 rtw8852bx_get_txpwr_sar_diff(struct rtw89_dev *rtwdev,
+					const struct rtw89_chan *chan)
+{
+	struct rtw89_sar_parm sar_parm = {
+		.center_freq = chan->freq,
+		.force_path = true,
+	};
+	s16 sar_bb_a, sar_bb_b;
+	s8 sar_mac;
+
+	sar_parm.path = RF_PATH_A;
+	sar_mac = rtw89_query_sar(rtwdev, &sar_parm);
+	sar_bb_a = rtw89_phy_txpwr_mac_to_bb(rtwdev, sar_mac);
+
+	sar_parm.path = RF_PATH_B;
+	sar_mac = rtw89_query_sar(rtwdev, &sar_parm);
+	sar_bb_b = rtw89_phy_txpwr_mac_to_bb(rtwdev, sar_mac);
+
+	return sar_bb_a - sar_bb_b;
+}
+
 static void rtw8852bx_set_txpwr_diff(struct rtw89_dev *rtwdev,
 				     const struct rtw89_chan *chan,
 				     enum rtw89_phy_idx phy_idx)
@@ -1343,6 +1364,7 @@ static void rtw8852bx_set_txpwr_diff(struct rtw89_dev *rtwdev,
 	s16 pwr_ofst;
 
 	pwr_ofst = rtw89_phy_ant_gain_pwr_offset(rtwdev, chan);
+	pwr_ofst += rtw8852bx_get_txpwr_sar_diff(rtwdev, chan);
 	rtw8852bx_set_txpwr_ref(rtwdev, phy_idx, pwr_ofst);
 }
 
diff --git a/sys/contrib/dev/rtw89/rtw8852b_rfk.c b/sys/contrib/dev/rtw89/rtw8852b_rfk.c
index ef47a5facc83..3e39a04c91e4 100644
--- a/sys/contrib/dev/rtw89/rtw8852b_rfk.c
+++ b/sys/contrib/dev/rtw89/rtw8852b_rfk.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2019-2022  Realtek Corporation
  */
 
+#include "chan.h"
 #include "coex.h"
 #include "debug.h"
 #include "mac.h"
@@ -1145,19 +1146,19 @@ static void _lok_res_table(struct rtw89_dev *rtwdev, u8 path, u8 ibias)
 
 static bool _lok_finetune_check(struct rtw89_dev *rtwdev, u8 path)
 {
+	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
 	struct rtw89_iqk_info *iqk_info = &rtwdev->iqk;
+	u8 ch = rfk_mcc->table_idx;
 	bool is_fail1, is_fail2;
 	u32 vbuff_i;
 	u32 vbuff_q;
 	u32 core_i;
 	u32 core_q;
 	u32 tmp;
-	u8 ch;
 
 	tmp = rtw89_read_rf(rtwdev, path, RR_TXMO, RFREG_MASK);
 	core_i = FIELD_GET(RR_TXMO_COI, tmp);
 	core_q = FIELD_GET(RR_TXMO_COQ, tmp);
-	ch = (iqk_info->iqk_times / 2) % RTW89_IQK_CHS_NR;
 
 	if (core_i < 0x2 || core_i > 0x1d || core_q < 0x2 || core_q > 0x1d)
 		is_fail1 = true;
@@ -1386,26 +1387,11 @@ static void _iqk_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, u
 			     enum rtw89_chanctx_idx chanctx_idx)
 {
 	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, chanctx_idx);
+	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
 	struct rtw89_iqk_info *iqk_info = &rtwdev->iqk;
+	u8 idx = rfk_mcc->table_idx;
 	u32 reg_rf18;
 	u32 reg_35c;
-	u8 idx;
-	u8 get_empty_table = false;
-
-	for (idx = 0; idx < RTW89_IQK_CHS_NR; idx++) {
-		if (iqk_info->iqk_mcc_ch[idx][path] == 0) {
-			get_empty_table = true;
-			break;
-		}
-	}
-	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK] (1)idx = %x\n", idx);
-
-	if (!get_empty_table) {
-		idx = iqk_info->iqk_table_idx[path] + 1;
-		if (idx > 1)
-			idx = 0;
-	}
-	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK] (2)idx = %x\n", idx);
 
 	reg_rf18 = rtw89_read_rf(rtwdev, path, RR_CFGCH, RFREG_MASK);
 	reg_35c = rtw89_phy_read32_mask(rtwdev, R_CIRST, B_CIRST_SYN);
@@ -1506,11 +1492,10 @@ static void _iqk_afebb_restore(struct rtw89_dev *rtwdev,
 
 static void _iqk_preset(struct rtw89_dev *rtwdev, u8 path)
 {
-	struct rtw89_iqk_info *iqk_info = &rtwdev->iqk;
-	u8 idx;
+	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
+	u8 idx = rfk_mcc->table_idx;
 
-	idx = iqk_info->iqk_table_idx[path];
-	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK] (3)idx = %x\n", idx);
+	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK] idx = %x\n", idx);
 
 	rtw89_phy_write32_mask(rtwdev, R_COEF_SEL + (path << 8), B_COEF_SEL_IQC, idx);
 	rtw89_phy_write32_mask(rtwdev, R_CFIR_LUT + (path << 8), B_CFIR_LUT_G3, idx);
@@ -3585,9 +3570,10 @@ static void _tssi_alimentk(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 	u8 ch_idx = _tssi_ch_to_idx(rtwdev, channel);
 	struct rtw8852bx_bb_tssi_bak tssi_bak;
 	s32 aliment_diff, tssi_cw_default;
-	u32 start_time, finish_time;
 	u32 bb_reg_backup[8] = {0};
+	ktime_t start_time;
 	const s16 *power;
+	s64 this_time;
 	u8 band;
 	bool ok;
 	u32 tmp;
@@ -3613,7 +3599,7 @@ static void _tssi_alimentk(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 		return;
 	}
 
-	start_time = ktime_get_ns();
+	start_time = ktime_get();
 
 	if (chan->band_type == RTW89_BAND_2G)
 		power = power_2g;
@@ -3738,12 +3724,17 @@ out:
 	rtw8852bx_bb_restore_tssi(rtwdev, phy, &tssi_bak);
 	rtw8852bx_bb_tx_mode_switch(rtwdev, phy, 0);
 
-	finish_time = ktime_get_ns();
-	tssi_info->tssi_alimk_time += finish_time - start_time;
+	this_time = ktime_us_delta(ktime_get(), start_time);
+	tssi_info->tssi_alimk_time += this_time;
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK,
-		    "[TSSI PA K] %s processing time = %d ms\n", __func__,
-		    tssi_info->tssi_alimk_time);
+#if defined(__linux__)
+		    "[TSSI PA K] %s processing time = %lld us (acc = %llu us)\n",
+		    __func__, this_time, tssi_info->tssi_alimk_time);
+#elif defined(__FreeBSD__)
+		    "[TSSI PA K] %s processing time = %jd us (acc = %ju us)\n",
+		    __func__, (intmax_t)this_time, (uintmax_t)tssi_info->tssi_alimk_time);
+#endif
 }
 
 void rtw8852b_dpk_init(struct rtw89_dev *rtwdev)
@@ -4178,3 +4169,49 @@ void rtw8852b_set_channel_rf(struct rtw89_dev *rtwdev,
 	rtw8852b_ctrl_bw_ch(rtwdev, phy_idx, chan->channel, chan->band_type,
 			    chan->band_width);
 }
+
+void rtw8852b_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx)
+{
+	const struct rtw89_chan *chan = rtw89_mgnt_chan_get(rtwdev, 0);
+	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
+	struct rtw89_rfk_chan_desc desc[__RTW89_RFK_CHS_NR_V0] = {};
+	u8 idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(desc); idx++) {
+		struct rtw89_rfk_chan_desc *p = &desc[idx];
+
+		p->ch = rfk_mcc->ch[idx];
+
+		p->has_band = true;
+		p->band = rfk_mcc->band[idx];
+	}
+
+	idx = rtw89_rfk_chan_lookup(rtwdev, desc, ARRAY_SIZE(desc), chan);
+
+	rfk_mcc->ch[idx] = chan->channel;
+	rfk_mcc->band[idx] = chan->band_type;
+	rfk_mcc->table_idx = idx;
+}
+
+void rtw8852b_rfk_chanctx_cb(struct rtw89_dev *rtwdev,
+			     enum rtw89_chanctx_state state)
+{
+	struct rtw89_dpk_info *dpk = &rtwdev->dpk;
+	u8 path;
+
+	switch (state) {
+	case RTW89_CHANCTX_STATE_MCC_START:
+		dpk->is_dpk_enable = false;
+		for (path = 0; path < RTW8852B_DPK_RF_PATH; path++)
+			_dpk_onoff(rtwdev, path, false);
+		break;
+	case RTW89_CHANCTX_STATE_MCC_STOP:
+		dpk->is_dpk_enable = true;
+		for (path = 0; path < RTW8852B_DPK_RF_PATH; path++)
+			_dpk_onoff(rtwdev, path, false);
+		rtw8852b_dpk(rtwdev, RTW89_PHY_0, RTW89_CHANCTX_0);
+		break;
+	default:
+		break;
+	}
+}
diff --git a/sys/contrib/dev/rtw89/rtw8852b_rfk.h b/sys/contrib/dev/rtw89/rtw8852b_rfk.h
index c31ba446e6e0..5fae980d5e2c 100644
--- a/sys/contrib/dev/rtw89/rtw8852b_rfk.h
+++ b/sys/contrib/dev/rtw89/rtw8852b_rfk.h
@@ -27,5 +27,8 @@ void rtw8852b_wifi_scan_notify(struct rtw89_dev *rtwdev, bool scan_start,
 void rtw8852b_set_channel_rf(struct rtw89_dev *rtwdev,
 			     const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx);
+void rtw8852b_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx);
+void rtw8852b_rfk_chanctx_cb(struct rtw89_dev *rtwdev,
+			     enum rtw89_chanctx_state state);
 
 #endif
diff --git a/sys/contrib/dev/rtw89/rtw8852be.c b/sys/contrib/dev/rtw89/rtw8852be.c
index b3b5b9b94f1a..832c0ee008a2 100644
--- a/sys/contrib/dev/rtw89/rtw8852be.c
+++ b/sys/contrib/dev/rtw89/rtw8852be.c
@@ -93,6 +93,7 @@ static struct pci_driver rtw89_8852be_driver = {
 	.probe		= rtw89_pci_probe,
 	.remove		= rtw89_pci_remove,
 	.driver.pm	= &rtw89_pm_ops,
+	.err_handler    = &rtw89_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name	= KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw89/rtw8852bt.c b/sys/contrib/dev/rtw89/rtw8852bt.c
index bde3e1fb7ca6..9427823aca2f 100644
--- a/sys/contrib/dev/rtw89/rtw8852bt.c
+++ b/sys/contrib/dev/rtw89/rtw8852bt.c
@@ -187,10 +187,17 @@ static const struct rtw89_edcca_regs rtw8852bt_edcca_regs = {
 	.edcca_p_mask			= B_EDCCA_LVL_MSK1,
 	.ppdu_level			= R_SEG0R_EDCCA_LVL_V1,
 	.ppdu_mask			= B_EDCCA_LVL_MSK3,
-	.rpt_a				= R_EDCCA_RPT_A,
-	.rpt_b				= R_EDCCA_RPT_B,
-	.rpt_sel			= R_EDCCA_RPT_SEL,
-	.rpt_sel_mask			= B_EDCCA_RPT_SEL_MSK,
+	.p = {{
+		.rpt_a			= R_EDCCA_RPT_A,
+		.rpt_b			= R_EDCCA_RPT_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_MSK,
+	}, {
+		.rpt_a			= R_EDCCA_RPT_P1_A,
+		.rpt_b			= R_EDCCA_RPT_P1_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_P1_MSK,
+	}},
 	.tx_collision_t2r_st		= R_TX_COLLISION_T2R_ST,
 	.tx_collision_t2r_st_mask	= B_TX_COLLISION_T2R_ST_M,
 };
@@ -526,8 +533,11 @@ static void rtw8852bt_set_channel_help(struct rtw89_dev *rtwdev, bool enter,
 
 static void rtw8852bt_rfk_init(struct rtw89_dev *rtwdev)
 {
+	struct rtw89_rfk_mcc_info *rfk_mcc = &rtwdev->rfk_mcc;
+
 	rtwdev->is_tssi_mode[RF_PATH_A] = false;
 	rtwdev->is_tssi_mode[RF_PATH_B] = false;
+	memset(rfk_mcc, 0, sizeof(*rfk_mcc));
 
 	rtw8852bt_dpk_init(rtwdev);
 	rtw8852bt_rck(rtwdev);
@@ -541,10 +551,18 @@ static void rtw8852bt_rfk_channel(struct rtw89_dev *rtwdev,
 	enum rtw89_chanctx_idx chanctx_idx = rtwvif_link->chanctx_idx;
 	enum rtw89_phy_idx phy_idx = rtwvif_link->phy_idx;
 
+	rtw8852bt_mcc_get_ch_info(rtwdev, phy_idx);
+	rtw89_btc_ntfy_conn_rfk(rtwdev, true);
+
 	rtw8852bt_rx_dck(rtwdev, phy_idx, chanctx_idx);
 	rtw8852bt_iqk(rtwdev, phy_idx, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852bt_tssi(rtwdev, phy_idx, true, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852bt_dpk(rtwdev, phy_idx, chanctx_idx);
+
+	rtw89_btc_ntfy_conn_rfk(rtwdev, false);
+	rtw89_fw_h2c_rf_ntfy_mcc(rtwdev);
 }
 
 static void rtw8852bt_rfk_band_changed(struct rtw89_dev *rtwdev,
@@ -676,6 +694,7 @@ static const struct rtw89_chip_ops rtw8852bt_chip_ops = {
 	.set_txpwr_ctrl		= rtw8852bx_set_txpwr_ctrl,
 	.init_txpwr_unit	= rtw8852bx_init_txpwr_unit,
 	.get_thermal		= rtw8852bx_get_thermal,
+	.chan_to_rf18_val	= NULL,
 	.ctrl_btg_bt_rx		= rtw8852bx_ctrl_btg_bt_rx,
 	.query_ppdu		= rtw8852bx_query_ppdu,
 	.convert_rpl_to_rssi	= rtw8852bx_convert_rpl_to_rssi,
@@ -697,6 +716,8 @@ static const struct rtw89_chip_ops rtw8852bt_chip_ops = {
 	.h2c_default_cmac_tbl	= rtw89_fw_h2c_default_cmac_tbl,
 	.h2c_assoc_cmac_tbl	= rtw89_fw_h2c_assoc_cmac_tbl,
 	.h2c_ampdu_cmac_tbl	= NULL,
+	.h2c_txtime_cmac_tbl	= rtw89_fw_h2c_txtime_cmac_tbl,
+	.h2c_punctured_cmac_tbl	= NULL,
 	.h2c_default_dmac_tbl	= NULL,
 	.h2c_update_beacon	= rtw89_fw_h2c_update_beacon,
 	.h2c_ba_cam		= rtw89_fw_h2c_ba_cam,
@@ -712,6 +733,10 @@ static const struct rtw89_chip_ops rtw8852bt_chip_ops = {
 	.btc_set_policy		= rtw89_btc_set_policy_v1,
 };
 
+static const struct rtw89_chanctx_listener rtw8852bt_chanctx_listener = {
+	.callbacks[RTW89_CHANCTX_CALLBACK_RFK] = rtw8852bt_rfk_chanctx_cb,
+};
+
 #ifdef CONFIG_PM
 static const struct wiphy_wowlan_support rtw_wowlan_stub_8852bt = {
 	.flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT,
@@ -732,14 +757,15 @@ const struct rtw89_chip_info rtw8852bt_chip_info = {
 	.try_ce_fw		= true,
 	.bbmcu_nr		= 0,
 	.needed_fw_elms		= RTW89_AX_GEN_DEF_NEEDED_FW_ELEMENTS_NO_6GHZ,
+	.fw_blacklist		= &rtw89_fw_blacklist_default,
 	.fifo_size		= 458752,
 	.small_fifo_size	= true,
 	.dle_scc_rsvd_size	= 98304,
 	.max_amsdu_limit	= 5000,
 	.dis_2g_40m_ul_ofdma	= true,
 	.rsvd_ple_ofst		= 0x6f800,
-	.hfc_param_ini		= rtw8852bt_hfc_param_ini_pcie,
-	.dle_mem		= rtw8852bt_dle_mem_pcie,
+	.hfc_param_ini		= {rtw8852bt_hfc_param_ini_pcie, NULL, NULL},
+	.dle_mem		= {rtw8852bt_dle_mem_pcie, NULL, NULL, NULL},
 	.wde_qempty_acq_grpnum	= 4,
 	.wde_qempty_mgq_grpsel	= 4,
 	.rf_base_addr		= {0xe000, 0xf000},
@@ -753,6 +779,7 @@ const struct rtw89_chip_info rtw8852bt_chip_info = {
 	.nctl_post_table	= NULL,
 	.dflt_parms		= NULL,
 	.rfe_parms_conf		= NULL,
+	.chanctx_listener	= &rtw8852bt_chanctx_listener,
 	.txpwr_factor_bb	= 3,
 	.txpwr_factor_rf	= 2,
 	.txpwr_factor_mac	= 1,
@@ -761,7 +788,7 @@ const struct rtw89_chip_info rtw8852bt_chip_info = {
 	.tssi_dbw_table		= NULL,
 	.support_macid_num	= RTW89_MAX_MAC_ID_NUM,
 	.support_link_num       = 0,
-	.support_chanctx_num	= 1,
+	.support_chanctx_num	= 2,
 	.support_rnr		= false,
 	.support_bands		= BIT(NL80211_BAND_2GHZ) |
 				  BIT(NL80211_BAND_5GHZ),
@@ -770,10 +797,15 @@ const struct rtw89_chip_info rtw8852bt_chip_info = {
 				  BIT(NL80211_CHAN_WIDTH_80),
 	.support_unii4		= true,
 	.support_ant_gain	= true,
+	.support_tas		= false,
+	.support_sar_by_ant	= true,
 	.ul_tb_waveform_ctrl	= true,
 	.ul_tb_pwr_diff		= false,
+	.rx_freq_frome_ie	= true,
 	.hw_sec_hdr		= false,
 	.hw_mgmt_tx_encrypt     = false,
+	.hw_tkip_crypto		= true,
+	.hw_mlo_bmc_crypto	= false,
 	.rf_path_num		= 2,
 	.tx_nss			= 2,
 	.rx_nss			= 2,
@@ -795,7 +827,6 @@ const struct rtw89_chip_info rtw8852bt_chip_info = {
 	.phycap_size		= 128,
 	.para_ver		= 0,
 	.wlcx_desired		= 0x070e0000,
-	.btcx_desired		= 0x7,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
 
diff --git a/sys/contrib/dev/rtw89/rtw8852bt_rfk.c b/sys/contrib/dev/rtw89/rtw8852bt_rfk.c
index 336a83e1d46b..fffa1dbb45b1 100644
--- a/sys/contrib/dev/rtw89/rtw8852bt_rfk.c
+++ b/sys/contrib/dev/rtw89/rtw8852bt_rfk.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2024 Realtek Corporation
  */
 
+#include "chan.h"
 #include "coex.h"
 #include "debug.h"
 #include "fw.h"
@@ -1529,26 +1530,11 @@ static void _iqk_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, u
 			     enum rtw89_chanctx_idx chanctx_idx)
 {
 	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, chanctx_idx);
+	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
 	struct rtw89_iqk_info *iqk_info = &rtwdev->iqk;
-	u8 get_empty_table = false;
+	u8 idx = rfk_mcc->table_idx;
 	u32 reg_rf18;
 	u32 reg_35c;
-	u8 idx;
-
-	for (idx = 0; idx < RTW89_IQK_CHS_NR; idx++) {
-		if (iqk_info->iqk_mcc_ch[idx][path] == 0) {
-			get_empty_table = true;
-			break;
-		}
-	}
-	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK] (1)idx = %x\n", idx);
-
-	if (!get_empty_table) {
-		idx = iqk_info->iqk_table_idx[path] + 1;
-		if (idx > 1)
-			idx = 0;
-	}
-	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK] (2)idx = %x\n", idx);
 
 	reg_rf18 = rtw89_read_rf(rtwdev, path, RR_CFGCH, RFREG_MASK);
 	reg_35c = rtw89_phy_read32_mask(rtwdev, R_CIRST, B_CIRST_SYN);
@@ -1640,7 +1626,8 @@ static void _iqk_afebb_restore(struct rtw89_dev *rtwdev,
 
 static void _iqk_preset(struct rtw89_dev *rtwdev, u8 path)
 {
-	u8 idx = 0;
+	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
+	u8 idx = rfk_mcc->table_idx;
 
 	rtw89_phy_write32_mask(rtwdev, R_COEF_SEL + (path << 8), 0x00000001, idx);
 	rtw89_phy_write32_mask(rtwdev, R_CFIR_LUT + (path << 8), 0x00000008, idx);
@@ -3663,9 +3650,10 @@ static void _tssi_alimentk(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 	u8 ch_idx = _tssi_ch_to_idx(rtwdev, channel);
 	struct rtw8852bx_bb_tssi_bak tssi_bak;
 	s32 aliment_diff, tssi_cw_default;
-	u32 start_time, finish_time;
 	u32 bb_reg_backup[8] = {};
+	ktime_t start_time;
 	const s16 *power;
+	s64 this_time;
 	u8 band;
 	bool ok;
 	u32 tmp;
@@ -3675,7 +3663,7 @@ static void _tssi_alimentk(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 		    "======> %s   channel=%d   path=%d\n", __func__, channel,
 		    path);
 
-	start_time = ktime_get_ns();
+	start_time = ktime_get();
 
 	if (chan->band_type == RTW89_BAND_2G)
 		power = power_2g;
@@ -3802,12 +3790,17 @@ out:
 	rtw8852bx_bb_restore_tssi(rtwdev, phy, &tssi_bak);
 	rtw8852bx_bb_tx_mode_switch(rtwdev, phy, 0);
 
-	finish_time = ktime_get_ns();
-	tssi_info->tssi_alimk_time += finish_time - start_time;
+	this_time = ktime_us_delta(ktime_get(), start_time);
+	tssi_info->tssi_alimk_time += this_time;
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK,
-		    "[TSSI PA K] %s processing time = %d ms\n", __func__,
-		    tssi_info->tssi_alimk_time);
+#if defined(__linux__)
+		    "[TSSI PA K] %s processing time = %lld us (acc = %llu us)\n",
+		    __func__, this_time, tssi_info->tssi_alimk_time);
+#elif defined(__FreeBSD__)
+		    "[TSSI PA K] %s processing time = %jd us (acc = %ju us)\n",
+		    __func__, (intmax_t)this_time, (uintmax_t)tssi_info->tssi_alimk_time);
+#endif
 }
 
 void rtw8852bt_dpk_init(struct rtw89_dev *rtwdev)
@@ -4251,3 +4244,49 @@ void rtw8852bt_set_channel_rf(struct rtw89_dev *rtwdev,
 	rtw8852bt_ctrl_bw_ch(rtwdev, phy_idx, chan->channel, chan->band_type,
 			     chan->band_width);
 }
+
+void rtw8852bt_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx)
+{
+	const struct rtw89_chan *chan = rtw89_mgnt_chan_get(rtwdev, 0);
+	struct rtw89_rfk_mcc_info_data *rfk_mcc = rtwdev->rfk_mcc.data;
+	struct rtw89_rfk_chan_desc desc[__RTW89_RFK_CHS_NR_V0] = {};
+	u8 idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(desc); idx++) {
+		struct rtw89_rfk_chan_desc *p = &desc[idx];
+
+		p->ch = rfk_mcc->ch[idx];
+
+		p->has_band = true;
+		p->band = rfk_mcc->band[idx];
+	}
+
+	idx = rtw89_rfk_chan_lookup(rtwdev, desc, ARRAY_SIZE(desc), chan);
+
+	rfk_mcc->ch[idx] = chan->channel;
+	rfk_mcc->band[idx] = chan->band_type;
+	rfk_mcc->table_idx = idx;
+}
+
+void rtw8852bt_rfk_chanctx_cb(struct rtw89_dev *rtwdev,
+			      enum rtw89_chanctx_state state)
+{
+	struct rtw89_dpk_info *dpk = &rtwdev->dpk;
+	u8 path;
+
+	switch (state) {
+	case RTW89_CHANCTX_STATE_MCC_START:
+		dpk->is_dpk_enable = false;
+		for (path = 0; path < RTW8852BT_SS; path++)
+			_dpk_onoff(rtwdev, path, false);
+		break;
+	case RTW89_CHANCTX_STATE_MCC_STOP:
+		dpk->is_dpk_enable = true;
+		for (path = 0; path < RTW8852BT_SS; path++)
+			_dpk_onoff(rtwdev, path, false);
+		rtw8852bt_dpk(rtwdev, RTW89_PHY_0, RTW89_CHANCTX_0);
+		break;
+	default:
+		break;
+	}
+}
diff --git a/sys/contrib/dev/rtw89/rtw8852bt_rfk.h b/sys/contrib/dev/rtw89/rtw8852bt_rfk.h
index e34560b4905f..a663bbda4075 100644
--- a/sys/contrib/dev/rtw89/rtw8852bt_rfk.h
+++ b/sys/contrib/dev/rtw89/rtw8852bt_rfk.h
@@ -27,5 +27,8 @@ void rtw8852bt_wifi_scan_notify(struct rtw89_dev *rtwdev, bool scan_start,
 void rtw8852bt_set_channel_rf(struct rtw89_dev *rtwdev,
 			      const struct rtw89_chan *chan,
 			      enum rtw89_phy_idx phy_idx);
+void rtw8852bt_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx);
+void rtw8852bt_rfk_chanctx_cb(struct rtw89_dev *rtwdev,
+			      enum rtw89_chanctx_state state);
 
 #endif
diff --git a/sys/contrib/dev/rtw89/rtw8852bte.c b/sys/contrib/dev/rtw89/rtw8852bte.c
index b69fa17beb33..a584c75b801d 100644
--- a/sys/contrib/dev/rtw89/rtw8852bte.c
+++ b/sys/contrib/dev/rtw89/rtw8852bte.c
@@ -95,6 +95,7 @@ static struct pci_driver rtw89_8852bte_driver = {
 	.probe		= rtw89_pci_probe,
 	.remove		= rtw89_pci_remove,
 	.driver.pm	= &rtw89_pm_ops,
+	.err_handler    = &rtw89_pci_err_handler,
 };
 module_pci_driver(rtw89_8852bte_driver);
 
diff --git a/sys/contrib/dev/rtw89/rtw8852bu.c b/sys/contrib/dev/rtw89/rtw8852bu.c
new file mode 100644
index 000000000000..b315cb997758
--- /dev/null
+++ b/sys/contrib/dev/rtw89/rtw8852bu.c
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/usb.h>
+#include "rtw8852b.h"
+#include "usb.h"
+
+static const struct rtw89_driver_info rtw89_8852bu_info = {
+	.chip = &rtw8852b_chip_info,
+	.variant = NULL,
+	.quirks = NULL,
+};
+
+static const struct usb_device_id rtw_8852bu_id_table[] = {
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0bda, 0xb832, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0bda, 0xb83a, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0bda, 0xb852, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0bda, 0xb85a, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0bda, 0xa85b, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0586, 0x3428, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0b05, 0x1a62, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0x6931, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x3574, 0x6121, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x35bc, 0x0100, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x35bc, 0x0108, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x7392, 0x6822, 0xff, 0xff, 0xff),
+	  .driver_info = (kernel_ulong_t)&rtw89_8852bu_info },
+	{},
+};
+MODULE_DEVICE_TABLE(usb, rtw_8852bu_id_table);
+
+static struct usb_driver rtw_8852bu_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = rtw_8852bu_id_table,
+	.probe = rtw89_usb_probe,
+	.disconnect = rtw89_usb_disconnect,
+};
+module_usb_driver(rtw_8852bu_driver);
+
+MODULE_AUTHOR("Bitterblue Smith <rtl8821cerfe2@gmail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852BU driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sys/contrib/dev/rtw89/rtw8852c.c b/sys/contrib/dev/rtw89/rtw8852c.c
index ef80d80dee2e..99f02c7702f9 100644
--- a/sys/contrib/dev/rtw89/rtw8852c.c
+++ b/sys/contrib/dev/rtw89/rtw8852c.c
@@ -12,9 +12,10 @@
 #include "rtw8852c.h"
 #include "rtw8852c_rfk.h"
 #include "rtw8852c_table.h"
+#include "sar.h"
 #include "util.h"
 
-#define RTW8852C_FW_FORMAT_MAX 1
+#define RTW8852C_FW_FORMAT_MAX 2
 #define RTW8852C_FW_BASENAME "rtw89/rtw8852c_fw"
 #define RTW8852C_MODULE_FIRMWARE \
 	RTW8852C_FW_BASENAME "-" __stringify(RTW8852C_FW_FORMAT_MAX) ".bin"
@@ -186,10 +187,17 @@ static const struct rtw89_edcca_regs rtw8852c_edcca_regs = {
 	.edcca_p_mask			= B_EDCCA_LVL_MSK1,
 	.ppdu_level			= R_SEG0R_EDCCA_LVL,
 	.ppdu_mask			= B_EDCCA_LVL_MSK3,
-	.rpt_a				= R_EDCCA_RPT_A,
-	.rpt_b				= R_EDCCA_RPT_B,
-	.rpt_sel			= R_EDCCA_RPT_SEL,
-	.rpt_sel_mask			= B_EDCCA_RPT_SEL_MSK,
+	.p = {{
+		.rpt_a			= R_EDCCA_RPT_A,
+		.rpt_b			= R_EDCCA_RPT_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_MSK,
+	}, {
+		.rpt_a			= R_EDCCA_RPT_P1_A,
+		.rpt_b			= R_EDCCA_RPT_P1_B,
+		.rpt_sel		= R_EDCCA_RPT_SEL,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_P1_MSK,
+	}},
 	.tx_collision_t2r_st		= R_TX_COLLISION_T2R_ST,
 	.tx_collision_t2r_st_mask	= B_TX_COLLISION_T2R_ST_M,
 };
@@ -1853,10 +1861,16 @@ static void rtw8852c_rfk_channel(struct rtw89_dev *rtwdev,
 	enum rtw89_phy_idx phy_idx = rtwvif_link->phy_idx;
 
 	rtw8852c_mcc_get_ch_info(rtwdev, phy_idx);
+	rtw89_btc_ntfy_conn_rfk(rtwdev, true);
+
 	rtw8852c_rx_dck(rtwdev, phy_idx, false);
 	rtw8852c_iqk(rtwdev, phy_idx, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852c_tssi(rtwdev, phy_idx, chanctx_idx);
+	rtw89_btc_ntfy_preserve_bt_time(rtwdev, 30);
 	rtw8852c_dpk(rtwdev, phy_idx, chanctx_idx);
+
+	rtw89_btc_ntfy_conn_rfk(rtwdev, false);
 	rtw89_fw_h2c_rf_ntfy_mcc(rtwdev);
 }
 
@@ -2071,6 +2085,31 @@ static void rtw8852c_set_txpwr_diff(struct rtw89_dev *rtwdev,
 	rtw8852c_set_txpwr_ref(rtwdev, phy_idx, pwr_ofst);
 }
 
+static void rtw8852c_set_txpwr_sar_diff(struct rtw89_dev *rtwdev,
+					const struct rtw89_chan *chan,
+					enum rtw89_phy_idx phy_idx)
+{
+	struct rtw89_sar_parm sar_parm = {
+		.center_freq = chan->freq,
+		.force_path = true,
+	};
+	s16 sar_rf;
+	s8 sar_mac;
+
+	if (phy_idx != RTW89_PHY_0)
+		return;
+
+	sar_parm.path = RF_PATH_A;
+	sar_mac = rtw89_query_sar(rtwdev, &sar_parm);
+	sar_rf = rtw89_phy_txpwr_mac_to_rf(rtwdev, sar_mac);
+	rtw89_phy_write32_mask(rtwdev, R_TXPWRB, B_TXPWRB_MAX, sar_rf);
+
+	sar_parm.path = RF_PATH_B;
+	sar_mac = rtw89_query_sar(rtwdev, &sar_parm);
+	sar_rf = rtw89_phy_txpwr_mac_to_rf(rtwdev, sar_mac);
+	rtw89_phy_write32_mask(rtwdev, R_P1_TXPWRB, B_TXPWRB_MAX, sar_rf);
+}
+
 static void rtw8852c_set_txpwr(struct rtw89_dev *rtwdev,
 			       const struct rtw89_chan *chan,
 			       enum rtw89_phy_idx phy_idx)
@@ -2081,6 +2120,7 @@ static void rtw8852c_set_txpwr(struct rtw89_dev *rtwdev,
 	rtw89_phy_set_txpwr_limit(rtwdev, chan, phy_idx);
 	rtw89_phy_set_txpwr_limit_ru(rtwdev, chan, phy_idx);
 	rtw8852c_set_txpwr_diff(rtwdev, chan, phy_idx);
+	rtw8852c_set_txpwr_sar_diff(rtwdev, chan, phy_idx);
 }
 
 static void rtw8852c_set_txpwr_ctrl(struct rtw89_dev *rtwdev,
@@ -2873,6 +2913,7 @@ static int rtw8852c_mac_disable_bb_rf(struct rtw89_dev *rtwdev)
 
 static const struct rtw89_chanctx_listener rtw8852c_chanctx_listener = {
 	.callbacks[RTW89_CHANCTX_CALLBACK_RFK] = rtw8852c_rfk_chanctx_cb,
+	.callbacks[RTW89_CHANCTX_CALLBACK_TAS] = rtw89_tas_chanctx_cb,
 };
 
 #ifdef CONFIG_PM
@@ -2913,6 +2954,7 @@ static const struct rtw89_chip_ops rtw8852c_chip_ops = {
 	.set_txpwr_ctrl		= rtw8852c_set_txpwr_ctrl,
 	.init_txpwr_unit	= rtw8852c_init_txpwr_unit,
 	.get_thermal		= rtw8852c_get_thermal,
+	.chan_to_rf18_val	= NULL,
 	.ctrl_btg_bt_rx		= rtw8852c_ctrl_btg_bt_rx,
 	.query_ppdu		= rtw8852c_query_ppdu,
 	.convert_rpl_to_rssi	= NULL,
@@ -2934,6 +2976,8 @@ static const struct rtw89_chip_ops rtw8852c_chip_ops = {
 	.h2c_default_cmac_tbl	= rtw89_fw_h2c_default_cmac_tbl,
 	.h2c_assoc_cmac_tbl	= rtw89_fw_h2c_assoc_cmac_tbl,
 	.h2c_ampdu_cmac_tbl	= NULL,
+	.h2c_txtime_cmac_tbl	= rtw89_fw_h2c_txtime_cmac_tbl,
+	.h2c_punctured_cmac_tbl	= NULL,
 	.h2c_default_dmac_tbl	= NULL,
 	.h2c_update_beacon	= rtw89_fw_h2c_update_beacon,
 	.h2c_ba_cam		= rtw89_fw_h2c_ba_cam,
@@ -2960,14 +3004,15 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.try_ce_fw		= false,
 	.bbmcu_nr		= 0,
 	.needed_fw_elms		= 0,
+	.fw_blacklist		= &rtw89_fw_blacklist_default,
 	.fifo_size		= 458752,
 	.small_fifo_size	= false,
 	.dle_scc_rsvd_size	= 0,
 	.max_amsdu_limit	= 8000,
 	.dis_2g_40m_ul_ofdma	= false,
 	.rsvd_ple_ofst		= 0x6f800,
-	.hfc_param_ini		= rtw8852c_hfc_param_ini_pcie,
-	.dle_mem		= rtw8852c_dle_mem_pcie,
+	.hfc_param_ini		= {rtw8852c_hfc_param_ini_pcie, NULL, NULL},
+	.dle_mem		= {rtw8852c_dle_mem_pcie, NULL, NULL, NULL},
 	.wde_qempty_acq_grpnum	= 16,
 	.wde_qempty_mgq_grpsel	= 16,
 	.rf_base_addr		= {0xe000, 0xf000},
@@ -3002,10 +3047,15 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 				  BIT(NL80211_CHAN_WIDTH_160),
 	.support_unii4		= true,
 	.support_ant_gain	= true,
+	.support_tas		= true,
+	.support_sar_by_ant	= true,
 	.ul_tb_waveform_ctrl	= false,
 	.ul_tb_pwr_diff		= true,
+	.rx_freq_frome_ie	= false,
 	.hw_sec_hdr		= true,
 	.hw_mgmt_tx_encrypt	= true,
+	.hw_tkip_crypto		= true,
+	.hw_mlo_bmc_crypto	= false,
 	.rf_path_num		= 2,
 	.tx_nss			= 2,
 	.rx_nss			= 2,
@@ -3027,7 +3077,6 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.phycap_size		= 0x60,
 	.para_ver		= 0x1,
 	.wlcx_desired		= 0x06000000,
-	.btcx_desired		= 0x7,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
 
diff --git a/sys/contrib/dev/rtw89/rtw8852ce.c b/sys/contrib/dev/rtw89/rtw8852ce.c
index 0785ed14461e..a7447daaf750 100644
--- a/sys/contrib/dev/rtw89/rtw8852ce.c
+++ b/sys/contrib/dev/rtw89/rtw8852ce.c
@@ -118,6 +118,7 @@ static struct pci_driver rtw89_8852ce_driver = {
 	.probe		= rtw89_pci_probe,
 	.remove		= rtw89_pci_remove,
 	.driver.pm	= &rtw89_pm_ops,
+	.err_handler    = &rtw89_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name	= KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw89/rtw8922a.c b/sys/contrib/dev/rtw89/rtw8922a.c
index 11d66bfceb15..36c641e3bc13 100644
--- a/sys/contrib/dev/rtw89/rtw8922a.c
+++ b/sys/contrib/dev/rtw89/rtw8922a.c
@@ -12,9 +12,10 @@
 #include "reg.h"
 #include "rtw8922a.h"
 #include "rtw8922a_rfk.h"
+#include "sar.h"
 #include "util.h"
 
-#define RTW8922A_FW_FORMAT_MAX 3
+#define RTW8922A_FW_FORMAT_MAX 4
 #define RTW8922A_FW_BASENAME "rtw89/rtw8922a_fw"
 #define RTW8922A_MODULE_FIRMWARE \
 	RTW8922A_FW_BASENAME "-" __stringify(RTW8922A_FW_FORMAT_MAX) ".bin"
@@ -205,10 +206,17 @@ static const struct rtw89_edcca_regs rtw8922a_edcca_regs = {
 	.edcca_p_mask			= B_EDCCA_LVL_MSK1,
 	.ppdu_level			= R_SEG0R_PPDU_LVL_BE,
 	.ppdu_mask			= B_EDCCA_LVL_MSK1,
-	.rpt_a				= R_EDCCA_RPT_A_BE,
-	.rpt_b				= R_EDCCA_RPT_B_BE,
-	.rpt_sel			= R_EDCCA_RPT_SEL_BE,
-	.rpt_sel_mask			= B_EDCCA_RPT_SEL_MSK,
+	.p = {{
+		.rpt_a			= R_EDCCA_RPT_A_BE,
+		.rpt_b			= R_EDCCA_RPT_B_BE,
+		.rpt_sel		= R_EDCCA_RPT_SEL_BE,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_MSK,
+	}, {
+		.rpt_a			= R_EDCCA_RPT_P1_A_BE,
+		.rpt_b			= R_EDCCA_RPT_P1_B_BE,
+		.rpt_sel		= R_EDCCA_RPT_SEL_BE,
+		.rpt_sel_mask		= B_EDCCA_RPT_SEL_P1_MSK,
+	}},
 	.rpt_sel_be			= R_EDCCA_RPTREG_SEL_BE,
 	.rpt_sel_be_mask		= B_EDCCA_RPTREG_SEL_BE_MSK,
 	.tx_collision_t2r_st		= R_TX_COLLISION_T2R_ST_BE,
@@ -2063,7 +2071,8 @@ static void __rtw8922a_rfk_init_late(struct rtw89_dev *rtwdev,
 	rtw89_phy_rfk_pre_ntfy_and_wait(rtwdev, phy_idx, 5);
 
 	rtw89_phy_rfk_dack_and_wait(rtwdev, phy_idx, chan, 58);
-	rtw89_phy_rfk_rxdck_and_wait(rtwdev, phy_idx, chan, false, 32);
+	if (!test_bit(RTW89_FLAG_SER_HANDLING, rtwdev->flags))
+		rtw89_phy_rfk_rxdck_and_wait(rtwdev, phy_idx, chan, false, 128);
 }
 
 static void rtw8922a_rfk_init_late(struct rtw89_dev *rtwdev)
@@ -2149,6 +2158,56 @@ static void rtw8922a_set_txpwr_ref(struct rtw89_dev *rtwdev,
 				     B_BE_PWR_REF_CTRL_CCK, ref_cck);
 }
 
+static const struct rtw89_reg_def rtw8922a_txpwr_ref[][3] = {
+	{{ .addr = R_TXAGC_REF_DBM_P0, .mask = B_TXAGC_OFDM_REF_DBM_P0},
+	 { .addr = R_TXAGC_REF_DBM_P0, .mask = B_TXAGC_CCK_REF_DBM_P0},
+	 { .addr = R_TSSI_K_P0, .mask = B_TSSI_K_OFDM_P0}
+	},
+	{{ .addr = R_TXAGC_REF_DBM_RF1_P0, .mask = B_TXAGC_OFDM_REF_DBM_RF1_P0},
+	 { .addr = R_TXAGC_REF_DBM_RF1_P0, .mask = B_TXAGC_CCK_REF_DBM_RF1_P0},
+	 { .addr = R_TSSI_K_RF1_P0, .mask = B_TSSI_K_OFDM_RF1_P0}
+	},
+};
+
+static void rtw8922a_set_txpwr_diff(struct rtw89_dev *rtwdev,
+				    const struct rtw89_chan *chan,
+				    enum rtw89_phy_idx phy_idx)
+{
+	s16 pwr_ofst = rtw89_phy_ant_gain_pwr_offset(rtwdev, chan);
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	static const u32 path_ofst[] = {0x0, 0x100};
+	const struct rtw89_reg_def *txpwr_ref;
+	static const s16 tssi_k_base = 0x12;
+	s16 tssi_k_ofst = abs(pwr_ofst) + tssi_k_base;
+	s16 ofst_dec[RF_PATH_NUM_8922A];
+	s16 tssi_k[RF_PATH_NUM_8922A];
+	s16 pwr_ref_ofst;
+	s16 pwr_ref = 0;
+	u8 i;
+
+	if (rtwdev->hal.cv == CHIP_CAV)
+		pwr_ref = 16;
+
+	pwr_ref <<= chip->txpwr_factor_rf;
+	pwr_ref_ofst = pwr_ref - rtw89_phy_txpwr_bb_to_rf(rtwdev, abs(pwr_ofst));
+
+	ofst_dec[RF_PATH_A] = pwr_ofst > 0 ? pwr_ref : pwr_ref_ofst;
+	ofst_dec[RF_PATH_B] = pwr_ofst > 0 ? pwr_ref_ofst : pwr_ref;
+	tssi_k[RF_PATH_A] = pwr_ofst > 0 ? tssi_k_base : tssi_k_ofst;
+	tssi_k[RF_PATH_B] = pwr_ofst > 0 ? tssi_k_ofst : tssi_k_base;
+
+	for (i = 0; i < RF_PATH_NUM_8922A; i++) {
+		txpwr_ref = rtw8922a_txpwr_ref[phy_idx];
+
+		rtw89_phy_write32_mask(rtwdev, txpwr_ref[0].addr + path_ofst[i],
+				       txpwr_ref[0].mask, ofst_dec[i]);
+		rtw89_phy_write32_mask(rtwdev, txpwr_ref[1].addr + path_ofst[i],
+				       txpwr_ref[1].mask, ofst_dec[i]);
+		rtw89_phy_write32_mask(rtwdev, txpwr_ref[2].addr + path_ofst[i],
+				       txpwr_ref[2].mask, tssi_k[i]);
+	}
+}
+
 static void rtw8922a_bb_tx_triangular(struct rtw89_dev *rtwdev, bool en,
 				      enum rtw89_phy_idx phy_idx)
 {
@@ -2176,6 +2235,31 @@ static void rtw8922a_set_tx_shape(struct rtw89_dev *rtwdev,
 		rtw8922a_bb_tx_triangular(rtwdev, true, phy_idx);
 }
 
+static void rtw8922a_set_txpwr_sar_diff(struct rtw89_dev *rtwdev,
+					const struct rtw89_chan *chan,
+					enum rtw89_phy_idx phy_idx)
+{
+	struct rtw89_sar_parm sar_parm = {
+		.center_freq = chan->freq,
+		.force_path = true,
+	};
+	s16 sar_rf;
+	s8 sar_mac;
+
+	if (phy_idx != RTW89_PHY_0)
+		return;
+
+	sar_parm.path = RF_PATH_A;
+	sar_mac = rtw89_query_sar(rtwdev, &sar_parm);
+	sar_rf = rtw89_phy_txpwr_mac_to_rf(rtwdev, sar_mac);
+	rtw89_phy_write32_mask(rtwdev, R_P0_TXPWRB_BE, B_TXPWRB_MAX_BE, sar_rf);
+
+	sar_parm.path = RF_PATH_B;
+	sar_mac = rtw89_query_sar(rtwdev, &sar_parm);
+	sar_rf = rtw89_phy_txpwr_mac_to_rf(rtwdev, sar_mac);
+	rtw89_phy_write32_mask(rtwdev, R_P1_TXPWRB_BE, B_TXPWRB_MAX_BE, sar_rf);
+}
+
 static void rtw8922a_set_txpwr(struct rtw89_dev *rtwdev,
 			       const struct rtw89_chan *chan,
 			       enum rtw89_phy_idx phy_idx)
@@ -2185,6 +2269,9 @@ static void rtw8922a_set_txpwr(struct rtw89_dev *rtwdev,
 	rtw8922a_set_tx_shape(rtwdev, chan, phy_idx);
 	rtw89_phy_set_txpwr_limit(rtwdev, chan, phy_idx);
 	rtw89_phy_set_txpwr_limit_ru(rtwdev, chan, phy_idx);
+	rtw8922a_set_txpwr_diff(rtwdev, chan, phy_idx);
+	rtw8922a_set_txpwr_ref(rtwdev, phy_idx);
+	rtw8922a_set_txpwr_sar_diff(rtwdev, chan, phy_idx);
 }
 
 static void rtw8922a_set_txpwr_ctrl(struct rtw89_dev *rtwdev,
@@ -2303,6 +2390,48 @@ static u8 rtw8922a_get_thermal(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_p
 	return clamp_t(int, th, 0, U8_MAX);
 }
 
+static u32 rtw8922a_chan_to_rf18_val(struct rtw89_dev *rtwdev,
+				     const struct rtw89_chan *chan)
+{
+	u32 val = u32_encode_bits(chan->channel, RR_CFGCH_CH);
+
+	switch (chan->band_type) {
+	case RTW89_BAND_2G:
+	default:
+		break;
+	case RTW89_BAND_5G:
+		val |= u32_encode_bits(CFGCH_BAND1_5G, RR_CFGCH_BAND1) |
+		       u32_encode_bits(CFGCH_BAND0_5G, RR_CFGCH_BAND0);
+		break;
+	case RTW89_BAND_6G:
+		val |= u32_encode_bits(CFGCH_BAND1_6G, RR_CFGCH_BAND1) |
+		       u32_encode_bits(CFGCH_BAND0_6G, RR_CFGCH_BAND0);
+		break;
+	}
+
+	switch (chan->band_width) {
+	case RTW89_CHANNEL_WIDTH_5:
+	case RTW89_CHANNEL_WIDTH_10:
+	case RTW89_CHANNEL_WIDTH_20:
+	default:
+		break;
+	case RTW89_CHANNEL_WIDTH_40:
+		val |= u32_encode_bits(CFGCH_BW_V2_40M, RR_CFGCH_BW_V2);
+		break;
+	case RTW89_CHANNEL_WIDTH_80:
+		val |= u32_encode_bits(CFGCH_BW_V2_80M, RR_CFGCH_BW_V2);
+		break;
+	case RTW89_CHANNEL_WIDTH_160:
+		val |= u32_encode_bits(CFGCH_BW_V2_160M, RR_CFGCH_BW_V2);
+		break;
+	case RTW89_CHANNEL_WIDTH_320:
+		val |= u32_encode_bits(CFGCH_BW_V2_320M, RR_CFGCH_BW_V2);
+		break;
+	}
+
+	return val;
+}
+
 static void rtw8922a_btc_set_rfe(struct rtw89_dev *rtwdev)
 {
 	union rtw89_btc_module_info *md = &rtwdev->btc.mdinfo;
@@ -2674,6 +2803,7 @@ static const struct rtw89_chip_ops rtw8922a_chip_ops = {
 	.set_txpwr_ctrl		= rtw8922a_set_txpwr_ctrl,
 	.init_txpwr_unit	= NULL,
 	.get_thermal		= rtw8922a_get_thermal,
+	.chan_to_rf18_val	= rtw8922a_chan_to_rf18_val,
 	.ctrl_btg_bt_rx		= rtw8922a_ctrl_btg_bt_rx,
 	.query_ppdu		= rtw8922a_query_ppdu,
 	.convert_rpl_to_rssi	= rtw8922a_convert_rpl_to_rssi,
@@ -2695,6 +2825,8 @@ static const struct rtw89_chip_ops rtw8922a_chip_ops = {
 	.h2c_default_cmac_tbl	= rtw89_fw_h2c_default_cmac_tbl_g7,
 	.h2c_assoc_cmac_tbl	= rtw89_fw_h2c_assoc_cmac_tbl_g7,
 	.h2c_ampdu_cmac_tbl	= rtw89_fw_h2c_ampdu_cmac_tbl_g7,
+	.h2c_txtime_cmac_tbl	= rtw89_fw_h2c_txtime_cmac_tbl_g7,
+	.h2c_punctured_cmac_tbl	= rtw89_fw_h2c_punctured_cmac_tbl_g7,
 	.h2c_default_dmac_tbl	= rtw89_fw_h2c_default_dmac_tbl_v2,
 	.h2c_update_beacon	= rtw89_fw_h2c_update_beacon_be,
 	.h2c_ba_cam		= rtw89_fw_h2c_ba_cam_v1,
@@ -2721,14 +2853,15 @@ const struct rtw89_chip_info rtw8922a_chip_info = {
 	.try_ce_fw		= false,
 	.bbmcu_nr		= 1,
 	.needed_fw_elms		= RTW89_BE_GEN_DEF_NEEDED_FW_ELEMENTS,
+	.fw_blacklist		= &rtw89_fw_blacklist_default,
 	.fifo_size		= 589824,
 	.small_fifo_size	= false,
 	.dle_scc_rsvd_size	= 0,
 	.max_amsdu_limit	= 8000,
 	.dis_2g_40m_ul_ofdma	= false,
 	.rsvd_ple_ofst		= 0x8f800,
-	.hfc_param_ini		= rtw8922a_hfc_param_ini_pcie,
-	.dle_mem		= rtw8922a_dle_mem_pcie,
+	.hfc_param_ini		= {rtw8922a_hfc_param_ini_pcie, NULL, NULL},
+	.dle_mem		= {rtw8922a_dle_mem_pcie, NULL, NULL, NULL},
 	.wde_qempty_acq_grpnum	= 4,
 	.wde_qempty_mgq_grpsel	= 4,
 	.rf_base_addr		= {0xe000, 0xf000},
@@ -2760,11 +2893,16 @@ const struct rtw89_chip_info rtw8922a_chip_info = {
 				  BIT(NL80211_CHAN_WIDTH_80) |
 				  BIT(NL80211_CHAN_WIDTH_160),
 	.support_unii4		= true,
-	.support_ant_gain	= false,
+	.support_ant_gain	= true,
+	.support_tas		= false,
+	.support_sar_by_ant	= true,
 	.ul_tb_waveform_ctrl	= false,
 	.ul_tb_pwr_diff		= false,
+	.rx_freq_frome_ie	= false,
 	.hw_sec_hdr		= true,
 	.hw_mgmt_tx_encrypt	= true,
+	.hw_tkip_crypto		= true,
+	.hw_mlo_bmc_crypto	= false,
 	.rf_path_num		= 2,
 	.tx_nss			= 2,
 	.rx_nss			= 2,
@@ -2786,7 +2924,6 @@ const struct rtw89_chip_info rtw8922a_chip_info = {
 	.phycap_size		= 0x38,
 	.para_ver		= 0xf,
 	.wlcx_desired		= 0x07110000,
-	.btcx_desired		= 0x7,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
 
diff --git a/sys/contrib/dev/rtw89/rtw8922a_rfk.c b/sys/contrib/dev/rtw89/rtw8922a_rfk.c
index c4c93f836a2f..fce094c7ce93 100644
--- a/sys/contrib/dev/rtw89/rtw8922a_rfk.c
+++ b/sys/contrib/dev/rtw89/rtw8922a_rfk.c
@@ -36,8 +36,7 @@ void rtw8922a_tssi_cont_en_phyidx(struct rtw89_dev *rtwdev, bool en, u8 phy_idx)
 
 static
 void rtw8922a_ctl_band_ch_bw(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
-			     u8 central_ch, enum rtw89_band band,
-			     enum rtw89_bandwidth bw)
+			     const struct rtw89_chan *chan)
 {
 	const u32 rf_addr[2] = {RR_CFGCH, RR_CFGCH_V1};
 	struct rtw89_hal *hal = &rtwdev->hal;
@@ -73,54 +72,9 @@ void rtw8922a_ctl_band_ch_bw(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 				return;
 			}
 
-			rf_reg[path][i] &= ~(RR_CFGCH_BAND1 | RR_CFGCH_BW |
+			rf_reg[path][i] &= ~(RR_CFGCH_BAND1 | RR_CFGCH_BW_V2 |
 					     RR_CFGCH_BAND0 | RR_CFGCH_CH);
-			rf_reg[path][i] |= u32_encode_bits(central_ch, RR_CFGCH_CH);
-
-			if (band == RTW89_BAND_2G)
-				rtw89_write_rf(rtwdev, path, RR_SMD, RR_VCO2, 0x0);
-			else
-				rtw89_write_rf(rtwdev, path, RR_SMD, RR_VCO2, 0x1);
-
-			switch (band) {
-			case RTW89_BAND_2G:
-			default:
-				break;
-			case RTW89_BAND_5G:
-				rf_reg[path][i] |=
-					u32_encode_bits(CFGCH_BAND1_5G, RR_CFGCH_BAND1) |
-					u32_encode_bits(CFGCH_BAND0_5G, RR_CFGCH_BAND0);
-				break;
-			case RTW89_BAND_6G:
-				rf_reg[path][i] |=
-					u32_encode_bits(CFGCH_BAND1_6G, RR_CFGCH_BAND1) |
-					u32_encode_bits(CFGCH_BAND0_6G, RR_CFGCH_BAND0);
-				break;
-			}
-
-			switch (bw) {
-			case RTW89_CHANNEL_WIDTH_5:
-			case RTW89_CHANNEL_WIDTH_10:
-			case RTW89_CHANNEL_WIDTH_20:
-			default:
-				break;
-			case RTW89_CHANNEL_WIDTH_40:
-				rf_reg[path][i] |=
-					u32_encode_bits(CFGCH_BW_V2_40M, RR_CFGCH_BW_V2);
-				break;
-			case RTW89_CHANNEL_WIDTH_80:
-				rf_reg[path][i] |=
-					u32_encode_bits(CFGCH_BW_V2_80M, RR_CFGCH_BW_V2);
-				break;
-			case RTW89_CHANNEL_WIDTH_160:
-				rf_reg[path][i] |=
-					u32_encode_bits(CFGCH_BW_V2_160M, RR_CFGCH_BW_V2);
-				break;
-			case RTW89_CHANNEL_WIDTH_320:
-				rf_reg[path][i] |=
-					u32_encode_bits(CFGCH_BW_V2_320M, RR_CFGCH_BW_V2);
-				break;
-			}
+			rf_reg[path][i] |= rtw89_chip_chan_to_rf18_val(rtwdev, chan);
 
 			rtw89_write_rf(rtwdev, path, rf_addr[i],
 				       RFREG_MASK, rf_reg[path][i]);
@@ -131,7 +85,7 @@ void rtw8922a_ctl_band_ch_bw(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 	if (hal->cv != CHIP_CAV)
 		return;
 
-	if (band == RTW89_BAND_2G) {
+	if (chan->band_type == RTW89_BAND_2G) {
 		rtw89_write_rf(rtwdev, RF_PATH_A, RR_LUTWE, RFREG_MASK, 0x80000);
 		rtw89_write_rf(rtwdev, RF_PATH_A, RR_LUTWA, RFREG_MASK, 0x00003);
 		rtw89_write_rf(rtwdev, RF_PATH_A, RR_LUTWD1, RFREG_MASK, 0x0c990);
@@ -150,8 +104,7 @@ void rtw8922a_set_channel_rf(struct rtw89_dev *rtwdev,
 			     const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx)
 {
-	rtw8922a_ctl_band_ch_bw(rtwdev, phy_idx, chan->channel, chan->band_type,
-				chan->band_width);
+	rtw8922a_ctl_band_ch_bw(rtwdev, phy_idx, chan);
 }
 
 enum _rf_syn_pow {
diff --git a/sys/contrib/dev/rtw89/rtw8922ae.c b/sys/contrib/dev/rtw89/rtw8922ae.c
index e8ccc5b4b759..f1eeb0dc61b4 100644
--- a/sys/contrib/dev/rtw89/rtw8922ae.c
+++ b/sys/contrib/dev/rtw89/rtw8922ae.c
@@ -106,6 +106,7 @@ static struct pci_driver rtw89_8922ae_driver = {
 	.probe		= rtw89_pci_probe,
 	.remove		= rtw89_pci_remove,
 	.driver.pm	= &rtw89_pm_ops_be,
+	.err_handler    = &rtw89_pci_err_handler,
 #if defined(__FreeBSD__)
 	.bsddriver.name	= KBUILD_MODNAME,
 #endif
diff --git a/sys/contrib/dev/rtw89/sar.c b/sys/contrib/dev/rtw89/sar.c
index 8b8c075780d4..5b16b98aa5b3 100644
--- a/sys/contrib/dev/rtw89/sar.c
+++ b/sys/contrib/dev/rtw89/sar.c
@@ -7,10 +7,16 @@
 #include "phy.h"
 #include "reg.h"
 #include "sar.h"
+#include "util.h"
 
 #define RTW89_TAS_FACTOR 2 /* unit: 0.25 dBm */
+#define RTW89_TAS_SAR_GAP (1 << RTW89_TAS_FACTOR)
 #define RTW89_TAS_DPR_GAP (1 << RTW89_TAS_FACTOR)
 #define RTW89_TAS_DELTA (2 << RTW89_TAS_FACTOR)
+#define RTW89_TAS_TX_RATIO_THRESHOLD 70
+#define RTW89_TAS_DFLT_TX_RATIO 80
+#define RTW89_TAS_DPR_ON_OFFSET (RTW89_TAS_DELTA + RTW89_TAS_SAR_GAP)
+#define RTW89_TAS_DPR_OFF_OFFSET (4 << RTW89_TAS_FACTOR)
 
 static enum rtw89_sar_subband rtw89_sar_get_subband(struct rtw89_dev *rtwdev,
 						    u32 center_freq)
@@ -51,10 +57,12 @@ static enum rtw89_sar_subband rtw89_sar_get_subband(struct rtw89_dev *rtwdev,
 }
 
 static int rtw89_query_sar_config_common(struct rtw89_dev *rtwdev,
-					 u32 center_freq, s32 *cfg)
+					 const struct rtw89_sar_parm *sar_parm,
+					 s32 *cfg)
 {
 	struct rtw89_sar_cfg_common *rtwsar = &rtwdev->sar.cfg_common;
 	enum rtw89_sar_subband subband_l, subband_h;
+	u32 center_freq = sar_parm->center_freq;
 	const struct rtw89_6ghz_span *span;
 
 	span = rtw89_get_6ghz_span(rtwdev, center_freq);
@@ -84,6 +92,93 @@ static int rtw89_query_sar_config_common(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static const struct rtw89_sar_entry_from_acpi *
+rtw89_sar_cfg_acpi_get_ent(const struct rtw89_sar_cfg_acpi *rtwsar,
+			   enum rtw89_rf_path path,
+			   enum rtw89_regulation_type regd)
+{
+	const struct rtw89_sar_indicator_from_acpi *ind = &rtwsar->indicator;
+	const struct rtw89_sar_table_from_acpi *tbl;
+	u8 sel;
+
+	sel = ind->tblsel[path];
+	tbl = &rtwsar->tables[sel];
+
+	return &tbl->entries[regd];
+}
+
+static
+s32 rtw89_sar_cfg_acpi_get_min(const struct rtw89_sar_entry_from_acpi *ent,
+			       enum rtw89_rf_path path,
+			       enum rtw89_acpi_sar_subband subband_low,
+			       enum rtw89_acpi_sar_subband subband_high)
+{
+	return min(ent->v[subband_low][path], ent->v[subband_high][path]);
+}
+
+static int rtw89_query_sar_config_acpi(struct rtw89_dev *rtwdev,
+				       const struct rtw89_sar_parm *sar_parm,
+				       s32 *cfg)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_sar_cfg_acpi *rtwsar = &rtwdev->sar.cfg_acpi;
+	const struct rtw89_sar_entry_from_acpi *ent_a, *ent_b;
+	enum rtw89_acpi_sar_subband subband_l, subband_h;
+	u32 center_freq = sar_parm->center_freq;
+	const struct rtw89_6ghz_span *span;
+	enum rtw89_regulation_type regd;
+	enum rtw89_band band;
+	s32 cfg_a, cfg_b;
+
+	span = rtw89_get_6ghz_span(rtwdev, center_freq);
+
+	if (span && RTW89_ACPI_SAR_SPAN_VALID(span)) {
+		subband_l = span->acpi_sar_subband_low;
+		subband_h = span->acpi_sar_subband_high;
+	} else {
+		subband_l = rtw89_acpi_sar_get_subband(rtwdev, center_freq);
+		subband_h = subband_l;
+	}
+
+	band = rtw89_acpi_sar_subband_to_band(rtwdev, subband_l);
+	regd = rtw89_regd_get(rtwdev, band);
+
+	ent_a = rtw89_sar_cfg_acpi_get_ent(rtwsar, RF_PATH_A, regd);
+	ent_b = rtw89_sar_cfg_acpi_get_ent(rtwsar, RF_PATH_B, regd);
+
+	cfg_a = rtw89_sar_cfg_acpi_get_min(ent_a, RF_PATH_A, subband_l, subband_h);
+	cfg_b = rtw89_sar_cfg_acpi_get_min(ent_b, RF_PATH_B, subband_l, subband_h);
+
+	if (chip->support_sar_by_ant) {
+		/* With declaration of support_sar_by_ant, relax the general
+		 * SAR querying to return the maximum between paths. However,
+		 * expect chip has dealt with the corresponding SAR settings
+		 * by path. (To get SAR for a given path, chip can then query
+		 * with force_path.)
+		 */
+		if (sar_parm->force_path) {
+			switch (sar_parm->path) {
+			default:
+			case RF_PATH_A:
+				*cfg = cfg_a;
+				break;
+			case RF_PATH_B:
+				*cfg = cfg_b;
+				break;
+			}
+		} else {
+			*cfg = max(cfg_a, cfg_b);
+		}
+	} else {
+		*cfg = min(cfg_a, cfg_b);
+	}
+
+	if (sar_parm->ntx == RTW89_2TX)
+		*cfg -= rtwsar->downgrade_2tx;
+
+	return 0;
+}
+
 static const
 struct rtw89_sar_handler rtw89_sar_handlers[RTW89_SAR_SOURCE_NR] = {
 	[RTW89_SAR_SOURCE_COMMON] = {
@@ -91,6 +186,11 @@ struct rtw89_sar_handler rtw89_sar_handlers[RTW89_SAR_SOURCE_NR] = {
 		.txpwr_factor_sar = 2,
 		.query_sar_config = rtw89_query_sar_config_common,
 	},
+	[RTW89_SAR_SOURCE_ACPI] = {
+		.descr_sar_source = "RTW89_SAR_SOURCE_ACPI",
+		.txpwr_factor_sar = TXPWR_FACTOR_OF_RTW89_ACPI_SAR,
+		.query_sar_config = rtw89_query_sar_config_acpi,
+	},
 };
 
 #define rtw89_sar_set_src(_dev, _src, _cfg_name, _cfg_data)		\
@@ -99,7 +199,8 @@ struct rtw89_sar_handler rtw89_sar_handlers[RTW89_SAR_SOURCE_NR] = {
 		typeof(_dev) _d = (_dev);				\
 		BUILD_BUG_ON(!rtw89_sar_handlers[_s].descr_sar_source);	\
 		BUILD_BUG_ON(!rtw89_sar_handlers[_s].query_sar_config);	\
-		lockdep_assert_held(&_d->mutex);			\
+		if (test_bit(RTW89_FLAG_PROBE_DONE, _d->flags))		\
+			lockdep_assert_wiphy(_d->hw->wiphy);		\
 		_d->sar._cfg_name = *(_cfg_data);			\
 		_d->sar.src = _s;					\
 	} while (0)
@@ -117,8 +218,8 @@ static s8 rtw89_txpwr_sar_to_mac(struct rtw89_dev *rtwdev, u8 fct, s32 cfg)
 			   RTW89_SAR_TXPWR_MAC_MAX);
 }
 
-static s8 rtw89_txpwr_tas_to_sar(const struct rtw89_sar_handler *sar_hdl,
-				 s8 cfg)
+static s32 rtw89_txpwr_tas_to_sar(const struct rtw89_sar_handler *sar_hdl,
+				  s32 cfg)
 {
 	const u8 fct = sar_hdl->txpwr_factor_sar;
 
@@ -128,8 +229,8 @@ static s8 rtw89_txpwr_tas_to_sar(const struct rtw89_sar_handler *sar_hdl,
 		return cfg >> (RTW89_TAS_FACTOR - fct);
 }
 
-static s8 rtw89_txpwr_sar_to_tas(const struct rtw89_sar_handler *sar_hdl,
-				 s8 cfg)
+static s32 rtw89_txpwr_sar_to_tas(const struct rtw89_sar_handler *sar_hdl,
+				  s32 cfg)
 {
 	const u8 fct = sar_hdl->txpwr_factor_sar;
 
@@ -139,35 +240,67 @@ static s8 rtw89_txpwr_sar_to_tas(const struct rtw89_sar_handler *sar_hdl,
 		return cfg << (RTW89_TAS_FACTOR - fct);
 }
 
-s8 rtw89_query_sar(struct rtw89_dev *rtwdev, u32 center_freq)
+static bool rtw89_tas_is_active(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	struct rtw89_vif *rtwvif;
+
+	if (!tas->enable)
+		return false;
+
+	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
+		if (ieee80211_vif_is_mld(rtwvif_to_vif(rtwvif)))
+			return false;
+	}
+
+	return true;
+}
+
+static const char *rtw89_tas_state_str(enum rtw89_tas_state state)
+{
+	switch (state) {
+	case RTW89_TAS_STATE_DPR_OFF:
+		return "DPR OFF";
+	case RTW89_TAS_STATE_DPR_ON:
+		return "DPR ON";
+	case RTW89_TAS_STATE_STATIC_SAR:
+		return "STATIC SAR";
+	default:
+		return NULL;
+	}
+}
+
+s8 rtw89_query_sar(struct rtw89_dev *rtwdev, const struct rtw89_sar_parm *sar_parm)
 {
 	const enum rtw89_sar_sources src = rtwdev->sar.src;
 	/* its members are protected by rtw89_sar_set_src() */
 	const struct rtw89_sar_handler *sar_hdl = &rtw89_sar_handlers[src];
 	struct rtw89_tas_info *tas = &rtwdev->tas;
-	s8 delta;
+	s32 offset;
 	int ret;
 	s32 cfg;
 	u8 fct;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (src == RTW89_SAR_SOURCE_NONE)
 		return RTW89_SAR_TXPWR_MAC_MAX;
 
-	ret = sar_hdl->query_sar_config(rtwdev, center_freq, &cfg);
+	ret = sar_hdl->query_sar_config(rtwdev, sar_parm, &cfg);
 	if (ret)
 		return RTW89_SAR_TXPWR_MAC_MAX;
 
-	if (tas->enable) {
+	if (rtw89_tas_is_active(rtwdev)) {
 		switch (tas->state) {
 		case RTW89_TAS_STATE_DPR_OFF:
-			return RTW89_SAR_TXPWR_MAC_MAX;
+			offset = rtw89_txpwr_tas_to_sar(sar_hdl, RTW89_TAS_DPR_OFF_OFFSET);
+			cfg += offset;
+			break;
 		case RTW89_TAS_STATE_DPR_ON:
-			delta = rtw89_txpwr_tas_to_sar(sar_hdl, tas->delta);
-			cfg -= delta;
+			offset = rtw89_txpwr_tas_to_sar(sar_hdl, RTW89_TAS_DPR_ON_OFFSET);
+			cfg -= offset;
 			break;
-		case RTW89_TAS_STATE_DPR_FORBID:
+		case RTW89_TAS_STATE_STATIC_SAR:
 		default:
 			break;
 		}
@@ -177,73 +310,86 @@ s8 rtw89_query_sar(struct rtw89_dev *rtwdev, u32 center_freq)
 
 	return rtw89_txpwr_sar_to_mac(rtwdev, fct, cfg);
 }
+EXPORT_SYMBOL(rtw89_query_sar);
 
-void rtw89_print_sar(struct seq_file *m, struct rtw89_dev *rtwdev, u32 center_freq)
+int rtw89_print_sar(struct rtw89_dev *rtwdev, char *buf, size_t bufsz,
+		    const struct rtw89_sar_parm *sar_parm)
 {
 	const enum rtw89_sar_sources src = rtwdev->sar.src;
 	/* its members are protected by rtw89_sar_set_src() */
 	const struct rtw89_sar_handler *sar_hdl = &rtw89_sar_handlers[src];
 	const u8 fct_mac = rtwdev->chip->txpwr_factor_mac;
+	char *p = buf, *end = buf + bufsz;
 	int ret;
 	s32 cfg;
 	u8 fct;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	if (src == RTW89_SAR_SOURCE_NONE) {
-		seq_puts(m, "no SAR is applied\n");
-		return;
+		p += scnprintf(p, end - p, "no SAR is applied\n");
+		goto out;
 	}
 
-	seq_printf(m, "source: %d (%s)\n", src, sar_hdl->descr_sar_source);
+	p += scnprintf(p, end - p, "source: %d (%s)\n", src,
+		       sar_hdl->descr_sar_source);
 
-	ret = sar_hdl->query_sar_config(rtwdev, center_freq, &cfg);
+	ret = sar_hdl->query_sar_config(rtwdev, sar_parm, &cfg);
 	if (ret) {
-		seq_printf(m, "config: return code: %d\n", ret);
-		seq_printf(m, "assign: max setting: %d (unit: 1/%lu dBm)\n",
-			   RTW89_SAR_TXPWR_MAC_MAX, BIT(fct_mac));
-		return;
+		p += scnprintf(p, end - p, "config: return code: %d\n", ret);
+		p += scnprintf(p, end - p,
+			       "assign: max setting: %d (unit: 1/%lu dBm)\n",
+			       RTW89_SAR_TXPWR_MAC_MAX, BIT(fct_mac));
+		goto out;
 	}
 
 	fct = sar_hdl->txpwr_factor_sar;
 
-	seq_printf(m, "config: %d (unit: 1/%lu dBm)\n", cfg, BIT(fct));
+	p += scnprintf(p, end - p, "config: %d (unit: 1/%lu dBm)\n", cfg,
+		       BIT(fct));
+
+	p += scnprintf(p, end - p, "support different configs by antenna: %s\n",
+		       str_yes_no(rtwdev->chip->support_sar_by_ant));
+out:
+	return p - buf;
 }
 
-void rtw89_print_tas(struct seq_file *m, struct rtw89_dev *rtwdev)
+int rtw89_print_tas(struct rtw89_dev *rtwdev, char *buf, size_t bufsz)
 {
 	struct rtw89_tas_info *tas = &rtwdev->tas;
+	char *p = buf, *end = buf + bufsz;
 
-	if (!tas->enable) {
-		seq_puts(m, "no TAS is applied\n");
-		return;
+	if (!rtw89_tas_is_active(rtwdev)) {
+		p += scnprintf(p, end - p, "no TAS is applied\n");
+		goto out;
 	}
 
-	seq_printf(m, "DPR gap: %d\n", tas->dpr_gap);
-	seq_printf(m, "TAS delta: %d\n", tas->delta);
+	p += scnprintf(p, end - p, "State: %s\n",
+		       rtw89_tas_state_str(tas->state));
+	p += scnprintf(p, end - p, "Average time: %d\n",
+		       tas->window_size * 2);
+	p += scnprintf(p, end - p, "SAR gap: %d dBm\n",
+		       RTW89_TAS_SAR_GAP >> RTW89_TAS_FACTOR);
+	p += scnprintf(p, end - p, "DPR gap: %d dBm\n",
+		       RTW89_TAS_DPR_GAP >> RTW89_TAS_FACTOR);
+	p += scnprintf(p, end - p, "DPR ON offset: %d dBm\n",
+		       RTW89_TAS_DPR_ON_OFFSET >> RTW89_TAS_FACTOR);
+	p += scnprintf(p, end - p, "DPR OFF offset: %d dBm\n",
+		       RTW89_TAS_DPR_OFF_OFFSET >> RTW89_TAS_FACTOR);
+
+out:
+	return p - buf;
 }
 
 static int rtw89_apply_sar_common(struct rtw89_dev *rtwdev,
 				  const struct rtw89_sar_cfg_common *sar)
 {
-	enum rtw89_sar_sources src;
-	int ret = 0;
-
-	mutex_lock(&rtwdev->mutex);
-
-	src = rtwdev->sar.src;
-	if (src != RTW89_SAR_SOURCE_NONE && src != RTW89_SAR_SOURCE_COMMON) {
-		rtw89_warn(rtwdev, "SAR source: %d is in use", src);
-		ret = -EBUSY;
-		goto exit;
-	}
-
+	/* let common SAR have the highest priority; always apply it */
 	rtw89_sar_set_src(rtwdev, RTW89_SAR_SOURCE_COMMON, cfg_common, sar);
 	rtw89_core_set_chip_txpwr(rtwdev);
+	rtw89_tas_reset(rtwdev, false);
 
-exit:
-	mutex_unlock(&rtwdev->mutex);
-	return ret;
+	return 0;
 }
 
 static const struct cfg80211_sar_freq_ranges rtw89_common_sar_freq_ranges[] = {
@@ -283,6 +429,8 @@ int rtw89_ops_set_sar_specs(struct ieee80211_hw *hw,
 	s32 power;
 	u32 i, idx;
 
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
 	if (sar->type != NL80211_SAR_TYPE_POWER)
 		return -EINVAL;
 
@@ -308,64 +456,254 @@ int rtw89_ops_set_sar_specs(struct ieee80211_hw *hw,
 	return rtw89_apply_sar_common(rtwdev, &sar_common);
 }
 
-static void rtw89_tas_state_update(struct rtw89_dev *rtwdev)
+static void rtw89_apply_sar_acpi(struct rtw89_dev *rtwdev,
+				 const struct rtw89_sar_cfg_acpi *sar)
+{
+	const struct rtw89_sar_table_from_acpi *tbl;
+	const struct rtw89_sar_entry_from_acpi *ent;
+	enum rtw89_sar_sources src;
+	unsigned int i, j, k;
+
+	src = rtwdev->sar.src;
+	if (src != RTW89_SAR_SOURCE_NONE) {
+		rtw89_warn(rtwdev, "SAR source: %d is in use", src);
+		return;
+	}
+
+	rtw89_debug(rtwdev, RTW89_DBG_SAR,
+		    "SAR-ACPI downgrade 2TX: %u (unit: 1/%lu dBm)\n",
+		    sar->downgrade_2tx, BIT(TXPWR_FACTOR_OF_RTW89_ACPI_SAR));
+
+	for (i = 0; i < sar->valid_num; i++) {
+		tbl = &sar->tables[i];
+
+		for (j = 0; j < RTW89_REGD_NUM; j++) {
+			ent = &tbl->entries[j];
+
+			rtw89_debug(rtwdev, RTW89_DBG_SAR,
+				    "SAR-ACPI-[%u] REGD-%s (unit: 1/%lu dBm)\n",
+				    i, rtw89_regd_get_string(j),
+				    BIT(TXPWR_FACTOR_OF_RTW89_ACPI_SAR));
+
+			for (k = 0; k < NUM_OF_RTW89_ACPI_SAR_SUBBAND; k++)
+				rtw89_debug(rtwdev, RTW89_DBG_SAR,
+					    "On subband %u, { %d, %d }\n", k,
+					    ent->v[k][RF_PATH_A], ent->v[k][RF_PATH_B]);
+		}
+	}
+
+	rtw89_sar_set_src(rtwdev, RTW89_SAR_SOURCE_ACPI, cfg_acpi, sar);
+
+	/* SAR via ACPI is only configured in the early initial phase, so
+	 * it does not seem necessary to reset txpwr related things here.
+	 */
+}
+
+static void rtw89_set_sar_from_acpi(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_sar_cfg_acpi *cfg;
+	int ret;
+
+	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+	if (!cfg)
+		return;
+
+	ret = rtw89_acpi_evaluate_sar(rtwdev, cfg);
+	if (ret) {
+		rtw89_debug(rtwdev, RTW89_DBG_SAR,
+			    "evaluating ACPI SAR returns %d\n", ret);
+		goto out;
+	}
+
+	if (unlikely(!cfg->valid_num)) {
+		rtw89_debug(rtwdev, RTW89_DBG_SAR, "no valid SAR table from ACPI\n");
+		goto out;
+	}
+
+	rtw89_apply_sar_acpi(rtwdev, cfg);
+
+out:
+	kfree(cfg);
+}
+
+static bool rtw89_tas_query_sar_config(struct rtw89_dev *rtwdev, s32 *cfg)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0);
 	const enum rtw89_sar_sources src = rtwdev->sar.src;
 	/* its members are protected by rtw89_sar_set_src() */
 	const struct rtw89_sar_handler *sar_hdl = &rtw89_sar_handlers[src];
-	struct rtw89_tas_info *tas = &rtwdev->tas;
-	s32 txpwr_avg = tas->total_txpwr / RTW89_TAS_MAX_WINDOW / PERCENT;
-	s32 dpr_on_threshold, dpr_off_threshold, cfg;
-	enum rtw89_tas_state state = tas->state;
-	const struct rtw89_chan *chan;
+	struct rtw89_sar_parm sar_parm = {};
 	int ret;
 
-	lockdep_assert_held(&rtwdev->mutex);
-
 	if (src == RTW89_SAR_SOURCE_NONE)
-		return;
+		return false;
 
-	chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0);
-	ret = sar_hdl->query_sar_config(rtwdev, chan->freq, &cfg);
+	sar_parm.center_freq = chan->freq;
+	ret = sar_hdl->query_sar_config(rtwdev, &sar_parm, cfg);
 	if (ret)
+		return false;
+
+	*cfg = rtw89_txpwr_sar_to_tas(sar_hdl, *cfg);
+
+	return true;
+}
+
+static bool __rtw89_tas_state_update(struct rtw89_dev *rtwdev,
+				     enum rtw89_tas_state state)
+{
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+
+	if (tas->state == state)
+		return false;
+
+	rtw89_debug(rtwdev, RTW89_DBG_SAR, "tas: switch state: %s -> %s\n",
+		    rtw89_tas_state_str(tas->state), rtw89_tas_state_str(state));
+
+	tas->state = state;
+	return true;
+}
+
+static void rtw89_tas_state_update(struct rtw89_dev *rtwdev,
+				   enum rtw89_tas_state state)
+{
+	if (!__rtw89_tas_state_update(rtwdev, state))
 		return;
 
-	cfg = rtw89_txpwr_sar_to_tas(sar_hdl, cfg);
+	rtw89_core_set_chip_txpwr(rtwdev);
+}
+
+static u32 rtw89_tas_get_window_size(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0);
+	u8 band = chan->band_type;
+	u8 regd = rtw89_regd_get(rtwdev, band);
 
-	if (tas->delta >= cfg) {
+	switch (regd) {
+	default:
 		rtw89_debug(rtwdev, RTW89_DBG_SAR,
-			    "TAS delta exceed SAR limit\n");
-		state = RTW89_TAS_STATE_DPR_FORBID;
-		goto out;
+			    "tas: regd: %u is unhandled\n", regd);
+		fallthrough;
+	case RTW89_IC:
+	case RTW89_KCC:
+		return 180;
+	case RTW89_FCC:
+		switch (band) {
+		case RTW89_BAND_2G:
+			return 50;
+		case RTW89_BAND_5G:
+			return 30;
+		case RTW89_BAND_6G:
+		default:
+			return 15;
+		}
+		break;
+	}
+}
+
+static void rtw89_tas_window_update(struct rtw89_dev *rtwdev)
+{
+	u32 window_size = rtw89_tas_get_window_size(rtwdev);
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	u64 total_txpwr = 0;
+	u8 head_idx;
+	u32 i, j;
+
+	WARN_ON_ONCE(tas->window_size > RTW89_TAS_TXPWR_WINDOW);
+
+	if (tas->window_size == window_size)
+		return;
+
+	rtw89_debug(rtwdev, RTW89_DBG_SAR, "tas: window update: %u -> %u\n",
+		    tas->window_size, window_size);
+
+	head_idx = (tas->txpwr_tail_idx - window_size + 1 + RTW89_TAS_TXPWR_WINDOW) %
+		   RTW89_TAS_TXPWR_WINDOW;
+	for (i = 0; i < window_size; i++) {
+		j = (head_idx + i) % RTW89_TAS_TXPWR_WINDOW;
+		total_txpwr += tas->txpwr_history[j];
+	}
+
+	tas->window_size = window_size;
+	tas->total_txpwr = total_txpwr;
+	tas->txpwr_head_idx = head_idx;
+}
+
+static void rtw89_tas_history_update(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_bb_ctx *bb = rtw89_get_bb_ctx(rtwdev, RTW89_PHY_0);
+	struct rtw89_env_monitor_info *env = &bb->env_monitor;
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	u8 tx_ratio = env->ifs_clm_tx_ratio;
+	u64 instant_txpwr, txpwr;
+
+	/* txpwr in unit of linear(mW) multiply by percentage */
+	if (tx_ratio == 0) {
+		/* special case: idle tx power
+		 * use -40 dBm * 100 tx ratio
+		 */
+		instant_txpwr = rtw89_db_to_linear(-40);
+		txpwr = instant_txpwr * 100;
+	} else {
+		instant_txpwr = tas->instant_txpwr;
+		txpwr = instant_txpwr * tx_ratio;
 	}
 
-	dpr_on_threshold = cfg;
-	dpr_off_threshold = cfg - tas->dpr_gap;
+	tas->total_txpwr += txpwr - tas->txpwr_history[tas->txpwr_head_idx];
+	tas->total_tx_ratio += tx_ratio - tas->tx_ratio_history[tas->tx_ratio_idx];
+	tas->tx_ratio_history[tas->tx_ratio_idx] = tx_ratio;
+
+	tas->txpwr_head_idx = (tas->txpwr_head_idx + 1) % RTW89_TAS_TXPWR_WINDOW;
+	tas->txpwr_tail_idx = (tas->txpwr_tail_idx + 1) % RTW89_TAS_TXPWR_WINDOW;
+	tas->tx_ratio_idx = (tas->tx_ratio_idx + 1) % RTW89_TAS_TX_RATIO_WINDOW;
+	tas->txpwr_history[tas->txpwr_tail_idx] = txpwr;
+
+	rtw89_debug(rtwdev, RTW89_DBG_SAR,
+		    "tas: instant_txpwr: %d, tx_ratio: %u, txpwr: %d\n",
+		    rtw89_linear_to_db_quarter(instant_txpwr), tx_ratio,
+		    rtw89_linear_to_db_quarter(div_u64(txpwr, PERCENT)));
+}
+
+static bool rtw89_tas_rolling_average(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	s32 dpr_on_threshold, dpr_off_threshold;
+	enum rtw89_tas_state state;
+	u16 tx_ratio_avg;
+	s32 txpwr_avg;
+	u64 linear;
+
+	linear = DIV_ROUND_DOWN_ULL(tas->total_txpwr, tas->window_size * PERCENT);
+	txpwr_avg = rtw89_linear_to_db_quarter(linear);
+	tx_ratio_avg = tas->total_tx_ratio / RTW89_TAS_TX_RATIO_WINDOW;
+	dpr_on_threshold = tas->dpr_on_threshold;
+	dpr_off_threshold = tas->dpr_off_threshold;
+
 	rtw89_debug(rtwdev, RTW89_DBG_SAR,
-		    "DPR_ON thold: %d, DPR_OFF thold: %d, txpwr_avg: %d\n",
-		    dpr_on_threshold, dpr_off_threshold, txpwr_avg);
+		    "tas: DPR_ON: %d, DPR_OFF: %d, txpwr_avg: %d, tx_ratio_avg: %u\n",
+		    dpr_on_threshold, dpr_off_threshold, txpwr_avg, tx_ratio_avg);
 
-	if (txpwr_avg >= dpr_on_threshold)
+	if (tx_ratio_avg >= RTW89_TAS_TX_RATIO_THRESHOLD)
+		state = RTW89_TAS_STATE_STATIC_SAR;
+	else if (txpwr_avg >= dpr_on_threshold)
 		state = RTW89_TAS_STATE_DPR_ON;
 	else if (txpwr_avg < dpr_off_threshold)
 		state = RTW89_TAS_STATE_DPR_OFF;
+	else
+		return false;
 
-out:
-	if (tas->state == state)
-		return;
-
-	rtw89_debug(rtwdev, RTW89_DBG_SAR,
-		    "TAS old state: %d, new state: %d\n", tas->state, state);
-	tas->state = state;
-	rtw89_core_set_chip_txpwr(rtwdev);
+	return __rtw89_tas_state_update(rtwdev, state);
 }
 
-void rtw89_tas_init(struct rtw89_dev *rtwdev)
+static void rtw89_tas_init(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_tas_info *tas = &rtwdev->tas;
+	const struct rtw89_acpi_policy_tas *ptr;
 	struct rtw89_acpi_dsm_result res = {};
 	int ret;
-	u8 val;
+
+	if (!chip->support_tas)
+		return;
 
 	ret = rtw89_acpi_evaluate_dsm(rtwdev, RTW89_ACPI_DSM_FUNC_TAS_EN, &res);
 	if (ret) {
@@ -374,8 +712,9 @@ void rtw89_tas_init(struct rtw89_dev *rtwdev)
 		return;
 	}
 
-	val = res.u.value;
-	switch (val) {
+	ptr = res.u.policy_tas;
+
+	switch (ptr->enable) {
 	case 0:
 		tas->enable = false;
 		break;
@@ -388,66 +727,170 @@ void rtw89_tas_init(struct rtw89_dev *rtwdev)
 
 	if (!tas->enable) {
 		rtw89_debug(rtwdev, RTW89_DBG_SAR, "TAS not enable\n");
-		return;
+		goto out;
 	}
 
-	tas->dpr_gap = RTW89_TAS_DPR_GAP;
-	tas->delta = RTW89_TAS_DELTA;
+	tas->enabled_countries = ptr->enabled_countries;
+
+out:
+	kfree(ptr);
 }
 
-void rtw89_tas_reset(struct rtw89_dev *rtwdev)
+void rtw89_tas_reset(struct rtw89_dev *rtwdev, bool force)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0);
 	struct rtw89_tas_info *tas = &rtwdev->tas;
+	u64 linear;
+	s32 cfg;
+	int i;
 
-	if (!tas->enable)
+	if (!rtw89_tas_is_active(rtwdev))
+		return;
+
+	if (!rtw89_tas_query_sar_config(rtwdev, &cfg))
 		return;
 
-	memset(&tas->txpwr_history, 0, sizeof(tas->txpwr_history));
-	tas->total_txpwr = 0;
-	tas->cur_idx = 0;
+	tas->dpr_on_threshold = cfg - RTW89_TAS_SAR_GAP;
+	tas->dpr_off_threshold = cfg - RTW89_TAS_SAR_GAP - RTW89_TAS_DPR_GAP;
+
+	/* avoid history reset after new SAR apply */
+	if (!force && tas->keep_history)
+		return;
+
+	linear = rtw89_db_quarter_to_linear(cfg) * RTW89_TAS_DFLT_TX_RATIO;
+	for (i = 0; i < RTW89_TAS_TXPWR_WINDOW; i++)
+		tas->txpwr_history[i] = linear;
+
+	for (i = 0; i < RTW89_TAS_TX_RATIO_WINDOW; i++)
+		tas->tx_ratio_history[i] = RTW89_TAS_DFLT_TX_RATIO;
+
+	tas->total_tx_ratio = RTW89_TAS_DFLT_TX_RATIO * RTW89_TAS_TX_RATIO_WINDOW;
+	tas->total_txpwr = linear * RTW89_TAS_TXPWR_WINDOW;
+	tas->window_size = RTW89_TAS_TXPWR_WINDOW;
+	tas->txpwr_head_idx = 0;
+	tas->txpwr_tail_idx = RTW89_TAS_TXPWR_WINDOW - 1;
+	tas->tx_ratio_idx = 0;
 	tas->state = RTW89_TAS_STATE_DPR_OFF;
+	tas->backup_state = RTW89_TAS_STATE_DPR_OFF;
+	tas->keep_history = true;
+
+	rtw89_debug(rtwdev, RTW89_DBG_SAR,
+		    "tas: band: %u, freq: %u\n", chan->band_type, chan->freq);
 }
 
-static const struct rtw89_reg_def txpwr_regs[] = {
-	{R_PATH0_TXPWR, B_PATH0_TXPWR},
-	{R_PATH1_TXPWR, B_PATH1_TXPWR},
-};
+static bool rtw89_tas_track(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	struct rtw89_hal *hal = &rtwdev->hal;
+	s32 cfg;
+
+	if (hal->disabled_dm_bitmap & BIT(RTW89_DM_TAS))
+		return false;
+
+	if (!rtw89_tas_is_active(rtwdev))
+		return false;
+
+	if (!rtw89_tas_query_sar_config(rtwdev, &cfg) || tas->block_regd)
+		return __rtw89_tas_state_update(rtwdev, RTW89_TAS_STATE_STATIC_SAR);
 
-void rtw89_tas_track(struct rtw89_dev *rtwdev)
+	if (tas->pause)
+		return false;
+
+	rtw89_tas_window_update(rtwdev);
+	rtw89_tas_history_update(rtwdev);
+
+	return rtw89_tas_rolling_average(rtwdev);
+}
+
+void rtw89_tas_scan(struct rtw89_dev *rtwdev, bool start)
 {
-	struct rtw89_env_monitor_info *env = &rtwdev->env_monitor;
-	const enum rtw89_sar_sources src = rtwdev->sar.src;
-	u8 max_nss_num = rtwdev->chip->rf_path_num;
 	struct rtw89_tas_info *tas = &rtwdev->tas;
-	s16 tmp, txpwr, instant_txpwr = 0;
-	u32 val;
-	int i;
+	s32 cfg;
+
+	if (!rtw89_tas_is_active(rtwdev))
+		return;
+
+	if (!rtw89_tas_query_sar_config(rtwdev, &cfg))
+		return;
+
+	if (start) {
+		tas->backup_state = tas->state;
+		rtw89_tas_state_update(rtwdev, RTW89_TAS_STATE_STATIC_SAR);
+	} else {
+		rtw89_tas_state_update(rtwdev, tas->backup_state);
+	}
+}
 
-	if (!tas->enable || src == RTW89_SAR_SOURCE_NONE)
+void rtw89_tas_chanctx_cb(struct rtw89_dev *rtwdev,
+			  enum rtw89_chanctx_state state)
+{
+	struct rtw89_tas_info *tas = &rtwdev->tas;
+	s32 cfg;
+
+	if (!rtw89_tas_is_active(rtwdev))
 		return;
 
-	if (env->ccx_watchdog_result != RTW89_PHY_ENV_MON_IFS_CLM)
+	if (!rtw89_tas_query_sar_config(rtwdev, &cfg))
 		return;
 
-	for (i = 0; i < max_nss_num; i++) {
-		val = rtw89_phy_read32_mask(rtwdev, txpwr_regs[i].addr,
-					    txpwr_regs[i].mask);
-		tmp = sign_extend32(val, 8);
-		if (tmp <= 0)
-			return;
-		instant_txpwr += tmp;
+	switch (state) {
+	case RTW89_CHANCTX_STATE_MCC_START:
+		tas->pause = true;
+		rtw89_tas_state_update(rtwdev, RTW89_TAS_STATE_STATIC_SAR);
+		break;
+	case RTW89_CHANCTX_STATE_MCC_STOP:
+		tas->pause = false;
+		break;
+	default:
+		break;
 	}
+}
+EXPORT_SYMBOL(rtw89_tas_chanctx_cb);
+
+void rtw89_sar_init(struct rtw89_dev *rtwdev)
+{
+	rtw89_set_sar_from_acpi(rtwdev);
+	rtw89_tas_init(rtwdev);
+}
+
+static bool rtw89_sar_track_acpi(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_sar_cfg_acpi *cfg = &rtwdev->sar.cfg_acpi;
+	struct rtw89_sar_indicator_from_acpi *ind = &cfg->indicator;
+	const enum rtw89_sar_sources src = rtwdev->sar.src;
+	bool changed;
+	int ret;
+
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
+	if (src != RTW89_SAR_SOURCE_ACPI)
+		return false;
+
+	if (!ind->enable_sync)
+		return false;
+
+	ret = rtw89_acpi_evaluate_dynamic_sar_indicator(rtwdev, cfg, &changed);
+	if (likely(!ret))
+		return changed;
 
-	instant_txpwr /= max_nss_num;
-	/* in unit of 0.25 dBm multiply by percentage */
-	txpwr = instant_txpwr * env->ifs_clm_tx_ratio;
-	tas->total_txpwr += txpwr - tas->txpwr_history[tas->cur_idx];
-	tas->txpwr_history[tas->cur_idx] = txpwr;
 	rtw89_debug(rtwdev, RTW89_DBG_SAR,
-		    "instant_txpwr: %d, tx_ratio: %d, txpwr: %d\n",
-		    instant_txpwr, env->ifs_clm_tx_ratio, txpwr);
+		    "%s: failed to track indicator: %d; reset and disable\n",
+		    __func__, ret);
 
-	tas->cur_idx = (tas->cur_idx + 1) % RTW89_TAS_MAX_WINDOW;
+	memset(ind->tblsel, 0, sizeof(ind->tblsel));
+	ind->enable_sync = false;
+	return true;
+}
+
+void rtw89_sar_track(struct rtw89_dev *rtwdev)
+{
+	unsigned int changes = 0;
+
+	changes += rtw89_sar_track_acpi(rtwdev);
+	changes += rtw89_tas_track(rtwdev);
 
-	rtw89_tas_state_update(rtwdev);
+	if (!changes)
+		return;
+
+	rtw89_core_set_chip_txpwr(rtwdev);
 }
diff --git a/sys/contrib/dev/rtw89/sar.h b/sys/contrib/dev/rtw89/sar.h
index 4ae081d2d3b4..4b7f3d44f57b 100644
--- a/sys/contrib/dev/rtw89/sar.h
+++ b/sys/contrib/dev/rtw89/sar.h
@@ -10,21 +10,34 @@
 #define RTW89_SAR_TXPWR_MAC_MAX 63
 #define RTW89_SAR_TXPWR_MAC_MIN -64
 
+struct rtw89_sar_parm {
+	u32 center_freq;
+	enum rtw89_ntx ntx;
+
+	bool force_path;
+	enum rtw89_rf_path path;
+};
+
 struct rtw89_sar_handler {
 	const char *descr_sar_source;
 	u8 txpwr_factor_sar;
-	int (*query_sar_config)(struct rtw89_dev *rtwdev, u32 center_freq, s32 *cfg);
+	int (*query_sar_config)(struct rtw89_dev *rtwdev,
+				const struct rtw89_sar_parm *sar_parm, s32 *cfg);
 };
 
 extern const struct cfg80211_sar_capa rtw89_sar_capa;
 
-s8 rtw89_query_sar(struct rtw89_dev *rtwdev, u32 center_freq);
-void rtw89_print_sar(struct seq_file *m, struct rtw89_dev *rtwdev, u32 center_freq);
-void rtw89_print_tas(struct seq_file *m, struct rtw89_dev *rtwdev);
+s8 rtw89_query_sar(struct rtw89_dev *rtwdev, const struct rtw89_sar_parm *sar_parm);
+int rtw89_print_sar(struct rtw89_dev *rtwdev, char *buf, size_t bufsz,
+		    const struct rtw89_sar_parm *sar_parm);
+int rtw89_print_tas(struct rtw89_dev *rtwdev, char *buf, size_t bufsz);
 int rtw89_ops_set_sar_specs(struct ieee80211_hw *hw,
 			    const struct cfg80211_sar_specs *sar);
-void rtw89_tas_init(struct rtw89_dev *rtwdev);
-void rtw89_tas_reset(struct rtw89_dev *rtwdev);
-void rtw89_tas_track(struct rtw89_dev *rtwdev);
+void rtw89_tas_reset(struct rtw89_dev *rtwdev, bool force);
+void rtw89_tas_scan(struct rtw89_dev *rtwdev, bool start);
+void rtw89_tas_chanctx_cb(struct rtw89_dev *rtwdev,
+			  enum rtw89_chanctx_state state);
+void rtw89_sar_init(struct rtw89_dev *rtwdev);
+void rtw89_sar_track(struct rtw89_dev *rtwdev);
 
 #endif
diff --git a/sys/contrib/dev/rtw89/ser.c b/sys/contrib/dev/rtw89/ser.c
index 6014e765c319..0459a7a73647 100644
--- a/sys/contrib/dev/rtw89/ser.c
+++ b/sys/contrib/dev/rtw89/ser.c
@@ -156,9 +156,9 @@ static void ser_state_run(struct rtw89_ser *ser, u8 evt)
 	rtw89_debug(rtwdev, RTW89_DBG_SER, "ser: %s receive %s\n",
 		    ser_st_name(ser), ser_ev_name(ser, evt));
 
-	mutex_lock(&rtwdev->mutex);
+	wiphy_lock(rtwdev->hw->wiphy);
 	rtw89_leave_lps(rtwdev);
-	mutex_unlock(&rtwdev->mutex);
+	wiphy_unlock(rtwdev->hw->wiphy);
 
 	ser->st_tbl[ser->state].st_func(ser, evt);
 }
@@ -309,6 +309,9 @@ static void ser_reset_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 		rtw89_core_release_bit_map(rtwdev->hw_port, rtwvif_link->port);
 		rtwvif_link->net_type = RTW89_NET_TYPE_NO_LINK;
 		rtwvif_link->trigger = false;
+		rtwvif_link->rand_tsf_done = false;
+
+		rtw89_p2p_noa_once_deinit(rtwvif_link);
 	}
 }
 
@@ -483,10 +486,14 @@ static void ser_l1_reset_pre_st_hdl(struct rtw89_ser *ser, u8 evt)
 static void ser_reset_trx_st_hdl(struct rtw89_ser *ser, u8 evt)
 {
 	struct rtw89_dev *rtwdev = container_of(ser, struct rtw89_dev, ser);
+	struct wiphy *wiphy = rtwdev->hw->wiphy;
 
 	switch (evt) {
 	case SER_EV_STATE_IN:
-		cancel_delayed_work_sync(&rtwdev->track_work);
+		wiphy_lock(wiphy);
+		wiphy_delayed_work_cancel(wiphy, &rtwdev->track_work);
+		wiphy_delayed_work_cancel(wiphy, &rtwdev->track_ps_work);
+		wiphy_unlock(wiphy);
 		drv_stop_tx(ser);
 
 		if (hal_stop_dma(ser)) {
@@ -517,8 +524,10 @@ static void ser_reset_trx_st_hdl(struct rtw89_ser *ser, u8 evt)
 		hal_enable_dma(ser);
 		drv_resume_rx(ser);
 		drv_resume_tx(ser);
-		ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->track_work,
-					     RTW89_TRACK_WORK_PERIOD);
+		wiphy_delayed_work_queue(wiphy, &rtwdev->track_work,
+					 RTW89_TRACK_WORK_PERIOD);
+		wiphy_delayed_work_queue(wiphy, &rtwdev->track_ps_work,
+					 RTW89_TRACK_PS_WORK_PERIOD);
 		break;
 
 	default:
@@ -560,21 +569,22 @@ static void ser_mac_mem_dump(struct rtw89_dev *rtwdev, u8 *buf,
 	const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def;
 	u32 filter_model_addr = mac->filter_model_addr;
 	u32 indir_access_addr = mac->indir_access_addr;
+	u32 mem_page_size = mac->mem_page_size;
 	u32 *ptr = (u32 *)buf;
 	u32 base_addr, start_page, residue;
 	u32 cnt = 0;
 	u32 i;
 
-	start_page = start_addr / MAC_MEM_DUMP_PAGE_SIZE;
-	residue = start_addr % MAC_MEM_DUMP_PAGE_SIZE;
+	start_page = start_addr / mem_page_size;
+	residue = start_addr % mem_page_size;
 	base_addr = mac->mem_base_addrs[sel];
-	base_addr += start_page * MAC_MEM_DUMP_PAGE_SIZE;
+	base_addr += start_page * mem_page_size;
 
 	while (cnt < len) {
 		rtw89_write32(rtwdev, filter_model_addr, base_addr);
 
 		for (i = indir_access_addr + residue;
-		     i < indir_access_addr + MAC_MEM_DUMP_PAGE_SIZE;
+		     i < indir_access_addr + mem_page_size;
 		     i += 4, ptr++) {
 			*ptr = rtw89_read32(rtwdev, i);
 			cnt += 4;
@@ -583,7 +593,7 @@ static void ser_mac_mem_dump(struct rtw89_dev *rtwdev, u8 *buf,
 		}
 
 		residue = 0;
-		base_addr += MAC_MEM_DUMP_PAGE_SIZE;
+		base_addr += mem_page_size;
 	}
 }
 
@@ -712,9 +722,9 @@ static void ser_l2_reset_st_hdl(struct rtw89_ser *ser, u8 evt)
 
 	switch (evt) {
 	case SER_EV_STATE_IN:
-		mutex_lock(&rtwdev->mutex);
+		wiphy_lock(rtwdev->hw->wiphy);
 		ser_l2_reset_st_pre_hdl(ser);
-		mutex_unlock(&rtwdev->mutex);
+		wiphy_unlock(rtwdev->hw->wiphy);
 
 		ieee80211_restart_hw(rtwdev->hw);
 		ser_set_alarm(ser, SER_RECFG_TIMEOUT, SER_EV_L2_RECFG_TIMEOUT);
diff --git a/sys/contrib/dev/rtw89/txrx.h b/sys/contrib/dev/rtw89/txrx.h
index 70fe7cebc9d5..ec01bfc363da 100644
--- a/sys/contrib/dev/rtw89/txrx.h
+++ b/sys/contrib/dev/rtw89/txrx.h
@@ -73,6 +73,7 @@ static inline u8 rtw89_get_data_nss(struct rtw89_dev *rtwdev, u16 hw_rate)
 #define RTW89_TXWD_BODY0_FW_DL BIT(20)
 #define RTW89_TXWD_BODY0_CHANNEL_DMA GENMASK(19, 16)
 #define RTW89_TXWD_BODY0_HDR_LLC_LEN GENMASK(15, 11)
+#define RTW89_TXWD_BODY0_STF_MODE BIT(10)
 #define RTW89_TXWD_BODY0_WD_PAGE BIT(7)
 #define RTW89_TXWD_BODY0_HW_AMSDU BIT(5)
 #define RTW89_TXWD_BODY0_HW_SSN_SEL GENMASK(3, 2)
@@ -712,6 +713,25 @@ static inline u8 rtw89_core_get_qsel(struct rtw89_dev *rtwdev, u8 tid)
 	}
 }
 
+static inline u8
+rtw89_core_get_qsel_mgmt(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
+{
+	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
+	struct rtw89_vif_link *rtwvif_link = tx_req->rtwvif_link;
+
+	if (desc_info->hiq) {
+		if (rtwvif_link->mac_idx == RTW89_MAC_1)
+			return RTW89_TX_QSEL_B1_HI;
+		else
+			return RTW89_TX_QSEL_B0_HI;
+	}
+
+	if (rtwvif_link->mac_idx == RTW89_MAC_1)
+		return RTW89_TX_QSEL_B1_MGMT;
+	else
+		return RTW89_TX_QSEL_B0_MGMT;
+}
+
 static inline u8 rtw89_core_get_ch_dma(struct rtw89_dev *rtwdev, u8 qsel)
 {
 	switch (qsel) {
@@ -719,12 +739,24 @@ static inline u8 rtw89_core_get_ch_dma(struct rtw89_dev *rtwdev, u8 qsel)
 		rtw89_warn(rtwdev, "Cannot map qsel to dma: %d\n", qsel);
 		fallthrough;
 	case RTW89_TX_QSEL_BE_0:
+	case RTW89_TX_QSEL_BE_1:
+	case RTW89_TX_QSEL_BE_2:
+	case RTW89_TX_QSEL_BE_3:
 		return RTW89_TXCH_ACH0;
 	case RTW89_TX_QSEL_BK_0:
+	case RTW89_TX_QSEL_BK_1:
+	case RTW89_TX_QSEL_BK_2:
+	case RTW89_TX_QSEL_BK_3:
 		return RTW89_TXCH_ACH1;
 	case RTW89_TX_QSEL_VI_0:
+	case RTW89_TX_QSEL_VI_1:
+	case RTW89_TX_QSEL_VI_2:
+	case RTW89_TX_QSEL_VI_3:
 		return RTW89_TXCH_ACH2;
 	case RTW89_TX_QSEL_VO_0:
+	case RTW89_TX_QSEL_VO_1:
+	case RTW89_TX_QSEL_VO_2:
+	case RTW89_TX_QSEL_VO_3:
 		return RTW89_TXCH_ACH3;
 	case RTW89_TX_QSEL_B0_MGMT:
 		return RTW89_TXCH_CH8;
diff --git a/sys/contrib/dev/rtw89/usb.c b/sys/contrib/dev/rtw89/usb.c
new file mode 100644
index 000000000000..6cf89aee252e
--- /dev/null
+++ b/sys/contrib/dev/rtw89/usb.c
@@ -0,0 +1,1042 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#include <linux/usb.h>
+#include "debug.h"
+#include "mac.h"
+#include "reg.h"
+#include "txrx.h"
+#include "usb.h"
+
+static void rtw89_usb_read_port_complete(struct urb *urb);
+
+static void rtw89_usb_vendorreq(struct rtw89_dev *rtwdev, u32 addr,
+				void *data, u16 len, u8 reqtype)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct usb_device *udev = rtwusb->udev;
+	unsigned int pipe;
+	u16 value, index;
+	int attempt, ret;
+
+	if (test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
+		return;
+
+	value = u32_get_bits(addr, GENMASK(15, 0));
+	index = u32_get_bits(addr, GENMASK(23, 16));
+
+	for (attempt = 0; attempt < 10; attempt++) {
+		*rtwusb->vendor_req_buf = 0;
+
+		if (reqtype == RTW89_USB_VENQT_READ) {
+			pipe = usb_rcvctrlpipe(udev, 0);
+		} else { /* RTW89_USB_VENQT_WRITE */
+			pipe = usb_sndctrlpipe(udev, 0);
+
+			memcpy(rtwusb->vendor_req_buf, data, len);
+		}
+
+		ret = usb_control_msg(udev, pipe, RTW89_USB_VENQT, reqtype,
+				      value, index, rtwusb->vendor_req_buf,
+				      len, 500);
+
+		if (ret == len) { /* Success */
+			atomic_set(&rtwusb->continual_io_error, 0);
+
+			if (reqtype == RTW89_USB_VENQT_READ)
+				memcpy(data, rtwusb->vendor_req_buf, len);
+
+			break;
+		}
+
+		if (ret == -ESHUTDOWN || ret == -ENODEV)
+			set_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags);
+		else if (ret < 0)
+			rtw89_warn(rtwdev,
+				   "usb %s%u 0x%x fail ret=%d value=0x%x attempt=%d\n",
+				   reqtype == RTW89_USB_VENQT_READ ? "read" : "write",
+				   len * 8, addr, ret,
+				   le32_to_cpup(rtwusb->vendor_req_buf),
+				   attempt);
+		else if (ret > 0 && reqtype == RTW89_USB_VENQT_READ)
+			memcpy(data, rtwusb->vendor_req_buf, len);
+
+		if (atomic_inc_return(&rtwusb->continual_io_error) > 4) {
+			set_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags);
+			break;
+		}
+	}
+}
+
+static u32 rtw89_usb_read_cmac(struct rtw89_dev *rtwdev, u32 addr)
+{
+	u32 addr32, val32, shift;
+	__le32 data = 0;
+	int count;
+
+	addr32 = addr & ~0x3;
+	shift = (addr & 0x3) * 8;
+
+	for (count = 0; ; count++) {
+		rtw89_usb_vendorreq(rtwdev, addr32, &data, 4,
+				    RTW89_USB_VENQT_READ);
+
+		val32 = le32_to_cpu(data);
+		if (val32 != RTW89_R32_DEAD)
+			break;
+
+		if (count >= MAC_REG_POOL_COUNT) {
+			rtw89_warn(rtwdev, "%s: addr %#x = %#x\n",
+				   __func__, addr32, val32);
+			val32 = RTW89_R32_DEAD;
+			break;
+		}
+
+		rtw89_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
+	}
+
+	return val32 >> shift;
+}
+
+static u8 rtw89_usb_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
+{
+	u8 data = 0;
+
+	if (ACCESS_CMAC(addr))
+		return rtw89_usb_read_cmac(rtwdev, addr);
+
+	rtw89_usb_vendorreq(rtwdev, addr, &data, 1, RTW89_USB_VENQT_READ);
+
+	return data;
+}
+
+static u16 rtw89_usb_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
+{
+	__le16 data = 0;
+
+	if (ACCESS_CMAC(addr))
+		return rtw89_usb_read_cmac(rtwdev, addr);
+
+	rtw89_usb_vendorreq(rtwdev, addr, &data, 2, RTW89_USB_VENQT_READ);
+
+	return le16_to_cpu(data);
+}
+
+static u32 rtw89_usb_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
+{
+	__le32 data = 0;
+
+	if (ACCESS_CMAC(addr))
+		return rtw89_usb_read_cmac(rtwdev, addr);
+
+	rtw89_usb_vendorreq(rtwdev, addr, &data, 4,
+			    RTW89_USB_VENQT_READ);
+
+	return le32_to_cpu(data);
+}
+
+static void rtw89_usb_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 val)
+{
+	u8 data = val;
+
+	rtw89_usb_vendorreq(rtwdev, addr, &data, 1, RTW89_USB_VENQT_WRITE);
+}
+
+static void rtw89_usb_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 val)
+{
+	__le16 data = cpu_to_le16(val);
+
+	rtw89_usb_vendorreq(rtwdev, addr, &data, 2, RTW89_USB_VENQT_WRITE);
+}
+
+static void rtw89_usb_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 val)
+{
+	__le32 data = cpu_to_le32(val);
+
+	rtw89_usb_vendorreq(rtwdev, addr, &data, 4, RTW89_USB_VENQT_WRITE);
+}
+
+static u32
+rtw89_usb_ops_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
+					    u8 txch)
+{
+	if (txch == RTW89_TXCH_CH12)
+		return 1;
+
+	return 42; /* TODO some kind of calculation? */
+}
+
+static u8 rtw89_usb_get_bulkout_id(u8 ch_dma)
+{
+	switch (ch_dma) {
+	case RTW89_DMA_ACH0:
+		return 3;
+	case RTW89_DMA_ACH1:
+		return 4;
+	case RTW89_DMA_ACH2:
+		return 5;
+	case RTW89_DMA_ACH3:
+		return 6;
+	default:
+	case RTW89_DMA_B0MG:
+		return 0;
+	case RTW89_DMA_B0HI:
+		return 1;
+	case RTW89_DMA_H2C:
+		return 2;
+	}
+}
+
+static void rtw89_usb_write_port_complete(struct urb *urb)
+{
+	struct rtw89_usb_tx_ctrl_block *txcb = urb->context;
+	struct rtw89_dev *rtwdev = txcb->rtwdev;
+	struct ieee80211_tx_info *info;
+	struct rtw89_txwd_body *txdesc;
+	struct sk_buff *skb;
+	u32 txdesc_size;
+
+	while (true) {
+		skb = skb_dequeue(&txcb->tx_ack_queue);
+		if (!skb)
+			break;
+
+		if (txcb->txch == RTW89_TXCH_CH12) {
+			dev_kfree_skb_any(skb);
+			continue;
+		}
+
+		txdesc = (struct rtw89_txwd_body *)skb->data;
+
+		txdesc_size = rtwdev->chip->txwd_body_size;
+		if (le32_get_bits(txdesc->dword0, RTW89_TXWD_BODY0_WD_INFO_EN))
+			txdesc_size += rtwdev->chip->txwd_info_size;
+
+		skb_pull(skb, txdesc_size);
+
+		info = IEEE80211_SKB_CB(skb);
+		ieee80211_tx_info_clear_status(info);
+
+		if (urb->status == 0) {
+			if (info->flags & IEEE80211_TX_CTL_NO_ACK)
+				info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
+			else
+				info->flags |= IEEE80211_TX_STAT_ACK;
+		}
+
+		ieee80211_tx_status_irqsafe(rtwdev->hw, skb);
+	}
+
+	switch (urb->status) {
+	case 0:
+	case -EPIPE:
+	case -EPROTO:
+	case -EINPROGRESS:
+	case -ENOENT:
+	case -ECONNRESET:
+		break;
+	default:
+		set_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags);
+		break;
+	}
+
+	kfree(txcb);
+	usb_free_urb(urb);
+}
+
+static int rtw89_usb_write_port(struct rtw89_dev *rtwdev, u8 ch_dma,
+				void *data, int len, void *context)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct usb_device *usbd = rtwusb->udev;
+	struct urb *urb;
+	u8 bulkout_id = rtw89_usb_get_bulkout_id(ch_dma);
+	unsigned int pipe;
+	int ret;
+
+	if (test_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags))
+		return 0;
+
+	urb = usb_alloc_urb(0, GFP_ATOMIC);
+	if (!urb)
+		return -ENOMEM;
+
+	pipe = usb_sndbulkpipe(usbd, rtwusb->out_pipe[bulkout_id]);
+
+	usb_fill_bulk_urb(urb, usbd, pipe, data, len,
+			  rtw89_usb_write_port_complete, context);
+	urb->transfer_flags |= URB_ZERO_PACKET;
+	ret = usb_submit_urb(urb, GFP_ATOMIC);
+
+	if (ret)
+		usb_free_urb(urb);
+
+	if (ret == -ENODEV)
+		set_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags);
+
+	return ret;
+}
+
+static void rtw89_usb_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct rtw89_usb_tx_ctrl_block *txcb;
+	struct sk_buff *skb;
+	int ret;
+
+	while (true) {
+		skb = skb_dequeue(&rtwusb->tx_queue[txch]);
+		if (!skb)
+			break;
+
+		txcb = kmalloc(sizeof(*txcb), GFP_ATOMIC);
+		if (!txcb) {
+			dev_kfree_skb_any(skb);
+			continue;
+		}
+
+		txcb->rtwdev = rtwdev;
+		txcb->txch = txch;
+		skb_queue_head_init(&txcb->tx_ack_queue);
+
+		skb_queue_tail(&txcb->tx_ack_queue, skb);
+
+		ret = rtw89_usb_write_port(rtwdev, txch, skb->data, skb->len,
+					   txcb);
+		if (ret) {
+			rtw89_err(rtwdev, "write port txch %d failed: %d\n",
+				  txch, ret);
+
+			skb_dequeue(&txcb->tx_ack_queue);
+			kfree(txcb);
+			dev_kfree_skb_any(skb);
+		}
+	}
+}
+
+static int rtw89_usb_tx_write_fwcmd(struct rtw89_dev *rtwdev,
+				    struct rtw89_core_tx_request *tx_req)
+{
+	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct sk_buff *skb = tx_req->skb;
+	struct sk_buff *skb512;
+	u32 txdesc_size = rtwdev->chip->h2c_desc_size;
+	void *txdesc;
+
+	if (((desc_info->pkt_size + txdesc_size) % 512) == 0) {
+		rtw89_debug(rtwdev, RTW89_DBG_HCI, "avoiding multiple of 512\n");
+
+		skb512 = dev_alloc_skb(txdesc_size + desc_info->pkt_size +
+				       RTW89_USB_MOD512_PADDING);
+		if (!skb512) {
+			rtw89_err(rtwdev, "%s: failed to allocate skb\n",
+				  __func__);
+
+			return -ENOMEM;
+		}
+
+		skb_pull(skb512, txdesc_size);
+		skb_put_data(skb512, skb->data, skb->len);
+		skb_put_zero(skb512, RTW89_USB_MOD512_PADDING);
+
+		dev_kfree_skb_any(skb);
+		skb = skb512;
+		tx_req->skb = skb512;
+
+		desc_info->pkt_size += RTW89_USB_MOD512_PADDING;
+	}
+
+	txdesc = skb_push(skb, txdesc_size);
+	memset(txdesc, 0, txdesc_size);
+	rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
+
+	skb_queue_tail(&rtwusb->tx_queue[desc_info->ch_dma], skb);
+
+	return 0;
+}
+
+static int rtw89_usb_ops_tx_write(struct rtw89_dev *rtwdev,
+				  struct rtw89_core_tx_request *tx_req)
+{
+	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct sk_buff *skb = tx_req->skb;
+	struct rtw89_txwd_body *txdesc;
+	u32 txdesc_size;
+
+	if ((desc_info->ch_dma == RTW89_TXCH_CH12 ||
+	     tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
+	    (desc_info->ch_dma != RTW89_TXCH_CH12 ||
+	     tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
+		rtw89_err(rtwdev, "dma channel %d/TX type %d mismatch\n",
+			  desc_info->ch_dma, tx_req->tx_type);
+		return -EINVAL;
+	}
+
+	if (desc_info->ch_dma == RTW89_TXCH_CH12)
+		return rtw89_usb_tx_write_fwcmd(rtwdev, tx_req);
+
+	txdesc_size = rtwdev->chip->txwd_body_size;
+	if (desc_info->en_wd_info)
+		txdesc_size += rtwdev->chip->txwd_info_size;
+
+	txdesc = skb_push(skb, txdesc_size);
+	memset(txdesc, 0, txdesc_size);
+	rtw89_chip_fill_txdesc(rtwdev, desc_info, txdesc);
+
+	le32p_replace_bits(&txdesc->dword0, 1, RTW89_TXWD_BODY0_STF_MODE);
+
+	skb_queue_tail(&rtwusb->tx_queue[desc_info->ch_dma], skb);
+
+	return 0;
+}
+
+static void rtw89_usb_rx_handler(struct work_struct *work)
+{
+	struct rtw89_usb *rtwusb = container_of(work, struct rtw89_usb, rx_work);
+	struct rtw89_dev *rtwdev = rtwusb->rtwdev;
+	struct rtw89_rx_desc_info desc_info;
+	struct sk_buff *rx_skb;
+	struct sk_buff *skb;
+	u32 pkt_offset;
+	int limit;
+
+	for (limit = 0; limit < 200; limit++) {
+		rx_skb = skb_dequeue(&rtwusb->rx_queue);
+		if (!rx_skb)
+			break;
+
+		if (skb_queue_len(&rtwusb->rx_queue) >= RTW89_USB_MAX_RXQ_LEN) {
+			rtw89_warn(rtwdev, "rx_queue overflow\n");
+			dev_kfree_skb_any(rx_skb);
+			continue;
+		}
+
+		memset(&desc_info, 0, sizeof(desc_info));
+		rtw89_chip_query_rxdesc(rtwdev, &desc_info, rx_skb->data, 0);
+
+		skb = rtw89_alloc_skb_for_rx(rtwdev, desc_info.pkt_size);
+		if (!skb) {
+			rtw89_debug(rtwdev, RTW89_DBG_HCI,
+				    "failed to allocate RX skb of size %u\n",
+				    desc_info.pkt_size);
+			continue;
+		}
+
+		pkt_offset = desc_info.offset + desc_info.rxd_len;
+
+		skb_put_data(skb, rx_skb->data + pkt_offset,
+			     desc_info.pkt_size);
+
+		rtw89_core_rx(rtwdev, &desc_info, skb);
+
+		if (skb_queue_len(&rtwusb->rx_free_queue) >= RTW89_USB_RX_SKB_NUM)
+			dev_kfree_skb_any(rx_skb);
+		else
+			skb_queue_tail(&rtwusb->rx_free_queue, rx_skb);
+	}
+
+	if (limit == 200) {
+		rtw89_debug(rtwdev, RTW89_DBG_HCI,
+			    "left %d rx skbs in the queue for later\n",
+			    skb_queue_len(&rtwusb->rx_queue));
+		queue_work(rtwusb->rxwq, &rtwusb->rx_work);
+	}
+}
+
+static void rtw89_usb_rx_resubmit(struct rtw89_usb *rtwusb,
+				  struct rtw89_usb_rx_ctrl_block *rxcb,
+				  gfp_t gfp)
+{
+	struct rtw89_dev *rtwdev = rtwusb->rtwdev;
+	struct sk_buff *rx_skb;
+	int ret;
+
+	rx_skb = skb_dequeue(&rtwusb->rx_free_queue);
+	if (!rx_skb)
+		rx_skb = alloc_skb(RTW89_USB_RECVBUF_SZ, gfp);
+
+	if (!rx_skb)
+		goto try_later;
+
+	skb_reset_tail_pointer(rx_skb);
+	rx_skb->len = 0;
+
+	rxcb->rx_skb = rx_skb;
+
+	usb_fill_bulk_urb(rxcb->rx_urb, rtwusb->udev,
+			  usb_rcvbulkpipe(rtwusb->udev, rtwusb->in_pipe),
+			  rxcb->rx_skb->data, RTW89_USB_RECVBUF_SZ,
+			  rtw89_usb_read_port_complete, rxcb);
+
+	ret = usb_submit_urb(rxcb->rx_urb, gfp);
+	if (ret) {
+		skb_queue_tail(&rtwusb->rx_free_queue, rxcb->rx_skb);
+
+		if (ret == -ENODEV)
+			set_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags);
+		else
+			rtw89_err(rtwdev, "Err sending rx data urb %d\n", ret);
+
+		if (ret == -ENOMEM)
+			goto try_later;
+	}
+
+	return;
+
+try_later:
+	rxcb->rx_skb = NULL;
+	queue_work(rtwusb->rxwq, &rtwusb->rx_urb_work);
+}
+
+static void rtw89_usb_rx_resubmit_work(struct work_struct *work)
+{
+	struct rtw89_usb *rtwusb = container_of(work, struct rtw89_usb, rx_urb_work);
+	struct rtw89_usb_rx_ctrl_block *rxcb;
+	int i;
+
+	for (i = 0; i < RTW89_USB_RXCB_NUM; i++) {
+		rxcb = &rtwusb->rx_cb[i];
+
+		if (!rxcb->rx_skb)
+			rtw89_usb_rx_resubmit(rtwusb, rxcb, GFP_ATOMIC);
+	}
+}
+
+static void rtw89_usb_read_port_complete(struct urb *urb)
+{
+	struct rtw89_usb_rx_ctrl_block *rxcb = urb->context;
+	struct rtw89_dev *rtwdev = rxcb->rtwdev;
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct sk_buff *skb = rxcb->rx_skb;
+
+	if (urb->status == 0) {
+		if (urb->actual_length > urb->transfer_buffer_length ||
+		    urb->actual_length < sizeof(struct rtw89_rxdesc_short)) {
+			rtw89_err(rtwdev, "failed to get urb length: %d\n",
+				  urb->actual_length);
+			skb_queue_tail(&rtwusb->rx_free_queue, skb);
+		} else {
+			skb_put(skb, urb->actual_length);
+			skb_queue_tail(&rtwusb->rx_queue, skb);
+			queue_work(rtwusb->rxwq, &rtwusb->rx_work);
+		}
+
+		rtw89_usb_rx_resubmit(rtwusb, rxcb, GFP_ATOMIC);
+	} else {
+		skb_queue_tail(&rtwusb->rx_free_queue, skb);
+
+		if (atomic_inc_return(&rtwusb->continual_io_error) > 4)
+			set_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags);
+
+		switch (urb->status) {
+		case -EINVAL:
+		case -EPIPE:
+		case -ENODEV:
+		case -ESHUTDOWN:
+			set_bit(RTW89_FLAG_UNPLUGGED, rtwdev->flags);
+			break;
+		case -EPROTO:
+		case -EILSEQ:
+		case -ETIME:
+		case -ECOMM:
+		case -EOVERFLOW:
+		case -ENOENT:
+			break;
+		case -EINPROGRESS:
+			rtw89_info(rtwdev, "URB is in progress\n");
+			break;
+		default:
+			rtw89_err(rtwdev, "%s status %d\n",
+				  __func__, urb->status);
+			break;
+		}
+	}
+}
+
+static void rtw89_usb_cancel_rx_bufs(struct rtw89_usb *rtwusb)
+{
+	struct rtw89_usb_rx_ctrl_block *rxcb;
+	int i;
+
+	for (i = 0; i < RTW89_USB_RXCB_NUM; i++) {
+		rxcb = &rtwusb->rx_cb[i];
+		usb_kill_urb(rxcb->rx_urb);
+	}
+}
+
+static void rtw89_usb_free_rx_bufs(struct rtw89_usb *rtwusb)
+{
+	struct rtw89_usb_rx_ctrl_block *rxcb;
+	int i;
+
+	for (i = 0; i < RTW89_USB_RXCB_NUM; i++) {
+		rxcb = &rtwusb->rx_cb[i];
+		usb_free_urb(rxcb->rx_urb);
+	}
+}
+
+static int rtw89_usb_alloc_rx_bufs(struct rtw89_usb *rtwusb)
+{
+	struct rtw89_usb_rx_ctrl_block *rxcb;
+	int i;
+
+	for (i = 0; i < RTW89_USB_RXCB_NUM; i++) {
+		rxcb = &rtwusb->rx_cb[i];
+
+		rxcb->rtwdev = rtwusb->rtwdev;
+		rxcb->rx_urb = usb_alloc_urb(0, GFP_KERNEL);
+		if (!rxcb->rx_urb) {
+			rtw89_usb_free_rx_bufs(rtwusb);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+static int rtw89_usb_init_rx(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct sk_buff *rx_skb;
+	int i;
+
+	rtwusb->rxwq = alloc_workqueue("rtw89_usb: rx wq", WQ_BH, 0);
+	if (!rtwusb->rxwq) {
+		rtw89_err(rtwdev, "failed to create RX work queue\n");
+		return -ENOMEM;
+	}
+
+	skb_queue_head_init(&rtwusb->rx_queue);
+	skb_queue_head_init(&rtwusb->rx_free_queue);
+
+	INIT_WORK(&rtwusb->rx_work, rtw89_usb_rx_handler);
+	INIT_WORK(&rtwusb->rx_urb_work, rtw89_usb_rx_resubmit_work);
+
+	for (i = 0; i < RTW89_USB_RX_SKB_NUM; i++) {
+		rx_skb = alloc_skb(RTW89_USB_RECVBUF_SZ, GFP_KERNEL);
+		if (rx_skb)
+			skb_queue_tail(&rtwusb->rx_free_queue, rx_skb);
+	}
+
+	return 0;
+}
+
+static void rtw89_usb_deinit_rx(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+
+	skb_queue_purge(&rtwusb->rx_queue);
+
+	destroy_workqueue(rtwusb->rxwq);
+
+	skb_queue_purge(&rtwusb->rx_free_queue);
+}
+
+static void rtw89_usb_start_rx(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	int i;
+
+	for (i = 0; i < RTW89_USB_RXCB_NUM; i++)
+		rtw89_usb_rx_resubmit(rtwusb, &rtwusb->rx_cb[i], GFP_KERNEL);
+}
+
+static void rtw89_usb_init_tx(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++)
+		skb_queue_head_init(&rtwusb->tx_queue[i]);
+}
+
+static void rtw89_usb_deinit_tx(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++) {
+		if (i == RTW89_TXCH_CH12)
+			skb_queue_purge(&rtwusb->tx_queue[i]);
+		else
+			ieee80211_purge_tx_queue(rtwdev->hw, &rtwusb->tx_queue[i]);
+	}
+}
+
+static void rtw89_usb_ops_reset(struct rtw89_dev *rtwdev)
+{
+	/* TODO: anything to do here? */
+}
+
+static int rtw89_usb_ops_start(struct rtw89_dev *rtwdev)
+{
+	return 0; /* Nothing to do. */
+}
+
+static void rtw89_usb_ops_stop(struct rtw89_dev *rtwdev)
+{
+	/* Nothing to do. */
+}
+
+static void rtw89_usb_ops_pause(struct rtw89_dev *rtwdev, bool pause)
+{
+	/* Nothing to do? */
+}
+
+static void rtw89_usb_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
+{
+	/* Nothing to do. */
+}
+
+static int rtw89_usb_ops_deinit(struct rtw89_dev *rtwdev)
+{
+	return 0; /* Nothing to do. */
+}
+
+static int rtw89_usb_ops_mac_pre_init(struct rtw89_dev *rtwdev)
+{
+	u32 val32;
+
+	rtw89_write32_set(rtwdev, R_AX_USB_HOST_REQUEST_2, B_AX_R_USBIO_MODE);
+
+	/* fix USB IO hang suggest by chihhanli@realtek.com */
+	rtw89_write32_clr(rtwdev, R_AX_USB_WLAN0_1,
+			  B_AX_USBRX_RST | B_AX_USBTX_RST);
+
+	val32 = rtw89_read32(rtwdev, R_AX_HCI_FUNC_EN);
+	val32 &= ~(B_AX_HCI_RXDMA_EN | B_AX_HCI_TXDMA_EN);
+	rtw89_write32(rtwdev, R_AX_HCI_FUNC_EN, val32);
+
+	val32 |= B_AX_HCI_RXDMA_EN | B_AX_HCI_TXDMA_EN;
+	rtw89_write32(rtwdev, R_AX_HCI_FUNC_EN, val32);
+	/* fix USB TRX hang suggest by chihhanli@realtek.com */
+
+	return 0;
+}
+
+static int rtw89_usb_ops_mac_pre_deinit(struct rtw89_dev *rtwdev)
+{
+	return 0; /* Nothing to do. */
+}
+
+static int rtw89_usb_ops_mac_post_init(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	enum usb_device_speed speed;
+	u32 ep;
+
+	rtw89_write32_clr(rtwdev, R_AX_USB3_MAC_NPI_CONFIG_INTF_0,
+			  B_AX_SSPHY_LFPS_FILTER);
+
+	speed = rtwusb->udev->speed;
+
+	if (speed == USB_SPEED_SUPER)
+		rtw89_write8(rtwdev, R_AX_RXDMA_SETTING, USB3_BULKSIZE);
+	else if (speed == USB_SPEED_HIGH)
+		rtw89_write8(rtwdev, R_AX_RXDMA_SETTING, USB2_BULKSIZE);
+	else
+		rtw89_write8(rtwdev, R_AX_RXDMA_SETTING, USB11_BULKSIZE);
+
+	for (ep = 5; ep <= 12; ep++) {
+		if (ep == 8)
+			continue;
+
+		rtw89_write8_mask(rtwdev, R_AX_USB_ENDPOINT_0,
+				  B_AX_EP_IDX, ep);
+		rtw89_write8(rtwdev, R_AX_USB_ENDPOINT_2 + 1, NUMP);
+	}
+
+	return 0;
+}
+
+static void rtw89_usb_ops_recalc_int_mit(struct rtw89_dev *rtwdev)
+{
+	/* Nothing to do. */
+}
+
+static int rtw89_usb_ops_mac_lv1_rcvy(struct rtw89_dev *rtwdev,
+				      enum rtw89_lv1_rcvy_step step)
+{
+	u32 reg, mask;
+
+	switch (rtwdev->chip->chip_id) {
+	case RTL8851B:
+	case RTL8852A:
+	case RTL8852B:
+		reg = R_AX_USB_WLAN0_1;
+		mask = B_AX_USBRX_RST | B_AX_USBTX_RST;
+		break;
+	case RTL8852C:
+		reg = R_AX_USB_WLAN0_1_V1;
+		mask = B_AX_USBRX_RST_V1 | B_AX_USBTX_RST_V1;
+		break;
+	default:
+		rtw89_err(rtwdev, "%s: fix me\n", __func__);
+		return -EOPNOTSUPP;
+	}
+
+	switch (step) {
+	case RTW89_LV1_RCVY_STEP_1:
+		rtw89_write32_set(rtwdev, reg, mask);
+
+		msleep(30);
+		break;
+	case RTW89_LV1_RCVY_STEP_2:
+		rtw89_write32_clr(rtwdev, reg, mask);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void rtw89_usb_ops_dump_err_status(struct rtw89_dev *rtwdev)
+{
+	rtw89_warn(rtwdev, "%s TODO\n", __func__);
+}
+
+static const struct rtw89_hci_ops rtw89_usb_ops = {
+	.tx_write	= rtw89_usb_ops_tx_write,
+	.tx_kick_off	= rtw89_usb_ops_tx_kick_off,
+	.flush_queues	= NULL, /* Not needed? */
+	.reset		= rtw89_usb_ops_reset,
+	.start		= rtw89_usb_ops_start,
+	.stop		= rtw89_usb_ops_stop,
+	.pause		= rtw89_usb_ops_pause,
+	.switch_mode	= rtw89_usb_ops_switch_mode,
+	.recalc_int_mit = rtw89_usb_ops_recalc_int_mit,
+
+	.read8		= rtw89_usb_ops_read8,
+	.read16		= rtw89_usb_ops_read16,
+	.read32		= rtw89_usb_ops_read32,
+	.write8		= rtw89_usb_ops_write8,
+	.write16	= rtw89_usb_ops_write16,
+	.write32	= rtw89_usb_ops_write32,
+
+	.mac_pre_init	= rtw89_usb_ops_mac_pre_init,
+	.mac_pre_deinit	= rtw89_usb_ops_mac_pre_deinit,
+	.mac_post_init	= rtw89_usb_ops_mac_post_init,
+	.deinit		= rtw89_usb_ops_deinit,
+
+	.check_and_reclaim_tx_resource = rtw89_usb_ops_check_and_reclaim_tx_resource,
+	.mac_lv1_rcvy	= rtw89_usb_ops_mac_lv1_rcvy,
+	.dump_err_status = rtw89_usb_ops_dump_err_status,
+	.napi_poll	= NULL,
+
+	.recovery_start = NULL,
+	.recovery_complete = NULL,
+
+	.ctrl_txdma_ch	= NULL,
+	.ctrl_txdma_fw_ch = NULL,
+	.ctrl_trxhci	= NULL,
+	.poll_txdma_ch_idle = NULL,
+
+	.clr_idx_all	= NULL,
+	.clear		= NULL,
+	.disable_intr	= NULL,
+	.enable_intr	= NULL,
+	.rst_bdram	= NULL,
+};
+
+static int rtw89_usb_parse(struct rtw89_dev *rtwdev,
+			   struct usb_interface *intf)
+{
+	struct usb_host_interface *host_interface = &intf->altsetting[0];
+	struct usb_interface_descriptor *intf_desc = &host_interface->desc;
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	struct usb_endpoint_descriptor *endpoint;
+	int num_out_pipes = 0;
+	u8 num;
+	int i;
+
+	if (intf_desc->bNumEndpoints > RTW89_MAX_ENDPOINT_NUM) {
+		rtw89_err(rtwdev, "found %d endpoints, expected %d max\n",
+			  intf_desc->bNumEndpoints, RTW89_MAX_ENDPOINT_NUM);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < intf_desc->bNumEndpoints; i++) {
+		endpoint = &host_interface->endpoint[i].desc;
+		num = usb_endpoint_num(endpoint);
+
+		if (usb_endpoint_dir_in(endpoint) &&
+		    usb_endpoint_xfer_bulk(endpoint)) {
+			if (rtwusb->in_pipe) {
+				rtw89_err(rtwdev,
+					  "found more than 1 bulk in endpoint\n");
+				return -EINVAL;
+			}
+
+			rtwusb->in_pipe = num;
+		}
+
+		if (usb_endpoint_dir_out(endpoint) &&
+		    usb_endpoint_xfer_bulk(endpoint)) {
+			if (num_out_pipes >= RTW89_MAX_BULKOUT_NUM) {
+				rtw89_err(rtwdev,
+					  "found more than %d bulk out endpoints\n",
+					  RTW89_MAX_BULKOUT_NUM);
+				return -EINVAL;
+			}
+
+			rtwusb->out_pipe[num_out_pipes++] = num;
+		}
+	}
+
+	if (num_out_pipes < 1) {
+		rtw89_err(rtwdev, "no bulk out endpoints found\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int rtw89_usb_intf_init(struct rtw89_dev *rtwdev,
+			       struct usb_interface *intf)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+	int ret;
+
+	ret = rtw89_usb_parse(rtwdev, intf);
+	if (ret)
+		return ret;
+
+	rtwusb->vendor_req_buf = kmalloc(sizeof(*rtwusb->vendor_req_buf),
+					 GFP_KERNEL);
+	if (!rtwusb->vendor_req_buf)
+		return -ENOMEM;
+
+	rtwusb->udev = usb_get_dev(interface_to_usbdev(intf));
+
+	usb_set_intfdata(intf, rtwdev->hw);
+
+	SET_IEEE80211_DEV(rtwdev->hw, &intf->dev);
+
+	return 0;
+}
+
+static void rtw89_usb_intf_deinit(struct rtw89_dev *rtwdev,
+				  struct usb_interface *intf)
+{
+	struct rtw89_usb *rtwusb = rtw89_usb_priv(rtwdev);
+
+	usb_put_dev(rtwusb->udev);
+	kfree(rtwusb->vendor_req_buf);
+	usb_set_intfdata(intf, NULL);
+}
+
+int rtw89_usb_probe(struct usb_interface *intf,
+		    const struct usb_device_id *id)
+{
+	const struct rtw89_driver_info *info;
+	struct rtw89_dev *rtwdev;
+	struct rtw89_usb *rtwusb;
+	int ret;
+
+	info = (const struct rtw89_driver_info *)id->driver_info;
+
+	rtwdev = rtw89_alloc_ieee80211_hw(&intf->dev,
+					  sizeof(struct rtw89_usb),
+					  info->chip, info->variant);
+	if (!rtwdev) {
+		dev_err(&intf->dev, "failed to allocate hw\n");
+		return -ENOMEM;
+	}
+
+	rtwusb = rtw89_usb_priv(rtwdev);
+	rtwusb->rtwdev = rtwdev;
+
+	rtwdev->hci.ops = &rtw89_usb_ops;
+	rtwdev->hci.type = RTW89_HCI_TYPE_USB;
+
+	ret = rtw89_usb_intf_init(rtwdev, intf);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to initialise intf: %d\n", ret);
+		goto err_free_hw;
+	}
+
+	if (rtwusb->udev->speed == USB_SPEED_SUPER)
+		rtwdev->hci.dle_type = RTW89_HCI_DLE_TYPE_USB3;
+	else
+		rtwdev->hci.dle_type = RTW89_HCI_DLE_TYPE_USB2;
+
+	rtw89_usb_init_tx(rtwdev);
+
+	ret = rtw89_usb_alloc_rx_bufs(rtwusb);
+	if (ret)
+		goto err_intf_deinit;
+
+	ret = rtw89_usb_init_rx(rtwdev);
+	if (ret)
+		goto err_free_rx_bufs;
+
+	ret = rtw89_core_init(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to initialise core: %d\n", ret);
+		goto err_deinit_rx;
+	}
+
+	ret = rtw89_chip_info_setup(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to setup chip information\n");
+		goto err_core_deinit;
+	}
+
+	ret = rtw89_core_register(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to register core\n");
+		goto err_core_deinit;
+	}
+
+	rtw89_usb_start_rx(rtwdev);
+
+	set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags);
+
+	return 0;
+
+err_core_deinit:
+	rtw89_core_deinit(rtwdev);
+err_deinit_rx:
+	rtw89_usb_deinit_rx(rtwdev);
+err_free_rx_bufs:
+	rtw89_usb_free_rx_bufs(rtwusb);
+err_intf_deinit:
+	rtw89_usb_intf_deinit(rtwdev, intf);
+err_free_hw:
+	rtw89_free_ieee80211_hw(rtwdev);
+
+	return ret;
+}
+EXPORT_SYMBOL(rtw89_usb_probe);
+
+void rtw89_usb_disconnect(struct usb_interface *intf)
+{
+	struct ieee80211_hw *hw = usb_get_intfdata(intf);
+	struct rtw89_dev *rtwdev;
+	struct rtw89_usb *rtwusb;
+
+	if (!hw)
+		return;
+
+	rtwdev = hw->priv;
+	rtwusb = rtw89_usb_priv(rtwdev);
+
+	rtw89_usb_cancel_rx_bufs(rtwusb);
+
+	rtw89_core_unregister(rtwdev);
+	rtw89_core_deinit(rtwdev);
+	rtw89_usb_deinit_rx(rtwdev);
+	rtw89_usb_free_rx_bufs(rtwusb);
+	rtw89_usb_deinit_tx(rtwdev);
+	rtw89_usb_intf_deinit(rtwdev, intf);
+	rtw89_free_ieee80211_hw(rtwdev);
+}
+EXPORT_SYMBOL(rtw89_usb_disconnect);
+
+MODULE_AUTHOR("Bitterblue Smith <rtl8821cerfe2@gmail.com>");
+MODULE_DESCRIPTION("Realtek USB 802.11ax wireless driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sys/contrib/dev/rtw89/usb.h b/sys/contrib/dev/rtw89/usb.h
new file mode 100644
index 000000000000..c1b4bfa20979
--- /dev/null
+++ b/sys/contrib/dev/rtw89/usb.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* Copyright(c) 2025  Realtek Corporation
+ */
+
+#ifndef __RTW89_USB_H__
+#define __RTW89_USB_H__
+
+#include "txrx.h"
+
+#define RTW89_USB_VENQT			0x05
+#define RTW89_USB_VENQT_READ		0xc0
+#define RTW89_USB_VENQT_WRITE		0x40
+
+#define RTW89_USB_RECVBUF_SZ		20480
+#define RTW89_USB_RXCB_NUM		8
+#define RTW89_USB_RX_SKB_NUM		16
+#define RTW89_USB_MAX_RXQ_LEN		512
+#define RTW89_USB_MOD512_PADDING	4
+
+#define RTW89_MAX_ENDPOINT_NUM		9
+#define RTW89_MAX_BULKOUT_NUM		7
+
+struct rtw89_usb_rx_ctrl_block {
+	struct rtw89_dev *rtwdev;
+	struct urb *rx_urb;
+	struct sk_buff *rx_skb;
+};
+
+struct rtw89_usb_tx_ctrl_block {
+	struct rtw89_dev *rtwdev;
+	u8 txch;
+	struct sk_buff_head tx_ack_queue;
+};
+
+struct rtw89_usb {
+	struct rtw89_dev *rtwdev;
+	struct usb_device *udev;
+
+	__le32 *vendor_req_buf;
+
+	atomic_t continual_io_error;
+
+	u8 in_pipe;
+	u8 out_pipe[RTW89_MAX_BULKOUT_NUM];
+
+	struct workqueue_struct *rxwq;
+	struct rtw89_usb_rx_ctrl_block rx_cb[RTW89_USB_RXCB_NUM];
+	struct sk_buff_head rx_queue;
+	struct sk_buff_head rx_free_queue;
+	struct work_struct rx_work;
+	struct work_struct rx_urb_work;
+
+	struct sk_buff_head tx_queue[RTW89_TXCH_NUM];
+};
+
+static inline struct rtw89_usb *rtw89_usb_priv(struct rtw89_dev *rtwdev)
+{
+	return (struct rtw89_usb *)rtwdev->priv;
+}
+
+int rtw89_usb_probe(struct usb_interface *intf,
+		    const struct usb_device_id *id);
+void rtw89_usb_disconnect(struct usb_interface *intf);
+
+#endif
diff --git a/sys/contrib/dev/rtw89/util.c b/sys/contrib/dev/rtw89/util.c
index e71956ce9853..073714db26f2 100644
--- a/sys/contrib/dev/rtw89/util.c
+++ b/sys/contrib/dev/rtw89/util.c
@@ -4,103 +4,159 @@
 
 #include "util.h"
 
-#define FRAC_ROWS 3
-#define FRAC_ROW_MAX (FRAC_ROWS - 1)
-#define NORM_ROW_MIN FRAC_ROWS
-
-static const u32 db_invert_table[12][8] = {
-	/* rows 0~2 in unit of U(32,3) */
-	{10, 13, 16, 20, 25, 32, 40, 50},
-	{64, 80, 101, 128, 160, 201, 256, 318},
-	{401, 505, 635, 800, 1007, 1268, 1596, 2010},
-	/* rows 3~11 in unit of U(32,0) */
-	{316, 398, 501, 631, 794, 1000, 1259, 1585},
-	{1995, 2512, 3162, 3981, 5012, 6310, 7943, 10000},
-	{12589, 15849, 19953, 25119, 31623, 39811, 50119, 63098},
-	{79433, 100000, 125893, 158489, 199526, 251189, 316228, 398107},
-	{501187, 630957, 794328, 1000000, 1258925, 1584893, 1995262, 2511886},
-	{3162278, 3981072, 5011872, 6309573, 7943282, 1000000, 12589254,
-	 15848932},
-	{19952623, 25118864, 31622777, 39810717, 50118723, 63095734, 79432823,
-	 100000000},
-	{125892541, 158489319, 199526232, 251188643, 316227766, 398107171,
-	 501187234, 630957345},
-	{794328235, 1000000000, 1258925412, 1584893192, 1995262315, 2511886432U,
-	 3162277660U, 3981071706U},
+#define RTW89_DBM_QUARTER_FACTOR 2
+#define RTW89_MIN_DBM (-41.25 * (1 << RTW89_DBM_QUARTER_FACTOR))
+#define RTW89_MAX_DBM (96 * (1 << RTW89_DBM_QUARTER_FACTOR))
+#define RTW89_DB_INVERT_TABLE_OFFSET (-RTW89_MIN_DBM)
+
+static const u64 db_invert_table[] = {
+	/* in unit of 0.000001 */
+	75, 79, 84, 89, 94, 100, 106, 112, 119, 126, 133, 141, 150, 158, 168, 178, 188,
+	200, 211, 224, 237, 251, 266, 282, 299, 316, 335, 355, 376, 398, 422, 447, 473,
+	501, 531, 562, 596, 631, 668, 708, 750, 794, 841, 891, 944, 1000, 1059, 1122, 1189,
+	1259, 1334, 1413, 1496, 1585, 1679, 1778, 1884, 1995, 2113, 2239, 2371, 2512, 2661,
+	2818, 2985, 3162, 3350, 3548, 3758, 3981, 4217, 4467, 4732, 5012, 5309, 5623, 5957,
+	6310, 6683, 7079, 7499, 7943, 8414, 8913, 9441, 10000, 10593, 11220, 11885, 12589,
+	13335, 14125, 14962, 15849, 16788, 17783, 18836, 19953, 21135, 22387, 23714, 25119,
+	26607, 28184, 29854, 31623, 33497, 35481, 37584, 39811, 42170, 44668, 47315, 50119,
+	53088, 56234, 59566, 63096, 66834, 70795, 74989, 79433, 84140, 89125, 94406, 100000,
+	105925, 112202, 118850, 125893, 133352, 141254, 149624, 158489, 167880, 177828,
+	188365, 199526, 211349, 223872, 237137, 251189, 266073, 281838, 298538, 316228,
+	334965, 354813, 375837, 398107, 421697, 446684, 473151, 501187, 530884, 562341,
+	595662, 630957, 668344, 707946, 749894, 794328, 841395, 891251, 944061, 1000000,
+	1059254, 1122018, 1188502, 1258925, 1333521, 1412538, 1496236, 1584893, 1678804,
+	1778279, 1883649, 1995262, 2113489, 2238721, 2371374, 2511886, 2660725, 2818383,
+	2985383, 3162278, 3349654, 3548134, 3758374, 3981072, 4216965, 4466836, 4731513,
+	5011872, 5308844, 5623413, 5956621, 6309573, 6683439, 7079458, 7498942, 7943282,
+	8413951, 8912509, 9440609, 10000000, 10592537, 11220185, 11885022, 12589254,
+	13335214, 14125375, 14962357, 15848932, 16788040, 17782794, 18836491, 19952623,
+	21134890, 22387211, 23713737, 25118864, 26607251, 28183829, 29853826, 31622777,
+	33496544, 35481339, 37583740, 39810717, 42169650, 44668359, 47315126, 50118723,
+	53088444, 56234133, 59566214, 63095734, 66834392, 70794578, 74989421, 79432823,
+	84139514, 89125094, 94406088, 100000000, 105925373, 112201845, 118850223, 125892541,
+	133352143, 141253754, 149623566, 158489319, 167880402, 177827941, 188364909, 199526231,
+	211348904, 223872114, 237137371, 251188643, 266072506, 281838293, 298538262, 316227766,
+	334965439, 354813389, 375837404, 398107171, 421696503, 446683592, 473151259, 501187234,
+	530884444, 562341325, 595662144, 630957344, 668343918, 707945784, 749894209, 794328235,
+	841395142, 891250938, 944060876, 1000000000, 1059253725, 1122018454, 1188502227,
+	1258925412, 1333521432, 1412537545, 1496235656, 1584893192, 1678804018, 1778279410,
+	1883649089, 1995262315, 2113489040, 2238721139, 2371373706, 2511886432, 2660725060,
+	2818382931, 2985382619, 3162277660, 3349654392, 3548133892, 3758374043, 3981071706,
+	4216965034, 4466835922ULL, 4731512590ULL, 5011872336ULL, 5308844442ULL, 5623413252ULL,
+	5956621435ULL, 6309573445ULL, 6683439176ULL, 7079457844ULL, 7498942093ULL,
+	7943282347ULL, 8413951416ULL, 8912509381ULL, 9440608763ULL, 10000000000ULL,
+	10592537252ULL, 11220184543ULL, 11885022274ULL, 12589254118ULL, 13335214322ULL,
+	14125375446ULL, 14962356561ULL, 15848931925ULL, 16788040181ULL, 17782794100ULL,
+	18836490895ULL, 19952623150ULL, 21134890398ULL, 22387211386ULL, 23713737057ULL,
+	25118864315ULL, 26607250598ULL, 28183829313ULL, 29853826189ULL, 31622776602ULL,
+	33496543916ULL, 35481338923ULL, 37583740429ULL, 39810717055ULL, 42169650343ULL,
+	44668359215ULL, 47315125896ULL, 50118723363ULL, 53088444423ULL, 56234132519ULL,
+	59566214353ULL, 63095734448ULL, 66834391757ULL, 70794578438ULL, 74989420933ULL,
+	79432823472ULL, 84139514165ULL, 89125093813ULL, 94406087629ULL, 100000000000ULL,
+	105925372518ULL, 112201845430ULL, 118850222744ULL, 125892541179ULL, 133352143216ULL,
+	141253754462ULL, 149623565609ULL, 158489319246ULL, 167880401812ULL, 177827941004ULL,
+	188364908949ULL, 199526231497ULL, 211348903984ULL, 223872113857ULL, 237137370566ULL,
+	251188643151ULL, 266072505980ULL, 281838293126ULL, 298538261892ULL, 316227766017ULL,
+	334965439158ULL, 354813389234ULL, 375837404288ULL, 398107170553ULL, 421696503429ULL,
+	446683592151ULL, 473151258961ULL, 501187233627ULL, 530884444231ULL, 562341325190ULL,
+	595662143529ULL, 630957344480ULL, 668343917569ULL, 707945784384ULL, 749894209332ULL,
+	794328234724ULL, 841395141645ULL, 891250938134ULL, 944060876286ULL, 1000000000000ULL,
+	1059253725177ULL, 1122018454302ULL, 1188502227437ULL, 1258925411794ULL,
+	1333521432163ULL, 1412537544623ULL, 1496235656094ULL, 1584893192461ULL,
+	1678804018123ULL, 1778279410039ULL, 1883649089490ULL, 1995262314969ULL,
+	2113489039837ULL, 2238721138568ULL, 2371373705662ULL, 2511886431510ULL,
+	2660725059799ULL, 2818382931264ULL, 2985382618918ULL, 3162277660168ULL,
+	3349654391578ULL, 3548133892336ULL, 3758374042884ULL, 3981071705535ULL,
+	4216965034286ULL, 4466835921510ULL, 4731512589615ULL, 5011872336273ULL,
+	5308844442310ULL, 5623413251904ULL, 5956621435290ULL, 6309573444802ULL,
+	6683439175686ULL, 7079457843841ULL, 7498942093325ULL, 7943282347243ULL,
+	8413951416452ULL, 8912509381337ULL, 9440608762859ULL, 10000000000000ULL,
+	10592537251773ULL, 11220184543020ULL, 11885022274370ULL, 12589254117942ULL,
+	13335214321633ULL, 14125375446228ULL, 14962356560944ULL, 15848931924611ULL,
+	16788040181226ULL, 17782794100389ULL, 18836490894898ULL, 19952623149689ULL,
+	21134890398367ULL, 22387211385683ULL, 23713737056617ULL, 25118864315096ULL,
+	26607250597988ULL, 28183829312645ULL, 29853826189180ULL, 31622776601684ULL,
+	33496543915783ULL, 35481338923358ULL, 37583740428845ULL, 39810717055350ULL,
+	42169650342858ULL, 44668359215096ULL, 47315125896148ULL, 50118723362727ULL,
+	53088444423099ULL, 56234132519035ULL, 59566214352901ULL, 63095734448019ULL,
+	66834391756862ULL, 70794578438414ULL, 74989420933246ULL, 79432823472428ULL,
+	84139514164520ULL, 89125093813375ULL, 94406087628593ULL, 100000000000000ULL,
+	105925372517729ULL, 112201845430197ULL, 118850222743702ULL, 125892541179417ULL,
+	133352143216332ULL, 141253754462276ULL, 149623565609444ULL, 158489319246111ULL,
+	167880401812256ULL, 177827941003893ULL, 188364908948981ULL, 199526231496888ULL,
+	211348903983664ULL, 223872113856834ULL, 237137370566166ULL, 251188643150958ULL,
+	266072505979882ULL, 281838293126446ULL, 298538261891796ULL, 316227766016838ULL,
+	334965439157829ULL, 354813389233577ULL, 375837404288444ULL, 398107170553497ULL,
+	421696503428583ULL, 446683592150964ULL, 473151258961482ULL, 501187233627272ULL,
+	530884444230989ULL, 562341325190350ULL, 595662143529011ULL, 630957344480196ULL,
+	668343917568615ULL, 707945784384138ULL, 749894209332456ULL, 794328234724284ULL,
+	841395141645198ULL, 891250938133745ULL, 944060876285923ULL, 1000000000000000ULL,
+	1059253725177290ULL, 1122018454301970ULL, 1188502227437020ULL, 1258925411794170ULL,
+	1333521432163330ULL, 1412537544622760ULL, 1496235656094440ULL, 1584893192461110ULL,
+	1678804018122560ULL, 1778279410038920ULL, 1883649089489810ULL, 1995262314968890ULL,
+	2113489039836650ULL, 2238721138568340ULL, 2371373705661660ULL, 2511886431509590ULL,
+	2660725059798820ULL, 2818382931264460ULL, 2985382618917960ULL, 3162277660168380ULL,
+	3349654391578280ULL, 3548133892335770ULL, 3758374042884440ULL, 3981071705534970ULL
 };
 
-u32 rtw89_linear_2_db(u64 val)
+s32 rtw89_linear_to_db_quarter(u64 val)
 {
-	u8 i, j;
-	u32 dB;
-
-	for (i = 0; i < 12; i++) {
-		for (j = 0; j < 8; j++) {
-			if (i <= FRAC_ROW_MAX &&
-			    (val << RTW89_LINEAR_FRAC_BITS) <= db_invert_table[i][j])
-				goto cnt;
-			else if (i > FRAC_ROW_MAX && val <= db_invert_table[i][j])
-				goto cnt;
-		}
-	}
+	int r = ARRAY_SIZE(db_invert_table) - 1;
+	int l = 0;
+	int m;
 
-	return 96; /* maximum 96 dB */
+	while (l <= r) {
+		m = l + (r - l) / 2;
 
-cnt:
-	/* special cases */
-	if (j == 0 && i == 0)
-		goto end;
+		if (db_invert_table[m] == val)
+			return m - (s32)RTW89_DB_INVERT_TABLE_OFFSET;
 
-	if (i == NORM_ROW_MIN && j == 0) {
-		if (db_invert_table[NORM_ROW_MIN][0] - val >
-		    val - (db_invert_table[FRAC_ROW_MAX][7] >> RTW89_LINEAR_FRAC_BITS)) {
-			i = FRAC_ROW_MAX;
-			j = 7;
-		}
-		goto end;
+		if (db_invert_table[m] > val)
+			r = m - 1;
+		else
+			l = m + 1;
 	}
 
-	if (i <= FRAC_ROW_MAX)
-		val <<= RTW89_LINEAR_FRAC_BITS;
-
-	/* compare difference to get precise dB */
-	if (j == 0) {
-		if (db_invert_table[i][j] - val >
-		    val - db_invert_table[i - 1][7]) {
-			i--;
-			j = 7;
-		}
-	} else {
-		if (db_invert_table[i][j] - val >
-		    val - db_invert_table[i][j - 1]) {
-			j--;
-		}
-	}
-end:
-	dB = (i << 3) + j + 1;
+	if (l >= ARRAY_SIZE(db_invert_table))
+		return RTW89_MAX_DBM;
+	else if (r < 0)
+		return RTW89_MIN_DBM;
+	else if (val - db_invert_table[r] <= db_invert_table[l] - val)
+		return r - (s32)RTW89_DB_INVERT_TABLE_OFFSET;
+	else
+		return l - (s32)RTW89_DB_INVERT_TABLE_OFFSET;
+}
+EXPORT_SYMBOL(rtw89_linear_to_db_quarter);
 
-	return dB;
+s32 rtw89_linear_to_db(u64 val)
+{
+	return rtw89_linear_to_db_quarter(val) >> RTW89_DBM_QUARTER_FACTOR;
 }
-EXPORT_SYMBOL(rtw89_linear_2_db);
+EXPORT_SYMBOL(rtw89_linear_to_db);
 
-u64 rtw89_db_2_linear(u32 db)
+u64 rtw89_db_quarter_to_linear(s32 db)
 {
-	u64 linear;
-	u8 i, j;
+	/* supported range -41.25 to 96 dBm, in unit of 0.25 dBm */
+	db = clamp_t(s32, db, RTW89_MIN_DBM, RTW89_MAX_DBM);
+	db += (s32)RTW89_DB_INVERT_TABLE_OFFSET;
 
-	if (db > 96)
-		db = 96;
-	else if (db < 1)
-		return 1;
+	return db_invert_table[db];
+}
+EXPORT_SYMBOL(rtw89_db_quarter_to_linear);
 
-	i = (db - 1) >> 3;
-	j = (db - 1) & 0x7;
+u64 rtw89_db_to_linear(s32 db)
+{
+	return rtw89_db_quarter_to_linear(db << RTW89_DBM_QUARTER_FACTOR);
+}
+EXPORT_SYMBOL(rtw89_db_to_linear);
 
-	linear = db_invert_table[i][j];
+void rtw89_might_trailing_ellipsis(char *buf, size_t size, ssize_t used)
+{
+	static const char ellipsis[] = "...";
 
-	if (i >= NORM_ROW_MIN)
-		linear = linear << RTW89_LINEAR_FRAC_BITS;
+	/* length of null terminiator isn't included in 'used' */
+	if (used + 1 < size || size < sizeof(ellipsis))
+		return;
 
-	return linear;
+	memcpy(buf + size - sizeof(ellipsis), ellipsis, sizeof(ellipsis));
 }
-EXPORT_SYMBOL(rtw89_db_2_linear);
diff --git a/sys/contrib/dev/rtw89/util.h b/sys/contrib/dev/rtw89/util.h
index e669544cafd3..bd08495301e4 100644
--- a/sys/contrib/dev/rtw89/util.h
+++ b/sys/contrib/dev/rtw89/util.h
@@ -6,13 +6,11 @@
 
 #include "core.h"
 
-#define RTW89_LINEAR_FRAC_BITS 3
-
 #define rtw89_iterate_vifs_bh(rtwdev, iterator, data)                          \
 	ieee80211_iterate_active_interfaces_atomic((rtwdev)->hw,               \
 			IEEE80211_IFACE_ITER_NORMAL, iterator, data)
 
-/* call this function with rtwdev->mutex is held */
+/* call this function with wiphy mutex is held */
 #define rtw89_for_each_rtwvif(rtwdev, rtwvif)				       \
 	list_for_each_entry(rtwvif, &(rtwdev)->rtwvifs_list, list)
 
@@ -25,7 +23,7 @@ static inline bool rtw89_rtwvif_in_list(struct rtw89_dev *rtwdev,
 {
 	struct rtw89_vif *rtwvif;
 
-	lockdep_assert_held(&rtwdev->mutex);
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
 
 	rtw89_for_each_rtwvif(rtwdev, rtwvif)
 		if (rtwvif == new)
@@ -75,7 +73,10 @@ static inline void ether_addr_copy_mask(u8 *dst, const u8 *src, u8 mask)
 	}
 }
 
-u32 rtw89_linear_2_db(u64 linear);
-u64 rtw89_db_2_linear(u32 db);
+s32 rtw89_linear_to_db_quarter(u64 val);
+s32 rtw89_linear_to_db(u64 val);
+u64 rtw89_db_quarter_to_linear(s32 db);
+u64 rtw89_db_to_linear(s32 db);
+void rtw89_might_trailing_ellipsis(char *buf, size_t size, ssize_t used);
 
 #endif
diff --git a/sys/contrib/dev/rtw89/wow.c b/sys/contrib/dev/rtw89/wow.c
index 311bf82b355e..695884e6cb22 100644
--- a/sys/contrib/dev/rtw89/wow.c
+++ b/sys/contrib/dev/rtw89/wow.c
@@ -12,7 +12,7 @@
 #include "util.h"
 #include "wow.h"
 
-void rtw89_wow_parse_akm(struct rtw89_dev *rtwdev, struct sk_buff *skb)
+void __rtw89_wow_parse_akm(struct rtw89_dev *rtwdev, struct sk_buff *skb)
 {
 	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
 	struct rtw89_wow_param *rtw_wow = &rtwdev->wow;
@@ -639,6 +639,8 @@ static struct ieee80211_key_conf *rtw89_wow_gtk_rekey(struct rtw89_dev *rtwdev,
 	struct ieee80211_key_conf *key;
 	u8 sz;
 
+	lockdep_assert_wiphy(rtwdev->hw->wiphy);
+
 	cipher_info = rtw89_cipher_alg_recognize(cipher);
 	sz = struct_size(rekey_conf, key, cipher_info->len);
 	rekey_conf = kmalloc(sz, GFP_KERNEL);
@@ -651,15 +653,13 @@ static struct ieee80211_key_conf *rtw89_wow_gtk_rekey(struct rtw89_dev *rtwdev,
 	memcpy(rekey_conf->key, gtk,
 	       flex_array_size(rekey_conf, key, cipher_info->len));
 
-	/* ieee80211_gtk_rekey_add() will call set_key(), therefore we
-	 * need to unlock mutex
-	 */
-	mutex_unlock(&rtwdev->mutex);
 	if (ieee80211_vif_is_mld(wow_vif))
-		key = ieee80211_gtk_rekey_add(wow_vif, rekey_conf, rtwvif_link->link_id);
+		key = ieee80211_gtk_rekey_add(wow_vif, keyidx, gtk,
+					      cipher_info->len,
+					      rtwvif_link->link_id);
 	else
-		key = ieee80211_gtk_rekey_add(wow_vif, rekey_conf, -1);
-	mutex_lock(&rtwdev->mutex);
+		key = ieee80211_gtk_rekey_add(wow_vif, keyidx, gtk,
+					      cipher_info->len, -1);
 
 	kfree(rekey_conf);
 	if (IS_ERR(key)) {
@@ -1124,8 +1124,7 @@ static int rtw89_wow_set_wakeups(struct rtw89_dev *rtwdev,
 		rtw89_wow_init_pno(rtwdev, wowlan->nd_config);
 
 	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
-		/* use the link on HW-0 to do wow flow */
-		rtwvif_link = rtw89_vif_get_link_inst(rtwvif, 0);
+		rtwvif_link = rtw89_get_designated_link(rtwvif);
 		if (!rtwvif_link)
 			continue;
 
@@ -1451,6 +1450,8 @@ static void rtw89_fw_release_pno_pkt_list(struct rtw89_dev *rtwdev,
 static int rtw89_pno_scan_update_probe_req(struct rtw89_dev *rtwdev,
 					   struct rtw89_vif_link *rtwvif_link)
 {
+	static const u8 basic_rate_ie[] = {WLAN_EID_SUPP_RATES, 0x08,
+		 0x0c, 0x12, 0x18, 0x24, 0x30, 0x48, 0x60, 0x6c};
 	struct rtw89_wow_param *rtw_wow = &rtwdev->wow;
 	struct cfg80211_sched_scan_request *nd_config = rtw_wow->nd_config;
 	u8 num = nd_config->n_match_sets, i;
@@ -1462,10 +1463,11 @@ static int rtw89_pno_scan_update_probe_req(struct rtw89_dev *rtwdev,
 		skb = ieee80211_probereq_get(rtwdev->hw, rtwvif_link->mac_addr,
 					     nd_config->match_sets[i].ssid.ssid,
 					     nd_config->match_sets[i].ssid.ssid_len,
-					     nd_config->ie_len);
+					     nd_config->ie_len + sizeof(basic_rate_ie));
 		if (!skb)
 			return -ENOMEM;
 
+		skb_put_data(skb, basic_rate_ie, sizeof(basic_rate_ie));
 		skb_put_data(skb, nd_config->ie, nd_config->ie_len);
 
 		info = kzalloc(sizeof(*info), GFP_KERNEL);
@@ -1516,7 +1518,7 @@ static int rtw89_pno_scan_offload(struct rtw89_dev *rtwdev, bool enable)
 	opt.enable = enable;
 	opt.repeat = RTW89_SCAN_NORMAL;
 	opt.norm_pd = max(interval, 1) * 10; /* in unit of 100ms */
-	opt.delay = max(rtw_wow->nd_config->delay, 1);
+	opt.delay = max(rtw_wow->nd_config->delay, 1) * 1000;
 
 	if (rtwdev->chip->chip_gen == RTW89_CHIP_BE) {
 		opt.operation = enable ? RTW89_SCAN_OP_START : RTW89_SCAN_OP_STOP;
@@ -1528,7 +1530,7 @@ static int rtw89_pno_scan_offload(struct rtw89_dev *rtwdev, bool enable)
 		opt.opch_end = RTW89_CHAN_INVALID;
 	}
 
-	mac->scan_offload(rtwdev, &opt, rtwvif_link, true);
+	rtw89_mac_scan_offload(rtwdev, &opt, rtwvif_link, true);
 
 	return 0;
 }
diff --git a/sys/contrib/dev/rtw89/wow.h b/sys/contrib/dev/rtw89/wow.h
index f91991e8f2e3..6606528d31c7 100644
--- a/sys/contrib/dev/rtw89/wow.h
+++ b/sys/contrib/dev/rtw89/wow.h
@@ -116,9 +116,21 @@ static inline bool rtw_wow_has_mgd_features(struct rtw89_dev *rtwdev)
 	return !bitmap_empty(rtw_wow->flags, RTW89_WOW_FLAG_NUM);
 }
 
+void __rtw89_wow_parse_akm(struct rtw89_dev *rtwdev, struct sk_buff *skb);
+
+static inline
+void rtw89_wow_parse_akm(struct rtw89_dev *rtwdev, struct sk_buff *skb)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+
+	if (likely(!ieee80211_is_assoc_req(hdr->frame_control)))
+		return;
+
+	__rtw89_wow_parse_akm(rtwdev, skb);
+}
+
 int rtw89_wow_suspend(struct rtw89_dev *rtwdev, struct cfg80211_wowlan *wowlan);
 int rtw89_wow_resume(struct rtw89_dev *rtwdev);
-void rtw89_wow_parse_akm(struct rtw89_dev *rtwdev, struct sk_buff *skb);
 #else
 static inline
 void rtw89_wow_parse_akm(struct rtw89_dev *rtwdev, struct sk_buff *skb)
diff --git a/sys/contrib/libnv/bsd_nvpair.c b/sys/contrib/libnv/bsd_nvpair.c
index c73bc2189121..b884dd260b84 100644
--- a/sys/contrib/libnv/bsd_nvpair.c
+++ b/sys/contrib/libnv/bsd_nvpair.c
@@ -985,13 +985,13 @@ nvpair_unpack_string_array(bool isbe __unused, nvpair_t *nvp,
 	size = nvp->nvp_datasize;
 	tmp = (const char *)ptr;
 	for (ii = 0; ii < nvp->nvp_nitems; ii++) {
-		len = strnlen(tmp, size - 1) + 1;
-		size -= len;
-		if (tmp[len - 1] != '\0') {
+		if (size <= 0) {
 			ERRNO_SET(EINVAL);
 			return (NULL);
 		}
-		if (size < 0) {
+		len = strnlen(tmp, size - 1) + 1;
+		size -= len;
+		if (tmp[len - 1] != '\0') {
 			ERRNO_SET(EINVAL);
 			return (NULL);
 		}
diff --git a/sys/contrib/libnv/nvlist.c b/sys/contrib/libnv/nvlist.c
index 41edc72322c3..73226ee51a78 100644
--- a/sys/contrib/libnv/nvlist.c
+++ b/sys/contrib/libnv/nvlist.c
@@ -478,7 +478,7 @@ nvlist_dump_error_check(const nvlist_t *nvl, int fd, int level)
 void
 nvlist_dump(const nvlist_t *nvl, int fd)
 {
-	const nvlist_t *tmpnvl;
+	const nvlist_t *tmpnvl, *top;
 	nvpair_t *nvp, *tmpnvp;
 	void *cookie;
 	int level;
@@ -487,6 +487,7 @@ nvlist_dump(const nvlist_t *nvl, int fd)
 	if (nvlist_dump_error_check(nvl, fd, level))
 		return;
 
+	top = nvl;
 	nvp = nvlist_first_nvpair(nvl);
 	while (nvp != NULL) {
 		dprintf(fd, "%*s%s (%s):", level * 4, "", nvpair_name(nvp),
@@ -645,6 +646,8 @@ nvlist_dump(const nvlist_t *nvl, int fd)
 
 		while ((nvp = nvlist_next_nvpair(nvl, nvp)) == NULL) {
 			do {
+				if (nvl == top)
+					return;
 				cookie = NULL;
 				if (nvlist_in_array(nvl))
 					dprintf(fd, "%*s,\n", level * 4, "");
@@ -847,7 +850,7 @@ nvlist_xpack(const nvlist_t *nvl, int64_t *fdidxp, size_t *sizep)
 {
 	unsigned char *buf, *ptr;
 	size_t left, size;
-	const nvlist_t *tmpnvl;
+	const nvlist_t *tmpnvl, *top;
 	nvpair_t *nvp, *tmpnvp;
 	void *cookie;
 
@@ -868,6 +871,7 @@ nvlist_xpack(const nvlist_t *nvl, int64_t *fdidxp, size_t *sizep)
 
 	ptr = nvlist_pack_header(nvl, ptr, &left);
 
+	top = nvl;
 	nvp = nvlist_first_nvpair(nvl);
 	while (nvp != NULL) {
 		NVPAIR_ASSERT(nvp);
@@ -958,6 +962,8 @@ nvlist_xpack(const nvlist_t *nvl, int64_t *fdidxp, size_t *sizep)
 			goto fail;
 		while ((nvp = nvlist_next_nvpair(nvl, nvp)) == NULL) {
 			do {
+				if (nvl == top)
+					goto out;
 				cookie = NULL;
 				if (nvlist_in_array(nvl)) {
 					ptr = nvpair_pack_nvlist_array_next(ptr,
diff --git a/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md b/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md
index 9b50a4a3d96e..f3d4316f6f67 100644
--- a/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md
@@ -14,7 +14,7 @@ Please check our issue tracker before opening a new feature request.
 Filling out the following template will help other contributors better understand your proposed feature.
 -->
 
-### Describe the feature would like to see added to OpenZFS
+### Describe the feature you would like to see added to OpenZFS
 
 <!--
 Provide a clear and concise description of the feature.
diff --git a/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md
index 79809179cf13..47edc8174603 100644
--- a/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md
+++ b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md
@@ -2,11 +2,6 @@
 
 <!--- Provide a general summary of your changes in the Title above -->
 
-<!---
-Documentation on ZFS Buildbot options can be found at
-https://openzfs.github.io/openzfs-docs/Developer%20Resources/Buildbot%20Options.html
--->
-
 ### Motivation and Context
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py b/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py
index b49255e8381d..08021aabcb61 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py
+++ b/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py
@@ -65,7 +65,7 @@ if __name__ == '__main__':
 
     # check last (HEAD) commit message
     last_commit_message_raw = subprocess.run([
-        'git', 'show', '-s', '--format=%B', 'HEAD'
+        'git', 'show', '-s', '--format=%B', head
     ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     for line in last_commit_message_raw.stdout.decode().splitlines():
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh
index de29ad1f57b6..0278264d9279 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh
@@ -6,6 +6,13 @@
 
 set -eu
 
+# We've been seeing this script take over 15min to run.  This may or
+# may not be normal.  Just to get a little more insight, print out
+# a message to stdout with the top running process, and do this every
+# 30 seconds.  We can delete this watchdog later once we get a better
+# handle on what the timeout value should be.
+(while [ 1 ] ; do sleep 30 && echo "[watchdog: $(ps -eo cmd --sort=-pcpu  | head -n 2 | tail -n 1)}')]"; done) &
+
 # install needed packages
 export DEBIAN_FRONTEND="noninteractive"
 sudo apt-get -y update
@@ -65,3 +72,6 @@ sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 -O relatime=off \
 for i in /sys/block/s*/queue/scheduler; do
   echo "none" | sudo tee $i
 done
+
+# Kill off our watchdog
+kill $(jobs -p)
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh
index 885a64037f89..422b3e9df388 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh
@@ -25,6 +25,10 @@ UBMIRROR="https://cloud-images.ubuntu.com"
 # default nic model for vm's
 NIC="virtio"
 
+# additional options for virt-install
+OPTS[0]=""
+OPTS[1]=""
+
 case "$OS" in
   almalinux8)
     OSNAME="AlmaLinux 8"
@@ -61,6 +65,14 @@ case "$OS" in
     OSNAME="Debian 12"
     URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
     ;;
+  debian13)
+    OSNAME="Debian 13"
+    # TODO: Overwrite OSv to debian13 for virt-install until it's added to osinfo
+    OSv="debian12"
+    URL="https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2"
+    OPTS[0]="--boot"
+    OPTS[1]="uefi=on"
+    ;;
   fedora41)
     OSNAME="Fedora 41"
     OSv="fedora-unknown"
@@ -109,7 +121,14 @@ case "$OS" in
     KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
     ;;
   freebsd15-0c)
-    FreeBSD="15.0-CURRENT"
+    FreeBSD="15.0-ALPHA4"
+    OSNAME="FreeBSD $FreeBSD"
+    OSv="freebsd14.0"
+    URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz"
+    KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
+    ;;
+  freebsd16-0c)
+    FreeBSD="16.0-CURRENT"
     OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
     URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz"
@@ -242,7 +261,7 @@ sudo virt-install \
   --network bridge=virbr0,model=$NIC,mac='52:54:00:83:79:00' \
   --cloud-init user-data=/tmp/user-data \
   --disk $DISK,bus=virtio,cache=none,format=raw,driver.discard=unmap \
-  --import --noautoconsole >/dev/null
+  --import --noautoconsole ${OPTS[0]} ${OPTS[1]} >/dev/null
 
 # Give the VMs hostnames so we don't have to refer to them with
 # hardcoded IP addresses.
@@ -275,7 +294,7 @@ else
   while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
     ssh 2>/dev/null root@vm0 "uname -a" && break
   done
-  ssh root@vm0 "pkg install -y bash ca_root_nss git qemu-guest-agent python3 py311-cloud-init"
+  ssh root@vm0 "env IGNORE_OSVERSION=yes pkg install -y bash ca_root_nss git qemu-guest-agent python3 py311-cloud-init"
   ssh root@vm0 "chsh -s $BASH root"
   ssh root@vm0 'sysrc qemu_guest_agent_enable="YES"'
   ssh root@vm0 'sysrc cloudinit_enable="YES"'
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh
index c41ecd09d52e..f67bb2f68e94 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh
@@ -20,7 +20,7 @@ function archlinux() {
   sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \
     fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \
     parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \
-    samba sysstat rng-tools rsync wget xxhash
+    samba strace sysstat rng-tools rsync wget xxhash
   echo "##[endgroup]"
 }
 
@@ -41,9 +41,10 @@ function debian() {
     libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \
     libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \
     lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \
-    python3-cffi python3-dev python3-distlib python3-packaging \
+    python3-cffi python3-dev python3-distlib python3-packaging libtirpc-dev \
     python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \
-    rsync samba sysstat uuid-dev watchdog wget xfslibs-dev  xxhash zlib1g-dev
+    rsync samba strace sysstat uuid-dev watchdog wget xfslibs-dev xxhash \
+    zlib1g-dev
   echo "##[endgroup]"
 }
 
@@ -87,8 +88,8 @@ function rhel() {
     libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \
     parted perf python3 python3-cffi python3-devel python3-packaging \
     kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \
-    rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \
-    zlib-devel
+    rpm-build rsync samba strace sysstat systemd watchdog wget xfsprogs-devel \
+    xxhash zlib-devel
   echo "##[endgroup]"
 }
 
@@ -104,7 +105,7 @@ function install_fedora_experimental_kernel {
   our_version="$1"
   sudo dnf -y copr enable @kernel-vanilla/stable
   sudo dnf -y copr enable @kernel-vanilla/mainline
-  all="$(sudo dnf list --showduplicates kernel-*)"
+  all="$(sudo dnf list --showduplicates kernel-* python3-perf* perf* bpftool*)"
   echo "Available versions:"
   echo "$all"
 
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh
index 17e976ebcc39..2807d9e77127 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh
@@ -5,12 +5,13 @@
 #
 # Usage:
 #
-#       qemu-4-build-vm.sh OS [--enable-debug][--dkms][--poweroff]
-#           [--release][--repo][--tarball]
+#       qemu-4-build-vm.sh OS [--enable-debug][--dkms][--patch-level NUM]
+#               [--poweroff][--release][--repo][--tarball]
 #
 # OS:           OS name like 'fedora41'
 # --enable-debug:  Build RPMs with '--enable-debug' (for testing)
 # --dkms:       Build DKMS RPMs as well
+# --patch-level NUM:    Use a custom patch level number for packages.
 # --poweroff:   Power-off the VM after building
 # --release     Build zfs-release*.rpm as well
 # --repo        After building everything, copy RPMs into /tmp/repo
@@ -21,6 +22,7 @@
 
 ENABLE_DEBUG=""
 DKMS=""
+PATCH_LEVEL=""
 POWEROFF=""
 RELEASE=""
 REPO=""
@@ -35,6 +37,11 @@ while [[ $# -gt 0 ]]; do
       DKMS=1
       shift
       ;;
+    --patch-level)
+      PATCH_LEVEL=$2
+      shift
+      shift
+      ;;
     --poweroff)
       POWEROFF=1
       shift
@@ -215,6 +222,10 @@ function rpm_build_and_install() {
   run ./autogen.sh
   echo "##[endgroup]"
 
+  if [ -n "$PATCH_LEVEL" ] ; then
+    sed -i -E 's/(Release:\s+)1/\1'$PATCH_LEVEL'/g' META
+  fi
+
   echo "##[group]Configure"
   run ./configure --enable-debuginfo $extra
   echo "##[endgroup]"
@@ -328,7 +339,13 @@ fi
 # almalinux9.5
 # fedora42
 source /etc/os-release
-sudo hostname "$ID$VERSION_ID"
+ if which hostnamectl &> /dev/null ; then
+  # Fedora 42+ use hostnamectl
+  sudo hostnamectl set-hostname "$ID$VERSION_ID"
+  sudo hostnamectl set-hostname --pretty "$ID$VERSION_ID"
+else
+  sudo hostname "$ID$VERSION_ID"
+fi
 
 # save some sysinfo
 uname -a > /var/tmp/uname.txt
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh
index 6bf10024a1a6..4869c1003e48 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh
@@ -12,16 +12,26 @@ source /var/tmp/env.txt
 # wait for poweroff to succeed
 PID=$(pidof /usr/bin/qemu-system-x86_64)
 tail --pid=$PID -f /dev/null
-sudo virsh undefine openzfs
+sudo virsh undefine --nvram openzfs
 
 # cpu pinning
 CPUSET=("0,1" "2,3")
 
+# additional options for virt-install
+OPTS[0]=""
+OPTS[1]=""
+
 case "$OS" in
   freebsd*)
     # FreeBSD needs only 6GiB
     RAM=6
     ;;
+  debian13)
+    RAM=8
+    # Boot Debian 13 with uefi=on and secureboot=off (ZFS Kernel Module not signed)
+    OPTS[0]="--boot"
+    OPTS[1]="firmware=efi,firmware.feature0.name=secure-boot,firmware.feature0.enabled=no"
+    ;;
   *)
     # Linux needs more memory, but can be optimized to share it via KSM
     RAM=8
@@ -79,7 +89,7 @@ EOF
     --network bridge=virbr0,model=$NIC,mac="52:54:00:83:79:0$i" \
     --disk $DISK-system,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
     --disk $DISK-tests,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
-    --import --noautoconsole >/dev/null
+    --import --noautoconsole ${OPTS[0]} ${OPTS[1]}
 done
 
 # generate some memory stats
@@ -98,19 +108,30 @@ echo '*/5 * * * *  /root/cronjob.sh' > crontab.txt
 sudo crontab crontab.txt
 rm crontab.txt
 
-# check if the machines are okay
-echo "Waiting for vm's to come up...  (${VMs}x CPU=$CPU RAM=$RAM)"
-for ((i=1; i<=VMs; i++)); do
-  .github/workflows/scripts/qemu-wait-for-vm.sh vm$i
-done
-echo "All $VMs VMs are up now."
-
 # Save the VM's serial output (ttyS0) to /var/tmp/console.txt
 # - ttyS0 on the VM corresponds to a local /dev/pty/N entry
 # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry
 for ((i=1; i<=VMs; i++)); do
   mkdir -p $RESPATH/vm$i
   read "pty" <<< $(sudo virsh ttyconsole vm$i)
+
+  # Create the file so we can tail it, even if there's no output.
+  touch $RESPATH/vm$i/console.txt
+
   sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" &
+
+  # Write all VM boot lines to the console to aid in debugging failed boots.
+  # The boot lines from all the VMs will be munged together, so prepend each
+  # line with the vm hostname (like 'vm1:').
+  (while IFS=$'\n' read -r line; do echo "vm$i: $line" ; done < <(sudo tail -f $RESPATH/vm$i/console.txt)) &
+
 done
 echo "Console logging for ${VMs}x $OS started."
+
+
+# check if the machines are okay
+echo "Waiting for vm's to come up...  (${VMs}x CPU=$CPU RAM=$RAM)"
+for ((i=1; i<=VMs; i++)); do
+  .github/workflows/scripts/qemu-wait-for-vm.sh vm$i
+done
+echo "All $VMs VMs are up now."
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh
index 5ab822f4f076..ca6ac77f146d 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh
@@ -111,7 +111,7 @@ fi
 sudo dmesg -c > dmesg-prerun.txt
 mount > mount.txt
 df -h > df-prerun.txt
-$TDIR/zfs-tests.sh -vK -s 3GB -T $TAGS
+$TDIR/zfs-tests.sh -vKO -s 3GB -T $TAGS
 RV=$?
 df -h > df-postrun.txt
 echo $RV > tests-exitcode.txt
diff --git a/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml b/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml
index 5b5afe746859..d8a95954fe1a 100644
--- a/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml
+++ b/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml
@@ -32,6 +32,11 @@ on:
         options:
         - "Build RPMs"
         - "Test repo"
+      patch_level:
+        type: string
+        required: false
+        default: ""
+        description: "(optional) patch level number"
       repo_url:
         type: string
         required: false
@@ -78,7 +83,13 @@ jobs:
                 mkdir -p /tmp/repo
                 ssh zfs@vm0 '$HOME/zfs/.github/workflows/scripts/qemu-test-repo-vm.sh' ${{ github.event.inputs.repo_url }}
         else
-                .github/workflows/scripts/qemu-4-build.sh --repo --release --dkms --tarball ${{ matrix.os }}
+                EXTRA=""
+                if [ -n "${{ github.event.inputs.patch_level }}" ] ; then
+                        EXTRA="--patch-level ${{ github.event.inputs.patch_level }}"
+                fi
+
+                .github/workflows/scripts/qemu-4-build.sh $EXTRA \
+                        --repo --release --dkms --tarball ${{ matrix.os }}
         fi
 
     - name: Prepare artifacts
diff --git a/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml b/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml
index cda620313189..3b164548f9be 100644
--- a/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml
+++ b/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Generate OS config and CI type
         id: os
         run: |
-          FULL_OS='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian11", "debian12", "fedora41", "fedora42", "freebsd13-5r", "freebsd14-3s", "freebsd15-0c", "ubuntu22", "ubuntu24"]'
+          FULL_OS='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian12", "debian13", "fedora41", "fedora42", "freebsd13-5r", "freebsd14-3s", "freebsd15-0c", "ubuntu22", "ubuntu24"]'
           QUICK_OS='["almalinux8", "almalinux9", "almalinux10", "debian12", "fedora42", "freebsd14-3s", "ubuntu24"]'
           # determine CI type when running on PR
           ci_type="full"
@@ -44,7 +44,7 @@ jobs:
             os_selection="$FULL_OS"
           fi
 
-          if [ ${{ github.event.inputs.fedora_kernel_ver }} != "" ] ; then
+          if ${{ github.event.inputs.fedora_kernel_ver != '' }}; then
               # They specified a custom kernel version for Fedora.  Use only
               # Fedora runners.
               os_json=$(echo ${os_selection} | jq -c '[.[] | select(startswith("fedora"))]')
@@ -53,9 +53,8 @@ jobs:
               os_json=$(echo ${os_selection} | jq -c)
           fi
 
-          echo $os_json
-          echo "os=$os_json" >> $GITHUB_OUTPUT
-          echo "ci_type=$ci_type" >> $GITHUB_OUTPUT
+          echo "os=$os_json" | tee -a $GITHUB_OUTPUT
+          echo "ci_type=$ci_type" | tee -a $GITHUB_OUTPUT
 
   qemu-vm:
     name: qemu-x86
@@ -63,13 +62,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # rhl:     almalinux8, almalinux9, centos-stream9, fedora41
-        # debian:  debian11, debian12, ubuntu22, ubuntu24
+        # rhl:     almalinux8, almalinux9, centos-stream9, fedora4x
+        # debian:  debian12, debian13, ubuntu22, ubuntu24
         # misc:    archlinux, tumbleweed
         # FreeBSD variants of 2025-06:
         # FreeBSD Release: freebsd13-5r, freebsd14-2r, freebsd14-3r
         # FreeBSD Stable:  freebsd13-5s, freebsd14-3s
-        # FreeBSD Current: freebsd15-0c
+        # FreeBSD Current: freebsd15-0c, freebsd16-0c
         os: ${{ fromJson(needs.test-config.outputs.test_os) }}
     runs-on: ubuntu-24.04
     steps:
@@ -78,8 +77,12 @@ jobs:
         ref: ${{ github.event.pull_request.head.sha }}
 
     - name: Setup QEMU
-      timeout-minutes: 10
-      run: .github/workflows/scripts/qemu-1-setup.sh
+      timeout-minutes: 20
+      run: |
+        # Add a timestamp to each line to debug timeouts
+        while IFS=$'\n' read -r line; do
+            echo "$(date +'%H:%M:%S') $line"
+        done < <(.github/workflows/scripts/qemu-1-setup.sh)
 
     - name: Start build machine
       timeout-minutes: 10
diff --git a/sys/contrib/openzfs/.mailmap b/sys/contrib/openzfs/.mailmap
index b6d942c000b8..3397fbc3745d 100644
--- a/sys/contrib/openzfs/.mailmap
+++ b/sys/contrib/openzfs/.mailmap
@@ -23,6 +23,7 @@
 # These maps are making names consistent where they have varied but the email
 # address has never changed. In most cases, the full name is in the
 # Signed-off-by of a commit with a matching author.
+Achill Gilgenast <achill@achill.org>
 Ahelenia Ziemiańska <nabijaczleweli@gmail.com>
 Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
 Alex John <alex@stty.io>
@@ -37,6 +38,7 @@ Crag Wang <crag0715@gmail.com>
 Damian Szuberski <szuberskidamian@gmail.com>
 Daniel Kolesa <daniel@octaforge.org>
 Debabrata Banerjee <dbavatar@gmail.com>
+Diwakar Kristappagari <diwakar-k@hpe.com>
 Finix Yan <yanchongwen@hotmail.com>
 Gaurav Kumar <gauravk.18@gmail.com>
 Gionatan Danti <g.danti@assyoma.it>
@@ -51,6 +53,7 @@ Jason Harmening <jason.harmening@gmail.com>
 Jeremy Faulkner <gldisater@gmail.com>
 Jinshan Xiong <jinshan.xiong@gmail.com>
 John Poduska <jpoduska@datto.com>
+Jo Zzsi <jozzsicsataban@gmail.com>
 Justin Scholz <git@justinscholz.de>
 Ka Ho Ng <khng300@gmail.com>
 Kash Pande <github@tripleback.net>
@@ -65,6 +68,7 @@ Michael Gmelin <grembo@FreeBSD.org>
 Olivier Mazouffre <olivier.mazouffre@ims-bordeaux.fr>
 Piotr Kubaj <pkubaj@anongoth.pl>
 Quentin Zdanis <zdanisq@gmail.com>
+Roberto Ricci <io@r-ricci.it>
 Roberto Ricci <ricci@disroot.org>
 Rob Norris <robn@despairlabs.com>
 Rob Norris <rob.norris@klarasystems.com>
@@ -81,7 +85,10 @@ Youzhong Yang <youzhong@gmail.com>
 # Signed-off-by: overriding Author:
 Alexander Ziaee <ziaee@FreeBSD.org> <concussious@runbox.com>
 Felix Schmidt <felixschmidt20@aol.com> <f.sch.prototype@gmail.com>
+Jean-Sébastien Pédron <dumbbell@FreeBSD.org> <jean-sebastien.pedron@dumbbell.fr>
+Konstantin Belousov <kib@FreeBSD.org> <kib@kib.kiev.ua>
 Olivier Certner <olce@FreeBSD.org> <olce.freebsd@certner.fr>
+Patrick Xia <patrickx@google.com> <octalc0de@aim.com>
 Phil Sutter <phil@nwl.cc> <p.github@nwl.cc>
 poscat <poscat@poscat.moe> <poscat0x04@outlook.com>
 Qiuhao Chen <chenqiuhao1997@gmail.com> <haohao0924@126.com>
@@ -123,6 +130,7 @@ buzzingwires <buzzingwires@outlook.com> <131118055+buzzingwires@users.noreply.gi
 Cedric Maunoury <cedric.maunoury@gmail.com> <38213715+cedricmaunoury@users.noreply.github.com>
 Charles Suh <charles.suh@gmail.com> <charlessuh@users.noreply.github.com>
 Chris Peredun <chris.peredun@ixsystems.com> <126915832+chrisperedun@users.noreply.github.com>
+classabbyamp <dev@placeviolette.net> <5366828+classabbyamp@users.noreply.github.com>
 Dacian Reece-Stremtan <dacianstremtan@gmail.com> <35844628+dacianstremtan@users.noreply.github.com>
 Damian Szuberski <szuberskidamian@gmail.com> <30863496+szubersk@users.noreply.github.com>
 Daniel Hiepler <d-git@coderdu.de> <32984777+heeplr@users.noreply.github.com>
@@ -145,6 +153,7 @@ Gaurav Kumar <gauravk.18@gmail.com> <gaurkuma@users.noreply.github.com>
 George Gaydarov <git@gg7.io> <gg7@users.noreply.github.com>
 Georgy Yakovlev <gyakovlev@gentoo.org> <168902+gyakovlev@users.noreply.github.com>
 Gerardwx <gerardw@alum.mit.edu> <Gerardwx@users.noreply.github.com>
+Germano Massullo <germano.massullo@gmail.com> <Germano0@users.noreply.github.com>
 Gian-Carlo DeFazio <defazio1@llnl.gov> <defaziogiancarlo@users.noreply.github.com>
 Giuseppe Di Natale <dinatale2@llnl.gov> <dinatale2@users.noreply.github.com>
 Hajo Möller <dasjoe@gmail.com> <dasjoe@users.noreply.github.com>
@@ -164,6 +173,7 @@ John Ramsden <johnramsden@riseup.net> <johnramsden@users.noreply.github.com>
 Jonathon Fernyhough <jonathon@m2x.dev> <559369+jonathonf@users.noreply.github.com>
 Jose Luis Duran <jlduran@gmail.com> <jlduran@users.noreply.github.com>
 Justin Hibbits <chmeeedalf@gmail.com> <chmeeedalf@users.noreply.github.com>
+Kaitlin Hoang <kthoang@amazon.com> <khoang98@users.noreply.github.com>
 Kevin Greene <kevin.greene@delphix.com> <104801862+kxgreene@users.noreply.github.com>
 Kevin Jin <lostking2008@hotmail.com> <33590050+jxdking@users.noreply.github.com>
 Kevin P. Fleming <kevin@km6g.us> <kpfleming@users.noreply.github.com>
@@ -181,6 +191,7 @@ Michael Niewöhner <foss@mniewoehner.de> <c0d3z3r0@users.noreply.github.com>
 Michael Zhivich <mzhivich@akamai.com> <33133421+mzhivich@users.noreply.github.com>
 MigeljanImeri <ImeriMigel@gmail.com> <78048439+MigeljanImeri@users.noreply.github.com>
 Mo Zhou <cdluminate@gmail.com> <5723047+cdluminate@users.noreply.github.com>
+nav1s <nav1s@proton.me> <42621369+nav1s@users.noreply.github.com>
 Nick Mattis <nickm970@gmail.com> <nmattis@users.noreply.github.com>
 omni <omni+vagant@hack.org> <79493359+omnivagant@users.noreply.github.com>
 Pablo Correa Gómez <ablocorrea@hotmail.com> <32678034+pablofsf@users.noreply.github.com>
@@ -202,6 +213,7 @@ Samuel Wycliffe <samuelwycliffe@gmail.com> <50765275+npc203@users.noreply.github
 Savyasachee Jha <hi@savyasacheejha.com> <savyajha@users.noreply.github.com>
 Scott Colby <scott@scolby.com> <scolby33@users.noreply.github.com>
 Sean Eric Fagan <kithrup@mac.com> <kithrup@users.noreply.github.com>
+Shreshth Srivastava <shreshthsrivastava2@gmail.com> <66148173+Shreshth3@users.noreply.github.com>
 Spencer Kinny <spencerkinny1995@gmail.com> <30333052+Spencer-Kinny@users.noreply.github.com>
 Srikanth N S <srikanth.nagasubbaraoseetharaman@hpe.com> <75025422+nssrikanth@users.noreply.github.com>
 Stefan Lendl <s.lendl@proxmox.com> <1321542+stfl@users.noreply.github.com>
diff --git a/sys/contrib/openzfs/AUTHORS b/sys/contrib/openzfs/AUTHORS
index a9d249a66f1e..e496c0e8a807 100644
--- a/sys/contrib/openzfs/AUTHORS
+++ b/sys/contrib/openzfs/AUTHORS
@@ -10,6 +10,7 @@ PAST MAINTAINERS:
 CONTRIBUTORS:
 
     Aaron Fineman <abyxcos@gmail.com>
+    Achill Gilgenast <achill@achill.org>
     Adam D. Moss <c@yotes.com>
     Adam Leventhal <ahl@delphix.com>
     Adam Stevko <adam.stevko@gmail.com>
@@ -59,6 +60,7 @@ CONTRIBUTORS:
     Andreas Buschmann <andreas.buschmann@tech.net.de>
     Andreas Dilger <adilger@intel.com>
     Andreas Vögele <andreas@andreasvoegele.com>
+    Andres <a-d-j-i@users.noreply.github.com>
     Andrew Barnes <barnes333@gmail.com>
     Andrew Hamilton <ahamilto@tjhsst.edu>
     Andrew Innes <andrew.c12@gmail.com>
@@ -72,6 +74,7 @@ CONTRIBUTORS:
     Andrey Prokopenko <job@terem.fr>
     Andrey Vesnovaty <andrey.vesnovaty@gmail.com>
     Andriy Gapon <avg@freebsd.org>
+    Andriy Tkachuk <andriy.tkachuk@seagate.com>
     Andy Bakun <github@thwartedefforts.org>
     Andy Fiddaman <omnios@citrus-it.co.uk>
     Aniruddha Shankar <k@191a.net>
@@ -120,6 +123,7 @@ CONTRIBUTORS:
     Caleb James DeLisle <calebdelisle@lavabit.com>
     Cameron Harr <harr1@llnl.gov>
     Cao Xuewen <cao.xuewen@zte.com.cn>
+    Carl George <carlwgeorge@gmail.com>
     Carlo Landmeter <clandmeter@gmail.com>
     Carlos Alberto Lopez Perez <clopez@igalia.com>
     Cedric Maunoury <cedric.maunoury@gmail.com>
@@ -150,6 +154,7 @@ CONTRIBUTORS:
     Chris Zubrzycki <github@mid-earth.net>
     Chuck Tuffli <ctuffli@gmail.com>
     Chunwei Chen <david.chen@nutanix.com>
+    classabbyamp <dev@placeviolette.net>
     Clemens Fruhwirth <clemens@endorphin.org>
     Clemens Lang <cl@clang.name>
     Clint Armstrong <clint@clintarmstrong.net>
@@ -157,6 +162,7 @@ CONTRIBUTORS:
     Colin Ian King <colin.king@canonical.com>
     Colin Percival <cperciva@tarsnap.com>
     Colm Buckley <colm@tuatha.org>
+    Cong Zhang <congzhangzh@users.noreply.github.com>
     Crag Wang <crag0715@gmail.com>
     Craig Loomis <cloomis@astro.princeton.edu>
     Craig Sanders <github@taz.net.au>
@@ -200,6 +206,7 @@ CONTRIBUTORS:
     Dimitri John Ledkov <xnox@ubuntu.com>
     Dimitry Andric <dimitry@andric.com>
     Dirkjan Bussink <d.bussink@gmail.com>
+    Diwakar Kristappagari <diwakar-k@hpe.com>
     Dmitry Khasanov <pik4ez@gmail.com>
     Dominic Pearson <dsp@technoanimal.net>
     Dominik Hassler <hadfl@omniosce.org>
@@ -212,6 +219,7 @@ CONTRIBUTORS:
     Eitan Adler <lists@eitanadler.com>
     Eli Rosenthal <eli.rosenthal@delphix.com>
     Eli Schwartz <eschwartz93@gmail.com>
+    Eric A. Borisch <eborisch@gmail.com>
     Eric Desrochers <eric.desrochers@canonical.com>
     Eric Dillmann <eric@jave.fr>
     Eric Schrock <Eric.Schrock@delphix.com>
@@ -250,6 +258,7 @@ CONTRIBUTORS:
     George Wilson <gwilson@delphix.com>
     Georgy Yakovlev <ya@sysdump.net>
     Gerardwx <gerardw@alum.mit.edu>
+    Germano Massullo <germano.massullo@gmail.com>
     Gian-Carlo DeFazio <defazio1@llnl.gov>
     Gionatan Danti <g.danti@assyoma.it>
     Giuseppe Di Natale <guss80@gmail.com>
@@ -282,17 +291,20 @@ CONTRIBUTORS:
     Henrik Riomar <henrik.riomar@gmail.com>
     Herb Wartens <wartens2@llnl.gov>
     Hiếu Lê <leorize+oss@disroot.org>
+    hoshinomori <hoshinomorimorimo@gmail.com>
     Huang Liu <liu.huang@zte.com.cn>
     HÃ¥kan Johansson <f96hajo@chalmers.se>
     Igor K <igor@dilos.org>
     Igor Kozhukhov <ikozhukhov@gmail.com>
     Igor Lvovsky <ilvovsky@gmail.com>
+    Igor Ostapenko <pm@igoro.pro>
     ilbsmart <wgqimut@gmail.com>
     Ilkka Sovanto <github@ilkka.kapsi.fi>
     illiliti <illiliti@protonmail.com>
     ilovezfs <ilovezfs@icloud.com>
     InsanePrawn <Insane.Prawny@gmail.com>
     Isaac Huang <he.huang@intel.com>
+    Ivan Shapovalov <intelfx@intelfx.name>
     Ivan Volosyuk <Ivan.Volosyuk@gmail.com>
     Jacek Fefliński <feflik@gmail.com>
     Jacob Adams <tookmund@gmail.com>
@@ -315,6 +327,7 @@ CONTRIBUTORS:
     Javen Wu <wu.javen@gmail.com>
     Jaydeep Kshirsagar <jkshirsagar@maxlinear.com>
     Jean-Baptiste Lallement <jean-baptiste@ubuntu.com>
+    Jean-Sébastien Pédron <dumbbell@FreeBSD.org>
     Jeff Dike <jdike@akamai.com>
     Jeremy Faulkner <gldisater@gmail.com>
     Jeremy Gill <jgill@parallax-innovations.com>
@@ -326,6 +339,7 @@ CONTRIBUTORS:
     Jinshan Xiong <jinshan.xiong@intel.com>
     Jitendra Patidar <jitendra.patidar@nutanix.com>
     JK Dingwall <james@dingwall.me.uk>
+    Joel Low <joel@joelsplace.sg>
     Joe Stein <joe.stein@delphix.com>
     John-Mark Gurney <jmg@funkthat.com>
     John Albietz <inthecloud247@gmail.com>
@@ -347,7 +361,9 @@ CONTRIBUTORS:
     Josh Soref <jsoref@users.noreply.github.com>
     Joshua M. Clulow <josh@sysmgr.org>
     José Luis Salvador Rufo <salvador.joseluis@gmail.com>
+    Jo Zzsi <jozzsicsataban@gmail.com>
     João Carlos Mendes Luís <jonny@jonny.eng.br>
+    JT Pennington <jt.pennington@klarasystems.com>
     Julian Brunner <julian.brunner@gmail.com>
     Julian Heuking <JulianH@beckhoff.com>
     jumbi77 <jumbi77@users.noreply.github.com>
@@ -374,11 +390,13 @@ CONTRIBUTORS:
     Kevin Jin <lostking2008@hotmail.com>
     Kevin P. Fleming <kevin@km6g.us>
     Kevin Tanguy <kevin.tanguy@ovh.net>
+    khoang98 <khoang98@users.noreply.github.com>
     KireinaHoro <i@jsteward.moe>
     Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl>
     Kleber Tarcísio <klebertarcisio@yahoo.com.br>
     Kody A Kantor <kody.kantor@gmail.com>
     Kohsuke Kawaguchi <kk@kohsuke.org>
+    Konstantin Belousov <kib@FreeBSD.org>
     Konstantin Khorenko <khorenko@virtuozzo.com>
     KORN Andras <korn@elan.rulez.org>
     kotauskas <v.toncharov@gmail.com>
@@ -407,6 +425,7 @@ CONTRIBUTORS:
     luozhengzheng <luo.zhengzheng@zte.com.cn>
     Luís Henriques <henrix@camandro.org>
     Madhav Suresh <madhav.suresh@delphix.com>
+    Maksym Shkolnyi <maksym.shkolnyi@workato.com>
     manfromafar <jonsonb10@gmail.com>
     Manoj Joseph <manoj.joseph@delphix.com>
     Manuel Amador (Rudd-O) <rudd-o@rudd-o.com>
@@ -447,6 +466,7 @@ CONTRIBUTORS:
     Max Zettlmeißl <max@zettlmeissl.de>
     Md Islam <mdnahian@outlook.com>
     megari <megari@iki.fi>
+    Meriel Luna Mittelbach <lunarlambda@gmail.com>
     Michael D Labriola <michael.d.labriola@gmail.com>
     Michael Franzl <michael@franzl.name>
     Michael Gebetsroither <michael@mgeb.org>
@@ -472,6 +492,7 @@ CONTRIBUTORS:
     Nathaniel Clark <Nathaniel.Clark@misrule.us>
     Nathaniel Wesley Filardo <nwf@cs.jhu.edu>
     Nathan Lewis <linux.robotdude@gmail.com>
+    nav1s <nav1s@proton.me>
     Nav Ravindranath <nav@delphix.com>
     Neal Gompa (ニール・ゴンパ) <ngompa13@gmail.com>
     Ned Bass <bass6@llnl.gov>
@@ -494,7 +515,9 @@ CONTRIBUTORS:
     Orivej Desh <orivej@gmx.fr>
     Pablo Correa Gómez <ablocorrea@hotmail.com>
     Palash Gandhi <pbg4930@rit.edu>
+    Patrick Fasano <patrick@patrickfasano.com>
     Patrick Mooney <pmooney@pfmooney.com>
+    Patrick Xia <patrickx@google.com>
     Patrik Greco <sikevux@sikevux.se>
     Paul B. Henson <henson@acm.org>
     Paul Dagnelie <pcd@delphix.com>
@@ -535,6 +558,7 @@ CONTRIBUTORS:
     Remy Blank <remy.blank@pobox.com>
     renelson <bnelson@nelsonbe.com>
     Reno Reckling <e-github@wthack.de>
+    René Wirnata <rene.wirnata@pandascience.net>
     Ricardo M. Correia <ricardo.correia@oracle.com>
     Riccardo Schirone <rschirone91@gmail.com>
     Richard Allen <belperite@gmail.com>
@@ -593,6 +617,7 @@ CONTRIBUTORS:
     Shengqi Chen <harry-chen@outlook.com>
     SHENGYI HONG <aokblast@FreeBSD.org>
     Shen Yan <shenyanxxxy@qq.com>
+    Shreshth Srivastava <shreshthsrivastava2@gmail.com>
     Sietse <sietse@wizdom.nu>
     Simon Guest <simon.guest@tesujimath.org>
     Simon Howard <fraggle@soulsphere.org>
@@ -640,6 +665,7 @@ CONTRIBUTORS:
     tleydxdy <shironeko.github@tesaguri.club>
     Tobin Harding <me@tobin.cc>
     Todd Seidelmann <seidelma@users.noreply.github.com>
+    Todd Zullinger <tmz@pobox.com>
     Tom Caputi <tcaputi@datto.com>
     Tom Matthews <tom@axiom-partners.com>
     Tomohiro Kusumi <kusumi.tomohiro@gmail.com>
@@ -652,6 +678,7 @@ CONTRIBUTORS:
     Toyam Cox <aviator45003@gmail.com>
     Trevor Bautista <trevrb@trevrb.net>
     Trey Dockendorf <treydock@gmail.com>
+    trick2011 <trick2011@users.noreply.github.com>
     Troels Nørgaard <tnn@tradeshift.com>
     tstabrawa <tstabrawa@users.noreply.github.com>
     Tulsi Jain <tulsi.jain@delphix.com>
diff --git a/sys/contrib/openzfs/META b/sys/contrib/openzfs/META
index 47f0795bfa11..bdb7aee48041 100644
--- a/sys/contrib/openzfs/META
+++ b/sys/contrib/openzfs/META
@@ -1,10 +1,10 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       2.3.99
+Version:       2.4.99
 Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 6.15
+Linux-Maximum: 6.17
 Linux-Minimum: 4.18
diff --git a/sys/contrib/openzfs/Makefile.am b/sys/contrib/openzfs/Makefile.am
index 5f09d170e730..30f78e490b78 100644
--- a/sys/contrib/openzfs/Makefile.am
+++ b/sys/contrib/openzfs/Makefile.am
@@ -1,6 +1,7 @@
 CLEANFILES =
 dist_noinst_DATA =
 INSTALL_DATA_HOOKS =
+INSTALL_EXEC_HOOKS =
 ALL_LOCAL =
 CLEAN_LOCAL =
 CHECKS = shellcheck checkbashisms
@@ -71,6 +72,9 @@ all: gitrev
 PHONY += install-data-hook $(INSTALL_DATA_HOOKS)
 install-data-hook: $(INSTALL_DATA_HOOKS)
 
+PHONY += install-exec-hook $(INSTALL_EXEC_HOOKS)
+install-exec-hook: $(INSTALL_EXEC_HOOKS)
+
 PHONY += maintainer-clean-local
 maintainer-clean-local:
 	-$(RM) $(GITREV)
diff --git a/sys/contrib/openzfs/cmd/Makefile.am b/sys/contrib/openzfs/cmd/Makefile.am
index 96040976e53e..ca94f6b77e06 100644
--- a/sys/contrib/openzfs/cmd/Makefile.am
+++ b/sys/contrib/openzfs/cmd/Makefile.am
@@ -98,17 +98,16 @@ endif
 
 
 if USING_PYTHON
-bin_SCRIPTS      += arc_summary     arcstat        dbufstat        zilstat
-CLEANFILES       += arc_summary     arcstat        dbufstat        zilstat
-dist_noinst_DATA += %D%/arc_summary %D%/arcstat.in %D%/dbufstat.in %D%/zilstat.in
+bin_SCRIPTS      += zarcsummary     zarcstat        dbufstat        zilstat
+CLEANFILES       += zarcsummary     zarcstat        dbufstat        zilstat
+dist_noinst_DATA += %D%/zarcsummary %D%/zarcstat.in %D%/dbufstat.in %D%/zilstat.in
 
-$(call SUBST,arcstat,%D%/)
+$(call SUBST,zarcstat,%D%/)
 $(call SUBST,dbufstat,%D%/)
 $(call SUBST,zilstat,%D%/)
-arc_summary: %D%/arc_summary
+zarcsummary: %D%/zarcsummary
 	$(AM_V_at)cp $< $@
 endif
 
-
 PHONY += cmd
 cmd: $(bin_SCRIPTS) $(bin_PROGRAMS) $(sbin_SCRIPTS) $(sbin_PROGRAMS) $(dist_bin_SCRIPTS) $(zfsexec_PROGRAMS) $(mounthelper_PROGRAMS)
diff --git a/sys/contrib/openzfs/cmd/arcstat.in b/sys/contrib/openzfs/cmd/zarcstat.in
index 6f9abb39c3fb..8ffd20481166 100755
--- a/sys/contrib/openzfs/cmd/arcstat.in
+++ b/sys/contrib/openzfs/cmd/zarcstat.in
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: CDDL-1.0
 #
 # Print out ZFS ARC Statistics exported via kstat(1)
-# For a definition of fields, or usage, use arcstat -v
+# For a definition of fields, or usage, use zarcstat -v
 #
 # This script was originally a fork of the original arcstat.pl (0.1)
 # by Neelakanth Nadgir, originally published on his Sun blog on
@@ -56,6 +56,7 @@ import time
 import getopt
 import re
 import copy
+import os
 
 from signal import signal, SIGINT, SIGWINCH, SIG_DFL
 
@@ -171,7 +172,7 @@ cols = {
     "zactive":    [7, 1000, "zfetch prefetches active per second"],
 }
 
-# ARC structural breakdown from arc_summary
+# ARC structural breakdown from zarcsummary
 structfields = {
     "cmp":      ["compressed", "Compressed"],
     "ovh":      ["overhead", "Overhead"],
@@ -187,7 +188,7 @@ structstats = {                             # size stats
     "sz":       ["_size", "size"],
 }
 
-# ARC types breakdown from arc_summary
+# ARC types breakdown from zarcsummary
 typefields = {
     "data":     ["data", "ARC data"],
     "meta":     ["metadata", "ARC metadata"],
@@ -198,7 +199,7 @@ typestats = {                               # size stats
     "sz":       ["_size", "size"],
 }
 
-# ARC states breakdown from arc_summary
+# ARC states breakdown from zarcsummary
 statefields = {
     "ano":      ["anon", "Anonymous"],
     "mfu":      ["mfu", "MFU"],
@@ -261,7 +262,7 @@ hdr_intr = 20          # Print header every 20 lines of output
 opfile = None
 sep = "  "              # Default separator is 2 spaces
 l2exist = False
-cmd = ("Usage: arcstat [-havxp] [-f fields] [-o file] [-s string] [interval "
+cmd = ("Usage: zarcstat [-havxp] [-f fields] [-o file] [-s string] [interval "
        "[count]]\n")
 cur = {}
 d = {}
@@ -348,10 +349,10 @@ def usage():
                      "character or string\n")
     sys.stderr.write("\t -p : Disable auto-scaling of numerical fields\n")
     sys.stderr.write("\nExamples:\n")
-    sys.stderr.write("\tarcstat -o /tmp/a.log 2 10\n")
-    sys.stderr.write("\tarcstat -s \",\" -o /tmp/a.log 2 10\n")
-    sys.stderr.write("\tarcstat -v\n")
-    sys.stderr.write("\tarcstat -f time,hit%,dh%,ph%,mh% 1\n")
+    sys.stderr.write("\tzarcstat -o /tmp/a.log 2 10\n")
+    sys.stderr.write("\tzarcstat -s \",\" -o /tmp/a.log 2 10\n")
+    sys.stderr.write("\tzarcstat -v\n")
+    sys.stderr.write("\tzarcstat -f time,hit%,dh%,ph%,mh% 1\n")
     sys.stderr.write("\n")
 
     sys.exit(1)
@@ -366,7 +367,7 @@ def snap_stats():
 
     cur = kstat
 
-    # fill in additional values from arc_summary
+    # fill in additional values from zarcsummary
     cur["caches_size"] = caches_size = cur["anon_data"]+cur["anon_metadata"]+\
         cur["mfu_data"]+cur["mfu_metadata"]+cur["mru_data"]+cur["mru_metadata"]+\
         cur["uncached_data"]+cur["uncached_metadata"]
@@ -766,6 +767,7 @@ def calculate():
 
 
 def main():
+
     global sint
     global count
     global hdr_intr
diff --git a/sys/contrib/openzfs/cmd/arc_summary b/sys/contrib/openzfs/cmd/zarcsummary
index e60c6b64e8a1..24a129d9ca70 100755
--- a/sys/contrib/openzfs/cmd/arc_summary
+++ b/sys/contrib/openzfs/cmd/zarcsummary
@@ -34,7 +34,7 @@ Provides basic information on the ARC, its efficiency, the L2ARC (if present),
 the Data Management Unit (DMU), Virtual Devices (VDEVs), and tunables. See
 the in-source documentation and code at
 https://github.com/openzfs/zfs/blob/master/module/zfs/arc.c for details.
-The original introduction to arc_summary can be found at
+The original introduction to zarcsummary can be found at
 http://cuddletech.com/?p=454
 """
 
@@ -161,7 +161,7 @@ elif sys.platform.startswith('linux'):
         return get_params(TUNABLES_PATH)
 
     def get_version_impl(request):
-        # The original arc_summary called /sbin/modinfo/{spl,zfs} to get
+        # The original zarcsummary called /sbin/modinfo/{spl,zfs} to get
         # the version information. We switch to /sys/module/{spl,zfs}/version
         # to make sure we get what is really loaded in the kernel
         try:
@@ -439,7 +439,7 @@ def print_header():
     """
 
     # datetime is now recommended over time but we keep the exact formatting
-    # from the older version of arc_summary in case there are scripts
+    # from the older version of zarcsummary in case there are scripts
     # that expect it in this way
     daydate = time.strftime(DATE_FORMAT)
     spc_date = LINE_LENGTH-len(daydate)
diff --git a/sys/contrib/openzfs/cmd/zdb/zdb.c b/sys/contrib/openzfs/cmd/zdb/zdb.c
index bf44d9c322b4..2560ad045db3 100644
--- a/sys/contrib/openzfs/cmd/zdb/zdb.c
+++ b/sys/contrib/openzfs/cmd/zdb/zdb.c
@@ -106,8 +106,14 @@ extern boolean_t spa_mode_readable_spacemaps;
 extern uint_t zfs_reconstruct_indirect_combinations_max;
 extern uint_t zfs_btree_verify_intensity;
 
+enum {
+	ARG_ALLOCATED = 256,
+	ARG_BLOCK_BIN_MODE,
+	ARG_BLOCK_CLASSES,
+};
+
 static const char cmdname[] = "zdb";
-uint8_t dump_opt[256];
+uint8_t dump_opt[512];
 
 typedef void object_viewer_t(objset_t *, uint64_t, void *data, size_t size);
 
@@ -127,6 +133,21 @@ static zfs_range_tree_t *mos_refd_objs;
 static spa_t *spa;
 static objset_t *os;
 static boolean_t kernel_init_done;
+static boolean_t corruption_found = B_FALSE;
+
+static enum {
+	BIN_AUTO = 0,
+	BIN_PSIZE,
+	BIN_LSIZE,
+	BIN_ASIZE,
+} block_bin_mode = BIN_AUTO;
+
+static enum {
+	CLASS_NORMAL = 1 << 1,
+	CLASS_SPECIAL = 1 << 2,
+	CLASS_DEDUP = 1 << 3,
+	CLASS_OTHER = 1 << 4,
+} block_classes = 0;
 
 static void snprintf_blkptr_compact(char *, size_t, const blkptr_t *,
     boolean_t);
@@ -176,7 +197,7 @@ static int
 sublivelist_verify_blkptr(void *arg, const blkptr_t *bp, boolean_t free,
     dmu_tx_t *tx)
 {
-	ASSERT3P(tx, ==, NULL);
+	ASSERT0P(tx);
 	struct sublivelist_verify *sv = arg;
 	sublivelist_verify_block_refcnt_t current = {
 			.svbr_blk = *bp,
@@ -250,6 +271,7 @@ sublivelist_verify_func(void *args, dsl_deadlist_entry_t *dle)
 		    &e->svbr_blk, B_TRUE);
 		(void) printf("\tERROR: %d unmatched FREE(s): %s\n",
 		    e->svbr_refcnt, blkbuf);
+		corruption_found = B_TRUE;
 	}
 	zfs_btree_destroy(&sv->sv_pair);
 
@@ -381,7 +403,7 @@ verify_livelist_allocs(metaslab_verify_t *mv, uint64_t txg,
 	sublivelist_verify_block_t svb = {{{0}}};
 	DVA_SET_VDEV(&svb.svb_dva, mv->mv_vdid);
 	DVA_SET_OFFSET(&svb.svb_dva, offset);
-	DVA_SET_ASIZE(&svb.svb_dva, size);
+	DVA_SET_ASIZE(&svb.svb_dva, 0);
 	zfs_btree_index_t where;
 	uint64_t end_offset = offset + size;
 
@@ -405,6 +427,7 @@ verify_livelist_allocs(metaslab_verify_t *mv, uint64_t txg,
 			    (u_longlong_t)DVA_GET_ASIZE(&found->svb_dva),
 			    (u_longlong_t)found->svb_allocated_txg,
 			    (u_longlong_t)txg);
+			corruption_found = B_TRUE;
 		}
 	}
 }
@@ -426,6 +449,7 @@ metaslab_spacemap_validation_cb(space_map_entry_t *sme, void *arg)
 			    (u_longlong_t)txg, (u_longlong_t)offset,
 			    (u_longlong_t)size, (u_longlong_t)mv->mv_vdid,
 			    (u_longlong_t)mv->mv_msid);
+			corruption_found = B_TRUE;
 		} else {
 			zfs_range_tree_add(mv->mv_allocated,
 			    offset, size);
@@ -439,6 +463,7 @@ metaslab_spacemap_validation_cb(space_map_entry_t *sme, void *arg)
 			    (u_longlong_t)txg, (u_longlong_t)offset,
 			    (u_longlong_t)size, (u_longlong_t)mv->mv_vdid,
 			    (u_longlong_t)mv->mv_msid);
+			corruption_found = B_TRUE;
 		} else {
 			zfs_range_tree_remove(mv->mv_allocated,
 			    offset, size);
@@ -526,6 +551,7 @@ mv_populate_livelist_allocs(metaslab_verify_t *mv, sublivelist_verify_t *sv)
 			    (u_longlong_t)DVA_GET_VDEV(&svb->svb_dva),
 			    (u_longlong_t)DVA_GET_OFFSET(&svb->svb_dva),
 			    (u_longlong_t)DVA_GET_ASIZE(&svb->svb_dva));
+			corruption_found = B_TRUE;
 			continue;
 		}
 
@@ -542,6 +568,7 @@ mv_populate_livelist_allocs(metaslab_verify_t *mv, sublivelist_verify_t *sv)
 			    (u_longlong_t)DVA_GET_VDEV(&svb->svb_dva),
 			    (u_longlong_t)DVA_GET_OFFSET(&svb->svb_dva),
 			    (u_longlong_t)DVA_GET_ASIZE(&svb->svb_dva));
+			corruption_found = B_TRUE;
 			continue;
 		}
 
@@ -655,6 +682,7 @@ livelist_metaslab_validate(spa_t *spa)
 	}
 	(void) printf("ERROR: Found livelist blocks marked as allocated "
 	    "for indirect vdevs:\n");
+	corruption_found = B_TRUE;
 
 	zfs_btree_index_t *where = NULL;
 	sublivelist_verify_block_t *svb;
@@ -739,6 +767,12 @@ usage(void)
 	(void) fprintf(stderr, "    Options to control amount of output:\n");
 	(void) fprintf(stderr, "        -b --block-stats             "
 	    "block statistics\n");
+	(void) fprintf(stderr, "           --bin=(lsize|psize|asize) "
+	    "bin blocks based on this size in all three columns\n");
+	(void) fprintf(stderr,
+	    "           --class=(normal|special|dedup|other)[,...]\n"
+	    "                                     only consider blocks from "
+	    "these allocation classes\n");
 	(void) fprintf(stderr, "        -B --backup                  "
 	    "backup stream\n");
 	(void) fprintf(stderr, "        -c --checksum                "
@@ -827,7 +861,7 @@ usage(void)
 	(void) fprintf(stderr, "Specify an option more than once (e.g. -bb) "
 	    "to make only that option verbose\n");
 	(void) fprintf(stderr, "Default is to dump everything non-verbosely\n");
-	zdb_exit(1);
+	zdb_exit(2);
 }
 
 static void
@@ -892,9 +926,9 @@ dump_packed_nvlist(objset_t *os, uint64_t object, void *data, size_t size)
 	size_t nvsize = *(uint64_t *)data;
 	char *packed = umem_alloc(nvsize, UMEM_NOFAIL);
 
-	VERIFY(0 == dmu_read(os, object, 0, nvsize, packed, DMU_READ_PREFETCH));
+	VERIFY0(dmu_read(os, object, 0, nvsize, packed, DMU_READ_PREFETCH));
 
-	VERIFY(nvlist_unpack(packed, nvsize, &nv, 0) == 0);
+	VERIFY0(nvlist_unpack(packed, nvsize, &nv, 0));
 
 	umem_free(packed, nvsize);
 
@@ -1455,8 +1489,8 @@ get_obsolete_refcount(vdev_t *vd)
 			refcount++;
 		}
 	} else {
-		ASSERT3P(vd->vdev_obsolete_sm, ==, NULL);
-		ASSERT3U(obsolete_sm_object, ==, 0);
+		ASSERT0P(vd->vdev_obsolete_sm);
+		ASSERT0(obsolete_sm_object);
 	}
 	for (unsigned c = 0; c < vd->vdev_children; c++) {
 		refcount += get_obsolete_refcount(vd->vdev_child[c]);
@@ -1578,9 +1612,8 @@ dump_spacemap(objset_t *os, space_map_t *sm)
 			continue;
 		}
 
-		uint8_t words;
 		char entry_type;
-		uint64_t entry_off, entry_run, entry_vdev = SM_NO_VDEVID;
+		uint64_t entry_off, entry_run, entry_vdev;
 
 		if (sm_entry_is_single_word(word)) {
 			entry_type = (SM_TYPE_DECODE(word) == SM_ALLOC) ?
@@ -1588,35 +1621,43 @@ dump_spacemap(objset_t *os, space_map_t *sm)
 			entry_off = (SM_OFFSET_DECODE(word) << mapshift) +
 			    sm->sm_start;
 			entry_run = SM_RUN_DECODE(word) << mapshift;
-			words = 1;
+
+			(void) printf("\t    [%6llu] %c "
+			    "range: %012llx-%012llx size: %08llx\n",
+			    (u_longlong_t)entry_id, entry_type,
+			    (u_longlong_t)entry_off,
+			    (u_longlong_t)(entry_off + entry_run - 1),
+			    (u_longlong_t)entry_run);
 		} else {
 			/* it is a two-word entry so we read another word */
 			ASSERT(sm_entry_is_double_word(word));
 
 			uint64_t extra_word;
 			offset += sizeof (extra_word);
+			ASSERT3U(offset, <, space_map_length(sm));
 			VERIFY0(dmu_read(os, space_map_object(sm), offset,
 			    sizeof (extra_word), &extra_word,
 			    DMU_READ_PREFETCH));
 
-			ASSERT3U(offset, <=, space_map_length(sm));
-
 			entry_run = SM2_RUN_DECODE(word) << mapshift;
 			entry_vdev = SM2_VDEV_DECODE(word);
 			entry_type = (SM2_TYPE_DECODE(extra_word) == SM_ALLOC) ?
 			    'A' : 'F';
 			entry_off = (SM2_OFFSET_DECODE(extra_word) <<
 			    mapshift) + sm->sm_start;
-			words = 2;
-		}
 
-		(void) printf("\t    [%6llu]    %c  range:"
-		    " %010llx-%010llx  size: %06llx vdev: %06llu words: %u\n",
-		    (u_longlong_t)entry_id,
-		    entry_type, (u_longlong_t)entry_off,
-		    (u_longlong_t)(entry_off + entry_run),
-		    (u_longlong_t)entry_run,
-		    (u_longlong_t)entry_vdev, words);
+			if (zopt_metaslab_args == 0 ||
+			    zopt_metaslab[0] == entry_vdev) {
+				(void) printf("\t    [%6llu] %c "
+				    "range: %012llx-%012llx size: %08llx "
+				    "vdev: %llu\n",
+				    (u_longlong_t)entry_id, entry_type,
+				    (u_longlong_t)entry_off,
+				    (u_longlong_t)(entry_off + entry_run - 1),
+				    (u_longlong_t)entry_run,
+				    (u_longlong_t)entry_vdev);
+			}
+		}
 
 		if (entry_type == 'A')
 			alloc += entry_run;
@@ -1652,6 +1693,16 @@ dump_metaslab_stats(metaslab_t *msp)
 }
 
 static void
+dump_allocated(void *arg, uint64_t start, uint64_t size)
+{
+	uint64_t *off = arg;
+	if (*off != start)
+		(void) printf("ALLOC: %"PRIu64" %"PRIu64"\n", *off,
+		    start - *off);
+	*off = start + size;
+}
+
+static void
 dump_metaslab(metaslab_t *msp)
 {
 	vdev_t *vd = msp->ms_group->mg_vd;
@@ -1667,13 +1718,24 @@ dump_metaslab(metaslab_t *msp)
 	    (u_longlong_t)msp->ms_id, (u_longlong_t)msp->ms_start,
 	    (u_longlong_t)space_map_object(sm), freebuf);
 
-	if (dump_opt['m'] > 2 && !dump_opt['L']) {
+	if (dump_opt[ARG_ALLOCATED] ||
+	    (dump_opt['m'] > 2 && !dump_opt['L'])) {
 		mutex_enter(&msp->ms_lock);
 		VERIFY0(metaslab_load(msp));
+	}
+
+	if (dump_opt['m'] > 2 && !dump_opt['L']) {
 		zfs_range_tree_stat_verify(msp->ms_allocatable);
 		dump_metaslab_stats(msp);
-		metaslab_unload(msp);
-		mutex_exit(&msp->ms_lock);
+	}
+
+	if (dump_opt[ARG_ALLOCATED]) {
+		uint64_t off = msp->ms_start;
+		zfs_range_tree_walk(msp->ms_allocatable, dump_allocated,
+		    &off);
+		if (off != msp->ms_start + msp->ms_size)
+			(void) printf("ALLOC: %"PRIu64" %"PRIu64"\n", off,
+			    msp->ms_size - off);
 	}
 
 	if (dump_opt['m'] > 1 && sm != NULL &&
@@ -1688,6 +1750,12 @@ dump_metaslab(metaslab_t *msp)
 		    SPACE_MAP_HISTOGRAM_SIZE, sm->sm_shift);
 	}
 
+	if (dump_opt[ARG_ALLOCATED] ||
+	    (dump_opt['m'] > 2 && !dump_opt['L'])) {
+		metaslab_unload(msp);
+		mutex_exit(&msp->ms_lock);
+	}
+
 	if (vd->vdev_ops == &vdev_draid_ops)
 		ASSERT3U(msp->ms_size, <=, 1ULL << vd->vdev_ms_shift);
 	else
@@ -1724,8 +1792,9 @@ print_vdev_metaslab_header(vdev_t *vd)
 		}
 	}
 
-	(void) printf("\tvdev %10llu   %s",
-	    (u_longlong_t)vd->vdev_id, bias_str);
+	(void) printf("\tvdev %10llu\t%s  metaslab shift %4llu",
+	    (u_longlong_t)vd->vdev_id, bias_str,
+	    (u_longlong_t)vd->vdev_ms_shift);
 
 	if (ms_flush_data_obj != 0) {
 		(void) printf("   ms_unflushed_phys object %llu",
@@ -1792,7 +1861,7 @@ print_vdev_indirect(vdev_t *vd)
 	vdev_indirect_births_t *vib = vd->vdev_indirect_births;
 
 	if (vim == NULL) {
-		ASSERT3P(vib, ==, NULL);
+		ASSERT0P(vib);
 		return;
 	}
 
@@ -1865,7 +1934,7 @@ dump_metaslabs(spa_t *spa)
 
 	(void) printf("\nMetaslabs:\n");
 
-	if (!dump_opt['d'] && zopt_metaslab_args > 0) {
+	if (zopt_metaslab_args > 0) {
 		c = zopt_metaslab[0];
 
 		if (c >= children)
@@ -2043,10 +2112,10 @@ dump_ddt_object(ddt_t *ddt, ddt_type_t type, ddt_class_t class)
 
 	if (error == ENOENT)
 		return;
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	error = ddt_object_count(ddt, type, class, &count);
-	ASSERT(error == 0);
+	ASSERT0(error);
 	if (count == 0)
 		return;
 
@@ -2583,19 +2652,17 @@ snprintf_blkptr_compact(char *blkbuf, size_t buflen, const blkptr_t *bp,
 	}
 }
 
-static void
+static u_longlong_t
 print_indirect(spa_t *spa, blkptr_t *bp, const zbookmark_phys_t *zb,
     const dnode_phys_t *dnp)
 {
 	char blkbuf[BP_SPRINTF_LEN];
+	u_longlong_t offset;
 	int l;
 
-	if (!BP_IS_EMBEDDED(bp)) {
-		ASSERT3U(BP_GET_TYPE(bp), ==, dnp->dn_type);
-		ASSERT3U(BP_GET_LEVEL(bp), ==, zb->zb_level);
-	}
+	offset = (u_longlong_t)blkid2offset(dnp, bp, zb);
 
-	(void) printf("%16llx ", (u_longlong_t)blkid2offset(dnp, bp, zb));
+	(void) printf("%16llx ", offset);
 
 	ASSERT(zb->zb_level >= 0);
 
@@ -2610,19 +2677,38 @@ print_indirect(spa_t *spa, blkptr_t *bp, const zbookmark_phys_t *zb,
 	snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp, B_FALSE);
 	if (dump_opt['Z'] && BP_GET_COMPRESS(bp) == ZIO_COMPRESS_ZSTD)
 		snprintf_zstd_header(spa, blkbuf, sizeof (blkbuf), bp);
-	(void) printf("%s\n", blkbuf);
+	(void) printf("%s", blkbuf);
+
+	if (!BP_IS_EMBEDDED(bp)) {
+		if (BP_GET_TYPE(bp) != dnp->dn_type) {
+			(void) printf(" (ERROR: Block pointer type "
+			    "(%llu) does not match dnode type (%hhu))",
+			    BP_GET_TYPE(bp), dnp->dn_type);
+			corruption_found = B_TRUE;
+		}
+		if (BP_GET_LEVEL(bp) != zb->zb_level) {
+			(void) printf(" (ERROR: Block pointer level "
+			    "(%llu) does not match bookmark level (%lld))",
+			    BP_GET_LEVEL(bp), (longlong_t)zb->zb_level);
+			corruption_found = B_TRUE;
+		}
+	}
+	(void) printf("\n");
+
+	return (offset);
 }
 
 static int
 visit_indirect(spa_t *spa, const dnode_phys_t *dnp,
     blkptr_t *bp, const zbookmark_phys_t *zb)
 {
+	u_longlong_t offset;
 	int err = 0;
 
 	if (BP_GET_BIRTH(bp) == 0)
 		return (0);
 
-	print_indirect(spa, bp, zb, dnp);
+	offset = print_indirect(spa, bp, zb, dnp);
 
 	if (BP_GET_LEVEL(bp) > 0 && !BP_IS_HOLE(bp)) {
 		arc_flags_t flags = ARC_FLAG_WAIT;
@@ -2652,8 +2738,15 @@ visit_indirect(spa_t *spa, const dnode_phys_t *dnp,
 				break;
 			fill += BP_GET_FILL(cbp);
 		}
-		if (!err)
-			ASSERT3U(fill, ==, BP_GET_FILL(bp));
+		if (!err) {
+			if (fill != BP_GET_FILL(bp)) {
+				(void) printf("%16llx: Block pointer "
+				    "fill (%llu) does not match calculated "
+				    "value (%llu)\n", offset, BP_GET_FILL(bp),
+				    (u_longlong_t)fill);
+				corruption_found = B_TRUE;
+			}
+		}
 		arc_buf_destroy(buf, &buf);
 	}
 
@@ -2909,6 +3002,7 @@ dump_full_bpobj(bpobj_t *bpo, const char *name, int indent)
 				(void) printf("ERROR %u while trying to open "
 				    "subobj id %llu\n",
 				    error, (u_longlong_t)subobj);
+				corruption_found = B_TRUE;
 				continue;
 			}
 			dump_full_bpobj(&subbpo, "subobj", indent + 1);
@@ -3088,6 +3182,7 @@ bpobj_count_refd(bpobj_t *bpo)
 				(void) printf("ERROR %u while trying to open "
 				    "subobj id %llu\n",
 				    error, (u_longlong_t)subobj);
+				corruption_found = B_TRUE;
 				continue;
 			}
 			bpobj_count_refd(&subbpo);
@@ -3109,7 +3204,7 @@ dsl_deadlist_entry_count_refd(void *arg, dsl_deadlist_entry_t *dle)
 static int
 dsl_deadlist_entry_dump(void *arg, dsl_deadlist_entry_t *dle)
 {
-	ASSERT(arg == NULL);
+	ASSERT0P(arg);
 	if (dump_opt['d'] >= 5) {
 		char buf[128];
 		(void) snprintf(buf, sizeof (buf),
@@ -3230,6 +3325,7 @@ zdb_derive_key(dsl_dir_t *dd, uint8_t *key_out)
 	uint64_t keyformat, salt, iters;
 	int i;
 	unsigned char c;
+	FILE *f;
 
 	VERIFY0(zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj,
 	    zfs_prop_to_name(ZFS_PROP_KEYFORMAT), sizeof (uint64_t),
@@ -3262,6 +3358,25 @@ zdb_derive_key(dsl_dir_t *dd, uint8_t *key_out)
 
 		break;
 
+	case ZFS_KEYFORMAT_RAW:
+		if ((f = fopen(key_material, "r")) == NULL)
+			return (B_FALSE);
+
+		if (fread(key_out, 1, WRAPPING_KEY_LEN, f) !=
+		    WRAPPING_KEY_LEN) {
+			(void) fclose(f);
+			return (B_FALSE);
+		}
+
+		/* Check the key length */
+		if (fgetc(f) != EOF) {
+			(void) fclose(f);
+			return (B_FALSE);
+		}
+
+		(void) fclose(f);
+		break;
+
 	default:
 		fatal("no support for key format %u\n",
 		    (unsigned int) keyformat);
@@ -3347,7 +3462,7 @@ open_objset(const char *path, const void *tag, objset_t **osp)
 	uint64_t sa_attrs = 0;
 	uint64_t version = 0;
 
-	VERIFY3P(sa_os, ==, NULL);
+	VERIFY0P(sa_os);
 
 	/*
 	 * We can't own an objset if it's redacted.  Therefore, we do this
@@ -3520,8 +3635,8 @@ dump_uidgid(objset_t *os, uint64_t uid, uint64_t gid)
 		uint64_t fuid_obj;
 
 		/* first find the fuid object.  It lives in the master node */
-		VERIFY(zap_lookup(os, MASTER_NODE_OBJ, ZFS_FUID_TABLES,
-		    8, 1, &fuid_obj) == 0);
+		VERIFY0(zap_lookup(os, MASTER_NODE_OBJ, ZFS_FUID_TABLES,
+		    8, 1, &fuid_obj));
 		zfs_fuid_avl_tree_create(&idx_tree, &domain_tree);
 		(void) zfs_fuid_table_load(os, fuid_obj,
 		    &idx_tree, &domain_tree);
@@ -5723,6 +5838,34 @@ dump_size_histograms(zdb_cb_t *zcb)
 
 
 	(void) printf("\nBlock Size Histogram\n");
+	switch (block_bin_mode) {
+	case BIN_PSIZE:
+		printf("(note: all categories are binned by %s)\n", "psize");
+		break;
+	case BIN_LSIZE:
+		printf("(note: all categories are binned by %s)\n", "lsize");
+		break;
+	case BIN_ASIZE:
+		printf("(note: all categories are binned by %s)\n", "asize");
+		break;
+	default:
+		printf("(note: all categories are binned separately)\n");
+		break;
+	}
+	if (block_classes != 0) {
+		char buf[256] = "";
+		if (block_classes & CLASS_NORMAL)
+			strlcat(buf, "\"normal\", ", sizeof (buf));
+		if (block_classes & CLASS_SPECIAL)
+			strlcat(buf, "\"special\", ", sizeof (buf));
+		if (block_classes & CLASS_DEDUP)
+			strlcat(buf, "\"dedup\", ", sizeof (buf));
+		if (block_classes & CLASS_OTHER)
+			strlcat(buf, "\"other\", ", sizeof (buf));
+		buf[strlen(buf)-2] = '\0';
+		printf("(note: only blocks in these classes are counted: %s)\n",
+		    buf);
+	}
 	/*
 	 * Print the first line titles
 	 */
@@ -6071,29 +6214,85 @@ skipped:
 		    [BPE_GET_PSIZE(bp)]++;
 		return;
 	}
+
+	if (block_classes != 0) {
+		spa_config_enter(zcb->zcb_spa, SCL_CONFIG, FTAG, RW_READER);
+
+		uint64_t vdev = DVA_GET_VDEV(&bp->blk_dva[0]);
+		uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[0]);
+		vdev_t *vd = vdev_lookup_top(zcb->zcb_spa, vdev);
+		ASSERT(vd != NULL);
+		metaslab_t *ms = vd->vdev_ms[offset >> vd->vdev_ms_shift];
+		ASSERT(ms != NULL);
+		metaslab_group_t *mg = ms->ms_group;
+		ASSERT(mg != NULL);
+		metaslab_class_t *mc = mg->mg_class;
+		ASSERT(mc != NULL);
+
+		spa_config_exit(zcb->zcb_spa, SCL_CONFIG, FTAG);
+
+		int class;
+		if (mc == spa_normal_class(zcb->zcb_spa)) {
+			class = CLASS_NORMAL;
+		} else if (mc == spa_special_class(zcb->zcb_spa)) {
+			class = CLASS_SPECIAL;
+		} else if (mc == spa_dedup_class(zcb->zcb_spa)) {
+			class = CLASS_DEDUP;
+		} else {
+			class = CLASS_OTHER;
+		}
+
+		if (!(block_classes & class)) {
+			goto hist_skipped;
+		}
+	}
+
 	/*
 	 * The binning histogram bins by powers of two up to
 	 * SPA_MAXBLOCKSIZE rather than creating bins for
 	 * every possible blocksize found in the pool.
 	 */
-	int bin = highbit64(BP_GET_PSIZE(bp)) - 1;
+	int bin;
+
+	/*
+	 * Binning strategy: each bin includes blocks up to and including
+	 * the given size (excluding blocks that fit into the previous bin).
+	 * This way, the "4K" bin includes blocks within the (2K; 4K] range.
+	 */
+#define	BIN(size) (highbit64((size) - 1))
+
+	switch (block_bin_mode) {
+	case BIN_PSIZE: bin = BIN(BP_GET_PSIZE(bp)); break;
+	case BIN_LSIZE: bin = BIN(BP_GET_LSIZE(bp)); break;
+	case BIN_ASIZE: bin = BIN(BP_GET_ASIZE(bp)); break;
+	case BIN_AUTO: break;
+	default: PANIC("bad block_bin_mode"); abort();
+	}
+
+	if (block_bin_mode == BIN_AUTO)
+		bin = BIN(BP_GET_PSIZE(bp));
 
 	zcb->zcb_psize_count[bin]++;
 	zcb->zcb_psize_len[bin] += BP_GET_PSIZE(bp);
 	zcb->zcb_psize_total += BP_GET_PSIZE(bp);
 
-	bin = highbit64(BP_GET_LSIZE(bp)) - 1;
+	if (block_bin_mode == BIN_AUTO)
+		bin = BIN(BP_GET_LSIZE(bp));
 
 	zcb->zcb_lsize_count[bin]++;
 	zcb->zcb_lsize_len[bin] += BP_GET_LSIZE(bp);
 	zcb->zcb_lsize_total += BP_GET_LSIZE(bp);
 
-	bin = highbit64(BP_GET_ASIZE(bp)) - 1;
+	if (block_bin_mode == BIN_AUTO)
+		bin = BIN(BP_GET_ASIZE(bp));
 
 	zcb->zcb_asize_count[bin]++;
 	zcb->zcb_asize_len[bin] += BP_GET_ASIZE(bp);
 	zcb->zcb_asize_total += BP_GET_ASIZE(bp);
 
+#undef BIN
+
+hist_skipped:
 	if (!do_claim)
 		return;
 
@@ -7016,7 +7215,7 @@ deleted_livelists_count_blocks(spa_t *spa, zdb_cb_t *zbc)
 static void
 dump_livelist_cb(dsl_deadlist_t *ll, void *arg)
 {
-	ASSERT3P(arg, ==, NULL);
+	ASSERT0P(arg);
 	global_feature_count[SPA_FEATURE_LIVELIST]++;
 	dump_blkptr_list(ll, "Deleted Livelist");
 	dsl_deadlist_iterate(ll, sublivelist_verify_lightweight, NULL);
@@ -7913,7 +8112,7 @@ verify_checkpoint_vdev_spacemaps(spa_t *checkpoint, spa_t *current)
 		for (uint64_t c = ckpoint_rvd->vdev_children;
 		    c < current_rvd->vdev_children; c++) {
 			vdev_t *current_vd = current_rvd->vdev_child[c];
-			VERIFY3P(current_vd->vdev_checkpoint_sm, ==, NULL);
+			VERIFY0P(current_vd->vdev_checkpoint_sm);
 		}
 	}
 
@@ -9334,6 +9533,12 @@ main(int argc, char **argv)
 		{"all-reconstruction",	no_argument,		NULL, 'Y'},
 		{"livelist",		no_argument,		NULL, 'y'},
 		{"zstd-headers",	no_argument,		NULL, 'Z'},
+		{"allocated-map",	no_argument,		NULL,
+		    ARG_ALLOCATED},
+		{"bin",			required_argument,	NULL,
+		    ARG_BLOCK_BIN_MODE},
+		{"class",		required_argument,	NULL,
+		    ARG_BLOCK_CLASSES},
 		{0, 0, 0, 0}
 	};
 
@@ -9364,6 +9569,7 @@ main(int argc, char **argv)
 		case 'u':
 		case 'y':
 		case 'Z':
+		case ARG_ALLOCATED:
 			dump_opt[c]++;
 			dump_all = 0;
 			break;
@@ -9446,6 +9652,59 @@ main(int argc, char **argv)
 		case 'x':
 			vn_dumpdir = optarg;
 			break;
+		case ARG_BLOCK_BIN_MODE:
+			if (strcmp(optarg, "lsize") == 0) {
+				block_bin_mode = BIN_LSIZE;
+			} else if (strcmp(optarg, "psize") == 0) {
+				block_bin_mode = BIN_PSIZE;
+			} else if (strcmp(optarg, "asize") == 0) {
+				block_bin_mode = BIN_ASIZE;
+			} else {
+				(void) fprintf(stderr,
+				    "--bin=\"%s\" must be one of \"lsize\", "
+				    "\"psize\" or \"asize\"\n", optarg);
+				usage();
+			}
+			break;
+
+		case ARG_BLOCK_CLASSES: {
+			char *buf = strdup(optarg), *tok = buf, *next,
+			    *save = NULL;
+
+			while ((next = strtok_r(tok, ",", &save)) != NULL) {
+				tok = NULL;
+
+				if (strcmp(next, "normal") == 0) {
+					block_classes |= CLASS_NORMAL;
+				} else if (strcmp(next, "special") == 0) {
+					block_classes |= CLASS_SPECIAL;
+				} else if (strcmp(next, "dedup") == 0) {
+					block_classes |= CLASS_DEDUP;
+				} else if (strcmp(next, "other") == 0) {
+					block_classes |= CLASS_OTHER;
+				} else {
+					(void) fprintf(stderr,
+					    "--class=\"%s\" must be a "
+					    "comma-separated list of either "
+					    "\"normal\", \"special\", "
+					    "\"asize\" or \"other\"; "
+					    "got \"%s\"\n",
+					    optarg, next);
+					usage();
+				}
+			}
+
+			if (block_classes == 0) {
+				(void) fprintf(stderr,
+				    "--class= must be a comma-separated "
+				    "list of either \"normal\", \"special\", "
+				    "\"asize\" or \"other\"; got empty\n");
+				usage();
+			}
+
+			free(buf);
+			break;
+		}
 		default:
 			usage();
 			break;
@@ -9488,6 +9747,9 @@ main(int argc, char **argv)
 	 */
 	spa_mode_readable_spacemaps = B_TRUE;
 
+	libspl_set_assert_ok((dump_opt['A'] == 1) || (dump_opt['A'] > 2));
+	zfs_recover = (dump_opt['A'] > 1);
+
 	if (dump_all)
 		verbose = MAX(verbose, 1);
 
@@ -9498,9 +9760,6 @@ main(int argc, char **argv)
 			dump_opt[c] += verbose;
 	}
 
-	libspl_set_assert_ok((dump_opt['A'] == 1) || (dump_opt['A'] > 2));
-	zfs_recover = (dump_opt['A'] > 1);
-
 	argc -= optind;
 	argv += optind;
 	if (argc < 2 && dump_opt['R'])
@@ -9634,7 +9893,7 @@ main(int argc, char **argv)
 		} else if (objset_str && !zdb_numeric(objset_str + 1) &&
 		    dump_opt['N']) {
 			printf("Supply a numeric objset ID with -N\n");
-			error = 1;
+			error = 2;
 			goto fini;
 		}
 	} else {
@@ -9743,7 +10002,7 @@ main(int argc, char **argv)
 	if (error == 0) {
 		if (dump_opt['k'] && (target_is_spa || dump_opt['R'])) {
 			ASSERT(checkpoint_pool != NULL);
-			ASSERT(checkpoint_target == NULL);
+			ASSERT0P(checkpoint_target);
 
 			error = spa_open(checkpoint_pool, &spa, FTAG);
 			if (error != 0) {
@@ -9936,5 +10195,8 @@ fini:
 	if (kernel_init_done)
 		kernel_fini();
 
+	if (corruption_found && error == 0)
+		error = 3;
+
 	return (error);
 }
diff --git a/sys/contrib/openzfs/cmd/zdb/zdb.h b/sys/contrib/openzfs/cmd/zdb/zdb.h
index 6b6c9169816b..48b561eb202c 100644
--- a/sys/contrib/openzfs/cmd/zdb/zdb.h
+++ b/sys/contrib/openzfs/cmd/zdb/zdb.h
@@ -29,6 +29,6 @@
 #define	_ZDB_H
 
 void dump_intent_log(zilog_t *);
-extern uint8_t dump_opt[256];
+extern uint8_t dump_opt[512];
 
 #endif	/* _ZDB_H */
diff --git a/sys/contrib/openzfs/cmd/zdb/zdb_il.c b/sys/contrib/openzfs/cmd/zdb/zdb_il.c
index 62e290cd122c..3d91fb28a4c7 100644
--- a/sys/contrib/openzfs/cmd/zdb/zdb_il.c
+++ b/sys/contrib/openzfs/cmd/zdb/zdb_il.c
@@ -48,8 +48,6 @@
 
 #include "zdb.h"
 
-extern uint8_t dump_opt[256];
-
 static char tab_prefix[4] = "\t\t\t";
 
 static void
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/Makefile.am b/sys/contrib/openzfs/cmd/zed/zed.d/Makefile.am
index 093a04c4636a..c0b161ecf248 100644
--- a/sys/contrib/openzfs/cmd/zed/zed.d/Makefile.am
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/Makefile.am
@@ -9,18 +9,18 @@ dist_zedexec_SCRIPTS = \
 	%D%/all-debug.sh \
 	%D%/all-syslog.sh \
 	%D%/data-notify.sh \
-	%D%/deadman-slot_off.sh \
+	%D%/deadman-sync-slot_off.sh \
 	%D%/generic-notify.sh \
-	%D%/pool_import-led.sh \
+	%D%/pool_import-sync-led.sh \
 	%D%/resilver_finish-notify.sh \
 	%D%/resilver_finish-start-scrub.sh \
 	%D%/scrub_finish-notify.sh \
-	%D%/statechange-led.sh \
+	%D%/statechange-sync-led.sh \
 	%D%/statechange-notify.sh \
-	%D%/statechange-slot_off.sh \
+	%D%/statechange-sync-slot_off.sh \
 	%D%/trim_finish-notify.sh \
-	%D%/vdev_attach-led.sh \
-	%D%/vdev_clear-led.sh
+	%D%/vdev_attach-sync-led.sh \
+	%D%/vdev_clear-sync-led.sh
 
 nodist_zedexec_SCRIPTS = \
 	%D%/history_event-zfs-list-cacher.sh
@@ -30,17 +30,17 @@ SUBSTFILES += $(nodist_zedexec_SCRIPTS)
 zedconfdefaults = \
 	all-syslog.sh \
 	data-notify.sh \
-	deadman-slot_off.sh \
+	deadman-sync-slot_off.sh \
 	history_event-zfs-list-cacher.sh \
-	pool_import-led.sh \
+	pool_import-sync-led.sh \
 	resilver_finish-notify.sh \
 	resilver_finish-start-scrub.sh \
 	scrub_finish-notify.sh \
-	statechange-led.sh \
+	statechange-sync-led.sh \
 	statechange-notify.sh \
-	statechange-slot_off.sh \
-	vdev_attach-led.sh \
-	vdev_clear-led.sh
+	statechange-sync-slot_off.sh \
+	vdev_attach-sync-led.sh \
+	vdev_clear-sync-led.sh
 
 dist_noinst_DATA += %D%/README
 
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/deadman-slot_off.sh b/sys/contrib/openzfs/cmd/zed/zed.d/deadman-sync-slot_off.sh
index 7b339b3add01..7b339b3add01 100755
--- a/sys/contrib/openzfs/cmd/zed/zed.d/deadman-slot_off.sh
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/deadman-sync-slot_off.sh
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/pool_import-led.sh b/sys/contrib/openzfs/cmd/zed/zed.d/pool_import-led.sh
deleted file mode 120000
index 7d7404398a4a..000000000000
--- a/sys/contrib/openzfs/cmd/zed/zed.d/pool_import-led.sh
+++ /dev/null
@@ -1 +0,0 @@
-statechange-led.sh
\ No newline at end of file
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/pool_import-sync-led.sh b/sys/contrib/openzfs/cmd/zed/zed.d/pool_import-sync-led.sh
new file mode 120000
index 000000000000..8b9c10c11ebb
--- /dev/null
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/pool_import-sync-led.sh
@@ -0,0 +1 @@
+statechange-sync-led.sh
\ No newline at end of file
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/statechange-led.sh b/sys/contrib/openzfs/cmd/zed/zed.d/statechange-sync-led.sh
index 40cb61f17307..40cb61f17307 100755
--- a/sys/contrib/openzfs/cmd/zed/zed.d/statechange-led.sh
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/statechange-sync-led.sh
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/statechange-slot_off.sh b/sys/contrib/openzfs/cmd/zed/zed.d/statechange-sync-slot_off.sh
index 06acce93b8aa..06acce93b8aa 100755
--- a/sys/contrib/openzfs/cmd/zed/zed.d/statechange-slot_off.sh
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/statechange-sync-slot_off.sh
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/vdev_attach-led.sh b/sys/contrib/openzfs/cmd/zed/zed.d/vdev_attach-led.sh
deleted file mode 120000
index 7d7404398a4a..000000000000
--- a/sys/contrib/openzfs/cmd/zed/zed.d/vdev_attach-led.sh
+++ /dev/null
@@ -1 +0,0 @@
-statechange-led.sh
\ No newline at end of file
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/vdev_attach-sync-led.sh b/sys/contrib/openzfs/cmd/zed/zed.d/vdev_attach-sync-led.sh
new file mode 120000
index 000000000000..8b9c10c11ebb
--- /dev/null
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/vdev_attach-sync-led.sh
@@ -0,0 +1 @@
+statechange-sync-led.sh
\ No newline at end of file
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/vdev_clear-led.sh b/sys/contrib/openzfs/cmd/zed/zed.d/vdev_clear-led.sh
deleted file mode 120000
index 7d7404398a4a..000000000000
--- a/sys/contrib/openzfs/cmd/zed/zed.d/vdev_clear-led.sh
+++ /dev/null
@@ -1 +0,0 @@
-statechange-led.sh
\ No newline at end of file
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/vdev_clear-sync-led.sh b/sys/contrib/openzfs/cmd/zed/zed.d/vdev_clear-sync-led.sh
new file mode 120000
index 000000000000..8b9c10c11ebb
--- /dev/null
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/vdev_clear-sync-led.sh
@@ -0,0 +1 @@
+statechange-sync-led.sh
\ No newline at end of file
diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/zed-functions.sh b/sys/contrib/openzfs/cmd/zed/zed.d/zed-functions.sh
index 6e00f153be1c..78d8f658ddd8 100644
--- a/sys/contrib/openzfs/cmd/zed/zed.d/zed-functions.sh
+++ b/sys/contrib/openzfs/cmd/zed/zed.d/zed-functions.sh
@@ -441,8 +441,9 @@ zed_notify_slack_webhook()
         "${pathname}")"
 
     # Construct the JSON message for posting.
+    # shellcheck disable=SC2016
     #
-    msg_json="$(printf '{"text": "*%s*\\n%s"}' "${subject}" "${msg_body}" )"
+    msg_json="$(printf '{"text": "*%s*\\n```%s```"}' "${subject}" "${msg_body}" )"
 
     # Send the POST request and check for errors.
     #
diff --git a/sys/contrib/openzfs/cmd/zed/zed_exec.c b/sys/contrib/openzfs/cmd/zed/zed_exec.c
index 036081decd64..a14af4f20a85 100644
--- a/sys/contrib/openzfs/cmd/zed/zed_exec.c
+++ b/sys/contrib/openzfs/cmd/zed/zed_exec.c
@@ -196,37 +196,29 @@ _nop(int sig)
 	(void) sig;
 }
 
-static void *
-_reap_children(void *arg)
+static void
+wait_for_children(boolean_t do_pause, boolean_t wait)
 {
-	(void) arg;
-	struct launched_process_node node, *pnode;
 	pid_t pid;
-	int status;
 	struct rusage usage;
-	struct sigaction sa = {};
-
-	(void) sigfillset(&sa.sa_mask);
-	(void) sigdelset(&sa.sa_mask, SIGCHLD);
-	(void) pthread_sigmask(SIG_SETMASK, &sa.sa_mask, NULL);
-
-	(void) sigemptyset(&sa.sa_mask);
-	sa.sa_handler = _nop;
-	sa.sa_flags = SA_NOCLDSTOP;
-	(void) sigaction(SIGCHLD, &sa, NULL);
+	int status;
+	struct launched_process_node node, *pnode;
 
 	for (_reap_children_stop = B_FALSE; !_reap_children_stop; ) {
 		(void) pthread_mutex_lock(&_launched_processes_lock);
-		pid = wait4(0, &status, WNOHANG, &usage);
-
+		pid = wait4(0, &status, wait ? 0 : WNOHANG, &usage);
 		if (pid == 0 || pid == (pid_t)-1) {
 			(void) pthread_mutex_unlock(&_launched_processes_lock);
-			if (pid == 0 || errno == ECHILD)
-				pause();
-			else if (errno != EINTR)
+			if ((pid == 0) || (errno == ECHILD)) {
+				if (do_pause)
+					pause();
+			} else if (errno != EINTR)
 				zed_log_msg(LOG_WARNING,
 				    "Failed to wait for children: %s",
 				    strerror(errno));
+			if (!do_pause)
+				return;
+
 		} else {
 			memset(&node, 0, sizeof (node));
 			node.pid = pid;
@@ -278,6 +270,25 @@ _reap_children(void *arg)
 		}
 	}
 
+}
+
+static void *
+_reap_children(void *arg)
+{
+	(void) arg;
+	struct sigaction sa = {};
+
+	(void) sigfillset(&sa.sa_mask);
+	(void) sigdelset(&sa.sa_mask, SIGCHLD);
+	(void) pthread_sigmask(SIG_SETMASK, &sa.sa_mask, NULL);
+
+	(void) sigemptyset(&sa.sa_mask);
+	sa.sa_handler = _nop;
+	sa.sa_flags = SA_NOCLDSTOP;
+	(void) sigaction(SIGCHLD, &sa, NULL);
+
+	wait_for_children(B_TRUE, B_FALSE);
+
 	return (NULL);
 }
 
@@ -307,6 +318,45 @@ zed_exec_fini(void)
 }
 
 /*
+ * Check if the zedlet name indicates if it is a synchronous zedlet
+ *
+ * Synchronous zedlets have a "-sync-" immediately following the event name in
+ * their zedlet filename, like:
+ *
+ * EVENT_NAME-sync-ZEDLETNAME.sh
+ *
+ * For example, if you wanted a synchronous statechange script:
+ *
+ * statechange-sync-myzedlet.sh
+ *
+ * Synchronous zedlets are guaranteed to be the only zedlet running.  No other
+ * zedlets may run in parallel with a synchronous zedlet.  A synchronous
+ * zedlet will wait for all previously spawned zedlets to finish before running.
+ * Users should be careful to only use synchronous zedlets when needed, since
+ * they decrease parallelism.
+ */
+static boolean_t
+zedlet_is_sync(const char *zedlet, const char *event)
+{
+	const char *sync_str = "-sync-";
+	size_t sync_str_len;
+	size_t zedlet_len;
+	size_t event_len;
+
+	sync_str_len = strlen(sync_str);
+	zedlet_len = strlen(zedlet);
+	event_len = strlen(event);
+
+	if (event_len + sync_str_len >= zedlet_len)
+		return (B_FALSE);
+
+	if (strncmp(&zedlet[event_len], sync_str, sync_str_len) == 0)
+		return (B_TRUE);
+
+	return (B_FALSE);
+}
+
+/*
  * Process the event [eid] by synchronously invoking all zedlets with a
  * matching class prefix.
  *
@@ -368,9 +418,28 @@ zed_exec_process(uint64_t eid, const char *class, const char *subclass,
 	    z = zed_strings_next(zcp->zedlets)) {
 		for (csp = class_strings; *csp; csp++) {
 			n = strlen(*csp);
-			if ((strncmp(z, *csp, n) == 0) && !isalpha(z[n]))
+			if ((strncmp(z, *csp, n) == 0) && !isalpha(z[n])) {
+				boolean_t is_sync = zedlet_is_sync(z, *csp);
+
+				if (is_sync) {
+					/*
+					 * Wait for previous zedlets to
+					 * finish
+					 */
+					wait_for_children(B_FALSE, B_TRUE);
+				}
+
 				_zed_exec_fork_child(eid, zcp->zedlet_dir,
 				    z, e, zcp->zevent_fd, zcp->do_foreground);
+
+				if (is_sync) {
+					/*
+					 * Wait for sync zedlet we just launched
+					 * to finish.
+					 */
+					wait_for_children(B_FALSE, B_TRUE);
+				}
+			}
 		}
 	}
 	free(e);
diff --git a/sys/contrib/openzfs/cmd/zfs/zfs_main.c b/sys/contrib/openzfs/cmd/zfs/zfs_main.c
index 363bb6da74ec..ccdd5ffef8e6 100644
--- a/sys/contrib/openzfs/cmd/zfs/zfs_main.c
+++ b/sys/contrib/openzfs/cmd/zfs/zfs_main.c
@@ -914,7 +914,11 @@ zfs_do_clone(int argc, char **argv)
 			log_history = B_FALSE;
 		}
 
-		ret = zfs_mount_and_share(g_zfs, argv[1], ZFS_TYPE_DATASET);
+		/*
+		 * Dataset cloned successfully, mount/share failures are
+		 * non-fatal.
+		 */
+		(void) zfs_mount_and_share(g_zfs, argv[1], ZFS_TYPE_DATASET);
 	}
 
 	zfs_close(zhp);
@@ -923,26 +927,22 @@ zfs_do_clone(int argc, char **argv)
 	return (!!ret);
 
 usage:
-	ASSERT3P(zhp, ==, NULL);
+	ASSERT0P(zhp);
 	nvlist_free(props);
 	usage(B_FALSE);
 	return (-1);
 }
 
 /*
- * Return a default volblocksize for the pool which always uses more than
- * half of the data sectors.  This primarily applies to dRAID which always
- * writes full stripe widths.
+ * Calculate the minimum allocation size based on the top-level vdevs.
  */
 static uint64_t
-default_volblocksize(zpool_handle_t *zhp, nvlist_t *props)
+calculate_volblocksize(nvlist_t *config)
 {
-	uint64_t volblocksize, asize = SPA_MINBLOCKSIZE;
+	uint64_t asize = SPA_MINBLOCKSIZE;
 	nvlist_t *tree, **vdevs;
 	uint_t nvdevs;
 
-	nvlist_t *config = zpool_get_config(zhp, NULL);
-
 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &tree) != 0 ||
 	    nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN,
 	    &vdevs, &nvdevs) != 0) {
@@ -973,6 +973,24 @@ default_volblocksize(zpool_handle_t *zhp, nvlist_t *props)
 		}
 	}
 
+	return (asize);
+}
+
+/*
+ * Return a default volblocksize for the pool which always uses more than
+ * half of the data sectors.  This primarily applies to dRAID which always
+ * writes full stripe widths.
+ */
+static uint64_t
+default_volblocksize(zpool_handle_t *zhp, nvlist_t *props)
+{
+	uint64_t volblocksize, asize = SPA_MINBLOCKSIZE;
+
+	nvlist_t *config = zpool_get_config(zhp, NULL);
+
+	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_MAX_ALLOC, &asize) != 0)
+		asize = calculate_volblocksize(config);
+
 	/*
 	 * Calculate the target volblocksize such that more than half
 	 * of the asize is used. The following table is for 4k sectors.
@@ -1319,7 +1337,9 @@ zfs_do_create(int argc, char **argv)
 		goto error;
 	}
 
-	ret = zfs_mount_and_share(g_zfs, argv[0], ZFS_TYPE_DATASET);
+	/* Dataset created successfully, mount/share failures are non-fatal */
+	ret = 0;
+	(void) zfs_mount_and_share(g_zfs, argv[0], ZFS_TYPE_DATASET);
 error:
 	nvlist_free(props);
 	return (ret);
@@ -5303,6 +5323,7 @@ zfs_do_receive(int argc, char **argv)
 #define	ZFS_DELEG_PERM_MOUNT		"mount"
 #define	ZFS_DELEG_PERM_SHARE		"share"
 #define	ZFS_DELEG_PERM_SEND		"send"
+#define	ZFS_DELEG_PERM_SEND_RAW		"send:raw"
 #define	ZFS_DELEG_PERM_RECEIVE		"receive"
 #define	ZFS_DELEG_PERM_RECEIVE_APPEND	"receive:append"
 #define	ZFS_DELEG_PERM_ALLOW		"allow"
@@ -5345,6 +5366,7 @@ static zfs_deleg_perm_tab_t zfs_deleg_perm_tbl[] = {
 	{ ZFS_DELEG_PERM_RENAME, ZFS_DELEG_NOTE_RENAME },
 	{ ZFS_DELEG_PERM_ROLLBACK, ZFS_DELEG_NOTE_ROLLBACK },
 	{ ZFS_DELEG_PERM_SEND, ZFS_DELEG_NOTE_SEND },
+	{ ZFS_DELEG_PERM_SEND_RAW, ZFS_DELEG_NOTE_SEND_RAW },
 	{ ZFS_DELEG_PERM_SHARE, ZFS_DELEG_NOTE_SHARE },
 	{ ZFS_DELEG_PERM_SNAPSHOT, ZFS_DELEG_NOTE_SNAPSHOT },
 	{ ZFS_DELEG_PERM_BOOKMARK, ZFS_DELEG_NOTE_BOOKMARK },
@@ -5929,6 +5951,10 @@ deleg_perm_comment(zfs_deleg_note_t note)
 	case ZFS_DELEG_NOTE_SEND:
 		str = gettext("");
 		break;
+	case ZFS_DELEG_NOTE_SEND_RAW:
+		str = gettext("Allow sending ONLY encrypted (raw) replication"
+		    "\n\t\t\t\tstreams");
+		break;
 	case ZFS_DELEG_NOTE_SHARE:
 		str = gettext("Allows sharing file systems over NFS or SMB"
 		    "\n\t\t\t\tprotocols");
@@ -6858,17 +6884,17 @@ print_holds(boolean_t scripted, int nwidth, int tagwidth, nvlist_t *nvl,
 
 			if (scripted) {
 				if (parsable) {
-					(void) printf("%s\t%s\t%ld\n", zname,
-					    tagname, (unsigned long)time);
+					(void) printf("%s\t%s\t%lld\n", zname,
+					    tagname, (long long)time);
 				} else {
 					(void) printf("%s\t%s\t%s\n", zname,
 					    tagname, tsbuf);
 				}
 			} else {
 				if (parsable) {
-					(void) printf("%-*s  %-*s  %ld\n",
+					(void) printf("%-*s  %-*s  %lld\n",
 					    nwidth, zname, tagwidth,
-					    tagname, (unsigned long)time);
+					    tagname, (long long)time);
 				} else {
 					(void) printf("%-*s  %-*s  %s\n",
 					    nwidth, zname, tagwidth,
diff --git a/sys/contrib/openzfs/cmd/zhack.c b/sys/contrib/openzfs/cmd/zhack.c
index 8244bc83fa0d..8ffbf91ffb30 100644
--- a/sys/contrib/openzfs/cmd/zhack.c
+++ b/sys/contrib/openzfs/cmd/zhack.c
@@ -54,6 +54,7 @@
 #include <sys/dmu_tx.h>
 #include <zfeature_common.h>
 #include <libzutil.h>
+#include <sys/metaslab_impl.h>
 
 static importargs_t g_importargs;
 static char *g_pool;
@@ -69,7 +70,8 @@ static __attribute__((noreturn)) void
 usage(void)
 {
 	(void) fprintf(stderr,
-	    "Usage: zhack [-c cachefile] [-d dir] <subcommand> <args> ...\n"
+	    "Usage: zhack [-o tunable] [-c cachefile] [-d dir] <subcommand> "
+	    "<args> ...\n"
 	    "where <subcommand> <args> is one of the following:\n"
 	    "\n");
 
@@ -93,7 +95,10 @@ usage(void)
 	    "        -c repair corrupted label checksums\n"
 	    "        -u restore the label on a detached device\n"
 	    "\n"
-	    "    <device> : path to vdev\n");
+	    "    <device> : path to vdev\n"
+	    "\n"
+	    "    metaslab leak <pool>\n"
+	    "        apply allocation map from zdb to specified pool\n");
 	exit(1);
 }
 
@@ -162,9 +167,9 @@ zhack_import(char *target, boolean_t readonly)
 
 	props = NULL;
 	if (readonly) {
-		VERIFY(nvlist_alloc(&props, NV_UNIQUE_NAME, 0) == 0);
-		VERIFY(nvlist_add_uint64(props,
-		    zpool_prop_to_name(ZPOOL_PROP_READONLY), 1) == 0);
+		VERIFY0(nvlist_alloc(&props, NV_UNIQUE_NAME, 0));
+		VERIFY0(nvlist_add_uint64(props,
+		    zpool_prop_to_name(ZPOOL_PROP_READONLY), 1));
 	}
 
 	zfeature_checks_disable = B_TRUE;
@@ -218,8 +223,8 @@ dump_obj(objset_t *os, uint64_t obj, const char *name)
 		} else {
 			ASSERT(za->za_integer_length == 1);
 			char val[1024];
-			VERIFY(zap_lookup(os, obj, za->za_name,
-			    1, sizeof (val), val) == 0);
+			VERIFY0(zap_lookup(os, obj, za->za_name,
+			    1, sizeof (val), val));
 			(void) printf("\t%s = %s\n", za->za_name, val);
 		}
 	}
@@ -363,10 +368,12 @@ feature_incr_sync(void *arg, dmu_tx_t *tx)
 	zfeature_info_t *feature = arg;
 	uint64_t refcount;
 
+	mutex_enter(&spa->spa_feat_stats_lock);
 	VERIFY0(feature_get_refcount_from_disk(spa, feature, &refcount));
 	feature_sync(spa, feature, refcount + 1, tx);
 	spa_history_log_internal(spa, "zhack feature incr", tx,
 	    "name=%s", feature->fi_guid);
+	mutex_exit(&spa->spa_feat_stats_lock);
 }
 
 static void
@@ -376,10 +383,12 @@ feature_decr_sync(void *arg, dmu_tx_t *tx)
 	zfeature_info_t *feature = arg;
 	uint64_t refcount;
 
+	mutex_enter(&spa->spa_feat_stats_lock);
 	VERIFY0(feature_get_refcount_from_disk(spa, feature, &refcount));
 	feature_sync(spa, feature, refcount - 1, tx);
 	spa_history_log_internal(spa, "zhack feature decr", tx,
 	    "name=%s", feature->fi_guid);
+	mutex_exit(&spa->spa_feat_stats_lock);
 }
 
 static void
@@ -496,6 +505,186 @@ zhack_do_feature(int argc, char **argv)
 	return (0);
 }
 
+static boolean_t
+strstarts(const char *a, const char *b)
+{
+	return (strncmp(a, b, strlen(b)) == 0);
+}
+
+static void
+metaslab_force_alloc(metaslab_t *msp, uint64_t start, uint64_t size,
+    dmu_tx_t *tx)
+{
+	ASSERT(msp->ms_disabled);
+	ASSERT(MUTEX_HELD(&msp->ms_lock));
+	uint64_t txg = dmu_tx_get_txg(tx);
+
+	uint64_t off = start;
+	while (off < start + size) {
+		uint64_t ostart, osize;
+		boolean_t found = zfs_range_tree_find_in(msp->ms_allocatable,
+		    off, start + size - off, &ostart, &osize);
+		if (!found)
+			break;
+		zfs_range_tree_remove(msp->ms_allocatable, ostart, osize);
+
+		if (zfs_range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK]))
+			vdev_dirty(msp->ms_group->mg_vd, VDD_METASLAB, msp,
+			    txg);
+
+		zfs_range_tree_add(msp->ms_allocating[txg & TXG_MASK], ostart,
+		    osize);
+		msp->ms_allocating_total += osize;
+		off = ostart + osize;
+	}
+}
+
+static void
+zhack_do_metaslab_leak(int argc, char **argv)
+{
+	int c;
+	char *target;
+	spa_t *spa;
+
+	optind = 1;
+	boolean_t force = B_FALSE;
+	while ((c = getopt(argc, argv, "f")) != -1) {
+		switch (c) {
+		case 'f':
+			force = B_TRUE;
+			break;
+		default:
+			usage();
+			break;
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 1) {
+		(void) fprintf(stderr, "error: missing pool name\n");
+		usage();
+	}
+	target = argv[0];
+
+	zhack_spa_open(target, B_FALSE, FTAG, &spa);
+	spa_config_enter(spa, SCL_VDEV | SCL_ALLOC, FTAG, RW_READER);
+
+	char *line = NULL;
+	size_t cap = 0;
+
+	vdev_t *vd = NULL;
+	metaslab_t *prev = NULL;
+	dmu_tx_t *tx = NULL;
+	while (getline(&line, &cap, stdin) > 0) {
+		if (strstarts(line, "\tvdev ")) {
+			uint64_t vdev_id, ms_shift;
+			if (sscanf(line,
+			    "\tvdev %10"PRIu64"\t%*s  metaslab shift %4"PRIu64,
+			    &vdev_id, &ms_shift) == 1) {
+				VERIFY3U(sscanf(line, "\tvdev %"PRIu64
+				    "\t  metaslab shift %4"PRIu64,
+				    &vdev_id, &ms_shift), ==, 2);
+			}
+			vd = vdev_lookup_top(spa, vdev_id);
+			if (vd == NULL) {
+				fprintf(stderr, "error: no such vdev with "
+				    "id %"PRIu64"\n", vdev_id);
+				break;
+			}
+			if (tx) {
+				dmu_tx_commit(tx);
+				mutex_exit(&prev->ms_lock);
+				metaslab_enable(prev, B_FALSE, B_FALSE);
+				tx = NULL;
+				prev = NULL;
+			}
+			if (vd->vdev_ms_shift != ms_shift) {
+				fprintf(stderr, "error: ms_shift mismatch: %"
+				    PRIu64" != %"PRIu64"\n", vd->vdev_ms_shift,
+				    ms_shift);
+				break;
+			}
+		} else if (strstarts(line, "\tmetaslabs ")) {
+			uint64_t ms_count;
+			VERIFY3U(sscanf(line, "\tmetaslabs %"PRIu64, &ms_count),
+			    ==, 1);
+			ASSERT(vd);
+			if (!force && vd->vdev_ms_count != ms_count) {
+				fprintf(stderr, "error: ms_count mismatch: %"
+				    PRIu64" != %"PRIu64"\n", vd->vdev_ms_count,
+				    ms_count);
+				break;
+			}
+		} else if (strstarts(line, "ALLOC:")) {
+			uint64_t start, size;
+			VERIFY3U(sscanf(line, "ALLOC: %"PRIu64" %"PRIu64"\n",
+			    &start, &size), ==, 2);
+
+			ASSERT(vd);
+			metaslab_t *cur =
+			    vd->vdev_ms[start >> vd->vdev_ms_shift];
+			if (prev != cur) {
+				if (prev) {
+					dmu_tx_commit(tx);
+					mutex_exit(&prev->ms_lock);
+					metaslab_enable(prev, B_FALSE, B_FALSE);
+				}
+				ASSERT(cur);
+				metaslab_disable(cur);
+				mutex_enter(&cur->ms_lock);
+				metaslab_load(cur);
+				prev = cur;
+				tx = dmu_tx_create_dd(
+				    spa_get_dsl(vd->vdev_spa)->dp_root_dir);
+				dmu_tx_assign(tx, DMU_TX_WAIT);
+			}
+
+			metaslab_force_alloc(cur, start, size, tx);
+		} else {
+			continue;
+		}
+	}
+	if (tx) {
+		dmu_tx_commit(tx);
+		mutex_exit(&prev->ms_lock);
+		metaslab_enable(prev, B_FALSE, B_FALSE);
+		tx = NULL;
+		prev = NULL;
+	}
+	if (line)
+		free(line);
+
+	spa_config_exit(spa, SCL_VDEV | SCL_ALLOC, FTAG);
+	spa_close(spa, FTAG);
+}
+
+static int
+zhack_do_metaslab(int argc, char **argv)
+{
+	char *subcommand;
+
+	argc--;
+	argv++;
+	if (argc == 0) {
+		(void) fprintf(stderr,
+		    "error: no metaslab operation specified\n");
+		usage();
+	}
+
+	subcommand = argv[0];
+	if (strcmp(subcommand, "leak") == 0) {
+		zhack_do_metaslab_leak(argc, argv);
+	} else {
+		(void) fprintf(stderr, "error: unknown subcommand: %s\n",
+		    subcommand);
+		usage();
+	}
+
+	return (0);
+}
+
 #define	ASHIFT_UBERBLOCK_SHIFT(ashift)	\
 	MIN(MAX(ashift, UBERBLOCK_SHIFT), \
 	MAX_UBERBLOCK_SHIFT)
@@ -525,6 +714,23 @@ zhack_repair_read_label(const int fd, vdev_label_t *vl,
 	return (0);
 }
 
+static int
+zhack_repair_get_byteswap(const zio_eck_t *vdev_eck, const int l, int *byteswap)
+{
+	if (vdev_eck->zec_magic == ZEC_MAGIC) {
+		*byteswap = B_FALSE;
+	} else if (vdev_eck->zec_magic == BSWAP_64((uint64_t)ZEC_MAGIC)) {
+		*byteswap = B_TRUE;
+	} else {
+		(void) fprintf(stderr, "error: label %d: "
+		    "Expected the nvlist checksum magic number but instead got "
+		    "0x%" PRIx64 "\n",
+		    l, vdev_eck->zec_magic);
+		return (1);
+	}
+	return (0);
+}
+
 static void
 zhack_repair_calc_cksum(const int byteswap, void *data, const uint64_t offset,
     const uint64_t abdsize, zio_eck_t *eck, zio_cksum_t *cksum)
@@ -551,33 +757,10 @@ zhack_repair_calc_cksum(const int byteswap, void *data, const uint64_t offset,
 }
 
 static int
-zhack_repair_check_label(uberblock_t *ub, const int l, const char **cfg_keys,
-    const size_t cfg_keys_len, nvlist_t *cfg, nvlist_t *vdev_tree_cfg,
-    uint64_t *ashift)
+zhack_repair_get_ashift(nvlist_t *cfg, const int l, uint64_t *ashift)
 {
 	int err;
-
-	if (ub->ub_txg != 0) {
-		(void) fprintf(stderr,
-		    "error: label %d: UB TXG of 0 expected, but got %"
-		    PRIu64 "\n",
-		    l, ub->ub_txg);
-		(void) fprintf(stderr, "It would appear the device was not "
-		    "properly removed.\n");
-		return (1);
-	}
-
-	for (int i = 0; i < cfg_keys_len; i++) {
-		uint64_t val;
-		err = nvlist_lookup_uint64(cfg, cfg_keys[i], &val);
-		if (err) {
-			(void) fprintf(stderr,
-			    "error: label %d, %d: "
-			    "cannot find nvlist key %s\n",
-			    l, i, cfg_keys[i]);
-			return (err);
-		}
-	}
+	nvlist_t *vdev_tree_cfg;
 
 	err = nvlist_lookup_nvlist(cfg,
 	    ZPOOL_CONFIG_VDEV_TREE, &vdev_tree_cfg);
@@ -601,7 +784,7 @@ zhack_repair_check_label(uberblock_t *ub, const int l, const char **cfg_keys,
 		(void) fprintf(stderr,
 		    "error: label %d: nvlist key %s is zero\n",
 		    l, ZPOOL_CONFIG_ASHIFT);
-		return (err);
+		return (1);
 	}
 
 	return (0);
@@ -616,30 +799,35 @@ zhack_repair_undetach(uberblock_t *ub, nvlist_t *cfg, const int l)
 	 */
 	if (BP_GET_LOGICAL_BIRTH(&ub->ub_rootbp) != 0) {
 		const uint64_t txg = BP_GET_LOGICAL_BIRTH(&ub->ub_rootbp);
+		int err;
+
 		ub->ub_txg = txg;
 
-		if (nvlist_remove_all(cfg, ZPOOL_CONFIG_CREATE_TXG) != 0) {
+		err = nvlist_remove_all(cfg, ZPOOL_CONFIG_CREATE_TXG);
+		if (err) {
 			(void) fprintf(stderr,
 			    "error: label %d: "
 			    "Failed to remove pool creation TXG\n",
 			    l);
-			return (1);
+			return (err);
 		}
 
-		if (nvlist_remove_all(cfg, ZPOOL_CONFIG_POOL_TXG) != 0) {
+		err = nvlist_remove_all(cfg, ZPOOL_CONFIG_POOL_TXG);
+		if (err) {
 			(void) fprintf(stderr,
 			    "error: label %d: Failed to remove pool TXG to "
 			    "be replaced.\n",
 			    l);
-			return (1);
+			return (err);
 		}
 
-		if (nvlist_add_uint64(cfg, ZPOOL_CONFIG_POOL_TXG, txg) != 0) {
+		err = nvlist_add_uint64(cfg, ZPOOL_CONFIG_POOL_TXG, txg);
+		if (err) {
 			(void) fprintf(stderr,
 			    "error: label %d: "
 			    "Failed to add pool TXG of %" PRIu64 "\n",
 			    l, txg);
-			return (1);
+			return (err);
 		}
 	}
 
@@ -733,6 +921,7 @@ zhack_repair_test_cksum(const int byteswap, void *vdev_data,
 	    BSWAP_64(ZEC_MAGIC) : ZEC_MAGIC;
 	const uint64_t actual_magic = vdev_eck->zec_magic;
 	int err = 0;
+
 	if (actual_magic != expected_magic) {
 		(void) fprintf(stderr, "error: label %d: "
 		    "Expected "
@@ -754,6 +943,36 @@ zhack_repair_test_cksum(const int byteswap, void *vdev_data,
 	return (err);
 }
 
+static int
+zhack_repair_unpack_cfg(vdev_label_t *vl, const int l, nvlist_t **cfg)
+{
+	const char *cfg_keys[] = { ZPOOL_CONFIG_VERSION,
+	    ZPOOL_CONFIG_POOL_STATE, ZPOOL_CONFIG_GUID };
+	int err;
+
+	err = nvlist_unpack(vl->vl_vdev_phys.vp_nvlist,
+	    VDEV_PHYS_SIZE - sizeof (zio_eck_t), cfg, 0);
+	if (err) {
+		(void) fprintf(stderr,
+		    "error: cannot unpack nvlist label %d\n", l);
+		return (err);
+	}
+
+	for (int i = 0; i < ARRAY_SIZE(cfg_keys); i++) {
+		uint64_t val;
+		err = nvlist_lookup_uint64(*cfg, cfg_keys[i], &val);
+		if (err) {
+			(void) fprintf(stderr,
+			    "error: label %d, %d: "
+			    "cannot find nvlist key %s\n",
+			    l, i, cfg_keys[i]);
+			return (err);
+		}
+	}
+
+	return (0);
+}
+
 static void
 zhack_repair_one_label(const zhack_repair_op_t op, const int fd,
     vdev_label_t *vl, const uint64_t label_offset, const int l,
@@ -767,10 +986,7 @@ zhack_repair_one_label(const zhack_repair_op_t op, const int fd,
 	    (zio_eck_t *)((char *)(vdev_data) + VDEV_PHYS_SIZE) - 1;
 	const uint64_t vdev_phys_offset =
 	    label_offset + offsetof(vdev_label_t, vl_vdev_phys);
-	const char *cfg_keys[] = { ZPOOL_CONFIG_VERSION,
-	    ZPOOL_CONFIG_POOL_STATE, ZPOOL_CONFIG_GUID };
 	nvlist_t *cfg;
-	nvlist_t *vdev_tree_cfg = NULL;
 	uint64_t ashift;
 	int byteswap;
 
@@ -778,18 +994,9 @@ zhack_repair_one_label(const zhack_repair_op_t op, const int fd,
 	if (err)
 		return;
 
-	if (vdev_eck->zec_magic == 0) {
-		(void) fprintf(stderr, "error: label %d: "
-		    "Expected the nvlist checksum magic number to not be zero"
-		    "\n",
-		    l);
-		(void) fprintf(stderr, "There should already be a checksum "
-		    "for the label.\n");
+	err = zhack_repair_get_byteswap(vdev_eck, l, &byteswap);
+	if (err)
 		return;
-	}
-
-	byteswap =
-	    (vdev_eck->zec_magic == BSWAP_64((uint64_t)ZEC_MAGIC));
 
 	if (byteswap) {
 		byteswap_uint64_array(&vdev_eck->zec_cksum,
@@ -805,16 +1012,7 @@ zhack_repair_one_label(const zhack_repair_op_t op, const int fd,
 		return;
 	}
 
-	err = nvlist_unpack(vl->vl_vdev_phys.vp_nvlist,
-	    VDEV_PHYS_SIZE - sizeof (zio_eck_t), &cfg, 0);
-	if (err) {
-		(void) fprintf(stderr,
-		    "error: cannot unpack nvlist label %d\n", l);
-		return;
-	}
-
-	err = zhack_repair_check_label(ub,
-	    l, cfg_keys, ARRAY_SIZE(cfg_keys), cfg, vdev_tree_cfg, &ashift);
+	err = zhack_repair_unpack_cfg(vl, l, &cfg);
 	if (err)
 		return;
 
@@ -822,6 +1020,19 @@ zhack_repair_one_label(const zhack_repair_op_t op, const int fd,
 		char *buf;
 		size_t buflen;
 
+		if (ub->ub_txg != 0) {
+			(void) fprintf(stderr,
+			    "error: label %d: UB TXG of 0 expected, but got %"
+			    PRIu64 "\n", l, ub->ub_txg);
+			(void) fprintf(stderr, "It would appear the device was "
+			    "not properly detached.\n");
+			return;
+		}
+
+		err = zhack_repair_get_ashift(cfg, l, &ashift);
+		if (err)
+			return;
+
 		err = zhack_repair_undetach(ub, cfg, l);
 		if (err)
 			return;
@@ -981,7 +1192,7 @@ main(int argc, char **argv)
 	dprintf_setup(&argc, argv);
 	zfs_prop_init();
 
-	while ((c = getopt(argc, argv, "+c:d:")) != -1) {
+	while ((c = getopt(argc, argv, "+c:d:o:")) != -1) {
 		switch (c) {
 		case 'c':
 			g_importargs.cachefile = optarg;
@@ -990,6 +1201,10 @@ main(int argc, char **argv)
 			assert(g_importargs.paths < MAX_NUM_PATHS);
 			g_importargs.path[g_importargs.paths++] = optarg;
 			break;
+		case 'o':
+			if (handle_tunable_option(optarg, B_FALSE) != 0)
+				exit(1);
+			break;
 		default:
 			usage();
 			break;
@@ -1011,6 +1226,8 @@ main(int argc, char **argv)
 		rv = zhack_do_feature(argc, argv);
 	} else if (strcmp(subcommand, "label") == 0) {
 		return (zhack_do_label(argc, argv));
+	} else if (strcmp(subcommand, "metaslab") == 0) {
+		rv = zhack_do_metaslab(argc, argv);
 	} else {
 		(void) fprintf(stderr, "error: unknown subcommand: %s\n",
 		    subcommand);
diff --git a/sys/contrib/openzfs/cmd/zilstat.in b/sys/contrib/openzfs/cmd/zilstat.in
index 4140398bf4a3..d01db9b0914b 100755
--- a/sys/contrib/openzfs/cmd/zilstat.in
+++ b/sys/contrib/openzfs/cmd/zilstat.in
@@ -47,6 +47,7 @@ cols = {
 	"cec":       [5,         1000,       "zil_commit_error_count"],
 	"csc":       [5,         1000,       "zil_commit_stall_count"],
 	"cSc":       [5,         1000,       "zil_commit_suspend_count"],
+	"cCc":       [5,         1000,       "zil_commit_crash_count"],
 	"ic":        [5,         1000,       "zil_itx_count"],
 	"iic":       [5,         1000,       "zil_itx_indirect_count"],
 	"iib":       [5,         1024,       "zil_itx_indirect_bytes"],
diff --git a/sys/contrib/openzfs/cmd/zinject/zinject.c b/sys/contrib/openzfs/cmd/zinject/zinject.c
index 113797c878b9..c2f646f2567d 100644
--- a/sys/contrib/openzfs/cmd/zinject/zinject.c
+++ b/sys/contrib/openzfs/cmd/zinject/zinject.c
@@ -107,6 +107,8 @@
  * 	zinject
  * 	zinject <-a | -u pool>
  * 	zinject -c <id|all>
+ * 	zinject -E <delay> [-a] [-m] [-f freq] [-l level] [-r range]
+ *	    [-T iotype] [-t type object | -b bookmark pool]
  * 	zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level]
  *	    [-r range] <object>
  * 	zinject [-f freq] [-a] [-m] [-u] -b objset:object:level:start:end pool
@@ -132,14 +134,18 @@
  * The '-f' flag controls the frequency of errors injected, expressed as a
  * real number percentage between 0.0001 and 100.  The default is 100.
  *
- * The this form is responsible for actually injecting the handler into the
+ * The <object> form is responsible for actually injecting the handler into the
  * framework.  It takes the arguments described above, translates them to the
  * internal tuple using libzpool, and then issues an ioctl() to register the
  * handler.
  *
- * The final form can target a specific bookmark, regardless of whether a
+ * The '-b' option can target a specific bookmark, regardless of whether a
  * human-readable interface has been designed.  It allows developers to specify
  * a particular block by number.
+ *
+ * The '-E' option injects pipeline ready stage delays for the given object or
+ * bookmark. The delay is specified in milliseconds, and it supports I/O type
+ * and range filters.
  */
 
 #include <errno.h>
@@ -346,6 +352,13 @@ usage(void)
 	    "\t\tsuch that the operation takes a minimum of supplied seconds\n"
 	    "\t\tto complete.\n"
 	    "\n"
+	    "\tzinject -E <delay> [-a] [-m] [-f freq] [-l level] [-r range]\n"
+	    "\t\t[-T iotype] [-t type object | -b bookmark pool]\n"
+	    "\n"
+	    "\t\tInject pipeline ready stage delays for the given object path\n"
+	    "\t\t(data or dnode) or raw bookmark. The delay is specified in\n"
+	    "\t\tmilliseconds.\n"
+	    "\n"
 	    "\tzinject -I [-s <seconds> | -g <txgs>] pool\n"
 	    "\t\tCause the pool to stop writing blocks yet not\n"
 	    "\t\treport errors for a duration.  Simulates buggy hardware\n"
@@ -724,12 +737,15 @@ register_handler(const char *pool, int flags, zinject_record_t *record,
 	if (quiet) {
 		(void) printf("%llu\n", (u_longlong_t)zc.zc_guid);
 	} else {
+		boolean_t show_object = B_FALSE;
+		boolean_t show_iotype = B_FALSE;
 		(void) printf("Added handler %llu with the following "
 		    "properties:\n", (u_longlong_t)zc.zc_guid);
 		(void) printf("  pool: %s\n", pool);
 		if (record->zi_guid) {
 			(void) printf("  vdev: %llx\n",
 			    (u_longlong_t)record->zi_guid);
+			show_iotype = B_TRUE;
 		} else if (record->zi_func[0] != '\0') {
 			(void) printf("  panic function: %s\n",
 			    record->zi_func);
@@ -742,7 +758,18 @@ register_handler(const char *pool, int flags, zinject_record_t *record,
 		} else if (record->zi_timer > 0) {
 			(void) printf(" timer: %lld ms\n",
 			    (u_longlong_t)NSEC2MSEC(record->zi_timer));
+			if (record->zi_cmd == ZINJECT_DELAY_READY) {
+				show_object = B_TRUE;
+				show_iotype = B_TRUE;
+			}
 		} else {
+			show_object = B_TRUE;
+		}
+		if (show_iotype) {
+			(void) printf("iotype: %s\n",
+			    iotype_to_str(record->zi_iotype));
+		}
+		if (show_object) {
 			(void) printf("objset: %llu\n",
 			    (u_longlong_t)record->zi_objset);
 			(void) printf("object: %llu\n",
@@ -910,6 +937,7 @@ main(int argc, char **argv)
 	int ret;
 	int flags = 0;
 	uint32_t dvas = 0;
+	hrtime_t ready_delay = -1;
 
 	if ((g_zfs = libzfs_init()) == NULL) {
 		(void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
@@ -940,7 +968,7 @@ main(int argc, char **argv)
 	}
 
 	while ((c = getopt(argc, argv,
-	    ":aA:b:C:d:D:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:")) != -1) {
+	    ":aA:b:C:d:D:E:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:")) != -1) {
 		switch (c) {
 		case 'a':
 			flags |= ZINJECT_FLUSH_ARC;
@@ -1113,6 +1141,18 @@ main(int argc, char **argv)
 		case 'u':
 			flags |= ZINJECT_UNLOAD_SPA;
 			break;
+		case 'E':
+			ready_delay = MSEC2NSEC(strtol(optarg, &end, 10));
+			if (ready_delay <= 0 || *end != '\0') {
+				(void) fprintf(stderr, "invalid delay '%s': "
+				    "must be a positive duration\n", optarg);
+				usage();
+				libzfs_fini(g_zfs);
+				return (1);
+			}
+			record.zi_cmd = ZINJECT_DELAY_READY;
+			record.zi_timer = ready_delay;
+			break;
 		case 'L':
 			if ((label = name_to_type(optarg)) == TYPE_INVAL &&
 			    !LABEL_TYPE(type)) {
@@ -1150,7 +1190,7 @@ main(int argc, char **argv)
 		 */
 		if (raw != NULL || range != NULL || type != TYPE_INVAL ||
 		    level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED ||
-		    record.zi_freq > 0 || dvas != 0) {
+		    record.zi_freq > 0 || dvas != 0 || ready_delay >= 0) {
 			(void) fprintf(stderr, "cancel (-c) incompatible with "
 			    "any other options\n");
 			usage();
@@ -1186,7 +1226,7 @@ main(int argc, char **argv)
 		 */
 		if (raw != NULL || range != NULL || type != TYPE_INVAL ||
 		    level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED ||
-		    dvas != 0) {
+		    dvas != 0 || ready_delay >= 0) {
 			(void) fprintf(stderr, "device (-d) incompatible with "
 			    "data error injection\n");
 			usage();
@@ -1276,13 +1316,23 @@ main(int argc, char **argv)
 			return (1);
 		}
 
-		record.zi_cmd = ZINJECT_DATA_FAULT;
+		if (record.zi_cmd == ZINJECT_UNINITIALIZED) {
+			record.zi_cmd = ZINJECT_DATA_FAULT;
+			if (!error)
+				error = EIO;
+		} else if (error != 0) {
+			(void) fprintf(stderr, "error type -e incompatible "
+			    "with delay injection\n");
+			libzfs_fini(g_zfs);
+			return (1);
+		} else {
+			record.zi_iotype = io_type;
+		}
+
 		if (translate_raw(raw, &record) != 0) {
 			libzfs_fini(g_zfs);
 			return (1);
 		}
-		if (!error)
-			error = EIO;
 	} else if (record.zi_cmd == ZINJECT_PANIC) {
 		if (raw != NULL || range != NULL || type != TYPE_INVAL ||
 		    level != 0 || device != NULL || record.zi_freq > 0 ||
@@ -1410,6 +1460,13 @@ main(int argc, char **argv)
 			record.zi_dvas = dvas;
 		}
 
+		if (record.zi_cmd != ZINJECT_UNINITIALIZED && error != 0) {
+			(void) fprintf(stderr, "error type -e incompatible "
+			    "with delay injection\n");
+			libzfs_fini(g_zfs);
+			return (1);
+		}
+
 		if (error == EACCES) {
 			if (type != TYPE_DATA) {
 				(void) fprintf(stderr, "decryption errors "
@@ -1425,8 +1482,12 @@ main(int argc, char **argv)
 			 * not found.
 			 */
 			error = ECKSUM;
-		} else {
+		} else if (record.zi_cmd == ZINJECT_UNINITIALIZED) {
 			record.zi_cmd = ZINJECT_DATA_FAULT;
+			if (!error)
+				error = EIO;
+		} else {
+			record.zi_iotype = io_type;
 		}
 
 		if (translate_record(type, argv[0], range, level, &record, pool,
@@ -1434,8 +1495,6 @@ main(int argc, char **argv)
 			libzfs_fini(g_zfs);
 			return (1);
 		}
-		if (!error)
-			error = EIO;
 	}
 
 	/*
diff --git a/sys/contrib/openzfs/cmd/zpool/Makefile.am b/sys/contrib/openzfs/cmd/zpool/Makefile.am
index 2f962408e5a3..5bb6d8160b18 100644
--- a/sys/contrib/openzfs/cmd/zpool/Makefile.am
+++ b/sys/contrib/openzfs/cmd/zpool/Makefile.am
@@ -148,6 +148,7 @@ dist_zpoolcompat_DATA = \
 	%D%/compatibility.d/openzfs-2.1-linux \
 	%D%/compatibility.d/openzfs-2.2 \
 	%D%/compatibility.d/openzfs-2.3 \
+	%D%/compatibility.d/openzfs-2.4 \
 	%D%/compatibility.d/openzfsonosx-1.7.0 \
 	%D%/compatibility.d/openzfsonosx-1.8.1 \
 	%D%/compatibility.d/openzfsonosx-1.9.3 \
@@ -187,7 +188,9 @@ zpoolcompatlinks = \
 	"openzfs-2.2		openzfs-2.2-linux" \
 	"openzfs-2.2		openzfs-2.2-freebsd" \
 	"openzfs-2.3		openzfs-2.3-linux" \
-	"openzfs-2.3		openzfs-2.3-freebsd"
+	"openzfs-2.3		openzfs-2.3-freebsd" \
+	"openzfs-2.4		openzfs-2.4-linux" \
+	"openzfs-2.4		openzfs-2.4-freebsd"
 
 zpoolconfdir = $(sysconfdir)/zfs/zpool.d
 INSTALL_DATA_HOOKS += zpool-install-data-hook
diff --git a/sys/contrib/openzfs/cmd/zpool/compatibility.d/openzfs-2.4 b/sys/contrib/openzfs/cmd/zpool/compatibility.d/openzfs-2.4
new file mode 100644
index 000000000000..3fbd91014c95
--- /dev/null
+++ b/sys/contrib/openzfs/cmd/zpool/compatibility.d/openzfs-2.4
@@ -0,0 +1,48 @@
+# Features supported by OpenZFS 2.4 on Linux and FreeBSD
+allocation_classes
+async_destroy
+blake3
+block_cloning
+block_cloning_endian
+bookmark_v2
+bookmark_written
+bookmarks
+device_rebuild
+device_removal
+draid
+dynamic_gang_header
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+fast_dedup
+filesystem_limits
+head_errlog
+hole_birth
+large_blocks
+large_dnode
+large_microzap
+livelist
+log_spacemap
+longname
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+physical_rewrite
+project_quota
+raidz_expansion
+redacted_datasets
+redaction_bookmarks
+redaction_list_spill
+resilver_defer
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+userobj_accounting
+vdev_zaps_v2
+zilsaxattr
+zpool_checkpoint
+zstd_compress
diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_iter.c b/sys/contrib/openzfs/cmd/zpool/zpool_iter.c
index 2eec9a95e24c..fef602736705 100644
--- a/sys/contrib/openzfs/cmd/zpool/zpool_iter.c
+++ b/sys/contrib/openzfs/cmd/zpool/zpool_iter.c
@@ -26,6 +26,7 @@
 
 /*
  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
+ * Copyright (c) 2025, Klara, Inc.
  */
 
 #include <libintl.h>
@@ -52,7 +53,7 @@
 typedef struct zpool_node {
 	zpool_handle_t	*zn_handle;
 	uu_avl_node_t	zn_avlnode;
-	int		zn_mark;
+	hrtime_t	zn_last_refresh;
 } zpool_node_t;
 
 struct zpool_list {
@@ -62,6 +63,7 @@ struct zpool_list {
 	uu_avl_pool_t	*zl_pool;
 	zprop_list_t	**zl_proplist;
 	zfs_type_t	zl_type;
+	hrtime_t	zl_last_refresh;
 };
 
 static int
@@ -81,26 +83,30 @@ zpool_compare(const void *larg, const void *rarg, void *unused)
  * of known pools.
  */
 static int
-add_pool(zpool_handle_t *zhp, void *data)
+add_pool(zpool_handle_t *zhp, zpool_list_t *zlp)
 {
-	zpool_list_t *zlp = data;
-	zpool_node_t *node = safe_malloc(sizeof (zpool_node_t));
+	zpool_node_t *node, *new = safe_malloc(sizeof (zpool_node_t));
 	uu_avl_index_t idx;
 
-	node->zn_handle = zhp;
-	uu_avl_node_init(node, &node->zn_avlnode, zlp->zl_pool);
-	if (uu_avl_find(zlp->zl_avl, node, NULL, &idx) == NULL) {
+	new->zn_handle = zhp;
+	uu_avl_node_init(new, &new->zn_avlnode, zlp->zl_pool);
+
+	node = uu_avl_find(zlp->zl_avl, new, NULL, &idx);
+	if (node == NULL) {
 		if (zlp->zl_proplist &&
 		    zpool_expand_proplist(zhp, zlp->zl_proplist,
 		    zlp->zl_type, zlp->zl_literal) != 0) {
 			zpool_close(zhp);
-			free(node);
+			free(new);
 			return (-1);
 		}
-		uu_avl_insert(zlp->zl_avl, node, idx);
+		new->zn_last_refresh = zlp->zl_last_refresh;
+		uu_avl_insert(zlp->zl_avl, new, idx);
 	} else {
+		zpool_refresh_stats_from_handle(node->zn_handle, zhp);
+		node->zn_last_refresh = zlp->zl_last_refresh;
 		zpool_close(zhp);
-		free(node);
+		free(new);
 		return (-1);
 	}
 
@@ -108,6 +114,18 @@ add_pool(zpool_handle_t *zhp, void *data)
 }
 
 /*
+ * add_pool(), but always returns 0. This allows zpool_iter() to continue
+ * even if a pool exists in the tree, or we fail to get the properties for
+ * a new one.
+ */
+static int
+add_pool_cb(zpool_handle_t *zhp, void *data)
+{
+	(void) add_pool(zhp, data);
+	return (0);
+}
+
+/*
  * Create a list of pools based on the given arguments.  If we're given no
  * arguments, then iterate over all pools in the system and add them to the AVL
  * tree.  Otherwise, add only those pool explicitly specified on the command
@@ -135,9 +153,10 @@ pool_list_get(int argc, char **argv, zprop_list_t **proplist, zfs_type_t type,
 	zlp->zl_type = type;
 
 	zlp->zl_literal = literal;
+	zlp->zl_last_refresh = gethrtime();
 
 	if (argc == 0) {
-		(void) zpool_iter(g_zfs, add_pool, zlp);
+		(void) zpool_iter(g_zfs, add_pool_cb, zlp);
 		zlp->zl_findall = B_TRUE;
 	} else {
 		int i;
@@ -159,15 +178,61 @@ pool_list_get(int argc, char **argv, zprop_list_t **proplist, zfs_type_t type,
 }
 
 /*
- * Search for any new pools, adding them to the list.  We only add pools when no
- * options were given on the command line.  Otherwise, we keep the list fixed as
- * those that were explicitly specified.
+ * Refresh the state of all pools on the list. Additionally, if no options were
+ * given on the command line, add any new pools and remove any that are no
+ * longer available.
  */
-void
-pool_list_update(zpool_list_t *zlp)
+int
+pool_list_refresh(zpool_list_t *zlp)
 {
-	if (zlp->zl_findall)
-		(void) zpool_iter(g_zfs, add_pool, zlp);
+	zlp->zl_last_refresh = gethrtime();
+
+	if (!zlp->zl_findall) {
+		/*
+		 * This list is a fixed list of pools, so we must not add
+		 * or remove any. Just walk over them and refresh their
+		 * state.
+		 */
+		int navail = 0;
+		for (zpool_node_t *node = uu_avl_first(zlp->zl_avl);
+		    node != NULL; node = uu_avl_next(zlp->zl_avl, node)) {
+			boolean_t missing;
+			zpool_refresh_stats(node->zn_handle, &missing);
+			navail += !missing;
+			node->zn_last_refresh = zlp->zl_last_refresh;
+		}
+		return (navail);
+	}
+
+	/* Search for any new pools and add them to the list. */
+	(void) zpool_iter(g_zfs, add_pool_cb, zlp);
+
+	/* Walk the list of existing pools, and update or remove them. */
+	zpool_node_t *node, *next;
+	for (node = uu_avl_first(zlp->zl_avl); node != NULL; node = next) {
+		next = uu_avl_next(zlp->zl_avl, node);
+
+		/*
+		 * Skip any that were refreshed and are online; they were added
+		 * by zpool_iter() and are already up to date.
+		 */
+		if (node->zn_last_refresh == zlp->zl_last_refresh &&
+		    zpool_get_state(node->zn_handle) != POOL_STATE_UNAVAIL)
+			continue;
+
+		/* Refresh and remove if necessary. */
+		boolean_t missing;
+		zpool_refresh_stats(node->zn_handle, &missing);
+		if (missing) {
+			uu_avl_remove(zlp->zl_avl, node);
+			zpool_close(node->zn_handle);
+			free(node);
+		} else {
+			node->zn_last_refresh = zlp->zl_last_refresh;
+		}
+	}
+
+	return (uu_avl_numnodes(zlp->zl_avl));
 }
 
 /*
@@ -191,23 +256,6 @@ pool_list_iter(zpool_list_t *zlp, int unavail, zpool_iter_f func,
 }
 
 /*
- * Remove the given pool from the list.  When running iostat, we want to remove
- * those pools that no longer exist.
- */
-void
-pool_list_remove(zpool_list_t *zlp, zpool_handle_t *zhp)
-{
-	zpool_node_t search, *node;
-
-	search.zn_handle = zhp;
-	if ((node = uu_avl_find(zlp->zl_avl, &search, NULL, NULL)) != NULL) {
-		uu_avl_remove(zlp->zl_avl, node);
-		zpool_close(node->zn_handle);
-		free(node);
-	}
-}
-
-/*
  * Free all the handles associated with this list.
  */
 void
diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_main.c b/sys/contrib/openzfs/cmd/zpool/zpool_main.c
index d401e087916b..a6658a9c2800 100644
--- a/sys/contrib/openzfs/cmd/zpool/zpool_main.c
+++ b/sys/contrib/openzfs/cmd/zpool/zpool_main.c
@@ -33,7 +33,7 @@
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
- * Copyright (c) 2021, 2023, Klara Inc.
+ * Copyright (c) 2021, 2023, 2025, Klara, Inc.
  * Copyright (c) 2021, 2025 Hewlett Packard Enterprise Development LP.
  */
 
@@ -456,7 +456,7 @@ get_usage(zpool_help_t idx)
 		    "<pool> <vdev> ...\n"));
 	case HELP_ATTACH:
 		return (gettext("\tattach [-fsw] [-o property=value] "
-		    "<pool> <device> <new-device>\n"));
+		    "<pool> <vdev> <new-device>\n"));
 	case HELP_CLEAR:
 		return (gettext("\tclear [[--power]|[-nF]] <pool> [device]\n"));
 	case HELP_CREATE:
@@ -752,10 +752,11 @@ usage(boolean_t requested)
 }
 
 /*
- * zpool initialize [-c | -s | -u] [-w] <pool> [<vdev> ...]
+ * zpool initialize [-c | -s | -u] [-w] <-a | pool> [<vdev> ...]
  * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
  * if none specified.
  *
+ *	-a	Use all pools.
  *	-c	Cancel. Ends active initializing.
  *	-s	Suspend. Initializing can then be restarted with no flags.
  *	-u	Uninitialize. Clears initialization state.
@@ -776,7 +777,7 @@ zpool_do_initialize(int argc, char **argv)
 		{"suspend",	no_argument,		NULL, 's'},
 		{"uninit",	no_argument,		NULL, 'u'},
 		{"wait",	no_argument,		NULL, 'w'},
-		{"all", 	no_argument,		NULL, 'a'},
+		{"all",		no_argument,		NULL, 'a'},
 		{0, 0, 0, 0}
 	};
 
@@ -5760,24 +5761,6 @@ children:
 	return (ret);
 }
 
-static int
-refresh_iostat(zpool_handle_t *zhp, void *data)
-{
-	iostat_cbdata_t *cb = data;
-	boolean_t missing;
-
-	/*
-	 * If the pool has disappeared, remove it from the list and continue.
-	 */
-	if (zpool_refresh_stats(zhp, &missing) != 0)
-		return (-1);
-
-	if (missing)
-		pool_list_remove(cb->cb_list, zhp);
-
-	return (0);
-}
-
 /*
  * Callback to print out the iostats for the given pool.
  */
@@ -6358,15 +6341,14 @@ get_namewidth_iostat(zpool_handle_t *zhp, void *data)
  * This command can be tricky because we want to be able to deal with pool
  * creation/destruction as well as vdev configuration changes.  The bulk of this
  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
- * on pool_list_update() to detect the addition of new pools.  Configuration
- * changes are all handled within libzfs.
+ * on pool_list_refresh() to detect the addition and removal of pools.
+ * Configuration changes are all handled within libzfs.
  */
 int
 zpool_do_iostat(int argc, char **argv)
 {
 	int c;
 	int ret;
-	int npools;
 	float interval = 0;
 	unsigned long count = 0;
 	zpool_list_t *list;
@@ -6617,10 +6599,24 @@ zpool_do_iostat(int argc, char **argv)
 		return (1);
 	}
 
+	int last_npools = 0;
 	for (;;) {
-		if ((npools = pool_list_count(list)) == 0)
+		/*
+		 * Refresh all pools in list, adding or removing pools as
+		 * necessary.
+		 */
+		int npools = pool_list_refresh(list);
+		if (npools == 0) {
 			(void) fprintf(stderr, gettext("no pools available\n"));
-		else {
+		} else {
+			/*
+			 * If the list of pools has changed since last time
+			 * around, reset the iteration count to force the
+			 * header to be redisplayed.
+			 */
+			if (last_npools != npools)
+				cb.cb_iteration = 0;
+
 			/*
 			 * If this is the first iteration and -y was supplied
 			 * we skip any printing.
@@ -6629,15 +6625,6 @@ zpool_do_iostat(int argc, char **argv)
 			    cb.cb_iteration == 0);
 
 			/*
-			 * Refresh all statistics.  This is done as an
-			 * explicit step before calculating the maximum name
-			 * width, so that any * configuration changes are
-			 * properly accounted for.
-			 */
-			(void) pool_list_iter(list, B_FALSE, refresh_iostat,
-			    &cb);
-
-			/*
 			 * Iterate over all pools to determine the maximum width
 			 * for the pool / device name column across all pools.
 			 */
@@ -6690,6 +6677,7 @@ zpool_do_iostat(int argc, char **argv)
 			if (skip) {
 				(void) fflush(stdout);
 				(void) fsleep(interval);
+				last_npools = npools;
 				continue;
 			}
 
@@ -6727,6 +6715,8 @@ zpool_do_iostat(int argc, char **argv)
 
 		(void) fflush(stdout);
 		(void) fsleep(interval);
+
+		last_npools = npools;
 	}
 
 	pool_list_free(list);
@@ -6985,7 +6975,6 @@ collect_vdev_prop(zpool_prop_t prop, uint64_t value, const char *str,
 
 /*
  * print static default line per vdev
- * not compatible with '-o' <proplist> option
  */
 static void
 collect_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
@@ -7041,48 +7030,98 @@ collect_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 		 * 'toplevel' boolean value is passed to the print_one_column()
 		 * to indicate that the value is valid.
 		 */
-		if (VDEV_STAT_VALID(vs_pspace, c) && vs->vs_pspace) {
-			collect_vdev_prop(ZPOOL_PROP_SIZE, vs->vs_pspace, NULL,
-			    scripted, B_TRUE, format, cb->cb_json, props,
-			    cb->cb_json_as_int);
-		} else {
-			collect_vdev_prop(ZPOOL_PROP_SIZE, vs->vs_space, NULL,
-			    scripted, toplevel, format, cb->cb_json, props,
-			    cb->cb_json_as_int);
+		for (zprop_list_t *pl = cb->cb_proplist; pl != NULL;
+		    pl = pl->pl_next) {
+			switch (pl->pl_prop) {
+			case ZPOOL_PROP_SIZE:
+				if (VDEV_STAT_VALID(vs_pspace, c) &&
+				    vs->vs_pspace) {
+					collect_vdev_prop(
+					    ZPOOL_PROP_SIZE, vs->vs_pspace,
+					    NULL, scripted, B_TRUE, format,
+					    cb->cb_json, props,
+					    cb->cb_json_as_int);
+				} else {
+					collect_vdev_prop(
+					    ZPOOL_PROP_SIZE, vs->vs_space, NULL,
+					    scripted, toplevel, format,
+					    cb->cb_json, props,
+					    cb->cb_json_as_int);
+				}
+				break;
+			case ZPOOL_PROP_ALLOCATED:
+				collect_vdev_prop(ZPOOL_PROP_ALLOCATED,
+				    vs->vs_alloc, NULL, scripted, toplevel,
+				    format, cb->cb_json, props,
+				    cb->cb_json_as_int);
+				break;
+
+			case ZPOOL_PROP_FREE:
+				collect_vdev_prop(ZPOOL_PROP_FREE,
+				    vs->vs_space - vs->vs_alloc, NULL, scripted,
+				    toplevel, format, cb->cb_json, props,
+				    cb->cb_json_as_int);
+				break;
+
+			case ZPOOL_PROP_CHECKPOINT:
+				collect_vdev_prop(ZPOOL_PROP_CHECKPOINT,
+				    vs->vs_checkpoint_space, NULL, scripted,
+				    toplevel, format, cb->cb_json, props,
+				    cb->cb_json_as_int);
+				break;
+
+			case ZPOOL_PROP_EXPANDSZ:
+				collect_vdev_prop(ZPOOL_PROP_EXPANDSZ,
+				    vs->vs_esize, NULL, scripted, B_TRUE,
+				    format, cb->cb_json, props,
+				    cb->cb_json_as_int);
+				break;
+
+			case ZPOOL_PROP_FRAGMENTATION:
+				collect_vdev_prop(
+				    ZPOOL_PROP_FRAGMENTATION,
+				    vs->vs_fragmentation, NULL, scripted,
+				    (vs->vs_fragmentation != ZFS_FRAG_INVALID &&
+				    toplevel),
+				    format, cb->cb_json, props,
+				    cb->cb_json_as_int);
+				break;
+
+			case ZPOOL_PROP_CAPACITY:
+				cap = (vs->vs_space == 0) ?
+				    0 : (vs->vs_alloc * 10000 / vs->vs_space);
+				collect_vdev_prop(ZPOOL_PROP_CAPACITY, cap,
+				    NULL, scripted, toplevel, format,
+				    cb->cb_json, props, cb->cb_json_as_int);
+				break;
+
+			case ZPOOL_PROP_HEALTH:
+				state = zpool_state_to_name(vs->vs_state,
+				    vs->vs_aux);
+				if (isspare) {
+					if (vs->vs_aux == VDEV_AUX_SPARED)
+						state = "INUSE";
+					else if (vs->vs_state ==
+					    VDEV_STATE_HEALTHY)
+						state = "AVAIL";
+				}
+				collect_vdev_prop(ZPOOL_PROP_HEALTH, 0, state,
+				    scripted, B_TRUE, format, cb->cb_json,
+				    props, cb->cb_json_as_int);
+				break;
+
+			case ZPOOL_PROP_NAME:
+				break;
+
+			default:
+				collect_vdev_prop(pl->pl_prop, 0,
+				    NULL, scripted, B_FALSE, format,
+				    cb->cb_json, props, cb->cb_json_as_int);
+
+			}
+
+
 		}
-		collect_vdev_prop(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, NULL,
-		    scripted, toplevel, format, cb->cb_json, props,
-		    cb->cb_json_as_int);
-		collect_vdev_prop(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
-		    NULL, scripted, toplevel, format, cb->cb_json, props,
-		    cb->cb_json_as_int);
-		collect_vdev_prop(ZPOOL_PROP_CHECKPOINT,
-		    vs->vs_checkpoint_space, NULL, scripted, toplevel, format,
-		    cb->cb_json, props, cb->cb_json_as_int);
-		collect_vdev_prop(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, NULL,
-		    scripted, B_TRUE, format, cb->cb_json, props,
-		    cb->cb_json_as_int);
-		collect_vdev_prop(ZPOOL_PROP_FRAGMENTATION,
-		    vs->vs_fragmentation, NULL, scripted,
-		    (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
-		    format, cb->cb_json, props, cb->cb_json_as_int);
-		cap = (vs->vs_space == 0) ? 0 :
-		    (vs->vs_alloc * 10000 / vs->vs_space);
-		collect_vdev_prop(ZPOOL_PROP_CAPACITY, cap, NULL,
-		    scripted, toplevel, format, cb->cb_json, props,
-		    cb->cb_json_as_int);
-		collect_vdev_prop(ZPOOL_PROP_DEDUPRATIO, 0, NULL,
-		    scripted, toplevel, format, cb->cb_json, props,
-		    cb->cb_json_as_int);
-		state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
-		if (isspare) {
-			if (vs->vs_aux == VDEV_AUX_SPARED)
-				state = "INUSE";
-			else if (vs->vs_state == VDEV_STATE_HEALTHY)
-				state = "AVAIL";
-		}
-		collect_vdev_prop(ZPOOL_PROP_HEALTH, 0, state, scripted,
-		    B_TRUE, format, cb->cb_json, props, cb->cb_json_as_int);
 
 		if (cb->cb_json) {
 			fnvlist_add_nvlist(ent, "properties", props);
@@ -7643,7 +7682,7 @@ zpool_do_replace(int argc, char **argv)
 }
 
 /*
- * zpool attach [-fsw] [-o property=value] <pool> <device>|<vdev> <new_device>
+ * zpool attach [-fsw] [-o property=value] <pool> <vdev> <new_device>
  *
  *	-f	Force attach, even if <new_device> appears to be in use.
  *	-s	Use sequential instead of healing reconstruction for resilver.
@@ -7651,9 +7690,9 @@ zpool_do_replace(int argc, char **argv)
  *	-w	Wait for resilvering (mirror) or expansion (raidz) to complete
  *		before returning.
  *
- * Attach <new_device> to a <device> or <vdev>, where the vdev can be of type
- * mirror or raidz. If <device> is not part of a mirror, then <device> will
- * be transformed into a mirror of <device> and <new_device>. When a mirror
+ * Attach <new_device> to a <vdev>, where the vdev can be of type
+ * device, mirror or raidz. If <vdev> is not part of a mirror, then <vdev> will
+ * be transformed into a mirror of <vdev> and <new_device>. When a mirror
  * is involved, <new_device> will begin life with a DTL of [0, now], and will
  * immediately begin to resilver itself. For the raidz case, a expansion will
  * commence and reflow the raidz data across all the disks including the
@@ -8446,8 +8485,9 @@ date_string_to_sec(const char *timestr, boolean_t rounding)
 }
 
 /*
- * zpool scrub [-e | -s | -p | -C | -E | -S] [-w] <pool> ...
+ * zpool scrub [-e | -s | -p | -C | -E | -S] [-w] [-a | <pool> ...]
  *
+ *	-a	Scrub all pools.
  *	-e	Only scrub blocks in the error log.
  *	-E	End date of scrub.
  *	-S	Start date of scrub.
@@ -8621,8 +8661,9 @@ zpool_do_resilver(int argc, char **argv)
 }
 
 /*
- * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
+ * zpool trim [-d] [-r <rate>] [-c | -s] <-a | pool> [<device> ...]
  *
+ *	-a		Trim all pools.
  *	-c		Cancel. Ends any in-progress trim.
  *	-d		Secure trim.  Requires kernel and device support.
  *	-r <rate>	Sets the TRIM rate in bytes (per second). Supports
@@ -12374,7 +12415,7 @@ zpool_do_events_next(ev_opts_t *opts)
 		nvlist_free(nvl);
 	}
 
-	VERIFY(0 == close(zevent_fd));
+	VERIFY0(close(zevent_fd));
 
 	return (ret);
 }
diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_util.h b/sys/contrib/openzfs/cmd/zpool/zpool_util.h
index 5ab7cb9750f1..3af23c52bd45 100644
--- a/sys/contrib/openzfs/cmd/zpool/zpool_util.h
+++ b/sys/contrib/openzfs/cmd/zpool/zpool_util.h
@@ -76,11 +76,10 @@ typedef struct zpool_list zpool_list_t;
 
 zpool_list_t *pool_list_get(int, char **, zprop_list_t **, zfs_type_t,
     boolean_t, int *);
-void pool_list_update(zpool_list_t *);
+int pool_list_refresh(zpool_list_t *);
 int pool_list_iter(zpool_list_t *, int unavail, zpool_iter_f, void *);
 void pool_list_free(zpool_list_t *);
 int pool_list_count(zpool_list_t *);
-void pool_list_remove(zpool_list_t *, zpool_handle_t *);
 
 extern libzfs_handle_t *g_zfs;
 
diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c b/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c
index 684b46a2d673..222b5524669e 100644
--- a/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c
+++ b/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c
@@ -270,14 +270,13 @@ is_spare(nvlist_t *config, const char *path)
  *	draid*		Virtual dRAID spare
  */
 static nvlist_t *
-make_leaf_vdev(nvlist_t *props, const char *arg, boolean_t is_primary)
+make_leaf_vdev(const char *arg, boolean_t is_primary, uint64_t ashift)
 {
 	char path[MAXPATHLEN];
 	struct stat64 statbuf;
 	nvlist_t *vdev = NULL;
 	const char *type = NULL;
 	boolean_t wholedisk = B_FALSE;
-	uint64_t ashift = 0;
 	int err;
 
 	/*
@@ -382,31 +381,6 @@ make_leaf_vdev(nvlist_t *props, const char *arg, boolean_t is_primary)
 		    (uint64_t)wholedisk) == 0);
 
 	/*
-	 * Override defaults if custom properties are provided.
-	 */
-	if (props != NULL) {
-		const char *value = NULL;
-
-		if (nvlist_lookup_string(props,
-		    zpool_prop_to_name(ZPOOL_PROP_ASHIFT), &value) == 0) {
-			if (zfs_nicestrtonum(NULL, value, &ashift) != 0) {
-				(void) fprintf(stderr,
-				    gettext("ashift must be a number.\n"));
-				return (NULL);
-			}
-			if (ashift != 0 &&
-			    (ashift < ASHIFT_MIN || ashift > ASHIFT_MAX)) {
-				(void) fprintf(stderr,
-				    gettext("invalid 'ashift=%" PRIu64 "' "
-				    "property: only values between %" PRId32 " "
-				    "and %" PRId32 " are allowed.\n"),
-				    ashift, ASHIFT_MIN, ASHIFT_MAX);
-				return (NULL);
-			}
-		}
-	}
-
-	/*
 	 * If the device is known to incorrectly report its physical sector
 	 * size explicitly provide the known correct value.
 	 */
@@ -609,22 +583,28 @@ get_replication(nvlist_t *nvroot, boolean_t fatal)
 				    ZPOOL_CONFIG_PATH, &path) == 0);
 
 				/*
+				 * Skip active spares they should never cause
+				 * the pool to be evaluated as inconsistent.
+				 */
+				if (is_spare(NULL, path))
+					continue;
+
+				/*
 				 * If we have a raidz/mirror that combines disks
-				 * with files, report it as an error.
+				 * with files, only report it as an error when
+				 * fatal is set to ensure all the replication
+				 * checks aren't skipped in check_replication().
 				 */
-				if (!dontreport && type != NULL &&
+				if (fatal && !dontreport && type != NULL &&
 				    strcmp(type, childtype) != 0) {
 					if (ret != NULL)
 						free(ret);
 					ret = NULL;
-					if (fatal)
-						vdev_error(gettext(
-						    "mismatched replication "
-						    "level: %s contains both "
-						    "files and devices\n"),
-						    rep.zprl_type);
-					else
-						return (NULL);
+					vdev_error(gettext(
+					    "mismatched replication "
+					    "level: %s contains both "
+					    "files and devices\n"),
+					    rep.zprl_type);
 					dontreport = B_TRUE;
 				}
 
@@ -1507,6 +1487,29 @@ construct_spec(nvlist_t *props, int argc, char **argv)
 	const char *type, *fulltype;
 	boolean_t is_log, is_special, is_dedup, is_spare;
 	boolean_t seen_logs;
+	uint64_t ashift = 0;
+
+	if (props != NULL) {
+		const char *value = NULL;
+
+		if (nvlist_lookup_string(props,
+		    zpool_prop_to_name(ZPOOL_PROP_ASHIFT), &value) == 0) {
+			if (zfs_nicestrtonum(NULL, value, &ashift) != 0) {
+				(void) fprintf(stderr,
+				    gettext("ashift must be a number.\n"));
+				return (NULL);
+			}
+			if (ashift != 0 &&
+			    (ashift < ASHIFT_MIN || ashift > ASHIFT_MAX)) {
+				(void) fprintf(stderr,
+				    gettext("invalid 'ashift=%" PRIu64 "' "
+				    "property: only values between %" PRId32 " "
+				    "and %" PRId32 " are allowed.\n"),
+				    ashift, ASHIFT_MIN, ASHIFT_MAX);
+				return (NULL);
+			}
+		}
+	}
 
 	top = NULL;
 	toplevels = 0;
@@ -1612,9 +1615,9 @@ construct_spec(nvlist_t *props, int argc, char **argv)
 				    children * sizeof (nvlist_t *));
 				if (child == NULL)
 					zpool_no_memory();
-				if ((nv = make_leaf_vdev(props, argv[c],
+				if ((nv = make_leaf_vdev(argv[c],
 				    !(is_log || is_special || is_dedup ||
-				    is_spare))) == NULL) {
+				    is_spare), ashift)) == NULL) {
 					for (c = 0; c < children - 1; c++)
 						nvlist_free(child[c]);
 					free(child);
@@ -1678,6 +1681,10 @@ construct_spec(nvlist_t *props, int argc, char **argv)
 					    ZPOOL_CONFIG_ALLOCATION_BIAS,
 					    VDEV_ALLOC_BIAS_DEDUP) == 0);
 				}
+				if (ashift > 0) {
+					fnvlist_add_uint64(nv,
+					    ZPOOL_CONFIG_ASHIFT, ashift);
+				}
 				if (strcmp(type, VDEV_TYPE_RAIDZ) == 0) {
 					verify(nvlist_add_uint64(nv,
 					    ZPOOL_CONFIG_NPARITY,
@@ -1705,8 +1712,9 @@ construct_spec(nvlist_t *props, int argc, char **argv)
 			 * We have a device.  Pass off to make_leaf_vdev() to
 			 * construct the appropriate nvlist describing the vdev.
 			 */
-			if ((nv = make_leaf_vdev(props, argv[0], !(is_log ||
-			    is_special || is_dedup || is_spare))) == NULL)
+			if ((nv = make_leaf_vdev(argv[0], !(is_log ||
+			    is_special || is_dedup || is_spare),
+			    ashift)) == NULL)
 				goto spec_out;
 
 			verify(nvlist_add_uint64(nv,
diff --git a/sys/contrib/openzfs/cmd/zstream/Makefile.am b/sys/contrib/openzfs/cmd/zstream/Makefile.am
index be3539fe905d..80ef1ea7ca11 100644
--- a/sys/contrib/openzfs/cmd/zstream/Makefile.am
+++ b/sys/contrib/openzfs/cmd/zstream/Makefile.am
@@ -18,6 +18,7 @@ zstream_LDADD = \
 	libzpool.la \
 	libnvpair.la
 
-PHONY += install-exec-hook
-install-exec-hook:
+cmd-zstream-install-exec-hook:
 	cd $(DESTDIR)$(sbindir) && $(LN_S) -f zstream zstreamdump
+
+INSTALL_EXEC_HOOKS += cmd-zstream-install-exec-hook
diff --git a/sys/contrib/openzfs/cmd/ztest.c b/sys/contrib/openzfs/cmd/ztest.c
index 2e88ae3e7994..89752dcb0f0f 100644
--- a/sys/contrib/openzfs/cmd/ztest.c
+++ b/sys/contrib/openzfs/cmd/ztest.c
@@ -273,7 +273,6 @@ extern int zfs_compressed_arc_enabled;
 extern int zfs_abd_scatter_enabled;
 extern uint_t dmu_object_alloc_chunk_shift;
 extern boolean_t zfs_force_some_double_word_sm_entries;
-extern unsigned long zio_decompress_fail_fraction;
 extern unsigned long zfs_reconstruct_indirect_damage_fraction;
 extern uint64_t raidz_expand_max_reflow_bytes;
 extern uint_t raidz_expand_pause_point;
@@ -829,8 +828,8 @@ static char *short_opts = NULL;
 static void
 init_options(void)
 {
-	ASSERT3P(long_opts, ==, NULL);
-	ASSERT3P(short_opts, ==, NULL);
+	ASSERT0P(long_opts);
+	ASSERT0P(short_opts);
 
 	int count = sizeof (option_table) / sizeof (option_table[0]);
 	long_opts = umem_alloc(sizeof (struct option) * count, UMEM_NOFAIL);
@@ -1686,7 +1685,7 @@ ztest_rll_init(rll_t *rll)
 static void
 ztest_rll_destroy(rll_t *rll)
 {
-	ASSERT3P(rll->rll_writer, ==, NULL);
+	ASSERT0P(rll->rll_writer);
 	ASSERT0(rll->rll_readers);
 	mutex_destroy(&rll->rll_lock);
 	cv_destroy(&rll->rll_cv);
@@ -1720,7 +1719,7 @@ ztest_rll_unlock(rll_t *rll)
 		rll->rll_writer = NULL;
 	} else {
 		ASSERT3S(rll->rll_readers, >, 0);
-		ASSERT3P(rll->rll_writer, ==, NULL);
+		ASSERT0P(rll->rll_writer);
 		rll->rll_readers--;
 	}
 
@@ -1996,7 +1995,7 @@ ztest_log_write(ztest_ds_t *zd, dmu_tx_t *tx, lr_write_t *lr)
 	    dmu_read(zd->zd_os, lr->lr_foid, lr->lr_offset, lr->lr_length,
 	    ((lr_write_t *)&itx->itx_lr) + 1, DMU_READ_NO_PREFETCH |
 	    DMU_KEEP_CACHING) != 0) {
-		zil_itx_destroy(itx);
+		zil_itx_destroy(itx, 0);
 		itx = zil_itx_create(TX_WRITE, sizeof (*lr));
 		write_state = WR_NEED_COPY;
 	}
@@ -2278,8 +2277,8 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
 
 			ztest_block_tag_t rbt;
 
-			VERIFY(dmu_read(os, lr->lr_foid, offset,
-			    sizeof (rbt), &rbt, flags) == 0);
+			VERIFY0(dmu_read(os, lr->lr_foid, offset,
+			    sizeof (rbt), &rbt, flags));
 			if (rbt.bt_magic == BT_MAGIC) {
 				ztest_bt_verify(&rbt, os, lr->lr_foid, 0,
 				    offset, gen, txg, crtxg);
@@ -2966,7 +2965,7 @@ ztest_zil_commit(ztest_ds_t *zd, uint64_t id)
 
 	(void) pthread_rwlock_rdlock(&zd->zd_zilog_lock);
 
-	zil_commit(zilog, ztest_random(ZTEST_OBJECTS));
+	VERIFY0(zil_commit(zilog, ztest_random(ZTEST_OBJECTS)));
 
 	/*
 	 * Remember the committed values in zd, which is in parent/child
@@ -4007,7 +4006,7 @@ raidz_scratch_verify(void)
 		 * requested by user, but scratch object was not created.
 		 */
 		case RRSS_SCRATCH_NOT_IN_USE:
-			ASSERT3U(offset, ==, 0);
+			ASSERT0(offset);
 			break;
 
 		/*
@@ -5537,8 +5536,8 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 			}
 
 			if (i == 1) {
-				VERIFY(dmu_buf_hold(os, bigobj, off,
-				    FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
+				VERIFY0(dmu_buf_hold(os, bigobj, off,
+				    FTAG, &dbt, DMU_READ_NO_PREFETCH));
 			}
 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
 				VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
@@ -7937,7 +7936,7 @@ ztest_freeze(void)
 	 */
 	while (BP_IS_HOLE(&zd->zd_zilog->zl_header->zh_log)) {
 		ztest_dmu_object_alloc_free(zd, 0);
-		zil_commit(zd->zd_zilog, 0);
+		VERIFY0(zil_commit(zd->zd_zilog, 0));
 	}
 
 	txg_wait_synced(spa_get_dsl(spa), 0);
@@ -7979,7 +7978,7 @@ ztest_freeze(void)
 	/*
 	 * Commit all of the changes we just generated.
 	 */
-	zil_commit(zd->zd_zilog, 0);
+	VERIFY0(zil_commit(zd->zd_zilog, 0));
 	txg_wait_synced(spa_get_dsl(spa), 0);
 
 	/*
@@ -8979,7 +8978,7 @@ main(int argc, char **argv)
 		exit(EXIT_FAILURE);
 	} else {
 		/* children should not be spawned if setting gvars fails */
-		VERIFY3S(err, ==, 0);
+		VERIFY0(err);
 	}
 
 	/* Override location of zpool.cache */
diff --git a/sys/contrib/openzfs/config/Shellcheck.am b/sys/contrib/openzfs/config/Shellcheck.am
index 1ab13516066c..87e6494056cf 100644
--- a/sys/contrib/openzfs/config/Shellcheck.am
+++ b/sys/contrib/openzfs/config/Shellcheck.am
@@ -16,10 +16,14 @@ SHELLCHECK_OPTS         = $(call JUST_SHELLCHECK_OPTS,$(1)) $(call JUST_CHECKBAS
 
 PHONY += shellcheck
 
+shellcheck_verbose = $(shellcheck_verbose_@AM_V@)
+shellcheck_verbose_ = $(shellcheck_verbose_@AM_DEFAULT_V@)
+shellcheck_verbose_0 = @echo SHELLCHECK $(_STGT);
+
 _STGT = $(subst ^,/,$(subst shellcheck-here-,,$@))
 shellcheck-here-%:
 if HAVE_SHELLCHECK
-	shellcheck --format=gcc --enable=all --exclude=SC1090,SC1091,SC2039,SC2250,SC2312,SC2317,SC3043 $$([ -n "$(SHELLCHECK_SHELL)" ] && echo "--shell=$(SHELLCHECK_SHELL)") "$$([ -e "$(_STGT)" ] || echo "$(srcdir)/")$(_STGT)"
+	$(shellcheck_verbose)shellcheck --format=gcc --enable=all --exclude=SC1090,SC1091,SC2039,SC2250,SC2312,SC2317,SC3043 $$([ -n "$(SHELLCHECK_SHELL)" ] && echo "--shell=$(SHELLCHECK_SHELL)") "$$([ -e "$(_STGT)" ] || echo "$(srcdir)/")$(_STGT)"
 else
 	@echo "skipping shellcheck of" $(_STGT) "because shellcheck is not installed"
 endif
@@ -29,11 +33,15 @@ shellcheck: $(SHELLCHECKSCRIPTS) $(call JUST_SHELLCHECK_OPTS,$(SHELLCHECKSCRIPTS
 
 PHONY += checkbashisms
 
+checkbashisms_verbose = $(checkbashisms_verbose_@AM_V@)
+checkbashisms_verbose_ = $(checkbashisms_verbose_@AM_DEFAULT_V@)
+checkbashisms_verbose_0 = @echo CHECKBASHISMS $(_BTGT);
+
 # command -v *is* specified by POSIX and every shell in existence supports it
 _BTGT = $(subst ^,/,$(subst checkbashisms-here-,,$@))
 checkbashisms-here-%:
 if HAVE_CHECKBASHISMS
-	! { [ -n "$(SHELLCHECK_SHELL)" ] && echo '#!/bin/$(SHELLCHECK_SHELL)'; cat "$$([ -e "$(_BTGT)" ] || echo "$(srcdir)/")$(_BTGT)"; } | \
+	$(checkbashisms_verbose)! { [ -n "$(SHELLCHECK_SHELL)" ] && echo '#!/bin/$(SHELLCHECK_SHELL)'; cat "$$([ -e "$(_BTGT)" ] || echo "$(srcdir)/")$(_BTGT)"; } | \
 	checkbashisms -npx 2>&1 | grep -vFe "'command' with option other than -p" -e 'command -v' -e 'any possible bashisms' $(CHECKBASHISMS_IGNORE) >&2
 else
 	@echo "skipping checkbashisms of" $(_BTGT) "because checkbashisms is not installed"
diff --git a/sys/contrib/openzfs/config/always-arch.m4 b/sys/contrib/openzfs/config/always-arch.m4
index 9f413eeddf95..d73b878916cb 100644
--- a/sys/contrib/openzfs/config/always-arch.m4
+++ b/sys/contrib/openzfs/config/always-arch.m4
@@ -34,8 +34,26 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [
 	esac
 
 	AM_CONDITIONAL([TARGET_CPU_AARCH64], test $TARGET_CPU = aarch64)
+	AM_CONDITIONAL([TARGET_CPU_I386],    test $TARGET_CPU = i386)
 	AM_CONDITIONAL([TARGET_CPU_X86_64],  test $TARGET_CPU = x86_64)
 	AM_CONDITIONAL([TARGET_CPU_POWERPC], test $TARGET_CPU = powerpc)
 	AM_CONDITIONAL([TARGET_CPU_SPARC64], test $TARGET_CPU = sparc64)
 	AM_CONDITIONAL([TARGET_CPU_ARM],     test $TARGET_CPU = arm)
 ])
+dnl #
+dnl # Check for conflicting environment variables
+dnl #
+dnl # If ARCH env variable is set up, then kernel Makefile in the /usr/src/kernel
+dnl # can misbehave during the zfs ./configure test of the module compilation.
+AC_DEFUN([ZFS_AC_CONFIG_CHECK_ARCH_VAR], [
+	AC_MSG_CHECKING([for conflicting environment variables])
+	if test -n "$ARCH"; then
+		AC_MSG_RESULT([warning])
+		AC_MSG_WARN(m4_normalize([ARCH environment variable is set to "$ARCH".
+    This can cause build kernel modules support check failure.
+    Please unset it.]))
+	else
+		AC_MSG_RESULT([done])
+	fi
+])
+
diff --git a/sys/contrib/openzfs/config/always-compiler-options.m4 b/sys/contrib/openzfs/config/always-compiler-options.m4
index 6383b12506ee..0e96435e3713 100644
--- a/sys/contrib/openzfs/config/always-compiler-options.m4
+++ b/sys/contrib/openzfs/config/always-compiler-options.m4
@@ -156,6 +156,34 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH], [
 ])
 
 dnl #
+dnl # Check if kernel cc supports -Wno-format-zero-length option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_NO_FORMAT_ZERO_LENGTH], [
+	saved_cc="$CC"
+	AS_IF(
+		[ test -n "$KERNEL_CC" ], [ CC="$KERNEL_CC" ],
+		[ test -n "$KERNEL_LLVM" ], [ CC="clang" ],
+		[ CC="gcc" ]
+	)
+	AC_MSG_CHECKING([whether $CC supports -Wno-format-zero-length])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Werror -Wno-format-zero-length"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		KERNEL_NO_FORMAT_ZERO_LENGTH=-Wno-format-zero-length
+		AC_MSG_RESULT([yes])
+	], [
+		KERNEL_NO_FORMAT_ZERO_LENGTH=
+		AC_MSG_RESULT([no])
+	])
+
+	CC="$saved_cc"
+	CFLAGS="$saved_flags"
+	AC_SUBST([KERNEL_NO_FORMAT_ZERO_LENGTH])
+])
+
+dnl #
 dnl # Check if cc supports -Wno-clobbered option.
 dnl #
 dnl # We actually invoke it with the -Wclobbered option
@@ -182,6 +210,27 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_CLOBBERED], [
 ])
 
 dnl #
+dnl # Check if cc supports -Wno-atomic-alignment option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_ATOMIC_ALIGNMENT], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-atomic-alignment])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Werror -Wno-atomic-alignment"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_ATOMIC_ALIGNMENT=-Wno-atomic-alignment
+		AC_MSG_RESULT([yes])
+	], [
+		NO_ATOMIC_ALIGNMENT=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_ATOMIC_ALIGNMENT])
+])
+
+dnl #
 dnl # Check if cc supports -Wimplicit-fallthrough option.
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_IMPLICIT_FALLTHROUGH], [
@@ -231,20 +280,17 @@ dnl #
 dnl # Check if kernel cc supports -Winfinite-recursion option.
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_INFINITE_RECURSION], [
-	AC_MSG_CHECKING([whether $KERNEL_CC supports -Winfinite-recursion])
-
 	saved_cc="$CC"
+	AS_IF(
+		[ test -n "$KERNEL_CC" ], [ CC="$KERNEL_CC" ],
+		[ test -n "$KERNEL_LLVM" ], [ CC="clang" ],
+		[ CC="gcc" ]
+	)
+	AC_MSG_CHECKING([whether $CC supports -Winfinite-recursion])
+
 	saved_flags="$CFLAGS"
-	CC="gcc"
 	CFLAGS="$CFLAGS -Werror -Winfinite-recursion"
 
-	AS_IF([ test -n "$KERNEL_CC" ], [
-		CC="$KERNEL_CC"
-	])
-	AS_IF([ test -n "$KERNEL_LLVM" ], [
-		CC="clang"
-	])
-
 	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
 		KERNEL_INFINITE_RECURSION=-Winfinite-recursion
 		AC_DEFINE([HAVE_KERNEL_INFINITE_RECURSION], 1,
@@ -329,20 +375,17 @@ dnl #
 dnl # Check if kernel cc supports -fno-ipa-sra option.
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_NO_IPA_SRA], [
-	AC_MSG_CHECKING([whether $KERNEL_CC supports -fno-ipa-sra])
-
 	saved_cc="$CC"
+	AS_IF(
+		[ test -n "$KERNEL_CC" ], [ CC="$KERNEL_CC" ],
+		[ test -n "$KERNEL_LLVM" ], [ CC="clang" ],
+		[ CC="gcc" ]
+	)
+	AC_MSG_CHECKING([whether $CC supports -fno-ipa-sra])
+
 	saved_flags="$CFLAGS"
-	CC="gcc"
 	CFLAGS="$CFLAGS -Werror -fno-ipa-sra"
 
-	AS_IF([ test -n "$KERNEL_CC" ], [
-		CC="$KERNEL_CC"
-	])
-	AS_IF([ test -n "$KERNEL_LLVM" ], [
-		CC="clang"
-	])
-
 	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
 		KERNEL_NO_IPA_SRA=-fno-ipa-sra
 		AC_MSG_RESULT([yes])
diff --git a/sys/contrib/openzfs/config/kernel-blkdev.m4 b/sys/contrib/openzfs/config/kernel-blkdev.m4
index 83190c6fbe3f..02011bf39fb2 100644
--- a/sys/contrib/openzfs/config/kernel-blkdev.m4
+++ b/sys/contrib/openzfs/config/kernel-blkdev.m4
@@ -29,9 +29,8 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH_4ARG], [
 		const char *path = "path";
 		fmode_t mode = 0;
 		void *holder = NULL;
-		struct blk_holder_ops h;
 
-		bdev = blkdev_get_by_path(path, mode, holder, &h);
+		bdev = blkdev_get_by_path(path, mode, holder, NULL);
 	])
 ])
 
@@ -48,9 +47,8 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_OPEN_BY_PATH], [
 		const char *path = "path";
 		fmode_t mode = 0;
 		void *holder = NULL;
-		struct blk_holder_ops h;
 
-		bdh = bdev_open_by_path(path, mode, holder, &h);
+		bdh = bdev_open_by_path(path, mode, holder, NULL);
 	])
 ])
 
@@ -68,9 +66,8 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BDEV_FILE_OPEN_BY_PATH], [
 		const char *path = "path";
 		fmode_t mode = 0;
 		void *holder = NULL;
-		struct blk_holder_ops h;
 
-		file = bdev_file_open_by_path(path, mode, holder, &h);
+		file = bdev_file_open_by_path(path, mode, holder, NULL);
 	])
 ])
 
diff --git a/sys/contrib/openzfs/config/kernel-block-device-operations.m4 b/sys/contrib/openzfs/config/kernel-block-device-operations.m4
index 4ff20b9c413d..1905340a9c7d 100644
--- a/sys/contrib/openzfs/config/kernel-block-device-operations.m4
+++ b/sys/contrib/openzfs/config/kernel-block-device-operations.m4
@@ -119,15 +119,49 @@ AC_DEFUN([ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK], [
 	])
 ])
 
+dnl #
+dnl # 6.18 API change
+dnl # block_device_operation->getgeo takes struct gendisk* as first arg
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_GETGEO_GENDISK], [
+	ZFS_LINUX_TEST_SRC([block_device_operations_getgeo_gendisk], [
+		#include <linux/blkdev.h>
+
+		static int blk_getgeo(struct gendisk *disk, struct hd_geometry *geo)
+		{
+			(void) disk, (void) geo;
+			return (0);
+		}
+
+		static const struct block_device_operations
+		    bops __attribute__ ((unused)) = {
+			.getgeo	= blk_getgeo,
+		};
+	], [], [])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_GETGEO_GENDISK], [
+	AC_MSG_CHECKING([whether bops->getgeo() takes gendisk as first arg])
+	ZFS_LINUX_TEST_RESULT([block_device_operations_getgeo_gendisk], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE([HAVE_BLOCK_DEVICE_OPERATIONS_GETGEO_GENDISK], [1],
+			[Define if getgeo() in block_device_operations takes struct gendisk * as its first arg])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS], [
 	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_CHECK_EVENTS
 	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_RELEASE_VOID
 	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG
 	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
+	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_GETGEO_GENDISK
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS], [
 	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_CHECK_EVENTS
 	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_RELEASE_VOID
 	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
+	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_GETGEO_GENDISK
 ])
diff --git a/sys/contrib/openzfs/config/kernel-dentry-operations.m4 b/sys/contrib/openzfs/config/kernel-dentry-operations.m4
index aa5a9f2aff39..ce0e6e5be959 100644
--- a/sys/contrib/openzfs/config/kernel-dentry-operations.m4
+++ b/sys/contrib/openzfs/config/kernel-dentry-operations.m4
@@ -24,6 +24,9 @@ dnl #
 dnl # 2.6.38 API change
 dnl # Added d_set_d_op() helper function.
 dnl #
+dnl # 6.17 API change
+dnl # d_set_d_op() removed. No direct replacement.
+dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_D_SET_D_OP], [
 	ZFS_LINUX_TEST_SRC([d_set_d_op], [
 		#include <linux/dcache.h>
@@ -34,22 +37,46 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_D_SET_D_OP], [
 
 AC_DEFUN([ZFS_AC_KERNEL_D_SET_D_OP], [
 	AC_MSG_CHECKING([whether d_set_d_op() is available])
-	ZFS_LINUX_TEST_RESULT_SYMBOL([d_set_d_op],
-	    [d_set_d_op], [fs/dcache.c], [
+	ZFS_LINUX_TEST_RESULT([d_set_d_op], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_D_SET_D_OP, 1,
+		    [Define if d_set_d_op() is available])
+	], [
+		AC_MSG_RESULT(no)
+	])
+])
+
+dnl #
+dnl # 6.17 API change
+dnl # sb->s_d_op removed; set_default_d_op(sb, dop) added
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SET_DEFAULT_D_OP], [
+	ZFS_LINUX_TEST_SRC([set_default_d_op], [
+		#include <linux/dcache.h>
+	], [
+		set_default_d_op(NULL, NULL);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SET_DEFAULT_D_OP], [
+	AC_MSG_CHECKING([whether set_default_d_op() is available])
+	ZFS_LINUX_TEST_RESULT([set_default_d_op], [
 		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_SET_DEFAULT_D_OP, 1,
+		    [Define if set_default_d_op() is available])
 	], [
-		ZFS_LINUX_TEST_ERROR([d_set_d_op])
+		AC_MSG_RESULT(no)
 	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_SRC_DENTRY], [
         ZFS_AC_KERNEL_SRC_D_OBTAIN_ALIAS
         ZFS_AC_KERNEL_SRC_D_SET_D_OP
-        ZFS_AC_KERNEL_SRC_S_D_OP
+        ZFS_AC_KERNEL_SRC_SET_DEFAULT_D_OP
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_DENTRY], [
         ZFS_AC_KERNEL_D_OBTAIN_ALIAS
         ZFS_AC_KERNEL_D_SET_D_OP
-        ZFS_AC_KERNEL_S_D_OP
+        ZFS_AC_KERNEL_SET_DEFAULT_D_OP
 ])
diff --git a/sys/contrib/openzfs/config/kernel-drop-inode.m4 b/sys/contrib/openzfs/config/kernel-drop-inode.m4
new file mode 100644
index 000000000000..6f2b12cadc02
--- /dev/null
+++ b/sys/contrib/openzfs/config/kernel-drop-inode.m4
@@ -0,0 +1,24 @@
+dnl #
+dnl # 6.18 API change
+dnl # - generic_drop_inode() renamed to inode_generic_drop()
+dnl # - generic_delete_inode() renamed to inode_just_drop()
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_GENERIC_DROP], [
+	ZFS_LINUX_TEST_SRC([inode_generic_drop], [
+		#include <linux/fs.h>
+	],[
+		struct inode *ip = NULL;
+		inode_generic_drop(ip);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_INODE_GENERIC_DROP], [
+	AC_MSG_CHECKING([whether inode_generic_drop() exists])
+	ZFS_LINUX_TEST_RESULT([inode_generic_drop], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_INODE_GENERIC_DROP, 1,
+			[inode_generic_drop() exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
diff --git a/sys/contrib/openzfs/config/kernel-mkdir.m4 b/sys/contrib/openzfs/config/kernel-mkdir.m4
index c1aebc387abe..78b32447c593 100644
--- a/sys/contrib/openzfs/config/kernel-mkdir.m4
+++ b/sys/contrib/openzfs/config/kernel-mkdir.m4
@@ -84,6 +84,8 @@ AC_DEFUN([ZFS_AC_KERNEL_MKDIR], [
 		AC_DEFINE(HAVE_IOPS_MKDIR_DENTRY, 1,
 		    [iops->mkdir() returns struct dentry*])
 	],[
+		AC_MSG_RESULT(no)
+
 		dnl #
 		dnl # 6.3 API change
 		dnl # mkdir() takes struct mnt_idmap * as the first arg
diff --git a/sys/contrib/openzfs/config/kernel-namespace.m4 b/sys/contrib/openzfs/config/kernel-namespace.m4
new file mode 100644
index 000000000000..9b0b12e4eab4
--- /dev/null
+++ b/sys/contrib/openzfs/config/kernel-namespace.m4
@@ -0,0 +1,31 @@
+dnl #
+dnl # 6.18 API change
+dnl # ns->ops->type was moved to ns->ns.ns_type (struct ns_common)
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_NS_COMMON_TYPE], [
+	ZFS_LINUX_TEST_SRC([ns_common_type], [
+		#include <linux/user_namespace.h>
+	],[
+		struct user_namespace ns;
+		ns.ns.ns_type = 0;
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_NS_COMMON_TYPE], [
+	AC_MSG_CHECKING([whether ns_type is accessible through ns_common])
+	ZFS_LINUX_TEST_RESULT([ns_common_type], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE([HAVE_NS_COMMON_TYPE], 1,
+			[Define if ns_type is accessible through ns_common])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_NAMESPACE], [
+	ZFS_AC_KERNEL_SRC_NS_COMMON_TYPE
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_NAMESPACE], [
+	ZFS_AC_KERNEL_NS_COMMON_TYPE
+])
diff --git a/sys/contrib/openzfs/config/kernel-userns-capabilities.m4 b/sys/contrib/openzfs/config/kernel-userns-capabilities.m4
deleted file mode 100644
index f4e24fb1606a..000000000000
--- a/sys/contrib/openzfs/config/kernel-userns-capabilities.m4
+++ /dev/null
@@ -1,79 +0,0 @@
-dnl #
-dnl # 2.6.38 API change
-dnl # ns_capable() was introduced
-dnl #
-AC_DEFUN([ZFS_AC_KERNEL_SRC_NS_CAPABLE], [
-	ZFS_LINUX_TEST_SRC([ns_capable], [
-		#include <linux/capability.h>
-	],[
-		ns_capable((struct user_namespace *)NULL, CAP_SYS_ADMIN);
-	])
-])
-
-AC_DEFUN([ZFS_AC_KERNEL_NS_CAPABLE], [
-	AC_MSG_CHECKING([whether ns_capable exists])
-	ZFS_LINUX_TEST_RESULT([ns_capable], [
-		AC_MSG_RESULT(yes)
-	],[
-		ZFS_LINUX_TEST_ERROR([ns_capable()])
-	])
-])
-
-dnl #
-dnl # 2.6.39 API change
-dnl # struct user_namespace was added to struct cred_t as cred->user_ns member
-dnl #
-AC_DEFUN([ZFS_AC_KERNEL_SRC_CRED_USER_NS], [
-	ZFS_LINUX_TEST_SRC([cred_user_ns], [
-		#include <linux/cred.h>
-	],[
-		struct cred cr;
-		cr.user_ns = (struct user_namespace *)NULL;
-	])
-])
-
-AC_DEFUN([ZFS_AC_KERNEL_CRED_USER_NS], [
-	AC_MSG_CHECKING([whether cred_t->user_ns exists])
-	ZFS_LINUX_TEST_RESULT([cred_user_ns], [
-		AC_MSG_RESULT(yes)
-	],[
-		ZFS_LINUX_TEST_ERROR([cred_t->user_ns()])
-	])
-])
-
-dnl #
-dnl # 3.4 API change
-dnl # kuid_has_mapping() and kgid_has_mapping() were added to distinguish
-dnl # between internal kernel uids/gids and user namespace uids/gids.
-dnl #
-AC_DEFUN([ZFS_AC_KERNEL_SRC_KUID_HAS_MAPPING], [
-	ZFS_LINUX_TEST_SRC([kuid_has_mapping], [
-		#include <linux/uidgid.h>
-	],[
-		kuid_has_mapping((struct user_namespace *)NULL, KUIDT_INIT(0));
-		kgid_has_mapping((struct user_namespace *)NULL, KGIDT_INIT(0));
-	])
-])
-
-AC_DEFUN([ZFS_AC_KERNEL_KUID_HAS_MAPPING], [
-	AC_MSG_CHECKING([whether kuid_has_mapping/kgid_has_mapping exist])
-	ZFS_LINUX_TEST_RESULT([kuid_has_mapping], [
-		AC_MSG_RESULT(yes)
-	],[
-		ZFS_LINUX_TEST_ERROR([kuid_has_mapping()])
-	])
-])
-
-AC_DEFUN([ZFS_AC_KERNEL_SRC_USERNS_CAPABILITIES], [
-	ZFS_AC_KERNEL_SRC_NS_CAPABLE
-	ZFS_AC_KERNEL_SRC_HAS_CAPABILITY
-	ZFS_AC_KERNEL_SRC_CRED_USER_NS
-	ZFS_AC_KERNEL_SRC_KUID_HAS_MAPPING
-])
-
-AC_DEFUN([ZFS_AC_KERNEL_USERNS_CAPABILITIES], [
-	ZFS_AC_KERNEL_NS_CAPABLE
-	ZFS_AC_KERNEL_HAS_CAPABILITY
-	ZFS_AC_KERNEL_CRED_USER_NS
-	ZFS_AC_KERNEL_KUID_HAS_MAPPING
-])
diff --git a/sys/contrib/openzfs/config/kernel-writeback.m4 b/sys/contrib/openzfs/config/kernel-writeback.m4
new file mode 100644
index 000000000000..334d65ef84b6
--- /dev/null
+++ b/sys/contrib/openzfs/config/kernel-writeback.m4
@@ -0,0 +1,58 @@
+AC_DEFUN([ZFS_AC_KERNEL_SRC_WRITEPAGE_T], [
+	dnl #
+	dnl # 6.3 API change
+	dnl # The writepage_t function type now has its first argument as
+	dnl # struct folio* instead of struct page*
+	dnl #
+	ZFS_LINUX_TEST_SRC([writepage_t_folio], [
+		#include <linux/writeback.h>
+		static int putpage(struct folio *folio,
+		    struct writeback_control *wbc, void *data)
+		{ return 0; }
+		writepage_t func = putpage;
+	],[])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_WRITEPAGE_T], [
+	AC_MSG_CHECKING([whether int (*writepage_t)() takes struct folio*])
+	ZFS_LINUX_TEST_RESULT([writepage_t_folio], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_WRITEPAGE_T_FOLIO, 1,
+		   [int (*writepage_t)() takes struct folio*])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_WRITE_CACHE_PAGES], [
+	dnl #
+	dnl # 6.18 API change
+	dnl # write_cache_pages() has been removed.
+	dnl #
+	ZFS_LINUX_TEST_SRC([write_cache_pages], [
+		#include <linux/writeback.h>
+	], [
+		(void) write_cache_pages(NULL, NULL, NULL, NULL);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_WRITE_CACHE_PAGES], [
+	AC_MSG_CHECKING([whether write_cache_pages() is available])
+	ZFS_LINUX_TEST_RESULT([write_cache_pages], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_WRITE_CACHE_PAGES, 1,
+		    [write_cache_pages() is available])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_WRITEBACK], [
+	ZFS_AC_KERNEL_SRC_WRITEPAGE_T
+	ZFS_AC_KERNEL_SRC_WRITE_CACHE_PAGES
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_WRITEBACK], [
+	ZFS_AC_KERNEL_WRITEPAGE_T
+	ZFS_AC_KERNEL_WRITE_CACHE_PAGES
+])
diff --git a/sys/contrib/openzfs/config/kernel-writepage_t.m4 b/sys/contrib/openzfs/config/kernel-writepage_t.m4
deleted file mode 100644
index a82cf370c9d4..000000000000
--- a/sys/contrib/openzfs/config/kernel-writepage_t.m4
+++ /dev/null
@@ -1,26 +0,0 @@
-AC_DEFUN([ZFS_AC_KERNEL_SRC_WRITEPAGE_T], [
-	dnl #
-	dnl # 6.3 API change
-	dnl # The writepage_t function type now has its first argument as
-	dnl # struct folio* instead of struct page*
-	dnl #
-	ZFS_LINUX_TEST_SRC([writepage_t_folio], [
-		#include <linux/writeback.h>
-		static int putpage(struct folio *folio,
-		    struct writeback_control *wbc, void *data)
-		{ return 0; }
-		writepage_t func = putpage;
-	],[])
-])
-
-AC_DEFUN([ZFS_AC_KERNEL_WRITEPAGE_T], [
-	AC_MSG_CHECKING([whether int (*writepage_t)() takes struct folio*])
-	ZFS_LINUX_TEST_RESULT([writepage_t_folio], [
-		AC_MSG_RESULT(yes)
-		AC_DEFINE(HAVE_WRITEPAGE_T_FOLIO, 1,
-		   [int (*writepage_t)() takes struct folio*])
-	],[
-		AC_MSG_RESULT(no)
-	])
-])
-
diff --git a/sys/contrib/openzfs/config/kernel.m4 b/sys/contrib/openzfs/config/kernel.m4
index e3e7625db7d8..40b7de739882 100644
--- a/sys/contrib/openzfs/config/kernel.m4
+++ b/sys/contrib/openzfs/config/kernel.m4
@@ -70,6 +70,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_COMMIT_METADATA
 	ZFS_AC_KERNEL_SRC_SETATTR_PREPARE
 	ZFS_AC_KERNEL_SRC_INSERT_INODE_LOCKED
+	ZFS_AC_KERNEL_SRC_DENTRY
 	ZFS_AC_KERNEL_SRC_TRUNCATE_SETSIZE
 	ZFS_AC_KERNEL_SRC_SECURITY_INODE
 	ZFS_AC_KERNEL_SRC_FST_MOUNT
@@ -120,7 +121,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_IDMAP_MNT_API
 	ZFS_AC_KERNEL_SRC_IDMAP_NO_USERNS
 	ZFS_AC_KERNEL_SRC_IATTR_VFSID
-	ZFS_AC_KERNEL_SRC_WRITEPAGE_T
+	ZFS_AC_KERNEL_SRC_WRITEBACK
 	ZFS_AC_KERNEL_SRC_RECLAIMED
 	ZFS_AC_KERNEL_SRC_REGISTER_SYSCTL_TABLE
 	ZFS_AC_KERNEL_SRC_REGISTER_SYSCTL_SZ
@@ -135,6 +136,8 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_TIMER
 	ZFS_AC_KERNEL_SRC_SUPER_BLOCK_S_WB_ERR
 	ZFS_AC_KERNEL_SRC_SOPS_FREE_INODE
+	ZFS_AC_KERNEL_SRC_NAMESPACE
+	ZFS_AC_KERNEL_SRC_INODE_GENERIC_DROP
 	case "$host_cpu" in
 		powerpc*)
 			ZFS_AC_KERNEL_SRC_CPU_HAS_FEATURE
@@ -188,6 +191,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_COMMIT_METADATA
 	ZFS_AC_KERNEL_SETATTR_PREPARE
 	ZFS_AC_KERNEL_INSERT_INODE_LOCKED
+	ZFS_AC_KERNEL_DENTRY
 	ZFS_AC_KERNEL_TRUNCATE_SETSIZE
 	ZFS_AC_KERNEL_SECURITY_INODE
 	ZFS_AC_KERNEL_FST_MOUNT
@@ -238,7 +242,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_IDMAP_MNT_API
 	ZFS_AC_KERNEL_IDMAP_NO_USERNS
 	ZFS_AC_KERNEL_IATTR_VFSID
-	ZFS_AC_KERNEL_WRITEPAGE_T
+	ZFS_AC_KERNEL_WRITEBACK
 	ZFS_AC_KERNEL_RECLAIMED
 	ZFS_AC_KERNEL_REGISTER_SYSCTL_TABLE
 	ZFS_AC_KERNEL_REGISTER_SYSCTL_SZ
@@ -254,6 +258,8 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_TIMER
 	ZFS_AC_KERNEL_SUPER_BLOCK_S_WB_ERR
 	ZFS_AC_KERNEL_SOPS_FREE_INODE
+	ZFS_AC_KERNEL_NAMESPACE
+	ZFS_AC_KERNEL_INODE_GENERIC_DROP
 	case "$host_cpu" in
 		powerpc*)
 			ZFS_AC_KERNEL_CPU_HAS_FEATURE
diff --git a/sys/contrib/openzfs/config/toolchain-simd.m4 b/sys/contrib/openzfs/config/toolchain-simd.m4
index 061576fd94e3..f18c91007cde 100644
--- a/sys/contrib/openzfs/config/toolchain-simd.m4
+++ b/sys/contrib/openzfs/config/toolchain-simd.m4
@@ -24,6 +24,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES
@@ -38,9 +40,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE], [
 	AC_MSG_CHECKING([whether host toolchain supports SSE])
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("xorps %xmm0, %xmm1");
+			return (0);
 		}
 	]])], [
 		AC_DEFINE([HAVE_SSE], 1, [Define if host toolchain supports SSE])
@@ -57,9 +60,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE2], [
 	AC_MSG_CHECKING([whether host toolchain supports SSE2])
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("pxor %xmm0, %xmm1");
+			return (0);
 		}
 	]])], [
 		AC_DEFINE([HAVE_SSE2], 1, [Define if host toolchain supports SSE2])
@@ -76,10 +80,11 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE3], [
 	AC_MSG_CHECKING([whether host toolchain supports SSE3])
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([[
-		void main()
+		int main()
 		{
 			char v[16];
 			__asm__ __volatile__("lddqu %0,%%xmm0" :: "m"(v[0]));
+			return (0);
 		}
 	]])], [
 		AC_DEFINE([HAVE_SSE3], 1, [Define if host toolchain supports SSE3])
@@ -96,9 +101,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSSE3], [
 	AC_MSG_CHECKING([whether host toolchain supports SSSE3])
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("pshufb %xmm0,%xmm1");
+			return (0);
 		}
 	]])], [
 		AC_DEFINE([HAVE_SSSE3], 1, [Define if host toolchain supports SSSE3])
@@ -115,9 +121,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE4_1], [
 	AC_MSG_CHECKING([whether host toolchain supports SSE4.1])
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("pmaxsb %xmm0,%xmm1");
+			return (0);
 		}
 	]])], [
 		AC_DEFINE([HAVE_SSE4_1], 1, [Define if host toolchain supports SSE4.1])
@@ -134,9 +141,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE4_2], [
 	AC_MSG_CHECKING([whether host toolchain supports SSE4.2])
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("pcmpgtq %xmm0, %xmm1");
+			return (0);
 		}
 	]])], [
 		AC_DEFINE([HAVE_SSE4_2], 1, [Define if host toolchain supports SSE4.2])
@@ -153,10 +161,11 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX], [
 	AC_MSG_CHECKING([whether host toolchain supports AVX])
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([[
-		void main()
+		int main()
 		{
 			char v[32];
 			__asm__ __volatile__("vmovdqa %0,%%ymm0" :: "m"(v[0]));
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -174,9 +183,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX2], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vpshufb %ymm0,%ymm1,%ymm2");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -194,9 +204,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512F], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vpandd %zmm0,%zmm1,%zmm2");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -214,9 +225,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512CD], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vplzcntd %zmm0,%zmm1");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -234,9 +246,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512DQ], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vandpd %zmm0,%zmm1,%zmm2");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -254,9 +267,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512BW], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vpshufb %zmm0,%zmm1,%zmm2");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -274,9 +288,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512IFMA], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vpmadd52luq %zmm0,%zmm1,%zmm2");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -294,9 +309,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VBMI], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vpermb %zmm0,%zmm1,%zmm2");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -314,9 +330,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512PF], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vgatherpf0dps (%rsi,%zmm0,4){%k1}");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -334,9 +351,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512ER], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vexp2pd %zmm0,%zmm1");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -354,9 +372,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VL], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("vpabsq %zmm0,%zmm1");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -374,9 +393,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("aesenc %xmm0, %xmm1");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -394,9 +414,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("pclmulqdq %0, %%xmm0, %%xmm1" :: "i"(0));
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -414,9 +435,10 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 			__asm__ __volatile__("movbe 0(%eax), %eax");
+			return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -427,6 +449,48 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [
 ])
 
 dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES], [
+	AC_MSG_CHECKING([whether host toolchain supports VAES])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		int main()
+		{
+			__asm__ __volatile__("vaesenc %ymm0, %ymm1, %ymm0");
+			return (0);
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_VAES], 1, [Define if host toolchain supports VAES])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ], [
+	AC_MSG_CHECKING([whether host toolchain supports VPCLMULQDQ])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		int main()
+		{
+			__asm__ __volatile__("vpclmulqdq %0, %%ymm4, %%ymm3, %%ymm5" :: "i"(0));
+			return (0);
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_VPCLMULQDQ], 1, [Define if host toolchain supports VPCLMULQDQ])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
 dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE], [
@@ -434,10 +498,11 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 		  char b[4096] __attribute__ ((aligned (64)));
 		  __asm__ __volatile__("xsave %[b]\n" : : [b] "m" (*b) : "memory");
+		  return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -455,10 +520,11 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 		  char b[4096] __attribute__ ((aligned (64)));
 		  __asm__ __volatile__("xsaveopt %[b]\n" : : [b] "m" (*b) : "memory");
+		  return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
@@ -476,10 +542,11 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES], [
 
 	AC_LINK_IFELSE([AC_LANG_SOURCE([
 	[
-		void main()
+		int main()
 		{
 		  char b[4096] __attribute__ ((aligned (64)));
 		  __asm__ __volatile__("xsaves %[b]\n" : : [b] "m" (*b) : "memory");
+		  return (0);
 		}
 	]])], [
 		AC_MSG_RESULT([yes])
diff --git a/sys/contrib/openzfs/config/user-statx.m4 b/sys/contrib/openzfs/config/user-statx.m4
index 0315f93e0c20..1ba74a40e9b8 100644
--- a/sys/contrib/openzfs/config/user-statx.m4
+++ b/sys/contrib/openzfs/config/user-statx.m4
@@ -2,7 +2,7 @@ dnl #
 dnl # Check for statx() function and STATX_MNT_ID availability
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_USER_STATX], [
-	AC_CHECK_HEADERS([linux/stat.h],
+	AC_CHECK_HEADERS([sys/stat.h],
 		[have_stat_headers=yes],
 		[have_stat_headers=no])
 
@@ -14,7 +14,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_STATX], [
 			AC_MSG_CHECKING([for STATX_MNT_ID])
 			AC_COMPILE_IFELSE([
 				AC_LANG_PROGRAM([[
-					#include <linux/stat.h>
+					#include <sys/stat.h>
 				]], [[
 					struct statx stx;
 					int mask = STATX_MNT_ID;
@@ -29,6 +29,6 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_STATX], [
 			])
 		])
 	], [
-		AC_MSG_WARN([linux/stat.h not found; skipping statx support])
+		AC_MSG_WARN([sys/stat.h not found; skipping statx support])
 	])
 ])  dnl end AC_DEFUN
diff --git a/sys/contrib/openzfs/config/zfs-build.m4 b/sys/contrib/openzfs/config/zfs-build.m4
index 7cf1b02d8757..161d390466db 100644
--- a/sys/contrib/openzfs/config/zfs-build.m4
+++ b/sys/contrib/openzfs/config/zfs-build.m4
@@ -252,10 +252,12 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_CLOBBERED
 	ZFS_AC_CONFIG_ALWAYS_CC_INFINITE_RECURSION
 	ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_INFINITE_RECURSION
+	ZFS_AC_CONFIG_ALWAYS_CC_NO_ATOMIC_ALIGNMENT
 	ZFS_AC_CONFIG_ALWAYS_CC_IMPLICIT_FALLTHROUGH
 	ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH
+	ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_NO_FORMAT_ZERO_LENGTH
 	ZFS_AC_CONFIG_ALWAYS_CC_FORMAT_OVERFLOW
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_OMIT_FRAME_POINTER
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_IPA_SRA
@@ -265,6 +267,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 	ZFS_AC_CONFIG_ALWAYS_SYSTEM
 	ZFS_AC_CONFIG_ALWAYS_ARCH
+	ZFS_AC_CONFIG_CHECK_ARCH_VAR
 	ZFS_AC_CONFIG_ALWAYS_PYTHON
 	ZFS_AC_CONFIG_ALWAYS_PYZFS
 	ZFS_AC_CONFIG_ALWAYS_SED
diff --git a/sys/contrib/openzfs/contrib/debian/control b/sys/contrib/openzfs/contrib/debian/control
index 96a2bdd88665..c5358dedc0fd 100644
--- a/sys/contrib/openzfs/contrib/debian/control
+++ b/sys/contrib/openzfs/contrib/debian/control
@@ -100,8 +100,8 @@ Depends: ${misc:Depends}, ${shlibs:Depends}
 # The libcurl4 is loaded through dlopen("libcurl.so.4").
 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=988521
 Recommends: libcurl4
-Breaks: libzfs2, libzfs4, libzfs4linux, libzfs6linux
-Replaces: libzfs2, libzfs4, libzfs4linux, libzfs6linux
+Breaks: libzfs2, libzfs4, libzfs4linux, libzfs6linux, openzfs-libzfs4
+Replaces: libzfs2, libzfs4, libzfs4linux, libzfs6linux, openzfs-libzfs4
 Conflicts: libzfs6linux
 Description: OpenZFS filesystem library for Linux - general support
  OpenZFS is a storage platform that encompasses the functionality of
@@ -128,8 +128,8 @@ Package: openzfs-libzpool6
 Section: contrib/libs
 Architecture: linux-any
 Depends: ${misc:Depends}, ${shlibs:Depends}
-Breaks: libzpool2, libzpool5, libzpool5linux, libzpool6linux
-Replaces: libzpool2, libzpool5, libzpool5linux, libzpool6linux
+Breaks: libzpool2, libzpool5, libzpool6linux
+Replaces: libzpool2, libzpool5, libzpool6linux
 Conflicts: libzpool6linux
 Description: OpenZFS pool library for Linux
  OpenZFS is a storage platform that encompasses the functionality of
diff --git a/sys/contrib/openzfs/contrib/debian/copyright b/sys/contrib/openzfs/contrib/debian/copyright
index 65c7d209d8eb..006f32fdf924 100644
--- a/sys/contrib/openzfs/contrib/debian/copyright
+++ b/sys/contrib/openzfs/contrib/debian/copyright
@@ -4,7 +4,7 @@ The detailed contributor information can be found in [2][3].
 
 Files: contrib/debian/*
 Copyright:
- 2013-2016, Aron Xu <aron@debian.org>
+ 2013-2025, Aron Xu <aron@debian.org>
  2016, Petter Reinholdtsen <pere@hungry.com>
  2013, Carlos Alberto Lopez Perez <clopez@igalia.com>
  2013, Turbo Fredriksson <turbo@bayour.com>
@@ -12,6 +12,8 @@ Copyright:
  2011-2013, Darik Horn <dajhorn@vanadac.com>
  2018-2019, Mo Zhou <cdluminate@gmail.com>
  2018-2020, Mo Zhou <lumin@debian.org>
+ 2023-2024, Shengqi Chen <harry-chen@outlook.com>
+ 2024-2025, Shengqi Chen <harry@debian.org>
 License: GPL-2+
 
 [1] https://tracker.debian.org/pkg/zfs-linux
diff --git a/sys/contrib/openzfs/contrib/debian/not-installed b/sys/contrib/openzfs/contrib/debian/not-installed
index 88557f76fcae..9c08da5a6a7b 100644
--- a/sys/contrib/openzfs/contrib/debian/not-installed
+++ b/sys/contrib/openzfs/contrib/debian/not-installed
@@ -1,4 +1,4 @@
-usr/bin/arc_summary.py
+usr/bin/zarcsummary.py
 usr/share/zfs/zfs-helpers.sh
 etc/default/zfs
 etc/init.d
@@ -9,4 +9,4 @@ etc/zfs/vdev_id.conf.sas_direct.example
 etc/zfs/vdev_id.conf.sas_switch.example
 etc/zfs/vdev_id.conf.scsi.example
 etc/zfs/zfs-functions
-lib/systemd/system/zfs-import.service
+usr/lib/systemd/system/zfs-import.service
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libnvpair3.install.in b/sys/contrib/openzfs/contrib/debian/openzfs-libnvpair3.install.in
index ed7b541e3607..fce542270dd8 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libnvpair3.install.in
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libnvpair3.install.in
@@ -1 +1 @@
-lib/@DEB_HOST_MULTIARCH@/libnvpair.so.*
+usr/lib/@DEB_HOST_MULTIARCH@/libnvpair.so.*
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.install b/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.install
index c33123f69a8d..bafdebe9bb91 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.install
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.install
@@ -1,2 +1,2 @@
-lib/*/security/pam_zfs_key.so
+usr/lib/*/security/pam_zfs_key.so
 usr/share/pam-configs/zfs_key
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.postinst b/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.postinst
index 03893454eee9..db4db73d6d5a 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.postinst
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libpam-zfs.postinst
@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e
 
-if ! $(ldd "/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security/pam_zfs_key.so" | grep -q "libasan") ; then
+if ! $(ldd "/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security/pam_zfs_key.so" | grep -q "libasan") ; then
 	pam-auth-update --package
 fi
 
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libuutil3.install.in b/sys/contrib/openzfs/contrib/debian/openzfs-libuutil3.install.in
index a197d030d743..bb33386791e1 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libuutil3.install.in
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libuutil3.install.in
@@ -1 +1 @@
-lib/@DEB_HOST_MULTIARCH@/libuutil.so.*
+usr/lib/@DEB_HOST_MULTIARCH@/libuutil.so.*
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libzfs-dev.install.in b/sys/contrib/openzfs/contrib/debian/openzfs-libzfs-dev.install.in
index eaa8c3925e24..5673e2661c6a 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libzfs-dev.install.in
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libzfs-dev.install.in
@@ -1,3 +1,5 @@
-lib/@DEB_HOST_MULTIARCH@/*.a usr/lib/@DEB_HOST_MULTIARCH@
+usr/lib/@DEB_HOST_MULTIARCH@/*.a
+usr/lib/@DEB_HOST_MULTIARCH@/*.so
+usr/lib/@DEB_HOST_MULTIARCH@/pkgconfig
 usr/include
-usr/lib/@DEB_HOST_MULTIARCH@
+
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libzfs6.install.in b/sys/contrib/openzfs/contrib/debian/openzfs-libzfs6.install.in
index 6765aaee59cc..a9054c14cc73 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libzfs6.install.in
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libzfs6.install.in
@@ -1,2 +1,2 @@
-lib/@DEB_HOST_MULTIARCH@/libzfs.so.*
-lib/@DEB_HOST_MULTIARCH@/libzfs_core.so.*
+usr/lib/@DEB_HOST_MULTIARCH@/libzfs.so.*
+usr/lib/@DEB_HOST_MULTIARCH@/libzfs_core.so.*
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libzfsbootenv1.install.in b/sys/contrib/openzfs/contrib/debian/openzfs-libzfsbootenv1.install.in
index 49216742433f..b61b8ab63265 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libzfsbootenv1.install.in
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libzfsbootenv1.install.in
@@ -1 +1 @@
-lib/@DEB_HOST_MULTIARCH@/libzfsbootenv.so.*
+usr/lib/@DEB_HOST_MULTIARCH@/libzfsbootenv.so.*
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-libzpool6.install.in b/sys/contrib/openzfs/contrib/debian/openzfs-libzpool6.install.in
index b9e872df9ba8..0e087a2709b3 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-libzpool6.install.in
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-libzpool6.install.in
@@ -1 +1 @@
-lib/@DEB_HOST_MULTIARCH@/libzpool.so.*
+usr/lib/@DEB_HOST_MULTIARCH@/libzpool.so.*
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-zfs-test.install b/sys/contrib/openzfs/contrib/debian/openzfs-zfs-test.install
index b3afef50dbd4..496cab2ad5e4 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-zfs-test.install
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-zfs-test.install
@@ -1,4 +1,4 @@
-sbin/ztest
+usr/sbin/ztest
 usr/bin/raidz_test
 usr/share/man/man1/raidz_test.1
 usr/share/man/man1/test-runner.1
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-zfs-zed.install b/sys/contrib/openzfs/contrib/debian/openzfs-zfs-zed.install
index a348ba828ee5..30699a8a98da 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-zfs-zed.install
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-zfs-zed.install
@@ -1,5 +1,5 @@
 etc/zfs/zed.d/*
-lib/systemd/system/zfs-zed.service
+usr/lib/systemd/system/zfs-zed.service
 usr/lib/zfs-linux/zed.d/*
 usr/sbin/zed
 usr/share/man/man8/zed.8
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-zfsutils.install b/sys/contrib/openzfs/contrib/debian/openzfs-zfsutils.install
index 37284a78ad18..6810108f2c5d 100644
--- a/sys/contrib/openzfs/contrib/debian/openzfs-zfsutils.install
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-zfsutils.install
@@ -1,48 +1,48 @@
 etc/default/zfs
 etc/zfs/zfs-functions
 etc/zfs/zpool.d/
-lib/systemd/system-generators/
-lib/systemd/system-preset/
-lib/systemd/system/zfs-import-cache.service
-lib/systemd/system/zfs-import-scan.service
-lib/systemd/system/zfs-import.target
-lib/systemd/system/zfs-load-key.service
-lib/systemd/system/zfs-mount.service
-lib/systemd/system/zfs-mount@.service
-lib/systemd/system/zfs-scrub-monthly@.timer
-lib/systemd/system/zfs-scrub-weekly@.timer
-lib/systemd/system/zfs-scrub@.service
-lib/systemd/system/zfs-trim-monthly@.timer
-lib/systemd/system/zfs-trim-weekly@.timer
-lib/systemd/system/zfs-trim@.service
-lib/systemd/system/zfs-share.service
-lib/systemd/system/zfs-volume-wait.service
-lib/systemd/system/zfs-volumes.target
-lib/systemd/system/zfs.target
-lib/udev/
-sbin/fsck.zfs
-sbin/mount.zfs
-sbin/zdb
-sbin/zfs
-sbin/zfs_ids_to_path
-sbin/zgenhostid
-sbin/zhack
-sbin/zinject
-sbin/zpool
-sbin/zstream
-sbin/zstreamdump
+usr/lib/systemd/system-generators/
+usr/lib/systemd/system-preset/
+usr/lib/systemd/system/zfs-import-cache.service
+usr/lib/systemd/system/zfs-import-scan.service
+usr/lib/systemd/system/zfs-import.target
+usr/lib/systemd/system/zfs-load-key.service
+usr/lib/systemd/system/zfs-mount.service
+usr/lib/systemd/system/zfs-mount@.service
+usr/lib/systemd/system/zfs-scrub-monthly@.timer
+usr/lib/systemd/system/zfs-scrub-weekly@.timer
+usr/lib/systemd/system/zfs-scrub@.service
+usr/lib/systemd/system/zfs-trim-monthly@.timer
+usr/lib/systemd/system/zfs-trim-weekly@.timer
+usr/lib/systemd/system/zfs-trim@.service
+usr/lib/systemd/system/zfs-share.service
+usr/lib/systemd/system/zfs-volume-wait.service
+usr/lib/systemd/system/zfs-volumes.target
+usr/lib/systemd/system/zfs.target
+usr/lib/udev/
+usr/sbin/fsck.zfs
+usr/sbin/mount.zfs
+usr/sbin/zdb
+usr/sbin/zfs
+usr/sbin/zfs_ids_to_path
+usr/sbin/zgenhostid
+usr/sbin/zhack
+usr/sbin/zinject
+usr/sbin/zpool
+usr/sbin/zstream
+usr/sbin/zstreamdump
 usr/bin/zvol_wait
-usr/lib/modules-load.d/ lib/
+usr/lib/modules-load.d/
 usr/lib/zfs-linux/zpool.d/
 usr/lib/zfs-linux/zpool_influxdb
 usr/lib/zfs-linux/zfs_prepare_disk
-usr/sbin/arc_summary
-usr/sbin/arcstat
-usr/sbin/dbufstat
-usr/sbin/zilstat
+usr/bin/zarcsummary
+usr/bin/zarcstat
+usr/bin/dbufstat usr/sbin
+usr/bin/zilstat
 usr/share/zfs/compatibility.d/
 usr/share/bash-completion/completions
-usr/share/man/man1/arcstat.1
+usr/share/man/man1/zarcstat.1
 usr/share/man/man1/zhack.1
 usr/share/man/man1/zvol_wait.1
 usr/share/man/man5/
diff --git a/sys/contrib/openzfs/contrib/debian/openzfs-zfsutils.links b/sys/contrib/openzfs/contrib/debian/openzfs-zfsutils.links
new file mode 100644
index 000000000000..54099e6573b0
--- /dev/null
+++ b/sys/contrib/openzfs/contrib/debian/openzfs-zfsutils.links
@@ -0,0 +1,3 @@
+usr/sbin/zfs      usr/bin/zfs
+usr/sbin/zpool    usr/bin/zpool
+usr/lib/zfs-linux/zpool_influxdb  usr/bin/zpool_influxdb
diff --git a/sys/contrib/openzfs/contrib/debian/rules.in b/sys/contrib/openzfs/contrib/debian/rules.in
index 3226d604546c..5087a7e18e16 100755
--- a/sys/contrib/openzfs/contrib/debian/rules.in
+++ b/sys/contrib/openzfs/contrib/debian/rules.in
@@ -37,18 +37,19 @@ override_dh_auto_configure:
 	@# Build the userland, but don't build the kernel modules.
 	dh_auto_configure -- @CFGOPTS@ \
 	  --bindir=/usr/bin \
-	  --sbindir=/sbin \
-	  --libdir=/lib/"$(DEB_HOST_MULTIARCH)" \
-	  --with-udevdir=/lib/udev \
+	  --sbindir=/usr/sbin \
+	  --with-mounthelperdir=/usr/sbin \
+	  --libdir=/usr/lib/"$(DEB_HOST_MULTIARCH)" \
+	  --with-udevdir=/usr/lib/udev \
 	  --with-zfsexecdir=/usr/lib/zfs-linux \
 	  --enable-systemd \
 	  --enable-pyzfs \
 	  --with-python=python3 \
-	  --with-pammoduledir='/lib/$(DEB_HOST_MULTIARCH)/security' \
+	  --with-pammoduledir='/usr/lib/$(DEB_HOST_MULTIARCH)/security' \
 	  --with-pkgconfigdir='/usr/lib/$(DEB_HOST_MULTIARCH)/pkgconfig' \
-	  --with-systemdunitdir=/lib/systemd/system \
-	  --with-systemdpresetdir=/lib/systemd/system-preset \
-	  --with-systemdgeneratordir=/lib/systemd/system-generators \
+	  --with-systemdunitdir=/usr/lib/systemd/system \
+	  --with-systemdpresetdir=/usr/lib/systemd/system-preset \
+	  --with-systemdgeneratordir=/usr/lib/systemd/system-generators \
 	  --with-config=user
 
 	for i in $(wildcard $(CURDIR)/debian/*.install.in) ; do \
@@ -77,23 +78,10 @@ override_dh_auto_install:
 	@# Install the utilities.
 	$(MAKE) install DESTDIR='$(CURDIR)/debian/tmp'
 
-	# Move from bin_dir to /usr/sbin
-	# Remove suffix (.py) as per policy 10.4 - Scripts
-	# https://www.debian.org/doc/debian-policy/ch-files.html#s-scripts
-	mkdir -p '$(CURDIR)/debian/tmp/usr/sbin/'
-	mv '$(CURDIR)/debian/tmp/usr/bin/arc_summary' '$(CURDIR)/debian/tmp/usr/sbin/arc_summary'
-	mv '$(CURDIR)/debian/tmp/usr/bin/arcstat' '$(CURDIR)/debian/tmp/usr/sbin/arcstat'
-	mv '$(CURDIR)/debian/tmp/usr/bin/dbufstat' '$(CURDIR)/debian/tmp/usr/sbin/dbufstat'
-	mv '$(CURDIR)/debian/tmp/usr/bin/zilstat' '$(CURDIR)/debian/tmp/usr/sbin/zilstat'
-
-	@# Zed has dependencies outside of the system root.
-	mv '$(CURDIR)/debian/tmp/sbin/zed' '$(CURDIR)/debian/tmp/usr/sbin/zed'
-	sed -i 's|ExecStart=/sbin/|ExecStart=/usr/sbin/|g' '$(CURDIR)/debian/tmp/lib/systemd/system/zfs-zed.service'
-
 	@# Install the DKMS source.
 	@# We only want the files needed to build the modules
 	install -D -t '$(CURDIR)/debian/tmp/usr/src/$(NAME)-$(DEB_VERSION_UPSTREAM)/scripts' \
-		'$(CURDIR)/scripts/dkms.postbuild'
+		'$(CURDIR)/scripts/dkms.postbuild' '$(CURDIR)/scripts/objtool-wrapper.in'
 	$(foreach file,$(DKMSFILES),mv '$(CURDIR)/$(NAME)-$(DEB_VERSION_UPSTREAM)/$(file)' '$(CURDIR)/debian/tmp/usr/src/$(NAME)-$(DEB_VERSION_UPSTREAM)' || exit 1;)
 
 	@# Only ever build Linux modules
@@ -108,8 +96,8 @@ override_dh_auto_install:
 	@#        - zfs.release$
 	@#  * Takes care of spaces and tabs
 	@#  * Remove reference to ZFS_AC_PACKAGE
-	awk '/^AC_CONFIG_FILES\(\[/,/^\]\)/ {\
-		if ($$0 !~ /^(AC_CONFIG_FILES\(\[([ \t]+)?$$|\]\)([ \t]+)?$$|([ \t]+)?(include\/(Makefile|sys|os\/(Makefile|linux))|module\/|Makefile([ \t]+)?$$|zfs\.release([ \t]+)?$$))/) \
+	awk '/^AC_CONFIG_FILES\(\[/,/\]\)/ {\
+		if ($$0 !~ /^(AC_CONFIG_FILES\(\[([ \t]+)?$$|\]\)([ \t]+)?$$|([ \t]+)?(include\/(Makefile|sys|os\/(Makefile|linux))|module\/|Makefile([ \t]+)?$$|zfs\.release([ \t]+)?$$))|scripts\/objtool-wrapper.*\]\)$$/) \
 		{next} } {print}' \
 		'$(CURDIR)/$(NAME)-$(DEB_VERSION_UPSTREAM)/configure.ac' | sed '/ZFS_AC_PACKAGE/d' > '$(CURDIR)/debian/tmp/usr/src/$(NAME)-$(DEB_VERSION_UPSTREAM)/configure.ac'
 	@# Set "SUBDIRS = module include" for CONFIG_KERNEL and remove SUBDIRS for all other configs.
@@ -131,11 +119,6 @@ override_dh_auto_install:
 	cd '$(CURDIR)/debian/tmp/usr/src/$(NAME)-$(DEB_VERSION_UPSTREAM)'; ./autogen.sh
 	rm -fr '$(CURDIR)/debian/tmp/usr/src/$(NAME)-$(DEB_VERSION_UPSTREAM)/autom4te.cache'
 
-	for i in `ls $(CURDIR)/debian/tmp/lib/$(DEB_HOST_MULTIARCH)/*.so`; do \
-		ln -s '/lib/$(DEB_HOST_MULTIARCH)/'`readlink $${i}` '$(CURDIR)/debian/tmp/usr/lib/$(DEB_HOST_MULTIARCH)/'`basename $${i}`; \
-		rm $${i}; \
-	done
-
 	chmod a-x '$(CURDIR)/debian/tmp/etc/zfs/zfs-functions'
 	chmod a-x '$(CURDIR)/debian/tmp/etc/default/zfs'
 
@@ -159,7 +142,7 @@ override_dh_auto_clean:
 	@if test -e META.orig; then mv META.orig META; fi
 
 override_dh_install:
-	find debian/tmp/lib -name '*.la' -delete
+	find debian/tmp/usr/lib -name '*.la' -delete
 	dh_install
 
 override_dh_missing:
@@ -173,8 +156,8 @@ override_dh_installinit:
 	dh_installinit -R --name zfs-zed
 
 override_dh_installsystemd:
-	mkdir -p debian/openzfs-zfsutils/lib/systemd/system
-	ln -sr /dev/null debian/openzfs-zfsutils/lib/systemd/system/zfs-import.service
+	mkdir -p debian/openzfs-zfsutils/usr/lib/systemd/system
+	ln -sr /dev/null debian/openzfs-zfsutils/usr/lib/systemd/system/zfs-import.service
 	dh_installsystemd --no-stop-on-upgrade -X zfs-zed.service
 	dh_installsystemd --name zfs-zed
 
diff --git a/sys/contrib/openzfs/contrib/debian/tree/zfs-initramfs/usr/share/initramfs-tools/hooks/zdev b/sys/contrib/openzfs/contrib/debian/tree/zfs-initramfs/usr/share/initramfs-tools/hooks/zdev
index 0cf21a4211a8..d4f968aed8f2 100755
--- a/sys/contrib/openzfs/contrib/debian/tree/zfs-initramfs/usr/share/initramfs-tools/hooks/zdev
+++ b/sys/contrib/openzfs/contrib/debian/tree/zfs-initramfs/usr/share/initramfs-tools/hooks/zdev
@@ -5,7 +5,7 @@
 
 PREREQ="udev"
 PREREQ_UDEV_RULES="60-zvol.rules 69-vdev.rules"
-COPY_EXEC_LIST="/lib/udev/zvol_id /lib/udev/vdev_id"
+COPY_EXEC_LIST="/usr/lib/udev/zvol_id /usr/lib/udev/vdev_id"
 
 # Generic result code.
 RC=0
diff --git a/sys/contrib/openzfs/contrib/dracut/90zfs/module-setup.sh.in b/sys/contrib/openzfs/contrib/dracut/90zfs/module-setup.sh.in
index acad468edfd1..130d94c70707 100755
--- a/sys/contrib/openzfs/contrib/dracut/90zfs/module-setup.sh.in
+++ b/sys/contrib/openzfs/contrib/dracut/90zfs/module-setup.sh.in
@@ -16,7 +16,8 @@ depends() {
 }
 
 installkernel() {
-	instmods -c zfs
+	hostonly='' instmods -c zfs
+	instmods mpt3sas virtio_blk
 }
 
 install() {
diff --git a/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/LICENSE b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/LICENSE
new file mode 100644
index 000000000000..04c03a37e0cb
--- /dev/null
+++ b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/LICENSE
@@ -0,0 +1,253 @@
+BoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL
+licensing. Files that are completely new have a Google copyright and an ISC
+license. This license is reproduced at the bottom of this file.
+
+Contributors to BoringSSL are required to follow the CLA rules for Chromium:
+https://cla.developers.google.com/clas
+
+Files in third_party/ have their own licenses, as described therein. The MIT
+license, for third_party/fiat, which, unlike other third_party directories, is
+compiled into non-test libraries, is included below.
+
+The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the
+OpenSSL License and the original SSLeay license apply to the toolkit. See below
+for the actual license texts. Actually both licenses are BSD-style Open Source
+licenses. In case of any license issues related to OpenSSL please contact
+openssl-core@openssl.org.
+
+The following are Google-internal bug numbers where explicit permission from
+some authors is recorded for use of their work. (This is purely for our own
+record keeping.)
+  27287199
+  27287880
+  27287883
+  263291445
+
+
+  OpenSSL License
+  ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2011 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com).  This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ *
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to.  The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ *
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    "This product includes cryptographic software written by
+ *     Eric Young (eay@cryptsoft.com)"
+ *    The word 'cryptographic' can be left out if the rouines from the library
+ *    being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from
+ *    the apps directory (application code) you must include an acknowledgement:
+ *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed.  i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */
+
+
+ISC license used for completely new code in BoringSSL:
+
+/* Copyright 2015 The BoringSSL Authors
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+
+The code in third_party/fiat carries the MIT license:
+
+Copyright (c) 2015-2016 the fiat-crypto authors (see
+https://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS).
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+Licenses for support code
+-------------------------
+
+Parts of the TLS test suite are under the Go license. This code is not included
+in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so
+distributing code linked against BoringSSL does not trigger this license:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+BoringSSL uses the Chromium test infrastructure to run a continuous build,
+trybots etc. The scripts which manage this, and the script for generating build
+metadata, are under the Chromium license. Distributing code linked against
+BoringSSL does not trigger this license.
+
+Copyright 2015 The Chromium Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/README b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/README
new file mode 100644
index 000000000000..aa6fb6d477fa
--- /dev/null
+++ b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/README
@@ -0,0 +1,11 @@
+This directory contains the original BoringSSL [1] GCM x86-64 assembly
+files [2].
+
+The assembler files where then further modified to fit the ICP conventions.
+
+The main purpose to include these files (and the original ones) here, is to
+serve as a reference if upstream changes need to be applied to the files
+included and modified in the ICP.
+
+[1] https://github.com/google/boringssl
+[2] https://github.com/google/boringssl/blob/d5440dd2c2c500ac2d3bba4afec47a054b4d99ae/gen/bcm/aes-gcm-avx2-x86_64-linux.S
diff --git a/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/aes-gcm-avx2-x86_64-linux.S b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/aes-gcm-avx2-x86_64-linux.S
new file mode 100644
index 000000000000..e7327c9de872
--- /dev/null
+++ b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/aes-gcm-avx2-x86_64-linux.S
@@ -0,0 +1,1328 @@
+// SPDX-License-Identifier: Apache-2.0
+// This file is generated from a similarly-named Perl script in the BoringSSL
+// source tree. Do not edit by hand.
+
+#include <openssl/asm_base.h>
+
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__)
+.section	.rodata
+.align	16
+
+
+.Lbswap_mask:
+.quad	0x08090a0b0c0d0e0f, 0x0001020304050607
+
+
+
+
+
+
+
+
+.Lgfpoly:
+.quad	1, 0xc200000000000000
+
+
+.Lgfpoly_and_internal_carrybit:
+.quad	1, 0xc200000000000001
+
+.align	32
+
+.Lctr_pattern:
+.quad	0, 0
+.quad	1, 0
+.Linc_2blocks:
+.quad	2, 0
+.quad	2, 0
+
+.text	
+.globl	gcm_init_vpclmulqdq_avx2
+.hidden gcm_init_vpclmulqdq_avx2
+.type	gcm_init_vpclmulqdq_avx2,@function
+.align	32
+gcm_init_vpclmulqdq_avx2:
+.cfi_startproc	
+
+_CET_ENDBR
+
+
+
+
+
+	vpshufd	$0x4e,(%rsi),%xmm3
+
+
+
+
+
+	vpshufd	$0xd3,%xmm3,%xmm0
+	vpsrad	$31,%xmm0,%xmm0
+	vpaddq	%xmm3,%xmm3,%xmm3
+	vpand	.Lgfpoly_and_internal_carrybit(%rip),%xmm0,%xmm0
+	vpxor	%xmm0,%xmm3,%xmm3
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm6
+
+
+	vpclmulqdq	$0x00,%xmm3,%xmm3,%xmm0
+	vpclmulqdq	$0x01,%xmm3,%xmm3,%xmm1
+	vpclmulqdq	$0x10,%xmm3,%xmm3,%xmm2
+	vpxor	%xmm2,%xmm1,%xmm1
+	vpclmulqdq	$0x01,%xmm0,%xmm6,%xmm2
+	vpshufd	$0x4e,%xmm0,%xmm0
+	vpxor	%xmm0,%xmm1,%xmm1
+	vpxor	%xmm2,%xmm1,%xmm1
+	vpclmulqdq	$0x11,%xmm3,%xmm3,%xmm5
+	vpclmulqdq	$0x01,%xmm1,%xmm6,%xmm0
+	vpshufd	$0x4e,%xmm1,%xmm1
+	vpxor	%xmm1,%xmm5,%xmm5
+	vpxor	%xmm0,%xmm5,%xmm5
+
+
+
+	vinserti128	$1,%xmm3,%ymm5,%ymm3
+	vinserti128	$1,%xmm5,%ymm5,%ymm5
+
+
+	vpclmulqdq	$0x00,%ymm5,%ymm3,%ymm0
+	vpclmulqdq	$0x01,%ymm5,%ymm3,%ymm1
+	vpclmulqdq	$0x10,%ymm5,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm6,%ymm2
+	vpshufd	$0x4e,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x11,%ymm5,%ymm3,%ymm4
+	vpclmulqdq	$0x01,%ymm1,%ymm6,%ymm0
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm4,%ymm4
+	vpxor	%ymm0,%ymm4,%ymm4
+
+
+
+	vmovdqu	%ymm3,96(%rdi)
+	vmovdqu	%ymm4,64(%rdi)
+
+
+
+	vpunpcklqdq	%ymm3,%ymm4,%ymm0
+	vpunpckhqdq	%ymm3,%ymm4,%ymm1
+	vpxor	%ymm1,%ymm0,%ymm0
+	vmovdqu	%ymm0,128+32(%rdi)
+
+
+	vpclmulqdq	$0x00,%ymm5,%ymm4,%ymm0
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm1
+	vpclmulqdq	$0x10,%ymm5,%ymm4,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm6,%ymm2
+	vpshufd	$0x4e,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x11,%ymm5,%ymm4,%ymm3
+	vpclmulqdq	$0x01,%ymm1,%ymm6,%ymm0
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpxor	%ymm0,%ymm3,%ymm3
+
+	vpclmulqdq	$0x00,%ymm5,%ymm3,%ymm0
+	vpclmulqdq	$0x01,%ymm5,%ymm3,%ymm1
+	vpclmulqdq	$0x10,%ymm5,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm6,%ymm2
+	vpshufd	$0x4e,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x11,%ymm5,%ymm3,%ymm4
+	vpclmulqdq	$0x01,%ymm1,%ymm6,%ymm0
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm4,%ymm4
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vmovdqu	%ymm3,32(%rdi)
+	vmovdqu	%ymm4,0(%rdi)
+
+
+
+	vpunpcklqdq	%ymm3,%ymm4,%ymm0
+	vpunpckhqdq	%ymm3,%ymm4,%ymm1
+	vpxor	%ymm1,%ymm0,%ymm0
+	vmovdqu	%ymm0,128(%rdi)
+
+	vzeroupper
+	ret
+
+.cfi_endproc	
+.size	gcm_init_vpclmulqdq_avx2, . - gcm_init_vpclmulqdq_avx2
+.globl	gcm_gmult_vpclmulqdq_avx2
+.hidden gcm_gmult_vpclmulqdq_avx2
+.type	gcm_gmult_vpclmulqdq_avx2,@function
+.align	32
+gcm_gmult_vpclmulqdq_avx2:
+.cfi_startproc	
+
+_CET_ENDBR
+
+
+
+	vmovdqu	(%rdi),%xmm0
+	vmovdqu	.Lbswap_mask(%rip),%xmm1
+	vmovdqu	128-16(%rsi),%xmm2
+	vmovdqu	.Lgfpoly(%rip),%xmm3
+	vpshufb	%xmm1,%xmm0,%xmm0
+
+	vpclmulqdq	$0x00,%xmm2,%xmm0,%xmm4
+	vpclmulqdq	$0x01,%xmm2,%xmm0,%xmm5
+	vpclmulqdq	$0x10,%xmm2,%xmm0,%xmm6
+	vpxor	%xmm6,%xmm5,%xmm5
+	vpclmulqdq	$0x01,%xmm4,%xmm3,%xmm6
+	vpshufd	$0x4e,%xmm4,%xmm4
+	vpxor	%xmm4,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm5,%xmm5
+	vpclmulqdq	$0x11,%xmm2,%xmm0,%xmm0
+	vpclmulqdq	$0x01,%xmm5,%xmm3,%xmm4
+	vpshufd	$0x4e,%xmm5,%xmm5
+	vpxor	%xmm5,%xmm0,%xmm0
+	vpxor	%xmm4,%xmm0,%xmm0
+
+
+	vpshufb	%xmm1,%xmm0,%xmm0
+	vmovdqu	%xmm0,(%rdi)
+	ret
+
+.cfi_endproc	
+.size	gcm_gmult_vpclmulqdq_avx2, . - gcm_gmult_vpclmulqdq_avx2
+.globl	gcm_ghash_vpclmulqdq_avx2
+.hidden gcm_ghash_vpclmulqdq_avx2
+.type	gcm_ghash_vpclmulqdq_avx2,@function
+.align	32
+gcm_ghash_vpclmulqdq_avx2:
+.cfi_startproc	
+
+_CET_ENDBR
+
+
+
+
+
+
+	vmovdqu	.Lbswap_mask(%rip),%xmm6
+	vmovdqu	.Lgfpoly(%rip),%xmm7
+
+
+	vmovdqu	(%rdi),%xmm5
+	vpshufb	%xmm6,%xmm5,%xmm5
+
+
+	cmpq	$32,%rcx
+	jb	.Lghash_lastblock
+
+
+
+	vinserti128	$1,%xmm6,%ymm6,%ymm6
+	vinserti128	$1,%xmm7,%ymm7,%ymm7
+
+	cmpq	$127,%rcx
+	jbe	.Lghash_loop_1x
+
+
+	vmovdqu	128(%rsi),%ymm8
+	vmovdqu	128+32(%rsi),%ymm9
+.Lghash_loop_4x:
+
+	vmovdqu	0(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	0(%rsi),%ymm2
+	vpxor	%ymm5,%ymm1,%ymm1
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x00,%ymm8,%ymm0,%ymm4
+
+	vmovdqu	32(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	32(%rsi),%ymm2
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm3,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x10,%ymm8,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vmovdqu	64(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	64(%rsi),%ymm2
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm3,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x00,%ymm9,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm4,%ymm4
+
+
+	vmovdqu	96(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	96(%rsi),%ymm2
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm3,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x10,%ymm9,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vpxor	%ymm3,%ymm4,%ymm4
+	vpxor	%ymm5,%ymm4,%ymm4
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm2
+	vpclmulqdq	$0x01,%ymm3,%ymm2,%ymm0
+	vpshufd	$0x4e,%ymm3,%ymm3
+	vpxor	%ymm3,%ymm4,%ymm4
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vpclmulqdq	$0x01,%ymm4,%ymm2,%ymm0
+	vpshufd	$0x4e,%ymm4,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpxor	%ymm0,%ymm5,%ymm5
+	vextracti128	$1,%ymm5,%xmm0
+	vpxor	%xmm0,%xmm5,%xmm5
+
+	subq	$-128,%rdx
+	addq	$-128,%rcx
+	cmpq	$127,%rcx
+	ja	.Lghash_loop_4x
+
+
+	cmpq	$32,%rcx
+	jb	.Lghash_loop_1x_done
+.Lghash_loop_1x:
+	vmovdqu	(%rdx),%ymm0
+	vpshufb	%ymm6,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vmovdqu	128-32(%rsi),%ymm0
+	vpclmulqdq	$0x00,%ymm0,%ymm5,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm5,%ymm2
+	vpclmulqdq	$0x10,%ymm0,%ymm5,%ymm3
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x01,%ymm1,%ymm7,%ymm3
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm2,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x11,%ymm0,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm7,%ymm1
+	vpshufd	$0x4e,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpxor	%ymm1,%ymm5,%ymm5
+
+	vextracti128	$1,%ymm5,%xmm0
+	vpxor	%xmm0,%xmm5,%xmm5
+	addq	$32,%rdx
+	subq	$32,%rcx
+	cmpq	$32,%rcx
+	jae	.Lghash_loop_1x
+.Lghash_loop_1x_done:
+
+
+.Lghash_lastblock:
+	testq	%rcx,%rcx
+	jz	.Lghash_done
+	vmovdqu	(%rdx),%xmm0
+	vpshufb	%xmm6,%xmm0,%xmm0
+	vpxor	%xmm0,%xmm5,%xmm5
+	vmovdqu	128-16(%rsi),%xmm0
+	vpclmulqdq	$0x00,%xmm0,%xmm5,%xmm1
+	vpclmulqdq	$0x01,%xmm0,%xmm5,%xmm2
+	vpclmulqdq	$0x10,%xmm0,%xmm5,%xmm3
+	vpxor	%xmm3,%xmm2,%xmm2
+	vpclmulqdq	$0x01,%xmm1,%xmm7,%xmm3
+	vpshufd	$0x4e,%xmm1,%xmm1
+	vpxor	%xmm1,%xmm2,%xmm2
+	vpxor	%xmm3,%xmm2,%xmm2
+	vpclmulqdq	$0x11,%xmm0,%xmm5,%xmm5
+	vpclmulqdq	$0x01,%xmm2,%xmm7,%xmm1
+	vpshufd	$0x4e,%xmm2,%xmm2
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpxor	%xmm1,%xmm5,%xmm5
+
+
+.Lghash_done:
+
+	vpshufb	%xmm6,%xmm5,%xmm5
+	vmovdqu	%xmm5,(%rdi)
+
+	vzeroupper
+	ret
+
+.cfi_endproc	
+.size	gcm_ghash_vpclmulqdq_avx2, . - gcm_ghash_vpclmulqdq_avx2
+.globl	aes_gcm_enc_update_vaes_avx2
+.hidden aes_gcm_enc_update_vaes_avx2
+.type	aes_gcm_enc_update_vaes_avx2,@function
+.align	32
+aes_gcm_enc_update_vaes_avx2:
+.cfi_startproc	
+
+_CET_ENDBR
+	pushq	%r12
+.cfi_adjust_cfa_offset	8
+.cfi_offset	%r12,-16
+
+	movq	16(%rsp),%r12
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern	BORINGSSL_function_hit
+.hidden BORINGSSL_function_hit
+	movb	$1,BORINGSSL_function_hit+8(%rip)
+#endif
+	vbroadcasti128	.Lbswap_mask(%rip),%ymm0
+
+
+
+	vmovdqu	(%r12),%xmm1
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vbroadcasti128	(%r8),%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm11
+
+
+
+	movl	240(%rcx),%r10d
+	leal	-20(,%r10,4),%r10d
+
+
+
+
+	leaq	96(%rcx,%r10,4),%r11
+	vbroadcasti128	(%rcx),%ymm9
+	vbroadcasti128	(%r11),%ymm10
+
+
+	vpaddd	.Lctr_pattern(%rip),%ymm11,%ymm11
+
+
+
+	cmpq	$127,%rdx
+	jbe	.Lcrypt_loop_4x_done__func1
+
+	vmovdqu	128(%r9),%ymm7
+	vmovdqu	128+32(%r9),%ymm8
+
+
+
+	vmovdqu	.Linc_2blocks(%rip),%ymm2
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm14
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm15
+	vpaddd	%ymm2,%ymm11,%ymm11
+
+
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	vpxor	%ymm9,%ymm14,%ymm14
+	vpxor	%ymm9,%ymm15,%ymm15
+
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_first_4_vecs__func1:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_first_4_vecs__func1
+	vpxor	0(%rdi),%ymm10,%ymm2
+	vpxor	32(%rdi),%ymm10,%ymm3
+	vpxor	64(%rdi),%ymm10,%ymm5
+	vpxor	96(%rdi),%ymm10,%ymm6
+	vaesenclast	%ymm2,%ymm12,%ymm12
+	vaesenclast	%ymm3,%ymm13,%ymm13
+	vaesenclast	%ymm5,%ymm14,%ymm14
+	vaesenclast	%ymm6,%ymm15,%ymm15
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+	vmovdqu	%ymm14,64(%rsi)
+	vmovdqu	%ymm15,96(%rsi)
+
+	subq	$-128,%rdi
+	addq	$-128,%rdx
+	cmpq	$127,%rdx
+	jbe	.Lghash_last_ciphertext_4x__func1
+.align	16
+.Lcrypt_loop_4x__func1:
+
+
+
+
+	vmovdqu	.Linc_2blocks(%rip),%ymm2
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm14
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm15
+	vpaddd	%ymm2,%ymm11,%ymm11
+
+
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	vpxor	%ymm9,%ymm14,%ymm14
+	vpxor	%ymm9,%ymm15,%ymm15
+
+	cmpl	$24,%r10d
+	jl	.Laes128__func1
+	je	.Laes192__func1
+
+	vbroadcasti128	-208(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-192(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes192__func1:
+	vbroadcasti128	-176(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-160(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes128__func1:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
+
+	vmovdqu	0(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	0(%r9),%ymm4
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm7,%ymm2,%ymm6
+
+	vbroadcasti128	-144(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vbroadcasti128	-128(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	32(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	32(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm7,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-112(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	64(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	64(%r9),%ymm4
+
+	vbroadcasti128	-96(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-80(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+
+	vmovdqu	96(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+
+	vbroadcasti128	-64(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vmovdqu	96(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-48(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm1,%ymm6,%ymm6
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm4
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-32(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpclmulqdq	$0x01,%ymm6,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-16(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vextracti128	$1,%ymm1,%xmm2
+	vpxor	%xmm2,%xmm1,%xmm1
+
+
+	subq	$-128,%rsi
+	vpxor	0(%rdi),%ymm10,%ymm2
+	vpxor	32(%rdi),%ymm10,%ymm3
+	vpxor	64(%rdi),%ymm10,%ymm5
+	vpxor	96(%rdi),%ymm10,%ymm6
+	vaesenclast	%ymm2,%ymm12,%ymm12
+	vaesenclast	%ymm3,%ymm13,%ymm13
+	vaesenclast	%ymm5,%ymm14,%ymm14
+	vaesenclast	%ymm6,%ymm15,%ymm15
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+	vmovdqu	%ymm14,64(%rsi)
+	vmovdqu	%ymm15,96(%rsi)
+
+	subq	$-128,%rdi
+
+	addq	$-128,%rdx
+	cmpq	$127,%rdx
+	ja	.Lcrypt_loop_4x__func1
+.Lghash_last_ciphertext_4x__func1:
+
+	vmovdqu	0(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	0(%r9),%ymm4
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm7,%ymm2,%ymm6
+
+	vmovdqu	32(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	32(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm7,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vmovdqu	64(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	64(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+
+	vmovdqu	96(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	96(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm1,%ymm6,%ymm6
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm4
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vpclmulqdq	$0x01,%ymm6,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vextracti128	$1,%ymm1,%xmm2
+	vpxor	%xmm2,%xmm1,%xmm1
+
+	subq	$-128,%rsi
+.Lcrypt_loop_4x_done__func1:
+
+	testq	%rdx,%rdx
+	jz	.Ldone__func1
+
+
+
+
+
+	leaq	128(%r9),%r8
+	subq	%rdx,%r8
+
+
+	vpxor	%xmm5,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm6,%xmm6
+	vpxor	%xmm7,%xmm7,%xmm7
+
+	cmpq	$64,%rdx
+	jb	.Llessthan64bytes__func1
+
+
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_1__func1:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_1__func1
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%ymm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%ymm3,%ymm13,%ymm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+
+
+	vpshufb	%ymm0,%ymm12,%ymm12
+	vpshufb	%ymm0,%ymm13,%ymm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%ymm3
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm7
+	vpclmulqdq	$0x00,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+	addq	$64,%r8
+	addq	$64,%rdi
+	addq	$64,%rsi
+	subq	$64,%rdx
+	jz	.Lreduce__func1
+
+	vpxor	%xmm1,%xmm1,%xmm1
+
+
+.Llessthan64bytes__func1:
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_2__func1:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_2__func1
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+
+
+	cmpq	$32,%rdx
+	jb	.Lxor_one_block__func1
+	je	.Lxor_two_blocks__func1
+
+.Lxor_three_blocks__func1:
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%xmm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%xmm3,%xmm13,%xmm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%xmm13,32(%rsi)
+
+	vpshufb	%ymm0,%ymm12,%ymm12
+	vpshufb	%xmm0,%xmm13,%xmm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%xmm3
+	vpclmulqdq	$0x00,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm7,%ymm7
+	jmp	.Lghash_mul_one_vec_unreduced__func1
+
+.Lxor_two_blocks__func1:
+	vmovdqu	(%rdi),%ymm2
+	vpxor	%ymm2,%ymm12,%ymm12
+	vmovdqu	%ymm12,(%rsi)
+	vpshufb	%ymm0,%ymm12,%ymm12
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	jmp	.Lghash_mul_one_vec_unreduced__func1
+
+.Lxor_one_block__func1:
+	vmovdqu	(%rdi),%xmm2
+	vpxor	%xmm2,%xmm12,%xmm12
+	vmovdqu	%xmm12,(%rsi)
+	vpshufb	%xmm0,%xmm12,%xmm12
+	vpxor	%xmm1,%xmm12,%xmm12
+	vmovdqu	(%r8),%xmm2
+
+.Lghash_mul_one_vec_unreduced__func1:
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+.Lreduce__func1:
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm2
+	vpclmulqdq	$0x01,%ymm5,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm3,%ymm6,%ymm6
+	vpclmulqdq	$0x01,%ymm6,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm7,%ymm7
+	vpxor	%ymm3,%ymm7,%ymm7
+	vextracti128	$1,%ymm7,%xmm1
+	vpxor	%xmm7,%xmm1,%xmm1
+
+.Ldone__func1:
+
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vmovdqu	%xmm1,(%r12)
+
+	vzeroupper
+	popq	%r12
+.cfi_adjust_cfa_offset	-8
+.cfi_restore	%r12
+	ret
+
+.cfi_endproc	
+.size	aes_gcm_enc_update_vaes_avx2, . - aes_gcm_enc_update_vaes_avx2
+.globl	aes_gcm_dec_update_vaes_avx2
+.hidden aes_gcm_dec_update_vaes_avx2
+.type	aes_gcm_dec_update_vaes_avx2,@function
+.align	32
+aes_gcm_dec_update_vaes_avx2:
+.cfi_startproc	
+
+_CET_ENDBR
+	pushq	%r12
+.cfi_adjust_cfa_offset	8
+.cfi_offset	%r12,-16
+
+	movq	16(%rsp),%r12
+	vbroadcasti128	.Lbswap_mask(%rip),%ymm0
+
+
+
+	vmovdqu	(%r12),%xmm1
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vbroadcasti128	(%r8),%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm11
+
+
+
+	movl	240(%rcx),%r10d
+	leal	-20(,%r10,4),%r10d
+
+
+
+
+	leaq	96(%rcx,%r10,4),%r11
+	vbroadcasti128	(%rcx),%ymm9
+	vbroadcasti128	(%r11),%ymm10
+
+
+	vpaddd	.Lctr_pattern(%rip),%ymm11,%ymm11
+
+
+
+	cmpq	$127,%rdx
+	jbe	.Lcrypt_loop_4x_done__func2
+
+	vmovdqu	128(%r9),%ymm7
+	vmovdqu	128+32(%r9),%ymm8
+.align	16
+.Lcrypt_loop_4x__func2:
+
+
+
+
+	vmovdqu	.Linc_2blocks(%rip),%ymm2
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm14
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm15
+	vpaddd	%ymm2,%ymm11,%ymm11
+
+
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	vpxor	%ymm9,%ymm14,%ymm14
+	vpxor	%ymm9,%ymm15,%ymm15
+
+	cmpl	$24,%r10d
+	jl	.Laes128__func2
+	je	.Laes192__func2
+
+	vbroadcasti128	-208(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-192(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes192__func2:
+	vbroadcasti128	-176(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-160(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes128__func2:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
+
+	vmovdqu	0(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	0(%r9),%ymm4
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm7,%ymm2,%ymm6
+
+	vbroadcasti128	-144(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vbroadcasti128	-128(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	32(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	32(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm7,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-112(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	64(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	64(%r9),%ymm4
+
+	vbroadcasti128	-96(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-80(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+
+	vmovdqu	96(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+
+	vbroadcasti128	-64(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vmovdqu	96(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-48(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm1,%ymm6,%ymm6
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm4
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-32(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpclmulqdq	$0x01,%ymm6,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-16(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vextracti128	$1,%ymm1,%xmm2
+	vpxor	%xmm2,%xmm1,%xmm1
+
+
+
+	vpxor	0(%rdi),%ymm10,%ymm2
+	vpxor	32(%rdi),%ymm10,%ymm3
+	vpxor	64(%rdi),%ymm10,%ymm5
+	vpxor	96(%rdi),%ymm10,%ymm6
+	vaesenclast	%ymm2,%ymm12,%ymm12
+	vaesenclast	%ymm3,%ymm13,%ymm13
+	vaesenclast	%ymm5,%ymm14,%ymm14
+	vaesenclast	%ymm6,%ymm15,%ymm15
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+	vmovdqu	%ymm14,64(%rsi)
+	vmovdqu	%ymm15,96(%rsi)
+
+	subq	$-128,%rdi
+	subq	$-128,%rsi
+	addq	$-128,%rdx
+	cmpq	$127,%rdx
+	ja	.Lcrypt_loop_4x__func2
+.Lcrypt_loop_4x_done__func2:
+
+	testq	%rdx,%rdx
+	jz	.Ldone__func2
+
+
+
+
+
+	leaq	128(%r9),%r8
+	subq	%rdx,%r8
+
+
+	vpxor	%xmm5,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm6,%xmm6
+	vpxor	%xmm7,%xmm7,%xmm7
+
+	cmpq	$64,%rdx
+	jb	.Llessthan64bytes__func2
+
+
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_1__func2:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_1__func2
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%ymm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%ymm3,%ymm13,%ymm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+
+
+	vpshufb	%ymm0,%ymm2,%ymm12
+	vpshufb	%ymm0,%ymm3,%ymm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%ymm3
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm7
+	vpclmulqdq	$0x00,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+	addq	$64,%r8
+	addq	$64,%rdi
+	addq	$64,%rsi
+	subq	$64,%rdx
+	jz	.Lreduce__func2
+
+	vpxor	%xmm1,%xmm1,%xmm1
+
+
+.Llessthan64bytes__func2:
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_2__func2:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_2__func2
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+
+
+	cmpq	$32,%rdx
+	jb	.Lxor_one_block__func2
+	je	.Lxor_two_blocks__func2
+
+.Lxor_three_blocks__func2:
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%xmm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%xmm3,%xmm13,%xmm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%xmm13,32(%rsi)
+
+	vpshufb	%ymm0,%ymm2,%ymm12
+	vpshufb	%xmm0,%xmm3,%xmm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%xmm3
+	vpclmulqdq	$0x00,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm7,%ymm7
+	jmp	.Lghash_mul_one_vec_unreduced__func2
+
+.Lxor_two_blocks__func2:
+	vmovdqu	(%rdi),%ymm2
+	vpxor	%ymm2,%ymm12,%ymm12
+	vmovdqu	%ymm12,(%rsi)
+	vpshufb	%ymm0,%ymm2,%ymm12
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	jmp	.Lghash_mul_one_vec_unreduced__func2
+
+.Lxor_one_block__func2:
+	vmovdqu	(%rdi),%xmm2
+	vpxor	%xmm2,%xmm12,%xmm12
+	vmovdqu	%xmm12,(%rsi)
+	vpshufb	%xmm0,%xmm2,%xmm12
+	vpxor	%xmm1,%xmm12,%xmm12
+	vmovdqu	(%r8),%xmm2
+
+.Lghash_mul_one_vec_unreduced__func2:
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+.Lreduce__func2:
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm2
+	vpclmulqdq	$0x01,%ymm5,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm3,%ymm6,%ymm6
+	vpclmulqdq	$0x01,%ymm6,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm7,%ymm7
+	vpxor	%ymm3,%ymm7,%ymm7
+	vextracti128	$1,%ymm7,%xmm1
+	vpxor	%xmm7,%xmm1,%xmm1
+
+.Ldone__func2:
+
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vmovdqu	%xmm1,(%r12)
+
+	vzeroupper
+	popq	%r12
+.cfi_adjust_cfa_offset	-8
+.cfi_restore	%r12
+	ret
+
+.cfi_endproc	
+.size	aes_gcm_dec_update_vaes_avx2, . - aes_gcm_dec_update_vaes_avx2
+#endif
diff --git a/sys/contrib/openzfs/contrib/initramfs/hooks/zfsunlock.in b/sys/contrib/openzfs/contrib/initramfs/hooks/zfsunlock.in
index 4776087d9a76..db9bf0e20274 100644
--- a/sys/contrib/openzfs/contrib/initramfs/hooks/zfsunlock.in
+++ b/sys/contrib/openzfs/contrib/initramfs/hooks/zfsunlock.in
@@ -8,3 +8,12 @@ fi
 . /usr/share/initramfs-tools/hook-functions
 
 copy_exec /usr/share/initramfs-tools/zfsunlock /usr/bin/zfsunlock
+
+if [ -f /etc/initramfs-tools/etc/motd ]; then
+	copy_file text /etc/initramfs-tools/etc/motd /etc/motd
+else
+	tmpf=$(mktemp)
+	echo "If you use zfs encrypted root filesystems, you can use \`zfsunlock\` to manually unlock it" > "$tmpf"
+	copy_file text "$tmpf" /etc/motd
+	rm -f "$tmpf"
+fi
diff --git a/sys/contrib/openzfs/contrib/initramfs/scripts/zfs b/sys/contrib/openzfs/contrib/initramfs/scripts/zfs
index c569b2528368..67707e9d80f4 100644
--- a/sys/contrib/openzfs/contrib/initramfs/scripts/zfs
+++ b/sys/contrib/openzfs/contrib/initramfs/scripts/zfs
@@ -979,7 +979,8 @@ mountroot()
 
 	touch /run/zfs_unlock_complete
 	if [ -e /run/zfs_unlock_complete_notify ]; then
-		read -r < /run/zfs_unlock_complete_notify
+		# shellcheck disable=SC2034
+		read -r zfs_unlock_complete_notify < /run/zfs_unlock_complete_notify
 	fi
 
 	# ------------
diff --git a/sys/contrib/openzfs/contrib/intel_qat/readme.md b/sys/contrib/openzfs/contrib/intel_qat/readme.md
index 7e45d395bb80..04c299b6404c 100644
--- a/sys/contrib/openzfs/contrib/intel_qat/readme.md
+++ b/sys/contrib/openzfs/contrib/intel_qat/readme.md
@@ -8,7 +8,7 @@ This contrib contains community compatibility patches to get Intel QAT working o
 These patches are based on the following Intel QAT version:
 [1.7.l.4.10.0-00014](https://01.org/sites/default/files/downloads/qat1.7.l.4.10.0-00014.tar.gz)
 
-When using QAT with above kernels versions, the following patches needs to be applied using:
+When using QAT with the above kernel versions, the following patches need to be applied using:
 patch -p1 < _$PATCH_
 _Where $PATCH refers to the path of the patch in question_
 
diff --git a/sys/contrib/openzfs/contrib/pam_zfs_key/pam_zfs_key.c b/sys/contrib/openzfs/contrib/pam_zfs_key/pam_zfs_key.c
index a0bc172c6f44..88698dedabbc 100644
--- a/sys/contrib/openzfs/contrib/pam_zfs_key/pam_zfs_key.c
+++ b/sys/contrib/openzfs/contrib/pam_zfs_key/pam_zfs_key.c
@@ -391,7 +391,11 @@ static int
 zfs_key_config_load(pam_handle_t *pamh, zfs_key_config_t *config,
     int argc, const char **argv)
 {
+#if	defined(__FreeBSD__)
+	config->homes_prefix = strdup("zroot/home");
+#else
 	config->homes_prefix = strdup("rpool/home");
+#endif
 	if (config->homes_prefix == NULL) {
 		pam_syslog(pamh, LOG_ERR, "strdup failure");
 		return (PAM_SERVICE_ERR);
diff --git a/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/exceptions.py b/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/exceptions.py
index b26a37f5de10..26d66a452726 100644
--- a/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/exceptions.py
+++ b/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/exceptions.py
@@ -604,5 +604,4 @@ class RaidzExpansionRunning(ZFSError):
     errno = ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS
     message = "A raidz device is currently expanding"
 
-
 # vim: softtabstop=4 tabstop=4 expandtab shiftwidth=4
diff --git a/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py b/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py
index 971aa1d0d493..bad1af2d1671 100644
--- a/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py
+++ b/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py
@@ -4223,7 +4223,7 @@ class _TempPool(object):
             self.getRoot().reset()
             return
 
-        # On the Buildbot builders this may fail with "pool is busy"
+        # On the CI builders this may fail with "pool is busy"
         # Retry 5 times before raising an error
         retry = 0
         while True:
diff --git a/sys/contrib/openzfs/etc/init.d/README.md b/sys/contrib/openzfs/etc/init.d/README.md
index da780fdc1222..3852dd9a6b2e 100644
--- a/sys/contrib/openzfs/etc/init.d/README.md
+++ b/sys/contrib/openzfs/etc/init.d/README.md
@@ -1,5 +1,5 @@
 DESCRIPTION
-  These script were written with the primary intention of being portable and
+  These scripts were written with the primary intention of being portable and
   usable on as many systems as possible.
 
   This is, in practice, usually not possible. But the intention is there.
diff --git a/sys/contrib/openzfs/include/libzfs.h b/sys/contrib/openzfs/include/libzfs.h
index 3fcdc176a621..14930fb90622 100644
--- a/sys/contrib/openzfs/include/libzfs.h
+++ b/sys/contrib/openzfs/include/libzfs.h
@@ -479,6 +479,8 @@ _LIBZFS_H zpool_status_t zpool_import_status(nvlist_t *, const char **,
 _LIBZFS_H nvlist_t *zpool_get_config(zpool_handle_t *, nvlist_t **);
 _LIBZFS_H nvlist_t *zpool_get_features(zpool_handle_t *);
 _LIBZFS_H int zpool_refresh_stats(zpool_handle_t *, boolean_t *);
+_LIBZFS_H void zpool_refresh_stats_from_handle(zpool_handle_t *,
+    zpool_handle_t *);
 _LIBZFS_H int zpool_get_errlog(zpool_handle_t *, nvlist_t **);
 _LIBZFS_H void zpool_add_propname(zpool_handle_t *, const char *);
 
diff --git a/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h b/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h
index 974704e92bbd..32bc02f3dc86 100644
--- a/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h
+++ b/sys/contrib/openzfs/include/os/freebsd/spl/sys/debug.h
@@ -69,6 +69,10 @@
 #define	__maybe_unused __attribute__((unused))
 #endif
 
+#ifndef __must_check
+#define	__must_check __attribute__((__warn_unused_result__))
+#endif
+
 /*
  * Without this, we see warnings from objtool during normal Linux builds when
  * the kernel is built with CONFIG_STACK_VALIDATION=y:
diff --git a/sys/contrib/openzfs/include/os/freebsd/spl/sys/proc.h b/sys/contrib/openzfs/include/os/freebsd/spl/sys/proc.h
index c6bc10d6babe..1cbd79ec893f 100644
--- a/sys/contrib/openzfs/include/os/freebsd/spl/sys/proc.h
+++ b/sys/contrib/openzfs/include/os/freebsd/spl/sys/proc.h
@@ -77,8 +77,8 @@ do_thread_create(caddr_t stk, size_t stksize, void (*proc)(void *), void *arg,
 	/*
 	 * Be sure there are no surprises.
 	 */
-	ASSERT(stk == NULL);
-	ASSERT(len == 0);
+	ASSERT0P(stk);
+	ASSERT0(len);
 	ASSERT(state == TS_RUN);
 
 	if (pp == &p0)
diff --git a/sys/contrib/openzfs/include/os/freebsd/spl/sys/time.h b/sys/contrib/openzfs/include/os/freebsd/spl/sys/time.h
index 2f5fe4619ef7..14b42f2e7087 100644
--- a/sys/contrib/openzfs/include/os/freebsd/spl/sys/time.h
+++ b/sys/contrib/openzfs/include/os/freebsd/spl/sys/time.h
@@ -63,6 +63,17 @@ typedef longlong_t	hrtime_t;
 #define	NSEC_TO_TICK(nsec)	((nsec) / (NANOSEC / hz))
 
 static __inline hrtime_t
+getlrtime(void)
+{
+	struct timespec ts;
+	hrtime_t nsec;
+
+	getnanouptime(&ts);
+	nsec = ((hrtime_t)ts.tv_sec * NANOSEC) + ts.tv_nsec;
+	return (nsec);
+}
+
+static __inline hrtime_t
 gethrtime(void)
 {
 	struct timespec ts;
diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
index 076dab8ba6dc..214f3ea0e787 100644
--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
@@ -542,24 +542,6 @@ blk_generic_alloc_queue(make_request_fn make_request, int node_id)
 }
 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
 
-/*
- * All the io_*() helper functions below can operate on a bio, or a rq, but
- * not both.  The older submit_bio() codepath will pass a bio, and the
- * newer blk-mq codepath will pass a rq.
- */
-static inline int
-io_data_dir(struct bio *bio, struct request *rq)
-{
-	if (rq != NULL) {
-		if (op_is_write(req_op(rq))) {
-			return (WRITE);
-		} else {
-			return (READ);
-		}
-	}
-	return (bio_data_dir(bio));
-}
-
 static inline int
 io_is_flush(struct bio *bio, struct request *rq)
 {
diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/dcache_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/dcache_compat.h
index 16e8a319a5f8..152e5a606f0e 100644
--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/dcache_compat.h
+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/dcache_compat.h
@@ -61,32 +61,6 @@
 #endif
 
 /*
- * 2.6.30 API change,
- * The const keyword was added to the 'struct dentry_operations' in
- * the dentry structure.  To handle this we define an appropriate
- * dentry_operations_t typedef which can be used.
- */
-typedef const struct dentry_operations	dentry_operations_t;
-
-/*
- * 2.6.38 API addition,
- * Added d_clear_d_op() helper function which clears some flags and the
- * registered dentry->d_op table.  This is required because d_set_d_op()
- * issues a warning when the dentry operations table is already set.
- * For the .zfs control directory to work properly we must be able to
- * override the default operations table and register custom .d_automount
- * and .d_revalidate callbacks.
- */
-static inline void
-d_clear_d_op(struct dentry *dentry)
-{
-	dentry->d_op = NULL;
-	dentry->d_flags &= ~(
-	    DCACHE_OP_HASH | DCACHE_OP_COMPARE |
-	    DCACHE_OP_REVALIDATE | DCACHE_OP_DELETE);
-}
-
-/*
  * Walk and invalidate all dentry aliases of an inode
  * unless it's a mountpoint
  */
diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h
index cd245a5f0135..326f471d7c9b 100644
--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h
+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h
@@ -139,15 +139,6 @@
  */
 #if defined(HAVE_KERNEL_FPU_INTERNAL)
 
-/*
- * For kernels not exporting *kfpu_{begin,end} we have to use inline assembly
- * with the XSAVE{,OPT,S} instructions, so we need the toolchain to support at
- * least XSAVE.
- */
-#if !defined(HAVE_XSAVE)
-#error "Toolchain needs to support the XSAVE assembler instruction"
-#endif
-
 #ifndef XFEATURE_MASK_XTILE
 /*
  * For kernels where this doesn't exist yet, we still don't want to break
@@ -335,9 +326,13 @@ kfpu_begin(void)
 		return;
 	}
 #endif
+#if defined(HAVE_XSAVE)
 	if (static_cpu_has(X86_FEATURE_XSAVE)) {
 		kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE);
-	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+		return;
+	}
+#endif
+	if (static_cpu_has(X86_FEATURE_FXSR)) {
 		kfpu_save_fxsr(state);
 	} else {
 		kfpu_save_fsave(state);
@@ -390,9 +385,13 @@ kfpu_end(void)
 		goto out;
 	}
 #endif
+#if defined(HAVE_XSAVE)
 	if (static_cpu_has(X86_FEATURE_XSAVE)) {
 		kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE);
-	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+		goto out;
+	}
+#endif
+	if (static_cpu_has(X86_FEATURE_FXSR)) {
 		kfpu_restore_fxsr(state);
 	} else {
 		kfpu_restore_fsave(state);
@@ -599,6 +598,32 @@ zfs_movbe_available(void)
 }
 
 /*
+ * Check if VAES instruction set is available
+ */
+static inline boolean_t
+zfs_vaes_available(void)
+{
+#if defined(X86_FEATURE_VAES)
+	return (!!boot_cpu_has(X86_FEATURE_VAES));
+#else
+	return (B_FALSE);
+#endif
+}
+
+/*
+ * Check if VPCLMULQDQ instruction set is available
+ */
+static inline boolean_t
+zfs_vpclmulqdq_available(void)
+{
+#if defined(X86_FEATURE_VPCLMULQDQ)
+	return (!!boot_cpu_has(X86_FEATURE_VPCLMULQDQ));
+#else
+	return (B_FALSE);
+#endif
+}
+
+/*
  * Check if SHA_NI instruction set is available
  */
 static inline boolean_t
diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/vfs_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/vfs_compat.h
index cbf14e28371f..d9dc904bc322 100644
--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/vfs_compat.h
+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/vfs_compat.h
@@ -23,6 +23,7 @@
 /*
  * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
  * Copyright (C) 2015 Jörg Thalheim.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  */
 
 #ifndef _ZFS_VFS_H
@@ -262,4 +263,10 @@ zpl_is_32bit_api(void)
 #define	zpl_generic_fillattr(user_ns, ip, sp)	generic_fillattr(ip, sp)
 #endif
 
+#ifdef HAVE_INODE_GENERIC_DROP
+/* 6.18 API change. These were renamed, alias the old names to the new. */
+#define	generic_delete_inode(ip)	inode_just_drop(ip)
+#define	generic_drop_inode(ip)		inode_generic_drop(ip)
+#endif
+
 #endif /* _ZFS_VFS_H */
diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h b/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h
index 1671ba4074da..85b96e1e23a7 100644
--- a/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h
+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/debug.h
@@ -69,6 +69,10 @@
 #define	__maybe_unused __attribute__((unused))
 #endif
 
+#ifndef __must_check
+#define	__must_check __attribute__((__warn_unused_result__))
+#endif
+
 /*
  * Without this, we see warnings from objtool during normal Linux builds when
  * the kernel is built with CONFIG_STACK_VALIDATION=y:
diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/mutex.h b/sys/contrib/openzfs/include/os/linux/spl/sys/mutex.h
index f000f53ab9b6..4eca2414fc5b 100644
--- a/sys/contrib/openzfs/include/os/linux/spl/sys/mutex.h
+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/mutex.h
@@ -111,7 +111,7 @@ spl_mutex_lockdep_on_maybe(kmutex_t *mp)			\
 #undef mutex_destroy
 #define	mutex_destroy(mp)					\
 {								\
-	VERIFY3P(mutex_owner(mp), ==, NULL);			\
+	VERIFY0P(mutex_owner(mp));			\
 }
 
 #define	mutex_tryenter(mp)					\
diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/rwlock.h b/sys/contrib/openzfs/include/os/linux/spl/sys/rwlock.h
index 563e0a19663d..c883836c2f83 100644
--- a/sys/contrib/openzfs/include/os/linux/spl/sys/rwlock.h
+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/rwlock.h
@@ -130,7 +130,7 @@ RW_READ_HELD(krwlock_t *rwp)
 /*
  * The Linux rwsem implementation does not require a matching destroy.
  */
-#define	rw_destroy(rwp)		((void) 0)
+#define	rw_destroy(rwp)		ASSERT(!(RW_LOCK_HELD(rwp)))
 
 /*
  * Upgrading a rwsem from a reader to a writer is not supported by the
diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/stat.h b/sys/contrib/openzfs/include/os/linux/spl/sys/stat.h
index 087389b57b34..ad2815e46394 100644
--- a/sys/contrib/openzfs/include/os/linux/spl/sys/stat.h
+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/stat.h
@@ -25,6 +25,6 @@
 #ifndef _SPL_STAT_H
 #define	_SPL_STAT_H
 
-#include <linux/stat.h>
+#include <sys/stat.h>
 
 #endif /* SPL_STAT_H */
diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/time.h b/sys/contrib/openzfs/include/os/linux/spl/sys/time.h
index 33b273b53996..4edc42a8aef9 100644
--- a/sys/contrib/openzfs/include/os/linux/spl/sys/time.h
+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/time.h
@@ -80,6 +80,14 @@ gethrestime_sec(void)
 }
 
 static inline hrtime_t
+getlrtime(void)
+{
+	inode_timespec_t ts;
+	ktime_get_coarse_ts64(&ts);
+	return (((hrtime_t)ts.tv_sec * NSEC_PER_SEC) + ts.tv_nsec);
+}
+
+static inline hrtime_t
 gethrtime(void)
 {
 	struct timespec64 ts;
diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h
index 955462c85d10..e34ea46b3fe8 100644
--- a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h
+++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h
@@ -139,18 +139,18 @@
 #define	ZCW_TP_STRUCT_ENTRY						    \
 		__field(lwb_t *,	zcw_lwb)			    \
 		__field(boolean_t,	zcw_done)			    \
-		__field(int,		zcw_zio_error)			    \
+		__field(int,		zcw_error)			    \
 
 #define	ZCW_TP_FAST_ASSIGN						    \
 		__entry->zcw_lwb		= zcw->zcw_lwb;		    \
 		__entry->zcw_done		= zcw->zcw_done;	    \
-		__entry->zcw_zio_error		= zcw->zcw_zio_error;
+		__entry->zcw_error		= zcw->zcw_error;
 
 #define	ZCW_TP_PRINTK_FMT						    \
 	"zcw { lwb %p done %u error %u }"
 
 #define	ZCW_TP_PRINTK_ARGS						    \
-	    __entry->zcw_lwb, __entry->zcw_done, __entry->zcw_zio_error
+	    __entry->zcw_lwb, __entry->zcw_done, __entry->zcw_error
 
 /*
  * Generic support for two argument tracepoints of the form:
diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/zpl.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/zpl.h
index f5a9105cd885..8994aab889fe 100644
--- a/sys/contrib/openzfs/include/os/linux/zfs/sys/zpl.h
+++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/zpl.h
@@ -55,6 +55,7 @@ extern const struct file_operations zpl_dir_file_operations;
 extern void zpl_prune_sb(uint64_t nr_to_scan, void *arg);
 
 extern const struct super_operations zpl_super_operations;
+extern const struct dentry_operations zpl_dentry_operations;
 extern const struct export_operations zpl_export_operations;
 extern struct file_system_type zpl_fs_type;
 
diff --git a/sys/contrib/openzfs/include/sys/dmu.h b/sys/contrib/openzfs/include/sys/dmu.h
index 7c2024a16d8f..aa5035862def 100644
--- a/sys/contrib/openzfs/include/sys/dmu.h
+++ b/sys/contrib/openzfs/include/sys/dmu.h
@@ -414,9 +414,9 @@ typedef struct dmu_buf {
 #define	DMU_POOL_ZPOOL_CHECKPOINT	"com.delphix:zpool_checkpoint"
 #define	DMU_POOL_LOG_SPACEMAP_ZAP	"com.delphix:log_spacemap_zap"
 #define	DMU_POOL_DELETED_CLONES		"com.delphix:deleted_clones"
-#define	DMU_POOL_TXG_LOG_TIME_MINUTES	"com.klaraystems:txg_log_time:minutes"
-#define	DMU_POOL_TXG_LOG_TIME_DAYS	"com.klaraystems:txg_log_time:days"
-#define	DMU_POOL_TXG_LOG_TIME_MONTHS	"com.klaraystems:txg_log_time:months"
+#define	DMU_POOL_TXG_LOG_TIME_MINUTES	"com.klarasystems:txg_log_time:minutes"
+#define	DMU_POOL_TXG_LOG_TIME_DAYS	"com.klarasystems:txg_log_time:days"
+#define	DMU_POOL_TXG_LOG_TIME_MONTHS	"com.klarasystems:txg_log_time:months"
 
 /*
  * Allocate an object from this objset.  The range of object numbers
@@ -742,8 +742,8 @@ dmu_buf_init_user(dmu_buf_user_t *dbu, dmu_buf_evict_func_t *evict_func_sync,
     dmu_buf_evict_func_t *evict_func_async,
     dmu_buf_t **clear_on_evict_dbufp __maybe_unused)
 {
-	ASSERT(dbu->dbu_evict_func_sync == NULL);
-	ASSERT(dbu->dbu_evict_func_async == NULL);
+	ASSERT0P(dbu->dbu_evict_func_sync);
+	ASSERT0P(dbu->dbu_evict_func_async);
 
 	/* must have at least one evict func */
 	IMPLY(evict_func_sync == NULL, evict_func_async != NULL);
diff --git a/sys/contrib/openzfs/include/sys/dmu_impl.h b/sys/contrib/openzfs/include/sys/dmu_impl.h
index 21a8b16a3ee6..bae872bd1907 100644
--- a/sys/contrib/openzfs/include/sys/dmu_impl.h
+++ b/sys/contrib/openzfs/include/sys/dmu_impl.h
@@ -168,12 +168,10 @@ extern "C" {
  * 	dn_allocated_txg
  * 	dn_free_txg
  * 	dn_assigned_txg
- * 	dn_dirty_txg
+ * 	dn_dirtycnt
  * 	dd_assigned_tx
  * 	dn_notxholds
  *	dn_nodnholds
- * 	dn_dirtyctx
- * 	dn_dirtyctx_firstset
  * 	(dn_phys copy fields?)
  * 	(dn_phys contents?)
  *   held from:
diff --git a/sys/contrib/openzfs/include/sys/dnode.h b/sys/contrib/openzfs/include/sys/dnode.h
index 76218c8b09ca..8bd1db5b7165 100644
--- a/sys/contrib/openzfs/include/sys/dnode.h
+++ b/sys/contrib/openzfs/include/sys/dnode.h
@@ -141,12 +141,6 @@ struct dmu_buf_impl;
 struct objset;
 struct zio;
 
-enum dnode_dirtycontext {
-	DN_UNDIRTIED,
-	DN_DIRTY_OPEN,
-	DN_DIRTY_SYNC
-};
-
 /* Is dn_used in bytes?  if not, it's in multiples of SPA_MINBLOCKSIZE */
 #define	DNODE_FLAG_USED_BYTES			(1 << 0)
 #define	DNODE_FLAG_USERUSED_ACCOUNTED		(1 << 1)
@@ -340,11 +334,9 @@ struct dnode {
 	uint64_t dn_allocated_txg;
 	uint64_t dn_free_txg;
 	uint64_t dn_assigned_txg;
-	uint64_t dn_dirty_txg;			/* txg dnode was last dirtied */
+	uint8_t dn_dirtycnt;
 	kcondvar_t dn_notxholds;
 	kcondvar_t dn_nodnholds;
-	enum dnode_dirtycontext dn_dirtyctx;
-	const void *dn_dirtyctx_firstset;	/* dbg: contents meaningless */
 
 	/* protected by own devices */
 	zfs_refcount_t dn_tx_holds;
@@ -440,7 +432,6 @@ void dnode_rele_and_unlock(dnode_t *dn, const void *tag, boolean_t evicting);
 int dnode_try_claim(objset_t *os, uint64_t object, int slots);
 boolean_t dnode_is_dirty(dnode_t *dn);
 void dnode_setdirty(dnode_t *dn, dmu_tx_t *tx);
-void dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, const void *tag);
 void dnode_sync(dnode_t *dn, dmu_tx_t *tx);
 void dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
     dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx);
@@ -468,9 +459,6 @@ void dnode_free_interior_slots(dnode_t *dn);
 
 void dnode_set_storage_type(dnode_t *dn, dmu_object_type_t type);
 
-#define	DNODE_IS_DIRTY(_dn)						\
-	((_dn)->dn_dirty_txg >= spa_syncing_txg((_dn)->dn_objset->os_spa))
-
 #define	DNODE_LEVEL_IS_CACHEABLE(_dn, _level)				\
 	((_dn)->dn_objset->os_primary_cache == ZFS_CACHE_ALL ||		\
 	(((_level) > 0 || DMU_OT_IS_METADATA((_dn)->dn_type)) &&	\
diff --git a/sys/contrib/openzfs/include/sys/dsl_deleg.h b/sys/contrib/openzfs/include/sys/dsl_deleg.h
index ae729b9f32ff..36dd6211219d 100644
--- a/sys/contrib/openzfs/include/sys/dsl_deleg.h
+++ b/sys/contrib/openzfs/include/sys/dsl_deleg.h
@@ -46,6 +46,7 @@ extern "C" {
 #define	ZFS_DELEG_PERM_MOUNT		"mount"
 #define	ZFS_DELEG_PERM_SHARE		"share"
 #define	ZFS_DELEG_PERM_SEND		"send"
+#define	ZFS_DELEG_PERM_SEND_RAW		"send:raw"
 #define	ZFS_DELEG_PERM_RECEIVE		"receive"
 #define	ZFS_DELEG_PERM_RECEIVE_APPEND	"receive:append"
 #define	ZFS_DELEG_PERM_ALLOW		"allow"
diff --git a/sys/contrib/openzfs/include/sys/fm/fs/zfs.h b/sys/contrib/openzfs/include/sys/fm/fs/zfs.h
index 659c64bf15a6..a771b11420fd 100644
--- a/sys/contrib/openzfs/include/sys/fm/fs/zfs.h
+++ b/sys/contrib/openzfs/include/sys/fm/fs/zfs.h
@@ -58,6 +58,7 @@ extern "C" {
 #define	FM_EREPORT_ZFS_PROBE_FAILURE		"probe_failure"
 #define	FM_EREPORT_ZFS_LOG_REPLAY		"log_replay"
 #define	FM_EREPORT_ZFS_CONFIG_CACHE_WRITE	"config_cache_write"
+#define	FM_EREPORT_ZFS_SITOUT			"sitout"
 
 #define	FM_EREPORT_PAYLOAD_ZFS_POOL		"pool"
 #define	FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE	"pool_failmode"
diff --git a/sys/contrib/openzfs/include/sys/fs/zfs.h b/sys/contrib/openzfs/include/sys/fs/zfs.h
index fc359c10365a..662fd81c5ee1 100644
--- a/sys/contrib/openzfs/include/sys/fs/zfs.h
+++ b/sys/contrib/openzfs/include/sys/fs/zfs.h
@@ -385,6 +385,8 @@ typedef enum {
 	VDEV_PROP_TRIM_SUPPORT,
 	VDEV_PROP_TRIM_ERRORS,
 	VDEV_PROP_SLOW_IOS,
+	VDEV_PROP_SIT_OUT,
+	VDEV_PROP_AUTOSIT,
 	VDEV_NUM_PROPS
 } vdev_prop_t;
 
@@ -746,6 +748,8 @@ typedef struct zpool_load_policy {
 #define	ZPOOL_CONFIG_METASLAB_SHIFT	"metaslab_shift"
 #define	ZPOOL_CONFIG_ASHIFT		"ashift"
 #define	ZPOOL_CONFIG_ASIZE		"asize"
+#define	ZPOOL_CONFIG_MIN_ALLOC		"min_alloc"
+#define	ZPOOL_CONFIG_MAX_ALLOC		"max_alloc"
 #define	ZPOOL_CONFIG_DTL		"DTL"
 #define	ZPOOL_CONFIG_SCAN_STATS		"scan_stats"	/* not stored on disk */
 #define	ZPOOL_CONFIG_REMOVAL_STATS	"removal_stats"	/* not stored on disk */
@@ -1673,6 +1677,7 @@ typedef enum {
 	ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS,
 	ZFS_ERR_ASHIFT_MISMATCH,
 	ZFS_ERR_STREAM_LARGE_MICROZAP,
+	ZFS_ERR_TOO_MANY_SITOUTS,
 } zfs_errno_t;
 
 /*
diff --git a/sys/contrib/openzfs/include/sys/range_tree.h b/sys/contrib/openzfs/include/sys/range_tree.h
index 0f6884682459..0f6def36f9f6 100644
--- a/sys/contrib/openzfs/include/sys/range_tree.h
+++ b/sys/contrib/openzfs/include/sys/range_tree.h
@@ -238,8 +238,7 @@ zfs_rs_set_end_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t end)
 }
 
 static inline void
-zfs_zfs_rs_set_fill_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt,
-    uint64_t fill)
+zfs_rs_set_fill_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t fill)
 {
 	ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES);
 	switch (rt->rt_type) {
@@ -277,7 +276,7 @@ static inline void
 zfs_rs_set_fill(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t fill)
 {
 	ASSERT(IS_P2ALIGNED(fill, 1ULL << rt->rt_shift));
-	zfs_zfs_rs_set_fill_raw(rs, rt, fill >> rt->rt_shift);
+	zfs_rs_set_fill_raw(rs, rt, fill >> rt->rt_shift);
 }
 
 typedef void zfs_range_tree_func_t(void *arg, uint64_t start, uint64_t size);
diff --git a/sys/contrib/openzfs/include/sys/spa.h b/sys/contrib/openzfs/include/sys/spa.h
index db6de332ae67..f172f2af6f07 100644
--- a/sys/contrib/openzfs/include/sys/spa.h
+++ b/sys/contrib/openzfs/include/sys/spa.h
@@ -880,7 +880,6 @@ extern kcondvar_t spa_namespace_cv;
 #define	SPA_CONFIG_UPDATE_VDEVS	1
 
 extern void spa_write_cachefile(spa_t *, boolean_t, boolean_t, boolean_t);
-extern void spa_config_load(void);
 extern int spa_all_configs(uint64_t *generation, nvlist_t **pools);
 extern void spa_config_set(spa_t *spa, nvlist_t *config);
 extern nvlist_t *spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg,
@@ -1031,7 +1030,7 @@ extern void spa_import_progress_set_notes_nolog(spa_t *spa,
 extern int spa_config_tryenter(spa_t *spa, int locks, const void *tag,
     krw_t rw);
 extern void spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw);
-extern void spa_config_enter_mmp(spa_t *spa, int locks, const void *tag,
+extern void spa_config_enter_priority(spa_t *spa, int locks, const void *tag,
     krw_t rw);
 extern void spa_config_exit(spa_t *spa, int locks, const void *tag);
 extern int spa_config_held(spa_t *spa, int locks, krw_t rw);
@@ -1085,6 +1084,7 @@ extern pool_state_t spa_state(spa_t *spa);
 extern spa_load_state_t spa_load_state(spa_t *spa);
 extern uint64_t spa_freeze_txg(spa_t *spa);
 extern uint64_t spa_get_worst_case_asize(spa_t *spa, uint64_t lsize);
+extern void spa_get_min_alloc_range(spa_t *spa, uint64_t *min, uint64_t *max);
 extern uint64_t spa_get_dspace(spa_t *spa);
 extern uint64_t spa_get_checkpoint_space(spa_t *spa);
 extern uint64_t spa_get_slop_space(spa_t *spa);
@@ -1244,7 +1244,6 @@ extern void vdev_mirror_stat_fini(void);
 /* Initialization and termination */
 extern void spa_init(spa_mode_t mode);
 extern void spa_fini(void);
-extern void spa_boot_init(void *);
 
 /* properties */
 extern int spa_prop_set(spa_t *spa, nvlist_t *nvp);
diff --git a/sys/contrib/openzfs/include/sys/spa_impl.h b/sys/contrib/openzfs/include/sys/spa_impl.h
index 07a959db3447..62b062984d36 100644
--- a/sys/contrib/openzfs/include/sys/spa_impl.h
+++ b/sys/contrib/openzfs/include/sys/spa_impl.h
@@ -265,6 +265,7 @@ struct spa {
 	uint64_t	spa_min_ashift;		/* of vdevs in normal class */
 	uint64_t	spa_max_ashift;		/* of vdevs in normal class */
 	uint64_t	spa_min_alloc;		/* of vdevs in normal class */
+	uint64_t	spa_max_alloc;		/* of vdevs in normal class */
 	uint64_t	spa_gcd_alloc;		/* of vdevs in normal class */
 	uint64_t	spa_config_guid;	/* config pool guid */
 	uint64_t	spa_load_guid;		/* spa_load initialized guid */
diff --git a/sys/contrib/openzfs/include/sys/vdev_impl.h b/sys/contrib/openzfs/include/sys/vdev_impl.h
index 4ab472bd6742..5a8c2f846be2 100644
--- a/sys/contrib/openzfs/include/sys/vdev_impl.h
+++ b/sys/contrib/openzfs/include/sys/vdev_impl.h
@@ -279,10 +279,12 @@ struct vdev {
 	uint64_t	vdev_noalloc;	/* device is passivated?	*/
 	uint64_t	vdev_removing;	/* device is being removed?	*/
 	uint64_t	vdev_failfast;	/* device failfast setting	*/
+	boolean_t	vdev_autosit;	/* automatic sitout management	*/
 	boolean_t	vdev_rz_expanding; /* raidz is being expanded?	*/
 	boolean_t	vdev_ishole;	/* is a hole in the namespace	*/
 	uint64_t	vdev_top_zap;
 	vdev_alloc_bias_t vdev_alloc_bias; /* metaslab allocation bias	*/
+	uint64_t	vdev_last_latency_check;
 
 	/* pool checkpoint related */
 	space_map_t	*vdev_checkpoint_sm;	/* contains reserved blocks */
@@ -431,6 +433,10 @@ struct vdev {
 	hrtime_t	vdev_mmp_pending; /* 0 if write finished	*/
 	uint64_t	vdev_mmp_kstat_id;	/* to find kstat entry */
 	uint64_t	vdev_expansion_time;	/* vdev's last expansion time */
+	/* used to calculate average read latency */
+	uint64_t	*vdev_prev_histo;
+	int64_t		vdev_outlier_count;	/* read outlier amongst peers */
+	hrtime_t	vdev_read_sit_out_expire; /* end of sit out period    */
 	list_node_t	vdev_leaf_node;		/* leaf vdev list */
 
 	/*
diff --git a/sys/contrib/openzfs/include/sys/vdev_raidz.h b/sys/contrib/openzfs/include/sys/vdev_raidz.h
index 3b02728cdbf3..df8c2aed4045 100644
--- a/sys/contrib/openzfs/include/sys/vdev_raidz.h
+++ b/sys/contrib/openzfs/include/sys/vdev_raidz.h
@@ -61,6 +61,9 @@ void vdev_raidz_checksum_error(zio_t *, struct raidz_col *, abd_t *);
 struct raidz_row *vdev_raidz_row_alloc(int, zio_t *);
 void vdev_raidz_reflow_copy_scratch(spa_t *);
 void raidz_dtl_reassessed(vdev_t *);
+boolean_t vdev_sit_out_reads(vdev_t *, zio_flag_t);
+void vdev_raidz_sit_child(vdev_t *, uint64_t);
+void vdev_raidz_unsit_child(vdev_t *);
 
 extern const zio_vsd_ops_t vdev_raidz_vsd_ops;
 
diff --git a/sys/contrib/openzfs/include/sys/vdev_raidz_impl.h b/sys/contrib/openzfs/include/sys/vdev_raidz_impl.h
index debce6f09a22..8c8dcfb077f6 100644
--- a/sys/contrib/openzfs/include/sys/vdev_raidz_impl.h
+++ b/sys/contrib/openzfs/include/sys/vdev_raidz_impl.h
@@ -119,6 +119,7 @@ typedef struct raidz_col {
 	uint8_t rc_need_orig_restore:1;	/* need to restore from orig_data? */
 	uint8_t rc_force_repair:1;	/* Write good data to this column */
 	uint8_t rc_allow_repair:1;	/* Allow repair I/O to this column */
+	uint8_t rc_latency_outlier:1;	/* Latency outlier for this device */
 	int rc_shadow_devidx;		/* for double write during expansion */
 	int rc_shadow_error;		/* for double write during expansion */
 	uint64_t rc_shadow_offset;	/* for double write during expansion */
@@ -133,6 +134,7 @@ typedef struct raidz_row {
 	int rr_firstdatacol;		/* First data column/parity count */
 	abd_t *rr_abd_empty;		/* dRAID empty sector buffer */
 	int rr_nempty;			/* empty sectors included in parity */
+	int rr_outlier_cnt;		/* Count of latency outlier devices */
 #ifdef ZFS_DEBUG
 	uint64_t rr_offset;		/* Logical offset for *_io_verify() */
 	uint64_t rr_size;		/* Physical size for *_io_verify() */
diff --git a/sys/contrib/openzfs/include/sys/zfs_file.h b/sys/contrib/openzfs/include/sys/zfs_file.h
index a1f344c2bb79..67abe9988aaa 100644
--- a/sys/contrib/openzfs/include/sys/zfs_file.h
+++ b/sys/contrib/openzfs/include/sys/zfs_file.h
@@ -46,7 +46,7 @@ void zfs_file_close(zfs_file_t *fp);
 
 int zfs_file_write(zfs_file_t *fp, const void *buf, size_t len, ssize_t *resid);
 int zfs_file_pwrite(zfs_file_t *fp, const void *buf, size_t len, loff_t off,
-    ssize_t *resid);
+    uint8_t ashift, ssize_t *resid);
 int zfs_file_read(zfs_file_t *fp, void *buf, size_t len, ssize_t *resid);
 int zfs_file_pread(zfs_file_t *fp, void *buf, size_t len, loff_t off,
     ssize_t *resid);
diff --git a/sys/contrib/openzfs/include/sys/zfs_ioctl.h b/sys/contrib/openzfs/include/sys/zfs_ioctl.h
index 8174242abdac..cfe11f43bb8e 100644
--- a/sys/contrib/openzfs/include/sys/zfs_ioctl.h
+++ b/sys/contrib/openzfs/include/sys/zfs_ioctl.h
@@ -455,6 +455,7 @@ typedef enum zinject_type {
 	ZINJECT_DECRYPT_FAULT,
 	ZINJECT_DELAY_IMPORT,
 	ZINJECT_DELAY_EXPORT,
+	ZINJECT_DELAY_READY,
 } zinject_type_t;
 
 typedef enum zinject_iotype {
diff --git a/sys/contrib/openzfs/include/sys/zfs_znode.h b/sys/contrib/openzfs/include/sys/zfs_znode.h
index ba577b80c98f..79b845a672a8 100644
--- a/sys/contrib/openzfs/include/sys/zfs_znode.h
+++ b/sys/contrib/openzfs/include/sys/zfs_znode.h
@@ -73,7 +73,7 @@ extern "C" {
 		pflags |= attr; \
 	else \
 		pflags &= ~attr; \
-	VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_FLAGS(ZTOZSB(zp)), \
+	VERIFY0(sa_update(zp->z_sa_hdl, SA_ZPL_FLAGS(ZTOZSB(zp)), \
 	    &pflags, sizeof (pflags), tx)); \
 }
 
diff --git a/sys/contrib/openzfs/include/sys/zil.h b/sys/contrib/openzfs/include/sys/zil.h
index 9d1fb47e2dfc..da085998879b 100644
--- a/sys/contrib/openzfs/include/sys/zil.h
+++ b/sys/contrib/openzfs/include/sys/zil.h
@@ -456,7 +456,7 @@ typedef enum {
 	WR_NUM_STATES	/* number of states */
 } itx_wr_state_t;
 
-typedef void (*zil_callback_t)(void *data);
+typedef void (*zil_callback_t)(void *data, int err);
 
 typedef struct itx {
 	list_node_t	itx_node;	/* linkage on zl_itx_list */
@@ -498,10 +498,13 @@ typedef struct zil_stats {
 	 *            (see zil_commit_writer_stall())
 	 * - suspend: ZIL suspended
 	 *            (see zil_commit(), zil_get_commit_list())
+	 * -   crash: ZIL crashed
+	 *            (see zil_crash(), zil_commit(), ...)
 	 */
 	kstat_named_t zil_commit_error_count;
 	kstat_named_t zil_commit_stall_count;
 	kstat_named_t zil_commit_suspend_count;
+	kstat_named_t zil_commit_crash_count;
 
 	/*
 	 * Number of transactions (reads, writes, renames, etc.)
@@ -549,6 +552,7 @@ typedef struct zil_sums {
 	wmsum_t zil_commit_error_count;
 	wmsum_t zil_commit_stall_count;
 	wmsum_t zil_commit_suspend_count;
+	wmsum_t zil_commit_crash_count;
 	wmsum_t zil_itx_count;
 	wmsum_t zil_itx_indirect_count;
 	wmsum_t zil_itx_indirect_bytes;
@@ -577,6 +581,25 @@ typedef struct zil_sums {
 #define	ZIL_STAT_BUMP(zil, stat) \
     ZIL_STAT_INCR(zil, stat, 1);
 
+/*
+ * Flags for zil_commit_flags(). zil_commit() is a shortcut for
+ * zil_commit_flags(ZIL_COMMIT_FAILMODE), which is the most common use.
+ */
+typedef enum {
+	/*
+	 * Try to commit the ZIL. If it fails, fall back to txg_wait_synced().
+	 * If that fails, return EIO.
+	 */
+	ZIL_COMMIT_NOW = 0,
+
+	/*
+	 * Like ZIL_COMMIT_NOW, but if the ZIL commit fails because the pool
+	 * suspended, act according to the pool's failmode= setting (wait for
+	 * the pool to resume, or return EIO).
+	 */
+	ZIL_COMMIT_FAILMODE = (1 << 1),
+} zil_commit_flag_t;
+
 typedef int zil_parse_blk_func_t(zilog_t *zilog, const blkptr_t *bp, void *arg,
     uint64_t txg);
 typedef int zil_parse_lr_func_t(zilog_t *zilog, const lr_t *lr, void *arg,
@@ -606,14 +629,16 @@ extern boolean_t zil_destroy(zilog_t *zilog, boolean_t keep_first);
 extern void	zil_destroy_sync(zilog_t *zilog, dmu_tx_t *tx);
 
 extern itx_t	*zil_itx_create(uint64_t txtype, size_t lrsize);
-extern void	zil_itx_destroy(itx_t *itx);
+extern void	zil_itx_destroy(itx_t *itx, int err);
 extern void	zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx);
 
 extern void	zil_async_to_sync(zilog_t *zilog, uint64_t oid);
-extern void	zil_commit(zilog_t *zilog, uint64_t oid);
-extern void	zil_commit_impl(zilog_t *zilog, uint64_t oid);
 extern void	zil_remove_async(zilog_t *zilog, uint64_t oid);
 
+extern int	zil_commit_flags(zilog_t *zilog, uint64_t oid,
+    zil_commit_flag_t flags);
+extern int __must_check	zil_commit(zilog_t *zilog, uint64_t oid);
+
 extern int	zil_reset(const char *osname, void *txarg);
 extern int	zil_claim(struct dsl_pool *dp,
     struct dsl_dataset *ds, void *txarg);
diff --git a/sys/contrib/openzfs/include/sys/zil_impl.h b/sys/contrib/openzfs/include/sys/zil_impl.h
index 252264b9eae9..ea1364a7e35a 100644
--- a/sys/contrib/openzfs/include/sys/zil_impl.h
+++ b/sys/contrib/openzfs/include/sys/zil_impl.h
@@ -41,8 +41,8 @@ extern "C" {
  *
  * An lwb will start out in the "new" state, and transition to the "opened"
  * state via a call to zil_lwb_write_open() on first itx assignment.  When
- * transitioning from "new" to "opened" the zilog's "zl_issuer_lock" must be
- * held.
+ * transitioning from "new" to "opened" the zilog's "zl_issuer_lock" and
+ * LWB's "lwb_lock" must be held.
  *
  * After the lwb is "opened", it can be assigned number of itxs and transition
  * into the "closed" state via zil_lwb_write_close() when full or on timeout.
@@ -100,16 +100,22 @@ typedef enum {
  * holding the "zl_issuer_lock". After the lwb is issued, the zilog's
  * "zl_lock" is used to protect the lwb against concurrent access.
  */
+typedef enum {
+	LWB_FLAG_SLIM =		(1<<0),	/* log block has slim format */
+	LWB_FLAG_SLOG =		(1<<1),	/* lwb_blk is on SLOG device */
+	LWB_FLAG_CRASHED =	(1<<2),	/* lwb is on the crash list */
+} lwb_flag_t;
+
 typedef struct lwb {
 	zilog_t		*lwb_zilog;	/* back pointer to log struct */
 	blkptr_t	lwb_blk;	/* on disk address of this log blk */
-	boolean_t	lwb_slim;	/* log block has slim format */
-	boolean_t	lwb_slog;	/* lwb_blk is on SLOG device */
+	lwb_flag_t	lwb_flags;	/* extra info about this lwb */
 	int		lwb_error;	/* log block allocation error */
 	int		lwb_nmax;	/* max bytes in the buffer */
 	int		lwb_nused;	/* # used bytes in buffer */
 	int		lwb_nfilled;	/* # filled bytes in buffer */
 	int		lwb_sz;		/* size of block and buffer */
+	int		lwb_min_sz;	/* min size for range allocation */
 	lwb_state_t	lwb_state;	/* the state of this lwb */
 	char		*lwb_buf;	/* log write buffer */
 	zio_t		*lwb_child_zio;	/* parent zio for children */
@@ -124,7 +130,7 @@ typedef struct lwb {
 	list_t		lwb_itxs;	/* list of itx's */
 	list_t		lwb_waiters;	/* list of zil_commit_waiter's */
 	avl_tree_t	lwb_vdev_tree;	/* vdevs to flush after lwb write */
-	kmutex_t	lwb_vdev_lock;	/* protects lwb_vdev_tree */
+	kmutex_t	lwb_lock;	/* protects lwb_vdev_tree and size */
 } lwb_t;
 
 /*
@@ -149,7 +155,7 @@ typedef struct zil_commit_waiter {
 	list_node_t	zcw_node;	/* linkage in lwb_t:lwb_waiter list */
 	lwb_t		*zcw_lwb;	/* back pointer to lwb when linked */
 	boolean_t	zcw_done;	/* B_TRUE when "done", else B_FALSE */
-	int		zcw_zio_error;	/* contains the zio io_error value */
+	int		zcw_error;	/* result to return from zil_commit() */
 } zil_commit_waiter_t;
 
 /*
@@ -221,6 +227,7 @@ struct zilog {
 	uint64_t	zl_cur_left;	/* current burst remaining size */
 	uint64_t	zl_cur_max;	/* biggest record in current burst */
 	list_t		zl_lwb_list;	/* in-flight log write list */
+	list_t		zl_lwb_crash_list; /* log writes in-flight at crash */
 	avl_tree_t	zl_bp_tree;	/* track bps during log parse */
 	clock_t		zl_replay_time;	/* lbolt of when replay started */
 	uint64_t	zl_replay_blks;	/* number of log blocks replayed */
@@ -245,6 +252,9 @@ struct zilog {
 	 */
 	uint64_t	zl_max_block_size;
 
+	/* After crash, txg to restart zil */
+	uint64_t	zl_restart_txg;
+
 	/* Pointer for per dataset zil sums */
 	zil_sums_t *zl_sums;
 };
diff --git a/sys/contrib/openzfs/include/sys/zio.h b/sys/contrib/openzfs/include/sys/zio.h
index 4f46eab3db89..acb0a03a36b2 100644
--- a/sys/contrib/openzfs/include/sys/zio.h
+++ b/sys/contrib/openzfs/include/sys/zio.h
@@ -82,7 +82,8 @@ gbh_nblkptrs(uint64_t size) {
 static inline zio_eck_t *
 gbh_eck(zio_gbh_phys_t *gbh, uint64_t size) {
 	ASSERT(IS_P2ALIGNED(size, sizeof (blkptr_t)));
-	return ((zio_eck_t *)((uintptr_t)gbh + (size_t)size - sizeof (zio_eck_t)));
+	return ((zio_eck_t *)((uintptr_t)gbh + (size_t)size -
+	    sizeof (zio_eck_t)));
 }
 
 static inline blkptr_t *
@@ -360,26 +361,26 @@ struct zbookmark_err_phys {
 	(zb)->zb_blkid == ZB_ROOT_BLKID)
 
 typedef struct zio_prop {
-	enum zio_checksum	zp_checksum;
-	enum zio_compress	zp_compress;
+	enum zio_checksum	zp_checksum:8;
+	enum zio_compress	zp_compress:8;
 	uint8_t			zp_complevel;
 	uint8_t			zp_level;
 	uint8_t			zp_copies;
 	uint8_t			zp_gang_copies;
-	dmu_object_type_t	zp_type;
-	boolean_t		zp_dedup;
-	boolean_t		zp_dedup_verify;
-	boolean_t		zp_nopwrite;
-	boolean_t		zp_brtwrite;
-	boolean_t		zp_encrypt;
-	boolean_t		zp_byteorder;
-	boolean_t		zp_direct_write;
-	boolean_t		zp_rewrite;
+	dmu_object_type_t	zp_type:8;
+	dmu_object_type_t	zp_storage_type:8;
+	boolean_t		zp_dedup:1;
+	boolean_t		zp_dedup_verify:1;
+	boolean_t		zp_nopwrite:1;
+	boolean_t		zp_brtwrite:1;
+	boolean_t		zp_encrypt:1;
+	boolean_t		zp_byteorder:1;
+	boolean_t		zp_direct_write:1;
+	boolean_t		zp_rewrite:1;
+	uint32_t		zp_zpl_smallblk;
 	uint8_t			zp_salt[ZIO_DATA_SALT_LEN];
 	uint8_t			zp_iv[ZIO_DATA_IV_LEN];
 	uint8_t			zp_mac[ZIO_DATA_MAC_LEN];
-	uint32_t		zp_zpl_smallblk;
-	dmu_object_type_t	zp_storage_type;
 } zio_prop_t;
 
 typedef struct zio_cksum_report zio_cksum_report_t;
@@ -622,7 +623,8 @@ extern zio_t *zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg,
     const blkptr_t *bp, zio_flag_t flags);
 
 extern int zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg,
-    blkptr_t *new_bp, uint64_t size, boolean_t *slog);
+    blkptr_t *new_bp, uint64_t min_size, uint64_t max_size, boolean_t *slog,
+    boolean_t allow_larger);
 extern void zio_flush(zio_t *zio, vdev_t *vd);
 extern void zio_shrink(zio_t *zio, uint64_t size);
 
@@ -716,6 +718,7 @@ extern void zio_handle_ignored_writes(zio_t *zio);
 extern hrtime_t zio_handle_io_delay(zio_t *zio);
 extern void zio_handle_import_delay(spa_t *spa, hrtime_t elapsed);
 extern void zio_handle_export_delay(spa_t *spa, hrtime_t elapsed);
+extern hrtime_t zio_handle_ready_delay(zio_t *zio);
 
 /*
  * Checksum ereport functions
diff --git a/sys/contrib/openzfs/include/sys/zvol.h b/sys/contrib/openzfs/include/sys/zvol.h
index cdc9dba2a28d..5791246e99e4 100644
--- a/sys/contrib/openzfs/include/sys/zvol.h
+++ b/sys/contrib/openzfs/include/sys/zvol.h
@@ -53,7 +53,7 @@ extern int zvol_set_volsize(const char *, uint64_t);
 extern int zvol_set_volthreading(const char *, boolean_t);
 extern int zvol_set_common(const char *, zfs_prop_t, zprop_source_t, uint64_t);
 extern int zvol_set_ro(const char *, boolean_t);
-extern zvol_state_handle_t *zvol_suspend(const char *);
+extern int zvol_suspend(const char *, zvol_state_handle_t **);
 extern int zvol_resume(zvol_state_handle_t *);
 extern void *zvol_tag(zvol_state_handle_t *);
 
diff --git a/sys/contrib/openzfs/include/sys/zvol_impl.h b/sys/contrib/openzfs/include/sys/zvol_impl.h
index f3dd9f26f23c..5422e66832c0 100644
--- a/sys/contrib/openzfs/include/sys/zvol_impl.h
+++ b/sys/contrib/openzfs/include/sys/zvol_impl.h
@@ -20,7 +20,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright (c) 2024, Klara, Inc.
+ * Copyright (c) 2024, 2025, Klara, Inc.
  */
 
 #ifndef	_SYS_ZVOL_IMPL_H
@@ -56,6 +56,7 @@ typedef struct zvol_state {
 	atomic_t		zv_suspend_ref;	/* refcount for suspend */
 	krwlock_t		zv_suspend_lock;	/* suspend lock */
 	kcondvar_t		zv_removing_cv;	/* ready to remove minor */
+	list_node_t		zv_remove_node;	/* node on removal list */
 	struct zvol_state_os	*zv_zso;	/* private platform state */
 	boolean_t		zv_threading;	/* volthreading property */
 } zvol_state_t;
@@ -135,7 +136,7 @@ int zvol_os_rename_minor(zvol_state_t *zv, const char *newname);
 int zvol_os_create_minor(const char *name);
 int zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize);
 boolean_t zvol_os_is_zvol(const char *path);
-void zvol_os_clear_private(zvol_state_t *zv);
+void zvol_os_remove_minor(zvol_state_t *zv);
 void zvol_os_set_disk_ro(zvol_state_t *zv, int flags);
 void zvol_os_set_capacity(zvol_state_t *zv, uint64_t capacity);
 
diff --git a/sys/contrib/openzfs/include/zfs_deleg.h b/sys/contrib/openzfs/include/zfs_deleg.h
index f80fe46d35f8..a7bbf1620ad5 100644
--- a/sys/contrib/openzfs/include/zfs_deleg.h
+++ b/sys/contrib/openzfs/include/zfs_deleg.h
@@ -55,6 +55,7 @@ typedef enum {
 	ZFS_DELEG_NOTE_PROMOTE,
 	ZFS_DELEG_NOTE_RENAME,
 	ZFS_DELEG_NOTE_SEND,
+	ZFS_DELEG_NOTE_SEND_RAW,
 	ZFS_DELEG_NOTE_RECEIVE,
 	ZFS_DELEG_NOTE_ALLOW,
 	ZFS_DELEG_NOTE_USERPROP,
diff --git a/sys/contrib/openzfs/lib/libicp/Makefile.am b/sys/contrib/openzfs/lib/libicp/Makefile.am
index ce24d13a760f..23adba10bc44 100644
--- a/sys/contrib/openzfs/lib/libicp/Makefile.am
+++ b/sys/contrib/openzfs/lib/libicp/Makefile.am
@@ -69,6 +69,7 @@ nodist_libicp_la_SOURCES += \
 	module/icp/asm-x86_64/aes/aes_aesni.S \
 	module/icp/asm-x86_64/modes/gcm_pclmulqdq.S \
 	module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S \
+	module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S \
 	module/icp/asm-x86_64/modes/ghash-x86_64.S \
 	module/icp/asm-x86_64/sha2/sha256-x86_64.S \
 	module/icp/asm-x86_64/sha2/sha512-x86_64.S \
diff --git a/sys/contrib/openzfs/lib/libspl/Makefile.am b/sys/contrib/openzfs/lib/libspl/Makefile.am
index 6640ecd582a7..0fd907d3011e 100644
--- a/sys/contrib/openzfs/lib/libspl/Makefile.am
+++ b/sys/contrib/openzfs/lib/libspl/Makefile.am
@@ -2,6 +2,9 @@ include $(srcdir)/%D%/include/Makefile.am
 
 libspl_assert_la_CFLAGS = $(AM_CFLAGS) $(LIBRARY_CFLAGS) $(LIBUNWIND_CFLAGS)
 libspl_la_CFLAGS        = $(libspl_assert_la_CFLAGS)
+if TARGET_CPU_I386
+libspl_la_CFLAGS        += $(NO_ATOMIC_ALIGNMENT)
+endif
 
 noinst_LTLIBRARIES += libspl_assert.la libspl.la
 CPPCHECKTARGETS    += libspl_assert.la libspl.la
diff --git a/sys/contrib/openzfs/lib/libspl/include/os/linux/sys/stat.h b/sys/contrib/openzfs/lib/libspl/include/os/linux/sys/stat.h
index a605af962a6d..13cc0b46ac93 100644
--- a/sys/contrib/openzfs/lib/libspl/include/os/linux/sys/stat.h
+++ b/sys/contrib/openzfs/lib/libspl/include/os/linux/sys/stat.h
@@ -33,7 +33,7 @@
 
 #ifdef HAVE_STATX
 #include <fcntl.h>
-#include <linux/stat.h>
+#include <sys/stat.h>
 #endif
 
 /*
diff --git a/sys/contrib/openzfs/lib/libspl/include/sys/debug.h b/sys/contrib/openzfs/lib/libspl/include/sys/debug.h
index cced309bd1bb..02f33a68b75b 100644
--- a/sys/contrib/openzfs/lib/libspl/include/sys/debug.h
+++ b/sys/contrib/openzfs/lib/libspl/include/sys/debug.h
@@ -38,4 +38,8 @@
 #define	__maybe_unused __attribute__((unused))
 #endif
 
+#ifndef __must_check
+#define	__must_check __attribute__((warn_unused_result))
+#endif
+
 #endif
diff --git a/sys/contrib/openzfs/lib/libspl/include/sys/simd.h b/sys/contrib/openzfs/lib/libspl/include/sys/simd.h
index 1ef24f5a7d39..4772a5416b2e 100644
--- a/sys/contrib/openzfs/lib/libspl/include/sys/simd.h
+++ b/sys/contrib/openzfs/lib/libspl/include/sys/simd.h
@@ -102,7 +102,9 @@ typedef enum cpuid_inst_sets {
 	AES,
 	PCLMULQDQ,
 	MOVBE,
-	SHA_NI
+	SHA_NI,
+	VAES,
+	VPCLMULQDQ
 } cpuid_inst_sets_t;
 
 /*
@@ -127,6 +129,8 @@ typedef struct cpuid_feature_desc {
 #define	_AES_BIT		(1U << 25)
 #define	_PCLMULQDQ_BIT		(1U << 1)
 #define	_MOVBE_BIT		(1U << 22)
+#define	_VAES_BIT		(1U << 9)
+#define	_VPCLMULQDQ_BIT		(1U << 10)
 #define	_SHA_NI_BIT		(1U << 29)
 
 /*
@@ -157,6 +161,8 @@ static const cpuid_feature_desc_t cpuid_features[] = {
 	[PCLMULQDQ]	= {1U, 0U, _PCLMULQDQ_BIT,	ECX	},
 	[MOVBE]		= {1U, 0U, _MOVBE_BIT,		ECX	},
 	[SHA_NI]	= {7U, 0U, _SHA_NI_BIT,		EBX	},
+	[VAES]		= {7U, 0U, _VAES_BIT,		ECX	},
+	[VPCLMULQDQ]	= {7U, 0U, _VPCLMULQDQ_BIT,	ECX	},
 };
 
 /*
@@ -231,6 +237,8 @@ CPUID_FEATURE_CHECK(aes, AES);
 CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);
 CPUID_FEATURE_CHECK(movbe, MOVBE);
 CPUID_FEATURE_CHECK(shani, SHA_NI);
+CPUID_FEATURE_CHECK(vaes, VAES);
+CPUID_FEATURE_CHECK(vpclmulqdq, VPCLMULQDQ);
 
 /*
  * Detect register set support
@@ -382,6 +390,24 @@ zfs_shani_available(void)
 }
 
 /*
+ * Check if VAES instruction is available
+ */
+static inline boolean_t
+zfs_vaes_available(void)
+{
+	return (__cpuid_has_vaes());
+}
+
+/*
+ * Check if VPCLMULQDQ instruction is available
+ */
+static inline boolean_t
+zfs_vpclmulqdq_available(void)
+{
+	return (__cpuid_has_vpclmulqdq());
+}
+
+/*
  * AVX-512 family of instruction sets:
  *
  * AVX512F	Foundation
diff --git a/sys/contrib/openzfs/lib/libspl/include/sys/time.h b/sys/contrib/openzfs/lib/libspl/include/sys/time.h
index da80a5852ae5..062c6ec979fc 100644
--- a/sys/contrib/openzfs/lib/libspl/include/sys/time.h
+++ b/sys/contrib/openzfs/lib/libspl/include/sys/time.h
@@ -98,6 +98,15 @@ gethrestime_sec(void)
 }
 
 static inline hrtime_t
+getlrtime(void)
+{
+	struct timeval tv;
+	(void) gettimeofday(&tv, NULL);
+	return ((((uint64_t)tv.tv_sec) * NANOSEC) +
+	    ((uint64_t)tv.tv_usec * NSEC_PER_USEC));
+}
+
+static inline hrtime_t
 gethrtime(void)
 {
 	struct timespec ts;
diff --git a/sys/contrib/openzfs/lib/libspl/include/sys/uio.h b/sys/contrib/openzfs/lib/libspl/include/sys/uio.h
index 93aa4984d734..9ada482be000 100644
--- a/sys/contrib/openzfs/lib/libspl/include/sys/uio.h
+++ b/sys/contrib/openzfs/lib/libspl/include/sys/uio.h
@@ -41,6 +41,7 @@
 #ifndef	_LIBSPL_SYS_UIO_H
 #define	_LIBSPL_SYS_UIO_H
 
+#include <sys/sysmacros.h>
 #include <sys/types.h>
 #include_next <sys/uio.h>
 
diff --git a/sys/contrib/openzfs/lib/libuutil/libuutil.abi b/sys/contrib/openzfs/lib/libuutil/libuutil.abi
index 6c736c61e4a5..2a740afa07ca 100644
--- a/sys/contrib/openzfs/lib/libuutil/libuutil.abi
+++ b/sys/contrib/openzfs/lib/libuutil/libuutil.abi
@@ -616,6 +616,7 @@
     <array-type-def dimensions='1' type-id='de572c22' size-in-bits='1472' id='6d3c2f42'>
       <subrange length='23' type-id='7359adad' id='fdd0f594'/>
     </array-type-def>
+    <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/>
     <array-type-def dimensions='1' type-id='3a47d82b' size-in-bits='256' id='a133ec23'>
       <subrange length='4' type-id='7359adad' id='16fe7105'/>
     </array-type-def>
@@ -1020,13 +1021,6 @@
     <array-type-def dimensions='1' type-id='03085adc' size-in-bits='192' id='083f8d58'>
       <subrange length='3' type-id='7359adad' id='56f209d2'/>
     </array-type-def>
-    <array-type-def dimensions='1' type-id='d315442e' size-in-bits='16' id='811205dc'>
-      <subrange length='1' type-id='7359adad' id='52f813b4'/>
-    </array-type-def>
-    <array-type-def dimensions='1' type-id='d3130597' size-in-bits='768' id='f63f23b9'>
-      <subrange length='12' type-id='7359adad' id='84827bdc'/>
-    </array-type-def>
-    <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/>
     <class-decl name='mnttab' size-in-bits='256' is-struct='yes' visibility='default' id='1b055409'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='mnt_special' type-id='26a90f95' visibility='default'/>
@@ -1061,93 +1055,6 @@
         <var-decl name='mnt_minor' type-id='3502e3ff' visibility='default'/>
       </data-member>
     </class-decl>
-    <typedef-decl name='__u16' type-id='8efea9e5' id='d315442e'/>
-    <typedef-decl name='__s32' type-id='95e97e5e' id='3158a266'/>
-    <typedef-decl name='__u32' type-id='f0981eeb' id='3f1a6b60'/>
-    <typedef-decl name='__s64' type-id='1eb56b1e' id='49659421'/>
-    <typedef-decl name='__u64' type-id='3a47d82b' id='d3130597'/>
-    <class-decl name='statx_timestamp' size-in-bits='128' is-struct='yes' visibility='default' id='94101016'>
-      <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='tv_sec' type-id='49659421' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='64'>
-        <var-decl name='tv_nsec' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='96'>
-        <var-decl name='__reserved' type-id='3158a266' visibility='default'/>
-      </data-member>
-    </class-decl>
-    <class-decl name='statx' size-in-bits='2048' is-struct='yes' visibility='default' id='720b04c5'>
-      <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='stx_mask' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='32'>
-        <var-decl name='stx_blksize' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='64'>
-        <var-decl name='stx_attributes' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='128'>
-        <var-decl name='stx_nlink' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='160'>
-        <var-decl name='stx_uid' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='192'>
-        <var-decl name='stx_gid' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='224'>
-        <var-decl name='stx_mode' type-id='d315442e' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='240'>
-        <var-decl name='__spare0' type-id='811205dc' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='256'>
-        <var-decl name='stx_ino' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='320'>
-        <var-decl name='stx_size' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='384'>
-        <var-decl name='stx_blocks' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='448'>
-        <var-decl name='stx_attributes_mask' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='512'>
-        <var-decl name='stx_atime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='640'>
-        <var-decl name='stx_btime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='768'>
-        <var-decl name='stx_ctime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='896'>
-        <var-decl name='stx_mtime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1024'>
-        <var-decl name='stx_rdev_major' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1056'>
-        <var-decl name='stx_rdev_minor' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1088'>
-        <var-decl name='stx_dev_major' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1120'>
-        <var-decl name='stx_dev_minor' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1152'>
-        <var-decl name='stx_mnt_id' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1216'>
-        <var-decl name='__spare2' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1280'>
-        <var-decl name='__spare3' type-id='f63f23b9' visibility='default'/>
-      </data-member>
-    </class-decl>
     <class-decl name='mntent' size-in-bits='320' is-struct='yes' visibility='default' id='56fe4a37'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='mnt_fsname' type-id='26a90f95' visibility='default'/>
@@ -1237,8 +1144,6 @@
     <pointer-type-def type-id='1b055409' size-in-bits='64' id='9d424d31'/>
     <pointer-type-def type-id='0bbec9cd' size-in-bits='64' id='62f7a03d'/>
     <qualified-type-def type-id='62f7a03d' restrict='yes' id='f1cadedf'/>
-    <pointer-type-def type-id='720b04c5' size-in-bits='64' id='936b8e35'/>
-    <qualified-type-def type-id='936b8e35' restrict='yes' id='31d265b7'/>
     <function-decl name='getmntent_r' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='e75a27e9'/>
       <parameter type-id='3cad23cd'/>
@@ -1254,14 +1159,6 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='26a90f95'/>
     </function-decl>
-    <function-decl name='statx' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='9d26089a'/>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='f0981eeb'/>
-      <parameter type-id='31d265b7'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
     <function-decl name='__fprintf_chk' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='e75a27e9'/>
       <parameter type-id='95e97e5e'/>
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs.abi b/sys/contrib/openzfs/lib/libzfs/libzfs.abi
index ba161d1ef10f..f988d27a286a 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs.abi
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs.abi
@@ -571,6 +571,7 @@
     <elf-symbol name='zpool_props_refresh' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_read_label' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_refresh_stats' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_refresh_stats_from_handle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_reguid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_reopen_one' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_scan' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -641,7 +642,7 @@
     <elf-symbol name='sa_protocol_names' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='spa_feature_table' size='2632' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfeature_checks_disable' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
-    <elf-symbol name='zfs_deleg_perm_tab' size='528' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zfs_deleg_perm_tab' size='544' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_history_event_names' size='328' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_max_dataset_nesting' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_userquota_prop_prefixes' size='96' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -1458,103 +1459,8 @@
     </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='lib/libspl/os/linux/getmntany.c' language='LANG_C99'>
-    <array-type-def dimensions='1' type-id='d315442e' size-in-bits='16' id='811205dc'>
-      <subrange length='1' type-id='7359adad' id='52f813b4'/>
-    </array-type-def>
-    <array-type-def dimensions='1' type-id='d3130597' size-in-bits='768' id='f63f23b9'>
-      <subrange length='12' type-id='7359adad' id='84827bdc'/>
-    </array-type-def>
-    <typedef-decl name='__u16' type-id='8efea9e5' id='d315442e'/>
-    <typedef-decl name='__s32' type-id='95e97e5e' id='3158a266'/>
-    <typedef-decl name='__u32' type-id='f0981eeb' id='3f1a6b60'/>
-    <typedef-decl name='__s64' type-id='1eb56b1e' id='49659421'/>
-    <typedef-decl name='__u64' type-id='3a47d82b' id='d3130597'/>
-    <class-decl name='statx_timestamp' size-in-bits='128' is-struct='yes' visibility='default' id='94101016'>
-      <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='tv_sec' type-id='49659421' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='64'>
-        <var-decl name='tv_nsec' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='96'>
-        <var-decl name='__reserved' type-id='3158a266' visibility='default'/>
-      </data-member>
-    </class-decl>
-    <class-decl name='statx' size-in-bits='2048' is-struct='yes' visibility='default' id='720b04c5'>
-      <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='stx_mask' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='32'>
-        <var-decl name='stx_blksize' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='64'>
-        <var-decl name='stx_attributes' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='128'>
-        <var-decl name='stx_nlink' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='160'>
-        <var-decl name='stx_uid' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='192'>
-        <var-decl name='stx_gid' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='224'>
-        <var-decl name='stx_mode' type-id='d315442e' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='240'>
-        <var-decl name='__spare0' type-id='811205dc' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='256'>
-        <var-decl name='stx_ino' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='320'>
-        <var-decl name='stx_size' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='384'>
-        <var-decl name='stx_blocks' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='448'>
-        <var-decl name='stx_attributes_mask' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='512'>
-        <var-decl name='stx_atime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='640'>
-        <var-decl name='stx_btime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='768'>
-        <var-decl name='stx_ctime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='896'>
-        <var-decl name='stx_mtime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1024'>
-        <var-decl name='stx_rdev_major' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1056'>
-        <var-decl name='stx_rdev_minor' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1088'>
-        <var-decl name='stx_dev_major' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1120'>
-        <var-decl name='stx_dev_minor' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1152'>
-        <var-decl name='stx_mnt_id' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1216'>
-        <var-decl name='__spare2' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1280'>
-        <var-decl name='__spare3' type-id='f63f23b9' visibility='default'/>
-      </data-member>
-    </class-decl>
     <pointer-type-def type-id='56fe4a37' size-in-bits='64' id='b6b61d2f'/>
     <qualified-type-def type-id='b6b61d2f' restrict='yes' id='3cad23cd'/>
-    <pointer-type-def type-id='720b04c5' size-in-bits='64' id='936b8e35'/>
-    <qualified-type-def type-id='936b8e35' restrict='yes' id='31d265b7'/>
     <function-decl name='getmntent_r' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='e75a27e9'/>
       <parameter type-id='3cad23cd'/>
@@ -1566,14 +1472,6 @@
       <parameter type-id='822cd80b'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='statx' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='9d26089a'/>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='f0981eeb'/>
-      <parameter type-id='31d265b7'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='lib/libspl/timestamp.c' language='LANG_C99'>
     <typedef-decl name='nl_item' type-id='95e97e5e' id='03b79a94'/>
@@ -3194,6 +3092,10 @@
       <parameter type-id='dace003f'/>
       <return type-id='80f4b756'/>
     </function-decl>
+    <function-decl name='fnvlist_dup' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='22cce67b'/>
+      <return type-id='5ce45b60'/>
+    </function-decl>
     <function-decl name='fnvpair_value_nvlist' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='3fa542f0'/>
       <return type-id='5ce45b60'/>
@@ -3238,6 +3140,11 @@
       <parameter type-id='37e3bd22' name='missing'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='zpool_refresh_stats_from_handle' mangled-name='zpool_refresh_stats_from_handle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_refresh_stats_from_handle'>
+      <parameter type-id='4c81de99' name='dzhp'/>
+      <parameter type-id='4c81de99' name='szhp'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='zpool_skip_pool' mangled-name='zpool_skip_pool' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_skip_pool'>
       <parameter type-id='80f4b756' name='poolname'/>
       <return type-id='c19b74c3'/>
@@ -6117,7 +6024,9 @@
       <enumerator name='VDEV_PROP_TRIM_SUPPORT' value='49'/>
       <enumerator name='VDEV_PROP_TRIM_ERRORS' value='50'/>
       <enumerator name='VDEV_PROP_SLOW_IOS' value='51'/>
-      <enumerator name='VDEV_NUM_PROPS' value='52'/>
+      <enumerator name='VDEV_PROP_SIT_OUT' value='52'/>
+      <enumerator name='VDEV_PROP_AUTOSIT' value='53'/>
+      <enumerator name='VDEV_NUM_PROPS' value='54'/>
     </enum-decl>
     <typedef-decl name='vdev_prop_t' type-id='1573bec8' id='5aa5c90c'/>
     <class-decl name='zpool_load_policy' size-in-bits='256' is-struct='yes' visibility='default' id='2f65b36f'>
@@ -9396,10 +9305,6 @@
       <parameter type-id='5ce45b60'/>
       <return type-id='48b5725f'/>
     </function-decl>
-    <function-decl name='fnvlist_dup' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='22cce67b'/>
-      <return type-id='5ce45b60'/>
-    </function-decl>
     <function-decl name='spl_pagesize' mangled-name='spl_pagesize' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='spl_pagesize'>
       <return type-id='b59d7dce'/>
     </function-decl>
@@ -9772,8 +9677,8 @@
     </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='module/zcommon/zfs_deleg.c' language='LANG_C99'>
-    <array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='4224' id='55e705e7'>
-      <subrange length='33' type-id='7359adad' id='6a5934df'/>
+    <array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='4352' id='55f84f08'>
+      <subrange length='34' type-id='7359adad' id='6a6a7e00'/>
     </array-type-def>
     <array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='infinite' id='7c00e69d'>
       <subrange length='infinite' id='031f2035'/>
@@ -9803,30 +9708,31 @@
       <enumerator name='ZFS_DELEG_NOTE_PROMOTE' value='5'/>
       <enumerator name='ZFS_DELEG_NOTE_RENAME' value='6'/>
       <enumerator name='ZFS_DELEG_NOTE_SEND' value='7'/>
-      <enumerator name='ZFS_DELEG_NOTE_RECEIVE' value='8'/>
-      <enumerator name='ZFS_DELEG_NOTE_ALLOW' value='9'/>
-      <enumerator name='ZFS_DELEG_NOTE_USERPROP' value='10'/>
-      <enumerator name='ZFS_DELEG_NOTE_MOUNT' value='11'/>
-      <enumerator name='ZFS_DELEG_NOTE_SHARE' value='12'/>
-      <enumerator name='ZFS_DELEG_NOTE_USERQUOTA' value='13'/>
-      <enumerator name='ZFS_DELEG_NOTE_GROUPQUOTA' value='14'/>
-      <enumerator name='ZFS_DELEG_NOTE_USERUSED' value='15'/>
-      <enumerator name='ZFS_DELEG_NOTE_GROUPUSED' value='16'/>
-      <enumerator name='ZFS_DELEG_NOTE_USEROBJQUOTA' value='17'/>
-      <enumerator name='ZFS_DELEG_NOTE_GROUPOBJQUOTA' value='18'/>
-      <enumerator name='ZFS_DELEG_NOTE_USEROBJUSED' value='19'/>
-      <enumerator name='ZFS_DELEG_NOTE_GROUPOBJUSED' value='20'/>
-      <enumerator name='ZFS_DELEG_NOTE_HOLD' value='21'/>
-      <enumerator name='ZFS_DELEG_NOTE_RELEASE' value='22'/>
-      <enumerator name='ZFS_DELEG_NOTE_DIFF' value='23'/>
-      <enumerator name='ZFS_DELEG_NOTE_BOOKMARK' value='24'/>
-      <enumerator name='ZFS_DELEG_NOTE_LOAD_KEY' value='25'/>
-      <enumerator name='ZFS_DELEG_NOTE_CHANGE_KEY' value='26'/>
-      <enumerator name='ZFS_DELEG_NOTE_PROJECTUSED' value='27'/>
-      <enumerator name='ZFS_DELEG_NOTE_PROJECTQUOTA' value='28'/>
-      <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJUSED' value='29'/>
-      <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJQUOTA' value='30'/>
-      <enumerator name='ZFS_DELEG_NOTE_NONE' value='31'/>
+      <enumerator name='ZFS_DELEG_NOTE_SEND_RAW' value='8'/>
+      <enumerator name='ZFS_DELEG_NOTE_RECEIVE' value='9'/>
+      <enumerator name='ZFS_DELEG_NOTE_ALLOW' value='10'/>
+      <enumerator name='ZFS_DELEG_NOTE_USERPROP' value='11'/>
+      <enumerator name='ZFS_DELEG_NOTE_MOUNT' value='12'/>
+      <enumerator name='ZFS_DELEG_NOTE_SHARE' value='13'/>
+      <enumerator name='ZFS_DELEG_NOTE_USERQUOTA' value='14'/>
+      <enumerator name='ZFS_DELEG_NOTE_GROUPQUOTA' value='15'/>
+      <enumerator name='ZFS_DELEG_NOTE_USERUSED' value='16'/>
+      <enumerator name='ZFS_DELEG_NOTE_GROUPUSED' value='17'/>
+      <enumerator name='ZFS_DELEG_NOTE_USEROBJQUOTA' value='18'/>
+      <enumerator name='ZFS_DELEG_NOTE_GROUPOBJQUOTA' value='19'/>
+      <enumerator name='ZFS_DELEG_NOTE_USEROBJUSED' value='20'/>
+      <enumerator name='ZFS_DELEG_NOTE_GROUPOBJUSED' value='21'/>
+      <enumerator name='ZFS_DELEG_NOTE_HOLD' value='22'/>
+      <enumerator name='ZFS_DELEG_NOTE_RELEASE' value='23'/>
+      <enumerator name='ZFS_DELEG_NOTE_DIFF' value='24'/>
+      <enumerator name='ZFS_DELEG_NOTE_BOOKMARK' value='25'/>
+      <enumerator name='ZFS_DELEG_NOTE_LOAD_KEY' value='26'/>
+      <enumerator name='ZFS_DELEG_NOTE_CHANGE_KEY' value='27'/>
+      <enumerator name='ZFS_DELEG_NOTE_PROJECTUSED' value='28'/>
+      <enumerator name='ZFS_DELEG_NOTE_PROJECTQUOTA' value='29'/>
+      <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJUSED' value='30'/>
+      <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJQUOTA' value='31'/>
+      <enumerator name='ZFS_DELEG_NOTE_NONE' value='32'/>
     </enum-decl>
     <typedef-decl name='zfs_deleg_note_t' type-id='729d4547' id='4613c173'/>
     <class-decl name='zfs_deleg_perm_tab' size-in-bits='128' is-struct='yes' visibility='default' id='5aa05c1f'>
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_config.c b/sys/contrib/openzfs/lib/libzfs/libzfs_config.c
index 0d2102191389..9d704e4303ff 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_config.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_config.c
@@ -308,6 +308,23 @@ zpool_refresh_stats(zpool_handle_t *zhp, boolean_t *missing)
 }
 
 /*
+ * Copies the pool config and state from szhp to dzhp. szhp and dzhp must
+ * represent the same pool. Used by pool_list_refresh() to avoid another
+ * round-trip into the kernel to get stats already collected earlier in the
+ * function.
+ */
+void
+zpool_refresh_stats_from_handle(zpool_handle_t *dzhp, zpool_handle_t *szhp)
+{
+	VERIFY0(strcmp(dzhp->zpool_name, szhp->zpool_name));
+	nvlist_free(dzhp->zpool_old_config);
+	dzhp->zpool_old_config = dzhp->zpool_config;
+	dzhp->zpool_config = fnvlist_dup(szhp->zpool_config);
+	dzhp->zpool_config_size = szhp->zpool_config_size;
+	dzhp->zpool_state = szhp->zpool_state;
+}
+
+/*
  * The following environment variables are undocumented
  * and should be used for testing purposes only:
  *
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_diff.c b/sys/contrib/openzfs/lib/libzfs/libzfs_diff.c
index 6aa0375f98d7..5f50bce531f7 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_diff.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_diff.c
@@ -81,7 +81,7 @@ get_stats_for_obj(differ_info_t *di, const char *dsname, uint64_t obj,
 	/* we can get stats even if we failed to get a path */
 	(void) memcpy(sb, &zc.zc_stat, sizeof (zfs_stat_t));
 	if (error == 0) {
-		ASSERT(di->zerr == 0);
+		ASSERT0(di->zerr);
 		(void) strlcpy(pn, zc.zc_value, maxlen);
 		return (0);
 	}
@@ -404,7 +404,7 @@ write_free_diffs(FILE *fp, differ_info_t *di, dmu_diff_record_t *dr)
 	(void) strlcpy(zc.zc_name, di->fromsnap, sizeof (zc.zc_name));
 	zc.zc_obj = dr->ddr_first - 1;
 
-	ASSERT(di->zerr == 0);
+	ASSERT0(di->zerr);
 
 	while (zc.zc_obj < dr->ddr_last) {
 		int err;
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_import.c b/sys/contrib/openzfs/lib/libzfs/libzfs_import.c
index 599e8e6f7819..7f276e9592c9 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_import.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_import.c
@@ -122,7 +122,7 @@ const pool_config_ops_t libzfs_config_ops = {
 static uint64_t
 label_offset(uint64_t size, int l)
 {
-	ASSERT(P2PHASE_TYPED(size, sizeof (vdev_label_t), uint64_t) == 0);
+	ASSERT0(P2PHASE_TYPED(size, sizeof (vdev_label_t), uint64_t));
 	return (l * sizeof (vdev_label_t) + (l < VDEV_LABELS / 2 ?
 	    0 : size - VDEV_LABELS * sizeof (vdev_label_t)));
 }
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_mount.c b/sys/contrib/openzfs/lib/libzfs/libzfs_mount.c
index 2a81b658d342..5c9e2199eed4 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_mount.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_mount.c
@@ -516,7 +516,7 @@ zfs_mount_at(zfs_handle_t *zhp, const char *options, int flags,
 		} else if (rc == ENOTSUP) {
 			int spa_version;
 
-			VERIFY(zfs_spa_version(zhp, &spa_version) == 0);
+			VERIFY0(zfs_spa_version(zhp, &spa_version));
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 			    "Can't mount a version %llu "
 			    "file system on a version %d pool. Pool must be"
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c b/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c
index 10b42720e963..ce154ae1a4cd 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_pool.c
@@ -5549,6 +5549,8 @@ zpool_get_vdev_prop_value(nvlist_t *nvprop, vdev_prop_t prop, char *prop_name,
 			/* Only use if provided by the RAIDZ VDEV above */
 			if (prop == VDEV_PROP_RAIDZ_EXPANDING)
 				return (ENOENT);
+			if (prop == VDEV_PROP_SIT_OUT)
+				return (ENOENT);
 		}
 		if (vdev_prop_index_to_string(prop, intval,
 		    (const char **)&strval) != 0)
@@ -5718,8 +5720,16 @@ zpool_set_vdev_prop(zpool_handle_t *zhp, const char *vdevname,
 	nvlist_free(nvl);
 	nvlist_free(outnvl);
 
-	if (ret)
-		(void) zpool_standard_error(zhp->zpool_hdl, errno, errbuf);
+	if (ret) {
+		if (errno == ENOTSUP) {
+			zfs_error_aux(zhp->zpool_hdl, dgettext(TEXT_DOMAIN,
+			    "property not supported for this vdev"));
+			(void) zfs_error(zhp->zpool_hdl, EZFS_PROPTYPE, errbuf);
+		} else {
+			(void) zpool_standard_error(zhp->zpool_hdl, errno,
+			    errbuf);
+		}
+	}
 
 	return (ret);
 }
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c b/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c
index 1ad10ebb3c15..77134d197904 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c
@@ -2505,7 +2505,7 @@ zfs_send_cb_impl(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 		err = ENOENT;
 
 	if (sdd.cleanup_fd != -1) {
-		VERIFY(0 == close(sdd.cleanup_fd));
+		VERIFY0(close(sdd.cleanup_fd));
 		sdd.cleanup_fd = -1;
 	}
 
@@ -2531,7 +2531,7 @@ err_out:
 	fnvlist_free(sdd.snapholds);
 
 	if (sdd.cleanup_fd != -1)
-		VERIFY(0 == close(sdd.cleanup_fd));
+		VERIFY0(close(sdd.cleanup_fd));
 	return (err);
 }
 
@@ -5108,7 +5108,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 		nvlist_t *holds, *errors = NULL;
 		int cleanup_fd = -1;
 
-		VERIFY(0 == nvlist_alloc(&holds, 0, KM_SLEEP));
+		VERIFY0(nvlist_alloc(&holds, 0, KM_SLEEP));
 		for (pair = nvlist_next_nvpair(snapholds_nvlist, NULL);
 		    pair != NULL;
 		    pair = nvlist_next_nvpair(snapholds_nvlist, pair)) {
@@ -5560,7 +5560,7 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap,
 			if ((cp = strchr(nonpackage_sendfs, '@')) != NULL)
 				*cp = '\0';
 			sendfs = nonpackage_sendfs;
-			VERIFY(finalsnap == NULL);
+			VERIFY0P(finalsnap);
 		}
 		return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags,
 		    &drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs,
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_status.c b/sys/contrib/openzfs/lib/libzfs/libzfs_status.c
index bdddefb92165..a589ca6896f0 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_status.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_status.c
@@ -98,57 +98,57 @@ static const char *const zfs_msgid_table[] = {
 #define	NMSGID	(sizeof (zfs_msgid_table) / sizeof (zfs_msgid_table[0]))
 
 static int
-vdev_missing(vdev_stat_t *vs, uint_t vsc)
+vdev_missing(vdev_stat_t *vs, uint_t vsc, void *arg)
 {
-	(void) vsc;
+	(void) vsc, (void) arg;
 	return (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 	    vs->vs_aux == VDEV_AUX_OPEN_FAILED);
 }
 
 static int
-vdev_faulted(vdev_stat_t *vs, uint_t vsc)
+vdev_faulted(vdev_stat_t *vs, uint_t vsc, void *arg)
 {
-	(void) vsc;
+	(void) vsc, (void) arg;
 	return (vs->vs_state == VDEV_STATE_FAULTED);
 }
 
 static int
-vdev_errors(vdev_stat_t *vs, uint_t vsc)
+vdev_errors(vdev_stat_t *vs, uint_t vsc, void *arg)
 {
-	(void) vsc;
+	(void) vsc, (void) arg;
 	return (vs->vs_state == VDEV_STATE_DEGRADED ||
 	    vs->vs_read_errors != 0 || vs->vs_write_errors != 0 ||
 	    vs->vs_checksum_errors != 0);
 }
 
 static int
-vdev_broken(vdev_stat_t *vs, uint_t vsc)
+vdev_broken(vdev_stat_t *vs, uint_t vsc, void *arg)
 {
-	(void) vsc;
+	(void) vsc, (void) arg;
 	return (vs->vs_state == VDEV_STATE_CANT_OPEN);
 }
 
 static int
-vdev_offlined(vdev_stat_t *vs, uint_t vsc)
+vdev_offlined(vdev_stat_t *vs, uint_t vsc, void *arg)
 {
-	(void) vsc;
+	(void) vsc, (void) arg;
 	return (vs->vs_state == VDEV_STATE_OFFLINE);
 }
 
 static int
-vdev_removed(vdev_stat_t *vs, uint_t vsc)
+vdev_removed(vdev_stat_t *vs, uint_t vsc, void *arg)
 {
-	(void) vsc;
+	(void) vsc, (void) arg;
 	return (vs->vs_state == VDEV_STATE_REMOVED);
 }
 
 static int
-vdev_non_native_ashift(vdev_stat_t *vs, uint_t vsc)
+vdev_non_native_ashift(vdev_stat_t *vs, uint_t vsc, void *arg)
 {
-	if (getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") != NULL)
-		return (0);
+	uint64_t ashift = *(uint64_t *)arg;
 
 	return (VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
+	    (ashift == 0 || vs->vs_configured_ashift < ashift) &&
 	    vs->vs_configured_ashift < vs->vs_physical_ashift);
 }
 
@@ -156,8 +156,8 @@ vdev_non_native_ashift(vdev_stat_t *vs, uint_t vsc)
  * Detect if any leaf devices that have seen errors or could not be opened.
  */
 static boolean_t
-find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t),
-    boolean_t ignore_replacing)
+find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t, void *),
+    void *arg, boolean_t ignore_replacing)
 {
 	nvlist_t **child;
 	uint_t c, children;
@@ -177,14 +177,16 @@ find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t),
 
 	if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_CHILDREN, &child,
 	    &children) == 0) {
-		for (c = 0; c < children; c++)
-			if (find_vdev_problem(child[c], func, ignore_replacing))
+		for (c = 0; c < children; c++) {
+			if (find_vdev_problem(child[c], func, arg,
+			    ignore_replacing))
 				return (B_TRUE);
+		}
 	} else {
 		uint_t vsc;
 		vdev_stat_t *vs = (vdev_stat_t *)fnvlist_lookup_uint64_array(
 		    vdev, ZPOOL_CONFIG_VDEV_STATS, &vsc);
-		if (func(vs, vsc) != 0)
+		if (func(vs, vsc, arg) != 0)
 			return (B_TRUE);
 	}
 
@@ -193,9 +195,11 @@ find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t),
 	 */
 	if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_L2CACHE, &child,
 	    &children) == 0) {
-		for (c = 0; c < children; c++)
-			if (find_vdev_problem(child[c], func, ignore_replacing))
+		for (c = 0; c < children; c++) {
+			if (find_vdev_problem(child[c], func, arg,
+			    ignore_replacing))
 				return (B_TRUE);
+		}
 	}
 
 	return (B_FALSE);
@@ -220,7 +224,7 @@ find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t),
  */
 static zpool_status_t
 check_status(nvlist_t *config, boolean_t isimport,
-    zpool_errata_t *erratap, const char *compat)
+    zpool_errata_t *erratap, const char *compat, uint64_t ashift)
 {
 	pool_scan_stat_t *ps = NULL;
 	uint_t vsc, psc;
@@ -371,15 +375,15 @@ check_status(nvlist_t *config, boolean_t isimport,
 	 * Bad devices in non-replicated config.
 	 */
 	if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
-	    find_vdev_problem(nvroot, vdev_faulted, B_TRUE))
+	    find_vdev_problem(nvroot, vdev_faulted, NULL, B_TRUE))
 		return (ZPOOL_STATUS_FAULTED_DEV_NR);
 
 	if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
-	    find_vdev_problem(nvroot, vdev_missing, B_TRUE))
+	    find_vdev_problem(nvroot, vdev_missing, NULL, B_TRUE))
 		return (ZPOOL_STATUS_MISSING_DEV_NR);
 
 	if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
-	    find_vdev_problem(nvroot, vdev_broken, B_TRUE))
+	    find_vdev_problem(nvroot, vdev_broken, NULL, B_TRUE))
 		return (ZPOOL_STATUS_CORRUPT_LABEL_NR);
 
 	/*
@@ -402,35 +406,37 @@ check_status(nvlist_t *config, boolean_t isimport,
 	/*
 	 * Missing devices in a replicated config.
 	 */
-	if (find_vdev_problem(nvroot, vdev_faulted, B_TRUE))
+	if (find_vdev_problem(nvroot, vdev_faulted, NULL, B_TRUE))
 		return (ZPOOL_STATUS_FAULTED_DEV_R);
-	if (find_vdev_problem(nvroot, vdev_missing, B_TRUE))
+	if (find_vdev_problem(nvroot, vdev_missing, NULL, B_TRUE))
 		return (ZPOOL_STATUS_MISSING_DEV_R);
-	if (find_vdev_problem(nvroot, vdev_broken, B_TRUE))
+	if (find_vdev_problem(nvroot, vdev_broken, NULL, B_TRUE))
 		return (ZPOOL_STATUS_CORRUPT_LABEL_R);
 
 	/*
 	 * Devices with errors
 	 */
-	if (!isimport && find_vdev_problem(nvroot, vdev_errors, B_TRUE))
+	if (!isimport && find_vdev_problem(nvroot, vdev_errors, NULL, B_TRUE))
 		return (ZPOOL_STATUS_FAILING_DEV);
 
 	/*
 	 * Offlined devices
 	 */
-	if (find_vdev_problem(nvroot, vdev_offlined, B_TRUE))
+	if (find_vdev_problem(nvroot, vdev_offlined, NULL, B_TRUE))
 		return (ZPOOL_STATUS_OFFLINE_DEV);
 
 	/*
 	 * Removed device
 	 */
-	if (find_vdev_problem(nvroot, vdev_removed, B_TRUE))
+	if (find_vdev_problem(nvroot, vdev_removed, NULL, B_TRUE))
 		return (ZPOOL_STATUS_REMOVED_DEV);
 
 	/*
 	 * Suboptimal, but usable, ashift configuration.
 	 */
-	if (find_vdev_problem(nvroot, vdev_non_native_ashift, B_FALSE))
+	if (!isimport &&
+	    getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL &&
+	    find_vdev_problem(nvroot, vdev_non_native_ashift, &ashift, B_FALSE))
 		return (ZPOOL_STATUS_NON_NATIVE_ASHIFT);
 
 	/*
@@ -510,8 +516,10 @@ zpool_get_status(zpool_handle_t *zhp, const char **msgid,
 	    ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
 		compatibility[0] = '\0';
 
+	uint64_t ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, NULL);
+
 	zpool_status_t ret = check_status(zhp->zpool_config, B_FALSE, errata,
-	    compatibility);
+	    compatibility, ashift);
 
 	if (msgid != NULL) {
 		if (ret >= NMSGID)
@@ -526,7 +534,7 @@ zpool_status_t
 zpool_import_status(nvlist_t *config, const char **msgid,
     zpool_errata_t *errata)
 {
-	zpool_status_t ret = check_status(config, B_TRUE, errata, NULL);
+	zpool_status_t ret = check_status(config, B_TRUE, errata, NULL, 0);
 
 	if (ret >= NMSGID)
 		*msgid = NULL;
diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_util.c b/sys/contrib/openzfs/lib/libzfs/libzfs_util.c
index 4edddc2a759b..26f5135dff62 100644
--- a/sys/contrib/openzfs/lib/libzfs/libzfs_util.c
+++ b/sys/contrib/openzfs/lib/libzfs/libzfs_util.c
@@ -776,6 +776,11 @@ zpool_standard_error_fmt(libzfs_handle_t *hdl, int error, const char *fmt, ...)
 	case ZFS_ERR_ASHIFT_MISMATCH:
 		zfs_verror(hdl, EZFS_ASHIFT_MISMATCH, fmt, ap);
 		break;
+	case ZFS_ERR_TOO_MANY_SITOUTS:
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "too many disks "
+		    "already sitting out"));
+		zfs_verror(hdl, EZFS_BUSY, fmt, ap);
+		break;
 	default:
 		zfs_error_aux(hdl, "%s", zfs_strerror(error));
 		zfs_verror(hdl, EZFS_UNKNOWN, fmt, ap);
diff --git a/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi b/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi
index 7464b3adb254..263cad045f7a 100644
--- a/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi
+++ b/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi
@@ -617,6 +617,7 @@
     <array-type-def dimensions='1' type-id='de572c22' size-in-bits='1472' id='6d3c2f42'>
       <subrange length='23' type-id='7359adad' id='fdd0f594'/>
     </array-type-def>
+    <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/>
     <array-type-def dimensions='1' type-id='3a47d82b' size-in-bits='256' id='a133ec23'>
       <subrange length='4' type-id='7359adad' id='16fe7105'/>
     </array-type-def>
@@ -988,13 +989,6 @@
     </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='lib/libspl/os/linux/getmntany.c' language='LANG_C99'>
-    <array-type-def dimensions='1' type-id='d315442e' size-in-bits='16' id='811205dc'>
-      <subrange length='1' type-id='7359adad' id='52f813b4'/>
-    </array-type-def>
-    <array-type-def dimensions='1' type-id='d3130597' size-in-bits='768' id='f63f23b9'>
-      <subrange length='12' type-id='7359adad' id='84827bdc'/>
-    </array-type-def>
-    <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/>
     <class-decl name='mnttab' size-in-bits='256' is-struct='yes' visibility='default' id='1b055409'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='mnt_special' type-id='26a90f95' visibility='default'/>
@@ -1029,93 +1023,6 @@
         <var-decl name='mnt_minor' type-id='3502e3ff' visibility='default'/>
       </data-member>
     </class-decl>
-    <typedef-decl name='__u16' type-id='8efea9e5' id='d315442e'/>
-    <typedef-decl name='__s32' type-id='95e97e5e' id='3158a266'/>
-    <typedef-decl name='__u32' type-id='f0981eeb' id='3f1a6b60'/>
-    <typedef-decl name='__s64' type-id='1eb56b1e' id='49659421'/>
-    <typedef-decl name='__u64' type-id='3a47d82b' id='d3130597'/>
-    <class-decl name='statx_timestamp' size-in-bits='128' is-struct='yes' visibility='default' id='94101016'>
-      <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='tv_sec' type-id='49659421' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='64'>
-        <var-decl name='tv_nsec' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='96'>
-        <var-decl name='__reserved' type-id='3158a266' visibility='default'/>
-      </data-member>
-    </class-decl>
-    <class-decl name='statx' size-in-bits='2048' is-struct='yes' visibility='default' id='720b04c5'>
-      <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='stx_mask' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='32'>
-        <var-decl name='stx_blksize' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='64'>
-        <var-decl name='stx_attributes' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='128'>
-        <var-decl name='stx_nlink' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='160'>
-        <var-decl name='stx_uid' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='192'>
-        <var-decl name='stx_gid' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='224'>
-        <var-decl name='stx_mode' type-id='d315442e' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='240'>
-        <var-decl name='__spare0' type-id='811205dc' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='256'>
-        <var-decl name='stx_ino' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='320'>
-        <var-decl name='stx_size' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='384'>
-        <var-decl name='stx_blocks' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='448'>
-        <var-decl name='stx_attributes_mask' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='512'>
-        <var-decl name='stx_atime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='640'>
-        <var-decl name='stx_btime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='768'>
-        <var-decl name='stx_ctime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='896'>
-        <var-decl name='stx_mtime' type-id='94101016' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1024'>
-        <var-decl name='stx_rdev_major' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1056'>
-        <var-decl name='stx_rdev_minor' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1088'>
-        <var-decl name='stx_dev_major' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1120'>
-        <var-decl name='stx_dev_minor' type-id='3f1a6b60' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1152'>
-        <var-decl name='stx_mnt_id' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1216'>
-        <var-decl name='__spare2' type-id='d3130597' visibility='default'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1280'>
-        <var-decl name='__spare3' type-id='f63f23b9' visibility='default'/>
-      </data-member>
-    </class-decl>
     <class-decl name='mntent' size-in-bits='320' is-struct='yes' visibility='default' id='56fe4a37'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='mnt_fsname' type-id='26a90f95' visibility='default'/>
@@ -1191,8 +1098,6 @@
     <pointer-type-def type-id='1b055409' size-in-bits='64' id='9d424d31'/>
     <pointer-type-def type-id='0bbec9cd' size-in-bits='64' id='62f7a03d'/>
     <qualified-type-def type-id='62f7a03d' restrict='yes' id='f1cadedf'/>
-    <pointer-type-def type-id='720b04c5' size-in-bits='64' id='936b8e35'/>
-    <qualified-type-def type-id='936b8e35' restrict='yes' id='31d265b7'/>
     <function-decl name='getmntent_r' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='e75a27e9'/>
       <parameter type-id='3cad23cd'/>
@@ -1208,14 +1113,6 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='26a90f95'/>
     </function-decl>
-    <function-decl name='statx' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='9d26089a'/>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='f0981eeb'/>
-      <parameter type-id='31d265b7'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
     <function-decl name='stat64' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='9d26089a'/>
       <parameter type-id='f1cadedf'/>
diff --git a/sys/contrib/openzfs/lib/libzpool/abd_os.c b/sys/contrib/openzfs/lib/libzpool/abd_os.c
index 0d5795de143a..8bd7a64ab24a 100644
--- a/sys/contrib/openzfs/lib/libzpool/abd_os.c
+++ b/sys/contrib/openzfs/lib/libzpool/abd_os.c
@@ -302,7 +302,7 @@ abd_iter_at_end(struct abd_iter *aiter)
 void
 abd_iter_advance(struct abd_iter *aiter, size_t amount)
 {
-	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
+	ASSERT0P(aiter->iter_mapaddr);
 	ASSERT0(aiter->iter_mapsize);
 
 	if (abd_iter_at_end(aiter))
@@ -315,7 +315,7 @@ abd_iter_advance(struct abd_iter *aiter, size_t amount)
 void
 abd_iter_map(struct abd_iter *aiter)
 {
-	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
+	ASSERT0P(aiter->iter_mapaddr);
 	ASSERT0(aiter->iter_mapsize);
 
 	if (abd_iter_at_end(aiter))
diff --git a/sys/contrib/openzfs/lib/libzpool/kernel.c b/sys/contrib/openzfs/lib/libzpool/kernel.c
index 2e8bf160465a..70eba5099119 100644
--- a/sys/contrib/openzfs/lib/libzpool/kernel.c
+++ b/sys/contrib/openzfs/lib/libzpool/kernel.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
+ * Copyright (c) 2025, Klara, Inc.
  */
 
 #include <assert.h>
@@ -38,6 +39,7 @@
 #include <sys/processor.h>
 #include <sys/rrwlock.h>
 #include <sys/spa.h>
+#include <sys/spa_impl.h>
 #include <sys/stat.h>
 #include <sys/systeminfo.h>
 #include <sys/time.h>
@@ -369,7 +371,7 @@ cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 	if (delta <= 0)
 		return (-1);
 
-	VERIFY(gettimeofday(&tv, NULL) == 0);
+	VERIFY0(gettimeofday(&tv, NULL));
 
 	ts.tv_sec = tv.tv_sec + delta / hz;
 	ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz);
@@ -644,39 +646,60 @@ __dprintf(boolean_t dprint, const char *file, const char *func,
  * cmn_err() and panic()
  * =========================================================================
  */
-static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
-static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 
-__attribute__((noreturn)) void
-vpanic(const char *fmt, va_list adx)
+static __attribute__((noreturn)) void
+panic_stop_or_abort(void)
 {
-	(void) fprintf(stderr, "error: ");
-	(void) vfprintf(stderr, fmt, adx);
-	(void) fprintf(stderr, "\n");
+	const char *stopenv = getenv("LIBZPOOL_PANIC_STOP");
+	if (stopenv != NULL && atoi(stopenv)) {
+		fputs("libzpool: LIBZPOOL_PANIC_STOP is set, sending "
+		    "SIGSTOP to process group\n", stderr);
+		fflush(stderr);
+
+		kill(0, SIGSTOP);
+
+		fputs("libzpool: continued after panic stop, "
+		    "aborting\n", stderr);
+	}
 
 	abort();	/* think of it as a "user-level crash dump" */
 }
 
-__attribute__((noreturn)) void
-panic(const char *fmt, ...)
-{
-	va_list adx;
+static void
+vcmn_msg(int ce, const char *fmt, va_list adx)
+{
+	switch (ce) {
+	case CE_IGNORE:
+		return;
+	case CE_CONT:
+		break;
+	case CE_NOTE:
+		fputs("libzpool: NOTICE: ", stderr);
+		break;
+	case CE_WARN:
+		fputs("libzpool: WARNING: ", stderr);
+		break;
+	case CE_PANIC:
+		fputs("libzpool: PANIC: ", stderr);
+		break;
+	default:
+		fputs("libzpool: [unknown severity %d]: ", stderr);
+		break;
+	}
 
-	va_start(adx, fmt);
-	vpanic(fmt, adx);
-	va_end(adx);
+	vfprintf(stderr, fmt, adx);
+	if (ce != CE_CONT)
+		fputc('\n', stderr);
+	fflush(stderr);
 }
 
 void
 vcmn_err(int ce, const char *fmt, va_list adx)
 {
+	vcmn_msg(ce, fmt, adx);
+
 	if (ce == CE_PANIC)
-		vpanic(fmt, adx);
-	if (ce != CE_NOTE) {	/* suppress noise in userland stress testing */
-		(void) fprintf(stderr, "%s", ce_prefix[ce]);
-		(void) vfprintf(stderr, fmt, adx);
-		(void) fprintf(stderr, "%s", ce_suffix[ce]);
-	}
+		panic_stop_or_abort();
 }
 
 void
@@ -689,6 +712,25 @@ cmn_err(int ce, const char *fmt, ...)
 	va_end(adx);
 }
 
+__attribute__((noreturn)) void
+panic(const char *fmt, ...)
+{
+	va_list adx;
+
+	va_start(adx, fmt);
+	vcmn_msg(CE_PANIC, fmt, adx);
+	va_end(adx);
+
+	panic_stop_or_abort();
+}
+
+__attribute__((noreturn)) void
+vpanic(const char *fmt, va_list adx)
+{
+	vcmn_msg(CE_PANIC, fmt, adx);
+	panic_stop_or_abort();
+}
+
 /*
  * =========================================================================
  * misc routines
@@ -811,6 +853,79 @@ umem_out_of_memory(void)
 	return (0);
 }
 
+static void
+spa_config_load(void)
+{
+	void *buf = NULL;
+	nvlist_t *nvlist, *child;
+	nvpair_t *nvpair;
+	char *pathname;
+	zfs_file_t *fp;
+	zfs_file_attr_t zfa;
+	uint64_t fsize;
+	int err;
+
+	/*
+	 * Open the configuration file.
+	 */
+	pathname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
+
+	(void) snprintf(pathname, MAXPATHLEN, "%s", spa_config_path);
+
+	err = zfs_file_open(pathname, O_RDONLY, 0, &fp);
+	if (err)
+		err = zfs_file_open(ZPOOL_CACHE_BOOT, O_RDONLY, 0, &fp);
+
+	kmem_free(pathname, MAXPATHLEN);
+
+	if (err)
+		return;
+
+	if (zfs_file_getattr(fp, &zfa))
+		goto out;
+
+	fsize = zfa.zfa_size;
+	buf = kmem_alloc(fsize, KM_SLEEP);
+
+	/*
+	 * Read the nvlist from the file.
+	 */
+	if (zfs_file_read(fp, buf, fsize, NULL) < 0)
+		goto out;
+
+	/*
+	 * Unpack the nvlist.
+	 */
+	if (nvlist_unpack(buf, fsize, &nvlist, KM_SLEEP) != 0)
+		goto out;
+
+	/*
+	 * Iterate over all elements in the nvlist, creating a new spa_t for
+	 * each one with the specified configuration.
+	 */
+	mutex_enter(&spa_namespace_lock);
+	nvpair = NULL;
+	while ((nvpair = nvlist_next_nvpair(nvlist, nvpair)) != NULL) {
+		if (nvpair_type(nvpair) != DATA_TYPE_NVLIST)
+			continue;
+
+		child = fnvpair_value_nvlist(nvpair);
+
+		if (spa_lookup(nvpair_name(nvpair)) != NULL)
+			continue;
+		(void) spa_add(nvpair_name(nvpair), child, NULL);
+	}
+	mutex_exit(&spa_namespace_lock);
+
+	nvlist_free(nvlist);
+
+out:
+	if (buf != NULL)
+		kmem_free(buf, fsize);
+
+	zfs_file_close(fp);
+}
+
 void
 kernel_init(int mode)
 {
@@ -835,6 +950,7 @@ kernel_init(int mode)
 	zstd_init();
 
 	spa_init((spa_mode_t)mode);
+	spa_config_load();
 
 	fletcher_4_init();
 
@@ -1163,7 +1279,7 @@ zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid)
  */
 int
 zfs_file_pwrite(zfs_file_t *fp, const void *buf,
-    size_t count, loff_t pos, ssize_t *resid)
+    size_t count, loff_t pos, uint8_t ashift, ssize_t *resid)
 {
 	ssize_t rc, split, done;
 	int sectors;
@@ -1173,8 +1289,8 @@ zfs_file_pwrite(zfs_file_t *fp, const void *buf,
 	 * system calls so that the process can be killed in between.
 	 * This is used by ztest to simulate realistic failure modes.
 	 */
-	sectors = count >> SPA_MINBLOCKSHIFT;
-	split = (sectors > 0 ? rand() % sectors : 0) << SPA_MINBLOCKSHIFT;
+	sectors = count >> ashift;
+	split = (sectors > 0 ? rand() % sectors : 0) << ashift;
 	rc = pwrite64(fp->f_fd, buf, split, pos);
 	if (rc != -1) {
 		done = rc;
diff --git a/sys/contrib/openzfs/lib/libzpool/util.c b/sys/contrib/openzfs/lib/libzpool/util.c
index 1d0d1a1e56d9..66d6f43967d5 100644
--- a/sys/contrib/openzfs/lib/libzpool/util.c
+++ b/sys/contrib/openzfs/lib/libzpool/util.c
@@ -137,12 +137,10 @@ show_pool_stats(spa_t *spa)
 	nvlist_t *config, *nvroot;
 	const char *name;
 
-	VERIFY(spa_get_stats(spa_name(spa), &config, NULL, 0) == 0);
+	VERIFY0(spa_get_stats(spa_name(spa), &config, NULL, 0));
 
-	VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
-	    &nvroot) == 0);
-	VERIFY(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
-	    &name) == 0);
+	VERIFY0(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot));
+	VERIFY0(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME, &name));
 
 	show_vdev_stats(name, ZPOOL_CONFIG_CHILDREN, nvroot, 0);
 	show_vdev_stats(NULL, ZPOOL_CONFIG_L2CACHE, nvroot, 0);
diff --git a/sys/contrib/openzfs/lib/libzutil/zutil_import.c b/sys/contrib/openzfs/lib/libzutil/zutil_import.c
index ccdc874076c3..08367f4c064d 100644
--- a/sys/contrib/openzfs/lib/libzutil/zutil_import.c
+++ b/sys/contrib/openzfs/lib/libzutil/zutil_import.c
@@ -917,7 +917,7 @@ error:
 static uint64_t
 label_offset(uint64_t size, int l)
 {
-	ASSERT(P2PHASE_TYPED(size, sizeof (vdev_label_t), uint64_t) == 0);
+	ASSERT0(P2PHASE_TYPED(size, sizeof (vdev_label_t), uint64_t));
 	return (l * sizeof (vdev_label_t) + (l < VDEV_LABELS / 2 ?
 	    0 : size - VDEV_LABELS * sizeof (vdev_label_t)));
 }
@@ -1769,7 +1769,7 @@ zpool_find_import_cached(libpc_handle_t *hdl, importargs_t *iarg)
 			fnvlist_add_nvlist(pools, nvpair_name(pair),
 			    fnvpair_value_nvlist(pair));
 
-			VERIFY3P(nvlist_next_nvpair(nv, pair), ==, NULL);
+			VERIFY0P(nvlist_next_nvpair(nv, pair));
 
 			iarg->guid = saved_guid;
 			iarg->poolname = saved_poolname;
@@ -1903,30 +1903,43 @@ zpool_find_config(libpc_handle_t *hdl, const char *target, nvlist_t **configp,
 		*sepp = '\0';
 
 	pools = zpool_search_import(hdl, args);
+	if (pools == NULL) {
+		zutil_error_aux(hdl, dgettext(TEXT_DOMAIN, "no pools found"));
+		(void) zutil_error_fmt(hdl, LPC_UNKNOWN, dgettext(TEXT_DOMAIN,
+		    "failed to find config for pool '%s'"), targetdup);
+		free(targetdup);
+		return (ENOENT);
+	}
 
-	if (pools != NULL) {
-		nvpair_t *elem = NULL;
-		while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
-			VERIFY0(nvpair_value_nvlist(elem, &config));
-			if (pool_match(config, targetdup)) {
-				count++;
-				if (match != NULL) {
-					/* multiple matches found */
-					continue;
-				} else {
-					match = fnvlist_dup(config);
-				}
+	nvpair_t *elem = NULL;
+	while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
+		VERIFY0(nvpair_value_nvlist(elem, &config));
+		if (pool_match(config, targetdup)) {
+			count++;
+			if (match != NULL) {
+				/* multiple matches found */
+				continue;
+			} else {
+				match = fnvlist_dup(config);
 			}
 		}
-		fnvlist_free(pools);
 	}
+	fnvlist_free(pools);
 
 	if (count == 0) {
+		zutil_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "no matching pools"));
+		(void) zutil_error_fmt(hdl, LPC_UNKNOWN, dgettext(TEXT_DOMAIN,
+		    "failed to find config for pool '%s'"), targetdup);
 		free(targetdup);
 		return (ENOENT);
 	}
 
 	if (count > 1) {
+		zutil_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "more than one matching pool"));
+		(void) zutil_error_fmt(hdl, LPC_UNKNOWN, dgettext(TEXT_DOMAIN,
+		    "failed to find config for pool '%s'"), targetdup);
 		free(targetdup);
 		fnvlist_free(match);
 		return (EINVAL);
diff --git a/sys/contrib/openzfs/man/Makefile.am b/sys/contrib/openzfs/man/Makefile.am
index 6a7b2d3e46b7..7a63641c1c39 100644
--- a/sys/contrib/openzfs/man/Makefile.am
+++ b/sys/contrib/openzfs/man/Makefile.am
@@ -2,7 +2,7 @@ dist_noinst_man_MANS = \
 	%D%/man1/cstyle.1
 
 dist_man_MANS = \
-	%D%/man1/arcstat.1 \
+	%D%/man1/zarcstat.1 \
 	%D%/man1/raidz_test.1 \
 	%D%/man1/test-runner.1 \
 	%D%/man1/zhack.1 \
@@ -124,10 +124,21 @@ dist_noinst_DATA += $(dist_noinst_man_MANS) $(dist_man_MANS)
 
 SUBSTFILES += $(nodist_man_MANS)
 
-CHECKS += mancheck
-mancheck:
-	$(top_srcdir)/scripts/mancheck.sh $(srcdir)/%D%
+MANFILES = $(dist_noinst_man_MANS) $(dist_man_MANS) $(nodist_man_MANS)
+
+PHONY += mancheck
+
+mancheck_verbose = $(mancheck_verbose_@AM_V@)
+mancheck_verbose_ = $(mancheck_verbose_@AM_DEFAULT_V@)
+mancheck_verbose_0 = @echo MANCHECK $(_MTGT);
 
+_MTGT = $(subst ^,/,$(subst mancheck-,,$@))
+mancheck-%:
+	$(mancheck_verbose)scripts/mancheck.sh $(_MTGT)
+
+mancheck: $(foreach manfile, $(MANFILES), $(addprefix mancheck-,$(subst /,^,$(manfile))))
+
+CHECKS += mancheck
 
 if BUILD_LINUX
 # The manual pager in most Linux distros defaults to "BSD" when .Os is blank,
diff --git a/sys/contrib/openzfs/man/man1/cstyle.1 b/sys/contrib/openzfs/man/man1/cstyle.1
index 241c82edd5a8..8f29129ce175 100644
--- a/sys/contrib/openzfs/man/man1/cstyle.1
+++ b/sys/contrib/openzfs/man/man1/cstyle.1
@@ -21,7 +21,7 @@
 .\"
 .\" CDDL HEADER END
 .\"
-.Dd May 26, 2021
+.Dd April 4, 2022
 .Dt CSTYLE 1
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man1/arcstat.1 b/sys/contrib/openzfs/man/man1/zarcstat.1
index f2474fbb701f..3633c5d417fe 100644
--- a/sys/contrib/openzfs/man/man1/arcstat.1
+++ b/sys/contrib/openzfs/man/man1/zarcstat.1
@@ -13,12 +13,12 @@
 .\" Copyright (c) 2015 by Delphix. All rights reserved.
 .\" Copyright (c) 2020 by AJ Jordan. All rights reserved.
 .\"
-.Dd December 23, 2022
-.Dt ARCSTAT 1
+.Dd September 19, 2024
+.Dt ZARCSTAT 1
 .Os
 .
 .Sh NAME
-.Nm arcstat
+.Nm zarcstat
 .Nd report ZFS ARC and L2ARC statistics
 .Sh SYNOPSIS
 .Nm
diff --git a/sys/contrib/openzfs/man/man1/zhack.1 b/sys/contrib/openzfs/man/man1/zhack.1
index f58c0527649b..63658cf930e9 100644
--- a/sys/contrib/openzfs/man/man1/zhack.1
+++ b/sys/contrib/openzfs/man/man1/zhack.1
@@ -23,7 +23,7 @@
 .\"
 .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS
 .\"
-.Dd May 26, 2021
+.Dd May 3, 2023
 .Dt ZHACK 1
 .Os
 .
@@ -122,6 +122,24 @@ Example:
 .Nm zhack Cm label repair Fl cu Ar device
   Fix checksums and undetach a device
 .
+.It Xo
+.Nm zhack
+.Cm metaslab leak
+.Op Fl f
+.Ar pool
+.Xc
+Apply a fragmentation profile generated by
+.Sy zdb
+to the specified
+.Ar pool Ns
+\&.
+.Pp
+The
+.Fl f
+flag forces the profile to apply even if the vdevs in the
+.Ar pool
+don't have the same number of metaslabs as the fragmentation profile.
+.
 .El
 .
 .Sh GLOBAL OPTIONS
@@ -143,6 +161,8 @@ Search for
 members in
 .Ar dir .
 Can be specified more than once.
+.It Fl o Ar var Ns = Ns Ar value
+Set the given tunable to the provided value.
 .El
 .
 .Sh EXAMPLES
diff --git a/sys/contrib/openzfs/man/man1/ztest.1 b/sys/contrib/openzfs/man/man1/ztest.1
index febbb62b1664..ae857bfea29c 100644
--- a/sys/contrib/openzfs/man/man1/ztest.1
+++ b/sys/contrib/openzfs/man/man1/ztest.1
@@ -24,7 +24,7 @@
 .\" reserved.
 .\" Copyright (c) 2017, Intel Corporation.
 .\"
-.Dd May 26, 2021
+.Dd July 12, 2025
 .Dt ZTEST 1
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man4/spl.4 b/sys/contrib/openzfs/man/man4/spl.4
index 683f8e2b631f..61dfe42e463d 100644
--- a/sys/contrib/openzfs/man/man4/spl.4
+++ b/sys/contrib/openzfs/man/man4/spl.4
@@ -15,7 +15,7 @@
 .\"
 .\" Copyright 2013 Turbo Fredriksson <turbo@bayour.com>. All rights reserved.
 .\"
-.Dd August 24, 2020
+.Dd May 7, 2025
 .Dt SPL 4
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man4/zfs.4 b/sys/contrib/openzfs/man/man4/zfs.4
index 4a5f9fd93f4f..11bcbf430210 100644
--- a/sys/contrib/openzfs/man/man4/zfs.4
+++ b/sys/contrib/openzfs/man/man4/zfs.4
@@ -4,6 +4,7 @@
 .\" Copyright (c) 2019, 2021 by Delphix. All rights reserved.
 .\" Copyright (c) 2019 Datto Inc.
 .\" Copyright (c) 2023, 2024, 2025, Klara, Inc.
+.\"
 .\" The contents of this file are subject to the terms of the Common Development
 .\" and Distribution License (the "License").  You may not use this file except
 .\" in compliance with the License. You can obtain a copy of the license at
@@ -17,7 +18,7 @@
 .\" own identifying information:
 .\" Portions Copyright [yyyy] [name of copyright owner]
 .\"
-.Dd May 29, 2025
+.Dd September 15, 2025
 .Dt ZFS 4
 .Os
 .
@@ -601,6 +602,42 @@ new format when enabling the
 feature.
 The default is to convert all log entries.
 .
+.It Sy vdev_read_sit_out_secs Ns = Ns Sy 600 Ns s Po 10 min Pc Pq ulong
+When a slow disk outlier is detected it is placed in a sit out state.
+While sitting out the disk will not participate in normal reads, instead its
+data will be reconstructed as needed from parity.
+Scrub operations will always read from a disk, even if it's sitting out.
+A number of disks in a RAID-Z or dRAID vdev may sit out at the same time, up
+to the number of parity devices.
+Writes will still be issued to a disk which is sitting out to maintain full
+redundancy.
+Defaults to 600 seconds and a value of zero disables disk sit-outs in general,
+including slow disk outlier detection.
+.
+.It Sy vdev_raidz_outlier_check_interval_ms Ns = Ns Sy 1000 Ns ms Po 1 sec Pc Pq ulong
+How often each RAID-Z and dRAID vdev will check for slow disk outliers.
+Increasing this interval will reduce the sensitivity of detection (since all
+I/Os since the last check are included in the statistics), but will slow the
+response to a disk developing a problem.
+Defaults to once per second; setting extremely small values may cause negative
+performance effects.
+.
+.It Sy vdev_raidz_outlier_insensitivity Ns = Ns Sy 50 Pq uint
+When performing slow outlier checks for RAID-Z and dRAID vdevs, this value is
+used to determine how far out an outlier must be before it counts as an event
+worth consdering.
+This is phrased as "insensitivity" because larger values result in fewer
+detections.
+Smaller values will result in more aggressive sitting out of disks that may have
+problems, but may significantly increase the rate of spurious sit-outs.
+.Pp
+To provide a more technical definition of this parameter, this is the multiple
+of the inter-quartile range (IQR) that is being used in a Tukey's Fence
+detection algorithm.
+This is much higher than a normal Tukey's Fence k-value, because the
+distribution under consideration is probably an extreme-value distribution,
+rather than a more typical Gaussian distribution.
+.
 .It Sy vdev_removal_max_span Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq uint
 During top-level vdev removal, chunks of data are copied from the vdev
 which may include free space in order to trade bandwidth for IOPS.
@@ -941,10 +978,6 @@ The target number of bytes the ARC should leave as free memory on the system.
 If zero, equivalent to the bigger of
 .Sy 512 KiB No and Sy all_system_memory/64 .
 .
-.It Sy zfs_autoimport_disable Ns = Ns Sy 1 Ns | Ns 0 Pq int
-Disable pool import at module load by ignoring the cache file
-.Pq Sy spa_config_path .
-.
 .It Sy zfs_checksum_events_per_second Ns = Ns Sy 20 Ns /s Pq uint
 Rate limit checksum events to this many per second.
 Note that this should not be set below the ZED thresholds
@@ -2550,6 +2583,49 @@ the xattr so as to not accumulate duplicates.
 .It Sy zio_requeue_io_start_cut_in_line Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Prioritize requeued I/O.
 .
+.It Sy zfs_delete_inode Ns = Ns Sy 0 Ns | Ns 1 Pq int
+Sets whether the kernel should free an inode structure when the last reference
+is released, or cache it in memory.
+Intended for testing/debugging.
+.Pp
+A live inode structure "pins" versious internal OpenZFS structures in memory,
+which can result in large amounts of "unusable" memory on systems with lots of
+infrequently-accessed files, until the kernel's memory pressure mechanism
+asks OpenZFS to release them.
+.Pp
+The default value of
+.Sy 0
+always caches inodes that appear to still exist on disk.
+Setting it to
+.Sy 1
+will immediately release unused inodes and their associated memory back to the
+dbuf cache or the ARC for reuse, but may reduce performance if inodes are
+frequently evicted and reloaded.
+.Pp
+This parameter is only available on Linux.
+.
+.It Sy zfs_delete_dentry Ns = Ns Sy 0 Ns | Ns 1 Pq int
+Sets whether the kernel should free a dentry structure when it is no longer
+required, or hold it in the dentry cache.
+Intended for testing/debugging.
+.
+Since a dentry structure holds an inode reference, a cached dentry can "pin"
+an inode in memory indefinitely, along with associated OpenZFS structures (See
+.Sy zfs_delete_inode ) .
+.Pp
+The default value of
+.Sy 0
+instructs the kernel to cache entries and their associated inodes when they
+are no longer directly referenced.
+They will be reclaimed as part of the kernel's normal cache management
+processes.
+Setting it to
+.Sy 1
+will instruct the kernel to release directory entries and their inodes as soon
+as they are no longer referenced by the filesystem.
+.Pp
+This parameter is only available on Linux.
+.
 .It Sy zio_taskq_batch_pct Ns = Ns Sy 80 Ns % Pq uint
 Percentage of online CPUs which will run a worker thread for I/O.
 These workers are responsible for I/O work such as compression, encryption,
diff --git a/sys/contrib/openzfs/man/man5/vdev_id.conf.5 b/sys/contrib/openzfs/man/man5/vdev_id.conf.5
index d2f817631c15..299a23720201 100644
--- a/sys/contrib/openzfs/man/man5/vdev_id.conf.5
+++ b/sys/contrib/openzfs/man/man5/vdev_id.conf.5
@@ -9,7 +9,7 @@
 .\" source.  A copy of the CDDL is also available via the Internet at
 .\" http://www.illumos.org/license/CDDL.
 .\"
-.Dd May 26, 2021
+.Dd October 8, 2024
 .Dt VDEV_ID.CONF 5
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man7/dracut.zfs.7 b/sys/contrib/openzfs/man/man7/dracut.zfs.7
index fb5da553af6e..3d051d4d3343 100644
--- a/sys/contrib/openzfs/man/man7/dracut.zfs.7
+++ b/sys/contrib/openzfs/man/man7/dracut.zfs.7
@@ -1,7 +1,7 @@
 .\" SPDX-License-Identifier: CDDL-1.0
 .\" SPDX-License-Identifier: 0BSD
 .\"
-.Dd March 28, 2023
+.Dd July 13, 2024
 .Dt DRACUT.ZFS 7
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man7/vdevprops.7 b/sys/contrib/openzfs/man/man7/vdevprops.7
index acabe6b6613a..0fb28d7db13c 100644
--- a/sys/contrib/openzfs/man/man7/vdevprops.7
+++ b/sys/contrib/openzfs/man/man7/vdevprops.7
@@ -19,9 +19,9 @@
 .\"
 .\" CDDL HEADER END
 .\"
-.\" Copyright (c) 2021 Klara, Inc.
+.\" Copyright (c) 2021, 2025, Klara, Inc.
 .\"
-.Dd October 30, 2022
+.Dd July 23, 2024
 .Dt VDEVPROPS 7
 .Os
 .
@@ -106,11 +106,17 @@ The number of children belonging to this vdev
 .It Sy read_errors , write_errors , checksum_errors , initialize_errors , trim_errors
 The number of errors of each type encountered by this vdev
 .It Sy slow_ios
-The number of slow I/Os encountered by this vdev,
-These represent I/O operations that didn't complete in
+This indicates the number of slow I/O operations encountered by this vdev.
+A slow I/O is defined as an operation that did not complete within the
 .Sy zio_slow_io_ms
-milliseconds
+threshold in milliseconds
 .Pq Sy 30000 No by default .
+For
+.Sy RAIDZ
+and
+.Sy DRAID
+configurations, this value also represents the number of times the vdev was
+identified as an outlier and excluded from participating in read I/O operations.
 .It Sy null_ops , read_ops , write_ops , free_ops , claim_ops , trim_ops
 The number of I/O operations of each type performed by this vdev
 .It Xo
@@ -150,6 +156,31 @@ The amount of space to reserve for the EFI system partition
 .It Sy failfast
 If this device should propagate BIO errors back to ZFS, used to disable
 failfast.
+.It Sy sit_out
+Only valid for
+.Sy RAIDZ
+and
+.Sy DRAID
+vdevs.
+True when a slow disk outlier was detected and the vdev is currently in a sit
+out state.
+This property can be manually set to cause vdevs to sit out.
+It will also be automatically set by the
+.Sy autosit
+logic if that is enabled.
+While sitting out, the vdev will not participate in normal reads, instead its
+data will be reconstructed as needed from parity.
+.It Sy autosit
+Only valid for
+.Sy RAIDZ
+and
+.Sy DRAID
+vdevs.
+If set, this enables the kernel-level slow disk detection logic.
+This logic automatically causes any vdevs that are significant negative
+performance outliers to sit out, as described in the
+.Sy sit_out
+property.
 .It Sy path
 The path to the device for this vdev
 .It Sy allocating
diff --git a/sys/contrib/openzfs/man/man7/zfsconcepts.7 b/sys/contrib/openzfs/man/man7/zfsconcepts.7
index 5c736e53670d..bb2178d85bcd 100644
--- a/sys/contrib/openzfs/man/man7/zfsconcepts.7
+++ b/sys/contrib/openzfs/man/man7/zfsconcepts.7
@@ -31,7 +31,7 @@
 .\" Copyright 2019 Joyent, Inc.
 .\" Copyright 2023 Klara, Inc.
 .\"
-.Dd October 6, 2023
+.Dd October 2, 2024
 .Dt ZFSCONCEPTS 7
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man7/zfsprops.7 b/sys/contrib/openzfs/man/man7/zfsprops.7
index ac3152cb5d51..77e994b912b6 100644
--- a/sys/contrib/openzfs/man/man7/zfsprops.7
+++ b/sys/contrib/openzfs/man/man7/zfsprops.7
@@ -39,7 +39,7 @@
 .\" Copyright (c) 2019, Kjeld Schouten-Lebbing
 .\" Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
 .\"
-.Dd June 29, 2024
+.Dd September 13, 2025
 .Dt ZFSPROPS 7
 .Os
 .
@@ -1192,18 +1192,26 @@ keylocation can be with either
 .Nm zfs Cm set
 or
 .Nm zfs Cm change-key .
+.Pp
 If
 .Sy prompt
-is selected ZFS will ask for the key at the command prompt when it is required
-to access the encrypted data (see
+is selected, ZFS will expect the key to be provided when it is required to
+access the encrypted data (see
 .Nm zfs Cm load-key
 for details).
-This setting will also allow the key to be passed in via the standard input
-stream,
-but users should be careful not to place keys which should be kept secret on
-the command line.
-If a file URI is selected, the key will be loaded from the
+If stdin is a TTY, then ZFS will ask for the key to be provided.
+Otherwise, stdin is expected to be the key to use and will be processed as such.
+Users should be careful not to place keys which should be kept secret on the
+command line, as most operating systems may expose command line arguments to
+other processes.
+If the
+.Dq raw
+.Sy keyformat
+was used, then the key must be provided via stdin.
+.Pp
+If a file URL is selected, the key will be loaded from the
 specified absolute file path.
+.Pp
 If an HTTPS or HTTP URL is selected, it will be GETted using
 .Xr fetch 3 ,
 libcurl, or nothing, depending on compile-time configuration and run-time
diff --git a/sys/contrib/openzfs/man/man7/zpool-features.7 b/sys/contrib/openzfs/man/man7/zpool-features.7
index 10dfd1f92936..b4404a6eb58d 100644
--- a/sys/contrib/openzfs/man/man7/zpool-features.7
+++ b/sys/contrib/openzfs/man/man7/zpool-features.7
@@ -19,7 +19,7 @@
 .\" Copyright (c) 2019, Allan Jude
 .\" Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
 .\"
-.Dd October 2, 2024
+.Dd July 23, 2025
 .Dt ZPOOL-FEATURES 7
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man7/zpoolconcepts.7 b/sys/contrib/openzfs/man/man7/zpoolconcepts.7
index dafe3bffc453..b9c8926d835d 100644
--- a/sys/contrib/openzfs/man/man7/zpoolconcepts.7
+++ b/sys/contrib/openzfs/man/man7/zpoolconcepts.7
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd April 7, 2023
+.Dd August 6, 2025
 .Dt ZPOOLCONCEPTS 7
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man7/zpoolprops.7 b/sys/contrib/openzfs/man/man7/zpoolprops.7
index 5d84753193ee..d3b4c2376943 100644
--- a/sys/contrib/openzfs/man/man7/zpoolprops.7
+++ b/sys/contrib/openzfs/man/man7/zpoolprops.7
@@ -29,7 +29,7 @@
 .\" Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
 .\" Copyright (c) 2023, Klara Inc.
 .\"
-.Dd November 18, 2024
+.Dd December 4, 2024
 .Dt ZPOOLPROPS 7
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zdb.8 b/sys/contrib/openzfs/man/man8/zdb.8
index 3984aaac5866..f51e24fa849c 100644
--- a/sys/contrib/openzfs/man/man8/zdb.8
+++ b/sys/contrib/openzfs/man/man8/zdb.8
@@ -15,7 +15,7 @@
 .\" Copyright (c) 2017 Lawrence Livermore National Security, LLC.
 .\" Copyright (c) 2017 Intel Corporation.
 .\"
-.Dd October 27, 2024
+.Dd August 12, 2025
 .Dt ZDB 8
 .Os
 .
@@ -69,6 +69,13 @@
 .Op Fl U Ar cache
 .Ar poolname Op Ar vdev Oo Ar metaslab Oc Ns …
 .Nm
+.Fl -allocated-map
+.Op Fl mAFLPXY
+.Op Fl e Oo Fl V Oc Oo Fl p Ar path Oc Ns …
+.Op Fl t Ar txg
+.Op Fl U Ar cache
+.Ar poolname Op Ar vdev Oo Ar metaslab Oc Ns …
+.Nm
 .Fl O
 .Op Fl K Ar key
 .Ar dataset path
@@ -128,10 +135,29 @@ that zdb may interpret inconsistent pool data and behave erratically.
 .Sh OPTIONS
 Display options:
 .Bl -tag -width Ds
+.It Fl Sy -allocated-map
+Prints out a list of all the allocated regions in the pool.
+Primarily intended for use with the
+.Nm zhack metaslab leak
+subcommand.
 .It Fl b , -block-stats
 Display statistics regarding the number, size
 .Pq logical, physical and allocated
 and deduplication of blocks.
+.It Fl -bin Ns = Ns ( Li lsize Ns | Ns Li psize Ns | Ns Li asize )
+When used with
+.Fl bb ,
+sort blocks into all three bins according to the given size (instead of binning
+a block for each size separately).
+.Pp
+For instance, with
+.Fl -bin Ns = Ns Li lsize ,
+a block with lsize of 16K and psize of 4K will be added to the 16K bin
+in all three columns.
+.It Fl -class Ns = Ns ( Li normal Ns | Ns Li special Ns | Ns Li dedup Ns | Ns Li other ) Ns Op , Ns …
+When used with
+.Fl bb ,
+only consider blocks from these allocation classes.
 .It Fl B , -backup
 Generate a backup stream, similar to
 .Nm zfs Cm send ,
@@ -531,6 +557,18 @@ option, with more occurrences enabling more verbosity.
 If no options are specified, all information about the named pool will be
 displayed at default verbosity.
 .
+.Sh EXIT STATUS
+The
+.Nm
+utility exits
+.Sy 0
+on success,
+.Sy 1
+if a fatal error occurs,
+.Sy 2
+if invalid command line options were specified, or
+.Sy 3
+if on-disk corruption was detected, but was not fatal.
 .Sh EXAMPLES
 .Ss Example 1 : No Display the configuration of imported pool Ar rpool
 .Bd -literal
diff --git a/sys/contrib/openzfs/man/man8/zed.8.in b/sys/contrib/openzfs/man/man8/zed.8.in
index c90a1834403b..2d19f2d8496b 100644
--- a/sys/contrib/openzfs/man/man8/zed.8.in
+++ b/sys/contrib/openzfs/man/man8/zed.8.in
@@ -13,7 +13,7 @@
 .\"
 .\" Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049)
 .\"
-.Dd May 26, 2021
+.Dd August 22, 2022
 .Dt ZED 8
 .Os
 .
@@ -158,6 +158,8 @@ Multiple ZEDLETs may be invoked for a given zevent.
 ZEDLETs are executables invoked by the ZED in response to a given zevent.
 They should be written under the presumption they can be invoked concurrently,
 and they should use appropriate locking to access any shared resources.
+The one exception to this are "synchronous zedlets", which are described later
+in this page.
 Common variables used by ZEDLETs can be stored in the default rc file which
 is sourced by scripts; these variables should be prefixed with
 .Sy ZED_ .
@@ -233,6 +235,36 @@ and
 .Sy ZPOOL .
 These variables may be overridden in the rc file.
 .
+.Sh Synchronous ZEDLETS
+ZED's normal behavior is to spawn off zedlets in parallel and ignore their
+completion order.
+This means that ZED can potentially
+have zedlets for event ID number 2 starting before zedlets for event ID number
+1 have finished.
+Most of the time this is fine, and it actually helps when the system is getting
+hammered with hundreds of events.
+.Pp
+However, there are times when you want your zedlets to be executed in sequence
+with the event ID.
+That is where synchronous zedlets come in.
+.Pp
+ZED will wait for all previously spawned zedlets to finish before running
+a synchronous zedlet.
+Synchronous zedlets are guaranteed to be the only
+zedlet running.
+No other zedlets may run in parallel with a synchronous zedlet.
+Users should be careful to only use synchronous zedlets when needed, since
+they decrease parallelism.
+.Pp
+To make a zedlet synchronous, simply add a "-sync-" immediately following the
+event name in the zedlet's file name:
+.Pp
+.Sy EVENT_NAME-sync-ZEDLETNAME.sh
+.Pp
+For example, if you wanted a synchronous statechange script:
+.Pp
+.Sy statechange-sync-myzedlet.sh
+.
 .Sh FILES
 .Bl -tag -width "-c"
 .It Pa @sysconfdir@/zfs/zed.d
diff --git a/sys/contrib/openzfs/man/man8/zfs-allow.8 b/sys/contrib/openzfs/man/man8/zfs-allow.8
index 5a8e80bf6a43..e3b0e1ab3e12 100644
--- a/sys/contrib/openzfs/man/man8/zfs-allow.8
+++ b/sys/contrib/openzfs/man/man8/zfs-allow.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd September 8, 2025
 .Dt ZFS-ALLOW 8
 .Os
 .
@@ -212,7 +212,8 @@ receive	subcommand	Must also have the \fBmount\fR and \fBcreate\fR ability, requ
 release	subcommand	Allows releasing a user hold which might destroy the snapshot
 rename	subcommand	Must also have the \fBmount\fR and \fBcreate\fR ability in the new parent
 rollback	subcommand	Must also have the \fBmount\fR ability
-send	subcommand
+send	subcommand	Allows sending a replication stream of a dataset.
+send:raw	subcommand	Only allows sending raw replication streams, preventing encrypted datasets being sent in decrypted form.
 share	subcommand	Allows sharing file systems over NFS or SMB protocols
 snapshot	subcommand	Must also have the \fBmount\fR ability
 
diff --git a/sys/contrib/openzfs/man/man8/zfs-bookmark.8 b/sys/contrib/openzfs/man/man8/zfs-bookmark.8
index 083ff46d241b..5a0933820020 100644
--- a/sys/contrib/openzfs/man/man8/zfs-bookmark.8
+++ b/sys/contrib/openzfs/man/man8/zfs-bookmark.8
@@ -31,7 +31,7 @@
 .\" Copyright 2019 Joyent, Inc.
 .\" Copyright (c) 2019, 2020 by Christian Schwarz. All Rights Reserved.
 .\"
-.Dd May 12, 2022
+.Dd July 11, 2022
 .Dt ZFS-BOOKMARK 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-clone.8 b/sys/contrib/openzfs/man/man8/zfs-clone.8
index cd412815f5fe..9609cf2ce36a 100644
--- a/sys/contrib/openzfs/man/man8/zfs-clone.8
+++ b/sys/contrib/openzfs/man/man8/zfs-clone.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZFS-CLONE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-create.8 b/sys/contrib/openzfs/man/man8/zfs-create.8
index 91878056cc7d..58bde5799240 100644
--- a/sys/contrib/openzfs/man/man8/zfs-create.8
+++ b/sys/contrib/openzfs/man/man8/zfs-create.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd June 2, 2023
 .Dt ZFS-CREATE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-destroy.8 b/sys/contrib/openzfs/man/man8/zfs-destroy.8
index 38359be02430..6a6791f7a44e 100644
--- a/sys/contrib/openzfs/man/man8/zfs-destroy.8
+++ b/sys/contrib/openzfs/man/man8/zfs-destroy.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd February 5, 2025
 .Dt ZFS-DESTROY 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-diff.8 b/sys/contrib/openzfs/man/man8/zfs-diff.8
index d4c48f4109be..5b94ea524666 100644
--- a/sys/contrib/openzfs/man/man8/zfs-diff.8
+++ b/sys/contrib/openzfs/man/man8/zfs-diff.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZFS-DIFF 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-hold.8 b/sys/contrib/openzfs/man/man8/zfs-hold.8
index 0c88937f0dc8..a877e428f88b 100644
--- a/sys/contrib/openzfs/man/man8/zfs-hold.8
+++ b/sys/contrib/openzfs/man/man8/zfs-hold.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd June 30, 2019
+.Dd November 8, 2022
 .Dt ZFS-HOLD 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-jail.8 b/sys/contrib/openzfs/man/man8/zfs-jail.8
index 53499a279d05..569f5f57eab4 100644
--- a/sys/contrib/openzfs/man/man8/zfs-jail.8
+++ b/sys/contrib/openzfs/man/man8/zfs-jail.8
@@ -37,7 +37,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd May 27, 2021
+.Dd July 11, 2022
 .Dt ZFS-JAIL 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-list.8 b/sys/contrib/openzfs/man/man8/zfs-list.8
index 677d8292e207..42eff94f9762 100644
--- a/sys/contrib/openzfs/man/man8/zfs-list.8
+++ b/sys/contrib/openzfs/man/man8/zfs-list.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd February 8, 2024
+.Dd August 25, 2025
 .Dt ZFS-LIST 8
 .Os
 .
@@ -50,27 +50,25 @@
 .Oo Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Oc Ns …
 .
 .Sh DESCRIPTION
-If specified, you can list property information by the absolute pathname or the
-relative pathname.
-By default, all file systems and volumes are displayed.
+By default, all file systems and volumes are displayed, with the following
+fields:
+.Sy name , Sy used , Sy available , Sy referenced , Sy mountpoint .
 Snapshots are displayed if the
 .Sy listsnapshots
 pool property is
 .Sy on
 .Po the default is
 .Sy off
-.Pc ,
+.Pc
 or if the
 .Fl t Sy snapshot
 or
 .Fl t Sy all
 options are specified.
-The following fields are displayed:
-.Sy name , Sy used , Sy available , Sy referenced , Sy mountpoint .
 .Bl -tag -width "-H"
 .It Fl H
 Used for scripting mode.
-Do not print headers and separate fields by a single tab instead of arbitrary
+Do not print headers, and separate fields by a single tab instead of arbitrary
 white space.
 .It Fl j , -json Op Ar --json-int
 Print the output in JSON format.
@@ -87,7 +85,7 @@ of
 will display only the dataset and its direct children.
 .It Fl o Ar property
 A comma-separated list of properties to display.
-The property must be:
+Each property must be:
 .Bl -bullet -compact
 .It
 One of the properties described in the
@@ -125,30 +123,41 @@ section of
 or the value
 .Sy name
 to sort by the dataset name.
-Multiple properties can be specified at one time using multiple
+Multiple properties can be specified to operate together using multiple
 .Fl s
-property options.
+or
+.Fl S
+options.
 Multiple
 .Fl s
-options are evaluated from left to right in decreasing order of importance.
-The following is a list of sorting criteria:
+and
+.Fl S
+options are evaluated from left to right to supply sort keys in
+decreasing order of priority.
+Property types operate as follows:
 .Bl -bullet -compact
 .It
 Numeric types sort in numeric order.
 .It
 String types sort in alphabetical order.
 .It
-Types inappropriate for a row sort that row to the literal bottom, regardless of
-the specified ordering.
+Types inappropriate for a row sort that row to the literal bottom,
+regardless of the specified ordering.
 .El
 .Pp
-If no sorting options are specified the existing behavior of
-.Nm zfs Cm list
-is preserved.
+If no sort columns are specified, or if two lines of output would sort
+equally across all specified columns, then datasets and bookmarks are
+sorted by name, whereas snapshots are sorted first by the name of their
+dataset and then by the time of their creation.
+When no sort columns are specified but snapshots are listed, this
+default behavior causes snapshots to be grouped under their datasets in
+chronological order by creation time.
 .It Fl S Ar property
 Same as
 .Fl s ,
-but sorts by property in descending order.
+but sorts by
+.Ar property
+in descending order.
 .It Fl t Ar type
 A comma-separated list of types to display, where
 .Ar type
diff --git a/sys/contrib/openzfs/man/man8/zfs-load-key.8 b/sys/contrib/openzfs/man/man8/zfs-load-key.8
index 7838c46d9e77..3a11cea99fd6 100644
--- a/sys/contrib/openzfs/man/man8/zfs-load-key.8
+++ b/sys/contrib/openzfs/man/man8/zfs-load-key.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd January 13, 2020
+.Dd July 11, 2022
 .Dt ZFS-LOAD-KEY 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-mount-generator.8.in b/sys/contrib/openzfs/man/man8/zfs-mount-generator.8.in
index ea470247daac..9e44ea30c636 100644
--- a/sys/contrib/openzfs/man/man8/zfs-mount-generator.8.in
+++ b/sys/contrib/openzfs/man/man8/zfs-mount-generator.8.in
@@ -23,7 +23,7 @@
 .\" OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 .\" WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 .\"
-.Dd May 31, 2021
+.Dd November 30, 2021
 .Dt ZFS-MOUNT-GENERATOR 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-mount.8 b/sys/contrib/openzfs/man/man8/zfs-mount.8
index 9fca6fffd5bb..2689b6dc345b 100644
--- a/sys/contrib/openzfs/man/man8/zfs-mount.8
+++ b/sys/contrib/openzfs/man/man8/zfs-mount.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd February 16, 2019
+.Dd October 12, 2024
 .Dt ZFS-MOUNT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-project.8 b/sys/contrib/openzfs/man/man8/zfs-project.8
index 36547680f53e..4ebfdf6ffe4f 100644
--- a/sys/contrib/openzfs/man/man8/zfs-project.8
+++ b/sys/contrib/openzfs/man/man8/zfs-project.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd May 27, 2021
+.Dd July 11, 2022
 .Dt ZFS-PROJECT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-promote.8 b/sys/contrib/openzfs/man/man8/zfs-promote.8
index 767045812607..435a7a5d0144 100644
--- a/sys/contrib/openzfs/man/man8/zfs-promote.8
+++ b/sys/contrib/openzfs/man/man8/zfs-promote.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZFS-PROMOTE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-rename.8 b/sys/contrib/openzfs/man/man8/zfs-rename.8
index 4cf192c0682b..8fedc67469e6 100644
--- a/sys/contrib/openzfs/man/man8/zfs-rename.8
+++ b/sys/contrib/openzfs/man/man8/zfs-rename.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZFS-RENAME 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-rewrite.8 b/sys/contrib/openzfs/man/man8/zfs-rewrite.8
index a3a037f3794a..ca5340c7e5eb 100644
--- a/sys/contrib/openzfs/man/man8/zfs-rewrite.8
+++ b/sys/contrib/openzfs/man/man8/zfs-rewrite.8
@@ -21,7 +21,7 @@
 .\"
 .\" Copyright (c) 2025 iXsystems, Inc.
 .\"
-.Dd May 6, 2025
+.Dd July 23, 2025
 .Dt ZFS-REWRITE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-send.8 b/sys/contrib/openzfs/man/man8/zfs-send.8
index c920a5a48798..6c5f6b94afd5 100644
--- a/sys/contrib/openzfs/man/man8/zfs-send.8
+++ b/sys/contrib/openzfs/man/man8/zfs-send.8
@@ -31,7 +31,7 @@
 .\" Copyright 2019 Joyent, Inc.
 .\" Copyright (c) 2024, Klara, Inc.
 .\"
-.Dd October 2, 2024
+.Dd August 29, 2025
 .Dt ZFS-SEND 8
 .Os
 .
@@ -173,8 +173,10 @@ The receiving system must have the
 feature enabled.
 If the
 .Sy lz4_compress
-feature is active on the sending system, then the receiving system must have
-that feature enabled as well.
+or
+.Sy zstd_compress
+features are active on the sending system, then the receiving system must have
+the corresponding features enabled as well.
 Datasets that are sent with this flag may not be
 received as an encrypted dataset, since encrypted datasets cannot use the
 .Sy embedded_data
@@ -201,8 +203,10 @@ property for details
 .Pc .
 If the
 .Sy lz4_compress
-feature is active on the sending system, then the receiving system must have
-that feature enabled as well.
+or
+.Sy zstd_compress
+features are active on the sending system, then the receiving system must have
+the corresponding features enabled as well.
 If the
 .Sy large_blocks
 feature is enabled on the sending system but the
@@ -357,8 +361,10 @@ property for details
 .Pc .
 If the
 .Sy lz4_compress
-feature is active on the sending system, then the receiving system must have
-that feature enabled as well.
+or
+.Sy zstd_compress
+features are active on the sending system, then the receiving system must have
+the corresponding features enabled as well.
 If the
 .Sy large_blocks
 feature is enabled on the sending system but the
@@ -400,8 +406,10 @@ The receiving system must have the
 feature enabled.
 If the
 .Sy lz4_compress
-feature is active on the sending system, then the receiving system must have
-that feature enabled as well.
+or
+.Sy zstd_compress
+features are active on the sending system, then the receiving system must have
+the corresponding features enabled as well.
 Datasets that are sent with this flag may not be received as an encrypted
 dataset,
 since encrypted datasets cannot use the
diff --git a/sys/contrib/openzfs/man/man8/zfs-set.8 b/sys/contrib/openzfs/man/man8/zfs-set.8
index 67f4d6eba171..08daf09d05f8 100644
--- a/sys/contrib/openzfs/man/man8/zfs-set.8
+++ b/sys/contrib/openzfs/man/man8/zfs-set.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd April 20, 2024
+.Dd October 12, 2024
 .Dt ZFS-SET 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-share.8 b/sys/contrib/openzfs/man/man8/zfs-share.8
index f7a09a189182..e9c32a44b0c7 100644
--- a/sys/contrib/openzfs/man/man8/zfs-share.8
+++ b/sys/contrib/openzfs/man/man8/zfs-share.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd May 17, 2021
+.Dd July 11, 2022
 .Dt ZFS-SHARE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-snapshot.8 b/sys/contrib/openzfs/man/man8/zfs-snapshot.8
index 3ddd1273c8e8..8f4b2c335f09 100644
--- a/sys/contrib/openzfs/man/man8/zfs-snapshot.8
+++ b/sys/contrib/openzfs/man/man8/zfs-snapshot.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZFS-SNAPSHOT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-upgrade.8 b/sys/contrib/openzfs/man/man8/zfs-upgrade.8
index bac74e37aef9..a5ce2b760da4 100644
--- a/sys/contrib/openzfs/man/man8/zfs-upgrade.8
+++ b/sys/contrib/openzfs/man/man8/zfs-upgrade.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd June 30, 2019
+.Dd July 11, 2022
 .Dt ZFS-UPGRADE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-userspace.8 b/sys/contrib/openzfs/man/man8/zfs-userspace.8
index d7a4d18e83b1..c255d911740d 100644
--- a/sys/contrib/openzfs/man/man8/zfs-userspace.8
+++ b/sys/contrib/openzfs/man/man8/zfs-userspace.8
@@ -30,7 +30,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd June 30, 2019
+.Dd July 11, 2022
 .Dt ZFS-USERSPACE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-wait.8 b/sys/contrib/openzfs/man/man8/zfs-wait.8
index 554a67455c60..e5c60010d2f9 100644
--- a/sys/contrib/openzfs/man/man8/zfs-wait.8
+++ b/sys/contrib/openzfs/man/man8/zfs-wait.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd May 31, 2021
+.Dd July 11, 2022
 .Dt ZFS-WAIT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs-zone.8 b/sys/contrib/openzfs/man/man8/zfs-zone.8
index 7ad0ac89463c..a56a304e82b2 100644
--- a/sys/contrib/openzfs/man/man8/zfs-zone.8
+++ b/sys/contrib/openzfs/man/man8/zfs-zone.8
@@ -38,7 +38,7 @@
 .\" Copyright 2019 Joyent, Inc.
 .\" Copyright 2021 Klara, Inc.
 .\"
-.Dd June 3, 2022
+.Dd July 11, 2022
 .Dt ZFS-ZONE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs.8 b/sys/contrib/openzfs/man/man8/zfs.8
index e16a3a82b672..b7566a727469 100644
--- a/sys/contrib/openzfs/man/man8/zfs.8
+++ b/sys/contrib/openzfs/man/man8/zfs.8
@@ -37,7 +37,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd April 18, 2025
+.Dd May 12, 2025
 .Dt ZFS 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zfs_ids_to_path.8 b/sys/contrib/openzfs/man/man8/zfs_ids_to_path.8
index eef0ce68f17b..465e336d170c 100644
--- a/sys/contrib/openzfs/man/man8/zfs_ids_to_path.8
+++ b/sys/contrib/openzfs/man/man8/zfs_ids_to_path.8
@@ -21,7 +21,7 @@
 .\"
 .\" Copyright (c) 2020 by Delphix. All rights reserved.
 .\"
-.Dd April 17, 2020
+.Dd July 11, 2022
 .Dt ZFS_IDS_TO_PATH 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zgenhostid.8 b/sys/contrib/openzfs/man/man8/zgenhostid.8
index 2b5b4fc18216..ff564880f97d 100644
--- a/sys/contrib/openzfs/man/man8/zgenhostid.8
+++ b/sys/contrib/openzfs/man/man8/zgenhostid.8
@@ -21,7 +21,7 @@
 .\"
 .\" Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
 .\"
-.Dd May 26, 2021
+.Dd July 11, 2022
 .Dt ZGENHOSTID 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zinject.8 b/sys/contrib/openzfs/man/man8/zinject.8
index 1d9e43aed5ec..704f6a7accd8 100644
--- a/sys/contrib/openzfs/man/man8/zinject.8
+++ b/sys/contrib/openzfs/man/man8/zinject.8
@@ -138,6 +138,20 @@ This injector is automatically cleared after the import is finished.
 .
 .It Xo
 .Nm zinject
+.Fl E Ar delay
+.Op Fl a
+.Op Fl m
+.Op Fl f Ar freq
+.Op Fl l Ar level
+.Op Fl r Ar range
+.Op Fl T Ar iotype
+.Op Fl t Ar type Ns | Ns Fl b Ar bookmark
+.Xc
+Inject pipeline ready stage delays for the given object or bookmark.
+The delay is specified in milliseconds.
+.
+.It Xo
+.Nm zinject
 .Fl I
 .Op Fl s Ar seconds Ns | Ns Fl g Ar txgs
 .Ar pool
diff --git a/sys/contrib/openzfs/man/man8/zpool-attach.8 b/sys/contrib/openzfs/man/man8/zpool-attach.8
index 51d876767666..04996ed4fa11 100644
--- a/sys/contrib/openzfs/man/man8/zpool-attach.8
+++ b/sys/contrib/openzfs/man/man8/zpool-attach.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd June 28, 2023
+.Dd November 8, 2023
 .Dt ZPOOL-ATTACH 8
 .Os
 .
@@ -39,24 +39,24 @@
 .Cm attach
 .Op Fl fsw
 .Oo Fl o Ar property Ns = Ns Ar value Oc
-.Ar pool device new_device
+.Ar pool vdev new_device
 .
 .Sh DESCRIPTION
 Attaches
 .Ar new_device
 to the existing
-.Ar device .
+.Ar vdev .
 The behavior differs depending on if the existing
-.Ar device
+.Ar vdev
 is a RAID-Z device, or a mirror/plain device.
 .Pp
-If the existing device is a mirror or plain device
+If the existing vdev is a mirror or plain device
 .Pq e.g. specified as Qo Li sda Qc or Qq Li mirror-7 ,
-the new device will be mirrored with the existing device, a resilver will be
+the new device will be mirrored with the existing vdev, a resilver will be
 initiated, and the new device will contribute to additional redundancy once the
 resilver completes.
 If
-.Ar device
+.Ar vdev
 is not currently part of a mirrored configuration,
 .Ar device
 automatically transforms into a two-way mirror of
@@ -64,7 +64,7 @@ automatically transforms into a two-way mirror of
 and
 .Ar new_device .
 If
-.Ar device
+.Ar vdev
 is part of a two-way mirror, attaching
 .Ar new_device
 creates a three-way mirror, and so on.
@@ -72,7 +72,7 @@ In either case,
 .Ar new_device
 begins to resilver immediately and any running scrub is canceled.
 .Pp
-If the existing device is a RAID-Z device
+If the existing vdev is a RAID-Z device
 .Pq e.g. specified as Qq Ar raidz2-0 ,
 the new device will become part of that RAID-Z group.
 A "raidz expansion" will be initiated, and once the expansion completes,
@@ -112,7 +112,7 @@ the checksums of all blocks which have been copied during the expansion.
 Forces use of
 .Ar new_device ,
 even if it appears to be in use.
-Not all devices can be overridden in this manner.
+Not all vdevs can be overridden in this manner.
 .It Fl o Ar property Ns = Ns Ar value
 Sets the given pool properties.
 See the
@@ -121,7 +121,7 @@ manual page for a list of valid properties that can be set.
 The only property supported at the moment is
 .Sy ashift .
 .It Fl s
-When attaching to a mirror or plain device, the
+When attaching to a mirror or plain vdev, the
 .Ar new_device
 is reconstructed sequentially to restore redundancy as quickly as possible.
 Checksums are not verified during sequential reconstruction so a scrub is
diff --git a/sys/contrib/openzfs/man/man8/zpool-checkpoint.8 b/sys/contrib/openzfs/man/man8/zpool-checkpoint.8
index d97d10d5df6e..b654f669cfa2 100644
--- a/sys/contrib/openzfs/man/man8/zpool-checkpoint.8
+++ b/sys/contrib/openzfs/man/man8/zpool-checkpoint.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd May 27, 2021
+.Dd July 11, 2022
 .Dt ZPOOL-CHECKPOINT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-clear.8 b/sys/contrib/openzfs/man/man8/zpool-clear.8
index 19cd4be36408..70cd8325bd0e 100644
--- a/sys/contrib/openzfs/man/man8/zpool-clear.8
+++ b/sys/contrib/openzfs/man/man8/zpool-clear.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd May 27, 2021
+.Dd April 29, 2024
 .Dt ZPOOL-CLEAR 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-create.8 b/sys/contrib/openzfs/man/man8/zpool-create.8
index 490c67629a20..a36ae260a158 100644
--- a/sys/contrib/openzfs/man/man8/zpool-create.8
+++ b/sys/contrib/openzfs/man/man8/zpool-create.8
@@ -28,7 +28,7 @@
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\" Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZPOOL-CREATE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-destroy.8 b/sys/contrib/openzfs/man/man8/zpool-destroy.8
index f49f29804ad7..82f3f3e203d6 100644
--- a/sys/contrib/openzfs/man/man8/zpool-destroy.8
+++ b/sys/contrib/openzfs/man/man8/zpool-destroy.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZPOOL-DESTROY 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-detach.8 b/sys/contrib/openzfs/man/man8/zpool-detach.8
index ae02dbc2d5b8..79a44310110d 100644
--- a/sys/contrib/openzfs/man/man8/zpool-detach.8
+++ b/sys/contrib/openzfs/man/man8/zpool-detach.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd August 9, 2019
+.Dd July 11, 2022
 .Dt ZPOOL-DETACH 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-events.8 b/sys/contrib/openzfs/man/man8/zpool-events.8
index 2d32dce2bb65..36a9864dc73b 100644
--- a/sys/contrib/openzfs/man/man8/zpool-events.8
+++ b/sys/contrib/openzfs/man/man8/zpool-events.8
@@ -190,6 +190,16 @@ Issued when a scrub is resumed on a pool.
 .It Sy scrub.paused
 Issued when a scrub is paused on a pool.
 .It Sy bootfs.vdev.attach
+.It Sy sitout
+Issued when a
+.Sy RAIDZ
+or
+.Sy DRAID
+vdev triggers the
+.Sy autosit
+logic.
+This logic detects when a disk in such a vdev is significantly slower than its
+peers, and sits them out temporarily to preserve the performance of the pool.
 .El
 .
 .Sh PAYLOADS
diff --git a/sys/contrib/openzfs/man/man8/zpool-export.8 b/sys/contrib/openzfs/man/man8/zpool-export.8
index 171a7541c6d2..02495c088f94 100644
--- a/sys/contrib/openzfs/man/man8/zpool-export.8
+++ b/sys/contrib/openzfs/man/man8/zpool-export.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZPOOL-EXPORT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-get.8 b/sys/contrib/openzfs/man/man8/zpool-get.8
index 1d6d1f08afa6..bfe1bae7619f 100644
--- a/sys/contrib/openzfs/man/man8/zpool-get.8
+++ b/sys/contrib/openzfs/man/man8/zpool-get.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd August 9, 2019
+.Dd October 12, 2024
 .Dt ZPOOL-GET 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-history.8 b/sys/contrib/openzfs/man/man8/zpool-history.8
index f15086eabc47..f02168951ff2 100644
--- a/sys/contrib/openzfs/man/man8/zpool-history.8
+++ b/sys/contrib/openzfs/man/man8/zpool-history.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd August 9, 2019
+.Dd July 11, 2022
 .Dt ZPOOL-HISTORY 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-import.8 b/sys/contrib/openzfs/man/man8/zpool-import.8
index 9076f5c34929..c6d5f222b6b2 100644
--- a/sys/contrib/openzfs/man/man8/zpool-import.8
+++ b/sys/contrib/openzfs/man/man8/zpool-import.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZPOOL-IMPORT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-initialize.8 b/sys/contrib/openzfs/man/man8/zpool-initialize.8
index 39579a58010e..5299a897cb97 100644
--- a/sys/contrib/openzfs/man/man8/zpool-initialize.8
+++ b/sys/contrib/openzfs/man/man8/zpool-initialize.8
@@ -28,7 +28,7 @@
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\" Copyright (c) 2025 Hewlett Packard Enterprise Development LP.
 .\"
-.Dd May 27, 2021
+.Dd July 30, 2025
 .Dt ZPOOL-INITIALIZE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-iostat.8 b/sys/contrib/openzfs/man/man8/zpool-iostat.8
index d8c21d0cfc6c..5dd9c9d55e20 100644
--- a/sys/contrib/openzfs/man/man8/zpool-iostat.8
+++ b/sys/contrib/openzfs/man/man8/zpool-iostat.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd March 16, 2022
+.Dd January 29, 2024
 .Dt ZPOOL-IOSTAT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-labelclear.8 b/sys/contrib/openzfs/man/man8/zpool-labelclear.8
index ba3d1509aa75..b807acaaede3 100644
--- a/sys/contrib/openzfs/man/man8/zpool-labelclear.8
+++ b/sys/contrib/openzfs/man/man8/zpool-labelclear.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd May 31, 2021
+.Dd July 11, 2022
 .Dt ZPOOL-LABELCLEAR 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-list.8 b/sys/contrib/openzfs/man/man8/zpool-list.8
index b720e203c1c9..106399941f98 100644
--- a/sys/contrib/openzfs/man/man8/zpool-list.8
+++ b/sys/contrib/openzfs/man/man8/zpool-list.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd March 16, 2022
+.Dd October 12, 2024
 .Dt ZPOOL-LIST 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-offline.8 b/sys/contrib/openzfs/man/man8/zpool-offline.8
index 49b1f34ad5d5..388c7634acce 100644
--- a/sys/contrib/openzfs/man/man8/zpool-offline.8
+++ b/sys/contrib/openzfs/man/man8/zpool-offline.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd August 9, 2019
+.Dd December 21, 2023
 .Dt ZPOOL-OFFLINE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-reguid.8 b/sys/contrib/openzfs/man/man8/zpool-reguid.8
index 77101fc07326..b98c88e320de 100644
--- a/sys/contrib/openzfs/man/man8/zpool-reguid.8
+++ b/sys/contrib/openzfs/man/man8/zpool-reguid.8
@@ -29,7 +29,7 @@
 .\" Copyright (c) 2024, Klara Inc.
 .\" Copyright (c) 2024, Mateusz Piotrowski
 .\"
-.Dd June 21, 2023
+.Dd August 26, 2024
 .Dt ZPOOL-REGUID 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-remove.8 b/sys/contrib/openzfs/man/man8/zpool-remove.8
index d10a92e49bbe..483d885f10fd 100644
--- a/sys/contrib/openzfs/man/man8/zpool-remove.8
+++ b/sys/contrib/openzfs/man/man8/zpool-remove.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd March 16, 2022
+.Dd November 19, 2024
 .Dt ZPOOL-REMOVE 8
 .Os
 .
@@ -58,8 +58,8 @@ This command supports removing hot spare, cache, log, and both mirrored and
 non-redundant primary top-level vdevs, including dedup and special vdevs.
 .Pp
 Top-level vdevs can only be removed if the primary pool storage does not contain
-a top-level raidz vdev, all top-level vdevs have the same sector size, and the
-keys for all encrypted datasets are loaded.
+a top-level raidz or draid vdev, all top-level vdevs have the same ashift size,
+and the keys for all encrypted datasets are loaded.
 .Pp
 Removing a top-level vdev reduces the total amount of space in the storage pool.
 The specified device will be evacuated by copying all allocated space from it to
diff --git a/sys/contrib/openzfs/man/man8/zpool-reopen.8 b/sys/contrib/openzfs/man/man8/zpool-reopen.8
index 594cff3d16d8..c4e10f0a546e 100644
--- a/sys/contrib/openzfs/man/man8/zpool-reopen.8
+++ b/sys/contrib/openzfs/man/man8/zpool-reopen.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd June 2, 2021
+.Dd July 11, 2022
 .Dt ZPOOL-REOPEN 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-replace.8 b/sys/contrib/openzfs/man/man8/zpool-replace.8
index 9f3156eeb3ef..651af13b19b8 100644
--- a/sys/contrib/openzfs/man/man8/zpool-replace.8
+++ b/sys/contrib/openzfs/man/man8/zpool-replace.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd May 29, 2021
+.Dd July 11, 2022
 .Dt ZPOOL-REPLACE 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-resilver.8 b/sys/contrib/openzfs/man/man8/zpool-resilver.8
index 2161d77f62ed..59c4be5db209 100644
--- a/sys/contrib/openzfs/man/man8/zpool-resilver.8
+++ b/sys/contrib/openzfs/man/man8/zpool-resilver.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd May 27, 2021
+.Dd July 11, 2022
 .Dt ZPOOL-RESILVER 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-scrub.8 b/sys/contrib/openzfs/man/man8/zpool-scrub.8
index 0ecf8bd3851f..cf7ead5788bf 100644
--- a/sys/contrib/openzfs/man/man8/zpool-scrub.8
+++ b/sys/contrib/openzfs/man/man8/zpool-scrub.8
@@ -28,7 +28,7 @@
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\" Copyright (c) 2025 Hewlett Packard Enterprise Development LP.
 .\"
-.Dd December 11, 2024
+.Dd August 6, 2025
 .Dt ZPOOL-SCRUB 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-split.8 b/sys/contrib/openzfs/man/man8/zpool-split.8
index a67c865cf30c..ee4c6384cf23 100644
--- a/sys/contrib/openzfs/man/man8/zpool-split.8
+++ b/sys/contrib/openzfs/man/man8/zpool-split.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd June 2, 2021
+.Dd July 11, 2022
 .Dt ZPOOL-SPLIT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-status.8 b/sys/contrib/openzfs/man/man8/zpool-status.8
index a7f3e088043b..108a1067b384 100644
--- a/sys/contrib/openzfs/man/man8/zpool-status.8
+++ b/sys/contrib/openzfs/man/man8/zpool-status.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd February 14, 2024
+.Dd May 20, 2025
 .Dt ZPOOL-STATUS 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-sync.8 b/sys/contrib/openzfs/man/man8/zpool-sync.8
index 8f438f363e83..d1dc05d0c202 100644
--- a/sys/contrib/openzfs/man/man8/zpool-sync.8
+++ b/sys/contrib/openzfs/man/man8/zpool-sync.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd August 9, 2019
+.Dd July 11, 2022
 .Dt ZPOOL-SYNC 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-trim.8 b/sys/contrib/openzfs/man/man8/zpool-trim.8
index 18723e1be0d2..c4e849019789 100644
--- a/sys/contrib/openzfs/man/man8/zpool-trim.8
+++ b/sys/contrib/openzfs/man/man8/zpool-trim.8
@@ -28,7 +28,7 @@
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\" Copyright (c) 2025 Hewlett Packard Enterprise Development LP.
 .\"
-.Dd May 27, 2021
+.Dd July 30, 2025
 .Dt ZPOOL-TRIM 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool-upgrade.8 b/sys/contrib/openzfs/man/man8/zpool-upgrade.8
index 20632ae4bba0..adae47f82eb1 100644
--- a/sys/contrib/openzfs/man/man8/zpool-upgrade.8
+++ b/sys/contrib/openzfs/man/man8/zpool-upgrade.8
@@ -28,7 +28,7 @@
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\" Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
 .\"
-.Dd March 16, 2022
+.Dd July 11, 2022
 .Dt ZPOOL-UPGRADE 8
 .Os
 .
@@ -65,10 +65,10 @@ property).
 .Cm upgrade
 .Fl v
 .Xc
-Displays legacy ZFS versions supported by the this version of ZFS.
+Displays legacy ZFS versions supported by this version of ZFS.
 See
 .Xr zpool-features 7
-for a description of feature flags features supported by this version of ZFS.
+for a description of features supported by this version of ZFS.
 .It Xo
 .Nm zpool
 .Cm upgrade
diff --git a/sys/contrib/openzfs/man/man8/zpool-wait.8 b/sys/contrib/openzfs/man/man8/zpool-wait.8
index 0ffb4badfb7b..28a51d29a913 100644
--- a/sys/contrib/openzfs/man/man8/zpool-wait.8
+++ b/sys/contrib/openzfs/man/man8/zpool-wait.8
@@ -28,7 +28,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd May 27, 2021
+.Dd January 29, 2024
 .Dt ZPOOL-WAIT 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zpool.8 b/sys/contrib/openzfs/man/man8/zpool.8
index b96944050594..3bfef780b298 100644
--- a/sys/contrib/openzfs/man/man8/zpool.8
+++ b/sys/contrib/openzfs/man/man8/zpool.8
@@ -27,7 +27,7 @@
 .\" Copyright 2017 Nexenta Systems, Inc.
 .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 .\"
-.Dd February 14, 2024
+.Dd November 19, 2024
 .Dt ZPOOL 8
 .Os
 .
diff --git a/sys/contrib/openzfs/man/man8/zstream.8 b/sys/contrib/openzfs/man/man8/zstream.8
index 03a8479c9e6a..5b3d063bc4a5 100644
--- a/sys/contrib/openzfs/man/man8/zstream.8
+++ b/sys/contrib/openzfs/man/man8/zstream.8
@@ -21,7 +21,7 @@
 .\"
 .\" Copyright (c) 2020 by Delphix. All rights reserved.
 .\"
-.Dd October 4, 2022
+.Dd November 10, 2022
 .Dt ZSTREAM 8
 .Os
 .
diff --git a/sys/contrib/openzfs/module/Kbuild.in b/sys/contrib/openzfs/module/Kbuild.in
index 3d6f288fa5da..58a80dc4402c 100644
--- a/sys/contrib/openzfs/module/Kbuild.in
+++ b/sys/contrib/openzfs/module/Kbuild.in
@@ -4,7 +4,7 @@
 
 ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement
 ZFS_MODULE_CFLAGS += -Wmissing-prototypes
-ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@  @NO_FORMAT_ZERO_LENGTH@
+ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@ @KERNEL_NO_FORMAT_ZERO_LENGTH@
 
 ifneq ($(KBUILD_EXTMOD),)
 zfs_include = @abs_top_srcdir@/include
@@ -135,6 +135,7 @@ ICP_OBJS_X86_64 := \
 	asm-x86_64/sha2/sha256-x86_64.o \
 	asm-x86_64/sha2/sha512-x86_64.o \
 	asm-x86_64/modes/aesni-gcm-x86_64.o \
+	asm-x86_64/modes/aesni-gcm-avx2-vaes.o \
 	asm-x86_64/modes/gcm_pclmulqdq.o \
 	asm-x86_64/modes/ghash-x86_64.o
 
diff --git a/sys/contrib/openzfs/module/avl/avl.c b/sys/contrib/openzfs/module/avl/avl.c
index b6c1c02bc3f2..67cbcd3adeec 100644
--- a/sys/contrib/openzfs/module/avl/avl.c
+++ b/sys/contrib/openzfs/module/avl/avl.c
@@ -225,7 +225,7 @@ avl_nearest(avl_tree_t *tree, avl_index_t where, int direction)
 	size_t off = tree->avl_offset;
 
 	if (node == NULL) {
-		ASSERT(tree->avl_root == NULL);
+		ASSERT0P(tree->avl_root);
 		return (NULL);
 	}
 	data = AVL_NODE2DATA(node, off);
@@ -478,7 +478,7 @@ avl_insert(avl_tree_t *tree, void *new_data, avl_index_t where)
 	size_t off = tree->avl_offset;
 
 #ifdef _LP64
-	ASSERT(((uintptr_t)new_data & 0x7) == 0);
+	ASSERT0(((uintptr_t)new_data & 0x7));
 #endif
 
 	node = AVL_DATA2NODE(new_data, off);
@@ -495,10 +495,10 @@ avl_insert(avl_tree_t *tree, void *new_data, avl_index_t where)
 	AVL_SETBALANCE(node, 0);
 	AVL_SETPARENT(node, parent);
 	if (parent != NULL) {
-		ASSERT(parent->avl_child[which_child] == NULL);
+		ASSERT0P(parent->avl_child[which_child]);
 		parent->avl_child[which_child] = node;
 	} else {
-		ASSERT(tree->avl_root == NULL);
+		ASSERT0P(tree->avl_root);
 		tree->avl_root = node;
 	}
 	/*
@@ -608,7 +608,7 @@ avl_insert_here(
 		ASSERT(diff > 0 ? child == 1 : child == 0);
 #endif
 	}
-	ASSERT(node->avl_child[child] == NULL);
+	ASSERT0P(node->avl_child[child]);
 
 	avl_insert(tree, new_data, AVL_MKINDEX(node, child));
 }
@@ -881,7 +881,7 @@ avl_create(avl_tree_t *tree, int (*compar) (const void *, const void *),
 	ASSERT(size > 0);
 	ASSERT(size >= offset + sizeof (avl_node_t));
 #ifdef _LP64
-	ASSERT((offset & 0x7) == 0);
+	ASSERT0((offset & 0x7));
 #endif
 
 	tree->avl_compar = compar;
@@ -897,8 +897,8 @@ void
 avl_destroy(avl_tree_t *tree)
 {
 	ASSERT(tree);
-	ASSERT(tree->avl_numnodes == 0);
-	ASSERT(tree->avl_root == NULL);
+	ASSERT0(tree->avl_numnodes);
+	ASSERT0P(tree->avl_root);
 }
 
 
diff --git a/sys/contrib/openzfs/module/icp/algs/modes/gcm.c b/sys/contrib/openzfs/module/icp/algs/modes/gcm.c
index c2a982b5a376..3cfa5b8165ce 100644
--- a/sys/contrib/openzfs/module/icp/algs/modes/gcm.c
+++ b/sys/contrib/openzfs/module/icp/algs/modes/gcm.c
@@ -46,6 +46,9 @@
 #define	IMPL_CYCLE	(UINT32_MAX-1)
 #ifdef CAN_USE_GCM_ASM
 #define	IMPL_AVX	(UINT32_MAX-2)
+#if CAN_USE_GCM_ASM >= 2
+#define	IMPL_AVX2	(UINT32_MAX-3)
+#endif
 #endif
 #define	GCM_IMPL_READ(i) (*(volatile uint32_t *) &(i))
 static uint32_t icp_gcm_impl = IMPL_FASTEST;
@@ -56,17 +59,16 @@ static uint32_t user_sel_impl = IMPL_FASTEST;
 boolean_t gcm_avx_can_use_movbe = B_FALSE;
 /*
  * Whether to use the optimized openssl gcm and ghash implementations.
- * Set to true if module parameter icp_gcm_impl == "avx".
  */
-static boolean_t gcm_use_avx = B_FALSE;
-#define	GCM_IMPL_USE_AVX	(*(volatile boolean_t *)&gcm_use_avx)
+static gcm_impl gcm_impl_used = GCM_IMPL_GENERIC;
+#define	GCM_IMPL_USED	(*(volatile gcm_impl *)&gcm_impl_used)
 
 extern boolean_t ASMABI atomic_toggle_boolean_nv(volatile boolean_t *);
 
 static inline boolean_t gcm_avx_will_work(void);
-static inline void gcm_set_avx(boolean_t);
-static inline boolean_t gcm_toggle_avx(void);
-static inline size_t gcm_simd_get_htab_size(boolean_t);
+static inline boolean_t gcm_avx2_will_work(void);
+static inline void gcm_use_impl(gcm_impl impl);
+static inline gcm_impl gcm_toggle_impl(void);
 
 static int gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *, char *, size_t,
     crypto_data_t *, size_t);
@@ -89,7 +91,7 @@ gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
     void (*xor_block)(uint8_t *, uint8_t *))
 {
 #ifdef CAN_USE_GCM_ASM
-	if (ctx->gcm_use_avx == B_TRUE)
+	if (ctx->impl != GCM_IMPL_GENERIC)
 		return (gcm_mode_encrypt_contiguous_blocks_avx(
 		    ctx, data, length, out, block_size));
 #endif
@@ -208,7 +210,7 @@ gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
 {
 	(void) copy_block;
 #ifdef CAN_USE_GCM_ASM
-	if (ctx->gcm_use_avx == B_TRUE)
+	if (ctx->impl != GCM_IMPL_GENERIC)
 		return (gcm_encrypt_final_avx(ctx, out, block_size));
 #endif
 
@@ -374,7 +376,7 @@ gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
     void (*xor_block)(uint8_t *, uint8_t *))
 {
 #ifdef CAN_USE_GCM_ASM
-	if (ctx->gcm_use_avx == B_TRUE)
+	if (ctx->impl != GCM_IMPL_GENERIC)
 		return (gcm_decrypt_final_avx(ctx, out, block_size));
 #endif
 
@@ -631,23 +633,23 @@ gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param,
 	    ((aes_key_t *)gcm_ctx->gcm_keysched)->ops->needs_byteswap;
 
 	if (GCM_IMPL_READ(icp_gcm_impl) != IMPL_CYCLE) {
-		gcm_ctx->gcm_use_avx = GCM_IMPL_USE_AVX;
+		gcm_ctx->impl = GCM_IMPL_USED;
 	} else {
 		/*
-		 * Handle the "cycle" implementation by creating avx and
-		 * non-avx contexts alternately.
+		 * Handle the "cycle" implementation by creating different
+		 * contexts, one per implementation.
 		 */
-		gcm_ctx->gcm_use_avx = gcm_toggle_avx();
+		gcm_ctx->impl = gcm_toggle_impl();
 
-		/* The avx impl. doesn't handle byte swapped key schedules. */
-		if (gcm_ctx->gcm_use_avx == B_TRUE && needs_bswap == B_TRUE) {
-			gcm_ctx->gcm_use_avx = B_FALSE;
+		/* The AVX impl. doesn't handle byte swapped key schedules. */
+		if (needs_bswap == B_TRUE) {
+			gcm_ctx->impl = GCM_IMPL_GENERIC;
 		}
 		/*
-		 * If this is a GCM context, use the MOVBE and the BSWAP
+		 * If this is an AVX context, use the MOVBE and the BSWAP
 		 * variants alternately.
 		 */
-		if (gcm_ctx->gcm_use_avx == B_TRUE &&
+		if (gcm_ctx->impl == GCM_IMPL_AVX &&
 		    zfs_movbe_available() == B_TRUE) {
 			(void) atomic_toggle_boolean_nv(
 			    (volatile boolean_t *)&gcm_avx_can_use_movbe);
@@ -658,12 +660,13 @@ gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param,
 	 * still they could be created by the aes generic implementation.
 	 * Make sure not to use them since we'll corrupt data if we do.
 	 */
-	if (gcm_ctx->gcm_use_avx == B_TRUE && needs_bswap == B_TRUE) {
-		gcm_ctx->gcm_use_avx = B_FALSE;
+	if (gcm_ctx->impl != GCM_IMPL_GENERIC && needs_bswap == B_TRUE) {
+		gcm_ctx->impl = GCM_IMPL_GENERIC;
 
 		cmn_err_once(CE_WARN,
 		    "ICP: Can't use the aes generic or cycle implementations "
-		    "in combination with the gcm avx implementation!");
+		    "in combination with the gcm avx or avx2-vaes "
+		    "implementation!");
 		cmn_err_once(CE_WARN,
 		    "ICP: Falling back to a compatible implementation, "
 		    "aes-gcm performance will likely be degraded.");
@@ -672,36 +675,20 @@ gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param,
 		    "restore performance.");
 	}
 
-	/* Allocate Htab memory as needed. */
-	if (gcm_ctx->gcm_use_avx == B_TRUE) {
-		size_t htab_len = gcm_simd_get_htab_size(gcm_ctx->gcm_use_avx);
-
-		if (htab_len == 0) {
-			return (CRYPTO_MECHANISM_PARAM_INVALID);
-		}
-		gcm_ctx->gcm_htab_len = htab_len;
-		gcm_ctx->gcm_Htable =
-		    kmem_alloc(htab_len, KM_SLEEP);
-
-		if (gcm_ctx->gcm_Htable == NULL) {
-			return (CRYPTO_HOST_MEMORY);
-		}
+	/*
+	 * AVX implementations use Htable with sizes depending on
+	 * implementation.
+	 */
+	if (gcm_ctx->impl != GCM_IMPL_GENERIC) {
+		rv = gcm_init_avx(gcm_ctx, iv, iv_len, aad, aad_len,
+		    block_size);
 	}
-	/* Avx and non avx context initialization differs from here on. */
-	if (gcm_ctx->gcm_use_avx == B_FALSE) {
+	else
 #endif /* ifdef CAN_USE_GCM_ASM */
-		if (gcm_init(gcm_ctx, iv, iv_len, aad, aad_len, block_size,
-		    encrypt_block, copy_block, xor_block) != CRYPTO_SUCCESS) {
-			rv = CRYPTO_MECHANISM_PARAM_INVALID;
-		}
-#ifdef CAN_USE_GCM_ASM
-	} else {
-		if (gcm_init_avx(gcm_ctx, iv, iv_len, aad, aad_len,
-		    block_size) != CRYPTO_SUCCESS) {
-			rv = CRYPTO_MECHANISM_PARAM_INVALID;
-		}
+	if (gcm_init(gcm_ctx, iv, iv_len, aad, aad_len, block_size,
+	    encrypt_block, copy_block, xor_block) != CRYPTO_SUCCESS) {
+		rv = CRYPTO_MECHANISM_PARAM_INVALID;
 	}
-#endif /* ifdef CAN_USE_GCM_ASM */
 
 	return (rv);
 }
@@ -767,6 +754,9 @@ gcm_impl_get_ops(void)
 		break;
 #ifdef CAN_USE_GCM_ASM
 	case IMPL_AVX:
+#if CAN_USE_GCM_ASM >= 2
+	case IMPL_AVX2:
+#endif
 		/*
 		 * Make sure that we return a valid implementation while
 		 * switching to the avx implementation since there still
@@ -828,6 +818,13 @@ gcm_impl_init(void)
 	 * Use the avx implementation if it's available and the implementation
 	 * hasn't changed from its default value of fastest on module load.
 	 */
+#if CAN_USE_GCM_ASM >= 2
+	if (gcm_avx2_will_work()) {
+		if (GCM_IMPL_READ(user_sel_impl) == IMPL_FASTEST) {
+			gcm_use_impl(GCM_IMPL_AVX2);
+		}
+	} else
+#endif
 	if (gcm_avx_will_work()) {
 #ifdef HAVE_MOVBE
 		if (zfs_movbe_available() == B_TRUE) {
@@ -835,7 +832,7 @@ gcm_impl_init(void)
 		}
 #endif
 		if (GCM_IMPL_READ(user_sel_impl) == IMPL_FASTEST) {
-			gcm_set_avx(B_TRUE);
+			gcm_use_impl(GCM_IMPL_AVX);
 		}
 	}
 #endif
@@ -852,6 +849,7 @@ static const struct {
 		{ "fastest",	IMPL_FASTEST },
 #ifdef CAN_USE_GCM_ASM
 		{ "avx",	IMPL_AVX },
+		{ "avx2-vaes",	IMPL_AVX2 },
 #endif
 };
 
@@ -887,7 +885,13 @@ gcm_impl_set(const char *val)
 	/* Check mandatory options */
 	for (i = 0; i < ARRAY_SIZE(gcm_impl_opts); i++) {
 #ifdef CAN_USE_GCM_ASM
+#if CAN_USE_GCM_ASM >= 2
 		/* Ignore avx implementation if it won't work. */
+		if (gcm_impl_opts[i].sel == IMPL_AVX2 &&
+		    !gcm_avx2_will_work()) {
+			continue;
+		}
+#endif
 		if (gcm_impl_opts[i].sel == IMPL_AVX && !gcm_avx_will_work()) {
 			continue;
 		}
@@ -915,11 +919,17 @@ gcm_impl_set(const char *val)
 	 * Use the avx implementation if available and the requested one is
 	 * avx or fastest.
 	 */
+#if CAN_USE_GCM_ASM >= 2
+	if (gcm_avx2_will_work() == B_TRUE &&
+	    (impl == IMPL_AVX2 || impl == IMPL_FASTEST)) {
+		gcm_use_impl(GCM_IMPL_AVX2);
+	} else
+#endif
 	if (gcm_avx_will_work() == B_TRUE &&
 	    (impl == IMPL_AVX || impl == IMPL_FASTEST)) {
-		gcm_set_avx(B_TRUE);
+		gcm_use_impl(GCM_IMPL_AVX);
 	} else {
-		gcm_set_avx(B_FALSE);
+		gcm_use_impl(GCM_IMPL_GENERIC);
 	}
 #endif
 
@@ -952,6 +962,12 @@ icp_gcm_impl_get(char *buffer, zfs_kernel_param_t *kp)
 	for (i = 0; i < ARRAY_SIZE(gcm_impl_opts); i++) {
 #ifdef CAN_USE_GCM_ASM
 		/* Ignore avx implementation if it won't work. */
+#if CAN_USE_GCM_ASM >= 2
+		if (gcm_impl_opts[i].sel == IMPL_AVX2 &&
+		    !gcm_avx2_will_work()) {
+			continue;
+		}
+#endif
 		if (gcm_impl_opts[i].sel == IMPL_AVX && !gcm_avx_will_work()) {
 			continue;
 		}
@@ -993,9 +1009,6 @@ MODULE_PARM_DESC(icp_gcm_impl, "Select gcm implementation.");
 
 /* Clear the FPU registers since they hold sensitive internal state. */
 #define	clear_fpu_regs() clear_fpu_regs_avx()
-#define	GHASH_AVX(ctx, in, len) \
-    gcm_ghash_avx((ctx)->gcm_ghash, (const uint64_t *)(ctx)->gcm_Htable, \
-    in, len)
 
 #define	gcm_incr_counter_block(ctx) gcm_incr_counter_block_by(ctx, 1)
 
@@ -1010,20 +1023,77 @@ MODULE_PARM_DESC(icp_gcm_impl, "Select gcm implementation.");
 static uint32_t gcm_avx_chunk_size =
 	((32 * 1024) / GCM_AVX_MIN_DECRYPT_BYTES) * GCM_AVX_MIN_DECRYPT_BYTES;
 
+/*
+ * GCM definitions: uint128_t is copied from include/crypto/modes.h
+ * Avoiding u128 because it is already defined in kernel sources.
+ */
+typedef struct {
+    uint64_t hi, lo;
+} uint128_t;
+
 extern void ASMABI clear_fpu_regs_avx(void);
 extern void ASMABI gcm_xor_avx(const uint8_t *src, uint8_t *dst);
 extern void ASMABI aes_encrypt_intel(const uint32_t rk[], int nr,
     const uint32_t pt[4], uint32_t ct[4]);
 
 extern void ASMABI gcm_init_htab_avx(uint64_t *Htable, const uint64_t H[2]);
+#if CAN_USE_GCM_ASM >= 2
+extern void ASMABI gcm_init_vpclmulqdq_avx2(uint128_t Htable[16],
+    const uint64_t H[2]);
+#endif
 extern void ASMABI gcm_ghash_avx(uint64_t ghash[2], const uint64_t *Htable,
     const uint8_t *in, size_t len);
+#if CAN_USE_GCM_ASM >= 2
+extern void ASMABI gcm_ghash_vpclmulqdq_avx2(uint64_t ghash[2],
+    const uint64_t *Htable, const uint8_t *in, size_t len);
+#endif
+static inline void GHASH_AVX(gcm_ctx_t *ctx, const uint8_t *in, size_t len)
+{
+	switch (ctx->impl) {
+#if CAN_USE_GCM_ASM >= 2
+		case GCM_IMPL_AVX2:
+			gcm_ghash_vpclmulqdq_avx2(ctx->gcm_ghash,
+			    (const uint64_t *)ctx->gcm_Htable, in, len);
+			break;
+#endif
+
+		case GCM_IMPL_AVX:
+			gcm_ghash_avx(ctx->gcm_ghash,
+			    (const uint64_t *)ctx->gcm_Htable, in, len);
+			break;
+
+		default:
+			VERIFY(B_FALSE);
+	}
+}
 
+typedef size_t ASMABI aesni_gcm_encrypt_impl(const uint8_t *, uint8_t *,
+    size_t, const void *, uint64_t *, const uint64_t *Htable, uint64_t *);
 extern size_t ASMABI aesni_gcm_encrypt(const uint8_t *, uint8_t *, size_t,
     const void *, uint64_t *, uint64_t *);
+#if CAN_USE_GCM_ASM >= 2
+extern void ASMABI aes_gcm_enc_update_vaes_avx2(const uint8_t *in,
+    uint8_t *out, size_t len, const void *key, const uint8_t ivec[16],
+    const uint128_t Htable[16], uint8_t Xi[16]);
+#endif
 
+typedef size_t ASMABI aesni_gcm_decrypt_impl(const uint8_t *, uint8_t *,
+    size_t, const void *, uint64_t *, const uint64_t *Htable, uint64_t *);
 extern size_t ASMABI aesni_gcm_decrypt(const uint8_t *, uint8_t *, size_t,
     const void *, uint64_t *, uint64_t *);
+#if CAN_USE_GCM_ASM >= 2
+extern void ASMABI aes_gcm_dec_update_vaes_avx2(const uint8_t *in,
+    uint8_t *out, size_t len, const void *key, const uint8_t ivec[16],
+    const uint128_t Htable[16], uint8_t Xi[16]);
+#endif
+
+static inline boolean_t
+gcm_avx2_will_work(void)
+{
+	return (kfpu_allowed() &&
+	    zfs_avx2_available() && zfs_vaes_available() &&
+	    zfs_vpclmulqdq_available());
+}
 
 static inline boolean_t
 gcm_avx_will_work(void)
@@ -1035,33 +1105,67 @@ gcm_avx_will_work(void)
 }
 
 static inline void
-gcm_set_avx(boolean_t val)
+gcm_use_impl(gcm_impl impl)
 {
-	if (gcm_avx_will_work() == B_TRUE) {
-		atomic_swap_32(&gcm_use_avx, val);
+	switch (impl) {
+#if CAN_USE_GCM_ASM >= 2
+		case GCM_IMPL_AVX2:
+			if (gcm_avx2_will_work() == B_TRUE) {
+				atomic_swap_32(&gcm_impl_used, impl);
+				return;
+			}
+
+			zfs_fallthrough;
+#endif
+
+		case GCM_IMPL_AVX:
+			if (gcm_avx_will_work() == B_TRUE) {
+				atomic_swap_32(&gcm_impl_used, impl);
+				return;
+			}
+
+			zfs_fallthrough;
+
+		default:
+			atomic_swap_32(&gcm_impl_used, GCM_IMPL_GENERIC);
 	}
 }
 
 static inline boolean_t
-gcm_toggle_avx(void)
+gcm_impl_will_work(gcm_impl impl)
 {
-	if (gcm_avx_will_work() == B_TRUE) {
-		return (atomic_toggle_boolean_nv(&GCM_IMPL_USE_AVX));
-	} else {
-		return (B_FALSE);
+	switch (impl) {
+#if CAN_USE_GCM_ASM >= 2
+		case GCM_IMPL_AVX2:
+			return (gcm_avx2_will_work());
+#endif
+
+		case GCM_IMPL_AVX:
+			return (gcm_avx_will_work());
+
+		default:
+			return (B_TRUE);
 	}
 }
 
-static inline size_t
-gcm_simd_get_htab_size(boolean_t simd_mode)
+static inline gcm_impl
+gcm_toggle_impl(void)
 {
-	switch (simd_mode) {
-	case B_TRUE:
-		return (2 * 6 * 2 * sizeof (uint64_t));
+	gcm_impl current_impl, new_impl;
+	do { /* handle races */
+		current_impl = atomic_load_32(&gcm_impl_used);
+		new_impl = current_impl;
+		while (B_TRUE) { /* handle incompatble implementations */
+			new_impl = (new_impl + 1) % GCM_IMPL_MAX;
+			if (gcm_impl_will_work(new_impl)) {
+				break;
+			}
+		}
 
-	default:
-		return (0);
-	}
+	} while (atomic_cas_32(&gcm_impl_used, current_impl, new_impl) !=
+	    current_impl);
+
+	return (new_impl);
 }
 
 
@@ -1077,6 +1181,50 @@ gcm_incr_counter_block_by(gcm_ctx_t *ctx, int n)
 	ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
 }
 
+static size_t aesni_gcm_encrypt_avx(const uint8_t *in, uint8_t *out,
+    size_t len, const void *key, uint64_t *iv, const uint64_t *Htable,
+    uint64_t *Xip)
+{
+	(void) Htable;
+	return (aesni_gcm_encrypt(in, out, len, key, iv, Xip));
+}
+
+#if CAN_USE_GCM_ASM >= 2
+// kSizeTWithoutLower4Bits is a mask that can be used to zero the lower four
+// bits of a |size_t|.
+// This is from boringssl/crypto/fipsmodule/aes/gcm.cc.inc
+static const size_t kSizeTWithoutLower4Bits = (size_t)-16;
+
+/* The following CRYPTO methods are from boringssl/crypto/internal.h */
+static inline uint32_t CRYPTO_bswap4(uint32_t x) {
+	return (__builtin_bswap32(x));
+}
+
+static inline uint32_t CRYPTO_load_u32_be(const void *in) {
+	uint32_t v;
+	memcpy(&v, in, sizeof (v));
+	return (CRYPTO_bswap4(v));
+}
+
+static inline void CRYPTO_store_u32_be(void *out, uint32_t v) {
+	v = CRYPTO_bswap4(v);
+	memcpy(out, &v, sizeof (v));
+}
+
+static size_t aesni_gcm_encrypt_avx2(const uint8_t *in, uint8_t *out,
+    size_t len, const void *key, uint64_t *iv, const uint64_t *Htable,
+    uint64_t *Xip)
+{
+	uint8_t *ivec = (uint8_t *)iv;
+	len &= kSizeTWithoutLower4Bits;
+	aes_gcm_enc_update_vaes_avx2(in, out, len, key, ivec,
+	    (const uint128_t *)Htable, (uint8_t *)Xip);
+	CRYPTO_store_u32_be(&ivec[12],
+	    CRYPTO_load_u32_be(&ivec[12]) + len / 16);
+	return (len);
+}
+#endif /* if CAN_USE_GCM_ASM >= 2 */
+
 /*
  * Encrypt multiple blocks of data in GCM mode.
  * This is done in gcm_avx_chunk_size chunks, utilizing AVX assembler routines
@@ -1091,8 +1239,15 @@ gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *ctx, char *data,
 	size_t done = 0;
 	uint8_t *datap = (uint8_t *)data;
 	size_t chunk_size = (size_t)GCM_CHUNK_SIZE_READ;
+	aesni_gcm_encrypt_impl *encrypt_blocks =
+#if CAN_USE_GCM_ASM >= 2
+	    ctx->impl == GCM_IMPL_AVX2 ?
+	    aesni_gcm_encrypt_avx2 :
+#endif
+	    aesni_gcm_encrypt_avx;
 	const aes_key_t *key = ((aes_key_t *)ctx->gcm_keysched);
 	uint64_t *ghash = ctx->gcm_ghash;
+	uint64_t *htable = ctx->gcm_Htable;
 	uint64_t *cb = ctx->gcm_cb;
 	uint8_t *ct_buf = NULL;
 	uint8_t *tmp = (uint8_t *)ctx->gcm_tmp;
@@ -1156,8 +1311,8 @@ gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *ctx, char *data,
 	/* Do the bulk encryption in chunk_size blocks. */
 	for (; bleft >= chunk_size; bleft -= chunk_size) {
 		kfpu_begin();
-		done = aesni_gcm_encrypt(
-		    datap, ct_buf, chunk_size, key, cb, ghash);
+		done = encrypt_blocks(
+		    datap, ct_buf, chunk_size, key, cb, htable, ghash);
 
 		clear_fpu_regs();
 		kfpu_end();
@@ -1180,7 +1335,8 @@ gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *ctx, char *data,
 	/* Bulk encrypt the remaining data. */
 	kfpu_begin();
 	if (bleft >= GCM_AVX_MIN_ENCRYPT_BYTES) {
-		done = aesni_gcm_encrypt(datap, ct_buf, bleft, key, cb, ghash);
+		done = encrypt_blocks(datap, ct_buf, bleft, key, cb, htable,
+		    ghash);
 		if (done == 0) {
 			rv = CRYPTO_FAILED;
 			goto out;
@@ -1293,6 +1449,29 @@ gcm_encrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size)
 	return (CRYPTO_SUCCESS);
 }
 
+static size_t aesni_gcm_decrypt_avx(const uint8_t *in, uint8_t *out,
+    size_t len, const void *key, uint64_t *iv, const uint64_t *Htable,
+    uint64_t *Xip)
+{
+	(void) Htable;
+	return (aesni_gcm_decrypt(in, out, len, key, iv, Xip));
+}
+
+#if CAN_USE_GCM_ASM >= 2
+static size_t aesni_gcm_decrypt_avx2(const uint8_t *in, uint8_t *out,
+    size_t len, const void *key, uint64_t *iv, const uint64_t *Htable,
+    uint64_t *Xip)
+{
+	uint8_t *ivec = (uint8_t *)iv;
+	len &= kSizeTWithoutLower4Bits;
+	aes_gcm_dec_update_vaes_avx2(in, out, len, key, ivec,
+	    (const uint128_t *)Htable, (uint8_t *)Xip);
+	CRYPTO_store_u32_be(&ivec[12],
+	    CRYPTO_load_u32_be(&ivec[12]) + len / 16);
+	return (len);
+}
+#endif /* if CAN_USE_GCM_ASM >= 2 */
+
 /*
  * Finalize decryption: We just have accumulated crypto text, so now we
  * decrypt it here inplace.
@@ -1306,10 +1485,17 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size)
 	    B_FALSE);
 
 	size_t chunk_size = (size_t)GCM_CHUNK_SIZE_READ;
+	aesni_gcm_decrypt_impl *decrypt_blocks =
+#if CAN_USE_GCM_ASM >= 2
+	    ctx->impl == GCM_IMPL_AVX2 ?
+	    aesni_gcm_decrypt_avx2 :
+#endif
+	    aesni_gcm_decrypt_avx;
 	size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
 	uint8_t *datap = ctx->gcm_pt_buf;
 	const aes_key_t *key = ((aes_key_t *)ctx->gcm_keysched);
 	uint32_t *cb = (uint32_t *)ctx->gcm_cb;
+	uint64_t *htable = ctx->gcm_Htable;
 	uint64_t *ghash = ctx->gcm_ghash;
 	uint32_t *tmp = (uint32_t *)ctx->gcm_tmp;
 	int rv = CRYPTO_SUCCESS;
@@ -1322,8 +1508,8 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size)
 	 */
 	for (bleft = pt_len; bleft >= chunk_size; bleft -= chunk_size) {
 		kfpu_begin();
-		done = aesni_gcm_decrypt(datap, datap, chunk_size,
-		    (const void *)key, ctx->gcm_cb, ghash);
+		done = decrypt_blocks(datap, datap, chunk_size,
+		    (const void *)key, ctx->gcm_cb, htable, ghash);
 		clear_fpu_regs();
 		kfpu_end();
 		if (done != chunk_size) {
@@ -1334,8 +1520,8 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size)
 	/* Decrypt remainder, which is less than chunk size, in one go. */
 	kfpu_begin();
 	if (bleft >= GCM_AVX_MIN_DECRYPT_BYTES) {
-		done = aesni_gcm_decrypt(datap, datap, bleft,
-		    (const void *)key, ctx->gcm_cb, ghash);
+		done = decrypt_blocks(datap, datap, bleft,
+		    (const void *)key, ctx->gcm_cb, htable, ghash);
 		if (done == 0) {
 			clear_fpu_regs();
 			kfpu_end();
@@ -1424,13 +1610,42 @@ gcm_init_avx(gcm_ctx_t *ctx, const uint8_t *iv, size_t iv_len,
 	ASSERT3S(((aes_key_t *)ctx->gcm_keysched)->ops->needs_byteswap, ==,
 	    B_FALSE);
 
+	size_t htab_len = 0;
+#if CAN_USE_GCM_ASM >= 2
+	if (ctx->impl == GCM_IMPL_AVX2) {
+		/*
+		 * BoringSSL's API specifies uint128_t[16] for htab; but only
+		 * uint128_t[12] are used.
+		 * See https://github.com/google/boringssl/blob/
+		 * 813840dd094f9e9c1b00a7368aa25e656554221f1/crypto/fipsmodule/
+		 * modes/asm/aes-gcm-avx2-x86_64.pl#L198-L200
+		 */
+		htab_len = (2 * 8 * sizeof (uint128_t));
+	} else
+#endif /* CAN_USE_GCM_ASM >= 2 */
+	{
+		htab_len = (2 * 6 * sizeof (uint128_t));
+	}
+
+	ctx->gcm_Htable = kmem_alloc(htab_len, KM_SLEEP);
+	if (ctx->gcm_Htable == NULL) {
+		return (CRYPTO_HOST_MEMORY);
+	}
+
 	/* Init H (encrypt zero block) and create the initial counter block. */
 	memset(H, 0, sizeof (ctx->gcm_H));
 	kfpu_begin();
 	aes_encrypt_intel(keysched, aes_rounds,
 	    (const uint32_t *)H, (uint32_t *)H);
 
-	gcm_init_htab_avx(ctx->gcm_Htable, H);
+#if CAN_USE_GCM_ASM >= 2
+	if (ctx->impl == GCM_IMPL_AVX2) {
+		gcm_init_vpclmulqdq_avx2((uint128_t *)ctx->gcm_Htable, H);
+	} else
+#endif /* if CAN_USE_GCM_ASM >= 2 */
+	{
+		gcm_init_htab_avx(ctx->gcm_Htable, H);
+	}
 
 	if (iv_len == 12) {
 		memcpy(cb, iv, 12);
diff --git a/sys/contrib/openzfs/module/icp/algs/modes/modes.c b/sys/contrib/openzfs/module/icp/algs/modes/modes.c
index 343591cd9691..ef3c1806e4b6 100644
--- a/sys/contrib/openzfs/module/icp/algs/modes/modes.c
+++ b/sys/contrib/openzfs/module/icp/algs/modes/modes.c
@@ -171,7 +171,7 @@ gcm_clear_ctx(gcm_ctx_t *ctx)
 	explicit_memset(ctx->gcm_remainder, 0, sizeof (ctx->gcm_remainder));
 	explicit_memset(ctx->gcm_H, 0, sizeof (ctx->gcm_H));
 #if defined(CAN_USE_GCM_ASM)
-	if (ctx->gcm_use_avx == B_TRUE) {
+	if (ctx->impl != GCM_IMPL_GENERIC) {
 		ASSERT3P(ctx->gcm_Htable, !=, NULL);
 		explicit_memset(ctx->gcm_Htable, 0, ctx->gcm_htab_len);
 		kmem_free(ctx->gcm_Htable, ctx->gcm_htab_len);
diff --git a/sys/contrib/openzfs/module/icp/algs/sha2/sha256_impl.c b/sys/contrib/openzfs/module/icp/algs/sha2/sha256_impl.c
index 6d3bcca9f995..dcb0a391dda4 100644
--- a/sys/contrib/openzfs/module/icp/algs/sha2/sha256_impl.c
+++ b/sys/contrib/openzfs/module/icp/algs/sha2/sha256_impl.c
@@ -38,11 +38,14 @@
 	kfpu_begin(); E(s, d, b); kfpu_end(); \
 }
 
+#if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \
+    defined(__PPC64__)
 /* some implementation is always okay */
 static inline boolean_t sha2_is_supported(void)
 {
 	return (B_TRUE);
 }
+#endif
 
 #if defined(__x86_64)
 
diff --git a/sys/contrib/openzfs/module/icp/algs/sha2/sha2_generic.c b/sys/contrib/openzfs/module/icp/algs/sha2/sha2_generic.c
index d0fcca798fa9..ad707341eec7 100644
--- a/sys/contrib/openzfs/module/icp/algs/sha2/sha2_generic.c
+++ b/sys/contrib/openzfs/module/icp/algs/sha2/sha2_generic.c
@@ -77,7 +77,8 @@ static const uint32_t SHA256_K[64] = {
 	h = g, g = f, f = e, e = d + T1; \
 	d = c, c = b, b = a, a = T1 + T2;
 
-static void sha256_generic(uint32_t state[8], const void *data, size_t num_blks)
+static void
+icp_sha256_generic(uint32_t state[8], const void *data, size_t num_blks)
 {
 	uint64_t blk;
 
@@ -173,7 +174,8 @@ static const uint64_t SHA512_K[80] = {
 	0x5fcb6fab3ad6faec, 0x6c44198c4a475817
 };
 
-static void sha512_generic(uint64_t state[8], const void *data, size_t num_blks)
+static void
+icp_sha512_generic(uint64_t state[8], const void *data, size_t num_blks)
 {
 	uint64_t blk;
 
@@ -226,7 +228,8 @@ static void sha512_generic(uint64_t state[8], const void *data, size_t num_blks)
 	}
 }
 
-static void sha256_update(sha256_ctx *ctx, const uint8_t *data, size_t len)
+static void
+icp_sha256_update(sha256_ctx *ctx, const uint8_t *data, size_t len)
 {
 	uint64_t pos = ctx->count[0];
 	uint64_t total = ctx->count[1];
@@ -258,7 +261,8 @@ static void sha256_update(sha256_ctx *ctx, const uint8_t *data, size_t len)
 	ctx->count[1] = total;
 }
 
-static void sha512_update(sha512_ctx *ctx, const uint8_t *data, size_t len)
+static void
+icp_sha512_update(sha512_ctx *ctx, const uint8_t *data, size_t len)
 {
 	uint64_t pos = ctx->count[0];
 	uint64_t total = ctx->count[1];
@@ -290,7 +294,8 @@ static void sha512_update(sha512_ctx *ctx, const uint8_t *data, size_t len)
 	ctx->count[1] = total;
 }
 
-static void sha256_final(sha256_ctx *ctx, uint8_t *result, int bits)
+static void
+icp_sha256_final(sha256_ctx *ctx, uint8_t *result, int bits)
 {
 	uint64_t mlen, pos = ctx->count[0];
 	uint8_t *m = ctx->wbuf;
@@ -334,7 +339,8 @@ static void sha256_final(sha256_ctx *ctx, uint8_t *result, int bits)
 	memset(ctx, 0, sizeof (*ctx));
 }
 
-static void sha512_final(sha512_ctx *ctx, uint8_t *result, int bits)
+static void
+icp_sha512_final(sha512_ctx *ctx, uint8_t *result, int bits)
 {
 	uint64_t mlen, pos = ctx->count[0];
 	uint8_t *m = ctx->wbuf, *r;
@@ -461,14 +467,14 @@ SHA2Update(SHA2_CTX *ctx, const void *data, size_t len)
 
 	switch (ctx->algotype) {
 		case SHA256:
-			sha256_update(&ctx->sha256, data, len);
+			icp_sha256_update(&ctx->sha256, data, len);
 			break;
 		case SHA512:
 		case SHA512_HMAC_MECH_INFO_TYPE:
-			sha512_update(&ctx->sha512, data, len);
+			icp_sha512_update(&ctx->sha512, data, len);
 			break;
 		case SHA512_256:
-			sha512_update(&ctx->sha512, data, len);
+			icp_sha512_update(&ctx->sha512, data, len);
 			break;
 	}
 }
@@ -479,32 +485,33 @@ SHA2Final(void *digest, SHA2_CTX *ctx)
 {
 	switch (ctx->algotype) {
 		case SHA256:
-			sha256_final(&ctx->sha256, digest, 256);
+			icp_sha256_final(&ctx->sha256, digest, 256);
 			break;
 		case SHA512:
 		case SHA512_HMAC_MECH_INFO_TYPE:
-			sha512_final(&ctx->sha512, digest, 512);
+			icp_sha512_final(&ctx->sha512, digest, 512);
 			break;
 		case SHA512_256:
-			sha512_final(&ctx->sha512, digest, 256);
+			icp_sha512_final(&ctx->sha512, digest, 256);
 			break;
 	}
 }
 
 /* the generic implementation is always okay */
-static boolean_t sha2_is_supported(void)
+static boolean_t
+icp_sha2_is_supported(void)
 {
 	return (B_TRUE);
 }
 
 const sha256_ops_t sha256_generic_impl = {
 	.name = "generic",
-	.transform = sha256_generic,
-	.is_supported = sha2_is_supported
+	.transform = icp_sha256_generic,
+	.is_supported = icp_sha2_is_supported
 };
 
 const sha512_ops_t sha512_generic_impl = {
 	.name = "generic",
-	.transform = sha512_generic,
-	.is_supported = sha2_is_supported
+	.transform = icp_sha512_generic,
+	.is_supported = icp_sha2_is_supported
 };
diff --git a/sys/contrib/openzfs/module/icp/algs/sha2/sha512_impl.c b/sys/contrib/openzfs/module/icp/algs/sha2/sha512_impl.c
index 2efd9fcf4c99..a85a71a83df4 100644
--- a/sys/contrib/openzfs/module/icp/algs/sha2/sha512_impl.c
+++ b/sys/contrib/openzfs/module/icp/algs/sha2/sha512_impl.c
@@ -38,11 +38,14 @@
 	kfpu_begin(); E(s, d, b); kfpu_end(); \
 }
 
+#if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \
+    defined(__aarch64__) || defined(__arm__) || defined(__PPC64__)
 /* some implementation is always okay */
 static inline boolean_t sha2_is_supported(void)
 {
 	return (B_TRUE);
 }
+#endif
 
 #if defined(__x86_64)
 
diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl
new file mode 100644
index 000000000000..04c03a37e0cb
--- /dev/null
+++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl
@@ -0,0 +1,253 @@
+BoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL
+licensing. Files that are completely new have a Google copyright and an ISC
+license. This license is reproduced at the bottom of this file.
+
+Contributors to BoringSSL are required to follow the CLA rules for Chromium:
+https://cla.developers.google.com/clas
+
+Files in third_party/ have their own licenses, as described therein. The MIT
+license, for third_party/fiat, which, unlike other third_party directories, is
+compiled into non-test libraries, is included below.
+
+The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the
+OpenSSL License and the original SSLeay license apply to the toolkit. See below
+for the actual license texts. Actually both licenses are BSD-style Open Source
+licenses. In case of any license issues related to OpenSSL please contact
+openssl-core@openssl.org.
+
+The following are Google-internal bug numbers where explicit permission from
+some authors is recorded for use of their work. (This is purely for our own
+record keeping.)
+  27287199
+  27287880
+  27287883
+  263291445
+
+
+  OpenSSL License
+  ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2011 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com).  This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ *
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to.  The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ *
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    "This product includes cryptographic software written by
+ *     Eric Young (eay@cryptsoft.com)"
+ *    The word 'cryptographic' can be left out if the rouines from the library
+ *    being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from
+ *    the apps directory (application code) you must include an acknowledgement:
+ *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed.  i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */
+
+
+ISC license used for completely new code in BoringSSL:
+
+/* Copyright 2015 The BoringSSL Authors
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+
+The code in third_party/fiat carries the MIT license:
+
+Copyright (c) 2015-2016 the fiat-crypto authors (see
+https://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS).
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+Licenses for support code
+-------------------------
+
+Parts of the TLS test suite are under the Go license. This code is not included
+in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so
+distributing code linked against BoringSSL does not trigger this license:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+BoringSSL uses the Chromium test infrastructure to run a continuous build,
+trybots etc. The scripts which manage this, and the script for generating build
+metadata, are under the Chromium license. Distributing code linked against
+BoringSSL does not trigger this license.
+
+Copyright 2015 The Chromium Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl.descrip b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl.descrip
new file mode 100644
index 000000000000..f63a67a4d2ae
--- /dev/null
+++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl.descrip
@@ -0,0 +1 @@
+PORTIONS OF AES GCM and GHASH FUNCTIONALITY
diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S
new file mode 100644
index 000000000000..3d1b045127e2
--- /dev/null
+++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S
@@ -0,0 +1,1323 @@
+// SPDX-License-Identifier: Apache-2.0
+// This file is generated from a similarly-named Perl script in the BoringSSL
+// source tree. Do not edit by hand.
+
+#if defined(__x86_64__) && defined(HAVE_AVX) && \
+    defined(HAVE_VAES) && defined(HAVE_VPCLMULQDQ)
+
+#define _ASM
+#include <sys/asm_linkage.h>
+
+/* Windows userland links with OpenSSL */
+#if !defined (_WIN32) || defined (_KERNEL)
+
+.section	.rodata
+.balign	16
+
+
+.Lbswap_mask:
+.quad	0x08090a0b0c0d0e0f, 0x0001020304050607
+
+
+
+
+
+
+
+
+.Lgfpoly:
+.quad	1, 0xc200000000000000
+
+
+.Lgfpoly_and_internal_carrybit:
+.quad	1, 0xc200000000000001
+
+.balign	32
+
+.Lctr_pattern:
+.quad	0, 0
+.quad	1, 0
+.Linc_2blocks:
+.quad	2, 0
+.quad	2, 0
+
+ENTRY_ALIGN(gcm_init_vpclmulqdq_avx2, 32)
+.cfi_startproc
+
+ENDBR
+
+
+
+
+
+	vmovdqu	(%rsi),%xmm3
+	// KCF/ICP stores H in network byte order with the hi qword first
+	// so we need to swap all bytes, not the 2 qwords.
+	vmovdqu	.Lbswap_mask(%rip),%xmm4
+	vpshufb	%xmm4,%xmm3,%xmm3
+
+
+
+
+
+	vpshufd	$0xd3,%xmm3,%xmm0
+	vpsrad	$31,%xmm0,%xmm0
+	vpaddq	%xmm3,%xmm3,%xmm3
+	vpand	.Lgfpoly_and_internal_carrybit(%rip),%xmm0,%xmm0
+	vpxor	%xmm0,%xmm3,%xmm3
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm6
+
+
+	vpclmulqdq	$0x00,%xmm3,%xmm3,%xmm0
+	vpclmulqdq	$0x11,%xmm3,%xmm3,%xmm5
+	vpclmulqdq	$0x01,%xmm0,%xmm6,%xmm1
+	vpshufd	$0x4e,%xmm0,%xmm0
+	vpxor	%xmm0,%xmm1,%xmm1
+	vpclmulqdq	$0x01,%xmm1,%xmm6,%xmm0
+	vpshufd	$0x4e,%xmm1,%xmm1
+	vpxor	%xmm1,%xmm5,%xmm5
+	vpxor	%xmm0,%xmm5,%xmm5
+
+
+
+	vinserti128	$1,%xmm3,%ymm5,%ymm3
+	vinserti128	$1,%xmm5,%ymm5,%ymm5
+
+
+	vpclmulqdq	$0x00,%ymm5,%ymm3,%ymm0
+	vpclmulqdq	$0x01,%ymm5,%ymm3,%ymm1
+	vpclmulqdq	$0x10,%ymm5,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm6,%ymm2
+	vpshufd	$0x4e,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x11,%ymm5,%ymm3,%ymm4
+	vpclmulqdq	$0x01,%ymm1,%ymm6,%ymm0
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm4,%ymm4
+	vpxor	%ymm0,%ymm4,%ymm4
+
+
+
+	vmovdqu	%ymm3,96(%rdi)
+	vmovdqu	%ymm4,64(%rdi)
+
+
+
+	vpunpcklqdq	%ymm3,%ymm4,%ymm0
+	vpunpckhqdq	%ymm3,%ymm4,%ymm1
+	vpxor	%ymm1,%ymm0,%ymm0
+	vmovdqu	%ymm0,128+32(%rdi)
+
+
+	vpclmulqdq	$0x00,%ymm5,%ymm4,%ymm0
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm1
+	vpclmulqdq	$0x10,%ymm5,%ymm4,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm6,%ymm2
+	vpshufd	$0x4e,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x11,%ymm5,%ymm4,%ymm3
+	vpclmulqdq	$0x01,%ymm1,%ymm6,%ymm0
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpxor	%ymm0,%ymm3,%ymm3
+
+	vpclmulqdq	$0x00,%ymm5,%ymm3,%ymm0
+	vpclmulqdq	$0x01,%ymm5,%ymm3,%ymm1
+	vpclmulqdq	$0x10,%ymm5,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm6,%ymm2
+	vpshufd	$0x4e,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpclmulqdq	$0x11,%ymm5,%ymm3,%ymm4
+	vpclmulqdq	$0x01,%ymm1,%ymm6,%ymm0
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm4,%ymm4
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vmovdqu	%ymm3,32(%rdi)
+	vmovdqu	%ymm4,0(%rdi)
+
+
+
+	vpunpcklqdq	%ymm3,%ymm4,%ymm0
+	vpunpckhqdq	%ymm3,%ymm4,%ymm1
+	vpxor	%ymm1,%ymm0,%ymm0
+	vmovdqu	%ymm0,128(%rdi)
+
+	vzeroupper
+	RET
+
+.cfi_endproc
+SET_SIZE(gcm_init_vpclmulqdq_avx2)
+ENTRY_ALIGN(gcm_gmult_vpclmulqdq_avx2, 32)
+.cfi_startproc
+
+ENDBR
+
+
+
+	vmovdqu	(%rdi),%xmm0
+	vmovdqu	.Lbswap_mask(%rip),%xmm1
+	vmovdqu	128-16(%rsi),%xmm2
+	vmovdqu	.Lgfpoly(%rip),%xmm3
+	vpshufb	%xmm1,%xmm0,%xmm0
+
+	vpclmulqdq	$0x00,%xmm2,%xmm0,%xmm4
+	vpclmulqdq	$0x01,%xmm2,%xmm0,%xmm5
+	vpclmulqdq	$0x10,%xmm2,%xmm0,%xmm6
+	vpxor	%xmm6,%xmm5,%xmm5
+	vpclmulqdq	$0x01,%xmm4,%xmm3,%xmm6
+	vpshufd	$0x4e,%xmm4,%xmm4
+	vpxor	%xmm4,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm5,%xmm5
+	vpclmulqdq	$0x11,%xmm2,%xmm0,%xmm0
+	vpclmulqdq	$0x01,%xmm5,%xmm3,%xmm4
+	vpshufd	$0x4e,%xmm5,%xmm5
+	vpxor	%xmm5,%xmm0,%xmm0
+	vpxor	%xmm4,%xmm0,%xmm0
+
+
+	vpshufb	%xmm1,%xmm0,%xmm0
+	vmovdqu	%xmm0,(%rdi)
+
+
+	RET
+
+.cfi_endproc
+SET_SIZE(gcm_gmult_vpclmulqdq_avx2)
+ENTRY_ALIGN(gcm_ghash_vpclmulqdq_avx2, 32)
+.cfi_startproc
+
+ENDBR
+
+
+
+
+
+
+	vmovdqu	.Lbswap_mask(%rip),%xmm6
+	vmovdqu	.Lgfpoly(%rip),%xmm7
+
+
+	vmovdqu	(%rdi),%xmm5
+	vpshufb	%xmm6,%xmm5,%xmm5
+
+
+	cmpq	$32,%rcx
+	jb	.Lghash_lastblock
+
+
+
+	vinserti128	$1,%xmm6,%ymm6,%ymm6
+	vinserti128	$1,%xmm7,%ymm7,%ymm7
+
+	cmpq	$127,%rcx
+	jbe	.Lghash_loop_1x
+
+
+	vmovdqu	128(%rsi),%ymm8
+	vmovdqu	128+32(%rsi),%ymm9
+.Lghash_loop_4x:
+
+	vmovdqu	0(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	0(%rsi),%ymm2
+	vpxor	%ymm5,%ymm1,%ymm1
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x00,%ymm8,%ymm0,%ymm4
+
+	vmovdqu	32(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	32(%rsi),%ymm2
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm3,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x10,%ymm8,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vmovdqu	64(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	64(%rsi),%ymm2
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm3,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x00,%ymm9,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm4,%ymm4
+
+
+	vmovdqu	96(%rdx),%ymm1
+	vpshufb	%ymm6,%ymm1,%ymm1
+	vmovdqu	96(%rsi),%ymm2
+	vpclmulqdq	$0x00,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm3,%ymm3
+	vpclmulqdq	$0x11,%ymm2,%ymm1,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vpunpckhqdq	%ymm1,%ymm1,%ymm0
+	vpxor	%ymm1,%ymm0,%ymm0
+	vpclmulqdq	$0x10,%ymm9,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vpxor	%ymm3,%ymm4,%ymm4
+	vpxor	%ymm5,%ymm4,%ymm4
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm2
+	vpclmulqdq	$0x01,%ymm3,%ymm2,%ymm0
+	vpshufd	$0x4e,%ymm3,%ymm3
+	vpxor	%ymm3,%ymm4,%ymm4
+	vpxor	%ymm0,%ymm4,%ymm4
+
+	vpclmulqdq	$0x01,%ymm4,%ymm2,%ymm0
+	vpshufd	$0x4e,%ymm4,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpxor	%ymm0,%ymm5,%ymm5
+	vextracti128	$1,%ymm5,%xmm0
+	vpxor	%xmm0,%xmm5,%xmm5
+
+	subq	$-128,%rdx
+	addq	$-128,%rcx
+	cmpq	$127,%rcx
+	ja	.Lghash_loop_4x
+
+
+	cmpq	$32,%rcx
+	jb	.Lghash_loop_1x_done
+.Lghash_loop_1x:
+	vmovdqu	(%rdx),%ymm0
+	vpshufb	%ymm6,%ymm0,%ymm0
+	vpxor	%ymm0,%ymm5,%ymm5
+	vmovdqu	128-32(%rsi),%ymm0
+	vpclmulqdq	$0x00,%ymm0,%ymm5,%ymm1
+	vpclmulqdq	$0x01,%ymm0,%ymm5,%ymm2
+	vpclmulqdq	$0x10,%ymm0,%ymm5,%ymm3
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x01,%ymm1,%ymm7,%ymm3
+	vpshufd	$0x4e,%ymm1,%ymm1
+	vpxor	%ymm1,%ymm2,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x11,%ymm0,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm7,%ymm1
+	vpshufd	$0x4e,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpxor	%ymm1,%ymm5,%ymm5
+
+	vextracti128	$1,%ymm5,%xmm0
+	vpxor	%xmm0,%xmm5,%xmm5
+	addq	$32,%rdx
+	subq	$32,%rcx
+	cmpq	$32,%rcx
+	jae	.Lghash_loop_1x
+.Lghash_loop_1x_done:
+
+
+.Lghash_lastblock:
+	testq	%rcx,%rcx
+	jz	.Lghash_done
+	vmovdqu	(%rdx),%xmm0
+	vpshufb	%xmm6,%xmm0,%xmm0
+	vpxor	%xmm0,%xmm5,%xmm5
+	vmovdqu	128-16(%rsi),%xmm0
+	vpclmulqdq	$0x00,%xmm0,%xmm5,%xmm1
+	vpclmulqdq	$0x01,%xmm0,%xmm5,%xmm2
+	vpclmulqdq	$0x10,%xmm0,%xmm5,%xmm3
+	vpxor	%xmm3,%xmm2,%xmm2
+	vpclmulqdq	$0x01,%xmm1,%xmm7,%xmm3
+	vpshufd	$0x4e,%xmm1,%xmm1
+	vpxor	%xmm1,%xmm2,%xmm2
+	vpxor	%xmm3,%xmm2,%xmm2
+	vpclmulqdq	$0x11,%xmm0,%xmm5,%xmm5
+	vpclmulqdq	$0x01,%xmm2,%xmm7,%xmm1
+	vpshufd	$0x4e,%xmm2,%xmm2
+	vpxor	%xmm2,%xmm5,%xmm5
+	vpxor	%xmm1,%xmm5,%xmm5
+
+
+.Lghash_done:
+
+	vpshufb	%xmm6,%xmm5,%xmm5
+	vmovdqu	%xmm5,(%rdi)
+
+	vzeroupper
+	RET
+
+.cfi_endproc
+SET_SIZE(gcm_ghash_vpclmulqdq_avx2)
+ENTRY_ALIGN(aes_gcm_enc_update_vaes_avx2, 32)
+.cfi_startproc
+
+ENDBR
+	pushq	%r12
+.cfi_adjust_cfa_offset	8
+.cfi_offset	%r12,-16
+
+	movq	16(%rsp),%r12
+#ifdef BORINGSSL_DISPATCH_TEST
+.extern	BORINGSSL_function_hit
+.hidden BORINGSSL_function_hit
+	movb	$1,BORINGSSL_function_hit+6(%rip)
+#endif
+	vbroadcasti128	.Lbswap_mask(%rip),%ymm0
+
+
+
+	vmovdqu	(%r12),%xmm1
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vbroadcasti128	(%r8),%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm11
+
+
+
+	movl	504(%rcx),%r10d		// ICP has a larger offset for rounds.
+	leal	-24(,%r10,4),%r10d	// ICP uses 10,12,14 not 9,11,13 for rounds.
+
+
+
+
+	leaq	96(%rcx,%r10,4),%r11
+	vbroadcasti128	(%rcx),%ymm9
+	vbroadcasti128	(%r11),%ymm10
+
+
+	vpaddd	.Lctr_pattern(%rip),%ymm11,%ymm11
+
+
+
+	cmpq	$127,%rdx
+	jbe	.Lcrypt_loop_4x_done__func1
+
+	vmovdqu	128(%r9),%ymm7
+	vmovdqu	128+32(%r9),%ymm8
+
+
+
+	vmovdqu	.Linc_2blocks(%rip),%ymm2
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm14
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm15
+	vpaddd	%ymm2,%ymm11,%ymm11
+
+
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	vpxor	%ymm9,%ymm14,%ymm14
+	vpxor	%ymm9,%ymm15,%ymm15
+
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_first_4_vecs__func1:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_first_4_vecs__func1
+	vpxor	0(%rdi),%ymm10,%ymm2
+	vpxor	32(%rdi),%ymm10,%ymm3
+	vpxor	64(%rdi),%ymm10,%ymm5
+	vpxor	96(%rdi),%ymm10,%ymm6
+	vaesenclast	%ymm2,%ymm12,%ymm12
+	vaesenclast	%ymm3,%ymm13,%ymm13
+	vaesenclast	%ymm5,%ymm14,%ymm14
+	vaesenclast	%ymm6,%ymm15,%ymm15
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+	vmovdqu	%ymm14,64(%rsi)
+	vmovdqu	%ymm15,96(%rsi)
+
+	subq	$-128,%rdi
+	addq	$-128,%rdx
+	cmpq	$127,%rdx
+	jbe	.Lghash_last_ciphertext_4x__func1
+.balign	16
+.Lcrypt_loop_4x__func1:
+
+
+
+
+	vmovdqu	.Linc_2blocks(%rip),%ymm2
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm14
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm15
+	vpaddd	%ymm2,%ymm11,%ymm11
+
+
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	vpxor	%ymm9,%ymm14,%ymm14
+	vpxor	%ymm9,%ymm15,%ymm15
+
+	cmpl	$24,%r10d
+	jl	.Laes128__func1
+	je	.Laes192__func1
+
+	vbroadcasti128	-208(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-192(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes192__func1:
+	vbroadcasti128	-176(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-160(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes128__func1:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
+
+	vmovdqu	0(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	0(%r9),%ymm4
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm7,%ymm2,%ymm6
+
+	vbroadcasti128	-144(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vbroadcasti128	-128(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	32(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	32(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm7,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-112(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	64(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	64(%r9),%ymm4
+
+	vbroadcasti128	-96(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-80(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+
+	vmovdqu	96(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+
+	vbroadcasti128	-64(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vmovdqu	96(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-48(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm1,%ymm6,%ymm6
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm4
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-32(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpclmulqdq	$0x01,%ymm6,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-16(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vextracti128	$1,%ymm1,%xmm2
+	vpxor	%xmm2,%xmm1,%xmm1
+
+
+	subq	$-128,%rsi
+	vpxor	0(%rdi),%ymm10,%ymm2
+	vpxor	32(%rdi),%ymm10,%ymm3
+	vpxor	64(%rdi),%ymm10,%ymm5
+	vpxor	96(%rdi),%ymm10,%ymm6
+	vaesenclast	%ymm2,%ymm12,%ymm12
+	vaesenclast	%ymm3,%ymm13,%ymm13
+	vaesenclast	%ymm5,%ymm14,%ymm14
+	vaesenclast	%ymm6,%ymm15,%ymm15
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+	vmovdqu	%ymm14,64(%rsi)
+	vmovdqu	%ymm15,96(%rsi)
+
+	subq	$-128,%rdi
+
+	addq	$-128,%rdx
+	cmpq	$127,%rdx
+	ja	.Lcrypt_loop_4x__func1
+.Lghash_last_ciphertext_4x__func1:
+
+	vmovdqu	0(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	0(%r9),%ymm4
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm7,%ymm2,%ymm6
+
+	vmovdqu	32(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	32(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm7,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vmovdqu	64(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	64(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+
+	vmovdqu	96(%rsi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	96(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm1,%ymm6,%ymm6
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm4
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vpclmulqdq	$0x01,%ymm6,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+	vextracti128	$1,%ymm1,%xmm2
+	vpxor	%xmm2,%xmm1,%xmm1
+
+	subq	$-128,%rsi
+.Lcrypt_loop_4x_done__func1:
+
+	testq	%rdx,%rdx
+	jz	.Ldone__func1
+
+
+
+
+
+	leaq	128(%r9),%r8
+	subq	%rdx,%r8
+
+
+	vpxor	%xmm5,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm6,%xmm6
+	vpxor	%xmm7,%xmm7,%xmm7
+
+	cmpq	$64,%rdx
+	jb	.Llessthan64bytes__func1
+
+
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_1__func1:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_1__func1
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%ymm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%ymm3,%ymm13,%ymm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+
+
+	vpshufb	%ymm0,%ymm12,%ymm12
+	vpshufb	%ymm0,%ymm13,%ymm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%ymm3
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm7
+	vpclmulqdq	$0x00,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+	addq	$64,%r8
+	addq	$64,%rdi
+	addq	$64,%rsi
+	subq	$64,%rdx
+	jz	.Lreduce__func1
+
+	vpxor	%xmm1,%xmm1,%xmm1
+
+
+.Llessthan64bytes__func1:
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_2__func1:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_2__func1
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+
+
+	cmpq	$32,%rdx
+	jb	.Lxor_one_block__func1
+	je	.Lxor_two_blocks__func1
+
+.Lxor_three_blocks__func1:
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%xmm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%xmm3,%xmm13,%xmm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%xmm13,32(%rsi)
+
+	vpshufb	%ymm0,%ymm12,%ymm12
+	vpshufb	%xmm0,%xmm13,%xmm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%xmm3
+	vpclmulqdq	$0x00,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm7,%ymm7
+	jmp	.Lghash_mul_one_vec_unreduced__func1
+
+.Lxor_two_blocks__func1:
+	vmovdqu	(%rdi),%ymm2
+	vpxor	%ymm2,%ymm12,%ymm12
+	vmovdqu	%ymm12,(%rsi)
+	vpshufb	%ymm0,%ymm12,%ymm12
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	jmp	.Lghash_mul_one_vec_unreduced__func1
+
+.Lxor_one_block__func1:
+	vmovdqu	(%rdi),%xmm2
+	vpxor	%xmm2,%xmm12,%xmm12
+	vmovdqu	%xmm12,(%rsi)
+	vpshufb	%xmm0,%xmm12,%xmm12
+	vpxor	%xmm1,%xmm12,%xmm12
+	vmovdqu	(%r8),%xmm2
+
+.Lghash_mul_one_vec_unreduced__func1:
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+.Lreduce__func1:
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm2
+	vpclmulqdq	$0x01,%ymm5,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm3,%ymm6,%ymm6
+	vpclmulqdq	$0x01,%ymm6,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm7,%ymm7
+	vpxor	%ymm3,%ymm7,%ymm7
+	vextracti128	$1,%ymm7,%xmm1
+	vpxor	%xmm7,%xmm1,%xmm1
+
+.Ldone__func1:
+
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vmovdqu	%xmm1,(%r12)
+
+	vzeroupper
+	popq	%r12
+.cfi_adjust_cfa_offset	-8
+.cfi_restore	%r12
+	RET
+
+.cfi_endproc
+SET_SIZE(aes_gcm_enc_update_vaes_avx2)
+ENTRY_ALIGN(aes_gcm_dec_update_vaes_avx2, 32)
+.cfi_startproc
+
+ENDBR
+	pushq	%r12
+.cfi_adjust_cfa_offset	8
+.cfi_offset	%r12,-16
+
+	movq	16(%rsp),%r12
+	vbroadcasti128	.Lbswap_mask(%rip),%ymm0
+
+
+
+	vmovdqu	(%r12),%xmm1
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vbroadcasti128	(%r8),%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm11
+
+
+
+	movl	504(%rcx),%r10d		// ICP has a larger offset for rounds.
+	leal	-24(,%r10,4),%r10d	// ICP uses 10,12,14 not 9,11,13 for rounds.
+
+
+
+
+	leaq	96(%rcx,%r10,4),%r11
+	vbroadcasti128	(%rcx),%ymm9
+	vbroadcasti128	(%r11),%ymm10
+
+
+	vpaddd	.Lctr_pattern(%rip),%ymm11,%ymm11
+
+
+
+	cmpq	$127,%rdx
+	jbe	.Lcrypt_loop_4x_done__func2
+
+	vmovdqu	128(%r9),%ymm7
+	vmovdqu	128+32(%r9),%ymm8
+.balign	16
+.Lcrypt_loop_4x__func2:
+
+
+
+
+	vmovdqu	.Linc_2blocks(%rip),%ymm2
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm14
+	vpaddd	%ymm2,%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm15
+	vpaddd	%ymm2,%ymm11,%ymm11
+
+
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	vpxor	%ymm9,%ymm14,%ymm14
+	vpxor	%ymm9,%ymm15,%ymm15
+
+	cmpl	$24,%r10d
+	jl	.Laes128__func2
+	je	.Laes192__func2
+
+	vbroadcasti128	-208(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-192(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes192__func2:
+	vbroadcasti128	-176(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vbroadcasti128	-160(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+.Laes128__func2:
+	prefetcht0	512(%rdi)
+	prefetcht0	512+64(%rdi)
+
+	vmovdqu	0(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	0(%r9),%ymm4
+	vpxor	%ymm1,%ymm3,%ymm3
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm7,%ymm2,%ymm6
+
+	vbroadcasti128	-144(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vbroadcasti128	-128(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	32(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	32(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm7,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-112(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vmovdqu	64(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+	vmovdqu	64(%r9),%ymm4
+
+	vbroadcasti128	-96(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-80(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x00,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+
+	vmovdqu	96(%rdi),%ymm3
+	vpshufb	%ymm0,%ymm3,%ymm3
+
+	vbroadcasti128	-64(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vmovdqu	96(%r9),%ymm4
+	vpclmulqdq	$0x00,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm5,%ymm5
+	vpclmulqdq	$0x11,%ymm4,%ymm3,%ymm2
+	vpxor	%ymm2,%ymm1,%ymm1
+	vpunpckhqdq	%ymm3,%ymm3,%ymm2
+	vpxor	%ymm3,%ymm2,%ymm2
+	vpclmulqdq	$0x10,%ymm8,%ymm2,%ymm2
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-48(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm1,%ymm6,%ymm6
+
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm4
+	vpclmulqdq	$0x01,%ymm5,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm2,%ymm6,%ymm6
+
+	vbroadcasti128	-32(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+
+	vpclmulqdq	$0x01,%ymm6,%ymm4,%ymm2
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm1,%ymm1
+	vpxor	%ymm2,%ymm1,%ymm1
+
+	vbroadcasti128	-16(%r11),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	vaesenc	%ymm2,%ymm14,%ymm14
+	vaesenc	%ymm2,%ymm15,%ymm15
+
+	vextracti128	$1,%ymm1,%xmm2
+	vpxor	%xmm2,%xmm1,%xmm1
+
+
+
+	vpxor	0(%rdi),%ymm10,%ymm2
+	vpxor	32(%rdi),%ymm10,%ymm3
+	vpxor	64(%rdi),%ymm10,%ymm5
+	vpxor	96(%rdi),%ymm10,%ymm6
+	vaesenclast	%ymm2,%ymm12,%ymm12
+	vaesenclast	%ymm3,%ymm13,%ymm13
+	vaesenclast	%ymm5,%ymm14,%ymm14
+	vaesenclast	%ymm6,%ymm15,%ymm15
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+	vmovdqu	%ymm14,64(%rsi)
+	vmovdqu	%ymm15,96(%rsi)
+
+	subq	$-128,%rdi
+	subq	$-128,%rsi
+	addq	$-128,%rdx
+	cmpq	$127,%rdx
+	ja	.Lcrypt_loop_4x__func2
+.Lcrypt_loop_4x_done__func2:
+
+	testq	%rdx,%rdx
+	jz	.Ldone__func2
+
+
+
+
+
+	leaq	128(%r9),%r8
+	subq	%rdx,%r8
+
+
+	vpxor	%xmm5,%xmm5,%xmm5
+	vpxor	%xmm6,%xmm6,%xmm6
+	vpxor	%xmm7,%xmm7,%xmm7
+
+	cmpq	$64,%rdx
+	jb	.Llessthan64bytes__func2
+
+
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_1__func2:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_1__func2
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%ymm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%ymm3,%ymm13,%ymm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%ymm13,32(%rsi)
+
+
+	vpshufb	%ymm0,%ymm2,%ymm12
+	vpshufb	%ymm0,%ymm3,%ymm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%ymm3
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm7
+	vpclmulqdq	$0x00,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm3,%ymm13,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+	addq	$64,%r8
+	addq	$64,%rdi
+	addq	$64,%rsi
+	subq	$64,%rdx
+	jz	.Lreduce__func2
+
+	vpxor	%xmm1,%xmm1,%xmm1
+
+
+.Llessthan64bytes__func2:
+	vpshufb	%ymm0,%ymm11,%ymm12
+	vpaddd	.Linc_2blocks(%rip),%ymm11,%ymm11
+	vpshufb	%ymm0,%ymm11,%ymm13
+	vpxor	%ymm9,%ymm12,%ymm12
+	vpxor	%ymm9,%ymm13,%ymm13
+	leaq	16(%rcx),%rax
+.Lvaesenc_loop_tail_2__func2:
+	vbroadcasti128	(%rax),%ymm2
+	vaesenc	%ymm2,%ymm12,%ymm12
+	vaesenc	%ymm2,%ymm13,%ymm13
+	addq	$16,%rax
+	cmpq	%rax,%r11
+	jne	.Lvaesenc_loop_tail_2__func2
+	vaesenclast	%ymm10,%ymm12,%ymm12
+	vaesenclast	%ymm10,%ymm13,%ymm13
+
+
+
+
+	cmpq	$32,%rdx
+	jb	.Lxor_one_block__func2
+	je	.Lxor_two_blocks__func2
+
+.Lxor_three_blocks__func2:
+	vmovdqu	0(%rdi),%ymm2
+	vmovdqu	32(%rdi),%xmm3
+	vpxor	%ymm2,%ymm12,%ymm12
+	vpxor	%xmm3,%xmm13,%xmm13
+	vmovdqu	%ymm12,0(%rsi)
+	vmovdqu	%xmm13,32(%rsi)
+
+	vpshufb	%ymm0,%ymm2,%ymm12
+	vpshufb	%xmm0,%xmm3,%xmm13
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	vmovdqu	32(%r8),%xmm3
+	vpclmulqdq	$0x00,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%xmm3,%xmm13,%xmm4
+	vpxor	%ymm4,%ymm7,%ymm7
+	jmp	.Lghash_mul_one_vec_unreduced__func2
+
+.Lxor_two_blocks__func2:
+	vmovdqu	(%rdi),%ymm2
+	vpxor	%ymm2,%ymm12,%ymm12
+	vmovdqu	%ymm12,(%rsi)
+	vpshufb	%ymm0,%ymm2,%ymm12
+	vpxor	%ymm1,%ymm12,%ymm12
+	vmovdqu	(%r8),%ymm2
+	jmp	.Lghash_mul_one_vec_unreduced__func2
+
+.Lxor_one_block__func2:
+	vmovdqu	(%rdi),%xmm2
+	vpxor	%xmm2,%xmm12,%xmm12
+	vmovdqu	%xmm12,(%rsi)
+	vpshufb	%xmm0,%xmm2,%xmm12
+	vpxor	%xmm1,%xmm12,%xmm12
+	vmovdqu	(%r8),%xmm2
+
+.Lghash_mul_one_vec_unreduced__func2:
+	vpclmulqdq	$0x00,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm5,%ymm5
+	vpclmulqdq	$0x01,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x10,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm6,%ymm6
+	vpclmulqdq	$0x11,%ymm2,%ymm12,%ymm4
+	vpxor	%ymm4,%ymm7,%ymm7
+
+.Lreduce__func2:
+
+	vbroadcasti128	.Lgfpoly(%rip),%ymm2
+	vpclmulqdq	$0x01,%ymm5,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm5,%ymm5
+	vpxor	%ymm5,%ymm6,%ymm6
+	vpxor	%ymm3,%ymm6,%ymm6
+	vpclmulqdq	$0x01,%ymm6,%ymm2,%ymm3
+	vpshufd	$0x4e,%ymm6,%ymm6
+	vpxor	%ymm6,%ymm7,%ymm7
+	vpxor	%ymm3,%ymm7,%ymm7
+	vextracti128	$1,%ymm7,%xmm1
+	vpxor	%xmm7,%xmm1,%xmm1
+
+.Ldone__func2:
+
+	vpshufb	%xmm0,%xmm1,%xmm1
+	vmovdqu	%xmm1,(%r12)
+
+	vzeroupper
+	popq	%r12
+.cfi_adjust_cfa_offset	-8
+.cfi_restore	%r12
+	RET
+
+.cfi_endproc
+SET_SIZE(aes_gcm_dec_update_vaes_avx2)
+
+#endif /* !_WIN32 || _KERNEL */
+
+/* Mark the stack non-executable. */
+#if defined(__linux__) && defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+#endif /* defined(__x86_64__) && defined(HAVE_AVX) && defined(HAVE_AES) ... */
diff --git a/sys/contrib/openzfs/module/icp/core/kcf_sched.c b/sys/contrib/openzfs/module/icp/core/kcf_sched.c
index 759f0d81d521..75e1052a4ed4 100644
--- a/sys/contrib/openzfs/module/icp/core/kcf_sched.c
+++ b/sys/contrib/openzfs/module/icp/core/kcf_sched.c
@@ -124,7 +124,7 @@ kcf_context_cache_destructor(void *buf, void *cdrarg)
 	(void) cdrarg;
 	kcf_context_t *kctx = (kcf_context_t *)buf;
 
-	ASSERT(kctx->kc_refcnt == 0);
+	ASSERT0(kctx->kc_refcnt);
 }
 
 void
diff --git a/sys/contrib/openzfs/module/icp/include/modes/modes.h b/sys/contrib/openzfs/module/icp/include/modes/modes.h
index ca734cf4f045..de11d9eafafb 100644
--- a/sys/contrib/openzfs/module/icp/include/modes/modes.h
+++ b/sys/contrib/openzfs/module/icp/include/modes/modes.h
@@ -42,7 +42,7 @@ extern "C" {
  */
 #if defined(__x86_64__) && defined(HAVE_AVX) && \
     defined(HAVE_AES) && defined(HAVE_PCLMULQDQ)
-#define	CAN_USE_GCM_ASM
+#define	CAN_USE_GCM_ASM (HAVE_VAES && HAVE_VPCLMULQDQ ? 2 : 1)
 extern boolean_t gcm_avx_can_use_movbe;
 #endif
 
@@ -129,6 +129,15 @@ typedef struct ccm_ctx {
 #define	ccm_copy_to		ccm_common.cc_copy_to
 #define	ccm_flags		ccm_common.cc_flags
 
+#ifdef CAN_USE_GCM_ASM
+typedef enum gcm_impl {
+	GCM_IMPL_GENERIC = 0,
+	GCM_IMPL_AVX,
+	GCM_IMPL_AVX2,
+	GCM_IMPL_MAX,
+} gcm_impl;
+#endif
+
 /*
  * gcm_tag_len:		Length of authentication tag.
  *
@@ -174,7 +183,7 @@ typedef struct gcm_ctx {
 	uint64_t gcm_len_a_len_c[2];
 	uint8_t *gcm_pt_buf;
 #ifdef CAN_USE_GCM_ASM
-	boolean_t gcm_use_avx;
+	enum gcm_impl impl;
 #endif
 } gcm_ctx_t;
 
diff --git a/sys/contrib/openzfs/module/icp/io/aes.c b/sys/contrib/openzfs/module/icp/io/aes.c
index ba703efa71fc..ca586eaf97ef 100644
--- a/sys/contrib/openzfs/module/icp/io/aes.c
+++ b/sys/contrib/openzfs/module/icp/io/aes.c
@@ -236,16 +236,16 @@ aes_encrypt_atomic(crypto_mechanism_t *mechanism,
 			    aes_xor_block);
 			if (ret != CRYPTO_SUCCESS)
 				goto out;
-			ASSERT(aes_ctx.ac_remainder_len == 0);
+			ASSERT0(aes_ctx.ac_remainder_len);
 		} else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE) {
 			ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
 			    ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
 			    aes_copy_block, aes_xor_block);
 			if (ret != CRYPTO_SUCCESS)
 				goto out;
-			ASSERT(aes_ctx.ac_remainder_len == 0);
+			ASSERT0(aes_ctx.ac_remainder_len);
 		} else {
-			ASSERT(aes_ctx.ac_remainder_len == 0);
+			ASSERT0(aes_ctx.ac_remainder_len);
 		}
 
 		if (plaintext != ciphertext) {
@@ -337,7 +337,7 @@ aes_decrypt_atomic(crypto_mechanism_t *mechanism,
 			ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
 			    plaintext, AES_BLOCK_LEN, aes_encrypt_block,
 			    aes_copy_block, aes_xor_block);
-			ASSERT(aes_ctx.ac_remainder_len == 0);
+			ASSERT0(aes_ctx.ac_remainder_len);
 			if ((ret == CRYPTO_SUCCESS) &&
 			    (ciphertext != plaintext)) {
 				plaintext->cd_length =
@@ -349,7 +349,7 @@ aes_decrypt_atomic(crypto_mechanism_t *mechanism,
 			ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
 			    plaintext, AES_BLOCK_LEN, aes_encrypt_block,
 			    aes_xor_block);
-			ASSERT(aes_ctx.ac_remainder_len == 0);
+			ASSERT0(aes_ctx.ac_remainder_len);
 			if ((ret == CRYPTO_SUCCESS) &&
 			    (ciphertext != plaintext)) {
 				plaintext->cd_length =
diff --git a/sys/contrib/openzfs/module/nvpair/nvpair.c b/sys/contrib/openzfs/module/nvpair/nvpair.c
index 811cfc87d7a4..eb8c14b4a783 100644
--- a/sys/contrib/openzfs/module/nvpair/nvpair.c
+++ b/sys/contrib/openzfs/module/nvpair/nvpair.c
@@ -265,7 +265,7 @@ nv_priv_alloc_embedded(nvpriv_t *priv)
 static int
 nvt_tab_alloc(nvpriv_t *priv, uint64_t buckets)
 {
-	ASSERT3P(priv->nvp_hashtable, ==, NULL);
+	ASSERT0P(priv->nvp_hashtable);
 	ASSERT0(priv->nvp_nbuckets);
 	ASSERT0(priv->nvp_nentries);
 
@@ -334,7 +334,7 @@ nvt_lookup_name_type(const nvlist_t *nvl, const char *name, data_type_t type)
 	i_nvp_t **tab = priv->nvp_hashtable;
 
 	if (tab == NULL) {
-		ASSERT3P(priv->nvp_list, ==, NULL);
+		ASSERT0P(priv->nvp_list);
 		ASSERT0(priv->nvp_nbuckets);
 		ASSERT0(priv->nvp_nentries);
 		return (NULL);
@@ -540,7 +540,7 @@ nvt_add_nvpair(nvlist_t *nvl, nvpair_t *nvp)
 
 	/* insert link at the beginning of the bucket */
 	i_nvp_t *new_entry = NVPAIR2I_NVP(nvp);
-	ASSERT3P(new_entry->nvi_hashtable_next, ==, NULL);
+	ASSERT0P(new_entry->nvi_hashtable_next);
 	new_entry->nvi_hashtable_next = bucket;
 	// cppcheck-suppress nullPointerRedundantCheck
 	tab[index] = new_entry;
diff --git a/sys/contrib/openzfs/module/os/freebsd/spl/spl_kmem.c b/sys/contrib/openzfs/module/os/freebsd/spl/spl_kmem.c
index 6d198fad5203..ae6e36d988c2 100644
--- a/sys/contrib/openzfs/module/os/freebsd/spl/spl_kmem.c
+++ b/sys/contrib/openzfs/module/os/freebsd/spl/spl_kmem.c
@@ -160,7 +160,7 @@ kmem_cache_create(const char *name, size_t bufsize, size_t align,
 {
 	kmem_cache_t *cache;
 
-	ASSERT3P(vmp, ==, NULL);
+	ASSERT0P(vmp);
 
 	cache = kmem_alloc(sizeof (*cache), KM_SLEEP);
 	strlcpy(cache->kc_name, name, sizeof (cache->kc_name));
diff --git a/sys/contrib/openzfs/module/os/freebsd/spl/spl_sysevent.c b/sys/contrib/openzfs/module/os/freebsd/spl/spl_sysevent.c
index 9da633c2b1be..3c2d39b20c09 100644
--- a/sys/contrib/openzfs/module/os/freebsd/spl/spl_sysevent.c
+++ b/sys/contrib/openzfs/module/os/freebsd/spl/spl_sysevent.c
@@ -256,7 +256,7 @@ sysevent_worker(void *arg __unused)
 	 * free `ze`, so just inline the free() here -- events have already
 	 * been drained.
 	 */
-	VERIFY3P(ze->ze_zevent, ==, NULL);
+	VERIFY0P(ze->ze_zevent);
 	kmem_free(ze, sizeof (zfs_zevent_t));
 
 	kthread_exit();
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c
index fbf67f6a14a8..4bf487cdc469 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c
@@ -507,7 +507,7 @@ abd_iter_at_end(struct abd_iter *aiter)
 void
 abd_iter_advance(struct abd_iter *aiter, size_t amount)
 {
-	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
+	ASSERT0P(aiter->iter_mapaddr);
 	ASSERT0(aiter->iter_mapsize);
 
 	/* There's nothing left to advance to, so do nothing */
@@ -526,7 +526,7 @@ abd_iter_map(struct abd_iter *aiter)
 {
 	void *paddr;
 
-	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
+	ASSERT0P(aiter->iter_mapaddr);
 	ASSERT0(aiter->iter_mapsize);
 
 	/* There's nothing left to iterate over, so do nothing */
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/dmu_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/dmu_os.c
index 364bbfc60abd..26cc7981bfcd 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/dmu_os.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/dmu_os.c
@@ -156,7 +156,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count,
 	if (dbp[0]->db_offset != 0 || numbufs > 1) {
 		for (i = 0; i < numbufs; i++) {
 			ASSERT(ISP2(dbp[i]->db_size));
-			ASSERT3U((dbp[i]->db_offset % dbp[i]->db_size), ==, 0);
+			ASSERT0((dbp[i]->db_offset % dbp[i]->db_size));
 			ASSERT3U(dbp[i]->db_size, ==, dbp[0]->db_size);
 		}
 	}
@@ -175,7 +175,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count,
 			vm_page_sunbusy(m);
 			break;
 		}
-		ASSERT3U(m->dirty, ==, 0);
+		ASSERT0(m->dirty);
 		ASSERT(!pmap_page_is_write_mapped(m));
 
 		ASSERT3U(db->db_size, >, PAGE_SIZE);
@@ -201,7 +201,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count,
 			if (m != bogus_page) {
 				vm_page_assert_xbusied(m);
 				ASSERT(vm_page_none_valid(m));
-				ASSERT3U(m->dirty, ==, 0);
+				ASSERT0(m->dirty);
 				ASSERT(!pmap_page_is_write_mapped(m));
 				va = zfs_map_page(m, &sf);
 			}
@@ -295,7 +295,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count,
 			vm_page_sunbusy(m);
 			break;
 		}
-		ASSERT3U(m->dirty, ==, 0);
+		ASSERT0(m->dirty);
 		ASSERT(!pmap_page_is_write_mapped(m));
 
 		ASSERT3U(db->db_size, >, PAGE_SIZE);
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c b/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c
index c114db14a916..b218c0da8125 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c
@@ -112,7 +112,6 @@ static int zfs__fini(void);
 static void zfs_shutdown(void *, int);
 
 static eventhandler_tag zfs_shutdown_event_tag;
-static eventhandler_tag zfs_mountroot_event_tag;
 
 #define	ZFS_MIN_KSTACK_PAGES 4
 
@@ -311,9 +310,6 @@ zfs_modevent(module_t mod, int type, void *unused __unused)
 			zfs_shutdown_event_tag = EVENTHANDLER_REGISTER(
 			    shutdown_post_sync, zfs_shutdown, NULL,
 			    SHUTDOWN_PRI_FIRST);
-			zfs_mountroot_event_tag = EVENTHANDLER_REGISTER(
-			    mountroot, spa_boot_init, NULL,
-			    SI_ORDER_ANY);
 		}
 		return (err);
 	case MOD_UNLOAD:
@@ -322,9 +318,6 @@ zfs_modevent(module_t mod, int type, void *unused __unused)
 			if (zfs_shutdown_event_tag != NULL)
 				EVENTHANDLER_DEREGISTER(shutdown_post_sync,
 				    zfs_shutdown_event_tag);
-			if (zfs_mountroot_event_tag != NULL)
-				EVENTHANDLER_DEREGISTER(mountroot,
-				    zfs_mountroot_event_tag);
 		}
 		return (err);
 	case MOD_SHUTDOWN:
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c
index ace2360c032d..ebc2c0eeb6d2 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c
@@ -163,6 +163,13 @@ param_set_arc_int(SYSCTL_HANDLER_ARGS)
 	return (0);
 }
 
+static void
+warn_deprecated_sysctl(const char *old, const char *new)
+{
+	printf("WARNING: sysctl vfs.zfs.%s is deprecated. Use vfs.zfs.%s instead.\n",
+	    old, new);
+}
+
 int
 param_set_arc_max(SYSCTL_HANDLER_ARGS)
 {
@@ -185,12 +192,15 @@ param_set_arc_max(SYSCTL_HANDLER_ARGS)
 	if (val != 0)
 		zfs_arc_max = arc_c_max;
 
+	if (arg2 != 0)
+		warn_deprecated_sysctl("arc_max", "arc.max");
+
 	return (0);
 }
 
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_max,
 	CTLTYPE_ULONG | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
-	NULL, 0, param_set_arc_max, "LU",
+	NULL, 1, param_set_arc_max, "LU",
 	"Maximum ARC size in bytes (LEGACY)");
 
 int
@@ -214,12 +224,15 @@ param_set_arc_min(SYSCTL_HANDLER_ARGS)
 	if (val != 0)
 		zfs_arc_min = arc_c_min;
 
+	if (arg2 != 0)
+		warn_deprecated_sysctl("arc_min", "arc.min");
+
 	return (0);
 }
 
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_min,
 	CTLTYPE_ULONG | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
-	NULL, 0, param_set_arc_min, "LU",
+	NULL, 1, param_set_arc_min, "LU",
 	"Minimum ARC size in bytes (LEGACY)");
 
 extern uint_t zfs_arc_free_target;
@@ -242,6 +255,9 @@ param_set_arc_free_target(SYSCTL_HANDLER_ARGS)
 
 	zfs_arc_free_target = val;
 
+	if (arg2 != 0)
+		warn_deprecated_sysctl("arc_free_target", "arc.free_target");
+
 	return (0);
 }
 
@@ -251,7 +267,7 @@ param_set_arc_free_target(SYSCTL_HANDLER_ARGS)
  */
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
 	CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
-	NULL, 0, param_set_arc_free_target, "IU",
+	NULL, 1, param_set_arc_free_target, "IU",
 	"Desired number of free pages below which ARC triggers reclaim"
 	" (LEGACY)");
 
@@ -270,12 +286,15 @@ param_set_arc_no_grow_shift(SYSCTL_HANDLER_ARGS)
 
 	arc_no_grow_shift = val;
 
+	if (arg2 != 0)
+		warn_deprecated_sysctl("arc_no_grow_shift", "arc.no_grow_shift");
+
 	return (0);
 }
 
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_no_grow_shift,
 	CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
-	NULL, 0, param_set_arc_no_grow_shift, "I",
+	NULL, 1, param_set_arc_no_grow_shift, "I",
 	"log2(fraction of ARC which must be free to allow growing) (LEGACY)");
 
 extern uint64_t l2arc_write_max;
@@ -746,12 +765,15 @@ param_set_min_auto_ashift(SYSCTL_HANDLER_ARGS)
 
 	zfs_vdev_min_auto_ashift = val;
 
+	if (arg2 != 0)
+		warn_deprecated_sysctl("min_auto_ashift",
+		    "vdev.min_auto_ashift");
+
 	return (0);
 }
 
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, min_auto_ashift,
-	CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
-	&zfs_vdev_min_auto_ashift, sizeof (zfs_vdev_min_auto_ashift),
+	CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, NULL, 1,
 	param_set_min_auto_ashift, "IU",
 	"Min ashift used when creating new top-level vdev. (LEGACY)");
 
@@ -771,12 +793,15 @@ param_set_max_auto_ashift(SYSCTL_HANDLER_ARGS)
 
 	zfs_vdev_max_auto_ashift = val;
 
+	if (arg2 != 0)
+		warn_deprecated_sysctl("max_auto_ashift",
+		    "vdev.max_auto_ashift");
+
 	return (0);
 }
 
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, max_auto_ashift,
-	CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
-	&zfs_vdev_max_auto_ashift, sizeof (zfs_vdev_max_auto_ashift),
+	CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, NULL, 1,
 	param_set_max_auto_ashift, "IU",
 	"Max ashift used when optimizing for logical -> physical sector size on"
 	" new top-level vdevs. (LEGACY)");
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/vdev_geom.c b/sys/contrib/openzfs/module/os/freebsd/zfs/vdev_geom.c
index c8ab7cc7cf8e..bbd1dafc69be 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/vdev_geom.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/vdev_geom.c
@@ -1236,7 +1236,7 @@ vdev_geom_io_done(zio_t *zio)
 	struct bio *bp = zio->io_bio;
 
 	if (zio->io_type != ZIO_TYPE_READ && zio->io_type != ZIO_TYPE_WRITE) {
-		ASSERT3P(bp, ==, NULL);
+		ASSERT0P(bp);
 		return;
 	}
 
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c
index 5c5adc6cc12b..cb5787269db2 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c
@@ -1175,7 +1175,7 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx)
 	int			count = 0;
 	zfs_acl_phys_t		acl_phys;
 
-	if (zp->z_zfsvfs->z_replay == B_FALSE) {
+	if (ZTOV(zp) != NULL && zp->z_zfsvfs->z_replay == B_FALSE) {
 		ASSERT_VOP_IN_SEQC(ZTOV(zp));
 	}
 
@@ -1632,7 +1632,7 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr,
 		if (zfsvfs->z_replay == B_FALSE)
 			ASSERT_VOP_ELOCKED(ZTOV(dzp), __func__);
 	} else
-		ASSERT3P(dzp->z_vnode, ==, NULL);
+		ASSERT0P(dzp->z_vnode);
 	memset(acl_ids, 0, sizeof (zfs_acl_ids_t));
 	acl_ids->z_mode = MAKEIMODE(vap->va_type, vap->va_mode);
 
@@ -2014,7 +2014,7 @@ top:
 
 	error = zfs_aclset_common(zp, aclp, cr, tx);
 	ASSERT0(error);
-	ASSERT3P(zp->z_acl_cached, ==, NULL);
+	ASSERT0P(zp->z_acl_cached);
 	zp->z_acl_cached = aclp;
 
 	if (fuid_dirtied)
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c
index 8d0ff9b25e30..d0a9c662e6f0 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c
@@ -357,7 +357,7 @@ zfsctl_create(zfsvfs_t *zfsvfs)
 	vnode_t *rvp;
 	uint64_t crtime[2];
 
-	ASSERT3P(zfsvfs->z_ctldir, ==, NULL);
+	ASSERT0P(zfsvfs->z_ctldir);
 
 	snapdir = sfs_alloc_node(sizeof (*snapdir), "snapshot", ZFSCTL_INO_ROOT,
 	    ZFSCTL_INO_SNAPDIR);
@@ -494,7 +494,7 @@ zfsctl_common_getattr(vnode_t *vp, vattr_t *vap)
 
 	vap->va_uid = 0;
 	vap->va_gid = 0;
-	vap->va_rdev = 0;
+	vap->va_rdev = NODEV;
 	/*
 	 * We are a purely virtual object, so we have no
 	 * blocksize or allocated blocks.
@@ -688,6 +688,8 @@ zfsctl_root_readdir(struct vop_readdir_args *ap)
 	 * count to return is 0.
 	 */
 	if (zfs_uio_offset(&uio) == 3 * sizeof (entry)) {
+		if (eofp != NULL)
+			*eofp = 1;
 		return (0);
 	}
 
@@ -760,8 +762,7 @@ zfsctl_common_pathconf(struct vop_pathconf_args *ap)
 		return (0);
 
 	case _PC_MIN_HOLE_SIZE:
-		*ap->a_retval = (int)SPA_MINBLOCKSIZE;
-		return (0);
+		return (EINVAL);
 
 	case _PC_ACL_EXTENDED:
 		*ap->a_retval = 0;
@@ -1367,7 +1368,7 @@ zfsctl_snapshot_unmount(const char *snapname, int flags __unused)
 
 	int err = getzfsvfs(snapname, &zfsvfs);
 	if (err != 0) {
-		ASSERT3P(zfsvfs, ==, NULL);
+		ASSERT0P(zfsvfs);
 		return (0);
 	}
 	vfsp = zfsvfs->z_vfs;
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c
index 191df832d726..75ba2ea0cb9e 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c
@@ -273,7 +273,7 @@ zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx)
 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
 
 	ASSERT(zp->z_unlinked);
-	ASSERT3U(zp->z_links, ==, 0);
+	ASSERT0(zp->z_links);
 
 	VERIFY0(zap_add_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx));
 
@@ -437,7 +437,7 @@ zfs_rmnode(znode_t *zp)
 	uint64_t	count;
 	int		error;
 
-	ASSERT3U(zp->z_links, ==, 0);
+	ASSERT0(zp->z_links);
 	if (zfsvfs->z_replay == B_FALSE)
 		ASSERT_VOP_ELOCKED(ZTOV(zp), __func__);
 
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_file_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_file_os.c
index 21e5f7938f9f..ca13569a1235 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_file_os.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_file_os.c
@@ -164,8 +164,9 @@ zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid)
 
 int
 zfs_file_pwrite(zfs_file_t *fp, const void *buf, size_t count, loff_t off,
-    ssize_t *resid)
+    uint8_t ashift, ssize_t *resid)
 {
+	(void) ashift;
 	return (zfs_file_write_impl(fp, buf, count, &off, resid));
 }
 
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c
index 0456552ed07e..79b784288911 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c
@@ -455,8 +455,13 @@ zfs_sync(vfs_t *vfsp, int waitfor)
 			return (0);
 		}
 
-		if (zfsvfs->z_log != NULL)
-			zil_commit(zfsvfs->z_log, 0);
+		if (zfsvfs->z_log != NULL) {
+			error = zil_commit(zfsvfs->z_log, 0);
+			if (error != 0) {
+				zfs_exit(zfsvfs, FTAG);
+				return (error);
+			}
+		}
 
 		zfs_exit(zfsvfs, FTAG);
 	} else {
@@ -1091,7 +1096,7 @@ zfsvfs_setup(zfsvfs_t *zfsvfs, boolean_t mounting)
 	if (mounting) {
 		boolean_t readonly;
 
-		ASSERT3P(zfsvfs->z_kstat.dk_kstats, ==, NULL);
+		ASSERT0P(zfsvfs->z_kstat.dk_kstats);
 		error = dataset_kstats_create(&zfsvfs->z_kstat, zfsvfs->z_os);
 		if (error)
 			return (error);
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c
index c4270d8b5d5c..f34a2fd37a77 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c
@@ -61,6 +61,7 @@
 #include <sys/fs/zfs.h>
 #include <sys/dmu.h>
 #include <sys/dmu_objset.h>
+#include <sys/dsl_dataset.h>
 #include <sys/spa.h>
 #include <sys/txg.h>
 #include <sys/dbuf.h>
@@ -388,7 +389,9 @@ zfs_ioctl(vnode_t *vp, ulong_t com, intptr_t data, int flag, cred_t *cred,
 		error = vn_lock(vp, LK_EXCLUSIVE);
 		if (error)
 			return (error);
+		vn_seqc_write_begin(vp);
 		error = zfs_ioctl_setxattr(vp, fsx, cred);
+		vn_seqc_write_end(vp);
 		VOP_UNLOCK(vp);
 		return (error);
 	}
@@ -1101,7 +1104,7 @@ zfs_create(znode_t *dzp, const char *name, vattr_t *vap, int excl, int mode,
 		zfs_exit(zfsvfs, FTAG);
 		return (error);
 	}
-	ASSERT3P(zp, ==, NULL);
+	ASSERT0P(zp);
 
 	/*
 	 * Create a new file object and update the directory
@@ -1193,8 +1196,8 @@ out:
 		*zpp = zp;
 	}
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1323,9 +1326,8 @@ out:
 	if (xzp)
 		vrele(ZTOV(xzp));
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
-
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1482,7 +1484,7 @@ zfs_mkdir(znode_t *dzp, const char *dirname, vattr_t *vap, znode_t **zpp,
 		zfs_exit(zfsvfs, FTAG);
 		return (error);
 	}
-	ASSERT3P(zp, ==, NULL);
+	ASSERT0P(zp);
 
 	if ((error = zfs_zaccess(dzp, ACE_ADD_SUBDIRECTORY, 0, B_FALSE, cr,
 	    mnt_ns))) {
@@ -1556,8 +1558,8 @@ out:
 
 	getnewvnode_drop_reserve();
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1637,8 +1639,8 @@ zfs_rmdir_(vnode_t *dvp, vnode_t *vp, const char *name, cred_t *cr)
 	if (zfsvfs->z_use_namecache)
 		cache_vop_rmdir(dvp, vp);
 out:
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1736,7 +1738,7 @@ zfs_readdir(vnode_t *vp, zfs_uio_t *uio, cred_t *cr, int *eofp,
 	/*
 	 * Quit if directory has been removed (posix)
 	 */
-	if ((*eofp = zp->z_unlinked) != 0) {
+	if ((*eofp = (zp->z_unlinked != 0)) != 0) {
 		zfs_exit(zfsvfs, FTAG);
 		return (0);
 	}
@@ -2014,7 +2016,7 @@ zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr)
 	if (vp->v_type == VBLK || vp->v_type == VCHR)
 		vap->va_rdev = zfs_cmpldev(rdev);
 	else
-		vap->va_rdev = 0;
+		vap->va_rdev = NODEV;
 	vap->va_gen = zp->z_gen;
 	vap->va_flags = 0;	/* FreeBSD: Reset chflags(2) flags. */
 	vap->va_filerev = zp->z_seq;
@@ -2204,6 +2206,7 @@ zfs_setattr_dir(znode_t *dzp)
 		if (err)
 			break;
 
+		vn_seqc_write_begin(ZTOV(zp));
 		mutex_enter(&dzp->z_lock);
 
 		if (zp->z_uid != dzp->z_uid) {
@@ -2253,6 +2256,7 @@ sa_add_projid_err:
 			dmu_tx_abort(tx);
 		}
 		tx = NULL;
+		vn_seqc_write_end(ZTOV(zp));
 		if (err != 0 && err != ENOENT)
 			break;
 
@@ -3009,8 +3013,8 @@ out:
 	}
 
 out2:
-	if (os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (err == 0 && os->os_sync == ZFS_SYNC_ALWAYS)
+		err = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (err);
@@ -3539,7 +3543,7 @@ out_seq:
 
 out:
 	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+		error = zil_commit(zilog, 0);
 	zfs_exit(zfsvfs, FTAG);
 
 	return (error);
@@ -3731,7 +3735,7 @@ zfs_symlink(znode_t *dzp, const char *name, vattr_t *vap,
 		*zpp = zp;
 
 		if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-			zil_commit(zilog, 0);
+			error = zil_commit(zilog, 0);
 	}
 
 	zfs_exit(zfsvfs, FTAG);
@@ -3921,8 +3925,8 @@ zfs_link(znode_t *tdzp, znode_t *szp, const char *name, cred_t *cr,
 		vnevent_link(ZTOV(szp), ct);
 	}
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -4112,6 +4116,7 @@ zfs_pathconf(vnode_t *vp, int cmd, ulong_t *valp, cred_t *cr,
 {
 	znode_t *zp;
 	zfsvfs_t *zfsvfs;
+	uint_t blksize, iosize;
 	int error;
 
 	switch (cmd) {
@@ -4123,8 +4128,20 @@ zfs_pathconf(vnode_t *vp, int cmd, ulong_t *valp, cred_t *cr,
 		*valp = 64;
 		return (0);
 	case _PC_MIN_HOLE_SIZE:
-		*valp = (int)SPA_MINBLOCKSIZE;
-		return (0);
+		iosize = vp->v_mount->mnt_stat.f_iosize;
+		if (vp->v_type == VREG) {
+			zp = VTOZ(vp);
+			blksize = zp->z_blksz;
+			if (zp->z_size <= blksize)
+				blksize = MAX(blksize, iosize);
+			*valp = (int)blksize;
+			return (0);
+		}
+		if (vp->v_type == VDIR) {
+			*valp = (int)iosize;
+			return (0);
+		}
+		return (EINVAL);
 	case _PC_ACL_EXTENDED:
 #if 0		/* POSIX ACLs are not implemented for ZFS on FreeBSD yet. */
 		zp = VTOZ(vp);
@@ -4206,8 +4223,20 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
 
 			zfs_vmobject_wlock(object);
 			(void) vm_page_grab_pages(object, OFF_TO_IDX(start),
-			    VM_ALLOC_NORMAL | VM_ALLOC_WAITOK | VM_ALLOC_ZERO,
+			    VM_ALLOC_NORMAL | VM_ALLOC_WAITOK,
 			    ma, count);
+			if (!vm_page_all_valid(ma[count - 1])) {
+				/*
+				 * Later in this function, we copy DMU data to
+				 * invalid pages only. The last page may not be
+				 * entirely filled though, if the file does not
+				 * end on a page boundary. Therefore, we zero
+				 * that last page here to make sure it does not
+				 * contain garbage after the end of file.
+				 */
+				ASSERT(vm_page_none_valid(ma[count - 1]));
+				vm_page_zero_invalid(ma[count - 1], FALSE);
+			}
 			zfs_vmobject_wunlock(object);
 		}
 		if (blksz == zp->z_blksz)
@@ -4313,7 +4342,7 @@ typedef struct {
 } putpage_commit_arg_t;
 
 static void
-zfs_putpage_commit_cb(void *arg)
+zfs_putpage_commit_cb(void *arg, int err)
 {
 	putpage_commit_arg_t *pca = arg;
 	vm_object_t object = pca->pca_pages[0]->object;
@@ -4322,7 +4351,17 @@ zfs_putpage_commit_cb(void *arg)
 
 	for (uint_t i = 0; i < pca->pca_npages; i++) {
 		vm_page_t pp = pca->pca_pages[i];
-		vm_page_undirty(pp);
+
+		if (err == 0) {
+			/*
+			 * Writeback succeeded, so undirty the page. If it
+			 * fails, we leave it in the same state it was. That's
+			 * most likely dirty, so it will get tried again some
+			 * other time.
+			 */
+			vm_page_undirty(pp);
+		}
+
 		vm_page_sunbusy(pp);
 	}
 
@@ -4510,8 +4549,13 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
 
 out:
 	zfs_rangelock_exit(lr);
-	if (commit)
-		zil_commit(zfsvfs->z_log, zp->z_id);
+	if (commit) {
+		err = zil_commit(zfsvfs->z_log, zp->z_id);
+		if (err != 0) {
+			zfs_exit(zfsvfs, FTAG);
+			return (err);
+		}
+	}
 
 	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, len);
 
@@ -5223,8 +5267,32 @@ struct vop_fsync_args {
 static int
 zfs_freebsd_fsync(struct vop_fsync_args *ap)
 {
+	vnode_t *vp = ap->a_vp;
+	int err = 0;
 
-	return (zfs_fsync(VTOZ(ap->a_vp), 0, ap->a_td->td_ucred));
+	/*
+	 * Push any dirty mmap()'d data out to the DMU and ZIL, ready for
+	 * zil_commit() to be called in zfs_fsync().
+	 */
+	if (vm_object_mightbedirty(vp->v_object)) {
+		zfs_vmobject_wlock(vp->v_object);
+		if (!vm_object_page_clean(vp->v_object, 0, 0, 0))
+			err = SET_ERROR(EIO);
+		zfs_vmobject_wunlock(vp->v_object);
+		if (err) {
+			/*
+			 * Unclear what state things are in. zfs_putpages()
+			 * will ensure the pages remain dirty if they haven't
+			 * been written down to the DMU, but because there may
+			 * be nothing logged, we can't assume that zfs_sync()
+			 * -> zil_commit() will give us a useful error. It's
+			 *  safest if we just error out here.
+			 */
+			return (err);
+		}
+	}
+
+	return (zfs_fsync(VTOZ(vp), 0, ap->a_td->td_ucred));
 }
 
 #ifndef _SYS_SYSPROTO_H_
@@ -5689,6 +5757,9 @@ zfs_freebsd_pathconf(struct vop_pathconf_args *ap)
 {
 	ulong_t val;
 	int error;
+#ifdef _PC_CLONE_BLKSIZE
+	zfsvfs_t *zfsvfs;
+#endif
 
 	error = zfs_pathconf(ap->a_vp, ap->a_name, &val,
 	    curthread->td_ucred, NULL);
@@ -5735,6 +5806,21 @@ zfs_freebsd_pathconf(struct vop_pathconf_args *ap)
 		*ap->a_retval = 1;
 		return (0);
 #endif
+#ifdef _PC_CLONE_BLKSIZE
+	case _PC_CLONE_BLKSIZE:
+		zfsvfs = (zfsvfs_t *)ap->a_vp->v_mount->mnt_data;
+		if (zfs_bclone_enabled &&
+		    spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
+		    SPA_FEATURE_BLOCK_CLONING))
+			*ap->a_retval = dsl_dataset_feature_is_active(
+			    zfsvfs->z_os->os_dsl_dataset,
+			    SPA_FEATURE_LARGE_BLOCKS) ?
+			    SPA_MAXBLOCKSIZE :
+			    SPA_OLD_MAXBLOCKSIZE;
+		else
+			*ap->a_retval = 0;
+		return (0);
+#endif
 	default:
 		return (vop_stdpathconf(ap));
 	}
@@ -6773,9 +6859,11 @@ zfs_deallocate(struct vop_deallocate_args *ap)
 	if (error == 0) {
 		if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS ||
 		    (ap->a_ioflag & IO_SYNC) != 0)
-			zil_commit(zilog, zp->z_id);
-		*ap->a_offset = off + len;
-		*ap->a_len = 0;
+			error = zil_commit(zilog, zp->z_id);
+		if (error == 0) {
+			*ap->a_offset = off + len;
+			*ap->a_len = 0;
+		}
 	}
 
 	zfs_exit(zfsvfs, FTAG);
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c
index 775f54a65f7d..649022ab5bcb 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c
@@ -161,15 +161,15 @@ zfs_znode_cache_destructor(void *buf, void *arg)
 	znode_t *zp = buf;
 
 	ASSERT(!POINTER_IS_VALID(zp->z_zfsvfs));
-	ASSERT3P(zp->z_vnode, ==, NULL);
+	ASSERT0P(zp->z_vnode);
 	ASSERT(!list_link_active(&zp->z_link_node));
 	mutex_destroy(&zp->z_lock);
 	mutex_destroy(&zp->z_acl_lock);
 	rw_destroy(&zp->z_xattr_lock);
 	zfs_rangelock_fini(&zp->z_rangelock);
 
-	ASSERT3P(zp->z_acl_cached, ==, NULL);
-	ASSERT3P(zp->z_xattr_cached, ==, NULL);
+	ASSERT0P(zp->z_acl_cached);
+	ASSERT0P(zp->z_xattr_cached);
 }
 
 
@@ -195,7 +195,7 @@ zfs_znode_init(void)
 	/*
 	 * Initialize zcache
 	 */
-	ASSERT3P(znode_uma_zone, ==, NULL);
+	ASSERT0P(znode_uma_zone);
 	znode_uma_zone = uma_zcreate("zfs_znode_cache",
 	    sizeof (znode_t), zfs_znode_cache_constructor_smr,
 	    zfs_znode_cache_destructor_smr, NULL, NULL, 0, 0);
@@ -224,7 +224,7 @@ zfs_znode_init(void)
 	/*
 	 * Initialize zcache
 	 */
-	ASSERT3P(znode_cache, ==, NULL);
+	ASSERT0P(znode_cache);
 	znode_cache = kmem_cache_create("zfs_znode_cache",
 	    sizeof (znode_t), 0, zfs_znode_cache_constructor,
 	    zfs_znode_cache_destructor, NULL, NULL, NULL, KMC_RECLAIMABLE);
@@ -353,8 +353,8 @@ zfs_znode_sa_init(zfsvfs_t *zfsvfs, znode_t *zp,
 	ASSERT(!POINTER_IS_VALID(zp->z_zfsvfs) || (zfsvfs == zp->z_zfsvfs));
 	ASSERT(MUTEX_HELD(ZFS_OBJ_MUTEX(zfsvfs, zp->z_id)));
 
-	ASSERT3P(zp->z_sa_hdl, ==, NULL);
-	ASSERT3P(zp->z_acl_cached, ==, NULL);
+	ASSERT0P(zp->z_sa_hdl);
+	ASSERT0P(zp->z_acl_cached);
 	if (sa_hdl == NULL) {
 		VERIFY0(sa_handle_get_from_db(zfsvfs->z_os, db, zp,
 		    SA_HDL_SHARED, &zp->z_sa_hdl));
@@ -817,6 +817,10 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr,
 	(*zpp)->z_dnodesize = dnodesize;
 	(*zpp)->z_projid = projid;
 
+	vnode_t *vp = ZTOV(*zpp);
+	if (!(flag & IS_ROOT_NODE))
+		vn_seqc_write_begin(vp);
+
 	if (vap->va_mask & AT_XVATTR)
 		zfs_xvattr_set(*zpp, (xvattr_t *)vap, tx);
 
@@ -825,7 +829,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr,
 		VERIFY0(zfs_aclset_common(*zpp, acl_ids->z_aclp, cr, tx));
 	}
 	if (!(flag & IS_ROOT_NODE)) {
-		vnode_t *vp = ZTOV(*zpp);
+		vn_seqc_write_end(vp);
 		vp->v_vflag |= VV_FORCEINSMQ;
 		int err = insmntque(vp, zfsvfs->z_vfs);
 		vp->v_vflag &= ~VV_FORCEINSMQ;
@@ -1127,7 +1131,7 @@ zfs_rezget(znode_t *zp)
 	}
 	rw_exit(&zp->z_xattr_lock);
 
-	ASSERT3P(zp->z_sa_hdl, ==, NULL);
+	ASSERT0P(zp->z_sa_hdl);
 	err = sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db);
 	if (err) {
 		ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num);
@@ -1298,7 +1302,7 @@ zfs_znode_free(znode_t *zp)
 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
 	char *symlink;
 
-	ASSERT3P(zp->z_sa_hdl, ==, NULL);
+	ASSERT0P(zp->z_sa_hdl);
 	zp->z_vnode = NULL;
 	mutex_enter(&zfsvfs->z_znodes_lock);
 	POINTER_INVALIDATE(&zp->z_zfsvfs);
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zio_crypt.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zio_crypt.c
index 91cf38016e00..8562c42b3220 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zio_crypt.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zio_crypt.c
@@ -437,6 +437,7 @@ zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
 
 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
 
+	memset(&cuio_s, 0, sizeof (cuio_s));
 	zfs_uio_init(&cuio, &cuio_s);
 
 	keydata_len = zio_crypt_table[crypt].ci_keylen;
@@ -519,6 +520,7 @@ zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
 	keydata_len = zio_crypt_table[crypt].ci_keylen;
 	rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
 
+	memset(&cuio_s, 0, sizeof (cuio_s));
 	zfs_uio_init(&cuio, &cuio_s);
 
 	/*
diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c
index 72a7c4ea082a..3ddbfcb97184 100644
--- a/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c
+++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c
@@ -31,7 +31,7 @@
  * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Integros [integros.com]
- * Copyright (c) 2024, Klara, Inc.
+ * Copyright (c) 2024, 2025, Klara, Inc.
  */
 
 /* Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org> */
@@ -183,6 +183,7 @@ static struct filterops zvol_filterops_vnode = {
 	.f_isfd = 1,
 	.f_detach = zvol_filter_detach,
 	.f_event = zvol_filter_vnode,
+	.f_copy = knote_triv_copy,
 };
 
 extern uint_t zfs_geom_probe_vdev_key;
@@ -196,7 +197,6 @@ DECLARE_GEOM_CLASS(zfs_zvol_class, zfs_zvol);
 
 static int zvol_geom_open(struct g_provider *pp, int flag, int count);
 static int zvol_geom_close(struct g_provider *pp, int flag, int count);
-static void zvol_geom_destroy(zvol_state_t *zv);
 static int zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace);
 static void zvol_geom_bio_start(struct bio *bp);
 static int zvol_geom_bio_getattr(struct bio *bp);
@@ -226,25 +226,14 @@ zvol_geom_open(struct g_provider *pp, int flag, int count)
 	}
 
 retry:
-	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
-	/*
-	 * Obtain a copy of private under zvol_state_lock to make sure either
-	 * the result of zvol free code setting private to NULL is observed,
-	 * or the zv is protected from being freed because of the positive
-	 * zv_open_count.
-	 */
-	zv = pp->private;
-	if (zv == NULL) {
-		rw_exit(&zvol_state_lock);
-		err = SET_ERROR(ENXIO);
-		goto out_locked;
-	}
+	zv = atomic_load_ptr(&pp->private);
+	if (zv == NULL)
+		return (SET_ERROR(ENXIO));
 
 	mutex_enter(&zv->zv_state_lock);
 	if (zv->zv_zso->zso_dying || zv->zv_flags & ZVOL_REMOVING) {
-		rw_exit(&zvol_state_lock);
 		err = SET_ERROR(ENXIO);
-		goto out_zv_locked;
+		goto out_locked;
 	}
 	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);
 
@@ -257,8 +246,24 @@ retry:
 		drop_suspend = B_TRUE;
 		if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) {
 			mutex_exit(&zv->zv_state_lock);
+
+			/*
+			 * Removal may happen while the locks are down, so
+			 * we can't trust zv any longer; we have to start over.
+			 */
+			zv = atomic_load_ptr(&pp->private);
+			if (zv == NULL)
+				return (SET_ERROR(ENXIO));
+
 			rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 			mutex_enter(&zv->zv_state_lock);
+
+			if (zv->zv_zso->zso_dying ||
+			    zv->zv_flags & ZVOL_REMOVING) {
+				err = SET_ERROR(ENXIO);
+				goto out_locked;
+			}
+
 			/* Check to see if zv_suspend_lock is needed. */
 			if (zv->zv_open_count != 0) {
 				rw_exit(&zv->zv_suspend_lock);
@@ -266,7 +271,6 @@ retry:
 			}
 		}
 	}
-	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
@@ -294,7 +298,7 @@ retry:
 		if (drop_namespace)
 			mutex_exit(&spa_namespace_lock);
 		if (err)
-			goto out_zv_locked;
+			goto out_locked;
 		pp->mediasize = zv->zv_volsize;
 		pp->stripeoffset = 0;
 		pp->stripesize = zv->zv_volblocksize;
@@ -329,9 +333,8 @@ out_opened:
 		zvol_last_close(zv);
 		wakeup(zv);
 	}
-out_zv_locked:
-	mutex_exit(&zv->zv_state_lock);
 out_locked:
+	mutex_exit(&zv->zv_state_lock);
 	if (drop_suspend)
 		rw_exit(&zv->zv_suspend_lock);
 	return (err);
@@ -345,12 +348,9 @@ zvol_geom_close(struct g_provider *pp, int flag, int count)
 	boolean_t drop_suspend = B_TRUE;
 	int new_open_count;
 
-	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
-	zv = pp->private;
-	if (zv == NULL) {
-		rw_exit(&zvol_state_lock);
+	zv = atomic_load_ptr(&pp->private);
+	if (zv == NULL)
 		return (SET_ERROR(ENXIO));
-	}
 
 	mutex_enter(&zv->zv_state_lock);
 	if (zv->zv_flags & ZVOL_EXCL) {
@@ -377,6 +377,15 @@ zvol_geom_close(struct g_provider *pp, int flag, int count)
 			mutex_exit(&zv->zv_state_lock);
 			rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 			mutex_enter(&zv->zv_state_lock);
+
+			/*
+			 * Unlike in zvol_geom_open(), we don't check if
+			 * removal started here, because we might be one of the
+			 * openers that needs to be thrown out! If we're the
+			 * last, we need to call zvol_last_close() below to
+			 * finish cleanup. So, no special treatment for us.
+			 */
+
 			/* Check to see if zv_suspend_lock is needed. */
 			new_open_count = zv->zv_open_count - count;
 			if (new_open_count != 0) {
@@ -387,7 +396,6 @@ zvol_geom_close(struct g_provider *pp, int flag, int count)
 	} else {
 		drop_suspend = B_FALSE;
 	}
-	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
@@ -408,20 +416,6 @@ zvol_geom_close(struct g_provider *pp, int flag, int count)
 	return (0);
 }
 
-static void
-zvol_geom_destroy(zvol_state_t *zv)
-{
-	struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
-	struct g_provider *pp = zsg->zsg_provider;
-
-	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);
-
-	g_topology_assert();
-
-	zsg->zsg_provider = NULL;
-	g_wither_geom(pp->geom, ENXIO);
-}
-
 void
 zvol_wait_close(zvol_state_t *zv)
 {
@@ -454,7 +448,7 @@ zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace)
 	    ("Unsupported access request to %s (acr=%d, acw=%d, ace=%d).",
 	    pp->name, acr, acw, ace));
 
-	if (pp->private == NULL) {
+	if (atomic_load_ptr(&pp->private) == NULL) {
 		if (acr <= 0 && acw <= 0 && ace <= 0)
 			return (0);
 		return (pp->error);
@@ -727,9 +721,9 @@ unlock:
 		break;
 	}
 
-	if (commit) {
+	if (error == 0 && commit) {
 commit:
-		zil_commit(zv->zv_zilog, ZVOL_OBJ);
+		error = zil_commit(zv->zv_zilog, ZVOL_OBJ);
 	}
 resume:
 	rw_exit(&zv->zv_suspend_lock);
@@ -906,8 +900,8 @@ zvol_cdev_write(struct cdev *dev, struct uio *uio_s, int ioflag)
 	zfs_rangelock_exit(lr);
 	int64_t nwritten = start_resid - zfs_uio_resid(&uio);
 	dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
-	if (commit)
-		zil_commit(zv->zv_zilog, ZVOL_OBJ);
+	if (error == 0 && commit)
+		error = zil_commit(zv->zv_zilog, ZVOL_OBJ);
 	rw_exit(&zv->zv_suspend_lock);
 
 	return (error);
@@ -921,25 +915,14 @@ zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td)
 	boolean_t drop_suspend = B_FALSE;
 
 retry:
-	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
-	/*
-	 * Obtain a copy of si_drv2 under zvol_state_lock to make sure either
-	 * the result of zvol free code setting si_drv2 to NULL is observed,
-	 * or the zv is protected from being freed because of the positive
-	 * zv_open_count.
-	 */
-	zv = dev->si_drv2;
-	if (zv == NULL) {
-		rw_exit(&zvol_state_lock);
-		err = SET_ERROR(ENXIO);
-		goto out_locked;
-	}
+	zv = atomic_load_ptr(&dev->si_drv2);
+	if (zv == NULL)
+		return (SET_ERROR(ENXIO));
 
 	mutex_enter(&zv->zv_state_lock);
-	if (zv->zv_zso->zso_dying) {
-		rw_exit(&zvol_state_lock);
+	if (zv->zv_zso->zso_dying || zv->zv_flags & ZVOL_REMOVING) {
 		err = SET_ERROR(ENXIO);
-		goto out_zv_locked;
+		goto out_locked;
 	}
 	ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_DEV);
 
@@ -954,6 +937,13 @@ retry:
 			mutex_exit(&zv->zv_state_lock);
 			rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 			mutex_enter(&zv->zv_state_lock);
+
+			if (unlikely(zv->zv_flags & ZVOL_REMOVING)) {
+				/* Removal started while locks were down. */
+				err = SET_ERROR(ENXIO);
+				goto out_locked;
+			}
+
 			/* Check to see if zv_suspend_lock is needed. */
 			if (zv->zv_open_count != 0) {
 				rw_exit(&zv->zv_suspend_lock);
@@ -961,7 +951,6 @@ retry:
 			}
 		}
 	}
-	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
@@ -989,7 +978,7 @@ retry:
 		if (drop_namespace)
 			mutex_exit(&spa_namespace_lock);
 		if (err)
-			goto out_zv_locked;
+			goto out_locked;
 	}
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
@@ -1016,9 +1005,8 @@ out_opened:
 		zvol_last_close(zv);
 		wakeup(zv);
 	}
-out_zv_locked:
-	mutex_exit(&zv->zv_state_lock);
 out_locked:
+	mutex_exit(&zv->zv_state_lock);
 	if (drop_suspend)
 		rw_exit(&zv->zv_suspend_lock);
 	return (err);
@@ -1030,12 +1018,9 @@ zvol_cdev_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 	zvol_state_t *zv;
 	boolean_t drop_suspend = B_TRUE;
 
-	rw_enter(&zvol_state_lock, ZVOL_RW_READER);
-	zv = dev->si_drv2;
-	if (zv == NULL) {
-		rw_exit(&zvol_state_lock);
+	zv = atomic_load_ptr(&dev->si_drv2);
+	if (zv == NULL)
 		return (SET_ERROR(ENXIO));
-	}
 
 	mutex_enter(&zv->zv_state_lock);
 	if (zv->zv_flags & ZVOL_EXCL) {
@@ -1060,6 +1045,15 @@ zvol_cdev_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 			mutex_exit(&zv->zv_state_lock);
 			rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 			mutex_enter(&zv->zv_state_lock);
+
+			/*
+			 * Unlike in zvol_cdev_open(), we don't check if
+			 * removal started here, because we might be one of the
+			 * openers that needs to be thrown out! If we're the
+			 * last, we need to call zvol_last_close() below to
+			 * finish cleanup. So, no special treatment for us.
+			 */
+
 			/* Check to see if zv_suspend_lock is needed. */
 			if (zv->zv_open_count != 1) {
 				rw_exit(&zv->zv_suspend_lock);
@@ -1069,7 +1063,6 @@ zvol_cdev_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 	} else {
 		drop_suspend = B_FALSE;
 	}
-	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
@@ -1101,7 +1094,8 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,
 	int error;
 	boolean_t sync;
 
-	zv = dev->si_drv2;
+	zv = atomic_load_ptr(&dev->si_drv2);
+	ASSERT3P(zv, !=, NULL);
 
 	error = 0;
 	KASSERT(zv->zv_open_count > 0,
@@ -1117,7 +1111,7 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,
 	case DIOCGFLUSH:
 		rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 		if (zv->zv_zilog != NULL)
-			zil_commit(zv->zv_zilog, ZVOL_OBJ);
+			error = zil_commit(zv->zv_zilog, ZVOL_OBJ);
 		rw_exit(&zv->zv_suspend_lock);
 		break;
 	case DIOCGDELETE:
@@ -1152,7 +1146,7 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,
 		}
 		zfs_rangelock_exit(lr);
 		if (sync)
-			zil_commit(zv->zv_zilog, ZVOL_OBJ);
+			error = zil_commit(zv->zv_zilog, ZVOL_OBJ);
 		rw_exit(&zv->zv_suspend_lock);
 		break;
 	case DIOCGSTRIPESIZE:
@@ -1162,6 +1156,7 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,
 		*(off_t *)data = 0;
 		break;
 	case DIOCGATTR: {
+		rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 		spa_t *spa = dmu_objset_spa(zv->zv_objset);
 		struct diocgattr_arg *arg = (struct diocgattr_arg *)data;
 		uint64_t refd, avail, usedobjs, availobjs;
@@ -1186,6 +1181,7 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,
 			arg->value.off = refd / DEV_BSIZE;
 		} else
 			error = SET_ERROR(ENOIOCTL);
+		rw_exit(&zv->zv_suspend_lock);
 		break;
 	}
 	case FIOSEEKHOLE:
@@ -1196,10 +1192,12 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,
 
 		hole = (cmd == FIOSEEKHOLE);
 		noff = *off;
+		rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 		lr = zfs_rangelock_enter(&zv->zv_rangelock, 0, UINT64_MAX,
 		    RL_READER);
 		error = dmu_offset_next(zv->zv_objset, ZVOL_OBJ, hole, &noff);
 		zfs_rangelock_exit(lr);
+		rw_exit(&zv->zv_suspend_lock);
 		*off = noff;
 		break;
 	}
@@ -1400,42 +1398,65 @@ zvol_alloc(const char *name, uint64_t volsize, uint64_t volblocksize,
  * Remove minor node for the specified volume.
  */
 void
-zvol_os_free(zvol_state_t *zv)
+zvol_os_remove_minor(zvol_state_t *zv)
 {
-	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
-	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
+	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 	ASSERT0(zv->zv_open_count);
+	ASSERT0(atomic_read(&zv->zv_suspend_ref));
+	ASSERT(zv->zv_flags & ZVOL_REMOVING);
 
-	ZFS_LOG(1, "ZVOL %s destroyed.", zv->zv_name);
-
-	rw_destroy(&zv->zv_suspend_lock);
-	zfs_rangelock_fini(&zv->zv_rangelock);
+	struct zvol_state_os *zso = zv->zv_zso;
+	zv->zv_zso = NULL;
 
 	if (zv->zv_volmode == ZFS_VOLMODE_GEOM) {
-		struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
-		struct g_provider *pp __maybe_unused = zsg->zsg_provider;
-
-		ASSERT3P(pp->private, ==, NULL);
+		struct zvol_state_geom *zsg = &zso->zso_geom;
+		struct g_provider *pp = zsg->zsg_provider;
+		atomic_store_ptr(&pp->private, NULL);
+		mutex_exit(&zv->zv_state_lock);
 
 		g_topology_lock();
-		zvol_geom_destroy(zv);
+		g_wither_geom(pp->geom, ENXIO);
 		g_topology_unlock();
 	} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
-		struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
+		struct zvol_state_dev *zsd = &zso->zso_dev;
 		struct cdev *dev = zsd->zsd_cdev;
 
+		if (dev != NULL)
+			atomic_store_ptr(&dev->si_drv2, NULL);
+		mutex_exit(&zv->zv_state_lock);
+
 		if (dev != NULL) {
-			ASSERT3P(dev->si_drv2, ==, NULL);
 			destroy_dev(dev);
 			knlist_clear(&zsd->zsd_selinfo.si_note, 0);
 			knlist_destroy(&zsd->zsd_selinfo.si_note);
 		}
 	}
 
+	kmem_free(zso, sizeof (struct zvol_state_os));
+
+	mutex_enter(&zv->zv_state_lock);
+}
+
+void
+zvol_os_free(zvol_state_t *zv)
+{
+	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
+	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
+	ASSERT0(zv->zv_open_count);
+	ASSERT0P(zv->zv_zso);
+
+	ASSERT0P(zv->zv_objset);
+	ASSERT0P(zv->zv_zilog);
+	ASSERT0P(zv->zv_dn);
+
+	ZFS_LOG(1, "ZVOL %s destroyed.", zv->zv_name);
+
+	rw_destroy(&zv->zv_suspend_lock);
+	zfs_rangelock_fini(&zv->zv_rangelock);
+
 	mutex_destroy(&zv->zv_state_lock);
 	cv_destroy(&zv->zv_removing_cv);
 	dataset_kstats_destroy(&zv->zv_kstat);
-	kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
 	kmem_free(zv, sizeof (zvol_state_t));
 	zvol_minors--;
 }
@@ -1493,11 +1514,11 @@ zvol_os_create_minor(const char *name)
 
 	zv->zv_objset = os;
 
-	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
+	ASSERT0P(zv->zv_kstat.dk_kstats);
 	error = dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
 	if (error)
 		goto out_dmu_objset_disown;
-	ASSERT3P(zv->zv_zilog, ==, NULL);
+	ASSERT0P(zv->zv_zilog);
 	zv->zv_zilog = zil_open(os, zvol_get_data, &zv->zv_kstat.dk_zil_sums);
 	if (spa_writeable(dmu_objset_spa(os))) {
 		if (zil_replay_disable)
@@ -1538,28 +1559,6 @@ out_doi:
 	return (error);
 }
 
-void
-zvol_os_clear_private(zvol_state_t *zv)
-{
-	ASSERT(RW_LOCK_HELD(&zvol_state_lock));
-	if (zv->zv_volmode == ZFS_VOLMODE_GEOM) {
-		struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
-		struct g_provider *pp = zsg->zsg_provider;
-
-		if (pp->private == NULL) /* already cleared */
-			return;
-
-		pp->private = NULL;
-		ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
-	} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
-		struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
-		struct cdev *dev = zsd->zsd_cdev;
-
-		if (dev != NULL)
-			dev->si_drv2 = NULL;
-	}
-}
-
 int
 zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize)
 {
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-condvar.c b/sys/contrib/openzfs/module/os/linux/spl/spl-condvar.c
index ce9c9e39e60c..aac5f2ebbfd2 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-condvar.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-condvar.c
@@ -66,9 +66,9 @@ void
 __cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg)
 {
 	ASSERT(cvp);
-	ASSERT(name == NULL);
+	ASSERT0P(name);
 	ASSERT(type == CV_DEFAULT);
-	ASSERT(arg == NULL);
+	ASSERT0P(arg);
 
 	cvp->cv_magic = CV_MAGIC;
 	init_waitqueue_head(&cvp->cv_event);
@@ -83,7 +83,7 @@ static int
 cv_destroy_wakeup(kcondvar_t *cvp)
 {
 	if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) {
-		ASSERT(cvp->cv_mutex == NULL);
+		ASSERT0P(cvp->cv_mutex);
 		ASSERT(!waitqueue_active(&cvp->cv_event));
 		return (1);
 	}
@@ -104,7 +104,7 @@ __cv_destroy(kcondvar_t *cvp)
 	while (cv_destroy_wakeup(cvp) == 0)
 		wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1);
 
-	ASSERT3P(cvp->cv_mutex, ==, NULL);
+	ASSERT0P(cvp->cv_mutex);
 	ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0);
 	ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0);
 	ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0);
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-generic.c b/sys/contrib/openzfs/module/os/linux/spl/spl-generic.c
index f37699b4347e..89ca4a648b2f 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-generic.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-generic.c
@@ -709,7 +709,7 @@ zone_get_hostid(void *zone)
 {
 	uint32_t hostid;
 
-	ASSERT3P(zone, ==, NULL);
+	ASSERT0P(zone);
 
 	if (spl_hostid != 0)
 		return ((uint32_t)(spl_hostid & HW_HOSTID_MASK));
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-kmem-cache.c b/sys/contrib/openzfs/module/os/linux/spl/spl-kmem-cache.c
index fab80289b278..22e4ed169d03 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-kmem-cache.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-kmem-cache.c
@@ -296,7 +296,7 @@ spl_slab_free(spl_kmem_slab_t *sks,
 	spl_kmem_cache_t *skc;
 
 	ASSERT(sks->sks_magic == SKS_MAGIC);
-	ASSERT(sks->sks_ref == 0);
+	ASSERT0(sks->sks_ref);
 
 	skc = sks->sks_cache;
 	ASSERT(skc->skc_magic == SKC_MAGIC);
@@ -598,7 +598,7 @@ static void
 spl_magazine_free(spl_kmem_magazine_t *skm)
 {
 	ASSERT(skm->skm_magic == SKM_MAGIC);
-	ASSERT(skm->skm_avail == 0);
+	ASSERT0(skm->skm_avail);
 	kfree(skm);
 }
 
@@ -610,7 +610,7 @@ spl_magazine_create(spl_kmem_cache_t *skc)
 {
 	int i = 0;
 
-	ASSERT((skc->skc_flags & KMC_SLAB) == 0);
+	ASSERT0((skc->skc_flags & KMC_SLAB));
 
 	skc->skc_mag = kzalloc(sizeof (spl_kmem_magazine_t *) *
 	    num_possible_cpus(), kmem_flags_convert(KM_SLEEP));
@@ -640,7 +640,7 @@ spl_magazine_destroy(spl_kmem_cache_t *skc)
 	spl_kmem_magazine_t *skm;
 	int i = 0;
 
-	ASSERT((skc->skc_flags & KMC_SLAB) == 0);
+	ASSERT0((skc->skc_flags & KMC_SLAB));
 
 	for_each_possible_cpu(i) {
 		skm = skc->skc_mag[i];
@@ -679,8 +679,8 @@ spl_kmem_cache_create(const char *name, size_t size, size_t align,
 	/*
 	 * Unsupported flags
 	 */
-	ASSERT(vmp == NULL);
-	ASSERT(reclaim == NULL);
+	ASSERT0P(vmp);
+	ASSERT0P(reclaim);
 
 	might_sleep();
 
@@ -863,11 +863,11 @@ spl_kmem_cache_destroy(spl_kmem_cache_t *skc)
 	 * Validate there are no objects in use and free all the
 	 * spl_kmem_slab_t, spl_kmem_obj_t, and object buffers.
 	 */
-	ASSERT3U(skc->skc_slab_alloc, ==, 0);
-	ASSERT3U(skc->skc_obj_alloc, ==, 0);
-	ASSERT3U(skc->skc_slab_total, ==, 0);
-	ASSERT3U(skc->skc_obj_total, ==, 0);
-	ASSERT3U(skc->skc_obj_emergency, ==, 0);
+	ASSERT0(skc->skc_slab_alloc);
+	ASSERT0(skc->skc_obj_alloc);
+	ASSERT0(skc->skc_slab_total);
+	ASSERT0(skc->skc_obj_total);
+	ASSERT0(skc->skc_obj_emergency);
 	ASSERT(list_empty(&skc->skc_complete_list));
 
 	ASSERT3U(percpu_counter_sum(&skc->skc_linux_alloc), ==, 0);
@@ -986,7 +986,7 @@ spl_cache_grow(spl_kmem_cache_t *skc, int flags, void **obj)
 
 	ASSERT0(flags & ~KM_PUBLIC_MASK);
 	ASSERT(skc->skc_magic == SKC_MAGIC);
-	ASSERT((skc->skc_flags & KMC_SLAB) == 0);
+	ASSERT0((skc->skc_flags & KMC_SLAB));
 
 	*obj = NULL;
 
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c b/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c
index 48f70b00c96b..02c5b42bc4a0 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c
@@ -541,7 +541,7 @@ __kstat_create(const char *ks_module, int ks_instance, const char *ks_name,
 	kstat_t *ksp;
 
 	ASSERT(ks_module);
-	ASSERT(ks_instance == 0);
+	ASSERT0(ks_instance);
 	ASSERT(ks_name);
 
 	if ((ks_type == KSTAT_TYPE_INTR) || (ks_type == KSTAT_TYPE_IO))
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c b/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c
index f42f455222de..8f5c73b13df5 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c
@@ -80,7 +80,7 @@ __thread_create(caddr_t stk, size_t  stksize, thread_func_t func,
 
 	/* Option pp is simply ignored */
 	/* Variable stack size unsupported */
-	ASSERT(stk == NULL);
+	ASSERT0P(stk);
 
 	tp = kmem_alloc(sizeof (thread_priv_t), KM_PUSHPAGE);
 	if (tp == NULL)
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-tsd.c b/sys/contrib/openzfs/module/os/linux/spl/spl-tsd.c
index 34a61bef7d4f..2e8cedf0dc87 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-tsd.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-tsd.c
@@ -161,7 +161,7 @@ tsd_hash_add(tsd_hash_table_t *table, uint_t key, pid_t pid, void *value)
 	ulong_t hash;
 	int rc = 0;
 
-	ASSERT3P(tsd_hash_search(table, key, pid), ==, NULL);
+	ASSERT0P(tsd_hash_search(table, key, pid));
 
 	/* New entry allocate structure, set value, and add to hash */
 	entry = kmem_alloc(sizeof (tsd_hash_entry_t), KM_PUSHPAGE);
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-zone.c b/sys/contrib/openzfs/module/os/linux/spl/spl-zone.c
index 45c2999a4bb1..b2eae5d00b10 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-zone.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-zone.c
@@ -25,6 +25,10 @@
  * SUCH DAMAGE.
  */
 
+/*
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
+ */
+
 #include <sys/types.h>
 #include <sys/sysmacros.h>
 #include <sys/kmem.h>
@@ -56,6 +60,19 @@ typedef struct zone_dataset {
 } zone_dataset_t;
 
 #ifdef CONFIG_USER_NS
+
+/*
+ * Linux 6.18 moved the generic namespace type away from ns->ops->type onto
+ * ns_common itself.
+ */
+#ifdef HAVE_NS_COMMON_TYPE
+#define	ns_is_newuser(ns)	\
+	((ns)->ns_type == CLONE_NEWUSER)
+#else
+#define	ns_is_newuser(ns)	\
+	((ns)->ops != NULL && (ns)->ops->type == CLONE_NEWUSER)
+#endif
+
 /*
  * Returns:
  * - 0 on success
@@ -84,7 +101,7 @@ user_ns_get(int fd, struct user_namespace **userns)
 		goto done;
 	}
 	ns = get_proc_ns(file_inode(nsfile));
-	if (ns->ops->type != CLONE_NEWUSER) {
+	if (!ns_is_newuser(ns)) {
 		error = ENOTTY;
 		goto done;
 	}
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c b/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c
index 248c9b7a6d3b..18f2426fbbfc 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/abd_os.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
  * Copyright (c) 2019 by Delphix. All rights reserved.
  * Copyright (c) 2023, 2024, Klara Inc.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  */
 
 /*
@@ -863,9 +864,9 @@ abd_iter_advance(struct abd_iter *aiter, size_t amount)
 	 * Ensure that last chunk is not in use. abd_iterate_*() must clear
 	 * this state (directly or abd_iter_unmap()) before advancing.
 	 */
-	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
+	ASSERT0P(aiter->iter_mapaddr);
 	ASSERT0(aiter->iter_mapsize);
-	ASSERT3P(aiter->iter_page, ==, NULL);
+	ASSERT0P(aiter->iter_page);
 	ASSERT0(aiter->iter_page_doff);
 	ASSERT0(aiter->iter_page_dsize);
 
@@ -897,7 +898,7 @@ abd_iter_map(struct abd_iter *aiter)
 	void *paddr;
 	size_t offset = 0;
 
-	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
+	ASSERT0P(aiter->iter_mapaddr);
 	ASSERT0(aiter->iter_mapsize);
 
 	/* There's nothing left to iterate over, so do nothing */
@@ -1109,6 +1110,14 @@ abd_return_buf_copy(abd_t *abd, void *buf, size_t n)
 #define	ABD_ITER_PAGE_SIZE(page)	(PAGESIZE)
 #endif
 
+#ifndef nth_page
+/*
+ * Since 6.18 nth_page() no longer exists, and is no longer required to iterate
+ * within a single SG entry, so we replace it with a simple addition.
+ */
+#define	nth_page(p, n)	((p)+(n))
+#endif
+
 void
 abd_iter_page(struct abd_iter *aiter)
 {
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c b/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
index 154ca22d9513..1bd3500e9f66 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
@@ -471,13 +471,17 @@ vdev_disk_close(vdev_t *v)
 	if (v->vdev_reopening || vd == NULL)
 		return;
 
+	rw_enter(&vd->vd_lock, RW_WRITER);
+
 	if (vd->vd_bdh != NULL)
 		vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa),
 		    zfs_vdev_holder);
 
+	v->vdev_tsd = NULL;
+
+	rw_exit(&vd->vd_lock);
 	rw_destroy(&vd->vd_lock);
 	kmem_free(vd, sizeof (vdev_disk_t));
-	v->vdev_tsd = NULL;
 }
 
 /*
@@ -552,7 +556,7 @@ vdev_bio_associate_blkg(struct bio *bio)
 #endif
 
 	ASSERT3P(q, !=, NULL);
-	ASSERT3P(bio->bi_blkg, ==, NULL);
+	ASSERT0P(bio->bi_blkg);
 
 	if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
 		bio->bi_blkg = q->root_blkg;
@@ -574,7 +578,7 @@ vdev_bio_set_dev(struct bio *bio, struct block_device *bdev)
 	bio->bi_bdev = bdev;
 
 	ASSERT3P(q, !=, NULL);
-	ASSERT3P(bio->bi_blkg, ==, NULL);
+	ASSERT0P(bio->bi_blkg);
 
 	if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
 		bio->bi_blkg = q->root_blkg;
@@ -806,7 +810,7 @@ vbio_completion(struct bio *bio)
 	 * here; instead we stash vbio on the zio and take care of it in the
 	 * done callback.
 	 */
-	ASSERT3P(zio->io_bio, ==, NULL);
+	ASSERT0P(zio->io_bio);
 	zio->io_bio = vbio;
 
 	zio_delay_interrupt(zio);
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c
index 1b169122f25b..934d74a112fd 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c
@@ -1900,7 +1900,7 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr,
 		if (!(flag & IS_ROOT_NODE) &&
 		    (dzp->z_pflags & ZFS_INHERIT_ACE) &&
 		    !(dzp->z_pflags & ZFS_XATTR)) {
-			VERIFY(0 == zfs_acl_node_read(dzp, B_TRUE,
+			VERIFY0(zfs_acl_node_read(dzp, B_TRUE,
 			    &paclp, B_FALSE));
 			acl_ids->z_aclp = zfs_acl_inherit(zfsvfs,
 			    vap->va_mode, paclp, acl_ids->z_mode, &need_chmod);
@@ -2204,8 +2204,8 @@ top:
 	}
 
 	error = zfs_aclset_common(zp, aclp, cr, tx);
-	ASSERT(error == 0);
-	ASSERT(zp->z_acl_cached == NULL);
+	ASSERT0(error);
+	ASSERT0P(zp->z_acl_cached);
 	zp->z_acl_cached = aclp;
 
 	if (fuid_dirtied)
@@ -2524,7 +2524,7 @@ zfs_zaccess_common(znode_t *zp, uint32_t v4_mode, uint32_t *working_mode,
 	 * Also note: DOS R/O is ignored for directories.
 	 */
 	if ((v4_mode & WRITE_MASK_DATA) &&
-	    S_ISDIR(ZTOI(zp)->i_mode) &&
+	    !S_ISDIR(ZTOI(zp)->i_mode) &&
 	    (zp->z_pflags & ZFS_READONLY)) {
 		return (SET_ERROR(EPERM));
 	}
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c
index 6552a933ce0a..fb4de50480a3 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c
@@ -494,9 +494,9 @@ zfsctl_inode_alloc(zfsvfs_t *zfsvfs, uint64_t id,
 	if (!creation)
 		now = current_time(ip);
 	zp = ITOZ(ip);
-	ASSERT3P(zp->z_dirlocks, ==, NULL);
-	ASSERT3P(zp->z_acl_cached, ==, NULL);
-	ASSERT3P(zp->z_xattr_cached, ==, NULL);
+	ASSERT0P(zp->z_dirlocks);
+	ASSERT0P(zp->z_acl_cached);
+	ASSERT0P(zp->z_xattr_cached);
 	zp->z_id = id;
 	zp->z_unlinked = B_FALSE;
 	zp->z_atime_dirty = B_FALSE;
@@ -590,7 +590,7 @@ zfsctl_inode_lookup(zfsvfs_t *zfsvfs, uint64_t id,
 int
 zfsctl_create(zfsvfs_t *zfsvfs)
 {
-	ASSERT(zfsvfs->z_ctldir == NULL);
+	ASSERT0P(zfsvfs->z_ctldir);
 
 	zfsvfs->z_ctldir = zfsctl_inode_alloc(zfsvfs, ZFSCTL_INO_ROOT,
 	    &zpl_fops_root, &zpl_ops_root, 0);
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c
index 2f935bb3fc8c..e8de536606e2 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c
@@ -463,7 +463,7 @@ zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx)
 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
 
 	ASSERT(zp->z_unlinked);
-	ASSERT(ZTOI(zp)->i_nlink == 0);
+	ASSERT0(ZTOI(zp)->i_nlink);
 
 	VERIFY3U(0, ==,
 	    zap_add_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx));
@@ -662,8 +662,8 @@ zfs_rmnode(znode_t *zp)
 	uint64_t	links;
 	int		error;
 
-	ASSERT(ZTOI(zp)->i_nlink == 0);
-	ASSERT(atomic_read(&ZTOI(zp)->i_count) == 0);
+	ASSERT0(ZTOI(zp)->i_nlink);
+	ASSERT0(atomic_read(&ZTOI(zp)->i_count));
 
 	/*
 	 * If this is an attribute directory, purge its contents.
@@ -710,7 +710,7 @@ zfs_rmnode(znode_t *zp)
 	    &xattr_obj, sizeof (xattr_obj));
 	if (error == 0 && xattr_obj) {
 		error = zfs_zget(zfsvfs, xattr_obj, &xzp);
-		ASSERT(error == 0);
+		ASSERT0(error);
 	}
 
 	acl_obj = zfs_external_acl(zp);
@@ -744,12 +744,12 @@ zfs_rmnode(znode_t *zp)
 	}
 
 	if (xzp) {
-		ASSERT(error == 0);
+		ASSERT0(error);
 		mutex_enter(&xzp->z_lock);
 		xzp->z_unlinked = B_TRUE;	/* mark xzp for deletion */
 		clear_nlink(ZTOI(xzp));		/* no more links to it */
 		links = 0;
-		VERIFY(0 == sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zfsvfs),
+		VERIFY0(sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zfsvfs),
 		    &links, sizeof (links), tx));
 		mutex_exit(&xzp->z_lock);
 		zfs_unlinked_add(xzp, tx);
@@ -872,7 +872,7 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag)
 		    ctime);
 	}
 	error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	mutex_exit(&zp->z_lock);
 
@@ -894,7 +894,7 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag)
 	    &dzp->z_pflags, sizeof (dzp->z_pflags));
 	zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime);
 	error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 	mutex_exit(&dzp->z_lock);
 
 	return (0);
@@ -986,7 +986,7 @@ zfs_drop_nlink_locked(znode_t *zp, dmu_tx_t *tx, boolean_t *unlinkedp)
 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs),
 	    NULL, &links, sizeof (links));
 	error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
-	ASSERT3U(error, ==, 0);
+	ASSERT0(error);
 
 	if (unlinkedp != NULL)
 		*unlinkedp = unlinked;
@@ -1058,7 +1058,7 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag,
 
 		/* The only error is !zfs_dirempty() and we checked earlier. */
 		error = zfs_drop_nlink_locked(zp, tx, &unlinked);
-		ASSERT3U(error, ==, 0);
+		ASSERT0(error);
 		mutex_exit(&zp->z_lock);
 	} else {
 		error = zfs_dropname(dl, zp, dzp, tx, flag);
@@ -1083,7 +1083,7 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag,
 	    NULL, &dzp->z_pflags, sizeof (dzp->z_pflags));
 	zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime);
 	error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 	mutex_exit(&dzp->z_lock);
 
 	if (unlinkedp != NULL)
@@ -1167,7 +1167,7 @@ zfs_make_xattrdir(znode_t *zp, vattr_t *vap, znode_t **xzpp, cred_t *cr)
 	ASSERT(error == 0 && parent == zp->z_id);
 #endif
 
-	VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &xzp->z_id,
+	VERIFY0(sa_update(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &xzp->z_id,
 	    sizeof (xzp->z_id), tx));
 
 	if (!zp->z_unlinked)
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_file_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_file_os.c
index c729947369c2..3fdcdbac6f68 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_file_os.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_file_os.c
@@ -115,8 +115,9 @@ zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid)
  */
 int
 zfs_file_pwrite(zfs_file_t *fp, const void *buf, size_t count, loff_t off,
-    ssize_t *resid)
+    uint8_t ashift, ssize_t *resid)
 {
+	(void) ashift;
 	ssize_t rc;
 
 	rc  = kernel_write(fp, buf, count, &off);
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_sysfs.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_sysfs.c
index 1c187d7b9cab..895d80b2d79e 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_sysfs.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_sysfs.c
@@ -223,7 +223,7 @@ zfs_kobj_add(zfs_mod_kobj_t *zkobj, struct kobject *parent, const char *name)
 {
 	/* zko_default_group.attrs must be NULL terminated */
 	ASSERT(zkobj->zko_default_group.attrs != NULL);
-	ASSERT(zkobj->zko_default_group.attrs[zkobj->zko_attr_count] == NULL);
+	ASSERT0P(zkobj->zko_default_group.attrs[zkobj->zko_attr_count]);
 
 	kobject_init(&zkobj->zko_kobj, &zkobj->zko_kobj_type);
 	return (kobject_add(&zkobj->zko_kobj, parent, name));
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c
index 396faef8f646..8a7d14ab6119 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c
@@ -279,19 +279,14 @@ zfs_sync(struct super_block *sb, int wait, cred_t *cr)
 		return (err);
 
 	/*
-	 * If the pool is suspended, just return an error. This is to help
-	 * with shutting down with pools suspended, as we don't want to block
-	 * in that case.
+	 * Sync any pending writes, but do not block if the pool is suspended.
+	 * This is to help with shutting down with pools suspended, as we don't
+	 * want to block in that case.
 	 */
-	if (spa_suspended(zfsvfs->z_os->os_spa)) {
-		zfs_exit(zfsvfs, FTAG);
-		return (SET_ERROR(EIO));
-	}
-
-	zil_commit(zfsvfs->z_log, 0);
+	err = zil_commit_flags(zfsvfs->z_log, 0, ZIL_COMMIT_NOW);
 	zfs_exit(zfsvfs, FTAG);
 
-	return (0);
+	return (err);
 }
 
 static void
@@ -883,7 +878,7 @@ zfsvfs_setup(zfsvfs_t *zfsvfs, boolean_t mounting)
 	 * operations out since we closed the ZIL.
 	 */
 	if (mounting) {
-		ASSERT3P(zfsvfs->z_kstat.dk_kstats, ==, NULL);
+		ASSERT0P(zfsvfs->z_kstat.dk_kstats);
 		error = dataset_kstats_create(&zfsvfs->z_kstat, zfsvfs->z_os);
 		if (error)
 			return (error);
@@ -1561,6 +1556,12 @@ zfs_domount(struct super_block *sb, zfs_mnt_t *zm, int silent)
 	sb->s_xattr = zpl_xattr_handlers;
 	sb->s_export_op = &zpl_export_operations;
 
+#ifdef HAVE_SET_DEFAULT_D_OP
+	set_default_d_op(sb, &zpl_dentry_operations);
+#else
+	sb->s_d_op = &zpl_dentry_operations;
+#endif
+
 	/* Set features for file system. */
 	zfs_set_fuid_feature(zfsvfs);
 
@@ -1676,7 +1677,7 @@ zfs_umount(struct super_block *sb)
 
 	if (zfsvfs->z_arc_prune != NULL)
 		arc_remove_prune_callback(zfsvfs->z_arc_prune);
-	VERIFY(zfsvfs_teardown(zfsvfs, B_TRUE) == 0);
+	VERIFY0(zfsvfs_teardown(zfsvfs, B_TRUE));
 	os = zfsvfs->z_os;
 
 	/*
@@ -1802,8 +1803,8 @@ zfs_vget(struct super_block *sb, struct inode **ipp, fid_t *fidp)
 		ASSERT(*ipp != NULL);
 
 		if (object == ZFSCTL_INO_SNAPDIR) {
-			VERIFY(zfsctl_root_lookup(*ipp, "snapshot", ipp,
-			    0, kcred, NULL, NULL) == 0);
+			VERIFY0(zfsctl_root_lookup(*ipp, "snapshot", ipp,
+			    0, kcred, NULL, NULL));
 		} else {
 			/*
 			 * Must have an existing ref, so igrab()
@@ -1905,7 +1906,7 @@ zfs_resume_fs(zfsvfs_t *zfsvfs, dsl_dataset_t *ds)
 		goto bail;
 
 	ds->ds_dir->dd_activity_cancelled = B_FALSE;
-	VERIFY(zfsvfs_setup(zfsvfs, B_FALSE) == 0);
+	VERIFY0(zfsvfs_setup(zfsvfs, B_FALSE));
 
 	zfs_set_fuid_feature(zfsvfs);
 	zfsvfs->z_rollback_time = jiffies;
@@ -2078,7 +2079,7 @@ zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers)
 		    ZFS_SA_ATTRS, 8, 1, &sa_obj, tx);
 		ASSERT0(error);
 
-		VERIFY(0 == sa_set_sa_object(os, sa_obj));
+		VERIFY0(sa_set_sa_object(os, sa_obj));
 		sa_register_update_callback(os, zfs_sa_upgrade);
 	}
 
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c
index 6a2fc5ad7935..e845ad69ad78 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c
@@ -841,8 +841,8 @@ out:
 		*zpp = zp;
 	}
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1203,8 +1203,8 @@ out:
 		zfs_zrele_async(xzp);
 	}
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1392,14 +1392,15 @@ out:
 
 	zfs_dirent_unlock(dl);
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
-
 	if (error != 0) {
 		zrele(zp);
 	} else {
 		zfs_znode_update_vfs(dzp);
 		zfs_znode_update_vfs(zp);
+
+		if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+			error = zil_commit(zilog, 0);
+
 	}
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1528,8 +1529,8 @@ out:
 	zfs_znode_update_vfs(zp);
 	zrele(zp);
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -2032,10 +2033,7 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns)
 		goto out3;
 	}
 
-	if ((mask & ATTR_SIZE) && (zp->z_pflags & ZFS_READONLY)) {
-		err = SET_ERROR(EPERM);
-		goto out3;
-	}
+	/* ZFS_READONLY will be handled in zfs_zaccess() */
 
 	/*
 	 * Verify timestamps doesn't overflow 32 bits.
@@ -2483,10 +2481,10 @@ top:
 			new_mode = zp->z_mode;
 		}
 		err = zfs_acl_chown_setattr(zp);
-		ASSERT(err == 0);
+		ASSERT0(err);
 		if (attrzp) {
 			err = zfs_acl_chown_setattr(attrzp);
-			ASSERT(err == 0);
+			ASSERT0(err);
 		}
 	}
 
@@ -2600,7 +2598,7 @@ out:
 	if (err == 0 && xattr_count > 0) {
 		err2 = sa_bulk_update(attrzp->z_sa_hdl, xattr_bulk,
 		    xattr_count, tx);
-		ASSERT(err2 == 0);
+		ASSERT0(err2);
 	}
 
 	if (aclp)
@@ -2630,8 +2628,8 @@ out:
 	}
 
 out2:
-	if (os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (err == 0 && os->os_sync == ZFS_SYNC_ALWAYS)
+		err = zil_commit(zilog, 0);
 
 out3:
 	kmem_free(xattr_bulk, sizeof (sa_bulk_attr_t) * bulks);
@@ -3157,7 +3155,7 @@ top:
 		 * zfs_link_create() to add back the same entry, but with a new
 		 * dnode (szp), should not fail.
 		 */
-		ASSERT3P(tzp, ==, NULL);
+		ASSERT0P(tzp);
 		goto commit_link_tzp;
 	}
 
@@ -3235,8 +3233,8 @@ out:
 	zfs_dirent_unlock(sdl);
 	zfs_dirent_unlock(tdl);
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -3436,7 +3434,7 @@ top:
 		*zpp = zp;
 
 		if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-			zil_commit(zilog, 0);
+			error = zil_commit(zilog, 0);
 	} else {
 		zrele(zp);
 	}
@@ -3654,8 +3652,8 @@ top:
 		 * operation are sync safe.
 		 */
 		if (is_tmpfile) {
-			VERIFY(zap_remove_int(zfsvfs->z_os,
-			    zfsvfs->z_unlinkedobj, szp->z_id, tx) == 0);
+			VERIFY0(zap_remove_int(zfsvfs->z_os,
+			    zfsvfs->z_unlinkedobj, szp->z_id, tx));
 		} else {
 			if (flags & FIGNORECASE)
 				txtype |= TX_CI;
@@ -3670,18 +3668,20 @@ top:
 
 	zfs_dirent_unlock(dl);
 
-	if (!is_tmpfile && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
-
-	if (is_tmpfile && zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
-		txg_wait_flag_t wait_flags =
-		    spa_get_failmode(dmu_objset_spa(zfsvfs->z_os)) ==
-		    ZIO_FAILURE_MODE_CONTINUE ? TXG_WAIT_SUSPEND : 0;
-		error = txg_wait_synced_flags(dmu_objset_pool(zfsvfs->z_os),
-		    txg, wait_flags);
-		if (error != 0) {
-			ASSERT3U(error, ==, ESHUTDOWN);
-			error = SET_ERROR(EIO);
+	if (error == 0) {
+		if (!is_tmpfile && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+			error = zil_commit(zilog, 0);
+
+		if (is_tmpfile && zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
+			txg_wait_flag_t wait_flags =
+			    spa_get_failmode(dmu_objset_spa(zfsvfs->z_os)) ==
+			    ZIO_FAILURE_MODE_CONTINUE ? TXG_WAIT_SUSPEND : 0;
+			error = txg_wait_synced_flags(
+			    dmu_objset_pool(zfsvfs->z_os), txg, wait_flags);
+			if (error != 0) {
+				ASSERT3U(error, ==, ESHUTDOWN);
+				error = SET_ERROR(EIO);
+			}
 		}
 	}
 
@@ -3691,16 +3691,42 @@ top:
 	return (error);
 }
 
-static void
-zfs_putpage_commit_cb(void *arg)
+/* Finish page writeback. */
+static inline void
+zfs_page_writeback_done(struct page *pp, int err)
 {
-	struct page *pp = arg;
+	if (err != 0) {
+		/*
+		 * Writeback failed. Re-dirty the page. It was undirtied before
+		 * the IO was issued (in zfs_putpage() or write_cache_pages()).
+		 * The kernel only considers writeback for dirty pages; if we
+		 * don't do this, it is eligible for eviction without being
+		 * written out, which we definitely don't want.
+		 */
+#ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
+		filemap_dirty_folio(page_mapping(pp), page_folio(pp));
+#else
+		__set_page_dirty_nobuffers(pp);
+#endif
+	}
 
 	ClearPageError(pp);
 	end_page_writeback(pp);
 }
 
 /*
+ * ZIL callback for page writeback. Passes to zfs_log_write() in zfs_putpage()
+ * for syncing writes. Called when the ZIL itx has been written to the log or
+ * the whole txg syncs, or if the ZIL crashes or the pool suspends. Any failure
+ * is passed as `err`.
+ */
+static void
+zfs_putpage_commit_cb(void *arg, int err)
+{
+	zfs_page_writeback_done(arg, err);
+}
+
+/*
  * Push a page out to disk, once the page is on stable storage the
  * registered commit callback will be run as notification of completion.
  *
@@ -3853,16 +3879,15 @@ zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc,
 	err = dmu_tx_assign(tx, DMU_TX_WAIT);
 	if (err != 0) {
 		dmu_tx_abort(tx);
-#ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
-		filemap_dirty_folio(page_mapping(pp), page_folio(pp));
-#else
-		__set_page_dirty_nobuffers(pp);
-#endif
-		ClearPageError(pp);
-		end_page_writeback(pp);
+		zfs_page_writeback_done(pp, err);
 		zfs_rangelock_exit(lr);
 		zfs_exit(zfsvfs, FTAG);
-		return (err);
+
+		/*
+		 * Don't return error for an async writeback; we've re-dirtied
+		 * the page so it will be tried again some other time.
+		 */
+		return (for_sync ? err : 0);
 	}
 
 	va = kmap(pp);
@@ -3916,7 +3941,7 @@ zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc,
 	 * ALL, zfs_putpage should do it.
 	 *
 	 * Summary:
-	 *   for_sync:  0=unlock immediately; 1 unlock once on disk
+	 *   for_sync:  0=unlock immediately; 1=unlock once on disk
 	 *   sync_mode: NONE=caller will commit; ALL=we will commit
 	 */
 	boolean_t need_commit = (wbc->sync_mode != WB_SYNC_NONE);
@@ -3931,16 +3956,24 @@ zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc,
 	    B_FALSE, for_sync ? zfs_putpage_commit_cb : NULL, pp);
 
 	if (!for_sync) {
-		ClearPageError(pp);
-		end_page_writeback(pp);
+		/*
+		 * Async writeback is logged and written to the DMU, so page
+		 * can now be unlocked.
+		 */
+		zfs_page_writeback_done(pp, 0);
 	}
 
 	dmu_tx_commit(tx);
 
 	zfs_rangelock_exit(lr);
 
-	if (need_commit)
-		zil_commit(zfsvfs->z_log, zp->z_id);
+	if (need_commit) {
+		err = zil_commit_flags(zfsvfs->z_log, zp->z_id, ZIL_COMMIT_NOW);
+		if (err != 0) {
+			zfs_exit(zfsvfs, FTAG);
+			return (err);
+		}
+	}
 
 	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, pglen);
 
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c
index 7683eeb3cf9f..bcaabeb32b8a 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c
@@ -144,9 +144,9 @@ zfs_znode_cache_destructor(void *buf, void *arg)
 	rw_destroy(&zp->z_xattr_lock);
 	zfs_rangelock_fini(&zp->z_rangelock);
 
-	ASSERT3P(zp->z_dirlocks, ==, NULL);
-	ASSERT3P(zp->z_acl_cached, ==, NULL);
-	ASSERT3P(zp->z_xattr_cached, ==, NULL);
+	ASSERT0P(zp->z_dirlocks);
+	ASSERT0P(zp->z_acl_cached);
+	ASSERT0P(zp->z_xattr_cached);
 }
 
 static int
@@ -178,13 +178,13 @@ zfs_znode_init(void)
 	 * backed by kmalloc() when on the Linux slab in order that any
 	 * wait_on_bit() operations on the related inode operate properly.
 	 */
-	ASSERT(znode_cache == NULL);
+	ASSERT0P(znode_cache);
 	znode_cache = kmem_cache_create("zfs_znode_cache",
 	    sizeof (znode_t), 0, zfs_znode_cache_constructor,
 	    zfs_znode_cache_destructor, NULL, NULL, NULL,
 	    KMC_SLAB | KMC_RECLAIMABLE);
 
-	ASSERT(znode_hold_cache == NULL);
+	ASSERT0P(znode_hold_cache);
 	znode_hold_cache = kmem_cache_create("zfs_znode_hold_cache",
 	    sizeof (znode_hold_t), 0, zfs_znode_hold_cache_constructor,
 	    zfs_znode_hold_cache_destructor, NULL, NULL, NULL, 0);
@@ -327,10 +327,10 @@ zfs_znode_sa_init(zfsvfs_t *zfsvfs, znode_t *zp,
 
 	mutex_enter(&zp->z_lock);
 
-	ASSERT(zp->z_sa_hdl == NULL);
-	ASSERT(zp->z_acl_cached == NULL);
+	ASSERT0P(zp->z_sa_hdl);
+	ASSERT0P(zp->z_acl_cached);
 	if (sa_hdl == NULL) {
-		VERIFY(0 == sa_handle_get_from_db(zfsvfs->z_os, db, zp,
+		VERIFY0(sa_handle_get_from_db(zfsvfs->z_os, db, zp,
 		    SA_HDL_SHARED, &zp->z_sa_hdl));
 	} else {
 		zp->z_sa_hdl = sa_hdl;
@@ -530,9 +530,9 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
 		return (NULL);
 
 	zp = ITOZ(ip);
-	ASSERT(zp->z_dirlocks == NULL);
-	ASSERT3P(zp->z_acl_cached, ==, NULL);
-	ASSERT3P(zp->z_xattr_cached, ==, NULL);
+	ASSERT0P(zp->z_dirlocks);
+	ASSERT0P(zp->z_acl_cached);
+	ASSERT0P(zp->z_xattr_cached);
 	zp->z_unlinked = B_FALSE;
 	zp->z_atime_dirty = B_FALSE;
 	zp->z_is_ctldir = B_FALSE;
@@ -611,7 +611,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
 	 * processing so do not hash unlinked znodes.
 	 */
 	if (links > 0)
-		VERIFY3S(insert_inode_locked(ip), ==, 0);
+		VERIFY0(insert_inode_locked(ip));
 
 	mutex_enter(&zfsvfs->z_znodes_lock);
 	list_insert_tail(&zfsvfs->z_all_znodes, zp);
@@ -811,7 +811,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr,
 	}
 
 	/* Now add in all of the "SA" attributes */
-	VERIFY(0 == sa_handle_get_from_db(zfsvfs->z_os, db, NULL, SA_HDL_SHARED,
+	VERIFY0(sa_handle_get_from_db(zfsvfs->z_os, db, NULL, SA_HDL_SHARED,
 	    &sa_hdl));
 
 	/*
@@ -901,7 +901,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr,
 		    acl_ids->z_fuid, acl_ids->z_fgid);
 	}
 
-	VERIFY(sa_replace_all_by_template(sa_hdl, sa_attrs, cnt, tx) == 0);
+	VERIFY0(sa_replace_all_by_template(sa_hdl, sa_attrs, cnt, tx));
 
 	if (!(flag & IS_ROOT_NODE)) {
 		/*
@@ -1200,7 +1200,7 @@ zfs_rezget(znode_t *zp)
 	}
 	rw_exit(&zp->z_xattr_lock);
 
-	ASSERT(zp->z_sa_hdl == NULL);
+	ASSERT0P(zp->z_sa_hdl);
 	err = sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db);
 	if (err) {
 		zfs_znode_hold_exit(zfsvfs, zh);
@@ -1314,9 +1314,9 @@ zfs_znode_delete(znode_t *zp, dmu_tx_t *tx)
 	zh = zfs_znode_hold_enter(zfsvfs, obj);
 	if (acl_obj) {
 		VERIFY(!zp->z_is_sa);
-		VERIFY(0 == dmu_object_free(os, acl_obj, tx));
+		VERIFY0(dmu_object_free(os, acl_obj, tx));
 	}
-	VERIFY(0 == dmu_object_free(os, obj, tx));
+	VERIFY0(dmu_object_free(os, obj, tx));
 	zfs_znode_dmu_fini(zp);
 	zfs_znode_hold_exit(zfsvfs, zh);
 }
@@ -1536,7 +1536,7 @@ zfs_extend(znode_t *zp, uint64_t end)
 
 	zp->z_size = end;
 
-	VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(ZTOZSB(zp)),
+	VERIFY0(sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(ZTOZSB(zp)),
 	    &zp->z_size, sizeof (zp->z_size), tx));
 
 	zfs_rangelock_exit(lr);
@@ -1726,7 +1726,7 @@ zfs_trunc(znode_t *zp, uint64_t end)
 		SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs),
 		    NULL, &zp->z_pflags, 8);
 	}
-	VERIFY(sa_bulk_update(zp->z_sa_hdl, bulk, count, tx) == 0);
+	VERIFY0(sa_bulk_update(zp->z_sa_hdl, bulk, count, tx));
 
 	dmu_tx_commit(tx);
 	zfs_rangelock_exit(lr);
@@ -1793,7 +1793,7 @@ log:
 	    NULL, &zp->z_pflags, 8);
 	zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
 	error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	zfs_log_truncate(zilog, tx, TX_TRUNCATE, zp, off, len);
 
@@ -1840,7 +1840,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	moid = MASTER_NODE_OBJ;
 	error = zap_create_claim(os, moid, DMU_OT_MASTER_NODE,
 	    DMU_OT_NONE, 0, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	/*
 	 * Set starting attributes.
@@ -1853,7 +1853,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 		const char *name;
 
 		ASSERT(nvpair_type(elem) == DATA_TYPE_UINT64);
-		VERIFY(nvpair_value_uint64(elem, &val) == 0);
+		VERIFY0(nvpair_value_uint64(elem, &val));
 		name = nvpair_name(elem);
 		if (strcmp(name, zfs_prop_to_name(ZFS_PROP_VERSION)) == 0) {
 			if (val < version)
@@ -1861,7 +1861,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 		} else {
 			error = zap_update(os, moid, name, 8, 1, &val, tx);
 		}
-		ASSERT(error == 0);
+		ASSERT0(error);
 		if (strcmp(name, zfs_prop_to_name(ZFS_PROP_NORMALIZE)) == 0)
 			norm = val;
 		else if (strcmp(name, zfs_prop_to_name(ZFS_PROP_CASE)) == 0)
@@ -1869,7 +1869,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	}
 	ASSERT(version != 0);
 	error = zap_update(os, moid, ZPL_VERSION_STR, 8, 1, &version, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	/*
 	 * Create zap object used for SA attribute registration
@@ -1879,7 +1879,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 		sa_obj = zap_create(os, DMU_OT_SA_MASTER_NODE,
 		    DMU_OT_NONE, 0, tx);
 		error = zap_add(os, moid, ZFS_SA_ATTRS, 8, 1, &sa_obj, tx);
-		ASSERT(error == 0);
+		ASSERT0(error);
 	} else {
 		sa_obj = 0;
 	}
@@ -1889,7 +1889,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	obj = zap_create(os, DMU_OT_UNLINKED_SET, DMU_OT_NONE, 0, tx);
 
 	error = zap_add(os, moid, ZFS_UNLINKED_SET, 8, 1, &obj, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	/*
 	 * Create root znode.  Create minimal znode/inode/zfsvfs/sb
@@ -1922,7 +1922,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	error = sa_setup(os, sa_obj, zfs_attr_table, ZPL_END,
 	    &zfsvfs->z_attr_table);
 
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	/*
 	 * Fold case on file systems that are always or sometimes case
@@ -1946,12 +1946,12 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 		mutex_init(&zfsvfs->z_hold_locks[i], NULL, MUTEX_DEFAULT, NULL);
 	}
 
-	VERIFY(0 == zfs_acl_ids_create(rootzp, IS_ROOT_NODE, &vattr,
+	VERIFY0(zfs_acl_ids_create(rootzp, IS_ROOT_NODE, &vattr,
 	    cr, NULL, &acl_ids, zfs_init_idmap));
 	zfs_mknode(rootzp, &vattr, tx, cr, IS_ROOT_NODE, &zp, &acl_ids);
 	ASSERT3P(zp, ==, rootzp);
 	error = zap_add(os, moid, ZFS_ROOT_OBJ, 8, 1, &rootzp->z_id, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 	zfs_acl_ids_free(&acl_ids);
 
 	atomic_set(&ZTOI(rootzp)->i_count, 0);
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c
index 48dae79a2373..81ac26cb0c93 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c
@@ -202,7 +202,7 @@ zpl_snapdir_revalidate(struct dentry *dentry, unsigned int flags)
 	return (!!dentry->d_inode);
 }
 
-static dentry_operations_t zpl_dops_snapdirs = {
+static const struct dentry_operations zpl_dops_snapdirs = {
 /*
  * Auto mounting of snapshots is only supported for 2.6.37 and
  * newer kernels.  Prior to this kernel the ops->follow_link()
@@ -215,6 +215,51 @@ static dentry_operations_t zpl_dops_snapdirs = {
 	.d_revalidate	= zpl_snapdir_revalidate,
 };
 
+/*
+ * For the .zfs control directory to work properly we must be able to override
+ * the default operations table and register custom .d_automount and
+ * .d_revalidate callbacks.
+ */
+static void
+set_snapdir_dentry_ops(struct dentry *dentry, unsigned int extraflags) {
+	static const unsigned int op_flags =
+	    DCACHE_OP_HASH | DCACHE_OP_COMPARE |
+	    DCACHE_OP_REVALIDATE | DCACHE_OP_DELETE |
+	    DCACHE_OP_PRUNE | DCACHE_OP_WEAK_REVALIDATE | DCACHE_OP_REAL;
+
+#ifdef HAVE_D_SET_D_OP
+	/*
+	 * d_set_d_op() will set the DCACHE_OP_ flags according to what it
+	 * finds in the passed dentry_operations, so we don't have to.
+	 *
+	 * We clear the flags and the old op table before calling d_set_d_op()
+	 * because issues a warning when the dentry operations table is already
+	 * set.
+	 */
+	dentry->d_op = NULL;
+	dentry->d_flags &= ~op_flags;
+	d_set_d_op(dentry, &zpl_dops_snapdirs);
+	dentry->d_flags |= extraflags;
+#else
+	/*
+	 * Since 6.17 there's no exported way to modify dentry ops, so we have
+	 * to reach in and do it ourselves. This should be safe for our very
+	 * narrow use case, which is to create or splice in an entry to give
+	 * access to a snapshot.
+	 *
+	 * We need to set the op flags directly. We hardcode
+	 * DCACHE_OP_REVALIDATE because that's the only operation we have; if
+	 * we ever extend zpl_dops_snapdirs we will need to update the op flags
+	 * to match.
+	 */
+	spin_lock(&dentry->d_lock);
+	dentry->d_op = &zpl_dops_snapdirs;
+	dentry->d_flags &= ~op_flags;
+	dentry->d_flags |= DCACHE_OP_REVALIDATE | extraflags;
+	spin_unlock(&dentry->d_lock);
+#endif
+}
+
 static struct dentry *
 zpl_snapdir_lookup(struct inode *dip, struct dentry *dentry,
     unsigned int flags)
@@ -236,10 +281,7 @@ zpl_snapdir_lookup(struct inode *dip, struct dentry *dentry,
 		return (ERR_PTR(error));
 
 	ASSERT(error == 0 || ip == NULL);
-	d_clear_d_op(dentry);
-	d_set_d_op(dentry, &zpl_dops_snapdirs);
-	dentry->d_flags |= DCACHE_NEED_AUTOMOUNT;
-
+	set_snapdir_dentry_ops(dentry, DCACHE_NEED_AUTOMOUNT);
 	return (d_splice_alias(ip, dentry));
 }
 
@@ -373,8 +415,7 @@ zpl_snapdir_mkdir(struct inode *dip, struct dentry *dentry, umode_t mode)
 
 	error = -zfsctl_snapdir_mkdir(dip, dname(dentry), vap, &ip, cr, 0);
 	if (error == 0) {
-		d_clear_d_op(dentry);
-		d_set_d_op(dentry, &zpl_dops_snapdirs);
+		set_snapdir_dentry_ops(dentry, 0);
 		d_instantiate(dentry, ip);
 	}
 
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
index ef7bd7352084..02965ac8cbee 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
@@ -22,6 +22,8 @@
 /*
  * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
+ * Copyright (c) 2025, Klara, Inc.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  */
 
 
@@ -106,6 +108,10 @@ zpl_iterate(struct file *filp, struct dir_context *ctx)
 	return (error);
 }
 
+static inline int
+zpl_write_cache_pages(struct address_space *mapping,
+    struct writeback_control *wbc, void *data);
+
 static int
 zpl_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
 {
@@ -115,9 +121,38 @@ zpl_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
 	int error;
 	fstrans_cookie_t cookie;
 
-	error = filemap_write_and_wait_range(inode->i_mapping, start, end);
-	if (error)
-		return (error);
+	/*
+	 * Force dirty pages in the range out to the DMU and the log, ready
+	 * for zil_commit() to write down.
+	 *
+	 * We call write_cache_pages() directly to ensure that zpl_putpage() is
+	 * called with the flags we need. We need WB_SYNC_NONE to avoid a call
+	 * to zil_commit() (since we're doing this as a kind of pre-sync); but
+	 * we do need for_sync so that the pages remain in writeback until
+	 * they're on disk, and so that we get an error if the DMU write fails.
+	 */
+	if (filemap_range_has_page(inode->i_mapping, start, end)) {
+		int for_sync = 1;
+		struct writeback_control wbc = {
+			.sync_mode = WB_SYNC_NONE,
+			.nr_to_write = LONG_MAX,
+			.range_start = start,
+			.range_end = end,
+		};
+		error =
+		    zpl_write_cache_pages(inode->i_mapping, &wbc, &for_sync);
+		if (error != 0) {
+			/*
+			 * Unclear what state things are in. zfs_putpage() will
+			 * ensure the pages remain dirty if they haven't been
+			 * written down to the DMU, but because there may be
+			 * nothing logged, we can't assume that zfs_sync() ->
+			 * zil_commit() will give us a useful error. It's
+			 * safest if we just error out here.
+			 */
+			return (error);
+		}
+	}
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
@@ -444,6 +479,7 @@ zpl_putpage(struct page *pp, struct writeback_control *wbc, void *data)
 	return (ret);
 }
 
+#ifdef HAVE_WRITE_CACHE_PAGES
 #ifdef HAVE_WRITEPAGE_T_FOLIO
 static int
 zpl_putfolio(struct folio *pp, struct writeback_control *wbc, void *data)
@@ -465,6 +501,78 @@ zpl_write_cache_pages(struct address_space *mapping,
 #endif
 	return (result);
 }
+#else
+static inline int
+zpl_write_cache_pages(struct address_space *mapping,
+    struct writeback_control *wbc, void *data)
+{
+	pgoff_t start = wbc->range_start >> PAGE_SHIFT;
+	pgoff_t end = wbc->range_end >> PAGE_SHIFT;
+
+	struct folio_batch fbatch;
+	folio_batch_init(&fbatch);
+
+	/*
+	 * This atomically (-ish) tags all DIRTY pages in the range with
+	 * TOWRITE, allowing users to continue dirtying or undirtying pages
+	 * while we get on with writeback, without us treading on each other.
+	 */
+	tag_pages_for_writeback(mapping, start, end);
+
+	int err = 0;
+	unsigned int npages;
+
+	/*
+	 * Grab references to the TOWRITE pages just flagged. This may not get
+	 * all of them, so we do it in a loop until there are none left.
+	 */
+	while ((npages = filemap_get_folios_tag(mapping, &start, end,
+	    PAGECACHE_TAG_TOWRITE, &fbatch)) != 0) {
+
+		/* Loop over each page and write it out. */
+		struct folio *folio;
+		while ((folio = folio_batch_next(&fbatch)) != NULL) {
+			folio_lock(folio);
+
+			/*
+			 * If the folio has been remapped, or is no longer
+			 * dirty, then there's nothing to do.
+			 */
+			if (folio->mapping != mapping ||
+			    !folio_test_dirty(folio)) {
+				folio_unlock(folio);
+				continue;
+			}
+
+			/*
+			 * If writeback is already in progress, wait for it to
+			 * finish. We continue after this even if the page
+			 * ends up clean; zfs_putpage() will skip it if no
+			 * further work is required.
+			 */
+			while (folio_test_writeback(folio))
+				folio_wait_bit(folio, PG_writeback);
+
+			/*
+			 * Write it out and collect any error. zfs_putpage()
+			 * will clear the TOWRITE and DIRTY flags, and return
+			 * with the page unlocked.
+			 */
+			int ferr = zpl_putpage(&folio->page, wbc, data);
+			if (err == 0 && ferr != 0)
+				err = ferr;
+
+			/* Housekeeping for the caller. */
+			wbc->nr_to_write -= folio_nr_pages(folio);
+		}
+
+		/* Release any remaining references on the batch. */
+		folio_batch_release(&fbatch);
+	}
+
+	return (err);
+}
+#endif
 
 static int
 zpl_writepages(struct address_space *mapping, struct writeback_control *wbc)
@@ -494,11 +602,30 @@ zpl_writepages(struct address_space *mapping, struct writeback_control *wbc)
 	if (sync_mode != wbc->sync_mode) {
 		if ((result = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
 			return (result);
-		if (zfsvfs->z_log != NULL)
-			zil_commit(zfsvfs->z_log, zp->z_id);
+
+		if (zfsvfs->z_log != NULL) {
+			/*
+			 * We don't want to block here if the pool suspends,
+			 * because this is not a syncing op by itself, but
+			 * might be part of one that the caller will
+			 * coordinate.
+			 */
+			result = -zil_commit_flags(zfsvfs->z_log, zp->z_id,
+			    ZIL_COMMIT_NOW);
+		}
+
 		zpl_exit(zfsvfs, FTAG);
 
 		/*
+		 * If zil_commit_flags() failed, it's unclear what state things
+		 * are currently in. putpage() has written back out what it can
+		 * to the DMU, but it may not be on disk. We have little choice
+		 * but to escape.
+		 */
+		if (result != 0)
+			return (result);
+
+		/*
 		 * We need to call write_cache_pages() again (we can't just
 		 * return after the commit) because the previous call in
 		 * non-SYNC mode does not guarantee that we got all the dirty
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c
index f9f6406f8b47..f97662d052c7 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c
@@ -247,7 +247,7 @@ zpl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode,
 	 * and fifos, but we want to know if this behavior ever changes.
 	 */
 	if (S_ISSOCK(mode) || S_ISFIFO(mode))
-		ASSERT(rdev == 0);
+		ASSERT0(rdev);
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c
index 94dcdd0b887d..347b352506e5 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c
@@ -22,6 +22,8 @@
 /*
  * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
  * Copyright (c) 2023, Datto Inc. All rights reserved.
+ * Copyright (c) 2025, Klara, Inc.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  */
 
 
@@ -32,7 +34,22 @@
 #include <sys/zpl.h>
 #include <linux/iversion.h>
 #include <linux/version.h>
+#include <linux/vfs_compat.h>
 
+/*
+ * What to do when the last reference to an inode is released. If 0, the kernel
+ * will cache it on the superblock. If 1, the inode will be freed immediately.
+ * See zpl_drop_inode().
+ */
+int zfs_delete_inode = 0;
+
+/*
+ * What to do when the last reference to a dentry is released. If 0, the kernel
+ * will cache it until the entry (file) is destroyed. If 1, the dentry will be
+ * marked for cleanup, at which time its inode reference will be released. See
+ * zpl_dentry_delete().
+ */
+int zfs_delete_dentry = 0;
 
 static struct inode *
 zpl_inode_alloc(struct super_block *sb)
@@ -49,7 +66,7 @@ zpl_inode_alloc(struct super_block *sb)
 static void
 zpl_inode_free(struct inode *ip)
 {
-	ASSERT(atomic_read(&ip->i_count) == 0);
+	ASSERT0(atomic_read(&ip->i_count));
 	zfs_inode_free(ip);
 }
 #endif
@@ -57,7 +74,7 @@ zpl_inode_free(struct inode *ip)
 static void
 zpl_inode_destroy(struct inode *ip)
 {
-	ASSERT(atomic_read(&ip->i_count) == 0);
+	ASSERT0(atomic_read(&ip->i_count));
 	zfs_inode_destroy(ip);
 }
 
@@ -77,11 +94,36 @@ zpl_dirty_inode(struct inode *ip, int flags)
 }
 
 /*
- * When ->drop_inode() is called its return value indicates if the
- * inode should be evicted from the inode cache.  If the inode is
- * unhashed and has no links the default policy is to evict it
- * immediately.
+ * ->drop_inode() is called when the last reference to an inode is released.
+ * Its return value indicates if the inode should be destroyed immediately, or
+ * cached on the superblock structure.
+ *
+ * By default (zfs_delete_inode=0), we call generic_drop_inode(), which returns
+ * "destroy immediately" if the inode is unhashed and has no links (roughly: no
+ * longer exists on disk). On datasets with millions of rarely-accessed files,
+ * this can cause a large amount of memory to be "pinned" by cached inodes,
+ * which in turn pin their associated dnodes and dbufs, until the kernel starts
+ * reporting memory pressure and requests OpenZFS release some memory (see
+ * zfs_prune()).
+ *
+ * When set to 1, we call generic_delete_inode(), which always returns "destroy
+ * immediately", resulting in inodes being destroyed immediately, releasing
+ * their associated dnodes and dbufs to the dbuf cached and the ARC to be
+ * evicted as normal.
  *
+ * Note that the "last reference" doesn't always mean the last _userspace_
+ * reference; the dentry cache also holds a reference, so "busy" inodes will
+ * still be kept alive that way (subject to dcache tuning).
+ */
+static int
+zpl_drop_inode(struct inode *ip)
+{
+	if (zfs_delete_inode)
+		return (generic_delete_inode(ip));
+	return (generic_drop_inode(ip));
+}
+
+/*
  * The ->evict_inode() callback must minimally truncate the inode pages,
  * and call clear_inode().  For 2.6.35 and later kernels this will
  * simply update the inode state, with the sync occurring before the
@@ -470,6 +512,7 @@ const struct super_operations zpl_super_operations = {
 	.destroy_inode		= zpl_inode_destroy,
 	.dirty_inode		= zpl_dirty_inode,
 	.write_inode		= NULL,
+	.drop_inode		= zpl_drop_inode,
 	.evict_inode		= zpl_evict_inode,
 	.put_super		= zpl_put_super,
 	.sync_fs		= zpl_sync_fs,
@@ -480,6 +523,35 @@ const struct super_operations zpl_super_operations = {
 	.show_stats		= NULL,
 };
 
+/*
+ * ->d_delete() is called when the last reference to a dentry is released. Its
+ *  return value indicates if the dentry should be destroyed immediately, or
+ *  retained in the dentry cache.
+ *
+ * By default (zfs_delete_dentry=0) the kernel will always cache unused
+ * entries.  Each dentry holds an inode reference, so cached dentries can hold
+ * the final inode reference indefinitely, leading to the inode and its related
+ * data being pinned (see zpl_drop_inode()).
+ *
+ * When set to 1, we signal that the dentry should be destroyed immediately and
+ * never cached. This reduces memory usage, at the cost of higher overheads to
+ * lookup a file, as the inode and its underlying data (dnode/dbuf) need to be
+ * reloaded and reinflated.
+ *
+ * Note that userspace does not have direct control over dentry references and
+ * reclaim; rather, this is part of the kernel's caching and reclaim subsystems
+ * (eg vm.vfs_cache_pressure).
+ */
+static int
+zpl_dentry_delete(const struct dentry *dentry)
+{
+	return (zfs_delete_dentry ? 1 : 0);
+}
+
+const struct dentry_operations zpl_dentry_operations = {
+	.d_delete = zpl_dentry_delete,
+};
+
 struct file_system_type zpl_fs_type = {
 	.owner			= THIS_MODULE,
 	.name			= ZFS_DRIVER,
@@ -491,3 +563,10 @@ struct file_system_type zpl_fs_type = {
 	.mount			= zpl_mount,
 	.kill_sb		= zpl_kill_sb,
 };
+
+ZFS_MODULE_PARAM(zfs, zfs_, delete_inode, INT, ZMOD_RW,
+	"Delete inodes as soon as the last reference is released.");
+
+ZFS_MODULE_PARAM(zfs, zfs_, delete_dentry, INT, ZMOD_RW,
+	"Delete dentries from dentry cache as soon as the last reference is "
+	"released.");
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c
index a098197e7448..d93282db815a 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c
@@ -1494,7 +1494,7 @@ zpl_posix_acl_free(void *arg)
 				acl_rel_head = NULL;
 				if (cmpxchg(&acl_rel_tail, &a->next,
 				    &acl_rel_head) == &a->next) {
-					ASSERT3P(a->next, ==, NULL);
+					ASSERT0P(a->next);
 					a->next = freelist;
 					freelist = a;
 					break;
@@ -1544,7 +1544,7 @@ zpl_posix_acl_release_impl(struct posix_acl *acl)
 	a->time = ddi_get_lbolt();
 	/* atomically points tail to us and get the previous tail */
 	prev = xchg(&acl_rel_tail, &a->next);
-	ASSERT3P(*prev, ==, NULL);
+	ASSERT0P(*prev);
 	*prev = a;
 	/* if it was empty before, schedule the free task */
 	if (prev == &acl_rel_head)
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
index a7431cc4da9d..fe939150b641 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
@@ -21,8 +21,8 @@
  */
 /*
  * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
- * Copyright (c) 2024, Rob Norris <robn@despairlabs.com>
- * Copyright (c) 2024, Klara, Inc.
+ * Copyright (c) 2024, 2025, Rob Norris <robn@despairlabs.com>
+ * Copyright (c) 2024, 2025, Klara, Inc.
  */
 
 #include <sys/dataset_kstats.h>
@@ -84,8 +84,9 @@ static unsigned int zvol_blk_mq_blocks_per_thread = 8;
 static inline void
 zvol_end_io(struct bio *bio, struct request *rq, int error)
 {
+	ASSERT3U(error, >=, 0);
 	if (bio) {
-		bio->bi_status = errno_to_bi_status(-error);
+		bio->bi_status = errno_to_bi_status(error);
 		bio_endio(bio);
 	} else {
 		blk_mq_end_request(rq, errno_to_bi_status(error));
@@ -208,8 +209,14 @@ zvol_write(zv_request_t *zvr)
 	disk = zv->zv_zso->zvo_disk;
 
 	/* bio marked as FLUSH need to flush before write */
-	if (io_is_flush(bio, rq))
-		zil_commit(zv->zv_zilog, ZVOL_OBJ);
+	if (io_is_flush(bio, rq)) {
+		error = zil_commit(zv->zv_zilog, ZVOL_OBJ);
+		if (error != 0) {
+			rw_exit(&zv->zv_suspend_lock);
+			zvol_end_io(bio, rq, -error);
+			return;
+		}
+	}
 
 	/* Some requests are just for flush and nothing else. */
 	if (io_size(bio, rq) == 0) {
@@ -273,8 +280,8 @@ zvol_write(zv_request_t *zvr)
 	dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
 	task_io_account_write(nwritten);
 
-	if (sync)
-		zil_commit(zv->zv_zilog, ZVOL_OBJ);
+	if (error == 0 && sync)
+		error = zil_commit(zv->zv_zilog, ZVOL_OBJ);
 
 	rw_exit(&zv->zv_suspend_lock);
 
@@ -282,7 +289,7 @@ zvol_write(zv_request_t *zvr)
 		blk_generic_end_io_acct(q, disk, WRITE, bio, start_time);
 	}
 
-	zvol_end_io(bio, rq, -error);
+	zvol_end_io(bio, rq, error);
 }
 
 static void
@@ -330,16 +337,14 @@ zvol_discard(zv_request_t *zvr)
 	}
 
 	/*
-	 * Align the request to volume block boundaries when a secure erase is
-	 * not required.  This will prevent dnode_free_range() from zeroing out
-	 * the unaligned parts which is slow (read-modify-write) and useless
-	 * since we are not freeing any space by doing so.
+	 * Align the request to volume block boundaries. This will prevent
+	 * dnode_free_range() from zeroing out the unaligned parts which is
+	 * slow (read-modify-write) and useless since we are not freeing any
+	 * space by doing so.
 	 */
-	if (!io_is_secure_erase(bio, rq)) {
-		start = P2ROUNDUP(start, zv->zv_volblocksize);
-		end = P2ALIGN_TYPED(end, zv->zv_volblocksize, uint64_t);
-		size = end - start;
-	}
+	start = P2ROUNDUP(start, zv->zv_volblocksize);
+	end = P2ALIGN_TYPED(end, zv->zv_volblocksize, uint64_t);
+	size = end - start;
 
 	if (start >= end)
 		goto unlock;
@@ -361,7 +366,7 @@ zvol_discard(zv_request_t *zvr)
 	zfs_rangelock_exit(lr);
 
 	if (error == 0 && sync)
-		zil_commit(zv->zv_zilog, ZVOL_OBJ);
+		error = zil_commit(zv->zv_zilog, ZVOL_OBJ);
 
 unlock:
 	rw_exit(&zv->zv_suspend_lock);
@@ -371,7 +376,7 @@ unlock:
 		    start_time);
 	}
 
-	zvol_end_io(bio, rq, -error);
+	zvol_end_io(bio, rq, error);
 }
 
 static void
@@ -449,7 +454,7 @@ zvol_read(zv_request_t *zvr)
 		blk_generic_end_io_acct(q, disk, READ, bio, start_time);
 	}
 
-	zvol_end_io(bio, rq, -error);
+	zvol_end_io(bio, rq, error);
 }
 
 static void
@@ -460,6 +465,24 @@ zvol_read_task(void *arg)
 	zv_request_task_free(task);
 }
 
+/*
+ * Note:
+ *
+ * The kernel uses different enum names for the IO opcode, depending on the
+ * kernel version ('req_opf', 'req_op').  To sidestep this, use macros rather
+ * than inline functions for these checks.
+ */
+/* Should this IO go down the zvol write path? */
+#define	ZVOL_OP_IS_WRITE(op) \
+	(op == REQ_OP_WRITE || \
+	op == REQ_OP_FLUSH || \
+	op == REQ_OP_DISCARD)
+
+/* Is this IO type supported by zvols? */
+#define	ZVOL_OP_IS_SUPPORTED(op) (op == REQ_OP_READ || ZVOL_OP_IS_WRITE(op))
+
+/* Get the IO opcode */
+#define	ZVOL_OP(bio, rq) (bio != NULL ? bio_op(bio) : req_op(rq))
 
 /*
  * Process a BIO or request
@@ -477,10 +500,36 @@ zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq,
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 	uint64_t offset = io_offset(bio, rq);
 	uint64_t size = io_size(bio, rq);
-	int rw = io_data_dir(bio, rq);
+	int rw;
+
+	if (unlikely(!ZVOL_OP_IS_SUPPORTED(ZVOL_OP(bio, rq)))) {
+		zfs_dbgmsg("Unsupported zvol %s, op=%d, flags=0x%x",
+		    rq != NULL ? "request" : "BIO",
+		    ZVOL_OP(bio, rq),
+		    rq != NULL ? rq->cmd_flags : bio->bi_opf);
+		ASSERT(ZVOL_OP_IS_SUPPORTED(ZVOL_OP(bio, rq)));
+		zvol_end_io(bio, rq, SET_ERROR(ENOTSUPP));
+		goto out;
+	}
+
+	if (ZVOL_OP_IS_WRITE(ZVOL_OP(bio, rq))) {
+		rw = WRITE;
+	} else {
+		rw = READ;
+	}
+
+	/*
+	 * Sanity check
+	 *
+	 * If we're a BIO, check our rw matches the kernel's
+	 * bio_data_dir(bio) rw.  We need to check because we support fewer
+	 * IO operations, and want to verify that what we think are reads and
+	 * writes from those operations match what the kernel thinks.
+	 */
+	ASSERT(rq != NULL || rw == bio_data_dir(bio));
 
 	if (unlikely(zv->zv_flags & ZVOL_REMOVING)) {
-		zvol_end_io(bio, rq, -SET_ERROR(ENXIO));
+		zvol_end_io(bio, rq, SET_ERROR(ENXIO));
 		goto out;
 	}
 
@@ -499,7 +548,7 @@ zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq,
 		    (long long unsigned)offset,
 		    (long unsigned)size);
 
-		zvol_end_io(bio, rq, -SET_ERROR(EIO));
+		zvol_end_io(bio, rq, SET_ERROR(EIO));
 		goto out;
 	}
 
@@ -512,8 +561,8 @@ zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq,
 #ifdef HAVE_BLK_MQ_RQ_HCTX
 		blk_mq_hw_queue = rq->mq_hctx->queue_num;
 #else
-		blk_mq_hw_queue =
-		    rq->q->queue_hw_ctx[rq->q->mq_map[rq->cpu]]->queue_num;
+		blk_mq_hw_queue = rq->q->queue_hw_ctx[
+		    rq->q->mq_map[raw_smp_processor_id()]]->queue_num;
 #endif
 	taskq_hash = cityhash3((uintptr_t)zv, offset >> ZVOL_TASKQ_OFFSET_SHIFT,
 	    blk_mq_hw_queue);
@@ -521,7 +570,7 @@ zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq,
 
 	if (rw == WRITE) {
 		if (unlikely(zv->zv_flags & ZVOL_RDONLY)) {
-			zvol_end_io(bio, rq, -SET_ERROR(EROFS));
+			zvol_end_io(bio, rq, SET_ERROR(EROFS));
 			goto out;
 		}
 
@@ -582,7 +631,7 @@ zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq,
 		 * interfaces lack this functionality (they block waiting for
 		 * the i/o to complete).
 		 */
-		if (io_is_discard(bio, rq) || io_is_secure_erase(bio, rq)) {
+		if (io_is_discard(bio, rq)) {
 			if (force_sync) {
 				zvol_discard(&zvr);
 			} else {
@@ -672,28 +721,19 @@ zvol_open(struct block_device *bdev, fmode_t flag)
 
 retry:
 #endif
-	rw_enter(&zvol_state_lock, RW_READER);
-	/*
-	 * Obtain a copy of private_data under the zvol_state_lock to make
-	 * sure that either the result of zvol free code path setting
-	 * disk->private_data to NULL is observed, or zvol_os_free()
-	 * is not called on this zv because of the positive zv_open_count.
-	 */
+
 #ifdef HAVE_BLK_MODE_T
-	zv = disk->private_data;
+	zv = atomic_load_ptr(&disk->private_data);
 #else
-	zv = bdev->bd_disk->private_data;
+	zv = atomic_load_ptr(&bdev->bd_disk->private_data);
 #endif
 	if (zv == NULL) {
-		rw_exit(&zvol_state_lock);
 		return (-SET_ERROR(ENXIO));
 	}
 
 	mutex_enter(&zv->zv_state_lock);
-
 	if (unlikely(zv->zv_flags & ZVOL_REMOVING)) {
 		mutex_exit(&zv->zv_state_lock);
-		rw_exit(&zvol_state_lock);
 		return (-SET_ERROR(ENXIO));
 	}
 
@@ -705,8 +745,28 @@ retry:
 	if (zv->zv_open_count == 0) {
 		if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
 			mutex_exit(&zv->zv_state_lock);
+
+			/*
+			 * Removal may happen while the locks are down, so
+			 * we can't trust zv any longer; we have to start over.
+			 */
+#ifdef HAVE_BLK_MODE_T
+			zv = atomic_load_ptr(&disk->private_data);
+#else
+			zv = atomic_load_ptr(&bdev->bd_disk->private_data);
+#endif
+			if (zv == NULL)
+				return (-SET_ERROR(ENXIO));
+
 			rw_enter(&zv->zv_suspend_lock, RW_READER);
 			mutex_enter(&zv->zv_state_lock);
+
+			if (unlikely(zv->zv_flags & ZVOL_REMOVING)) {
+				mutex_exit(&zv->zv_state_lock);
+				rw_exit(&zv->zv_suspend_lock);
+				return (-SET_ERROR(ENXIO));
+			}
+
 			/* check to see if zv_suspend_lock is needed */
 			if (zv->zv_open_count != 0) {
 				rw_exit(&zv->zv_suspend_lock);
@@ -717,7 +777,6 @@ retry:
 			drop_suspend = B_TRUE;
 		}
 	}
-	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
@@ -814,11 +873,11 @@ zvol_release(struct gendisk *disk, fmode_t unused)
 #if !defined(HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG)
 	(void) unused;
 #endif
-	zvol_state_t *zv;
 	boolean_t drop_suspend = B_TRUE;
 
-	rw_enter(&zvol_state_lock, RW_READER);
-	zv = disk->private_data;
+	zvol_state_t *zv = atomic_load_ptr(&disk->private_data);
+	if (zv == NULL)
+		return;
 
 	mutex_enter(&zv->zv_state_lock);
 	ASSERT3U(zv->zv_open_count, >, 0);
@@ -832,6 +891,15 @@ zvol_release(struct gendisk *disk, fmode_t unused)
 			mutex_exit(&zv->zv_state_lock);
 			rw_enter(&zv->zv_suspend_lock, RW_READER);
 			mutex_enter(&zv->zv_state_lock);
+
+			/*
+			 * Unlike in zvol_open(), we don't check if removal
+			 * started here, because we might be one of the openers
+			 * that needs to be thrown out! If we're the last, we
+			 * need to call zvol_last_close() below to finish
+			 * cleanup. So, no special treatment for us.
+			 */
+
 			/* check to see if zv_suspend_lock is needed */
 			if (zv->zv_open_count != 1) {
 				rw_exit(&zv->zv_suspend_lock);
@@ -841,7 +909,6 @@ zvol_release(struct gendisk *disk, fmode_t unused)
 	} else {
 		drop_suspend = B_FALSE;
 	}
-	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
@@ -861,9 +928,10 @@ static int
 zvol_ioctl(struct block_device *bdev, fmode_t mode,
     unsigned int cmd, unsigned long arg)
 {
-	zvol_state_t *zv = bdev->bd_disk->private_data;
 	int error = 0;
 
+	zvol_state_t *zv = atomic_load_ptr(&bdev->bd_disk->private_data);
+	ASSERT3P(zv, !=, NULL);
 	ASSERT3U(zv->zv_open_count, >, 0);
 
 	switch (cmd) {
@@ -886,16 +954,18 @@ zvol_ioctl(struct block_device *bdev, fmode_t mode,
 
 	case BLKZNAME:
 		mutex_enter(&zv->zv_state_lock);
-		error = copy_to_user((void *)arg, zv->zv_name, MAXNAMELEN);
+		error = -copy_to_user((void *)arg, zv->zv_name, MAXNAMELEN);
 		mutex_exit(&zv->zv_state_lock);
+		if (error)
+			error = SET_ERROR(error);
 		break;
 
 	default:
-		error = -ENOTTY;
+		error = SET_ERROR(ENOTTY);
 		break;
 	}
 
-	return (SET_ERROR(error));
+	return (-error);
 }
 
 #ifdef CONFIG_COMPAT
@@ -914,9 +984,8 @@ zvol_check_events(struct gendisk *disk, unsigned int clearing)
 {
 	unsigned int mask = 0;
 
-	rw_enter(&zvol_state_lock, RW_READER);
+	zvol_state_t *zv = atomic_load_ptr(&disk->private_data);
 
-	zvol_state_t *zv = disk->private_data;
 	if (zv != NULL) {
 		mutex_enter(&zv->zv_state_lock);
 		mask = zv->zv_changed ? DISK_EVENT_MEDIA_CHANGE : 0;
@@ -924,17 +993,14 @@ zvol_check_events(struct gendisk *disk, unsigned int clearing)
 		mutex_exit(&zv->zv_state_lock);
 	}
 
-	rw_exit(&zvol_state_lock);
-
 	return (mask);
 }
 
 static int
 zvol_revalidate_disk(struct gendisk *disk)
 {
-	rw_enter(&zvol_state_lock, RW_READER);
+	zvol_state_t *zv = atomic_load_ptr(&disk->private_data);
 
-	zvol_state_t *zv = disk->private_data;
 	if (zv != NULL) {
 		mutex_enter(&zv->zv_state_lock);
 		set_capacity(zv->zv_zso->zvo_disk,
@@ -942,8 +1008,6 @@ zvol_revalidate_disk(struct gendisk *disk)
 		mutex_exit(&zv->zv_state_lock);
 	}
 
-	rw_exit(&zvol_state_lock);
-
 	return (0);
 }
 
@@ -962,28 +1026,19 @@ zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize)
 	return (0);
 }
 
-void
-zvol_os_clear_private(zvol_state_t *zv)
-{
-	/*
-	 * Cleared while holding zvol_state_lock as a writer
-	 * which will prevent zvol_open() from opening it.
-	 */
-	zv->zv_zso->zvo_disk->private_data = NULL;
-}
-
 /*
  * Provide a simple virtual geometry for legacy compatibility.  For devices
  * smaller than 1 MiB a small head and sector count is used to allow very
  * tiny devices.  For devices over 1 Mib a standard head and sector count
  * is used to keep the cylinders count reasonable.
  */
-static int
-zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo)
+static inline int
+zvol_getgeo_impl(struct gendisk *disk, struct hd_geometry *geo)
 {
-	zvol_state_t *zv = bdev->bd_disk->private_data;
+	zvol_state_t *zv = atomic_load_ptr(&disk->private_data);
 	sector_t sectors;
 
+	ASSERT3P(zv, !=, NULL);
 	ASSERT3U(zv->zv_open_count, >, 0);
 
 	sectors = get_capacity(zv->zv_zso->zvo_disk);
@@ -1002,6 +1057,20 @@ zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 	return (0);
 }
 
+#ifdef HAVE_BLOCK_DEVICE_OPERATIONS_GETGEO_GENDISK
+static int
+zvol_getgeo(struct gendisk *disk, struct hd_geometry *geo)
+{
+	return (zvol_getgeo_impl(disk, geo));
+}
+#else
+static int
+zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo)
+{
+	return (zvol_getgeo_impl(bdev->bd_disk, geo));
+}
+#endif
+
 /*
  * Why have two separate block_device_operations structs?
  *
@@ -1408,53 +1477,70 @@ out_kmem:
 	return (ret);
 }
 
-/*
- * Cleanup then free a zvol_state_t which was created by zvol_alloc().
- * At this time, the structure is not opened by anyone, is taken off
- * the zvol_state_list, and has its private data set to NULL.
- * The zvol_state_lock is dropped.
- *
- * This function may take many milliseconds to complete (e.g. we've seen
- * it take over 256ms), due to the calls to "blk_cleanup_queue" and
- * "del_gendisk". Thus, consumers need to be careful to account for this
- * latency when calling this function.
- */
 void
-zvol_os_free(zvol_state_t *zv)
+zvol_os_remove_minor(zvol_state_t *zv)
 {
-
-	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
-	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
+	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 	ASSERT0(zv->zv_open_count);
-	ASSERT3P(zv->zv_zso->zvo_disk->private_data, ==, NULL);
+	ASSERT0(atomic_read(&zv->zv_suspend_ref));
+	ASSERT(zv->zv_flags & ZVOL_REMOVING);
 
-	rw_destroy(&zv->zv_suspend_lock);
-	zfs_rangelock_fini(&zv->zv_rangelock);
+	struct zvol_state_os *zso = zv->zv_zso;
+	zv->zv_zso = NULL;
 
-	del_gendisk(zv->zv_zso->zvo_disk);
+	/* Clearing private_data will make new callers return immediately. */
+	atomic_store_ptr(&zso->zvo_disk->private_data, NULL);
+
+	/*
+	 * Drop the state lock before calling del_gendisk(). There may be
+	 * callers waiting to acquire it, but del_gendisk() will block until
+	 * they exit, which would deadlock.
+	 */
+	mutex_exit(&zv->zv_state_lock);
+
+	del_gendisk(zso->zvo_disk);
 #if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) && \
 	(defined(HAVE_BLK_ALLOC_DISK) || defined(HAVE_BLK_ALLOC_DISK_2ARG))
 #if defined(HAVE_BLK_CLEANUP_DISK)
-	blk_cleanup_disk(zv->zv_zso->zvo_disk);
+	blk_cleanup_disk(zso->zvo_disk);
 #else
-	put_disk(zv->zv_zso->zvo_disk);
+	put_disk(zso->zvo_disk);
 #endif
 #else
-	blk_cleanup_queue(zv->zv_zso->zvo_queue);
-	put_disk(zv->zv_zso->zvo_disk);
+	blk_cleanup_queue(zso->zvo_queue);
+	put_disk(zso->zvo_disk);
 #endif
 
-	if (zv->zv_zso->use_blk_mq)
-		blk_mq_free_tag_set(&zv->zv_zso->tag_set);
+	if (zso->use_blk_mq)
+		blk_mq_free_tag_set(&zso->tag_set);
 
-	ida_simple_remove(&zvol_ida,
-	    MINOR(zv->zv_zso->zvo_dev) >> ZVOL_MINOR_BITS);
+	ida_free(&zvol_ida, MINOR(zso->zvo_dev) >> ZVOL_MINOR_BITS);
+
+	kmem_free(zso, sizeof (struct zvol_state_os));
+
+	mutex_enter(&zv->zv_state_lock);
+}
+
+void
+zvol_os_free(zvol_state_t *zv)
+{
+
+	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
+	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
+	ASSERT0(zv->zv_open_count);
+	ASSERT0P(zv->zv_zso);
+
+	ASSERT0P(zv->zv_objset);
+	ASSERT0P(zv->zv_zilog);
+	ASSERT0P(zv->zv_dn);
+
+	rw_destroy(&zv->zv_suspend_lock);
+	zfs_rangelock_fini(&zv->zv_rangelock);
 
 	cv_destroy(&zv->zv_removing_cv);
 	mutex_destroy(&zv->zv_state_lock);
 	dataset_kstats_destroy(&zv->zv_kstat);
 
-	kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
 	kmem_free(zv, sizeof (zvol_state_t));
 }
 
@@ -1474,7 +1560,9 @@ __zvol_os_add_disk(struct gendisk *disk)
 {
 	int error = 0;
 #ifdef HAVE_ADD_DISK_RET
-	error = add_disk(disk);
+	error = -add_disk(disk);
+	if (error)
+		error = SET_ERROR(error);
 #else
 	add_disk(disk);
 #endif
@@ -1581,7 +1669,7 @@ zvol_os_create_minor(const char *name)
 	if (zvol_inhibit_dev)
 		return (0);
 
-	idx = ida_simple_get(&zvol_ida, 0, 0, kmem_flags_convert(KM_SLEEP));
+	idx = ida_alloc(&zvol_ida, kmem_flags_convert(KM_SLEEP));
 	if (idx < 0)
 		return (SET_ERROR(-idx));
 	minor = idx << ZVOL_MINOR_BITS;
@@ -1589,7 +1677,7 @@ zvol_os_create_minor(const char *name)
 		/* too many partitions can cause an overflow */
 		zfs_dbgmsg("zvol: create minor overflow: %s, minor %u/%u",
 		    name, minor, MINOR(minor));
-		ida_simple_remove(&zvol_ida, idx);
+		ida_free(&zvol_ida, idx);
 		return (SET_ERROR(EINVAL));
 	}
 
@@ -1597,7 +1685,7 @@ zvol_os_create_minor(const char *name)
 	if (zv) {
 		ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 		mutex_exit(&zv->zv_state_lock);
-		ida_simple_remove(&zvol_ida, idx);
+		ida_free(&zvol_ida, idx);
 		return (SET_ERROR(EEXIST));
 	}
 
@@ -1649,11 +1737,11 @@ zvol_os_create_minor(const char *name)
 	blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, zv->zv_zso->zvo_queue);
 #endif
 
-	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
+	ASSERT0P(zv->zv_kstat.dk_kstats);
 	error = dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
 	if (error)
 		goto out_dmu_objset_disown;
-	ASSERT3P(zv->zv_zilog, ==, NULL);
+	ASSERT0P(zv->zv_zilog);
 	zv->zv_zilog = zil_open(os, zvol_get_data, &zv->zv_kstat.dk_zil_sums);
 	if (spa_writeable(dmu_objset_spa(os))) {
 		if (zil_replay_disable)
@@ -1697,7 +1785,7 @@ out_doi:
 		rw_exit(&zvol_state_lock);
 		error = zvol_os_add_disk(zv->zv_zso->zvo_disk);
 	} else {
-		ida_simple_remove(&zvol_ida, idx);
+		ida_free(&zvol_ida, idx);
 	}
 
 	return (error);
@@ -1759,10 +1847,10 @@ zvol_init(void)
 		return (error);
 	}
 
-	error = register_blkdev(zvol_major, ZVOL_DRIVER);
+	error = -register_blkdev(zvol_major, ZVOL_DRIVER);
 	if (error) {
 		printk(KERN_INFO "ZFS: register_blkdev() failed %d\n", error);
-		return (error);
+		return (SET_ERROR(error));
 	}
 
 	if (zvol_blk_mq_queue_depth == 0) {
diff --git a/sys/contrib/openzfs/module/zcommon/simd_stat.c b/sys/contrib/openzfs/module/zcommon/simd_stat.c
index 11e2080ff9f2..007ae9e4fbbc 100644
--- a/sys/contrib/openzfs/module/zcommon/simd_stat.c
+++ b/sys/contrib/openzfs/module/zcommon/simd_stat.c
@@ -118,6 +118,10 @@ simd_stat_kstat_data(char *buf, size_t size, void *data)
 		    "pclmulqdq", zfs_pclmulqdq_available());
 		off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
 		    "movbe", zfs_movbe_available());
+		off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
+		    "vaes", zfs_vaes_available());
+		off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
+		    "vpclmulqdq", zfs_vpclmulqdq_available());
 
 		off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
 		    "osxsave", boot_cpu_has(X86_FEATURE_OSXSAVE));
diff --git a/sys/contrib/openzfs/module/zcommon/zfs_deleg.c b/sys/contrib/openzfs/module/zcommon/zfs_deleg.c
index 49bb534ca26c..87596558c9a1 100644
--- a/sys/contrib/openzfs/module/zcommon/zfs_deleg.c
+++ b/sys/contrib/openzfs/module/zcommon/zfs_deleg.c
@@ -59,6 +59,7 @@ const zfs_deleg_perm_tab_t zfs_deleg_perm_tab[] = {
 	{ZFS_DELEG_PERM_SNAPSHOT},
 	{ZFS_DELEG_PERM_SHARE},
 	{ZFS_DELEG_PERM_SEND},
+	{ZFS_DELEG_PERM_SEND_RAW},
 	{ZFS_DELEG_PERM_USERPROP},
 	{ZFS_DELEG_PERM_USERQUOTA},
 	{ZFS_DELEG_PERM_GROUPQUOTA},
diff --git a/sys/contrib/openzfs/module/zcommon/zfs_prop.c b/sys/contrib/openzfs/module/zcommon/zfs_prop.c
index 864e3898b365..9190ae0362ea 100644
--- a/sys/contrib/openzfs/module/zcommon/zfs_prop.c
+++ b/sys/contrib/openzfs/module/zcommon/zfs_prop.c
@@ -364,8 +364,8 @@ zfs_prop_init(void)
 
 	static const zprop_index_t xattr_table[] = {
 		{ "off",	ZFS_XATTR_OFF },
-		{ "on",		ZFS_XATTR_SA },
 		{ "sa",		ZFS_XATTR_SA },
+		{ "on",		ZFS_XATTR_SA },
 		{ "dir",	ZFS_XATTR_DIR },
 		{ NULL }
 	};
diff --git a/sys/contrib/openzfs/module/zcommon/zpool_prop.c b/sys/contrib/openzfs/module/zcommon/zpool_prop.c
index 04ae9f986d8f..07819ba2be8b 100644
--- a/sys/contrib/openzfs/module/zcommon/zpool_prop.c
+++ b/sys/contrib/openzfs/module/zcommon/zpool_prop.c
@@ -467,9 +467,15 @@ vdev_prop_init(void)
 	zprop_register_index(VDEV_PROP_RAIDZ_EXPANDING, "raidz_expanding", 0,
 	    PROP_READONLY, ZFS_TYPE_VDEV, "on | off", "RAIDZ_EXPANDING",
 	    boolean_table, sfeatures);
+	zprop_register_index(VDEV_PROP_SIT_OUT, "sit_out", 0,
+	    PROP_DEFAULT, ZFS_TYPE_VDEV, "on | off", "SIT_OUT", boolean_table,
+	    sfeatures);
 	zprop_register_index(VDEV_PROP_TRIM_SUPPORT, "trim_support", 0,
 	    PROP_READONLY, ZFS_TYPE_VDEV, "on | off", "TRIMSUP",
 	    boolean_table, sfeatures);
+	zprop_register_index(VDEV_PROP_AUTOSIT, "autosit", 0,
+	    PROP_DEFAULT, ZFS_TYPE_VDEV, "on | off", "AUTOSIT", boolean_table,
+	    sfeatures);
 
 	/* default index properties */
 	zprop_register_index(VDEV_PROP_FAILFAST, "failfast", B_TRUE,
diff --git a/sys/contrib/openzfs/module/zfs/abd.c b/sys/contrib/openzfs/module/zfs/abd.c
index 826928e67350..bf9b13c30509 100644
--- a/sys/contrib/openzfs/module/zfs/abd.c
+++ b/sys/contrib/openzfs/module/zfs/abd.c
@@ -563,7 +563,7 @@ abd_get_offset_impl(abd_t *abd, abd_t *sabd, size_t off, size_t size)
 			left -= csize;
 			off = 0;
 		}
-		ASSERT3U(left, ==, 0);
+		ASSERT0(left);
 	} else {
 		abd = abd_get_offset_scatter(abd, sabd, off, size);
 	}
diff --git a/sys/contrib/openzfs/module/zfs/arc.c b/sys/contrib/openzfs/module/zfs/arc.c
index 3483be64ec57..dbb5e942e2e6 100644
--- a/sys/contrib/openzfs/module/zfs/arc.c
+++ b/sys/contrib/openzfs/module/zfs/arc.c
@@ -1157,7 +1157,7 @@ buf_fini(void)
 #if defined(_KERNEL)
 	/*
 	 * Large allocations which do not require contiguous pages
-	 * should be using vmem_free() in the linux kernel\
+	 * should be using vmem_free() in the linux kernel.
 	 */
 	vmem_free(buf_hash_table.ht_table,
 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
@@ -1392,6 +1392,7 @@ arc_get_complevel(arc_buf_t *buf)
 	return (buf->b_hdr->b_complevel);
 }
 
+__maybe_unused
 static inline boolean_t
 arc_buf_is_shared(arc_buf_t *buf)
 {
@@ -2239,8 +2240,8 @@ arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	if (GHOST_STATE(state)) {
-		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
-		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_buf);
+		ASSERT0P(hdr->b_l1hdr.b_pabd);
 		ASSERT(!HDR_HAS_RABD(hdr));
 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
 		    HDR_GET_LSIZE(hdr), hdr);
@@ -2278,8 +2279,8 @@ arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	if (GHOST_STATE(state)) {
-		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
-		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_buf);
+		ASSERT0P(hdr->b_l1hdr.b_pabd);
 		ASSERT(!HDR_HAS_RABD(hdr));
 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
 		    HDR_GET_LSIZE(hdr), hdr);
@@ -2319,7 +2320,7 @@ add_reference(arc_buf_hdr_t *hdr, const void *tag)
 	if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {
 		ASSERT(state == arc_anon);
 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
-		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_buf);
 	}
 
 	if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
@@ -2503,7 +2504,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
 			(void) zfs_refcount_add_many(
 			    &new_state->arcs_size[type],
 			    HDR_GET_LSIZE(hdr), hdr);
-			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+			ASSERT0P(hdr->b_l1hdr.b_pabd);
 			ASSERT(!HDR_HAS_RABD(hdr));
 		} else {
 
@@ -2547,7 +2548,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
 	if (update_old && old_state != arc_l2c_only) {
 		ASSERT(HDR_HAS_L1HDR(hdr));
 		if (GHOST_STATE(old_state)) {
-			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+			ASSERT0P(hdr->b_l1hdr.b_pabd);
 			ASSERT(!HDR_HAS_RABD(hdr));
 
 			/*
@@ -2758,7 +2759,7 @@ arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
 	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
 	    hdr->b_type == ARC_BUFC_METADATA);
 	ASSERT3P(ret, !=, NULL);
-	ASSERT3P(*ret, ==, NULL);
+	ASSERT0P(*ret);
 	IMPLY(encrypted, compressed);
 
 	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
@@ -2982,7 +2983,7 @@ static void
 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
 {
 	ASSERT(arc_can_share(hdr, buf));
-	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+	ASSERT0P(hdr->b_l1hdr.b_pabd);
 	ASSERT(!ARC_BUF_ENCRYPTED(buf));
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 
@@ -3201,14 +3202,14 @@ arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)
 
 	if (alloc_rdata) {
 		size = HDR_GET_PSIZE(hdr);
-		ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
+		ASSERT0P(hdr->b_crypt_hdr.b_rabd);
 		hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
 		    alloc_flags);
 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
 		ARCSTAT_INCR(arcstat_raw_size, size);
 	} else {
 		size = arc_hdr_size(hdr);
-		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_pabd);
 		hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
 		    alloc_flags);
 		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
@@ -3290,7 +3291,7 @@ arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
 
 	ASSERT(HDR_EMPTY(hdr));
 #ifdef ZFS_DEBUG
-	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
+	ASSERT0P(hdr->b_l1hdr.b_freeze_cksum);
 #endif
 	HDR_SET_PSIZE(hdr, psize);
 	HDR_SET_LSIZE(hdr, lsize);
@@ -3351,12 +3352,12 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 		nhdr->b_l1hdr.b_state = arc_l2c_only;
 
 		/* Verify previous threads set to NULL before freeing */
-		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
+		ASSERT0P(nhdr->b_l1hdr.b_pabd);
 		ASSERT(!HDR_HAS_RABD(hdr));
 	} else {
-		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_buf);
 #ifdef ZFS_DEBUG
-		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_freeze_cksum);
 #endif
 
 		/*
@@ -3375,7 +3376,7 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 		 * might try to be accessed, even though it was removed.
 		 */
 		VERIFY(!HDR_L2_WRITING(hdr));
-		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+		VERIFY0P(hdr->b_l1hdr.b_pabd);
 		ASSERT(!HDR_HAS_RABD(hdr));
 
 		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
@@ -3698,12 +3699,12 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 			arc_hdr_free_abd(hdr, B_TRUE);
 	}
 
-	ASSERT3P(hdr->b_hash_next, ==, NULL);
+	ASSERT0P(hdr->b_hash_next);
 	if (HDR_HAS_L1HDR(hdr)) {
 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
-		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_acb);
 #ifdef ZFS_DEBUG
-		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_freeze_cksum);
 #endif
 		kmem_cache_free(hdr_full_cache, hdr);
 	} else {
@@ -3771,7 +3772,7 @@ arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
+	ASSERT0P(hdr->b_l1hdr.b_buf);
 	ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
 
 	*real_evicted = 0;
@@ -3796,7 +3797,7 @@ arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
 		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
 
 		if (HDR_HAS_L2HDR(hdr)) {
-			ASSERT(hdr->b_l1hdr.b_pabd == NULL);
+			ASSERT0P(hdr->b_l1hdr.b_pabd);
 			ASSERT(!HDR_HAS_RABD(hdr));
 			/*
 			 * This buffer is cached on the 2nd Level ARC;
@@ -4650,10 +4651,10 @@ arc_flush_task(void *arg)
 	arc_flush_impl(spa_guid, B_FALSE);
 	arc_async_flush_remove(spa_guid, af->af_cache_level);
 
-	uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
-	if (elaspsed > 0) {
+	uint64_t elapsed = NSEC2MSEC(gethrtime() - start_time);
+	if (elapsed > 0) {
 		zfs_dbgmsg("spa %llu arc flushed in %llu ms",
-		    (u_longlong_t)spa_guid, (u_longlong_t)elaspsed);
+		    (u_longlong_t)spa_guid, (u_longlong_t)elapsed);
 	}
 }
 
@@ -5554,7 +5555,7 @@ static void
 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
 {
 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
-		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
+		ASSERT0(HDR_GET_PSIZE(hdr));
 		ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);
 	} else {
 		if (HDR_COMPRESSION_ENABLED(hdr)) {
@@ -6132,14 +6133,14 @@ top:
 			}
 
 			if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
-				ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+				ASSERT0P(hdr->b_l1hdr.b_pabd);
 				ASSERT(!HDR_HAS_RABD(hdr));
 				ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 				ASSERT0(zfs_refcount_count(
 				    &hdr->b_l1hdr.b_refcnt));
-				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
+				ASSERT0P(hdr->b_l1hdr.b_buf);
 #ifdef ZFS_DEBUG
-				ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
+				ASSERT0P(hdr->b_l1hdr.b_freeze_cksum);
 #endif
 			} else if (HDR_IO_IN_PROGRESS(hdr)) {
 				/*
@@ -6233,7 +6234,7 @@ top:
 		acb->acb_nobuf = no_buf;
 		acb->acb_zb = *zb;
 
-		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
+		ASSERT0P(hdr->b_l1hdr.b_acb);
 		hdr->b_l1hdr.b_acb = acb;
 
 		if (HDR_HAS_L2HDR(hdr) &&
@@ -6717,7 +6718,7 @@ arc_release(arc_buf_t *buf, const void *tag)
 
 		nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
 		    compress, hdr->b_complevel, type);
-		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
+		ASSERT0P(nhdr->b_l1hdr.b_buf);
 		ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
 		VERIFY3U(nhdr->b_type, ==, type);
 		ASSERT(!HDR_SHARED_DATA(nhdr));
@@ -6804,7 +6805,7 @@ arc_write_ready(zio_t *zio)
 		if (HDR_HAS_RABD(hdr))
 			arc_hdr_free_abd(hdr, B_TRUE);
 	}
-	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+	ASSERT0P(hdr->b_l1hdr.b_pabd);
 	ASSERT(!HDR_HAS_RABD(hdr));
 	ASSERT(!HDR_SHARED_DATA(hdr));
 	ASSERT(!arc_buf_is_shared(buf));
@@ -6948,7 +6949,7 @@ arc_write_done(zio_t *zio)
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
-	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
+	ASSERT0P(hdr->b_l1hdr.b_acb);
 
 	if (zio->io_error == 0) {
 		arc_hdr_verify(hdr, zio->io_bp);
@@ -6973,7 +6974,7 @@ arc_write_done(zio_t *zio)
 		arc_buf_hdr_t *exists;
 		kmutex_t *hash_lock;
 
-		ASSERT3U(zio->io_error, ==, 0);
+		ASSERT0(zio->io_error);
 
 		arc_cksum_verify(buf);
 
@@ -6994,7 +6995,7 @@ arc_write_done(zio_t *zio)
 				arc_hdr_destroy(exists);
 				mutex_exit(hash_lock);
 				exists = buf_hash_insert(hdr, &hash_lock);
-				ASSERT3P(exists, ==, NULL);
+				ASSERT0P(exists);
 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
 				/* nopwrite */
 				ASSERT(zio->io_prop.zp_nopwrite);
@@ -7007,7 +7008,7 @@ arc_write_done(zio_t *zio)
 				ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
 				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
 				ASSERT(BP_GET_DEDUP(zio->io_bp));
-				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
+				ASSERT0(BP_GET_LEVEL(zio->io_bp));
 			}
 		}
 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
@@ -7044,7 +7045,7 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
 	ASSERT3P(done, !=, NULL);
 	ASSERT(!HDR_IO_ERROR(hdr));
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
+	ASSERT0P(hdr->b_l1hdr.b_acb);
 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
 	if (uncached)
 		arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
@@ -7113,7 +7114,7 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
 		arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
 
 	ASSERT(!arc_buf_is_shared(buf));
-	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
+	ASSERT0P(hdr->b_l1hdr.b_pabd);
 
 	zio = zio_write(pio, spa, txg, bp,
 	    abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
@@ -9151,7 +9152,7 @@ top:
 		if (dev->l2ad_first) {
 			/*
 			 * This is the first sweep through the device. There is
-			 * nothing to evict. We have already trimmmed the
+			 * nothing to evict. We have already trimmed the
 			 * whole device.
 			 */
 			goto out;
@@ -10085,12 +10086,12 @@ l2arc_device_teardown(void *arg)
 	kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize);
 	vmem_free(remdev, sizeof (l2arc_dev_t));
 
-	uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
-	if (elaspsed > 0) {
+	uint64_t elapsed = NSEC2MSEC(gethrtime() - start_time);
+	if (elapsed > 0) {
 		zfs_dbgmsg("spa %llu, vdev %llu removed in %llu ms",
 		    (u_longlong_t)rva->rva_spa_gid,
 		    (u_longlong_t)rva->rva_vdev_gid,
-		    (u_longlong_t)elaspsed);
+		    (u_longlong_t)elapsed);
 	}
 
 	if (rva->rva_async)
diff --git a/sys/contrib/openzfs/module/zfs/bpobj.c b/sys/contrib/openzfs/module/zfs/bpobj.c
index 0a8a077edf63..ea9fbd036c6e 100644
--- a/sys/contrib/openzfs/module/zfs/bpobj.c
+++ b/sys/contrib/openzfs/module/zfs/bpobj.c
@@ -160,8 +160,8 @@ bpobj_open(bpobj_t *bpo, objset_t *os, uint64_t object)
 	memset(bpo, 0, sizeof (*bpo));
 	mutex_init(&bpo->bpo_lock, NULL, MUTEX_DEFAULT, NULL);
 
-	ASSERT(bpo->bpo_dbuf == NULL);
-	ASSERT(bpo->bpo_phys == NULL);
+	ASSERT0P(bpo->bpo_dbuf);
+	ASSERT0P(bpo->bpo_phys);
 	ASSERT(object != 0);
 	ASSERT3U(doi.doi_type, ==, DMU_OT_BPOBJ);
 	ASSERT3U(doi.doi_bonus_type, ==, DMU_OT_BPOBJ_HDR);
@@ -478,7 +478,7 @@ bpobj_iterate_impl(bpobj_t *initial_bpo, bpobj_itor_t func, void *arg,
 			 * We have unprocessed subobjs. Process the next one.
 			 */
 			ASSERT(bpo->bpo_havecomp);
-			ASSERT3P(bpobj_size, ==, NULL);
+			ASSERT0P(bpobj_size);
 
 			/* Add the last subobj to stack. */
 			int64_t i = bpi->bpi_unprocessed_subobjs - 1;
diff --git a/sys/contrib/openzfs/module/zfs/btree.c b/sys/contrib/openzfs/module/zfs/btree.c
index aa282f711bc3..725b96a3b2c7 100644
--- a/sys/contrib/openzfs/module/zfs/btree.c
+++ b/sys/contrib/openzfs/module/zfs/btree.c
@@ -1110,7 +1110,7 @@ zfs_btree_add_idx(zfs_btree_t *tree, const void *value,
 	if (where->bti_node == NULL) {
 		ASSERT3U(tree->bt_num_elems, ==, 1);
 		ASSERT3S(tree->bt_height, ==, -1);
-		ASSERT3P(tree->bt_root, ==, NULL);
+		ASSERT0P(tree->bt_root);
 		ASSERT0(where->bti_offset);
 
 		tree->bt_num_nodes++;
@@ -1947,7 +1947,7 @@ void
 zfs_btree_destroy(zfs_btree_t *tree)
 {
 	ASSERT0(tree->bt_num_elems);
-	ASSERT3P(tree->bt_root, ==, NULL);
+	ASSERT0P(tree->bt_root);
 }
 
 /* Verify that every child of this node has the correct parent pointer. */
@@ -1969,10 +1969,10 @@ static void
 zfs_btree_verify_pointers(zfs_btree_t *tree)
 {
 	if (tree->bt_height == -1) {
-		VERIFY3P(tree->bt_root, ==, NULL);
+		VERIFY0P(tree->bt_root);
 		return;
 	}
-	VERIFY3P(tree->bt_root->bth_parent, ==, NULL);
+	VERIFY0P(tree->bt_root->bth_parent);
 	zfs_btree_verify_pointers_helper(tree, tree->bt_root);
 }
 
diff --git a/sys/contrib/openzfs/module/zfs/dataset_kstats.c b/sys/contrib/openzfs/module/zfs/dataset_kstats.c
index d3baabd6169f..e5abcd2044cf 100644
--- a/sys/contrib/openzfs/module/zfs/dataset_kstats.c
+++ b/sys/contrib/openzfs/module/zfs/dataset_kstats.c
@@ -44,6 +44,7 @@ static dataset_kstat_values_t empty_dataset_kstats = {
 	{ "zil_commit_error_count",		KSTAT_DATA_UINT64 },
 	{ "zil_commit_stall_count",		KSTAT_DATA_UINT64 },
 	{ "zil_commit_suspend_count",		KSTAT_DATA_UINT64 },
+	{ "zil_commit_crash_count",		KSTAT_DATA_UINT64 },
 	{ "zil_itx_count",			KSTAT_DATA_UINT64 },
 	{ "zil_itx_indirect_count",		KSTAT_DATA_UINT64 },
 	{ "zil_itx_indirect_bytes",		KSTAT_DATA_UINT64 },
diff --git a/sys/contrib/openzfs/module/zfs/dbuf.c b/sys/contrib/openzfs/module/zfs/dbuf.c
index 432c99cec960..fccc4c5b5b94 100644
--- a/sys/contrib/openzfs/module/zfs/dbuf.c
+++ b/sys/contrib/openzfs/module/zfs/dbuf.c
@@ -523,7 +523,7 @@ dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
 		return;
 
 	/* Only data blocks support the attachment of user data. */
-	ASSERT(db->db_level == 0);
+	ASSERT0(db->db_level);
 
 	/* Clients must resolve a dbuf before attaching user data. */
 	ASSERT(db->db.db_data != NULL);
@@ -1128,8 +1128,8 @@ dbuf_verify(dmu_buf_impl_t *db)
 	DB_DNODE_ENTER(db);
 	dn = DB_DNODE(db);
 	if (dn == NULL) {
-		ASSERT(db->db_parent == NULL);
-		ASSERT(db->db_blkptr == NULL);
+		ASSERT0P(db->db_parent);
+		ASSERT0P(db->db_blkptr);
 	} else {
 		ASSERT3U(db->db.db_object, ==, dn->dn_object);
 		ASSERT3P(db->db_objset, ==, dn->dn_objset);
@@ -1180,7 +1180,7 @@ dbuf_verify(dmu_buf_impl_t *db)
 			/* db is pointed to by the dnode */
 			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
 			if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
-				ASSERT(db->db_parent == NULL);
+				ASSERT0P(db->db_parent);
 			else
 				ASSERT(db->db_parent != NULL);
 			if (db->db_blkid != DMU_SPILL_BLKID)
@@ -1219,7 +1219,7 @@ dbuf_verify(dmu_buf_impl_t *db)
 				int i;
 
 				for (i = 0; i < db->db.db_size >> 3; i++) {
-					ASSERT(buf[i] == 0);
+					ASSERT0(buf[i]);
 				}
 			} else {
 				blkptr_t *bps = db->db.db_data;
@@ -1259,7 +1259,7 @@ dbuf_clear_data(dmu_buf_impl_t *db)
 {
 	ASSERT(MUTEX_HELD(&db->db_mtx));
 	dbuf_evict_user(db);
-	ASSERT3P(db->db_buf, ==, NULL);
+	ASSERT0P(db->db_buf);
 	db->db.db_data = NULL;
 	if (db->db_state != DB_NOFILL) {
 		db->db_state = DB_UNCACHED;
@@ -1384,13 +1384,13 @@ dbuf_read_done(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
 	 * All reads are synchronous, so we must have a hold on the dbuf
 	 */
 	ASSERT(zfs_refcount_count(&db->db_holds) > 0);
-	ASSERT(db->db_buf == NULL);
-	ASSERT(db->db.db_data == NULL);
+	ASSERT0P(db->db_buf);
+	ASSERT0P(db->db.db_data);
 	if (buf == NULL) {
 		/* i/o error */
 		ASSERT(zio == NULL || zio->io_error != 0);
 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
-		ASSERT3P(db->db_buf, ==, NULL);
+		ASSERT0P(db->db_buf);
 		db->db_state = DB_UNCACHED;
 		DTRACE_SET_STATE(db, "i/o error");
 	} else if (db->db_level == 0 && db->db_freed_in_flight) {
@@ -1584,7 +1584,7 @@ dbuf_read_impl(dmu_buf_impl_t *db, dnode_t *dn, zio_t *zio, dmu_flags_t flags,
 	ASSERT(!zfs_refcount_is_zero(&db->db_holds));
 	ASSERT(MUTEX_HELD(&db->db_mtx));
 	ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
-	ASSERT(db->db_buf == NULL);
+	ASSERT0P(db->db_buf);
 	ASSERT(db->db_parent == NULL ||
 	    RW_LOCK_HELD(&db->db_parent->db_rwlock));
 
@@ -1682,7 +1682,7 @@ dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
 
 	ASSERT(MUTEX_HELD(&db->db_mtx));
 	ASSERT(db->db.db_data != NULL);
-	ASSERT(db->db_level == 0);
+	ASSERT0(db->db_level);
 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
 
 	if (dr == NULL ||
@@ -1901,8 +1901,8 @@ dbuf_noread(dmu_buf_impl_t *db, dmu_flags_t flags)
 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
 		cv_wait(&db->db_changed, &db->db_mtx);
 	if (db->db_state == DB_UNCACHED) {
-		ASSERT(db->db_buf == NULL);
-		ASSERT(db->db.db_data == NULL);
+		ASSERT0P(db->db_buf);
+		ASSERT0P(db->db.db_data);
 		dbuf_set_data(db, dbuf_alloc_arcbuf(db));
 		db->db_state = DB_FILL;
 		DTRACE_SET_STATE(db, "assigning filled buffer");
@@ -1929,7 +1929,7 @@ dbuf_unoverride(dbuf_dirty_record_t *dr)
 	 * comes from dbuf_dirty() callers who must also hold a range lock.
 	 */
 	ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
-	ASSERT(db->db_level == 0);
+	ASSERT0(db->db_level);
 
 	if (db->db_blkid == DMU_BONUS_BLKID ||
 	    dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
@@ -1994,7 +1994,7 @@ dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
 
 	mutex_enter(&dn->dn_dbufs_mtx);
 	db = avl_find(&dn->dn_dbufs, db_search, &where);
-	ASSERT3P(db, ==, NULL);
+	ASSERT0P(db);
 
 	db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
 
@@ -2017,7 +2017,7 @@ dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
 		if (db->db_state == DB_UNCACHED ||
 		    db->db_state == DB_NOFILL ||
 		    db->db_state == DB_EVICTING) {
-			ASSERT(db->db.db_data == NULL);
+			ASSERT0P(db->db.db_data);
 			mutex_exit(&db->db_mtx);
 			continue;
 		}
@@ -2270,14 +2270,6 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 	if (dn->dn_objset->os_dsl_dataset != NULL)
 		rrw_exit(&dn->dn_objset->os_dsl_dataset->ds_bp_rwlock, FTAG);
 #endif
-	/*
-	 * We make this assert for private objects as well, but after we
-	 * check if we're already dirty.  They are allowed to re-dirty
-	 * in syncing context.
-	 */
-	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
-	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
-	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
 
 	mutex_enter(&db->db_mtx);
 	/*
@@ -2289,12 +2281,6 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 	    db->db_state == DB_CACHED || db->db_state == DB_FILL ||
 	    db->db_state == DB_NOFILL);
 
-	mutex_enter(&dn->dn_mtx);
-	dnode_set_dirtyctx(dn, tx, db);
-	if (tx->tx_txg > dn->dn_dirty_txg)
-		dn->dn_dirty_txg = tx->tx_txg;
-	mutex_exit(&dn->dn_mtx);
-
 	if (db->db_blkid == DMU_SPILL_BLKID)
 		dn->dn_have_spill = B_TRUE;
 
@@ -2313,13 +2299,6 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 		return (dr_next);
 	}
 
-	/*
-	 * Only valid if not already dirty.
-	 */
-	ASSERT(dn->dn_object == 0 ||
-	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
-	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
-
 	ASSERT3U(dn->dn_nlevels, >, db->db_level);
 
 	/*
@@ -2557,12 +2536,13 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 
 	/*
 	 * Due to our use of dn_nlevels below, this can only be called
-	 * in open context, unless we are operating on the MOS.
-	 * From syncing context, dn_nlevels may be different from the
-	 * dn_nlevels used when dbuf was dirtied.
+	 * in open context, unless we are operating on the MOS or it's
+	 * a special object. From syncing context, dn_nlevels may be
+	 * different from the dn_nlevels used when dbuf was dirtied.
 	 */
 	ASSERT(db->db_objset ==
 	    dmu_objset_pool(db->db_objset)->dp_meta_objset ||
+	    DMU_OBJECT_IS_SPECIAL(db->db.db_object) ||
 	    txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 	ASSERT0(db->db_level);
@@ -2896,8 +2876,8 @@ dmu_buf_will_clone_or_dio(dmu_buf_t *db_fake, dmu_tx_t *tx)
 		dbuf_clear_data(db);
 	}
 
-	ASSERT3P(db->db_buf, ==, NULL);
-	ASSERT3P(db->db.db_data, ==, NULL);
+	ASSERT0P(db->db_buf);
+	ASSERT0P(db->db.db_data);
 
 	db->db_state = DB_NOFILL;
 	DTRACE_SET_STATE(db,
@@ -2932,7 +2912,7 @@ dmu_buf_will_fill_flags(dmu_buf_t *db_fake, dmu_tx_t *tx, boolean_t canfail,
 
 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 	ASSERT(tx->tx_txg != 0);
-	ASSERT(db->db_level == 0);
+	ASSERT0(db->db_level);
 	ASSERT(!zfs_refcount_is_zero(&db->db_holds));
 
 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
@@ -3144,7 +3124,7 @@ dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx,
 {
 	ASSERT(!zfs_refcount_is_zero(&db->db_holds));
 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
-	ASSERT(db->db_level == 0);
+	ASSERT0(db->db_level);
 	ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf));
 	ASSERT(buf != NULL);
 	ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size);
@@ -3209,7 +3189,7 @@ dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx,
 		VERIFY(!dbuf_undirty(db, tx));
 		db->db_state = DB_UNCACHED;
 	}
-	ASSERT(db->db_buf == NULL);
+	ASSERT0P(db->db_buf);
 	dbuf_set_data(db, buf);
 	db->db_state = DB_FILL;
 	DTRACE_SET_STATE(db, "filling assigned arcbuf");
@@ -3269,7 +3249,7 @@ dbuf_destroy(dmu_buf_impl_t *db)
 	}
 
 	ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
-	ASSERT(db->db_data_pending == NULL);
+	ASSERT0P(db->db_data_pending);
 	ASSERT(list_is_empty(&db->db_dirty_records));
 
 	db->db_state = DB_EVICTING;
@@ -3321,11 +3301,11 @@ dbuf_destroy(dmu_buf_impl_t *db)
 
 	db->db_parent = NULL;
 
-	ASSERT(db->db_buf == NULL);
-	ASSERT(db->db.db_data == NULL);
-	ASSERT(db->db_hash_next == NULL);
-	ASSERT(db->db_blkptr == NULL);
-	ASSERT(db->db_data_pending == NULL);
+	ASSERT0P(db->db_buf);
+	ASSERT0P(db->db.db_data);
+	ASSERT0P(db->db_hash_next);
+	ASSERT0P(db->db_blkptr);
+	ASSERT0P(db->db_data_pending);
 	ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);
 	ASSERT(!multilist_link_active(&db->db_cache_link));
 
@@ -3960,7 +3940,7 @@ dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid,
 		if (fail_uncached)
 			return (SET_ERROR(ENOENT));
 
-		ASSERT3P(parent, ==, NULL);
+		ASSERT0P(parent);
 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
 		if (fail_sparse) {
 			if (err == 0 && bp && BP_IS_HOLE(bp))
@@ -4064,7 +4044,7 @@ dbuf_create_bonus(dnode_t *dn)
 {
 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
 
-	ASSERT(dn->dn_bonus == NULL);
+	ASSERT0P(dn->dn_bonus);
 	dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL,
 	    dbuf_hash(dn->dn_objset, dn->dn_object, 0, DMU_BONUS_BLKID));
 	dn->dn_bonus->db_pending_evict = FALSE;
@@ -4416,7 +4396,7 @@ dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
 		 * inappropriate to hook it in (i.e., nlevels mismatch).
 		 */
 		ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
-		ASSERT(db->db_parent == NULL);
+		ASSERT0P(db->db_parent);
 		db->db_parent = dn->dn_dbuf;
 		db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
 		DBUF_VERIFY(db);
@@ -4477,7 +4457,7 @@ dbuf_prepare_encrypted_dnode_leaf(dbuf_dirty_record_t *dr)
 
 	ASSERT(MUTEX_HELD(&db->db_mtx));
 	ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT);
-	ASSERT3U(db->db_level, ==, 0);
+	ASSERT0(db->db_level);
 
 	if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) {
 		zbookmark_phys_t zb;
@@ -4588,7 +4568,7 @@ dbuf_sync_leaf_verify_bonus_dnode(dbuf_dirty_record_t *dr)
 
 	/* ensure that everything is zero after our data */
 	for (; datap_end < datap_max; datap_end++)
-		ASSERT(*datap_end == 0);
+		ASSERT0(*datap_end);
 #endif
 }
 
@@ -4596,7 +4576,7 @@ static blkptr_t *
 dbuf_lightweight_bp(dbuf_dirty_record_t *dr)
 {
 	/* This must be a lightweight dirty record. */
-	ASSERT3P(dr->dr_dbuf, ==, NULL);
+	ASSERT0P(dr->dr_dbuf);
 	dnode_t *dn = dr->dr_dnode;
 
 	if (dn->dn_phys->dn_nlevels == 1) {
@@ -4739,7 +4719,7 @@ dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
 	 */
 	if (db->db_state == DB_UNCACHED) {
 		/* This buffer has been freed since it was dirtied */
-		ASSERT3P(db->db.db_data, ==, NULL);
+		ASSERT0P(db->db.db_data);
 	} else if (db->db_state == DB_FILL) {
 		/* This buffer was freed and is now being re-filled */
 		ASSERT(db->db.db_data != dr->dt.dl.dr_data);
@@ -4756,9 +4736,9 @@ dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
 		 */
 		dbuf_dirty_record_t *dr_head =
 		    list_head(&db->db_dirty_records);
-		ASSERT3P(db->db_buf, ==, NULL);
-		ASSERT3P(db->db.db_data, ==, NULL);
-		ASSERT3P(dr_head->dt.dl.dr_data, ==, NULL);
+		ASSERT0P(db->db_buf);
+		ASSERT0P(db->db.db_data);
+		ASSERT0P(dr_head->dt.dl.dr_data);
 		ASSERT3U(dr_head->dt.dl.dr_override_state, ==, DR_OVERRIDDEN);
 	} else {
 		ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
diff --git a/sys/contrib/openzfs/module/zfs/ddt.c b/sys/contrib/openzfs/module/zfs/ddt.c
index e0b9fc3951ff..0dc9adc7fd4f 100644
--- a/sys/contrib/openzfs/module/zfs/ddt.c
+++ b/sys/contrib/openzfs/module/zfs/ddt.c
@@ -397,7 +397,7 @@ ddt_object_create(ddt_t *ddt, ddt_type_t type, ddt_class_t class,
 
 	ddt_object_name(ddt, type, class, name);
 
-	ASSERT3U(*objectp, ==, 0);
+	ASSERT0(*objectp);
 	VERIFY0(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash));
 	ASSERT3U(*objectp, !=, 0);
 
@@ -1011,7 +1011,7 @@ ddt_free(const ddt_t *ddt, ddt_entry_t *dde)
 {
 	if (dde->dde_io != NULL) {
 		for (int p = 0; p < DDT_NPHYS(ddt); p++)
-			ASSERT3P(dde->dde_io->dde_lead_zio[p], ==, NULL);
+			ASSERT0P(dde->dde_io->dde_lead_zio[p]);
 
 		if (dde->dde_io->dde_repair_abd != NULL)
 			abd_free(dde->dde_io->dde_repair_abd);
@@ -1421,7 +1421,7 @@ ddt_key_compare(const void *x1, const void *x2)
 static void
 ddt_create_dir(ddt_t *ddt, dmu_tx_t *tx)
 {
-	ASSERT3U(ddt->ddt_dir_object, ==, 0);
+	ASSERT0(ddt->ddt_dir_object);
 	ASSERT3U(ddt->ddt_version, ==, DDT_VERSION_FDT);
 
 	char name[DDT_NAMELEN];
@@ -1701,9 +1701,11 @@ ddt_load(spa_t *spa)
 			}
 		}
 
-		error = ddt_log_load(ddt);
-		if (error != 0 && error != ENOENT)
-			return (error);
+		if (ddt->ddt_flags & DDT_FLAG_LOG) {
+			error = ddt_log_load(ddt);
+			if (error != 0 && error != ENOENT)
+				return (error);
+		}
 
 		DDT_KSTAT_SET(ddt, dds_log_active_entries,
 		    avl_numnodes(&ddt->ddt_log_active->ddl_tree));
@@ -2395,7 +2397,7 @@ ddt_sync(spa_t *spa, uint64_t txg)
 	 * scan's root zio here so that we can wait for any scan IOs in
 	 * addition to the regular ddt IOs.
 	 */
-	ASSERT3P(scn->scn_zio_root, ==, NULL);
+	ASSERT0P(scn->scn_zio_root);
 	scn->scn_zio_root = rio;
 
 	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
diff --git a/sys/contrib/openzfs/module/zfs/ddt_log.c b/sys/contrib/openzfs/module/zfs/ddt_log.c
index dbd381aa9609..c7a2426f3a77 100644
--- a/sys/contrib/openzfs/module/zfs/ddt_log.c
+++ b/sys/contrib/openzfs/module/zfs/ddt_log.c
@@ -116,7 +116,7 @@ static void
 ddt_log_create_one(ddt_t *ddt, ddt_log_t *ddl, uint_t n, dmu_tx_t *tx)
 {
 	ASSERT3U(ddt->ddt_dir_object, >, 0);
-	ASSERT3U(ddl->ddl_object, ==, 0);
+	ASSERT0(ddl->ddl_object);
 
 	char name[DDT_NAMELEN];
 	ddt_log_name(ddt, name, n);
@@ -176,11 +176,13 @@ ddt_log_update_stats(ddt_t *ddt)
 	 * that's reasonable to expect anyway.
 	 */
 	dmu_object_info_t doi;
-	uint64_t nblocks;
-	dmu_object_info(ddt->ddt_os, ddt->ddt_log_active->ddl_object, &doi);
-	nblocks = doi.doi_physical_blocks_512;
-	dmu_object_info(ddt->ddt_os, ddt->ddt_log_flushing->ddl_object, &doi);
-	nblocks += doi.doi_physical_blocks_512;
+	uint64_t nblocks = 0;
+	if (dmu_object_info(ddt->ddt_os, ddt->ddt_log_active->ddl_object,
+	    &doi) == 0)
+		nblocks += doi.doi_physical_blocks_512;
+	if (dmu_object_info(ddt->ddt_os, ddt->ddt_log_flushing->ddl_object,
+	    &doi) == 0)
+		nblocks += doi.doi_physical_blocks_512;
 
 	ddt_object_t *ddo = &ddt->ddt_log_stats;
 	ddo->ddo_count =
@@ -194,7 +196,7 @@ void
 ddt_log_begin(ddt_t *ddt, size_t nentries, dmu_tx_t *tx, ddt_log_update_t *dlu)
 {
 	ASSERT3U(nentries, >, 0);
-	ASSERT3P(dlu->dlu_dbp, ==, NULL);
+	ASSERT0P(dlu->dlu_dbp);
 
 	if (ddt->ddt_log_active->ddl_object == 0)
 		ddt_log_create(ddt, tx);
@@ -243,6 +245,13 @@ ddt_log_alloc_entry(ddt_t *ddt)
 }
 
 static void
+ddt_log_free_entry(ddt_t *ddt, ddt_log_entry_t *ddle)
+{
+	kmem_cache_free(ddt->ddt_flags & DDT_FLAG_FLAT ?
+	    ddt_log_entry_flat_cache : ddt_log_entry_trad_cache, ddle);
+}
+
+static void
 ddt_log_update_entry(ddt_t *ddt, ddt_log_t *ddl, ddt_lightweight_entry_t *ddlwe)
 {
 	/* Create the log tree entry from a live or stored entry */
@@ -347,8 +356,7 @@ ddt_log_take_first(ddt_t *ddt, ddt_log_t *ddl, ddt_lightweight_entry_t *ddlwe)
 	ddt_histogram_sub_entry(ddt, &ddt->ddt_log_histogram, ddlwe);
 
 	avl_remove(&ddl->ddl_tree, ddle);
-	kmem_cache_free(ddt->ddt_flags & DDT_FLAG_FLAT ?
-	    ddt_log_entry_flat_cache : ddt_log_entry_trad_cache, ddle);
+	ddt_log_free_entry(ddt, ddle);
 
 	return (B_TRUE);
 }
@@ -365,8 +373,7 @@ ddt_log_remove_key(ddt_t *ddt, ddt_log_t *ddl, const ddt_key_t *ddk)
 	ddt_histogram_sub_entry(ddt, &ddt->ddt_log_histogram, &ddlwe);
 
 	avl_remove(&ddl->ddl_tree, ddle);
-	kmem_cache_free(ddt->ddt_flags & DDT_FLAG_FLAT ?
-	    ddt_log_entry_flat_cache : ddt_log_entry_trad_cache, ddle);
+	ddt_log_free_entry(ddt, ddle);
 
 	return (B_TRUE);
 }
@@ -527,8 +534,7 @@ ddt_log_empty(ddt_t *ddt, ddt_log_t *ddl)
 	IMPLY(ddt->ddt_version == UINT64_MAX, avl_is_empty(&ddl->ddl_tree));
 	while ((ddle =
 	    avl_destroy_nodes(&ddl->ddl_tree, &cookie)) != NULL) {
-		kmem_cache_free(ddt->ddt_flags & DDT_FLAG_FLAT ?
-		    ddt_log_entry_flat_cache : ddt_log_entry_trad_cache, ddle);
+		ddt_log_free_entry(ddt, ddle);
 	}
 	ASSERT(avl_is_empty(&ddl->ddl_tree));
 }
@@ -727,7 +733,7 @@ ddt_log_load(ddt_t *ddt)
 				ddle = fe;
 				fe = AVL_NEXT(fl, fe);
 				avl_remove(fl, ddle);
-
+				ddt_log_free_entry(ddt, ddle);
 				ddle = ae;
 				ae = AVL_NEXT(al, ae);
 			}
@@ -748,8 +754,8 @@ ddt_log_load(ddt_t *ddt)
 void
 ddt_log_alloc(ddt_t *ddt)
 {
-	ASSERT3P(ddt->ddt_log_active, ==, NULL);
-	ASSERT3P(ddt->ddt_log_flushing, ==, NULL);
+	ASSERT0P(ddt->ddt_log_active);
+	ASSERT0P(ddt->ddt_log_flushing);
 
 	avl_create(&ddt->ddt_log[0].ddl_tree, ddt_key_compare,
 	    sizeof (ddt_log_entry_t), offsetof(ddt_log_entry_t, ddle_node));
diff --git a/sys/contrib/openzfs/module/zfs/dmu.c b/sys/contrib/openzfs/module/zfs/dmu.c
index 296e58ef9cd8..a7a5c89bdafb 100644
--- a/sys/contrib/openzfs/module/zfs/dmu.c
+++ b/sys/contrib/openzfs/module/zfs/dmu.c
@@ -759,6 +759,8 @@ dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
 		 */
 		uint8_t ibps = ibs - SPA_BLKPTRSHIFT;
 		limit = P2ROUNDUP(dmu_prefetch_max, 1 << ibs) >> ibs;
+		if (limit == 0)
+			end2 = start2;
 		do {
 			level2++;
 			start2 = P2ROUNDUP(start2, 1 << ibps) >> ibps;
@@ -1343,7 +1345,7 @@ dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
 	if (size == 0)
 		return;
 
-	VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
+	VERIFY0(dmu_buf_hold_array(os, object, offset, size,
 	    FALSE, FTAG, &numbufs, &dbp));
 
 	for (i = 0; i < numbufs; i++) {
@@ -1689,8 +1691,8 @@ dmu_object_cached_size(objset_t *os, uint64_t object,
 
 	dmu_object_info_from_dnode(dn, &doi);
 
-	for (uint64_t off = 0; off < doi.doi_max_offset;
-	    off += dmu_prefetch_max) {
+	for (uint64_t off = 0; off < doi.doi_max_offset &&
+	    dmu_prefetch_max > 0; off += dmu_prefetch_max) {
 		/* dbuf_read doesn't prefetch L1 blocks. */
 		dmu_prefetch_by_dnode(dn, 1, off,
 		    dmu_prefetch_max, ZIO_PRIORITY_SYNC_READ);
@@ -1872,7 +1874,7 @@ dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg)
 			 */
 			BP_SET_LSIZE(bp, db->db_size);
 		} else if (!BP_IS_EMBEDDED(bp)) {
-			ASSERT(BP_GET_LEVEL(bp) == 0);
+			ASSERT0(BP_GET_LEVEL(bp));
 			BP_SET_FILL(bp, 1);
 		}
 	}
@@ -2405,7 +2407,7 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
 			}
 		}
 	} else if (wp & WP_NOFILL) {
-		ASSERT(level == 0);
+		ASSERT0(level);
 
 		/*
 		 * If we're writing preallocated blocks, we aren't actually
@@ -2865,7 +2867,7 @@ byteswap_uint64_array(void *vbuf, size_t size)
 	size_t count = size >> 3;
 	int i;
 
-	ASSERT((size & 7) == 0);
+	ASSERT0((size & 7));
 
 	for (i = 0; i < count; i++)
 		buf[i] = BSWAP_64(buf[i]);
@@ -2878,7 +2880,7 @@ byteswap_uint32_array(void *vbuf, size_t size)
 	size_t count = size >> 2;
 	int i;
 
-	ASSERT((size & 3) == 0);
+	ASSERT0((size & 3));
 
 	for (i = 0; i < count; i++)
 		buf[i] = BSWAP_32(buf[i]);
@@ -2891,7 +2893,7 @@ byteswap_uint16_array(void *vbuf, size_t size)
 	size_t count = size >> 1;
 	int i;
 
-	ASSERT((size & 1) == 0);
+	ASSERT0((size & 1));
 
 	for (i = 0; i < count; i++)
 		buf[i] = BSWAP_16(buf[i]);
diff --git a/sys/contrib/openzfs/module/zfs/dmu_direct.c b/sys/contrib/openzfs/module/zfs/dmu_direct.c
index 930ff101eca3..d44c686088fc 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_direct.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_direct.c
@@ -95,9 +95,9 @@ dmu_write_direct_done(zio_t *zio)
 	abd_free(zio->io_abd);
 
 	mutex_enter(&db->db_mtx);
-	ASSERT3P(db->db_buf, ==, NULL);
-	ASSERT3P(dr->dt.dl.dr_data, ==, NULL);
-	ASSERT3P(db->db.db_data, ==, NULL);
+	ASSERT0P(db->db_buf);
+	ASSERT0P(dr->dt.dl.dr_data);
+	ASSERT0P(db->db.db_data);
 	db->db_state = DB_UNCACHED;
 	mutex_exit(&db->db_mtx);
 
diff --git a/sys/contrib/openzfs/module/zfs/dmu_object.c b/sys/contrib/openzfs/module/zfs/dmu_object.c
index b4ff7d224cc9..207cc6d0e713 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_object.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_object.c
@@ -90,7 +90,7 @@ dmu_object_alloc_impl(objset_t *os, dmu_object_type_t ot, int blocksize,
 	if (allocated_dnode != NULL) {
 		ASSERT3P(tag, !=, NULL);
 	} else {
-		ASSERT3P(tag, ==, NULL);
+		ASSERT0P(tag);
 		tag = FTAG;
 	}
 
diff --git a/sys/contrib/openzfs/module/zfs/dmu_objset.c b/sys/contrib/openzfs/module/zfs/dmu_objset.c
index c135f620800f..8e6b569c2100 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_objset.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_objset.c
@@ -724,7 +724,7 @@ dmu_objset_from_ds(dsl_dataset_t *ds, objset_t **osp)
 
 		if (err == 0) {
 			mutex_enter(&ds->ds_lock);
-			ASSERT(ds->ds_objset == NULL);
+			ASSERT0P(ds->ds_objset);
 			ds->ds_objset = os;
 			mutex_exit(&ds->ds_lock);
 		}
@@ -2037,6 +2037,8 @@ userquota_updates_task(void *arg)
 				dn->dn_id_flags |= DN_ID_CHKED_BONUS;
 		}
 		dn->dn_id_flags &= ~(DN_ID_NEW_EXIST);
+		ASSERT3U(dn->dn_dirtycnt, >, 0);
+		dn->dn_dirtycnt--;
 		mutex_exit(&dn->dn_mtx);
 
 		multilist_sublist_remove(list, dn);
@@ -2070,6 +2072,10 @@ dnode_rele_task(void *arg)
 
 	dnode_t *dn;
 	while ((dn = multilist_sublist_head(list)) != NULL) {
+		mutex_enter(&dn->dn_mtx);
+		ASSERT3U(dn->dn_dirtycnt, >, 0);
+		dn->dn_dirtycnt--;
+		mutex_exit(&dn->dn_mtx);
 		multilist_sublist_remove(list, dn);
 		dnode_rele(dn, &os->os_synced_dnodes);
 	}
@@ -2226,7 +2232,7 @@ dmu_objset_userquota_get_ids(dnode_t *dn, boolean_t before, dmu_tx_t *tx)
 				rf |= DB_RF_HAVESTRUCT;
 			error = dmu_spill_hold_by_dnode(dn, rf,
 			    FTAG, (dmu_buf_t **)&db);
-			ASSERT(error == 0);
+			ASSERT0(error);
 			mutex_enter(&db->db_mtx);
 			data = (before) ? db->db.db_data :
 			    dmu_objset_userquota_find_data(db, tx);
diff --git a/sys/contrib/openzfs/module/zfs/dmu_recv.c b/sys/contrib/openzfs/module/zfs/dmu_recv.c
index 73227b58c140..45c7af2bdcd2 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_recv.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_recv.c
@@ -866,7 +866,7 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 	 */
 	if (dcp == NULL && drrb->drr_fromguid == 0 &&
 	    drba->drba_origin == NULL) {
-		ASSERT3P(dcp, ==, NULL);
+		ASSERT0P(dcp);
 		dcp = &dummy_dcp;
 
 		if (featureflags & DMU_BACKUP_FEATURE_RAW)
@@ -881,7 +881,7 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 		if (drba->drba_cookie->drc_fromsnapobj != 0) {
 			VERIFY0(dsl_dataset_hold_obj(dp,
 			    drba->drba_cookie->drc_fromsnapobj, FTAG, &snap));
-			ASSERT3P(dcp, ==, NULL);
+			ASSERT0P(dcp);
 		}
 		if (drc->drc_heal) {
 			/* When healing we want to use the provided snapshot */
@@ -905,7 +905,7 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 		if (drba->drba_origin != NULL) {
 			VERIFY0(dsl_dataset_hold(dp, drba->drba_origin,
 			    FTAG, &origin));
-			ASSERT3P(dcp, ==, NULL);
+			ASSERT0P(dcp);
 		}
 
 		/* Create new dataset. */
@@ -2792,7 +2792,7 @@ receive_read_payload_and_next_header(dmu_recv_cookie_t *drc, int len, void *buf)
 			drc->drc_rrd->bytes_read = drc->drc_bytes_read;
 		}
 	} else {
-		ASSERT3P(buf, ==, NULL);
+		ASSERT0P(buf);
 	}
 
 	drc->drc_prev_cksum = drc->drc_cksum;
@@ -3450,7 +3450,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp)
 			break;
 		}
 
-		ASSERT3P(drc->drc_rrd, ==, NULL);
+		ASSERT0P(drc->drc_rrd);
 		drc->drc_rrd = drc->drc_next_rrd;
 		drc->drc_next_rrd = NULL;
 		/* Allocates and loads header into drc->drc_next_rrd */
@@ -3468,7 +3468,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp)
 		drc->drc_rrd = NULL;
 	}
 
-	ASSERT3P(drc->drc_rrd, ==, NULL);
+	ASSERT0P(drc->drc_rrd);
 	drc->drc_rrd = kmem_zalloc(sizeof (*drc->drc_rrd), KM_SLEEP);
 	drc->drc_rrd->eos_marker = B_TRUE;
 	bqueue_enqueue_flush(&rwa->q, drc->drc_rrd, 1);
diff --git a/sys/contrib/openzfs/module/zfs/dmu_redact.c b/sys/contrib/openzfs/module/zfs/dmu_redact.c
index 9226ac9e4b80..5a22ed71a5fe 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_redact.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_redact.c
@@ -1067,7 +1067,7 @@ dmu_redact_snap(const char *snapname, nvlist_t *redactnvl,
 	}
 	if (err != 0)
 		goto out;
-	VERIFY3P(nvlist_next_nvpair(redactnvl, pair), ==, NULL);
+	VERIFY0P(nvlist_next_nvpair(redactnvl, pair));
 
 	boolean_t resuming = B_FALSE;
 	zfs_bookmark_phys_t bookmark;
diff --git a/sys/contrib/openzfs/module/zfs/dmu_send.c b/sys/contrib/openzfs/module/zfs/dmu_send.c
index deeba29e159a..8ecb99d5f57c 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_send.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_send.c
@@ -962,7 +962,7 @@ do_dump(dmu_send_cookie_t *dscp, struct send_range *range)
 		char *data = NULL;
 		if (srdp->abd != NULL) {
 			data = abd_to_buf(srdp->abd);
-			ASSERT3P(srdp->abuf, ==, NULL);
+			ASSERT0P(srdp->abuf);
 		} else if (srdp->abuf != NULL) {
 			data = srdp->abuf->b_data;
 		}
@@ -2514,7 +2514,7 @@ dmu_send_impl(struct dmu_send_params *dspp)
 	 * list in the stream.
 	 */
 	if (dspp->numfromredactsnaps != NUM_SNAPS_NOT_REDACTED) {
-		ASSERT3P(from_rl, ==, NULL);
+		ASSERT0P(from_rl);
 		fnvlist_add_uint64_array(nvl, BEGINNV_REDACT_FROM_SNAPS,
 		    dspp->fromredactsnaps, (uint_t)dspp->numfromredactsnaps);
 		if (dspp->numfromredactsnaps > 0) {
@@ -2891,7 +2891,7 @@ dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
 			    &fromds);
 
 			if (err != 0) {
-				ASSERT3P(fromds, ==, NULL);
+				ASSERT0P(fromds);
 			} else {
 				/*
 				 * We need to make a deep copy of the redact
diff --git a/sys/contrib/openzfs/module/zfs/dmu_tx.c b/sys/contrib/openzfs/module/zfs/dmu_tx.c
index d85d8b89423e..40c0b3402a05 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_tx.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_tx.c
@@ -126,7 +126,7 @@ dmu_tx_hold_dnode_impl(dmu_tx_t *tx, dnode_t *dn, enum dmu_tx_hold_type type,
 			 * problem, but there's no way for it to happen (for
 			 * now, at least).
 			 */
-			ASSERT(dn->dn_assigned_txg == 0);
+			ASSERT0(dn->dn_assigned_txg);
 			dn->dn_assigned_txg = tx->tx_txg;
 			(void) zfs_refcount_add(&dn->dn_tx_holds, tx);
 			mutex_exit(&dn->dn_mtx);
@@ -443,7 +443,7 @@ dmu_tx_count_free(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
 	dnode_t *dn = txh->txh_dnode;
 	int err;
 
-	ASSERT(tx->tx_txg == 0);
+	ASSERT0(tx->tx_txg);
 
 	if (off >= (dn->dn_maxblkid + 1) * dn->dn_datablksz)
 		return;
@@ -607,7 +607,7 @@ dmu_tx_hold_zap_impl(dmu_tx_hold_t *txh, const char *name)
 	dnode_t *dn = txh->txh_dnode;
 	int err;
 
-	ASSERT(tx->tx_txg == 0);
+	ASSERT0(tx->tx_txg);
 
 	dmu_tx_count_dnode(txh);
 
@@ -681,7 +681,7 @@ dmu_tx_hold_bonus(dmu_tx_t *tx, uint64_t object)
 {
 	dmu_tx_hold_t *txh;
 
-	ASSERT(tx->tx_txg == 0);
+	ASSERT0(tx->tx_txg);
 
 	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
 	    object, THT_BONUS, 0, 0);
@@ -706,7 +706,7 @@ dmu_tx_hold_space(dmu_tx_t *tx, uint64_t space)
 {
 	dmu_tx_hold_t *txh;
 
-	ASSERT(tx->tx_txg == 0);
+	ASSERT0(tx->tx_txg);
 
 	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
 	    DMU_NEW_OBJECT, THT_SPACE, space, 0);
@@ -1232,7 +1232,7 @@ dmu_tx_assign(dmu_tx_t *tx, dmu_tx_flag_t flags)
 {
 	int err;
 
-	ASSERT(tx->tx_txg == 0);
+	ASSERT0(tx->tx_txg);
 	ASSERT0(flags & ~(DMU_TX_WAIT | DMU_TX_NOTHROTTLE | DMU_TX_SUSPEND));
 	IMPLY(flags & DMU_TX_SUSPEND, flags & DMU_TX_WAIT);
 	ASSERT(!dsl_pool_sync_context(tx->tx_pool));
@@ -1328,7 +1328,7 @@ dmu_tx_wait(dmu_tx_t *tx)
 	dsl_pool_t *dp = tx->tx_pool;
 	hrtime_t before;
 
-	ASSERT(tx->tx_txg == 0);
+	ASSERT0(tx->tx_txg);
 	ASSERT(!dsl_pool_config_held(tx->tx_pool));
 
 	/*
@@ -1644,12 +1644,12 @@ dmu_tx_hold_sa(dmu_tx_t *tx, sa_handle_t *hdl, boolean_t may_grow)
 		dmu_tx_hold_zap(tx, sa->sa_layout_attr_obj, B_TRUE, NULL);
 
 	if (sa->sa_force_spill || may_grow || hdl->sa_spill) {
-		ASSERT(tx->tx_txg == 0);
+		ASSERT0(tx->tx_txg);
 		dmu_tx_hold_spill(tx, object);
 	} else {
 		DB_DNODE_ENTER(db);
 		if (DB_DNODE(db)->dn_have_spill) {
-			ASSERT(tx->tx_txg == 0);
+			ASSERT0(tx->tx_txg);
 			dmu_tx_hold_spill(tx, object);
 		}
 		DB_DNODE_EXIT(db);
diff --git a/sys/contrib/openzfs/module/zfs/dnode.c b/sys/contrib/openzfs/module/zfs/dnode.c
index 451e1533efa0..e88d394b5229 100644
--- a/sys/contrib/openzfs/module/zfs/dnode.c
+++ b/sys/contrib/openzfs/module/zfs/dnode.c
@@ -173,9 +173,7 @@ dnode_cons(void *arg, void *unused, int kmflag)
 	dn->dn_allocated_txg = 0;
 	dn->dn_free_txg = 0;
 	dn->dn_assigned_txg = 0;
-	dn->dn_dirty_txg = 0;
-	dn->dn_dirtyctx = 0;
-	dn->dn_dirtyctx_firstset = NULL;
+	dn->dn_dirtycnt = 0;
 	dn->dn_bonus = NULL;
 	dn->dn_have_spill = B_FALSE;
 	dn->dn_zio = NULL;
@@ -214,7 +212,7 @@ dnode_dest(void *arg, void *unused)
 
 	for (int i = 0; i < TXG_SIZE; i++) {
 		ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
-		ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
+		ASSERT0P(dn->dn_free_ranges[i]);
 		list_destroy(&dn->dn_dirty_records[i]);
 		ASSERT0(dn->dn_next_nblkptr[i]);
 		ASSERT0(dn->dn_next_nlevels[i]);
@@ -229,12 +227,10 @@ dnode_dest(void *arg, void *unused)
 	ASSERT0(dn->dn_allocated_txg);
 	ASSERT0(dn->dn_free_txg);
 	ASSERT0(dn->dn_assigned_txg);
-	ASSERT0(dn->dn_dirty_txg);
-	ASSERT0(dn->dn_dirtyctx);
-	ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL);
-	ASSERT3P(dn->dn_bonus, ==, NULL);
+	ASSERT0(dn->dn_dirtycnt);
+	ASSERT0P(dn->dn_bonus);
 	ASSERT(!dn->dn_have_spill);
-	ASSERT3P(dn->dn_zio, ==, NULL);
+	ASSERT0P(dn->dn_zio);
 	ASSERT0(dn->dn_oldused);
 	ASSERT0(dn->dn_oldflags);
 	ASSERT0(dn->dn_olduid);
@@ -318,7 +314,7 @@ dnode_kstats_update(kstat_t *ksp, int rw)
 void
 dnode_init(void)
 {
-	ASSERT(dnode_cache == NULL);
+	ASSERT0P(dnode_cache);
 	dnode_cache = kmem_cache_create("dnode_t", sizeof (dnode_t),
 	    0, dnode_cons, dnode_dest, NULL, NULL, NULL, KMC_RECLAIMABLE);
 	kmem_cache_set_move(dnode_cache, dnode_move);
@@ -509,7 +505,7 @@ dnode_buf_byteswap(void *vbuf, size_t size)
 	int i = 0;
 
 	ASSERT3U(sizeof (dnode_phys_t), ==, (1<<DNODE_SHIFT));
-	ASSERT((size & (sizeof (dnode_phys_t)-1)) == 0);
+	ASSERT0((size & (sizeof (dnode_phys_t)-1)));
 
 	while (i < size) {
 		dnode_phys_t *dnp = (void *)(((char *)vbuf) + i);
@@ -673,7 +669,7 @@ dnode_destroy(dnode_t *dn)
 	objset_t *os = dn->dn_objset;
 	boolean_t complete_os_eviction = B_FALSE;
 
-	ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
+	ASSERT0((dn->dn_id_flags & DN_ID_NEW_EXIST));
 
 	mutex_enter(&os->os_lock);
 	POINTER_INVALIDATE(&dn->dn_objset);
@@ -692,10 +688,8 @@ dnode_destroy(dnode_t *dn)
 	dn->dn_allocated_txg = 0;
 	dn->dn_free_txg = 0;
 	dn->dn_assigned_txg = 0;
-	dn->dn_dirty_txg = 0;
+	dn->dn_dirtycnt = 0;
 
-	dn->dn_dirtyctx = 0;
-	dn->dn_dirtyctx_firstset = NULL;
 	if (dn->dn_bonus != NULL) {
 		mutex_enter(&dn->dn_bonus->db_mtx);
 		dbuf_destroy(dn->dn_bonus);
@@ -780,7 +774,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
 		ASSERT0(dn->dn_next_maxblkid[i]);
 		ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
 		ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL);
-		ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
+		ASSERT0P(dn->dn_free_ranges[i]);
 	}
 
 	dn->dn_type = ot;
@@ -800,11 +794,9 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
 	dn->dn_bonuslen = bonuslen;
 	dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
 	dn->dn_compress = ZIO_COMPRESS_INHERIT;
-	dn->dn_dirtyctx = 0;
 
 	dn->dn_free_txg = 0;
-	dn->dn_dirtyctx_firstset = NULL;
-	dn->dn_dirty_txg = 0;
+	dn->dn_dirtycnt = 0;
 
 	dn->dn_allocated_txg = tx->tx_txg;
 	dn->dn_id_flags = 0;
@@ -955,10 +947,8 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn)
 	ndn->dn_allocated_txg = odn->dn_allocated_txg;
 	ndn->dn_free_txg = odn->dn_free_txg;
 	ndn->dn_assigned_txg = odn->dn_assigned_txg;
-	ndn->dn_dirty_txg = odn->dn_dirty_txg;
-	ndn->dn_dirtyctx = odn->dn_dirtyctx;
-	ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
-	ASSERT(zfs_refcount_count(&odn->dn_tx_holds) == 0);
+	ndn->dn_dirtycnt = odn->dn_dirtycnt;
+	ASSERT0(zfs_refcount_count(&odn->dn_tx_holds));
 	zfs_refcount_transfer(&ndn->dn_holds, &odn->dn_holds);
 	ASSERT(avl_is_empty(&ndn->dn_dbufs));
 	avl_swap(&ndn->dn_dbufs, &odn->dn_dbufs);
@@ -1020,9 +1010,7 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn)
 	odn->dn_allocated_txg = 0;
 	odn->dn_free_txg = 0;
 	odn->dn_assigned_txg = 0;
-	odn->dn_dirty_txg = 0;
-	odn->dn_dirtyctx = 0;
-	odn->dn_dirtyctx_firstset = NULL;
+	odn->dn_dirtycnt = 0;
 	odn->dn_have_spill = B_FALSE;
 	odn->dn_zio = NULL;
 	odn->dn_oldused = 0;
@@ -1273,8 +1261,8 @@ dnode_check_slots_free(dnode_children_t *children, int idx, int slots)
 		} else if (DN_SLOT_IS_PTR(dn)) {
 			mutex_enter(&dn->dn_mtx);
 			boolean_t can_free = (dn->dn_type == DMU_OT_NONE &&
-			    zfs_refcount_is_zero(&dn->dn_holds) &&
-			    !DNODE_IS_DIRTY(dn));
+			    dn->dn_dirtycnt == 0 &&
+			    zfs_refcount_is_zero(&dn->dn_holds));
 			mutex_exit(&dn->dn_mtx);
 
 			if (!can_free)
@@ -1757,17 +1745,23 @@ dnode_hold(objset_t *os, uint64_t object, const void *tag, dnode_t **dnp)
  * reference on the dnode.  Returns FALSE if unable to add a
  * new reference.
  */
+static boolean_t
+dnode_add_ref_locked(dnode_t *dn, const void *tag)
+{
+	ASSERT(MUTEX_HELD(&dn->dn_mtx));
+	if (zfs_refcount_is_zero(&dn->dn_holds))
+		return (FALSE);
+	VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag));
+	return (TRUE);
+}
+
 boolean_t
 dnode_add_ref(dnode_t *dn, const void *tag)
 {
 	mutex_enter(&dn->dn_mtx);
-	if (zfs_refcount_is_zero(&dn->dn_holds)) {
-		mutex_exit(&dn->dn_mtx);
-		return (FALSE);
-	}
-	VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag));
+	boolean_t r = dnode_add_ref_locked(dn, tag);
 	mutex_exit(&dn->dn_mtx);
-	return (TRUE);
+	return (r);
 }
 
 void
@@ -1830,31 +1824,20 @@ dnode_try_claim(objset_t *os, uint64_t object, int slots)
 }
 
 /*
- * Checks if the dnode itself is dirty, or is carrying any uncommitted records.
- * It is important to check both conditions, as some operations (eg appending
- * to a file) can dirty both as a single logical unit, but they are not synced
- * out atomically, so checking one and not the other can result in an object
- * appearing to be clean mid-way through a commit.
+ * Test if the dnode is dirty, or carrying uncommitted records.
  *
- * Do not change this lightly! If you get it wrong, dmu_offset_next() can
- * detect a hole where there is really data, leading to silent corruption.
+ * dn_dirtycnt is the number of txgs this dnode is dirty on. It's incremented
+ * in dnode_setdirty() the first time the dnode is dirtied on a txg, and
+ * decremented in either dnode_rele_task() or userquota_updates_task() when the
+ * txg is synced out.
  */
 boolean_t
 dnode_is_dirty(dnode_t *dn)
 {
 	mutex_enter(&dn->dn_mtx);
-
-	for (int i = 0; i < TXG_SIZE; i++) {
-		if (multilist_link_active(&dn->dn_dirty_link[i]) ||
-		    !list_is_empty(&dn->dn_dirty_records[i])) {
-			mutex_exit(&dn->dn_mtx);
-			return (B_TRUE);
-		}
-	}
-
+	boolean_t dirty = (dn->dn_dirtycnt != 0);
 	mutex_exit(&dn->dn_mtx);
-
-	return (B_FALSE);
+	return (dirty);
 }
 
 void
@@ -1916,7 +1899,11 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
 	 * dnode will hang around after we finish processing its
 	 * children.
 	 */
-	VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg));
+	mutex_enter(&dn->dn_mtx);
+	VERIFY(dnode_add_ref_locked(dn, (void *)(uintptr_t)tx->tx_txg));
+	dn->dn_dirtycnt++;
+	ASSERT3U(dn->dn_dirtycnt, <=, 3);
+	mutex_exit(&dn->dn_mtx);
 
 	(void) dbuf_dirty(dn->dn_dbuf, tx);
 
@@ -2221,32 +2208,6 @@ dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
 	mutex_exit(&dn->dn_dbufs_mtx);
 }
 
-void
-dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, const void *tag)
-{
-	/*
-	 * Don't set dirtyctx to SYNC if we're just modifying this as we
-	 * initialize the objset.
-	 */
-	if (dn->dn_dirtyctx == DN_UNDIRTIED) {
-		dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
-
-		if (ds != NULL) {
-			rrw_enter(&ds->ds_bp_rwlock, RW_READER, tag);
-		}
-		if (!BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
-			if (dmu_tx_is_syncing(tx))
-				dn->dn_dirtyctx = DN_DIRTY_SYNC;
-			else
-				dn->dn_dirtyctx = DN_DIRTY_OPEN;
-			dn->dn_dirtyctx_firstset = tag;
-		}
-		if (ds != NULL) {
-			rrw_exit(&ds->ds_bp_rwlock, tag);
-		}
-	}
-}
-
 static void
 dnode_partial_zero(dnode_t *dn, uint64_t off, uint64_t blkoff, uint64_t len,
     dmu_tx_t *tx)
@@ -2304,7 +2265,7 @@ dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
 		if ((off >> blkshift) > dn->dn_maxblkid)
 			return;
 	} else {
-		ASSERT(dn->dn_maxblkid == 0);
+		ASSERT0(dn->dn_maxblkid);
 		if (off == 0 && len >= blksz) {
 			/*
 			 * Freeing the whole block; fast-track this request.
@@ -2524,7 +2485,7 @@ dnode_diduse_space(dnode_t *dn, int64_t delta)
 	}
 	space += delta;
 	if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) {
-		ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0);
+		ASSERT0((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES));
 		ASSERT0(P2PHASE(space, 1<<DEV_BSHIFT));
 		dn->dn_phys->dn_used = space >> DEV_BSHIFT;
 	} else {
@@ -2695,6 +2656,32 @@ dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset,
 }
 
 /*
+ * Adjust *offset to the next (or previous) block byte offset at lvl.
+ * Returns FALSE if *offset would overflow or underflow.
+ */
+static boolean_t
+dnode_next_block(dnode_t *dn, int flags, uint64_t *offset, int lvl)
+{
+	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
+	int span = lvl * epbs + dn->dn_datablkshift;
+	uint64_t blkid, maxblkid;
+
+	if (span >= 8 * sizeof (uint64_t))
+		return (B_FALSE);
+
+	blkid = *offset >> span;
+	maxblkid = 1ULL << (8 * sizeof (*offset) - span);
+	if (!(flags & DNODE_FIND_BACKWARDS) && blkid + 1 < maxblkid)
+		*offset = (blkid + 1) << span;
+	else if ((flags & DNODE_FIND_BACKWARDS) && blkid > 0)
+		*offset = (blkid << span) - 1;
+	else
+		return (B_FALSE);
+
+	return (B_TRUE);
+}
+
+/*
  * Find the next hole, data, or sparse region at or after *offset.
  * The value 'blkfill' tells us how many items we expect to find
  * in an L0 data block; this value is 1 for normal objects,
@@ -2721,7 +2708,7 @@ int
 dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset,
     int minlvl, uint64_t blkfill, uint64_t txg)
 {
-	uint64_t initial_offset = *offset;
+	uint64_t matched = *offset;
 	int lvl, maxlvl;
 	int error = 0;
 
@@ -2745,16 +2732,36 @@ dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset,
 
 	maxlvl = dn->dn_phys->dn_nlevels;
 
-	for (lvl = minlvl; lvl <= maxlvl; lvl++) {
+	for (lvl = minlvl; lvl <= maxlvl; ) {
 		error = dnode_next_offset_level(dn,
 		    flags, offset, lvl, blkfill, txg);
-		if (error != ESRCH)
+		if (error == 0 && lvl > minlvl) {
+			--lvl;
+			matched = *offset;
+		} else if (error == ESRCH && lvl < maxlvl &&
+		    dnode_next_block(dn, flags, &matched, lvl)) {
+			/*
+			 * Continue search at next/prev offset in lvl+1 block.
+			 *
+			 * Usually we only search upwards at the start of the
+			 * search as higher level blocks point at a matching
+			 * minlvl block in most cases, but we backtrack if not.
+			 *
+			 * This can happen for txg > 0 searches if the block
+			 * contains only BPs/dnodes freed at that txg. It also
+			 * happens if we are still syncing out the tree, and
+			 * some BP's at higher levels are not updated yet.
+			 *
+			 * We must adjust offset to avoid coming back to the
+			 * same offset and getting stuck looping forever. This
+			 * also deals with the case where offset is already at
+			 * the beginning or end of the object.
+			 */
+			++lvl;
+			*offset = matched;
+		} else {
 			break;
-	}
-
-	while (error == 0 && --lvl >= minlvl) {
-		error = dnode_next_offset_level(dn,
-		    flags, offset, lvl, blkfill, txg);
+		}
 	}
 
 	/*
@@ -2766,9 +2773,6 @@ dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset,
 		error = 0;
 	}
 
-	if (error == 0 && (flags & DNODE_FIND_BACKWARDS ?
-	    initial_offset < *offset : initial_offset > *offset))
-		error = SET_ERROR(ESRCH);
 out:
 	if (!(flags & DNODE_FIND_HAVELOCK))
 		rw_exit(&dn->dn_struct_rwlock);
diff --git a/sys/contrib/openzfs/module/zfs/dnode_sync.c b/sys/contrib/openzfs/module/zfs/dnode_sync.c
index 4067f221f1bf..046ceddb3609 100644
--- a/sys/contrib/openzfs/module/zfs/dnode_sync.c
+++ b/sys/contrib/openzfs/module/zfs/dnode_sync.c
@@ -209,8 +209,8 @@ free_verify(dmu_buf_impl_t *db, uint64_t start, uint64_t end, dmu_tx_t *tx)
 		rw_exit(&dn->dn_struct_rwlock);
 		if (err == ENOENT)
 			continue;
-		ASSERT(err == 0);
-		ASSERT(child->db_level == 0);
+		ASSERT0(err);
+		ASSERT0(child->db_level);
 		dr = dbuf_find_dirty_eq(child, txg);
 
 		/* data_old better be zeroed */
@@ -868,7 +868,7 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx)
 	dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx);
 
 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
-		ASSERT3P(list_head(list), ==, NULL);
+		ASSERT0P(list_head(list));
 		dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);
 	}
 
diff --git a/sys/contrib/openzfs/module/zfs/dsl_bookmark.c b/sys/contrib/openzfs/module/zfs/dsl_bookmark.c
index fdc8b7b198f0..ee574c499f9f 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_bookmark.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_bookmark.c
@@ -243,7 +243,7 @@ dsl_bookmark_create_check_impl(dsl_pool_t *dp,
 	/* error is retval of the following if-cascade */
 	if (strchr(source, '@') != NULL) {
 		dsl_dataset_t *source_snap_ds;
-		ASSERT3S(snapshot_namecheck(source, NULL, NULL), ==, 0);
+		ASSERT0(snapshot_namecheck(source, NULL, NULL));
 		error = dsl_dataset_hold(dp, source, FTAG, &source_snap_ds);
 		if (error == 0) {
 			VERIFY(source_snap_ds->ds_is_snapshot);
@@ -258,7 +258,7 @@ dsl_bookmark_create_check_impl(dsl_pool_t *dp,
 		}
 	} else if (strchr(source, '#') != NULL) {
 		zfs_bookmark_phys_t source_phys;
-		ASSERT3S(bookmark_namecheck(source, NULL, NULL), ==, 0);
+		ASSERT0(bookmark_namecheck(source, NULL, NULL));
 		/*
 		 * Source must exists and be an earlier point in newbm_ds's
 		 * timeline (newbm_ds's origin may be a snap of source's ds)
@@ -501,7 +501,7 @@ dsl_bookmark_create_sync_impl_snap(const char *bookmark, const char *snapshot,
 		    sizeof (uint64_t) * num_redact_snaps);
 		local_rl->rl_phys->rlp_num_snaps = num_redact_snaps;
 		if (bookmark_redacted) {
-			ASSERT3P(redaction_list, ==, NULL);
+			ASSERT0P(redaction_list);
 			local_rl->rl_phys->rlp_last_blkid = UINT64_MAX;
 			local_rl->rl_phys->rlp_last_object = UINT64_MAX;
 			dsl_redaction_list_long_rele(local_rl, tag);
diff --git a/sys/contrib/openzfs/module/zfs/dsl_crypt.c b/sys/contrib/openzfs/module/zfs/dsl_crypt.c
index 6b6bb8d45b6b..f519b937edc0 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_crypt.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_crypt.c
@@ -534,7 +534,7 @@ out:
 static void
 dsl_crypto_key_free(dsl_crypto_key_t *dck)
 {
-	ASSERT(zfs_refcount_count(&dck->dck_holds) == 0);
+	ASSERT0(zfs_refcount_count(&dck->dck_holds));
 
 	/* destroy the zio_crypt_key_t */
 	zio_crypt_key_destroy(&dck->dck_key);
@@ -1912,7 +1912,7 @@ dsl_dataset_create_crypt_sync(uint64_t dsobj, dsl_dir_t *dd,
 
 	/* clones always use their origin's wrapping key */
 	if (dsl_dir_is_clone(dd)) {
-		ASSERT3P(dcp, ==, NULL);
+		ASSERT0P(dcp);
 
 		/*
 		 * If this is an encrypted clone we just need to clone the
diff --git a/sys/contrib/openzfs/module/zfs/dsl_dataset.c b/sys/contrib/openzfs/module/zfs/dsl_dataset.c
index b767c9641419..420687480a76 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_dataset.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_dataset.c
@@ -450,7 +450,7 @@ dsl_dataset_evict_sync(void *dbu)
 {
 	dsl_dataset_t *ds = dbu;
 
-	ASSERT(ds->ds_owner == NULL);
+	ASSERT0P(ds->ds_owner);
 
 	unique_remove(ds->ds_fsid_guid);
 }
@@ -460,7 +460,7 @@ dsl_dataset_evict_async(void *dbu)
 {
 	dsl_dataset_t *ds = dbu;
 
-	ASSERT(ds->ds_owner == NULL);
+	ASSERT0P(ds->ds_owner);
 
 	ds->ds_dbuf = NULL;
 
@@ -1187,7 +1187,7 @@ dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
 	ASSERT(origin == NULL || origin->ds_dir->dd_pool == dp);
 	ASSERT(origin == NULL || dsl_dataset_phys(origin)->ds_num_children > 0);
 	ASSERT(dmu_tx_is_syncing(tx));
-	ASSERT(dsl_dir_phys(dd)->dd_head_dataset_obj == 0);
+	ASSERT0(dsl_dir_phys(dd)->dd_head_dataset_obj);
 
 	dsobj = dmu_object_alloc(mos, DMU_OT_DSL_DATASET, 0,
 	    DMU_OT_DSL_DATASET, sizeof (dsl_dataset_phys_t), tx);
@@ -2112,7 +2112,7 @@ dsl_dataset_sync(dsl_dataset_t *ds, zio_t *rio, dmu_tx_t *tx)
 {
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(ds->ds_objset != NULL);
-	ASSERT(dsl_dataset_phys(ds)->ds_next_snap_obj == 0);
+	ASSERT0(dsl_dataset_phys(ds)->ds_next_snap_obj);
 
 	/*
 	 * in case we had to change ds_fsid_guid when we opened it,
@@ -4180,7 +4180,7 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	int64_t unused_refres_delta;
 
-	ASSERT(clone->ds_reserved == 0);
+	ASSERT0(clone->ds_reserved);
 	/*
 	 * NOTE: On DEBUG kernels there could be a race between this and
 	 * the check function if spa_asize_inflation is adjusted...
diff --git a/sys/contrib/openzfs/module/zfs/dsl_deadlist.c b/sys/contrib/openzfs/module/zfs/dsl_deadlist.c
index 9ffc998ac173..41ac72bf1c16 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_deadlist.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_deadlist.c
@@ -1037,7 +1037,7 @@ dsl_livelist_iterate(void *arg, const blkptr_t *bp, boolean_t bp_freed,
 	avl_tree_t *avl = lia->avl;
 	bplist_t *to_free = lia->to_free;
 	zthr_t *t = lia->t;
-	ASSERT(tx == NULL);
+	ASSERT0P(tx);
 
 	if ((t != NULL) && (zthr_has_waiters(t) || zthr_iscancelled(t)))
 		return (SET_ERROR(EINTR));
@@ -1049,7 +1049,8 @@ dsl_livelist_iterate(void *arg, const blkptr_t *bp, boolean_t bp_freed,
 		ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(&found->le_bp));
 		ASSERT3U(BP_GET_CHECKSUM(bp), ==,
 		    BP_GET_CHECKSUM(&found->le_bp));
-		ASSERT3U(BP_GET_BIRTH(bp), ==, BP_GET_BIRTH(&found->le_bp));
+		ASSERT3U(BP_GET_PHYSICAL_BIRTH(bp), ==,
+		    BP_GET_PHYSICAL_BIRTH(&found->le_bp));
 	}
 	if (bp_freed) {
 		if (found == NULL) {
diff --git a/sys/contrib/openzfs/module/zfs/dsl_deleg.c b/sys/contrib/openzfs/module/zfs/dsl_deleg.c
index c01a06e98340..fdd37b36e280 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_deleg.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_deleg.c
@@ -102,7 +102,7 @@ dsl_deleg_can_allow(char *ddname, nvlist_t *nvp, cred_t *cr)
 		nvlist_t *perms;
 		nvpair_t *permpair = NULL;
 
-		VERIFY(nvpair_value_nvlist(whopair, &perms) == 0);
+		VERIFY0(nvpair_value_nvlist(whopair, &perms));
 
 		while ((permpair = nvlist_next_nvpair(perms, permpair))) {
 			const char *perm = nvpair_name(permpair);
@@ -189,8 +189,7 @@ dsl_deleg_set_sync(void *arg, dmu_tx_t *tx)
 			const char *perm = nvpair_name(permpair);
 			uint64_t n = 0;
 
-			VERIFY(zap_update(mos, jumpobj,
-			    perm, 8, 1, &n, tx) == 0);
+			VERIFY0(zap_update(mos, jumpobj, perm, 8, 1, &n, tx));
 			spa_history_log_internal_dd(dd, "permission update", tx,
 			    "%s %s", whokey, perm);
 		}
@@ -225,7 +224,7 @@ dsl_deleg_unset_sync(void *arg, dmu_tx_t *tx)
 			if (zap_lookup(mos, zapobj, whokey, 8,
 			    1, &jumpobj) == 0) {
 				(void) zap_remove(mos, zapobj, whokey, tx);
-				VERIFY(0 == zap_destroy(mos, jumpobj, tx));
+				VERIFY0(zap_destroy(mos, jumpobj, tx));
 			}
 			spa_history_log_internal_dd(dd, "permission who remove",
 			    tx, "%s", whokey);
@@ -243,7 +242,7 @@ dsl_deleg_unset_sync(void *arg, dmu_tx_t *tx)
 			if (zap_count(mos, jumpobj, &n) == 0 && n == 0) {
 				(void) zap_remove(mos, zapobj,
 				    whokey, tx);
-				VERIFY(0 == zap_destroy(mos,
+				VERIFY0(zap_destroy(mos,
 				    jumpobj, tx));
 			}
 			spa_history_log_internal_dd(dd, "permission remove", tx,
@@ -332,7 +331,7 @@ dsl_deleg_get(const char *ddname, nvlist_t **nvp)
 	basezc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
 	baseza = zap_attribute_alloc();
 	source = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
-	VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+	VERIFY0(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP));
 
 	for (dd = startdd; dd != NULL; dd = dd->dd_parent) {
 		nvlist_t *sp_nvp;
@@ -706,7 +705,7 @@ copy_create_perms(dsl_dir_t *dd, uint64_t pzapobj,
 	    ZFS_DELEG_LOCAL, &uid);
 	if (zap_lookup(mos, zapobj, whokey, 8, 1, &jumpobj) == ENOENT) {
 		jumpobj = zap_create(mos, DMU_OT_DSL_PERMS, DMU_OT_NONE, 0, tx);
-		VERIFY(zap_add(mos, zapobj, whokey, 8, 1, &jumpobj, tx) == 0);
+		VERIFY0(zap_add(mos, zapobj, whokey, 8, 1, &jumpobj, tx));
 	}
 
 	za = zap_attribute_alloc();
@@ -716,8 +715,7 @@ copy_create_perms(dsl_dir_t *dd, uint64_t pzapobj,
 		uint64_t zero = 0;
 		ASSERT(za->za_integer_length == 8 && za->za_num_integers == 1);
 
-		VERIFY(zap_update(mos, jumpobj, za->za_name,
-		    8, 1, &zero, tx) == 0);
+		VERIFY0(zap_update(mos, jumpobj, za->za_name, 8, 1, &zero, tx));
 	}
 	zap_cursor_fini(&zc);
 	zap_attribute_free(za);
@@ -761,10 +759,10 @@ dsl_deleg_destroy(objset_t *mos, uint64_t zapobj, dmu_tx_t *tx)
 	    zap_cursor_retrieve(&zc, za) == 0;
 	    zap_cursor_advance(&zc)) {
 		ASSERT(za->za_integer_length == 8 && za->za_num_integers == 1);
-		VERIFY(0 == zap_destroy(mos, za->za_first_integer, tx));
+		VERIFY0(zap_destroy(mos, za->za_first_integer, tx));
 	}
 	zap_cursor_fini(&zc);
-	VERIFY(0 == zap_destroy(mos, zapobj, tx));
+	VERIFY0(zap_destroy(mos, zapobj, tx));
 	zap_attribute_free(za);
 	return (0);
 }
diff --git a/sys/contrib/openzfs/module/zfs/dsl_destroy.c b/sys/contrib/openzfs/module/zfs/dsl_destroy.c
index fff49c97f4d2..ea01ee586f8b 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_destroy.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_destroy.c
@@ -350,7 +350,7 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 			dsl_dataset_deactivate_feature(ds, f, tx);
 	}
 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
-		ASSERT3P(ds->ds_prev, ==, NULL);
+		ASSERT0P(ds->ds_prev);
 		VERIFY0(dsl_dataset_hold_obj(dp,
 		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &ds_prev));
 		after_branch_point =
@@ -465,7 +465,7 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 		    &used, &comp, &uncomp);
 		dsl_dataset_phys(ds_next)->ds_unique_bytes += used;
 		dsl_dataset_rele(ds_nextnext, FTAG);
-		ASSERT3P(ds_next->ds_prev, ==, NULL);
+		ASSERT0P(ds_next->ds_prev);
 
 		/* Collapse range in this head. */
 		dsl_dataset_t *hds;
@@ -525,7 +525,7 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 
 	/* remove from snapshot namespace */
 	dsl_dataset_t *ds_head;
-	ASSERT(dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0);
+	ASSERT0(dsl_dataset_phys(ds)->ds_snapnames_zapobj);
 	VERIFY0(dsl_dataset_hold_obj(dp,
 	    dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj, FTAG, &ds_head));
 	VERIFY0(dsl_dataset_get_snapname(ds));
@@ -728,7 +728,7 @@ kill_blkptr(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 		 */
 		dsl_free(ka->tx->tx_pool, ka->tx->tx_txg, bp);
 	} else {
-		ASSERT(zilog == NULL);
+		ASSERT0P(zilog);
 		ASSERT3U(BP_GET_BIRTH(bp), >,
 		    dsl_dataset_phys(ka->ds)->ds_prev_snap_txg);
 		(void) dsl_dataset_block_kill(ka->ds, bp, tx, B_FALSE);
diff --git a/sys/contrib/openzfs/module/zfs/dsl_dir.c b/sys/contrib/openzfs/module/zfs/dsl_dir.c
index f24cd2049533..6ce1890cfea1 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_dir.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_dir.c
@@ -151,8 +151,8 @@ dsl_dir_evict_async(void *dbu)
 
 	for (t = 0; t < TXG_SIZE; t++) {
 		ASSERT(!txg_list_member(&dp->dp_dirty_dirs, dd, t));
-		ASSERT(dd->dd_tempreserved[t] == 0);
-		ASSERT(dd->dd_space_towrite[t] == 0);
+		ASSERT0(dd->dd_tempreserved[t]);
+		ASSERT0(dd->dd_space_towrite[t]);
 	}
 
 	if (dd->dd_parent)
diff --git a/sys/contrib/openzfs/module/zfs/dsl_pool.c b/sys/contrib/openzfs/module/zfs/dsl_pool.c
index 4f1f66b835f2..f47822df8b53 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_pool.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_pool.c
@@ -522,8 +522,8 @@ dsl_pool_create(spa_t *spa, nvlist_t *zplprops __attribute__((unused)),
 
 		/* create and open the free_bplist */
 		obj = bpobj_alloc(dp->dp_meta_objset, SPA_OLD_MAXBLOCKSIZE, tx);
-		VERIFY(zap_add(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
-		    DMU_POOL_FREE_BPOBJ, sizeof (uint64_t), 1, &obj, tx) == 0);
+		VERIFY0(zap_add(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
+		    DMU_POOL_FREE_BPOBJ, sizeof (uint64_t), 1, &obj, tx));
 		VERIFY0(bpobj_open(&dp->dp_free_bpobj,
 		    dp->dp_meta_objset, obj));
 	}
@@ -1077,7 +1077,7 @@ upgrade_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 		dsl_dataset_phys(prev)->ds_num_children++;
 
 		if (dsl_dataset_phys(ds)->ds_next_snap_obj == 0) {
-			ASSERT(ds->ds_prev == NULL);
+			ASSERT0P(ds->ds_prev);
 			VERIFY0(dsl_dataset_hold_obj(dp,
 			    dsl_dataset_phys(ds)->ds_prev_snap_obj,
 			    ds, &ds->ds_prev));
@@ -1173,7 +1173,7 @@ dsl_pool_create_origin(dsl_pool_t *dp, dmu_tx_t *tx)
 	dsl_dataset_t *ds;
 
 	ASSERT(dmu_tx_is_syncing(tx));
-	ASSERT(dp->dp_origin_snap == NULL);
+	ASSERT0P(dp->dp_origin_snap);
 	ASSERT(rrw_held(&dp->dp_config_rwlock, RW_WRITER));
 
 	/* create the origin dir, ds, & snap-ds */
@@ -1250,7 +1250,7 @@ dsl_pool_user_hold_create_obj(dsl_pool_t *dp, dmu_tx_t *tx)
 {
 	objset_t *mos = dp->dp_meta_objset;
 
-	ASSERT(dp->dp_tmp_userrefs_obj == 0);
+	ASSERT0(dp->dp_tmp_userrefs_obj);
 	ASSERT(dmu_tx_is_syncing(tx));
 
 	dp->dp_tmp_userrefs_obj = zap_create_link(mos, DMU_OT_USERREFS,
diff --git a/sys/contrib/openzfs/module/zfs/dsl_prop.c b/sys/contrib/openzfs/module/zfs/dsl_prop.c
index b76f22df61e2..51f624da5689 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_prop.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_prop.c
@@ -815,7 +815,7 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 		 */
 		err = zap_update(mos, zapobj, recvdstr,
 		    intsz, numints, value, tx);
-		ASSERT(err == 0);
+		ASSERT0(err);
 		break;
 	case (ZPROP_SRC_NONE | ZPROP_SRC_LOCAL | ZPROP_SRC_RECEIVED):
 		/*
@@ -1166,7 +1166,7 @@ dsl_prop_get_all_impl(objset_t *mos, uint64_t propobj,
 		if (nvlist_exists(nv, propname))
 			continue;
 
-		VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+		VERIFY0(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP));
 		if (za->za_integer_length == 1) {
 			/*
 			 * String property
@@ -1179,8 +1179,7 @@ dsl_prop_get_all_impl(objset_t *mos, uint64_t propobj,
 				kmem_free(tmp, za->za_num_integers);
 				break;
 			}
-			VERIFY(nvlist_add_string(propval, ZPROP_VALUE,
-			    tmp) == 0);
+			VERIFY0(nvlist_add_string(propval, ZPROP_VALUE, tmp));
 			kmem_free(tmp, za->za_num_integers);
 		} else {
 			/*
@@ -1191,8 +1190,8 @@ dsl_prop_get_all_impl(objset_t *mos, uint64_t propobj,
 			    za->za_first_integer);
 		}
 
-		VERIFY(nvlist_add_string(propval, ZPROP_SOURCE, source) == 0);
-		VERIFY(nvlist_add_nvlist(nv, propname, propval) == 0);
+		VERIFY0(nvlist_add_string(propval, ZPROP_SOURCE, source));
+		VERIFY0(nvlist_add_nvlist(nv, propname, propval));
 		nvlist_free(propval);
 	}
 	zap_cursor_fini(&zc);
@@ -1215,7 +1214,7 @@ dsl_prop_get_all_ds(dsl_dataset_t *ds, nvlist_t **nvp,
 	int err = 0;
 	char setpoint[ZFS_MAX_DATASET_NAME_LEN];
 
-	VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+	VERIFY0(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP));
 
 	if (ds->ds_is_snapshot)
 		flags |= DSL_PROP_GET_SNAPSHOT;
@@ -1333,18 +1332,18 @@ dsl_prop_nvlist_add_uint64(nvlist_t *nv, zfs_prop_t prop, uint64_t value)
 	uint64_t default_value;
 
 	if (nvlist_lookup_nvlist(nv, propname, &propval) == 0) {
-		VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, value) == 0);
+		VERIFY0(nvlist_add_uint64(propval, ZPROP_VALUE, value));
 		return;
 	}
 
-	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
-	VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, value) == 0);
+	VERIFY0(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP));
+	VERIFY0(nvlist_add_uint64(propval, ZPROP_VALUE, value));
 	/* Indicate the default source if we can. */
 	if (dodefault(prop, 8, 1, &default_value) == 0 &&
 	    value == default_value) {
-		VERIFY(nvlist_add_string(propval, ZPROP_SOURCE, "") == 0);
+		VERIFY0(nvlist_add_string(propval, ZPROP_SOURCE, ""));
 	}
-	VERIFY(nvlist_add_nvlist(nv, propname, propval) == 0);
+	VERIFY0(nvlist_add_nvlist(nv, propname, propval));
 	nvlist_free(propval);
 }
 
@@ -1355,13 +1354,13 @@ dsl_prop_nvlist_add_string(nvlist_t *nv, zfs_prop_t prop, const char *value)
 	const char *propname = zfs_prop_to_name(prop);
 
 	if (nvlist_lookup_nvlist(nv, propname, &propval) == 0) {
-		VERIFY(nvlist_add_string(propval, ZPROP_VALUE, value) == 0);
+		VERIFY0(nvlist_add_string(propval, ZPROP_VALUE, value));
 		return;
 	}
 
-	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
-	VERIFY(nvlist_add_string(propval, ZPROP_VALUE, value) == 0);
-	VERIFY(nvlist_add_nvlist(nv, propname, propval) == 0);
+	VERIFY0(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP));
+	VERIFY0(nvlist_add_string(propval, ZPROP_VALUE, value));
+	VERIFY0(nvlist_add_nvlist(nv, propname, propval));
 	nvlist_free(propval);
 }
 
diff --git a/sys/contrib/openzfs/module/zfs/dsl_scan.c b/sys/contrib/openzfs/module/zfs/dsl_scan.c
index 5052992d775c..fcd50c459d07 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_scan.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_scan.c
@@ -1784,7 +1784,7 @@ dsl_scan_zil_block(zilog_t *zilog, const blkptr_t *bp, void *arg,
 	SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET],
 	    ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]);
 
-	VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
+	VERIFY0(scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
 	return (0);
 }
 
@@ -1820,7 +1820,7 @@ dsl_scan_zil_record(zilog_t *zilog, const lr_t *lrc, void *arg,
 		    lr->lr_foid, ZB_ZIL_LEVEL,
 		    lr->lr_offset / BP_GET_LSIZE(bp));
 
-		VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
+		VERIFY0(scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
 	}
 	return (0);
 }
@@ -5141,7 +5141,7 @@ dsl_scan_io_queue_vdev_xfer(vdev_t *svd, vdev_t *tvd)
 	mutex_enter(&svd->vdev_scan_io_queue_lock);
 	mutex_enter(&tvd->vdev_scan_io_queue_lock);
 
-	VERIFY3P(tvd->vdev_scan_io_queue, ==, NULL);
+	VERIFY0P(tvd->vdev_scan_io_queue);
 	tvd->vdev_scan_io_queue = svd->vdev_scan_io_queue;
 	svd->vdev_scan_io_queue = NULL;
 	if (tvd->vdev_scan_io_queue != NULL)
diff --git a/sys/contrib/openzfs/module/zfs/dsl_userhold.c b/sys/contrib/openzfs/module/zfs/dsl_userhold.c
index 57c70e4ce3d2..f91b7a1eb69a 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_userhold.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_userhold.c
@@ -335,7 +335,7 @@ dsl_dataset_user_hold(nvlist_t *holds, minor_t cleanup_minor, nvlist_t *errlist)
 
 	dduha.dduha_holds = holds;
 	/* chkholds can have non-unique name */
-	VERIFY(0 == nvlist_alloc(&dduha.dduha_chkholds, 0, KM_SLEEP));
+	VERIFY0(nvlist_alloc(&dduha.dduha_chkholds, 0, KM_SLEEP));
 	dduha.dduha_errlist = errlist;
 	dduha.dduha_minor = cleanup_minor;
 
diff --git a/sys/contrib/openzfs/module/zfs/fm.c b/sys/contrib/openzfs/module/zfs/fm.c
index a092817efedd..ae788b2310d8 100644
--- a/sys/contrib/openzfs/module/zfs/fm.c
+++ b/sys/contrib/openzfs/module/zfs/fm.c
@@ -337,7 +337,7 @@ zfs_zevent_next(zfs_zevent_t *ze, nvlist_t **event, uint64_t *event_size,
 		}
 	}
 
-	VERIFY(nvlist_size(ev->ev_nvl, &size, NV_ENCODE_NATIVE) == 0);
+	VERIFY0(nvlist_size(ev->ev_nvl, &size, NV_ENCODE_NATIVE));
 	if (size > *event_size) {
 		*event_size = size;
 		error = ENOMEM;
diff --git a/sys/contrib/openzfs/module/zfs/metaslab.c b/sys/contrib/openzfs/module/zfs/metaslab.c
index 0e5f09b2724c..9f4399af56bd 100644
--- a/sys/contrib/openzfs/module/zfs/metaslab.c
+++ b/sys/contrib/openzfs/module/zfs/metaslab.c
@@ -391,7 +391,7 @@ static kstat_t *metaslab_ksp;
 void
 metaslab_stat_init(void)
 {
-	ASSERT(metaslab_alloc_trace_cache == NULL);
+	ASSERT0P(metaslab_alloc_trace_cache);
 	metaslab_alloc_trace_cache = kmem_cache_create(
 	    "metaslab_alloc_trace_cache", sizeof (metaslab_alloc_trace_t),
 	    0, NULL, NULL, NULL, NULL, NULL, 0);
@@ -456,16 +456,16 @@ metaslab_class_destroy(metaslab_class_t *mc)
 {
 	spa_t *spa = mc->mc_spa;
 
-	ASSERT(mc->mc_alloc == 0);
-	ASSERT(mc->mc_deferred == 0);
-	ASSERT(mc->mc_space == 0);
-	ASSERT(mc->mc_dspace == 0);
+	ASSERT0(mc->mc_alloc);
+	ASSERT0(mc->mc_deferred);
+	ASSERT0(mc->mc_space);
+	ASSERT0(mc->mc_dspace);
 
 	for (int i = 0; i < spa->spa_alloc_count; i++) {
 		metaslab_class_allocator_t *mca = &mc->mc_allocator[i];
 		avl_destroy(&mca->mca_tree);
 		mutex_destroy(&mca->mca_lock);
-		ASSERT(mca->mca_rotor == NULL);
+		ASSERT0P(mca->mca_rotor);
 		ASSERT0(mca->mca_reserved);
 	}
 	mutex_destroy(&mc->mc_lock);
@@ -1087,8 +1087,8 @@ metaslab_group_destroy(metaslab_group_t *mg)
 {
 	spa_t *spa = mg->mg_class->mc_spa;
 
-	ASSERT(mg->mg_prev == NULL);
-	ASSERT(mg->mg_next == NULL);
+	ASSERT0P(mg->mg_prev);
+	ASSERT0P(mg->mg_next);
 	/*
 	 * We may have gone below zero with the activation count
 	 * either because we never activated in the first place or
@@ -1118,8 +1118,8 @@ metaslab_group_activate(metaslab_group_t *mg)
 
 	ASSERT3U(spa_config_held(spa, SCL_ALLOC, RW_WRITER), !=, 0);
 
-	ASSERT(mg->mg_prev == NULL);
-	ASSERT(mg->mg_next == NULL);
+	ASSERT0P(mg->mg_prev);
+	ASSERT0P(mg->mg_next);
 	ASSERT(mg->mg_activation_count <= 0);
 
 	if (++mg->mg_activation_count <= 0)
@@ -1164,8 +1164,8 @@ metaslab_group_passivate(metaslab_group_t *mg)
 	if (--mg->mg_activation_count != 0) {
 		for (int i = 0; i < spa->spa_alloc_count; i++)
 			ASSERT(mc->mc_allocator[i].mca_rotor != mg);
-		ASSERT(mg->mg_prev == NULL);
-		ASSERT(mg->mg_next == NULL);
+		ASSERT0P(mg->mg_prev);
+		ASSERT0P(mg->mg_next);
 		ASSERT(mg->mg_activation_count < 0);
 		return;
 	}
@@ -1345,7 +1345,7 @@ metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp)
 static void
 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp)
 {
-	ASSERT(msp->ms_group == NULL);
+	ASSERT0P(msp->ms_group);
 	mutex_enter(&mg->mg_lock);
 	msp->ms_group = mg;
 	msp->ms_weight = 0;
@@ -3017,7 +3017,7 @@ metaslab_fini(metaslab_t *msp)
 	metaslab_group_remove(mg, msp);
 
 	mutex_enter(&msp->ms_lock);
-	VERIFY(msp->ms_group == NULL);
+	VERIFY0P(msp->ms_group);
 
 	/*
 	 * If this metaslab hasn't been through metaslab_sync_done() yet its
@@ -5739,7 +5739,7 @@ metaslab_unalloc_dva(spa_t *spa, const dva_t *dva, uint64_t txg)
 	ASSERT(!vd->vdev_removing);
 	ASSERT(vdev_is_concrete(vd));
 	ASSERT0(vd->vdev_indirect_config.vic_mapping_object);
-	ASSERT3P(vd->vdev_indirect_mapping, ==, NULL);
+	ASSERT0P(vd->vdev_indirect_mapping);
 
 	if (DVA_GET_GANG(dva))
 		size = vdev_gang_header_asize(vd);
@@ -5997,7 +5997,7 @@ metaslab_alloc_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
 	}
 
 	ASSERT(ndvas > 0 && ndvas <= spa_max_replication(spa));
-	ASSERT(BP_GET_NDVAS(bp) == 0);
+	ASSERT0(BP_GET_NDVAS(bp));
 	ASSERT(hintbp == NULL || ndvas <= BP_GET_NDVAS(hintbp));
 	ASSERT3P(zal, !=, NULL);
 
@@ -6029,7 +6029,7 @@ metaslab_alloc_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
 				smallest_psize = MIN(cur_psize, smallest_psize);
 		}
 	}
-	ASSERT(error == 0);
+	ASSERT0(error);
 	ASSERT(BP_GET_NDVAS(bp) == ndvas);
 	if (actual_psize)
 		*actual_psize = smallest_psize;
diff --git a/sys/contrib/openzfs/module/zfs/mmp.c b/sys/contrib/openzfs/module/zfs/mmp.c
index f3665d29b8b4..fd46127b6068 100644
--- a/sys/contrib/openzfs/module/zfs/mmp.c
+++ b/sys/contrib/openzfs/module/zfs/mmp.c
@@ -260,7 +260,7 @@ mmp_thread_stop(spa_t *spa)
 	zfs_dbgmsg("MMP thread stopped pool '%s' gethrtime %llu",
 	    spa_name(spa), gethrtime());
 
-	ASSERT(mmp->mmp_thread == NULL);
+	ASSERT0P(mmp->mmp_thread);
 	mmp->mmp_thread_exiting = 0;
 }
 
@@ -446,7 +446,7 @@ mmp_write_uberblock(spa_t *spa)
 	uint64_t offset;
 
 	hrtime_t lock_acquire_time = gethrtime();
-	spa_config_enter_mmp(spa, SCL_STATE, mmp_tag, RW_READER);
+	spa_config_enter_priority(spa, SCL_STATE, mmp_tag, RW_READER);
 	lock_acquire_time = gethrtime() - lock_acquire_time;
 	if (lock_acquire_time > (MSEC2NSEC(MMP_MIN_INTERVAL) / 10))
 		zfs_dbgmsg("MMP SCL_STATE acquisition pool '%s' took %llu ns "
diff --git a/sys/contrib/openzfs/module/zfs/multilist.c b/sys/contrib/openzfs/module/zfs/multilist.c
index 7b85d19e19ee..46fb79269310 100644
--- a/sys/contrib/openzfs/module/zfs/multilist.c
+++ b/sys/contrib/openzfs/module/zfs/multilist.c
@@ -81,7 +81,7 @@ multilist_create_impl(multilist_t *ml, size_t size, size_t offset,
 	ml->ml_num_sublists = num;
 	ml->ml_index_func = index_func;
 
-	ml->ml_sublists = kmem_zalloc(sizeof (multilist_sublist_t) *
+	ml->ml_sublists = vmem_zalloc(sizeof (multilist_sublist_t) *
 	    ml->ml_num_sublists, KM_SLEEP);
 
 	ASSERT3P(ml->ml_sublists, !=, NULL);
@@ -134,7 +134,7 @@ multilist_destroy(multilist_t *ml)
 	}
 
 	ASSERT3P(ml->ml_sublists, !=, NULL);
-	kmem_free(ml->ml_sublists,
+	vmem_free(ml->ml_sublists,
 	    sizeof (multilist_sublist_t) * ml->ml_num_sublists);
 
 	ml->ml_num_sublists = 0;
diff --git a/sys/contrib/openzfs/module/zfs/range_tree.c b/sys/contrib/openzfs/module/zfs/range_tree.c
index fc2b17606bd2..d73195f1a21f 100644
--- a/sys/contrib/openzfs/module/zfs/range_tree.c
+++ b/sys/contrib/openzfs/module/zfs/range_tree.c
@@ -377,7 +377,7 @@ zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill)
 		return;
 	}
 
-	ASSERT3P(rs, ==, NULL);
+	ASSERT0P(rs);
 
 	/*
 	 * Determine whether or not we will have to merge with our neighbors.
@@ -585,7 +585,7 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size,
 		 * the size, since we do not support removing partial segments
 		 * of range trees with gaps.
 		 */
-		zfs_zfs_rs_set_fill_raw(rs, rt, zfs_rs_get_end_raw(rs, rt) -
+		zfs_rs_set_fill_raw(rs, rt, zfs_rs_get_end_raw(rs, rt) -
 		    zfs_rs_get_start_raw(rs, rt));
 		zfs_range_tree_stat_incr(rt, &rs_tmp);
 
@@ -867,7 +867,7 @@ zfs_range_tree_remove_xor_add_segment(uint64_t start, uint64_t end,
 
 		next = zfs_btree_next(&removefrom->rt_root, &where, &where);
 	}
-	VERIFY3P(curr, ==, NULL);
+	VERIFY0P(curr);
 
 	if (start != end) {
 		VERIFY3U(start, <, end);
diff --git a/sys/contrib/openzfs/module/zfs/rrwlock.c b/sys/contrib/openzfs/module/zfs/rrwlock.c
index 8ee784619839..d0df39b93560 100644
--- a/sys/contrib/openzfs/module/zfs/rrwlock.c
+++ b/sys/contrib/openzfs/module/zfs/rrwlock.c
@@ -108,7 +108,7 @@ rrn_add(rrwlock_t *rrl, const void *tag)
 	rn->rn_rrl = rrl;
 	rn->rn_next = tsd_get(rrw_tsd_key);
 	rn->rn_tag = tag;
-	VERIFY(tsd_set(rrw_tsd_key, rn) == 0);
+	VERIFY0(tsd_set(rrw_tsd_key, rn));
 }
 
 /*
@@ -129,7 +129,7 @@ rrn_find_and_remove(rrwlock_t *rrl, const void *tag)
 			if (prev)
 				prev->rn_next = rn->rn_next;
 			else
-				VERIFY(tsd_set(rrw_tsd_key, rn->rn_next) == 0);
+				VERIFY0(tsd_set(rrw_tsd_key, rn->rn_next));
 			kmem_free(rn, sizeof (*rn));
 			return (B_TRUE);
 		}
@@ -155,7 +155,7 @@ rrw_destroy(rrwlock_t *rrl)
 {
 	mutex_destroy(&rrl->rr_lock);
 	cv_destroy(&rrl->rr_cv);
-	ASSERT(rrl->rr_writer == NULL);
+	ASSERT0P(rrl->rr_writer);
 	zfs_refcount_destroy(&rrl->rr_anon_rcount);
 	zfs_refcount_destroy(&rrl->rr_linked_rcount);
 }
@@ -188,7 +188,7 @@ rrw_enter_read_impl(rrwlock_t *rrl, boolean_t prio, const void *tag)
 	} else {
 		(void) zfs_refcount_add(&rrl->rr_anon_rcount, tag);
 	}
-	ASSERT(rrl->rr_writer == NULL);
+	ASSERT0P(rrl->rr_writer);
 	mutex_exit(&rrl->rr_lock);
 }
 
diff --git a/sys/contrib/openzfs/module/zfs/sa.c b/sys/contrib/openzfs/module/zfs/sa.c
index 5db470ce6242..7ad25d4d85ba 100644
--- a/sys/contrib/openzfs/module/zfs/sa.c
+++ b/sys/contrib/openzfs/module/zfs/sa.c
@@ -304,7 +304,7 @@ sa_get_spill(sa_handle_t *hdl)
 	if (hdl->sa_spill == NULL) {
 		if ((rc = dmu_spill_hold_existing(hdl->sa_bonus, NULL,
 		    &hdl->sa_spill)) == 0)
-			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
+			VERIFY0(sa_build_index(hdl, SA_SPILL));
 	} else {
 		rc = 0;
 	}
@@ -432,7 +432,7 @@ sa_add_layout_entry(objset_t *os, const sa_attr_type_t *attrs, int attr_count,
 
 		(void) snprintf(attr_name, sizeof (attr_name),
 		    "%d", (int)lot_num);
-		VERIFY(0 == zap_update(os, os->os_sa->sa_layout_attr_obj,
+		VERIFY0(zap_update(os, os->os_sa->sa_layout_attr_obj,
 		    attr_name, 2, attr_count, attrs, tx));
 	}
 
@@ -505,7 +505,7 @@ sa_resize_spill(sa_handle_t *hdl, uint32_t size, dmu_tx_t *tx)
 	}
 
 	error = dbuf_spill_set_blksz(hdl->sa_spill, blocksize, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 	return (error);
 }
 
@@ -717,7 +717,7 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
 
 		if (BUF_SPACE_NEEDED(spill_used, spillhdrsize) >
 		    hdl->sa_spill->db_size)
-			VERIFY(0 == sa_resize_spill(hdl,
+			VERIFY0(sa_resize_spill(hdl,
 			    BUF_SPACE_NEEDED(spill_used, spillhdrsize), tx));
 	}
 
@@ -791,7 +791,7 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
 		hdl->sa_bonus_tab = NULL;
 	}
 	if (!sa->sa_force_spill)
-		VERIFY(0 == sa_build_index(hdl, SA_BONUS));
+		VERIFY0(sa_build_index(hdl, SA_BONUS));
 	if (hdl->sa_spill) {
 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
 		if (!spilling) {
@@ -801,10 +801,10 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
 			dmu_buf_rele(hdl->sa_spill, NULL);
 			hdl->sa_spill = NULL;
 			hdl->sa_spill_tab = NULL;
-			VERIFY(0 == dmu_rm_spill(hdl->sa_os,
+			VERIFY0(dmu_rm_spill(hdl->sa_os,
 			    sa_handle_object(hdl), tx));
 		} else {
-			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
+			VERIFY0(sa_build_index(hdl, SA_SPILL));
 		}
 	}
 
@@ -1733,10 +1733,10 @@ sa_add_projid(sa_handle_t *hdl, dmu_tx_t *tx, uint64_t projid)
 		    NULL, dxattr_obj, dxattr_size);
 	}
 
-	VERIFY(dmu_set_bonustype(db, DMU_OT_SA, tx) == 0);
-	VERIFY(sa_replace_all_by_template_locked(hdl, attrs, count, tx) == 0);
+	VERIFY0(dmu_set_bonustype(db, DMU_OT_SA, tx));
+	VERIFY0(sa_replace_all_by_template_locked(hdl, attrs, count, tx));
 	if (znode_acl.z_acl_extern_obj) {
-		VERIFY(0 == dmu_object_free(zfsvfs->z_os,
+		VERIFY0(dmu_object_free(zfsvfs->z_os,
 		    znode_acl.z_acl_extern_obj, tx));
 	}
 
@@ -1858,7 +1858,7 @@ sa_attr_register_sync(sa_handle_t *hdl, dmu_tx_t *tx)
 			continue;
 		ATTR_ENCODE(attr_value, tb[i].sa_attr, tb[i].sa_length,
 		    tb[i].sa_byteswap);
-		VERIFY(0 == zap_update(hdl->sa_os, sa->sa_reg_attr_obj,
+		VERIFY0(zap_update(hdl->sa_os, sa->sa_reg_attr_obj,
 		    tb[i].sa_name, 8, 1, &attr_value, tx));
 		tb[i].sa_registered = B_TRUE;
 	}
@@ -2013,7 +2013,7 @@ sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
 				 * Only a variable-sized attribute can be
 				 * replaced here, and its size must be changing.
 				 */
-				ASSERT3U(reg_length, ==, 0);
+				ASSERT0(reg_length);
 				ASSERT3U(length, !=, buflen);
 				SA_ADD_BULK_ATTR(attr_desc, j, attr,
 				    locator, datastart, buflen);
diff --git a/sys/contrib/openzfs/module/zfs/spa.c b/sys/contrib/openzfs/module/zfs/spa.c
index 5ecb175fbd63..b3bb46da263b 100644
--- a/sys/contrib/openzfs/module/zfs/spa.c
+++ b/sys/contrib/openzfs/module/zfs/spa.c
@@ -426,10 +426,10 @@ spa_prop_add_user(nvlist_t *nvl, const char *propname, char *strval,
 {
 	nvlist_t *propval;
 
-	VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
-	VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0);
-	VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0);
-	VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0);
+	VERIFY0(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP));
+	VERIFY0(nvlist_add_uint64(propval, ZPROP_SOURCE, src));
+	VERIFY0(nvlist_add_string(propval, ZPROP_VALUE, strval));
+	VERIFY0(nvlist_add_nvlist(nvl, propname, propval));
 	nvlist_free(propval);
 }
 
@@ -965,7 +965,7 @@ spa_prop_set(spa_t *spa, nvlist_t *nvp)
 			uint64_t ver = 0;
 
 			if (prop == ZPOOL_PROP_VERSION) {
-				VERIFY(nvpair_value_uint64(elem, &ver) == 0);
+				VERIFY0(nvpair_value_uint64(elem, &ver));
 			} else {
 				ASSERT(zpool_prop_feature(nvpair_name(elem)));
 				ver = SPA_VERSION_FEATURES;
@@ -1295,7 +1295,7 @@ spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q)
 	spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
 
 	if (tqs->stqs_taskq == NULL) {
-		ASSERT3U(tqs->stqs_count, ==, 0);
+		ASSERT0(tqs->stqs_count);
 		return;
 	}
 
@@ -1836,9 +1836,9 @@ static void
 spa_deactivate(spa_t *spa)
 {
 	ASSERT(spa->spa_sync_on == B_FALSE);
-	ASSERT(spa->spa_dsl_pool == NULL);
-	ASSERT(spa->spa_root_vdev == NULL);
-	ASSERT(spa->spa_async_zio_root == NULL);
+	ASSERT0P(spa->spa_dsl_pool);
+	ASSERT0P(spa->spa_root_vdev);
+	ASSERT0P(spa->spa_async_zio_root);
 	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
 
 	spa_evicting_os_wait(spa);
@@ -2021,7 +2021,7 @@ spa_unload_log_sm_flush_all(spa_t *spa)
 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
 	VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND));
 
-	ASSERT3U(spa->spa_log_flushall_txg, ==, 0);
+	ASSERT0(spa->spa_log_flushall_txg);
 	spa->spa_log_flushall_txg = dmu_tx_get_txg(tx);
 
 	dmu_tx_commit(tx);
@@ -2280,7 +2280,7 @@ spa_unload(spa_t *spa)
 	 */
 	if (spa->spa_root_vdev)
 		vdev_free(spa->spa_root_vdev);
-	ASSERT(spa->spa_root_vdev == NULL);
+	ASSERT0P(spa->spa_root_vdev);
 
 	/*
 	 * Close the dsl pool.
@@ -2418,8 +2418,8 @@ spa_load_spares(spa_t *spa)
 	spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *),
 	    KM_SLEEP);
 	for (i = 0; i < spa->spa_spares.sav_count; i++) {
-		VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0,
-		    VDEV_ALLOC_SPARE) == 0);
+		VERIFY0(spa_config_parse(spa, &vd, spares[i], NULL, 0,
+		    VDEV_ALLOC_SPARE));
 		ASSERT(vd != NULL);
 
 		spa->spa_spares.sav_vdevs[i] = vd;
@@ -2546,8 +2546,8 @@ spa_load_l2cache(spa_t *spa)
 			/*
 			 * Create new vdev
 			 */
-			VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
-			    VDEV_ALLOC_L2CACHE) == 0);
+			VERIFY0(spa_config_parse(spa, &vd, l2cache[i], NULL, 0,
+			    VDEV_ALLOC_L2CACHE));
 			ASSERT(vd != NULL);
 			newvdevs[i] = vd;
 
@@ -2799,7 +2799,7 @@ spa_passivate_log(spa_t *spa)
 		vdev_t *tvd = rvd->vdev_child[c];
 
 		if (tvd->vdev_islog) {
-			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
+			ASSERT0P(tvd->vdev_log_mg);
 			metaslab_group_passivate(tvd->vdev_mg);
 			slog_found = B_TRUE;
 		}
@@ -2822,7 +2822,7 @@ spa_activate_log(spa_t *spa)
 		vdev_t *tvd = rvd->vdev_child[c];
 
 		if (tvd->vdev_islog) {
-			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
+			ASSERT0P(tvd->vdev_log_mg);
 			metaslab_group_activate(tvd->vdev_mg);
 		}
 	}
@@ -3259,7 +3259,7 @@ spa_livelist_delete_cb(void *arg, zthr_t *z)
 static void
 spa_start_livelist_destroy_thread(spa_t *spa)
 {
-	ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL);
+	ASSERT0P(spa->spa_livelist_delete_zthr);
 	spa->spa_livelist_delete_zthr =
 	    zthr_create("z_livelist_destroy",
 	    spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa,
@@ -3275,7 +3275,7 @@ static int
 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
     dmu_tx_t *tx)
 {
-	ASSERT(tx == NULL);
+	ASSERT0P(tx);
 	livelist_new_arg_t *lna = arg;
 	if (bp_freed) {
 		bplist_append(lna->frees, bp);
@@ -3469,7 +3469,7 @@ spa_start_livelist_condensing_thread(spa_t *spa)
 	spa->spa_to_condense.syncing = B_FALSE;
 	spa->spa_to_condense.cancelled = B_FALSE;
 
-	ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL);
+	ASSERT0P(spa->spa_livelist_condense_zthr);
 	spa->spa_livelist_condense_zthr =
 	    zthr_create("z_livelist_condense",
 	    spa_livelist_condense_cb_check,
@@ -3486,7 +3486,7 @@ spa_spawn_aux_threads(spa_t *spa)
 	spa_start_livelist_destroy_thread(spa);
 	spa_start_livelist_condensing_thread(spa);
 
-	ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL);
+	ASSERT0P(spa->spa_checkpoint_discard_zthr);
 	spa->spa_checkpoint_discard_zthr =
 	    zthr_create("z_checkpoint_discard",
 	    spa_checkpoint_discard_thread_check,
@@ -4091,11 +4091,11 @@ spa_ld_parse_config(spa_t *spa, spa_import_type_t type)
 	nvlist_free(spa->spa_load_info);
 	spa->spa_load_info = fnvlist_alloc();
 
-	ASSERT(spa->spa_comment == NULL);
+	ASSERT0P(spa->spa_comment);
 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
 		spa->spa_comment = spa_strdup(comment);
 
-	ASSERT(spa->spa_compatibility == NULL);
+	ASSERT0P(spa->spa_compatibility);
 	if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY,
 	    &compatibility) == 0)
 		spa->spa_compatibility = spa_strdup(compatibility);
@@ -5913,7 +5913,7 @@ spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request,
 		nvlist_free(config);
 
 	if (state == SPA_LOAD_RECOVER) {
-		ASSERT3P(loadinfo, ==, NULL);
+		ASSERT0P(loadinfo);
 		spa_import_progress_remove(spa_guid(spa));
 		return (rewind_error);
 	} else {
@@ -9091,7 +9091,7 @@ spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru)
 int
 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd)
 {
-	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
+	ASSERT0(spa_config_held(spa, SCL_ALL, RW_WRITER));
 
 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
 		return (SET_ERROR(EBUSY));
@@ -9102,7 +9102,7 @@ spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd)
 int
 spa_scan_stop(spa_t *spa)
 {
-	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
+	ASSERT0(spa_config_held(spa, SCL_ALL, RW_WRITER));
 	if (dsl_scan_resilvering(spa->spa_dsl_pool))
 		return (SET_ERROR(EBUSY));
 
@@ -9119,7 +9119,7 @@ int
 spa_scan_range(spa_t *spa, pool_scan_func_t func, uint64_t txgstart,
     uint64_t txgend)
 {
-	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0);
+	ASSERT0(spa_config_held(spa, SCL_ALL, RW_WRITER));
 
 	if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE)
 		return (SET_ERROR(ENOTSUP));
@@ -9548,7 +9548,7 @@ spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx)
 {
 	zio_t *zio = zio_root(spa, NULL, NULL, 0);
 	bplist_iterate(bpl, spa_free_sync_cb, zio, tx);
-	VERIFY(zio_wait(zio) == 0);
+	VERIFY0(zio_wait(zio));
 }
 
 /*
@@ -9587,7 +9587,7 @@ spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx)
 	size_t nvsize = 0;
 	dmu_buf_t *db;
 
-	VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0);
+	VERIFY0(nvlist_size(nv, &nvsize, NV_ENCODE_XDR));
 
 	/*
 	 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration
@@ -9597,15 +9597,15 @@ spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx)
 	bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE);
 	packed = vmem_alloc(bufsize, KM_SLEEP);
 
-	VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
-	    KM_SLEEP) == 0);
+	VERIFY0(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR,
+	    KM_SLEEP));
 	memset(packed + nvsize, 0, bufsize - nvsize);
 
 	dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx);
 
 	vmem_free(packed, bufsize);
 
-	VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
+	VERIFY0(dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
 	dmu_buf_will_dirty(db, tx);
 	*(uint64_t *)db->db_data = nvsize;
 	dmu_buf_rele(db, FTAG);
@@ -10541,7 +10541,7 @@ spa_sync_tq_create(spa_t *spa, const char *name)
 {
 	kthread_t **kthreads;
 
-	ASSERT(spa->spa_sync_tq == NULL);
+	ASSERT0P(spa->spa_sync_tq);
 	ASSERT3S(spa->spa_alloc_count, <=, boot_ncpus);
 
 	/*
diff --git a/sys/contrib/openzfs/module/zfs/spa_config.c b/sys/contrib/openzfs/module/zfs/spa_config.c
index 7d4d06659146..f615591e826b 100644
--- a/sys/contrib/openzfs/module/zfs/spa_config.c
+++ b/sys/contrib/openzfs/module/zfs/spa_config.c
@@ -48,18 +48,17 @@
 /*
  * Pool configuration repository.
  *
- * Pool configuration is stored as a packed nvlist on the filesystem.  By
- * default, all pools are stored in /etc/zfs/zpool.cache and loaded on boot
- * (when the ZFS module is loaded).  Pools can also have the 'cachefile'
- * property set that allows them to be stored in an alternate location until
- * the control of external software.
+ * Pool configuration is stored as a packed nvlist on the filesystem.  When
+ * pools are imported they are added to the /etc/zfs/zpool.cache file and
+ * removed from it when exported.  For each cache file, we have a single nvlist
+ * which holds all the configuration information.  Pools can also have the
+ * 'cachefile' property set which allows this config to be stored in an
+ * alternate location under the control of external software.
  *
- * For each cache file, we have a single nvlist which holds all the
- * configuration information.  When the module loads, we read this information
- * from /etc/zfs/zpool.cache and populate the SPA namespace.  This namespace is
- * maintained independently in spa.c.  Whenever the namespace is modified, or
- * the configuration of a pool is changed, we call spa_write_cachefile(), which
- * walks through all the active pools and writes the configuration to disk.
+ * The kernel independantly maintains an AVL tree of imported pools.  See the
+ * "SPA locking" comment in spa.c.  Whenever a pool configuration is modified
+ * we call spa_write_cachefile() which walks through all the active pools and
+ * writes the updated configuration to to /etc/zfs/zpool.cache file.
  */
 
 static uint64_t spa_config_generation = 1;
@@ -69,94 +68,6 @@ static uint64_t spa_config_generation = 1;
  * userland pools when doing testing.
  */
 char *spa_config_path = (char *)ZPOOL_CACHE;
-#ifdef _KERNEL
-static int zfs_autoimport_disable = B_TRUE;
-#endif
-
-/*
- * Called when the module is first loaded, this routine loads the configuration
- * file into the SPA namespace.  It does not actually open or load the pools; it
- * only populates the namespace.
- */
-void
-spa_config_load(void)
-{
-	void *buf = NULL;
-	nvlist_t *nvlist, *child;
-	nvpair_t *nvpair;
-	char *pathname;
-	zfs_file_t *fp;
-	zfs_file_attr_t zfa;
-	uint64_t fsize;
-	int err;
-
-#ifdef _KERNEL
-	if (zfs_autoimport_disable)
-		return;
-#endif
-
-	/*
-	 * Open the configuration file.
-	 */
-	pathname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
-
-	(void) snprintf(pathname, MAXPATHLEN, "%s", spa_config_path);
-
-	err = zfs_file_open(pathname, O_RDONLY, 0, &fp);
-
-#ifdef __FreeBSD__
-	if (err)
-		err = zfs_file_open(ZPOOL_CACHE_BOOT, O_RDONLY, 0, &fp);
-#endif
-	kmem_free(pathname, MAXPATHLEN);
-
-	if (err)
-		return;
-
-	if (zfs_file_getattr(fp, &zfa))
-		goto out;
-
-	fsize = zfa.zfa_size;
-	buf = kmem_alloc(fsize, KM_SLEEP);
-
-	/*
-	 * Read the nvlist from the file.
-	 */
-	if (zfs_file_read(fp, buf, fsize, NULL) < 0)
-		goto out;
-
-	/*
-	 * Unpack the nvlist.
-	 */
-	if (nvlist_unpack(buf, fsize, &nvlist, KM_SLEEP) != 0)
-		goto out;
-
-	/*
-	 * Iterate over all elements in the nvlist, creating a new spa_t for
-	 * each one with the specified configuration.
-	 */
-	mutex_enter(&spa_namespace_lock);
-	nvpair = NULL;
-	while ((nvpair = nvlist_next_nvpair(nvlist, nvpair)) != NULL) {
-		if (nvpair_type(nvpair) != DATA_TYPE_NVLIST)
-			continue;
-
-		child = fnvpair_value_nvlist(nvpair);
-
-		if (spa_lookup(nvpair_name(nvpair)) != NULL)
-			continue;
-		(void) spa_add(nvpair_name(nvpair), child, NULL);
-	}
-	mutex_exit(&spa_namespace_lock);
-
-	nvlist_free(nvlist);
-
-out:
-	if (buf != NULL)
-		kmem_free(buf, fsize);
-
-	zfs_file_close(fp);
-}
 
 static int
 spa_config_remove(spa_config_dirent_t *dp)
@@ -461,6 +372,8 @@ spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, int getstats)
 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, txg);
 	fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, spa_guid(spa));
 	fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, spa->spa_errata);
+	fnvlist_add_uint64(config, ZPOOL_CONFIG_MIN_ALLOC, spa->spa_min_alloc);
+	fnvlist_add_uint64(config, ZPOOL_CONFIG_MAX_ALLOC, spa->spa_max_alloc);
 	if (spa->spa_comment != NULL)
 		fnvlist_add_string(config, ZPOOL_CONFIG_COMMENT,
 		    spa->spa_comment);
@@ -623,7 +536,6 @@ spa_config_update(spa_t *spa, int what)
 		spa_config_update(spa, SPA_CONFIG_UPDATE_VDEVS);
 }
 
-EXPORT_SYMBOL(spa_config_load);
 EXPORT_SYMBOL(spa_all_configs);
 EXPORT_SYMBOL(spa_config_set);
 EXPORT_SYMBOL(spa_config_generate);
@@ -634,8 +546,3 @@ EXPORT_SYMBOL(spa_config_update);
 ZFS_MODULE_PARAM(zfs_spa, spa_, config_path, STRING, ZMOD_RD,
 	"SPA config file (/etc/zfs/zpool.cache)");
 #endif
-
-#ifdef _KERNEL
-ZFS_MODULE_PARAM(zfs, zfs_, autoimport_disable, INT, ZMOD_RW,
-	"Disable pool import at module load");
-#endif
diff --git a/sys/contrib/openzfs/module/zfs/spa_misc.c b/sys/contrib/openzfs/module/zfs/spa_misc.c
index 2eba8362a166..0bead6d49666 100644
--- a/sys/contrib/openzfs/module/zfs/spa_misc.c
+++ b/sys/contrib/openzfs/module/zfs/spa_misc.c
@@ -251,11 +251,11 @@ spa_mode_t spa_mode_global = SPA_MODE_UNINIT;
 
 #ifdef ZFS_DEBUG
 /*
- * Everything except dprintf, set_error, spa, and indirect_remap is on
- * by default in debug builds.
+ * Everything except dprintf, set_error, indirect_remap, and raidz_reconstruct
+ * is on by default in debug builds.
  */
 int zfs_flags = ~(ZFS_DEBUG_DPRINTF | ZFS_DEBUG_SET_ERROR |
-    ZFS_DEBUG_INDIRECT_REMAP);
+    ZFS_DEBUG_INDIRECT_REMAP | ZFS_DEBUG_RAIDZ_RECONSTRUCT);
 #else
 int zfs_flags = 0;
 #endif
@@ -471,9 +471,9 @@ spa_config_lock_destroy(spa_t *spa)
 		spa_config_lock_t *scl = &spa->spa_config_lock[i];
 		mutex_destroy(&scl->scl_lock);
 		cv_destroy(&scl->scl_cv);
-		ASSERT(scl->scl_writer == NULL);
-		ASSERT(scl->scl_write_wanted == 0);
-		ASSERT(scl->scl_count == 0);
+		ASSERT0P(scl->scl_writer);
+		ASSERT0(scl->scl_write_wanted);
+		ASSERT0(scl->scl_count);
 	}
 }
 
@@ -510,7 +510,7 @@ spa_config_tryenter(spa_t *spa, int locks, const void *tag, krw_t rw)
 
 static void
 spa_config_enter_impl(spa_t *spa, int locks, const void *tag, krw_t rw,
-    int mmp_flag)
+    int priority_flag)
 {
 	(void) tag;
 	int wlocks_held = 0;
@@ -526,7 +526,7 @@ spa_config_enter_impl(spa_t *spa, int locks, const void *tag, krw_t rw,
 		mutex_enter(&scl->scl_lock);
 		if (rw == RW_READER) {
 			while (scl->scl_writer ||
-			    (!mmp_flag && scl->scl_write_wanted)) {
+			    (!priority_flag && scl->scl_write_wanted)) {
 				cv_wait(&scl->scl_cv, &scl->scl_lock);
 			}
 		} else {
@@ -551,7 +551,7 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw)
 }
 
 /*
- * The spa_config_enter_mmp() allows the mmp thread to cut in front of
+ * The spa_config_enter_priority() allows the mmp thread to cut in front of
  * outstanding write lock requests. This is needed since the mmp updates are
  * time sensitive and failure to service them promptly will result in a
  * suspended pool. This pool suspension has been seen in practice when there is
@@ -560,7 +560,7 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw)
  */
 
 void
-spa_config_enter_mmp(spa_t *spa, int locks, const void *tag, krw_t rw)
+spa_config_enter_priority(spa_t *spa, int locks, const void *tag, krw_t rw)
 {
 	spa_config_enter_impl(spa, locks, tag, rw, 1);
 }
@@ -784,29 +784,29 @@ spa_add(const char *name, nvlist_t *config, const char *altroot)
 	dp->scd_path = altroot ? NULL : spa_strdup(spa_config_path);
 	list_insert_head(&spa->spa_config_list, dp);
 
-	VERIFY(nvlist_alloc(&spa->spa_load_info, NV_UNIQUE_NAME,
-	    KM_SLEEP) == 0);
+	VERIFY0(nvlist_alloc(&spa->spa_load_info, NV_UNIQUE_NAME, KM_SLEEP));
 
 	if (config != NULL) {
 		nvlist_t *features;
 
 		if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_FEATURES_FOR_READ,
 		    &features) == 0) {
-			VERIFY(nvlist_dup(features, &spa->spa_label_features,
-			    0) == 0);
+			VERIFY0(nvlist_dup(features,
+			    &spa->spa_label_features, 0));
 		}
 
-		VERIFY(nvlist_dup(config, &spa->spa_config, 0) == 0);
+		VERIFY0(nvlist_dup(config, &spa->spa_config, 0));
 	}
 
 	if (spa->spa_label_features == NULL) {
-		VERIFY(nvlist_alloc(&spa->spa_label_features, NV_UNIQUE_NAME,
-		    KM_SLEEP) == 0);
+		VERIFY0(nvlist_alloc(&spa->spa_label_features, NV_UNIQUE_NAME,
+		    KM_SLEEP));
 	}
 
 	spa->spa_min_ashift = INT_MAX;
 	spa->spa_max_ashift = 0;
 	spa->spa_min_alloc = INT_MAX;
+	spa->spa_max_alloc = 0;
 	spa->spa_gcd_alloc = INT_MAX;
 
 	/* Reset cached value */
@@ -1866,6 +1866,19 @@ spa_get_worst_case_asize(spa_t *spa, uint64_t lsize)
 }
 
 /*
+ * Return the range of minimum allocation sizes for the normal allocation
+ * class. This can be used by external consumers of the DMU to estimate
+ * potential wasted capacity when setting the recordsize for an object.
+ * This is mainly for dRAID pools which always pad to a full stripe width.
+ */
+void
+spa_get_min_alloc_range(spa_t *spa, uint64_t *min_alloc, uint64_t *max_alloc)
+{
+	*min_alloc = spa->spa_min_alloc;
+	*max_alloc = spa->spa_max_alloc;
+}
+
+/*
  * Return the amount of slop space in bytes.  It is typically 1/32 of the pool
  * (3.2%), minus the embedded log space.  On very small pools, it may be
  * slightly larger than this.  On very large pools, it will be capped to
@@ -2549,13 +2562,6 @@ spa_name_compare(const void *a1, const void *a2)
 }
 
 void
-spa_boot_init(void *unused)
-{
-	(void) unused;
-	spa_config_load();
-}
-
-void
 spa_init(spa_mode_t mode)
 {
 	mutex_init(&spa_namespace_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -2608,7 +2614,6 @@ spa_init(spa_mode_t mode)
 	chksum_init();
 	zpool_prop_init();
 	zpool_feature_init();
-	spa_config_load();
 	vdev_prop_init();
 	l2arc_start();
 	scan_init();
@@ -3094,6 +3099,7 @@ EXPORT_SYMBOL(spa_version);
 EXPORT_SYMBOL(spa_state);
 EXPORT_SYMBOL(spa_load_state);
 EXPORT_SYMBOL(spa_freeze_txg);
+EXPORT_SYMBOL(spa_get_min_alloc_range); /* for Lustre */
 EXPORT_SYMBOL(spa_get_dspace);
 EXPORT_SYMBOL(spa_update_dspace);
 EXPORT_SYMBOL(spa_deflate);
diff --git a/sys/contrib/openzfs/module/zfs/spa_stats.c b/sys/contrib/openzfs/module/zfs/spa_stats.c
index 6d7cabcf766d..2c87122a0aa9 100644
--- a/sys/contrib/openzfs/module/zfs/spa_stats.c
+++ b/sys/contrib/openzfs/module/zfs/spa_stats.c
@@ -718,7 +718,7 @@ spa_mmp_history_set(spa_t *spa, uint64_t mmp_node_id, int io_error,
 	for (smh = list_tail(&shl->procfs_list.pl_list); smh != NULL;
 	    smh = list_prev(&shl->procfs_list.pl_list, smh)) {
 		if (smh->mmp_node_id == mmp_node_id) {
-			ASSERT(smh->io_error == 0);
+			ASSERT0(smh->io_error);
 			smh->io_error = io_error;
 			smh->duration = duration;
 			error = 0;
diff --git a/sys/contrib/openzfs/module/zfs/space_map.c b/sys/contrib/openzfs/module/zfs/space_map.c
index c429e0edd168..5f24963f2291 100644
--- a/sys/contrib/openzfs/module/zfs/space_map.c
+++ b/sys/contrib/openzfs/module/zfs/space_map.c
@@ -817,7 +817,7 @@ space_map_open(space_map_t **smp, objset_t *os, uint64_t object,
 	space_map_t *sm;
 	int error;
 
-	ASSERT(*smp == NULL);
+	ASSERT0P(*smp);
 	ASSERT(os != NULL);
 	ASSERT(object != 0);
 
diff --git a/sys/contrib/openzfs/module/zfs/space_reftree.c b/sys/contrib/openzfs/module/zfs/space_reftree.c
index 9b2d5ed31dc9..889980e08c06 100644
--- a/sys/contrib/openzfs/module/zfs/space_reftree.c
+++ b/sys/contrib/openzfs/module/zfs/space_reftree.c
@@ -149,6 +149,6 @@ space_reftree_generate_map(avl_tree_t *t, zfs_range_tree_t *rt, int64_t minref)
 			}
 		}
 	}
-	ASSERT(refcnt == 0);
+	ASSERT0(refcnt);
 	ASSERT(start == -1ULL);
 }
diff --git a/sys/contrib/openzfs/module/zfs/vdev.c b/sys/contrib/openzfs/module/zfs/vdev.c
index 70b14fb9b2c8..c8d7280387a2 100644
--- a/sys/contrib/openzfs/module/zfs/vdev.c
+++ b/sys/contrib/openzfs/module/zfs/vdev.c
@@ -29,7 +29,7 @@
  * Copyright 2017 Joyent, Inc.
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019, Datto Inc. All rights reserved.
- * Copyright (c) 2021, Klara Inc.
+ * Copyright (c) 2021, 2025, Klara, Inc.
  * Copyright (c) 2021, 2023 Hewlett Packard Enterprise Development LP.
  */
 
@@ -554,7 +554,7 @@ vdev_add_child(vdev_t *pvd, vdev_t *cvd)
 	vdev_t **newchild;
 
 	ASSERT(spa_config_held(cvd->vdev_spa, SCL_ALL, RW_WRITER) == SCL_ALL);
-	ASSERT(cvd->vdev_parent == NULL);
+	ASSERT0P(cvd->vdev_parent);
 
 	cvd->vdev_parent = pvd;
 
@@ -578,7 +578,7 @@ vdev_add_child(vdev_t *pvd, vdev_t *cvd)
 	pvd->vdev_nonrot &= cvd->vdev_nonrot;
 
 	cvd->vdev_top = (pvd->vdev_top ? pvd->vdev_top: cvd);
-	ASSERT(cvd->vdev_top->vdev_parent->vdev_parent == NULL);
+	ASSERT0P(cvd->vdev_top->vdev_parent->vdev_parent);
 
 	/*
 	 * Walk up all ancestors to update guid sum.
@@ -1086,6 +1086,10 @@ vdev_alloc(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, uint_t id,
 		}
 	}
 
+	if (top_level && (ops == &vdev_raidz_ops || ops == &vdev_draid_ops))
+		vd->vdev_autosit =
+		    vdev_prop_default_numeric(VDEV_PROP_AUTOSIT);
+
 	/*
 	 * Add ourselves to the parent's list of children.
 	 */
@@ -1101,10 +1105,10 @@ vdev_free(vdev_t *vd)
 {
 	spa_t *spa = vd->vdev_spa;
 
-	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
-	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
-	ASSERT3P(vd->vdev_autotrim_thread, ==, NULL);
-	ASSERT3P(vd->vdev_rebuild_thread, ==, NULL);
+	ASSERT0P(vd->vdev_initialize_thread);
+	ASSERT0P(vd->vdev_trim_thread);
+	ASSERT0P(vd->vdev_autotrim_thread);
+	ASSERT0P(vd->vdev_rebuild_thread);
 
 	/*
 	 * Scan queues are normally destroyed at the end of a scan. If the
@@ -1133,7 +1137,7 @@ vdev_free(vdev_t *vd)
 	for (int c = 0; c < vd->vdev_children; c++)
 		vdev_free(vd->vdev_child[c]);
 
-	ASSERT(vd->vdev_child == NULL);
+	ASSERT0P(vd->vdev_child);
 	ASSERT(vd->vdev_guid_sum == vd->vdev_guid);
 
 	if (vd->vdev_ops->vdev_op_fini != NULL)
@@ -1162,7 +1166,7 @@ vdev_free(vdev_t *vd)
 	 */
 	vdev_remove_child(vd->vdev_parent, vd);
 
-	ASSERT(vd->vdev_parent == NULL);
+	ASSERT0P(vd->vdev_parent);
 	ASSERT(!list_link_active(&vd->vdev_leaf_node));
 
 	/*
@@ -1187,6 +1191,9 @@ vdev_free(vdev_t *vd)
 		spa_spare_remove(vd);
 	if (vd->vdev_isl2cache)
 		spa_l2cache_remove(vd);
+	if (vd->vdev_prev_histo)
+		kmem_free(vd->vdev_prev_histo,
+		    sizeof (uint64_t) * VDEV_L_HISTO_BUCKETS);
 
 	txg_list_destroy(&vd->vdev_ms_list);
 	txg_list_destroy(&vd->vdev_dtl_list);
@@ -1309,9 +1316,9 @@ vdev_top_transfer(vdev_t *svd, vdev_t *tvd)
 	ASSERT0(tvd->vdev_indirect_config.vic_births_object);
 	ASSERT0(tvd->vdev_indirect_config.vic_mapping_object);
 	ASSERT3U(tvd->vdev_indirect_config.vic_prev_indirect_vdev, ==, -1ULL);
-	ASSERT3P(tvd->vdev_indirect_mapping, ==, NULL);
-	ASSERT3P(tvd->vdev_indirect_births, ==, NULL);
-	ASSERT3P(tvd->vdev_obsolete_sm, ==, NULL);
+	ASSERT0P(tvd->vdev_indirect_mapping);
+	ASSERT0P(tvd->vdev_indirect_births);
+	ASSERT0P(tvd->vdev_obsolete_sm);
 	ASSERT0(tvd->vdev_noalloc);
 	ASSERT0(tvd->vdev_removing);
 	ASSERT0(tvd->vdev_rebuilding);
@@ -1464,7 +1471,7 @@ vdev_remove_parent(vdev_t *cvd)
 	if (cvd == cvd->vdev_top)
 		vdev_top_transfer(mvd, cvd);
 
-	ASSERT(mvd->vdev_children == 0);
+	ASSERT0(mvd->vdev_children);
 	vdev_free(mvd);
 }
 
@@ -1490,12 +1497,14 @@ vdev_spa_set_alloc(spa_t *spa, uint64_t min_alloc)
 {
 	if (min_alloc < spa->spa_min_alloc)
 		spa->spa_min_alloc = min_alloc;
-	if (spa->spa_gcd_alloc == INT_MAX) {
+
+	if (min_alloc > spa->spa_max_alloc)
+		spa->spa_max_alloc = min_alloc;
+
+	if (spa->spa_gcd_alloc == INT_MAX)
 		spa->spa_gcd_alloc = min_alloc;
-	} else {
-		spa->spa_gcd_alloc = vdev_gcd(min_alloc,
-		    spa->spa_gcd_alloc);
-	}
+	else
+		spa->spa_gcd_alloc = vdev_gcd(min_alloc, spa->spa_gcd_alloc);
 }
 
 void
@@ -1553,8 +1562,7 @@ vdev_metaslab_group_create(vdev_t *vd)
 			if (vd->vdev_ashift < spa->spa_min_ashift)
 				spa->spa_min_ashift = vd->vdev_ashift;
 
-			uint64_t min_alloc = vdev_get_min_alloc(vd);
-			vdev_spa_set_alloc(spa, min_alloc);
+			vdev_spa_set_alloc(spa, vdev_get_min_alloc(vd));
 		}
 	}
 }
@@ -2134,14 +2142,14 @@ vdev_open(vdev_t *vd)
 	 * faulted, bail out of the open.
 	 */
 	if (!vd->vdev_removed && vd->vdev_faulted) {
-		ASSERT(vd->vdev_children == 0);
+		ASSERT0(vd->vdev_children);
 		ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED ||
 		    vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
 		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
 		    vd->vdev_label_aux);
 		return (SET_ERROR(ENXIO));
 	} else if (vd->vdev_offline) {
-		ASSERT(vd->vdev_children == 0);
+		ASSERT0(vd->vdev_children);
 		vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE, VDEV_AUX_NONE);
 		return (SET_ERROR(ENXIO));
 	}
@@ -2197,7 +2205,7 @@ vdev_open(vdev_t *vd)
 	 * the vdev is accessible.  If we're faulted, bail.
 	 */
 	if (vd->vdev_faulted) {
-		ASSERT(vd->vdev_children == 0);
+		ASSERT0(vd->vdev_children);
 		ASSERT(vd->vdev_label_aux == VDEV_AUX_ERR_EXCEEDED ||
 		    vd->vdev_label_aux == VDEV_AUX_EXTERNAL);
 		vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
@@ -2206,7 +2214,7 @@ vdev_open(vdev_t *vd)
 	}
 
 	if (vd->vdev_degraded) {
-		ASSERT(vd->vdev_children == 0);
+		ASSERT0(vd->vdev_children);
 		vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
 		    VDEV_AUX_ERR_EXCEEDED);
 	} else {
@@ -3857,6 +3865,26 @@ vdev_load(vdev_t *vd)
 		}
 	}
 
+	if (vd == vd->vdev_top && vd->vdev_top_zap != 0) {
+		spa_t *spa = vd->vdev_spa;
+		uint64_t autosit;
+
+		error = zap_lookup(spa->spa_meta_objset, vd->vdev_top_zap,
+		    vdev_prop_to_name(VDEV_PROP_AUTOSIT), sizeof (autosit),
+		    1, &autosit);
+		if (error == 0) {
+			vd->vdev_autosit = autosit == 1;
+		} else if (error == ENOENT) {
+			vd->vdev_autosit = vdev_prop_default_numeric(
+			    VDEV_PROP_AUTOSIT);
+		} else {
+			vdev_dbgmsg(vd,
+			    "vdev_load: zap_lookup(top_zap=%llu) "
+			    "failed [error=%d]",
+			    (u_longlong_t)vd->vdev_top_zap, error);
+		}
+	}
+
 	/*
 	 * Load any rebuild state from the top-level vdev zap.
 	 */
@@ -3945,7 +3973,7 @@ vdev_load(vdev_t *vd)
 		if (error == 0 && checkpoint_sm_obj != 0) {
 			objset_t *mos = spa_meta_objset(vd->vdev_spa);
 			ASSERT(vd->vdev_asize != 0);
-			ASSERT3P(vd->vdev_checkpoint_sm, ==, NULL);
+			ASSERT0P(vd->vdev_checkpoint_sm);
 
 			error = space_map_open(&vd->vdev_checkpoint_sm,
 			    mos, checkpoint_sm_obj, 0, vd->vdev_asize,
@@ -3993,7 +4021,7 @@ vdev_load(vdev_t *vd)
 	if (error == 0 && obsolete_sm_object != 0) {
 		objset_t *mos = vd->vdev_spa->spa_meta_objset;
 		ASSERT(vd->vdev_asize != 0);
-		ASSERT3P(vd->vdev_obsolete_sm, ==, NULL);
+		ASSERT0P(vd->vdev_obsolete_sm);
 
 		if ((error = space_map_open(&vd->vdev_obsolete_sm, mos,
 		    obsolete_sm_object, 0, vd->vdev_asize, 0))) {
@@ -4521,7 +4549,7 @@ top:
 			/*
 			 * Prevent any future allocations.
 			 */
-			ASSERT3P(tvd->vdev_log_mg, ==, NULL);
+			ASSERT0P(tvd->vdev_log_mg);
 			metaslab_group_passivate(mg);
 			(void) spa_vdev_state_exit(spa, vd, 0);
 
@@ -4616,6 +4644,8 @@ vdev_clear(spa_t *spa, vdev_t *vd)
 	vd->vdev_stat.vs_checksum_errors = 0;
 	vd->vdev_stat.vs_dio_verify_errors = 0;
 	vd->vdev_stat.vs_slow_ios = 0;
+	atomic_store_64(&vd->vdev_outlier_count, 0);
+	vd->vdev_read_sit_out_expire = 0;
 
 	for (int c = 0; c < vd->vdev_children; c++)
 		vdev_clear(spa, vd->vdev_child[c]);
@@ -5194,7 +5224,7 @@ vdev_stat_update(zio_t *zio, uint64_t psize)
 int64_t
 vdev_deflated_space(vdev_t *vd, int64_t space)
 {
-	ASSERT((space & (SPA_MINBLOCKSIZE-1)) == 0);
+	ASSERT0((space & (SPA_MINBLOCKSIZE-1)));
 	ASSERT(vd->vdev_deflate_ratio != 0 || vd->vdev_isl2cache);
 
 	return ((space >> SPA_MINBLOCKSHIFT) * vd->vdev_deflate_ratio);
@@ -5286,8 +5316,8 @@ vdev_config_dirty(vdev_t *vd)
 
 		if (nvlist_lookup_nvlist_array(sav->sav_config,
 		    ZPOOL_CONFIG_L2CACHE, &aux, &naux) != 0) {
-			VERIFY(nvlist_lookup_nvlist_array(sav->sav_config,
-			    ZPOOL_CONFIG_SPARES, &aux, &naux) == 0);
+			VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config,
+			    ZPOOL_CONFIG_SPARES, &aux, &naux));
 		}
 
 		ASSERT(c < naux);
@@ -5675,7 +5705,7 @@ vdev_expand(vdev_t *vd, uint64_t txg)
 	    (vd->vdev_asize >> vd->vdev_ms_shift) > vd->vdev_ms_count &&
 	    vdev_is_concrete(vd)) {
 		vdev_metaslab_group_create(vd);
-		VERIFY(vdev_metaslab_init(vd, txg) == 0);
+		VERIFY0(vdev_metaslab_init(vd, txg));
 		vdev_config_dirty(vd);
 	}
 }
@@ -6107,6 +6137,56 @@ vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
 			}
 			vd->vdev_failfast = intval & 1;
 			break;
+		case VDEV_PROP_SIT_OUT:
+			/* Only expose this for a draid or raidz leaf */
+			if (!vd->vdev_ops->vdev_op_leaf ||
+			    vd->vdev_top == NULL ||
+			    (vd->vdev_top->vdev_ops != &vdev_raidz_ops &&
+			    vd->vdev_top->vdev_ops != &vdev_draid_ops)) {
+				error = ENOTSUP;
+				break;
+			}
+			if (nvpair_value_uint64(elem, &intval) != 0) {
+				error = EINVAL;
+				break;
+			}
+			if (intval == 1) {
+				vdev_t *ancestor = vd;
+				while (ancestor->vdev_parent != vd->vdev_top)
+					ancestor = ancestor->vdev_parent;
+				vdev_t *pvd = vd->vdev_top;
+				uint_t sitouts = 0;
+				for (int i = 0; i < pvd->vdev_children; i++) {
+					if (pvd->vdev_child[i] == ancestor)
+						continue;
+					if (vdev_sit_out_reads(
+					    pvd->vdev_child[i], 0)) {
+						sitouts++;
+					}
+				}
+				if (sitouts >= vdev_get_nparity(pvd)) {
+					error = ZFS_ERR_TOO_MANY_SITOUTS;
+					break;
+				}
+				if (error == 0)
+					vdev_raidz_sit_child(vd,
+					    INT64_MAX - gethrestime_sec());
+			} else {
+				vdev_raidz_unsit_child(vd);
+			}
+			break;
+		case VDEV_PROP_AUTOSIT:
+			if (vd->vdev_ops != &vdev_raidz_ops &&
+			    vd->vdev_ops != &vdev_draid_ops) {
+				error = ENOTSUP;
+				break;
+			}
+			if (nvpair_value_uint64(elem, &intval) != 0) {
+				error = EINVAL;
+				break;
+			}
+			vd->vdev_autosit = intval == 1;
+			break;
 		case VDEV_PROP_CHECKSUM_N:
 			if (nvpair_value_uint64(elem, &intval) != 0) {
 				error = EINVAL;
@@ -6456,6 +6536,19 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
 					    ZPROP_SRC_NONE);
 				}
 				continue;
+			case VDEV_PROP_SIT_OUT:
+				/* Only expose this for a draid or raidz leaf */
+				if (vd->vdev_ops->vdev_op_leaf &&
+				    vd->vdev_top != NULL &&
+				    (vd->vdev_top->vdev_ops ==
+				    &vdev_raidz_ops ||
+				    vd->vdev_top->vdev_ops ==
+				    &vdev_draid_ops)) {
+					vdev_prop_add_list(outnvl, propname,
+					    NULL, vdev_sit_out_reads(vd, 0),
+					    ZPROP_SRC_NONE);
+				}
+				continue;
 			case VDEV_PROP_TRIM_SUPPORT:
 				/* only valid for leaf vdevs */
 				if (vd->vdev_ops->vdev_op_leaf) {
@@ -6506,6 +6599,29 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
 				vdev_prop_add_list(outnvl, propname, strval,
 				    intval, src);
 				break;
+			case VDEV_PROP_AUTOSIT:
+				/* Only raidz vdevs cannot have this property */
+				if (vd->vdev_ops != &vdev_raidz_ops &&
+				    vd->vdev_ops != &vdev_draid_ops) {
+					src = ZPROP_SRC_NONE;
+					intval = ZPROP_BOOLEAN_NA;
+				} else {
+					err = vdev_prop_get_int(vd, prop,
+					    &intval);
+					if (err && err != ENOENT)
+						break;
+
+					if (intval ==
+					    vdev_prop_default_numeric(prop))
+						src = ZPROP_SRC_DEFAULT;
+					else
+						src = ZPROP_SRC_LOCAL;
+				}
+
+				vdev_prop_add_list(outnvl, propname, NULL,
+				    intval, src);
+				break;
+
 			case VDEV_PROP_CHECKSUM_N:
 			case VDEV_PROP_CHECKSUM_T:
 			case VDEV_PROP_IO_N:
diff --git a/sys/contrib/openzfs/module/zfs/vdev_draid.c b/sys/contrib/openzfs/module/zfs/vdev_draid.c
index feec5fd3ce17..8588cfee3f7d 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_draid.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_draid.c
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2018 Intel Corporation.
  * Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+ * Copyright (c) 2025, Klara, Inc.
  */
 
 #include <sys/zfs_context.h>
@@ -477,7 +478,7 @@ vdev_draid_generate_perms(const draid_map_t *map, uint8_t **permsp)
 	VERIFY3U(map->dm_children, <=, VDEV_DRAID_MAX_CHILDREN);
 	VERIFY3U(map->dm_seed, !=, 0);
 	VERIFY3U(map->dm_nperms, !=, 0);
-	VERIFY3P(map->dm_perms, ==, NULL);
+	VERIFY0P(map->dm_perms);
 
 #ifdef _KERNEL
 	/*
@@ -590,7 +591,7 @@ vdev_draid_psize_to_asize(vdev_t *vd, uint64_t psize, uint64_t txg)
 	uint64_t asize = (rows * vdc->vdc_groupwidth) << ashift;
 
 	ASSERT3U(asize, !=, 0);
-	ASSERT3U(asize % (vdc->vdc_groupwidth), ==, 0);
+	ASSERT0(asize % (vdc->vdc_groupwidth));
 
 	return (asize);
 }
@@ -704,7 +705,7 @@ vdev_draid_map_alloc_scrub(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr)
 	uint64_t skip_off = 0;
 
 	ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);
-	ASSERT3P(rr->rr_abd_empty, ==, NULL);
+	ASSERT0P(rr->rr_abd_empty);
 
 	if (rr->rr_nempty > 0) {
 		rr->rr_abd_empty = abd_alloc_linear(rr->rr_nempty * skip_size,
@@ -793,7 +794,7 @@ vdev_draid_map_alloc_empty(zio_t *zio, raidz_row_t *rr)
 	uint64_t skip_off = 0;
 
 	ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);
-	ASSERT3P(rr->rr_abd_empty, ==, NULL);
+	ASSERT0P(rr->rr_abd_empty);
 
 	if (rr->rr_nempty > 0) {
 		rr->rr_abd_empty = abd_alloc_linear(rr->rr_nempty * skip_size,
@@ -807,7 +808,7 @@ vdev_draid_map_alloc_empty(zio_t *zio, raidz_row_t *rr)
 			/* empty data column (small read), add a skip sector */
 			ASSERT3U(skip_size, ==, parity_size);
 			ASSERT3U(rr->rr_nempty, !=, 0);
-			ASSERT3P(rc->rc_abd, ==, NULL);
+			ASSERT0P(rc->rc_abd);
 			rc->rc_abd = abd_get_offset_size(rr->rr_abd_empty,
 			    skip_off, skip_size);
 			skip_off += skip_size;
@@ -1623,7 +1624,7 @@ vdev_draid_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize,
 	    SPA_MAXBLOCKSIZE);
 
 	ASSERT3U(vdev_draid_get_astart(vd, start), ==, start);
-	ASSERT3U(asize % (vdc->vdc_groupwidth << ashift), ==, 0);
+	ASSERT0(asize % (vdc->vdc_groupwidth << ashift));
 
 	/* Chunks must evenly span all data columns in the group. */
 	psize = (((psize >> ashift) / ndata) * ndata) << ashift;
@@ -1634,7 +1635,7 @@ vdev_draid_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize,
 	uint64_t left = vdev_draid_group_to_offset(vd, group + 1) - start;
 	chunk_size = MIN(chunk_size, left);
 
-	ASSERT3U(chunk_size % (vdc->vdc_groupwidth << ashift), ==, 0);
+	ASSERT0(chunk_size % (vdc->vdc_groupwidth << ashift));
 	ASSERT3U(vdev_draid_offset_to_group(vd, start), ==,
 	    vdev_draid_offset_to_group(vd, start + chunk_size - 1));
 
@@ -1996,6 +1997,33 @@ vdev_draid_io_start_read(zio_t *zio, raidz_row_t *rr)
 				rc->rc_allow_repair = 1;
 			}
 		}
+
+		if (vdev_sit_out_reads(cvd, zio->io_flags)) {
+			rr->rr_outlier_cnt++;
+			ASSERT0(rc->rc_latency_outlier);
+			rc->rc_latency_outlier = 1;
+		}
+	}
+
+	/*
+	 * When the row contains a latency outlier and sufficient parity
+	 * exists to reconstruct the column data, then skip reading the
+	 * known slow child vdev as a performance optimization.
+	 */
+	if (rr->rr_outlier_cnt > 0 &&
+	    (rr->rr_firstdatacol - rr->rr_missingparity) >=
+	    (rr->rr_missingdata + 1)) {
+
+		for (int c = rr->rr_cols - 1; c >= rr->rr_firstdatacol; c--) {
+			raidz_col_t *rc = &rr->rr_col[c];
+
+			if (rc->rc_error == 0 && rc->rc_latency_outlier) {
+				rr->rr_missingdata++;
+				rc->rc_error = SET_ERROR(EAGAIN);
+				rc->rc_skipped = 1;
+				break;
+			}
+		}
 	}
 
 	/*
@@ -2272,7 +2300,7 @@ vdev_draid_init(spa_t *spa, nvlist_t *nv, void **tsd)
 	ASSERT3U(vdc->vdc_groupwidth, <=, vdc->vdc_ndisks);
 	ASSERT3U(vdc->vdc_groupsz, >=, 2 * VDEV_DRAID_ROWHEIGHT);
 	ASSERT3U(vdc->vdc_devslicesz, >=, VDEV_DRAID_ROWHEIGHT);
-	ASSERT3U(vdc->vdc_devslicesz % VDEV_DRAID_ROWHEIGHT, ==, 0);
+	ASSERT0(vdc->vdc_devslicesz % VDEV_DRAID_ROWHEIGHT);
 	ASSERT3U((vdc->vdc_groupwidth * vdc->vdc_ngroups) %
 	    vdc->vdc_ndisks, ==, 0);
 
diff --git a/sys/contrib/openzfs/module/zfs/vdev_file.c b/sys/contrib/openzfs/module/zfs/vdev_file.c
index f457669bc809..20b4db65ec06 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_file.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_file.c
@@ -228,7 +228,8 @@ vdev_file_io_strategy(void *arg)
 		abd_return_buf_copy(zio->io_abd, buf, size);
 	} else {
 		buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size);
-		err = zfs_file_pwrite(vf->vf_file, buf, size, off, &resid);
+		err = zfs_file_pwrite(vf->vf_file, buf, size, off,
+		    vd->vdev_ashift, &resid);
 		abd_return_buf(zio->io_abd, buf, size);
 	}
 	zio->io_error = err;
diff --git a/sys/contrib/openzfs/module/zfs/vdev_indirect.c b/sys/contrib/openzfs/module/zfs/vdev_indirect.c
index 9fc71fa0e03e..7538f471e63c 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_indirect.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_indirect.c
@@ -792,7 +792,7 @@ spa_condense_indirect_start_sync(vdev_t *vd, dmu_tx_t *tx)
 	    DMU_POOL_CONDENSING_INDIRECT, sizeof (uint64_t),
 	    sizeof (*scip) / sizeof (uint64_t), scip, tx));
 
-	ASSERT3P(spa->spa_condensing_indirect, ==, NULL);
+	ASSERT0P(spa->spa_condensing_indirect);
 	spa->spa_condensing_indirect = spa_condensing_indirect_create(spa);
 
 	zfs_dbgmsg("starting condense of vdev %llu in txg %llu: "
@@ -882,7 +882,7 @@ spa_condense_fini(spa_t *spa)
 void
 spa_start_indirect_condensing_thread(spa_t *spa)
 {
-	ASSERT3P(spa->spa_condense_zthr, ==, NULL);
+	ASSERT0P(spa->spa_condense_zthr);
 	spa->spa_condense_zthr = zthr_create("z_indirect_condense",
 	    spa_condense_indirect_thread_check,
 	    spa_condense_indirect_thread, spa, minclsyspri);
@@ -1504,7 +1504,7 @@ vdev_indirect_splits_checksum_validate(indirect_vsd_t *iv, zio_t *zio)
 	    is != NULL; is = list_next(&iv->iv_splits, is)) {
 
 		ASSERT3P(is->is_good_child->ic_data, !=, NULL);
-		ASSERT3P(is->is_good_child->ic_duplicate, ==, NULL);
+		ASSERT0P(is->is_good_child->ic_duplicate);
 
 		abd_copy_off(zio->io_abd, is->is_good_child->ic_data,
 		    is->is_split_offset, 0, is->is_size);
diff --git a/sys/contrib/openzfs/module/zfs/vdev_initialize.c b/sys/contrib/openzfs/module/zfs/vdev_initialize.c
index 9243c76e810d..27188c46e561 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_initialize.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_initialize.c
@@ -632,7 +632,7 @@ vdev_initialize(vdev_t *vd)
 	ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 	ASSERT(vd->vdev_ops->vdev_op_leaf);
 	ASSERT(vdev_is_concrete(vd));
-	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
+	ASSERT0P(vd->vdev_initialize_thread);
 	ASSERT(!vd->vdev_detached);
 	ASSERT(!vd->vdev_initialize_exit_wanted);
 	ASSERT(!vd->vdev_top->vdev_removing);
@@ -653,7 +653,7 @@ vdev_uninitialize(vdev_t *vd)
 	ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 	ASSERT(vd->vdev_ops->vdev_op_leaf);
 	ASSERT(vdev_is_concrete(vd));
-	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
+	ASSERT0P(vd->vdev_initialize_thread);
 	ASSERT(!vd->vdev_detached);
 	ASSERT(!vd->vdev_initialize_exit_wanted);
 	ASSERT(!vd->vdev_top->vdev_removing);
@@ -672,7 +672,7 @@ vdev_initialize_stop_wait_impl(vdev_t *vd)
 	while (vd->vdev_initialize_thread != NULL)
 		cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock);
 
-	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
+	ASSERT0P(vd->vdev_initialize_thread);
 	vd->vdev_initialize_exit_wanted = B_FALSE;
 }
 
diff --git a/sys/contrib/openzfs/module/zfs/vdev_label.c b/sys/contrib/openzfs/module/zfs/vdev_label.c
index 6baa6236aac2..0d4fdaa77ba0 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_label.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_label.c
@@ -163,7 +163,7 @@ uint64_t
 vdev_label_offset(uint64_t psize, int l, uint64_t offset)
 {
 	ASSERT(offset < sizeof (vdev_label_t));
-	ASSERT(P2PHASE_TYPED(psize, sizeof (vdev_label_t), uint64_t) == 0);
+	ASSERT0(P2PHASE_TYPED(psize, sizeof (vdev_label_t), uint64_t));
 
 	return (offset + l * sizeof (vdev_label_t) + (l < VDEV_LABELS / 2 ?
 	    0 : psize - VDEV_LABELS * sizeof (vdev_label_t)));
@@ -511,6 +511,8 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats,
 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift);
 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASIZE,
 		    vd->vdev_asize);
+		fnvlist_add_uint64(nv, ZPOOL_CONFIG_MIN_ALLOC,
+		    vdev_get_min_alloc(vd));
 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_LOG, vd->vdev_islog);
 		if (vd->vdev_noalloc) {
 			fnvlist_add_uint64(nv, ZPOOL_CONFIG_NONALLOCATING,
@@ -768,12 +770,12 @@ vdev_top_config_generate(spa_t *spa, nvlist_t *config)
 	}
 
 	if (idx) {
-		VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_HOLE_ARRAY,
-		    array, idx) == 0);
+		VERIFY0(nvlist_add_uint64_array(config,
+		    ZPOOL_CONFIG_HOLE_ARRAY, array, idx));
 	}
 
-	VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VDEV_CHILDREN,
-	    rvd->vdev_children) == 0);
+	VERIFY0(nvlist_add_uint64(config, ZPOOL_CONFIG_VDEV_CHILDREN,
+	    rvd->vdev_children));
 
 	kmem_free(array, rvd->vdev_children * sizeof (uint64_t));
 }
@@ -1189,8 +1191,8 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
 		 * vdev uses as described above, and automatically expires if we
 		 * fail.
 		 */
-		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_CREATE_TXG,
-		    crtxg) == 0);
+		VERIFY0(nvlist_add_uint64(label, ZPOOL_CONFIG_CREATE_TXG,
+		    crtxg));
 	}
 
 	buf = vp->vp_nvlist;
diff --git a/sys/contrib/openzfs/module/zfs/vdev_queue.c b/sys/contrib/openzfs/module/zfs/vdev_queue.c
index aa41f7066036..c12713b107bf 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_queue.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_queue.c
@@ -780,7 +780,7 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
 			if (dio->io_flags & ZIO_FLAG_NODATA) {
 				/* allocate a buffer for a write gap */
 				ASSERT3U(dio->io_type, ==, ZIO_TYPE_WRITE);
-				ASSERT3P(dio->io_abd, ==, NULL);
+				ASSERT0P(dio->io_abd);
 				abd_gang_add(aio->io_abd,
 				    abd_get_zeros(dio->io_size), B_TRUE);
 			} else {
diff --git a/sys/contrib/openzfs/module/zfs/vdev_raidz.c b/sys/contrib/openzfs/module/zfs/vdev_raidz.c
index 210cdcab1ecc..56b8e3b60b22 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_raidz.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_raidz.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2016 Gvozden Nešković. All rights reserved.
+ * Copyright (c) 2025, Klara, Inc.
  */
 
 #include <sys/zfs_context.h>
@@ -356,6 +357,32 @@ unsigned long raidz_expand_max_reflow_bytes = 0;
 uint_t raidz_expand_pause_point = 0;
 
 /*
+ * This represents the duration for a slow drive read sit out.
+ */
+static unsigned long vdev_read_sit_out_secs = 600;
+
+/*
+ * How often each RAID-Z and dRAID vdev will check for slow disk outliers.
+ * Increasing this interval will reduce the sensitivity of detection (since all
+ * I/Os since the last check are included in the statistics), but will slow the
+ * response to a disk developing a problem.
+ *
+ * Defaults to once per second; setting extremely small values may cause
+ * negative performance effects.
+ */
+static hrtime_t vdev_raidz_outlier_check_interval_ms = 1000;
+
+/*
+ * When performing slow outlier checks for RAID-Z and dRAID vdevs, this value is
+ * used to determine how far out an outlier must be before it counts as an event
+ * worth consdering.
+ *
+ * Smaller values will result in more aggressive sitting out of disks that may
+ * have problems, but may significantly increase the rate of spurious sit-outs.
+ */
+static uint32_t vdev_raidz_outlier_insensitivity = 50;
+
+/*
  * Maximum amount of copy io's outstanding at once.
  */
 #ifdef _ILP32
@@ -412,7 +439,7 @@ vdev_raidz_map_free(raidz_map_t *rm)
 		    rm->rm_nphys_cols);
 	}
 
-	ASSERT3P(rm->rm_lr, ==, NULL);
+	ASSERT0P(rm->rm_lr);
 	kmem_free(rm, offsetof(raidz_map_t, rm_row[rm->rm_nrows]));
 }
 
@@ -2311,6 +2338,41 @@ vdev_raidz_min_asize(vdev_t *vd)
 	    vd->vdev_children);
 }
 
+/*
+ * return B_TRUE if a read should be skipped due to being too slow.
+ *
+ * In vdev_child_slow_outlier() it looks for outliers based on disk
+ * latency from the most recent child reads.  Here we're checking if,
+ * over time, a disk has has been an outlier too many times and is
+ * now in a sit out period.
+ */
+boolean_t
+vdev_sit_out_reads(vdev_t *vd, zio_flag_t io_flags)
+{
+	if (vdev_read_sit_out_secs == 0)
+		return (B_FALSE);
+
+	/* Avoid skipping a data column read when scrubbing */
+	if (io_flags & ZIO_FLAG_SCRUB)
+		return (B_FALSE);
+
+	if (!vd->vdev_ops->vdev_op_leaf) {
+		boolean_t sitting = B_FALSE;
+		for (int c = 0; c < vd->vdev_children; c++) {
+			sitting |= vdev_sit_out_reads(vd->vdev_child[c],
+			    io_flags);
+		}
+		return (sitting);
+	}
+
+	if (vd->vdev_read_sit_out_expire >= gethrestime_sec())
+		return (B_TRUE);
+
+	vd->vdev_read_sit_out_expire = 0;
+
+	return (B_FALSE);
+}
+
 void
 vdev_raidz_child_done(zio_t *zio)
 {
@@ -2431,7 +2493,7 @@ raidz_start_skip_writes(zio_t *zio)
 		vdev_t *cvd = vd->vdev_child[rc->rc_devidx];
 		if (rc->rc_size != 0)
 			continue;
-		ASSERT3P(rc->rc_abd, ==, NULL);
+		ASSERT0P(rc->rc_abd);
 
 		ASSERT3U(rc->rc_offset, <,
 		    cvd->vdev_psize - VDEV_LABEL_END_SIZE);
@@ -2475,6 +2537,45 @@ vdev_raidz_io_start_read_row(zio_t *zio, raidz_row_t *rr, boolean_t forceparity)
 			rc->rc_skipped = 1;
 			continue;
 		}
+
+		if (vdev_sit_out_reads(cvd, zio->io_flags)) {
+			rr->rr_outlier_cnt++;
+			ASSERT0(rc->rc_latency_outlier);
+			rc->rc_latency_outlier = 1;
+		}
+	}
+
+	/*
+	 * When the row contains a latency outlier and sufficient parity
+	 * exists to reconstruct the column data, then skip reading the
+	 * known slow child vdev as a performance optimization.
+	 */
+	if (rr->rr_outlier_cnt > 0 &&
+	    (rr->rr_firstdatacol - rr->rr_missingparity) >=
+	    (rr->rr_missingdata + 1)) {
+
+		for (int c = rr->rr_cols - 1; c >= 0; c--) {
+			raidz_col_t *rc = &rr->rr_col[c];
+
+			if (rc->rc_error == 0 && rc->rc_latency_outlier) {
+				if (c >= rr->rr_firstdatacol)
+					rr->rr_missingdata++;
+				else
+					rr->rr_missingparity++;
+				rc->rc_error = SET_ERROR(EAGAIN);
+				rc->rc_skipped = 1;
+				break;
+			}
+		}
+	}
+
+	for (int c = rr->rr_cols - 1; c >= 0; c--) {
+		raidz_col_t *rc = &rr->rr_col[c];
+		vdev_t *cvd = vd->vdev_child[rc->rc_devidx];
+
+		if (rc->rc_error || rc->rc_size == 0)
+			continue;
+
 		if (forceparity ||
 		    c >= rr->rr_firstdatacol || rr->rr_missingdata > 0 ||
 		    (zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) {
@@ -2498,6 +2599,7 @@ vdev_raidz_io_start_read_phys_cols(zio_t *zio, raidz_map_t *rm)
 
 		ASSERT3U(prc->rc_devidx, ==, i);
 		vdev_t *cvd = vd->vdev_child[i];
+
 		if (!vdev_readable(cvd)) {
 			prc->rc_error = SET_ERROR(ENXIO);
 			prc->rc_tried = 1;	/* don't even try */
@@ -2774,6 +2876,239 @@ vdev_raidz_worst_error(raidz_row_t *rr)
 	return (error);
 }
 
+/*
+ * Find the median value from a set of n values
+ */
+static uint64_t
+latency_median_value(const uint64_t *data, size_t n)
+{
+	uint64_t m;
+
+	if (n % 2 == 0)
+		m = (data[(n >> 1) - 1] + data[n >> 1]) >> 1;
+	else
+		m = data[((n + 1) >> 1) - 1];
+
+	return (m);
+}
+
+/*
+ * Calculate the outlier fence from a set of n latency values
+ *
+ * fence = Q3 + vdev_raidz_outlier_insensitivity x (Q3 - Q1)
+ */
+static uint64_t
+latency_quartiles_fence(const uint64_t *data, size_t n, uint64_t *iqr)
+{
+	uint64_t q1 = latency_median_value(&data[0], n >> 1);
+	uint64_t q3 = latency_median_value(&data[(n + 1) >> 1], n >> 1);
+
+	/*
+	 * To avoid detecting false positive outliers when N is small and
+	 * and the latencies values are very close, make sure the IQR
+	 * is at least 25% larger than Q1.
+	 */
+	*iqr = MAX(q3 - q1, q1 / 4);
+
+	return (q3 + (*iqr * vdev_raidz_outlier_insensitivity));
+}
+#define	LAT_CHILDREN_MIN	5
+#define	LAT_OUTLIER_LIMIT	20
+
+static int
+latency_compare(const void *arg1, const void *arg2)
+{
+	const uint64_t *l1 = (uint64_t *)arg1;
+	const uint64_t *l2 = (uint64_t *)arg2;
+
+	return (TREE_CMP(*l1, *l2));
+}
+
+void
+vdev_raidz_sit_child(vdev_t *svd, uint64_t secs)
+{
+	for (int c = 0; c < svd->vdev_children; c++)
+		vdev_raidz_sit_child(svd->vdev_child[c], secs);
+
+	if (!svd->vdev_ops->vdev_op_leaf)
+		return;
+
+	/* Begin a sit out period for this slow drive */
+	svd->vdev_read_sit_out_expire = gethrestime_sec() +
+	    secs;
+
+	/* Count each slow io period */
+	mutex_enter(&svd->vdev_stat_lock);
+	svd->vdev_stat.vs_slow_ios++;
+	mutex_exit(&svd->vdev_stat_lock);
+}
+
+void
+vdev_raidz_unsit_child(vdev_t *vd)
+{
+	for (int c = 0; c < vd->vdev_children; c++)
+		vdev_raidz_unsit_child(vd->vdev_child[c]);
+
+	if (!vd->vdev_ops->vdev_op_leaf)
+		return;
+
+	vd->vdev_read_sit_out_expire = 0;
+}
+
+/*
+ * Check for any latency outlier from latest set of child reads.
+ *
+ * Uses a Tukey's fence, with K = 50, for detecting extreme outliers. This
+ * rule defines extreme outliers as data points outside the fence of the
+ * third quartile plus fifty times the Interquartile Range (IQR). This range
+ * is the distance between the first and third quartile.
+ *
+ * Fifty is an extremely large value for Tukey's fence, but the outliers we're
+ * attempting to detect here are orders of magnitude times larger than the
+ * median. This large value should capture any truly fault disk quickly,
+ * without causing spurious sit-outs.
+ *
+ * To further avoid spurious sit-outs, vdevs must be detected multiple times
+ * as an outlier before they are sat, and outlier counts will gradually decay.
+ * Every nchildren times we have detected an outlier, we subtract 2 from the
+ * outlier count of all children. If detected outliers are close to uniformly
+ * distributed, this will result in the outlier count remaining close to 0
+ * (in expectation; over long enough time-scales, spurious sit-outs are still
+ * possible).
+ */
+static void
+vdev_child_slow_outlier(zio_t *zio)
+{
+	vdev_t *vd = zio->io_vd;
+	if (!vd->vdev_autosit || vdev_read_sit_out_secs == 0 ||
+	    vd->vdev_children < LAT_CHILDREN_MIN)
+		return;
+
+	hrtime_t now = getlrtime();
+	uint64_t last = atomic_load_64(&vd->vdev_last_latency_check);
+
+	if ((now - last) < MSEC2NSEC(vdev_raidz_outlier_check_interval_ms))
+		return;
+
+	/* Allow a single winner when there are racing callers. */
+	if (atomic_cas_64(&vd->vdev_last_latency_check, last, now) != last)
+		return;
+
+	int children = vd->vdev_children;
+	uint64_t *lat_data = kmem_alloc(sizeof (uint64_t) * children, KM_SLEEP);
+
+	for (int c = 0; c < children; c++) {
+		vdev_t *cvd = vd->vdev_child[c];
+		if (cvd->vdev_prev_histo == NULL) {
+			mutex_enter(&cvd->vdev_stat_lock);
+			size_t size =
+			    sizeof (cvd->vdev_stat_ex.vsx_disk_histo[0]);
+			cvd->vdev_prev_histo = kmem_zalloc(size, KM_SLEEP);
+			memcpy(cvd->vdev_prev_histo,
+			    cvd->vdev_stat_ex.vsx_disk_histo[ZIO_TYPE_READ],
+			    size);
+			mutex_exit(&cvd->vdev_stat_lock);
+		}
+	}
+	uint64_t max = 0;
+	vdev_t *svd = NULL;
+	uint_t sitouts = 0;
+	boolean_t skip = B_FALSE, svd_sitting = B_FALSE;
+	for (int c = 0; c < children; c++) {
+		vdev_t *cvd = vd->vdev_child[c];
+		boolean_t sitting = vdev_sit_out_reads(cvd, 0) ||
+		    cvd->vdev_state != VDEV_STATE_HEALTHY;
+
+		/* We can't sit out more disks than we have parity */
+		if (sitting && ++sitouts >= vdev_get_nparity(vd))
+			skip = B_TRUE;
+
+		mutex_enter(&cvd->vdev_stat_lock);
+
+		uint64_t *prev_histo = cvd->vdev_prev_histo;
+		uint64_t *histo =
+		    cvd->vdev_stat_ex.vsx_disk_histo[ZIO_TYPE_READ];
+		if (skip) {
+			size_t size =
+			    sizeof (cvd->vdev_stat_ex.vsx_disk_histo[0]);
+			memcpy(prev_histo, histo, size);
+			mutex_exit(&cvd->vdev_stat_lock);
+			continue;
+		}
+		uint64_t count = 0;
+		lat_data[c] = 0;
+		for (int i = 0; i < VDEV_L_HISTO_BUCKETS; i++) {
+			uint64_t this_count = histo[i] - prev_histo[i];
+			lat_data[c] += (1ULL << i) * this_count;
+			count += this_count;
+		}
+		size_t size = sizeof (cvd->vdev_stat_ex.vsx_disk_histo[0]);
+		memcpy(prev_histo, histo, size);
+		mutex_exit(&cvd->vdev_stat_lock);
+		lat_data[c] /= MAX(1, count);
+
+		/* Wait until all disks have been read from */
+		if (lat_data[c] == 0 && !sitting) {
+			skip = B_TRUE;
+			continue;
+		}
+
+		/* Keep track of the vdev with largest value */
+		if (lat_data[c] > max) {
+			max = lat_data[c];
+			svd = cvd;
+			svd_sitting = sitting;
+		}
+	}
+
+	if (skip) {
+		kmem_free(lat_data, sizeof (uint64_t) * children);
+		return;
+	}
+
+	qsort((void *)lat_data, children, sizeof (uint64_t), latency_compare);
+
+	uint64_t iqr;
+	uint64_t fence = latency_quartiles_fence(lat_data, children, &iqr);
+
+	ASSERT3U(lat_data[children - 1], ==, max);
+	if (max > fence && !svd_sitting) {
+		ASSERT3U(iqr, >, 0);
+		uint64_t incr = MAX(1, MIN((max - fence) / iqr,
+		    LAT_OUTLIER_LIMIT / 4));
+		vd->vdev_outlier_count += incr;
+		if (vd->vdev_outlier_count >= children) {
+			for (int c = 0; c < children; c++) {
+				vdev_t *cvd = vd->vdev_child[c];
+				cvd->vdev_outlier_count -= 2;
+				cvd->vdev_outlier_count = MAX(0,
+				    cvd->vdev_outlier_count);
+			}
+			vd->vdev_outlier_count = 0;
+		}
+		/*
+		 * Keep track of how many times this child has had
+		 * an outlier read. A disk that persitently has a
+		 * higher than peers outlier count will be considered
+		 * a slow disk.
+		 */
+		svd->vdev_outlier_count += incr;
+		if (svd->vdev_outlier_count > LAT_OUTLIER_LIMIT) {
+			ASSERT0(svd->vdev_read_sit_out_expire);
+			vdev_raidz_sit_child(svd, vdev_read_sit_out_secs);
+			(void) zfs_ereport_post(FM_EREPORT_ZFS_SITOUT,
+			    zio->io_spa, svd, NULL, NULL, 0);
+			vdev_dbgmsg(svd, "begin read sit out for %d secs",
+			    (int)vdev_read_sit_out_secs);
+
+			for (int c = 0; c < vd->vdev_children; c++)
+				vd->vdev_child[c]->vdev_outlier_count = 0;
+		}
+	}
+
+	kmem_free(lat_data, sizeof (uint64_t) * children);
+}
+
 static void
 vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr)
 {
@@ -3363,7 +3698,7 @@ vdev_raidz_io_done_reconstruct_known_missing(zio_t *zio, raidz_map_t *rm,
 		 * also have been fewer parity errors than parity
 		 * columns or, again, we wouldn't be in this code path.
 		 */
-		ASSERT(parity_untried == 0);
+		ASSERT0(parity_untried);
 		ASSERT(parity_errors < rr->rr_firstdatacol);
 
 		/*
@@ -3515,6 +3850,9 @@ vdev_raidz_io_done(zio_t *zio)
 				raidz_row_t *rr = rm->rm_row[i];
 				vdev_raidz_io_done_verified(zio, rr);
 			}
+			/* Periodically check for a read outlier */
+			if (zio->io_type == ZIO_TYPE_READ)
+				vdev_child_slow_outlier(zio);
 			zio_checksum_verified(zio);
 		} else {
 			/*
@@ -4743,7 +5081,7 @@ spa_raidz_expand_thread(void *arg, zthr_t *zthr)
 void
 spa_start_raidz_expansion_thread(spa_t *spa)
 {
-	ASSERT3P(spa->spa_raidz_expand_zthr, ==, NULL);
+	ASSERT0P(spa->spa_raidz_expand_zthr);
 	spa->spa_raidz_expand_zthr = zthr_create("raidz_expand",
 	    spa_raidz_expand_thread_check, spa_raidz_expand_thread,
 	    spa, defclsyspri);
@@ -5155,3 +5493,10 @@ ZFS_MODULE_PARAM(zfs_vdev, raidz_, io_aggregate_rows, ULONG, ZMOD_RW,
 ZFS_MODULE_PARAM(zfs, zfs_, scrub_after_expand, INT, ZMOD_RW,
 	"For expanded RAIDZ, automatically start a pool scrub when expansion "
 	"completes");
+ZFS_MODULE_PARAM(zfs_vdev, vdev_, read_sit_out_secs, ULONG, ZMOD_RW,
+	"Raidz/draid slow disk sit out time period in seconds");
+ZFS_MODULE_PARAM(zfs_vdev, vdev_, raidz_outlier_check_interval_ms, U64,
+	ZMOD_RW, "Interval to check for slow raidz/draid children");
+ZFS_MODULE_PARAM(zfs_vdev, vdev_, raidz_outlier_insensitivity, UINT,
+	ZMOD_RW, "How insensitive the slow raidz/draid child check should be");
+/* END CSTYLED */
diff --git a/sys/contrib/openzfs/module/zfs/vdev_rebuild.c b/sys/contrib/openzfs/module/zfs/vdev_rebuild.c
index cf259788ccf4..47b3b9921abe 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_rebuild.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_rebuild.c
@@ -256,7 +256,7 @@ vdev_rebuild_initiate_sync(void *arg, dmu_tx_t *tx)
 	    "vdev_id=%llu vdev_guid=%llu started",
 	    (u_longlong_t)vd->vdev_id, (u_longlong_t)vd->vdev_guid);
 
-	ASSERT3P(vd->vdev_rebuild_thread, ==, NULL);
+	ASSERT0P(vd->vdev_rebuild_thread);
 	vd->vdev_rebuild_thread = thread_create(NULL, 0,
 	    vdev_rebuild_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
 
@@ -413,7 +413,7 @@ vdev_rebuild_reset_sync(void *arg, dmu_tx_t *tx)
 	mutex_enter(&vd->vdev_rebuild_lock);
 
 	ASSERT(vrp->vrp_rebuild_state == VDEV_REBUILD_ACTIVE);
-	ASSERT3P(vd->vdev_rebuild_thread, ==, NULL);
+	ASSERT0P(vd->vdev_rebuild_thread);
 
 	vrp->vrp_last_offset = 0;
 	vrp->vrp_min_txg = 0;
diff --git a/sys/contrib/openzfs/module/zfs/vdev_removal.c b/sys/contrib/openzfs/module/zfs/vdev_removal.c
index 3887be4bd548..abb71543e3ab 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_removal.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_removal.c
@@ -51,34 +51,70 @@
 #include <sys/trace_zfs.h>
 
 /*
- * This file contains the necessary logic to remove vdevs from a
- * storage pool.  Currently, the only devices that can be removed
- * are log, cache, and spare devices; and top level vdevs from a pool
- * w/o raidz or mirrors.  (Note that members of a mirror can be removed
- * by the detach operation.)
+ * This file contains the necessary logic to remove vdevs from a storage
+ * pool. Note that members of a mirror can be removed by the detach
+ * operation. Currently, the only devices that can be removed are:
  *
- * Log vdevs are removed by evacuating them and then turning the vdev
- * into a hole vdev while holding spa config locks.
+ * 1) Traditional hot spare and cache vdevs. Note that draid distributed
+ *    spares are fixed at creation time and cannot be removed.
  *
- * Top level vdevs are removed and converted into an indirect vdev via
- * a multi-step process:
+ * 2) Log vdevs are removed by evacuating them and then turning the vdev
+ *    into a hole vdev while holding spa config locks.
  *
- *  - Disable allocations from this device (spa_vdev_remove_top).
+ * 3) Top-level singleton and mirror vdevs, including dedup and special
+ *    vdevs, are removed and converted into an indirect vdev via a
+ *    multi-step process:
  *
- *  - From a new thread (spa_vdev_remove_thread), copy data from
- *    the removing vdev to a different vdev.  The copy happens in open
- *    context (spa_vdev_copy_impl) and issues a sync task
- *    (vdev_mapping_sync) so the sync thread can update the partial
- *    indirect mappings in core and on disk.
+ *    - Disable allocations from this device (spa_vdev_remove_top).
  *
- *  - If a free happens during a removal, it is freed from the
- *    removing vdev, and if it has already been copied, from the new
- *    location as well (free_from_removing_vdev).
+ *    - From a new thread (spa_vdev_remove_thread), copy data from the
+ *      removing vdev to a different vdev. The copy happens in open context
+ *      (spa_vdev_copy_impl) and issues a sync task (vdev_mapping_sync) so
+ *      the sync thread can update the partial indirect mappings in core
+ *      and on disk.
  *
- *  - After the removal is completed, the copy thread converts the vdev
- *    into an indirect vdev (vdev_remove_complete) before instructing
- *    the sync thread to destroy the space maps and finish the removal
- *    (spa_finish_removal).
+ *    - If a free happens during a removal, it is freed from the removing
+ *      vdev, and if it has already been copied, from the new location as
+ *      well (free_from_removing_vdev).
+ *
+ *    - After the removal is completed, the copy thread converts the vdev
+ *      into an indirect vdev (vdev_remove_complete) before instructing
+ *      the sync thread to destroy the space maps and finish the removal
+ *      (spa_finish_removal).
+ *
+ *   The following constraints currently apply primary device removal:
+ *
+ *     - All vdevs must be online, healthy, and not be missing any data
+ *       according to the DTLs.
+ *
+ *     - When removing a singleton or mirror vdev, regardless of it's a
+ *       special, dedup, or primary device, it must have the same ashift
+ *       as the devices in the normal allocation class. Furthermore, all
+ *       vdevs in the normal allocation class must have the same ashift to
+ *       ensure the new allocations never includes additional padding.
+ *
+ *     - The normal allocation class cannot contain any raidz or draid
+ *       top-level vdevs since segments are copied without regard for block
+ *       boundaries. This makes it impossible to calculate the required
+ *       parity columns when using these vdev types as the destination.
+ *
+ *     - The encryption keys must be loaded so the ZIL logs can be reset
+ *       in order to prevent writing to the device being removed.
+ *
+ * N.B. ashift and raidz/draid constraints for primary top-level device
+ * removal could be slightly relaxed if it were possible to request that
+ * DVAs from a mirror or singleton in the specified allocation class be
+ * used (metaslab_alloc_dva).
+ *
+ * This flexibility would be particularly useful for raidz/draid pools which
+ * often include a mirrored special device. If a mistakenly added top-level
+ * singleton were added it could then still be removed at the cost of some
+ * special device capacity. This may be a worthwhile tradeoff depending on
+ * the pool capacity and expense (cost, complexity, time) of creating a new
+ * pool and copying all of the data to correct the configuration.
+ *
+ * Furthermore, while not currently supported it should be possible to allow
+ * vdevs of any type to be removed as long as they've never been written to.
  */
 
 typedef struct vdev_copy_arg {
@@ -344,10 +380,10 @@ spa_vdev_remove_aux(nvlist_t *config, const char *name, nvlist_t **dev,
 	for (int i = 0, j = 0; i < count; i++) {
 		if (dev[i] == dev_to_remove)
 			continue;
-		VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP) == 0);
+		VERIFY0(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP));
 	}
 
-	VERIFY(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY) == 0);
+	VERIFY0(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY));
 	fnvlist_add_nvlist_array(config, name, (const nvlist_t * const *)newdev,
 	    count - 1);
 
@@ -423,7 +459,7 @@ vdev_remove_initiate_sync(void *arg, dmu_tx_t *tx)
 	svr = spa_vdev_removal_create(vd);
 
 	ASSERT(vd->vdev_removing);
-	ASSERT3P(vd->vdev_indirect_mapping, ==, NULL);
+	ASSERT0P(vd->vdev_indirect_mapping);
 
 	spa_feature_incr(spa, SPA_FEATURE_DEVICE_REMOVAL, tx);
 	if (spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)) {
@@ -529,7 +565,7 @@ vdev_remove_initiate_sync(void *arg, dmu_tx_t *tx)
 	 * but in any case only when there are outstanding free i/os, which
 	 * there are not).
 	 */
-	ASSERT3P(spa->spa_vdev_removal, ==, NULL);
+	ASSERT0P(spa->spa_vdev_removal);
 	spa->spa_vdev_removal = svr;
 	svr->svr_thread = thread_create(NULL, 0,
 	    spa_vdev_remove_thread, spa, 0, &p0, TS_RUN, minclsyspri);
@@ -1362,11 +1398,11 @@ vdev_remove_complete(spa_t *spa)
 	txg_wait_synced(spa->spa_dsl_pool, 0);
 	txg = spa_vdev_enter(spa);
 	vdev_t *vd = vdev_lookup_top(spa, spa->spa_vdev_removal->svr_vdev_id);
-	ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
-	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
-	ASSERT3P(vd->vdev_autotrim_thread, ==, NULL);
+	ASSERT0P(vd->vdev_initialize_thread);
+	ASSERT0P(vd->vdev_trim_thread);
+	ASSERT0P(vd->vdev_autotrim_thread);
 	vdev_rebuild_stop_wait(vd);
-	ASSERT3P(vd->vdev_rebuild_thread, ==, NULL);
+	ASSERT0P(vd->vdev_rebuild_thread);
 
 	sysevent_t *ev = spa_event_create(spa, vd, NULL,
 	    ESC_ZFS_VDEV_REMOVE_DEV);
@@ -1868,7 +1904,7 @@ spa_vdev_remove_cancel_sync(void *arg, dmu_tx_t *tx)
 	vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
 	objset_t *mos = spa->spa_meta_objset;
 
-	ASSERT3P(svr->svr_thread, ==, NULL);
+	ASSERT0P(svr->svr_thread);
 
 	spa_feature_decr(spa, SPA_FEATURE_DEVICE_REMOVAL, tx);
 
@@ -2076,7 +2112,7 @@ spa_vdev_remove_log(vdev_t *vd, uint64_t *txg)
 
 	ASSERT(vd->vdev_islog);
 	ASSERT(vd == vd->vdev_top);
-	ASSERT3P(vd->vdev_log_mg, ==, NULL);
+	ASSERT0P(vd->vdev_log_mg);
 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
 
 	/*
@@ -2112,7 +2148,7 @@ spa_vdev_remove_log(vdev_t *vd, uint64_t *txg)
 
 	if (error != 0) {
 		metaslab_group_activate(mg);
-		ASSERT3P(vd->vdev_log_mg, ==, NULL);
+		ASSERT0P(vd->vdev_log_mg);
 		return (error);
 	}
 	ASSERT0(vd->vdev_stat.vs_alloc);
diff --git a/sys/contrib/openzfs/module/zfs/vdev_trim.c b/sys/contrib/openzfs/module/zfs/vdev_trim.c
index fc8d5b8e9a8a..eee18b367909 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_trim.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_trim.c
@@ -1010,7 +1010,7 @@ vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)
 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
 	ASSERT(vd->vdev_ops->vdev_op_leaf);
 	ASSERT(vdev_is_concrete(vd));
-	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
+	ASSERT0P(vd->vdev_trim_thread);
 	ASSERT(!vd->vdev_detached);
 	ASSERT(!vd->vdev_trim_exit_wanted);
 	ASSERT(!vd->vdev_top->vdev_removing);
@@ -1032,7 +1032,7 @@ vdev_trim_stop_wait_impl(vdev_t *vd)
 	while (vd->vdev_trim_thread != NULL)
 		cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);
 
-	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
+	ASSERT0P(vd->vdev_trim_thread);
 	vd->vdev_trim_exit_wanted = B_FALSE;
 }
 
@@ -1539,7 +1539,7 @@ vdev_autotrim_stop_wait(vdev_t *tvd)
 		cv_wait(&tvd->vdev_autotrim_cv,
 		    &tvd->vdev_autotrim_lock);
 
-		ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL);
+		ASSERT0P(tvd->vdev_autotrim_thread);
 		tvd->vdev_autotrim_exit_wanted = B_FALSE;
 	}
 	mutex_exit(&tvd->vdev_autotrim_lock);
@@ -1712,7 +1712,7 @@ vdev_trim_l2arc(spa_t *spa)
 		mutex_enter(&vd->vdev_trim_lock);
 		ASSERT(vd->vdev_ops->vdev_op_leaf);
 		ASSERT(vdev_is_concrete(vd));
-		ASSERT3P(vd->vdev_trim_thread, ==, NULL);
+		ASSERT0P(vd->vdev_trim_thread);
 		ASSERT(!vd->vdev_detached);
 		ASSERT(!vd->vdev_trim_exit_wanted);
 		ASSERT(!vd->vdev_top->vdev_removing);
diff --git a/sys/contrib/openzfs/module/zfs/zap.c b/sys/contrib/openzfs/module/zfs/zap.c
index 0896690c97e3..3e4e997798a3 100644
--- a/sys/contrib/openzfs/module/zfs/zap.c
+++ b/sys/contrib/openzfs/module/zfs/zap.c
@@ -921,7 +921,7 @@ fzap_add_cd(zap_name_t *zn,
 
 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));
 	ASSERT(!zap->zap_ismicro);
-	ASSERT(fzap_check(zn, integer_size, num_integers) == 0);
+	ASSERT0(fzap_check(zn, integer_size, num_integers));
 
 	err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l);
 	if (err != 0)
@@ -1386,7 +1386,7 @@ again:
 		}
 		err = zap_entry_read_name(zap, &zeh,
 		    za->za_name_len, za->za_name);
-		ASSERT(err == 0);
+		ASSERT0(err);
 
 		za->za_normalization_conflict =
 		    zap_entry_normalization_conflict(&zeh,
@@ -1546,7 +1546,7 @@ zap_shrink(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx)
 	boolean_t trunc = B_FALSE;
 	int err = 0;
 
-	ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_nentries, ==, 0);
+	ASSERT0(zap_leaf_phys(l)->l_hdr.lh_nentries);
 	ASSERT3U(prefix_len, <=, zap_f_phys(zap)->zap_ptrtbl.zt_shift);
 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));
 	ASSERT3U(ZAP_HASH_IDX(hash, prefix_len), ==, prefix);
@@ -1564,7 +1564,7 @@ zap_shrink(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx)
 		uint64_t sl_hash = ZAP_PREFIX_HASH(sl_prefix, prefix_len);
 		int slbit = prefix & 1;
 
-		ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_nentries, ==, 0);
+		ASSERT0(zap_leaf_phys(l)->l_hdr.lh_nentries);
 
 		/*
 		 * Check if there is a sibling by reading ptrtbl ptrs.
diff --git a/sys/contrib/openzfs/module/zfs/zap_micro.c b/sys/contrib/openzfs/module/zfs/zap_micro.c
index 411b1a9db5ab..ea4e3117a8b9 100644
--- a/sys/contrib/openzfs/module/zfs/zap_micro.c
+++ b/sys/contrib/openzfs/module/zfs/zap_micro.c
@@ -346,7 +346,7 @@ zap_name_alloc_uint64(zap_t *zap, const uint64_t *key, int numints)
 {
 	zap_name_t *zn = kmem_cache_alloc(zap_name_cache, KM_SLEEP);
 
-	ASSERT(zap->zap_normflags == 0);
+	ASSERT0(zap->zap_normflags);
 	zn->zn_zap = zap;
 	zn->zn_key_intlen = sizeof (*key);
 	zn->zn_key_orig = zn->zn_key_norm = key;
@@ -1876,7 +1876,7 @@ zap_cursor_serialize(zap_cursor_t *zc)
 		return (-1ULL);
 	if (zc->zc_zap == NULL)
 		return (zc->zc_serialized);
-	ASSERT((zc->zc_hash & zap_maxcd(zc->zc_zap)) == 0);
+	ASSERT0((zc->zc_hash & zap_maxcd(zc->zc_zap)));
 	ASSERT(zc->zc_cd < zap_maxcd(zc->zc_zap));
 
 	/*
@@ -1911,7 +1911,7 @@ zap_cursor_retrieve(zap_cursor_t *zc, zap_attribute_t *za)
 		 * we must add to the existing zc_cd, which may already
 		 * be 1 due to the zap_cursor_advance.
 		 */
-		ASSERT(zc->zc_hash == 0);
+		ASSERT0(zc->zc_hash);
 		hb = zap_hashbits(zc->zc_zap);
 		zc->zc_hash = zc->zc_serialized << (64 - hb);
 		zc->zc_cd += zc->zc_serialized >> hb;
diff --git a/sys/contrib/openzfs/module/zfs/zcp.c b/sys/contrib/openzfs/module/zfs/zcp.c
index 9aecf67fd256..c6684f453e95 100644
--- a/sys/contrib/openzfs/module/zfs/zcp.c
+++ b/sys/contrib/openzfs/module/zfs/zcp.c
@@ -765,7 +765,7 @@ zcp_lua_alloc(void *ud, void *ptr, size_t osize, size_t nsize)
 			return (NULL);
 		}
 		(void) memcpy(luabuf, ptr, osize);
-		VERIFY3P(zcp_lua_alloc(ud, ptr, osize, 0), ==, NULL);
+		VERIFY0P(zcp_lua_alloc(ud, ptr, osize, 0));
 		return (luabuf);
 	}
 }
diff --git a/sys/contrib/openzfs/module/zfs/zfeature.c b/sys/contrib/openzfs/module/zfs/zfeature.c
index 7dfe00d42a08..4cf9e0dbb405 100644
--- a/sys/contrib/openzfs/module/zfs/zfeature.c
+++ b/sys/contrib/openzfs/module/zfs/zfeature.c
@@ -210,8 +210,8 @@ spa_features_check(spa_t *spa, boolean_t for_write,
 				    za->za_name, 1, MAXPATHLEN, buf) == 0)
 					desc = buf;
 
-				VERIFY(nvlist_add_string(unsup_feat,
-				    za->za_name, desc) == 0);
+				VERIFY0(nvlist_add_string(unsup_feat,
+				    za->za_name, desc));
 			}
 		}
 	}
@@ -308,6 +308,7 @@ feature_sync(spa_t *spa, zfeature_info_t *feature, uint64_t refcount,
 	ASSERT(VALID_FEATURE_OR_NONE(feature->fi_feature));
 	uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
 	    spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj;
+	ASSERT(MUTEX_HELD(&spa->spa_feat_stats_lock));
 	VERIFY0(zap_update(spa->spa_meta_objset, zapobj, feature->fi_guid,
 	    sizeof (uint64_t), 1, &refcount, tx));
 
@@ -360,7 +361,9 @@ feature_enable_sync(spa_t *spa, zfeature_info_t *feature, dmu_tx_t *tx)
 	    feature->fi_guid, 1, strlen(feature->fi_desc) + 1,
 	    feature->fi_desc, tx));
 
+	mutex_enter(&spa->spa_feat_stats_lock);
 	feature_sync(spa, feature, initial_refcount, tx);
+	mutex_exit(&spa->spa_feat_stats_lock);
 
 	if (spa_feature_is_enabled(spa, SPA_FEATURE_ENABLED_TXG)) {
 		uint64_t enabling_txg = dmu_tx_get_txg(tx);
@@ -416,6 +419,7 @@ feature_do_action(spa_t *spa, spa_feature_t fid, feature_action_t action,
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT3U(spa_version(spa), >=, SPA_VERSION_FEATURES);
 
+	mutex_enter(&spa->spa_feat_stats_lock);
 	VERIFY3U(feature_get_refcount(spa, feature, &refcount), !=, ENOTSUP);
 
 	switch (action) {
@@ -433,6 +437,7 @@ feature_do_action(spa_t *spa, spa_feature_t fid, feature_action_t action,
 	}
 
 	feature_sync(spa, feature, refcount, tx);
+	mutex_exit(&spa->spa_feat_stats_lock);
 }
 
 void
diff --git a/sys/contrib/openzfs/module/zfs/zfs_crrd.c b/sys/contrib/openzfs/module/zfs/zfs_crrd.c
index f9267ed41d71..30d4c7c36897 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_crrd.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_crrd.c
@@ -162,9 +162,9 @@ dbrrd_add(dbrrd_t *db, hrtime_t time, uint64_t txg)
 	daydiff = time - rrd_tail(&db->dbr_days);
 	monthdiff = time - rrd_tail(&db->dbr_months);
 
-	if (monthdiff >= 0 && monthdiff >= SEC2NSEC(30 * 24 * 60 * 60))
+	if (monthdiff >= 0 && monthdiff >= 30 * 24 * 60 * 60)
 		rrd_add(&db->dbr_months, time, txg);
-	else if (daydiff >= 0 && daydiff >= SEC2NSEC(24 * 60 * 60))
+	else if (daydiff >= 0 && daydiff >= 24 * 60 * 60)
 		rrd_add(&db->dbr_days, time, txg);
 	else if (minutedif >= 0)
 		rrd_add(&db->dbr_minutes, time, txg);
@@ -208,7 +208,8 @@ dbrrd_closest(hrtime_t tv, const rrd_data_t *r1, const rrd_data_t *r2)
 	if (r2 == NULL)
 		return (r1);
 
-	return (ABS(tv - r1->rrdd_time) < ABS(tv - r2->rrdd_time) ? r1 : r2);
+	return (ABS(tv - (hrtime_t)r1->rrdd_time) <
+	    ABS(tv - (hrtime_t)r2->rrdd_time) ? r1 : r2);
 }
 
 uint64_t
diff --git a/sys/contrib/openzfs/module/zfs/zfs_fuid.c b/sys/contrib/openzfs/module/zfs/zfs_fuid.c
index 10a6d289fbf8..2af1efe82e62 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_fuid.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_fuid.c
@@ -112,8 +112,7 @@ zfs_fuid_table_load(objset_t *os, uint64_t fuid_obj, avl_tree_t *idx_tree,
 	uint64_t fuid_size;
 
 	ASSERT(fuid_obj != 0);
-	VERIFY(0 == dmu_bonus_hold(os, fuid_obj,
-	    FTAG, &db));
+	VERIFY0(dmu_bonus_hold(os, fuid_obj, FTAG, &db));
 	fuid_size = *(uint64_t *)db->db_data;
 	dmu_buf_rele(db, FTAG);
 
@@ -125,22 +124,21 @@ zfs_fuid_table_load(objset_t *os, uint64_t fuid_obj, avl_tree_t *idx_tree,
 		int i;
 
 		packed = kmem_alloc(fuid_size, KM_SLEEP);
-		VERIFY(dmu_read(os, fuid_obj, 0,
-		    fuid_size, packed, DMU_READ_PREFETCH) == 0);
-		VERIFY(nvlist_unpack(packed, fuid_size,
-		    &nvp, 0) == 0);
-		VERIFY(nvlist_lookup_nvlist_array(nvp, FUID_NVP_ARRAY,
-		    &fuidnvp, &count) == 0);
+		VERIFY0(dmu_read(os, fuid_obj, 0,
+		    fuid_size, packed, DMU_READ_PREFETCH));
+		VERIFY0(nvlist_unpack(packed, fuid_size, &nvp, 0));
+		VERIFY0(nvlist_lookup_nvlist_array(nvp, FUID_NVP_ARRAY,
+		    &fuidnvp, &count));
 
 		for (i = 0; i != count; i++) {
 			fuid_domain_t *domnode;
 			const char *domain;
 			uint64_t idx;
 
-			VERIFY(nvlist_lookup_string(fuidnvp[i], FUID_DOMAIN,
-			    &domain) == 0);
-			VERIFY(nvlist_lookup_uint64(fuidnvp[i], FUID_IDX,
-			    &idx) == 0);
+			VERIFY0(nvlist_lookup_string(fuidnvp[i], FUID_DOMAIN,
+			    &domain));
+			VERIFY0(nvlist_lookup_uint64(fuidnvp[i], FUID_IDX,
+			    &idx));
 
 			domnode = kmem_alloc(sizeof (fuid_domain_t), KM_SLEEP);
 
@@ -246,35 +244,33 @@ zfs_fuid_sync(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
 		    &zfsvfs->z_fuid_obj, tx) == 0);
 	}
 
-	VERIFY(nvlist_alloc(&nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+	VERIFY0(nvlist_alloc(&nvp, NV_UNIQUE_NAME, KM_SLEEP));
 
 	numnodes = avl_numnodes(&zfsvfs->z_fuid_idx);
 	fuids = kmem_alloc(numnodes * sizeof (void *), KM_SLEEP);
 	for (i = 0, domnode = avl_first(&zfsvfs->z_fuid_domain); domnode; i++,
 	    domnode = AVL_NEXT(&zfsvfs->z_fuid_domain, domnode)) {
-		VERIFY(nvlist_alloc(&fuids[i], NV_UNIQUE_NAME, KM_SLEEP) == 0);
-		VERIFY(nvlist_add_uint64(fuids[i], FUID_IDX,
-		    domnode->f_idx) == 0);
-		VERIFY(nvlist_add_uint64(fuids[i], FUID_OFFSET, 0) == 0);
-		VERIFY(nvlist_add_string(fuids[i], FUID_DOMAIN,
-		    domnode->f_ksid->kd_name) == 0);
+		VERIFY0(nvlist_alloc(&fuids[i], NV_UNIQUE_NAME, KM_SLEEP));
+		VERIFY0(nvlist_add_uint64(fuids[i], FUID_IDX,
+		    domnode->f_idx));
+		VERIFY0(nvlist_add_uint64(fuids[i], FUID_OFFSET, 0));
+		VERIFY0(nvlist_add_string(fuids[i], FUID_DOMAIN,
+		    domnode->f_ksid->kd_name));
 	}
 	fnvlist_add_nvlist_array(nvp, FUID_NVP_ARRAY,
 	    (const nvlist_t * const *)fuids, numnodes);
 	for (i = 0; i != numnodes; i++)
 		nvlist_free(fuids[i]);
 	kmem_free(fuids, numnodes * sizeof (void *));
-	VERIFY(nvlist_size(nvp, &nvsize, NV_ENCODE_XDR) == 0);
+	VERIFY0(nvlist_size(nvp, &nvsize, NV_ENCODE_XDR));
 	packed = kmem_alloc(nvsize, KM_SLEEP);
-	VERIFY(nvlist_pack(nvp, &packed, &nvsize,
-	    NV_ENCODE_XDR, KM_SLEEP) == 0);
+	VERIFY0(nvlist_pack(nvp, &packed, &nvsize, NV_ENCODE_XDR, KM_SLEEP));
 	nvlist_free(nvp);
 	zfsvfs->z_fuid_size = nvsize;
 	dmu_write(zfsvfs->z_os, zfsvfs->z_fuid_obj, 0,
 	    zfsvfs->z_fuid_size, packed, tx);
 	kmem_free(packed, zfsvfs->z_fuid_size);
-	VERIFY(0 == dmu_bonus_hold(zfsvfs->z_os, zfsvfs->z_fuid_obj,
-	    FTAG, &db));
+	VERIFY0(dmu_bonus_hold(zfsvfs->z_os, zfsvfs->z_fuid_obj, FTAG, &db));
 	dmu_buf_will_dirty(db, tx);
 	*(uint64_t *)db->db_data = zfsvfs->z_fuid_size;
 	dmu_buf_rele(db, FTAG);
diff --git a/sys/contrib/openzfs/module/zfs/zfs_ioctl.c b/sys/contrib/openzfs/module/zfs/zfs_ioctl.c
index dcb71229f96a..5ca7c2320c4e 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_ioctl.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_ioctl.c
@@ -683,6 +683,7 @@ zfs_secpolicy_send(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr)
 	dsl_dataset_t *ds;
 	const char *cp;
 	int error;
+	boolean_t rawok = (zc->zc_flags & 0x8);
 
 	/*
 	 * Generate the current snapshot name from the given objsetid, then
@@ -705,6 +706,10 @@ zfs_secpolicy_send(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr)
 
 	error = zfs_secpolicy_write_perms_ds(zc->zc_name, ds,
 	    ZFS_DELEG_PERM_SEND, cr);
+	if (error != 0 && rawok == B_TRUE) {
+		error = zfs_secpolicy_write_perms_ds(zc->zc_name, ds,
+		    ZFS_DELEG_PERM_SEND_RAW, cr);
+	}
 	dsl_dataset_rele(ds, FTAG);
 	dsl_pool_rele(dp, FTAG);
 
@@ -714,9 +719,17 @@ zfs_secpolicy_send(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr)
 static int
 zfs_secpolicy_send_new(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr)
 {
+	boolean_t rawok = nvlist_exists(innvl, "rawok");
+	int error;
+
 	(void) innvl;
-	return (zfs_secpolicy_write_perms(zc->zc_name,
-	    ZFS_DELEG_PERM_SEND, cr));
+	error = zfs_secpolicy_write_perms(zc->zc_name,
+	    ZFS_DELEG_PERM_SEND, cr);
+	if (error != 0 && rawok == B_TRUE) {
+		error = zfs_secpolicy_write_perms(zc->zc_name,
+		    ZFS_DELEG_PERM_SEND_RAW, cr);
+	}
+	return (error);
 }
 
 static int
@@ -1493,7 +1506,7 @@ zfs_ioc_pool_create(zfs_cmd_t *zc)
 			goto pool_props_bad;
 		(void) nvlist_remove_all(props, ZPOOL_HIDDEN_ARGS);
 
-		VERIFY(nvlist_alloc(&zplprops, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+		VERIFY0(nvlist_alloc(&zplprops, NV_UNIQUE_NAME, KM_SLEEP));
 		error = zfs_fill_zplprops_root(version, rootprops,
 		    zplprops, NULL);
 		if (error != 0)
@@ -2245,7 +2258,7 @@ nvl_add_zplprop(objset_t *os, nvlist_t *props, zfs_prop_t prop)
 	 */
 	if ((error = zfs_get_zplprop(os, prop, &value)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(props, zfs_prop_to_name(prop), value) == 0);
+	VERIFY0(nvlist_add_uint64(props, zfs_prop_to_name(prop), value));
 	return (0);
 }
 
@@ -2280,7 +2293,7 @@ zfs_ioc_objset_zplprops(zfs_cmd_t *zc)
 	    dmu_objset_type(os) == DMU_OST_ZFS) {
 		nvlist_t *nv;
 
-		VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+		VERIFY0(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP));
 		if ((err = nvl_add_zplprop(os, nv, ZFS_PROP_VERSION)) == 0 &&
 		    (err = nvl_add_zplprop(os, nv, ZFS_PROP_NORMALIZE)) == 0 &&
 		    (err = nvl_add_zplprop(os, nv, ZFS_PROP_UTF8ONLY)) == 0 &&
@@ -2483,7 +2496,7 @@ zfs_prop_set_userquota(const char *dsname, nvpair_t *pair)
 
 	if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
 		nvlist_t *attrs;
-		VERIFY(nvpair_value_nvlist(pair, &attrs) == 0);
+		VERIFY0(nvpair_value_nvlist(pair, &attrs));
 		if (nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
 		    &pair) != 0)
 			return (SET_ERROR(EINVAL));
@@ -2538,9 +2551,8 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source,
 
 	if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
 		nvlist_t *attrs;
-		VERIFY(nvpair_value_nvlist(pair, &attrs) == 0);
-		VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
-		    &pair) == 0);
+		VERIFY0(nvpair_value_nvlist(pair, &attrs));
+		VERIFY0(nvlist_lookup_nvpair(attrs, ZPROP_VALUE, &pair));
 	}
 
 	/* all special properties are numeric except for keylocation */
@@ -2932,14 +2944,14 @@ props_skip(nvlist_t *props, nvlist_t *skipped, nvlist_t **newprops)
 {
 	nvpair_t *pair;
 
-	VERIFY(nvlist_alloc(newprops, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+	VERIFY0(nvlist_alloc(newprops, NV_UNIQUE_NAME, KM_SLEEP));
 
 	pair = NULL;
 	while ((pair = nvlist_next_nvpair(props, pair)) != NULL) {
 		if (nvlist_exists(skipped, nvpair_name(pair)))
 			continue;
 
-		VERIFY(nvlist_add_nvpair(*newprops, pair) == 0);
+		VERIFY0(nvlist_add_nvpair(*newprops, pair));
 	}
 }
 
@@ -3064,11 +3076,11 @@ zfs_ioc_inherit_prop(zfs_cmd_t *zc)
 
 	switch (type) {
 	case PROP_TYPE_STRING:
-		VERIFY(0 == nvlist_add_string(dummy, propname, ""));
+		VERIFY0(nvlist_add_string(dummy, propname, ""));
 		break;
 	case PROP_TYPE_NUMBER:
 	case PROP_TYPE_INDEX:
-		VERIFY(0 == nvlist_add_uint64(dummy, propname, 0));
+		VERIFY0(nvlist_add_uint64(dummy, propname, 0));
 		break;
 	default:
 		err = SET_ERROR(EINVAL);
@@ -3454,14 +3466,14 @@ zfs_fill_zplprops_impl(objset_t *os, uint64_t zplver,
 	/*
 	 * Put the version in the zplprops
 	 */
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_VERSION), zplver) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_VERSION), zplver));
 
 	if (norm == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_NORMALIZE, &norm)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_NORMALIZE), norm) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_NORMALIZE), norm));
 
 	/*
 	 * If we're normalizing, names must always be valid UTF-8 strings.
@@ -3471,55 +3483,55 @@ zfs_fill_zplprops_impl(objset_t *os, uint64_t zplver,
 	if (u8 == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_UTF8ONLY, &u8)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_UTF8ONLY), u8) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_UTF8ONLY), u8));
 
 	if (sense == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_CASE, &sense)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_CASE), sense) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_CASE), sense));
 
 	if (duq == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_DEFAULTUSERQUOTA, &duq)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_DEFAULTUSERQUOTA), duq) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_DEFAULTUSERQUOTA), duq));
 
 	if (dgq == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_DEFAULTGROUPQUOTA,
 	    &dgq)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_DEFAULTGROUPQUOTA), dgq) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_DEFAULTGROUPQUOTA), dgq));
 
 	if (dpq == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_DEFAULTPROJECTQUOTA,
 	    &dpq)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_DEFAULTPROJECTQUOTA), dpq) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_DEFAULTPROJECTQUOTA), dpq));
 
 	if (duoq == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_DEFAULTUSEROBJQUOTA,
 	    &duoq)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_DEFAULTUSEROBJQUOTA), duoq) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_DEFAULTUSEROBJQUOTA), duoq));
 
 	if (dgoq == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_DEFAULTGROUPOBJQUOTA,
 	    &dgoq)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_DEFAULTGROUPOBJQUOTA), dgoq) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_DEFAULTGROUPOBJQUOTA), dgoq));
 
 	if (dpoq == ZFS_PROP_UNDEFINED &&
 	    (error = zfs_get_zplprop(os, ZFS_PROP_DEFAULTPROJECTOBJQUOTA,
 	    &dpoq)) != 0)
 		return (error);
-	VERIFY(nvlist_add_uint64(zplprops,
-	    zfs_prop_to_name(ZFS_PROP_DEFAULTPROJECTOBJQUOTA), dpoq) == 0);
+	VERIFY0(nvlist_add_uint64(zplprops,
+	    zfs_prop_to_name(ZFS_PROP_DEFAULTPROJECTOBJQUOTA), dpoq));
 
 	if (is_ci)
 		*is_ci = (sense == ZFS_CASE_INSENSITIVE);
@@ -3668,8 +3680,8 @@ zfs_ioc_create(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
 		 * file system creation, so go figure them out
 		 * now.
 		 */
-		VERIFY(nvlist_alloc(&zct.zct_zplprops,
-		    NV_UNIQUE_NAME, KM_SLEEP) == 0);
+		VERIFY0(nvlist_alloc(&zct.zct_zplprops,
+		    NV_UNIQUE_NAME, KM_SLEEP));
 		error = zfs_fill_zplprops(fsname, nvprops,
 		    zct.zct_zplprops, &is_insensitive);
 		if (error != 0) {
@@ -4727,7 +4739,7 @@ zfs_ioc_rollback(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
 			error = error ? error : resume_err;
 		}
 		zfs_vfs_rele(zfsvfs);
-	} else if ((zv = zvol_suspend(fsname)) != NULL) {
+	} else if (zvol_suspend(fsname, &zv) == 0) {
 		error = dsl_dataset_rollback(fsname, target, zvol_tag(zv),
 		    outnvl);
 		zvol_resume(zv);
@@ -4916,9 +4928,8 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr)
 		 * format.
 		 */
 		nvlist_t *attrs;
-		VERIFY(nvpair_value_nvlist(pair, &attrs) == 0);
-		VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
-		    &pair) == 0);
+		VERIFY0(nvpair_value_nvlist(pair, &attrs));
+		VERIFY0(nvlist_lookup_nvpair(attrs, ZPROP_VALUE, &pair));
 	}
 
 	/*
@@ -5103,7 +5114,7 @@ zfs_check_clearable(const char *dataset, nvlist_t *props, nvlist_t **errlist)
 	if (props == NULL)
 		return (0);
 
-	VERIFY(nvlist_alloc(&errors, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+	VERIFY0(nvlist_alloc(&errors, NV_UNIQUE_NAME, KM_SLEEP));
 
 	zc = kmem_alloc(sizeof (zfs_cmd_t), KM_SLEEP);
 	(void) strlcpy(zc->zc_name, dataset, sizeof (zc->zc_name));
@@ -5115,9 +5126,8 @@ zfs_check_clearable(const char *dataset, nvlist_t *props, nvlist_t **errlist)
 		    sizeof (zc->zc_value));
 		if ((err = zfs_check_settable(dataset, pair, CRED())) != 0 ||
 		    (err = zfs_secpolicy_inherit_prop(zc, NULL, CRED())) != 0) {
-			VERIFY(nvlist_remove_nvpair(props, pair) == 0);
-			VERIFY(nvlist_add_int32(errors,
-			    zc->zc_value, err) == 0);
+			VERIFY0(nvlist_remove_nvpair(props, pair));
+			VERIFY0(nvlist_add_int32(errors, zc->zc_value, err));
 		}
 		pair = next_pair;
 	}
@@ -5127,7 +5137,7 @@ zfs_check_clearable(const char *dataset, nvlist_t *props, nvlist_t **errlist)
 		nvlist_free(errors);
 		errors = NULL;
 	} else {
-		VERIFY(nvpair_value_int32(pair, &rv) == 0);
+		VERIFY0(nvpair_value_int32(pair, &rv));
 	}
 
 	if (errlist == NULL)
@@ -5144,16 +5154,14 @@ propval_equals(nvpair_t *p1, nvpair_t *p2)
 	if (nvpair_type(p1) == DATA_TYPE_NVLIST) {
 		/* dsl_prop_get_all_impl() format */
 		nvlist_t *attrs;
-		VERIFY(nvpair_value_nvlist(p1, &attrs) == 0);
-		VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
-		    &p1) == 0);
+		VERIFY0(nvpair_value_nvlist(p1, &attrs));
+		VERIFY0(nvlist_lookup_nvpair(attrs, ZPROP_VALUE, &p1));
 	}
 
 	if (nvpair_type(p2) == DATA_TYPE_NVLIST) {
 		nvlist_t *attrs;
-		VERIFY(nvpair_value_nvlist(p2, &attrs) == 0);
-		VERIFY(nvlist_lookup_nvpair(attrs, ZPROP_VALUE,
-		    &p2) == 0);
+		VERIFY0(nvpair_value_nvlist(p2, &attrs));
+		VERIFY0(nvlist_lookup_nvpair(attrs, ZPROP_VALUE, &p2));
 	}
 
 	if (nvpair_type(p1) != nvpair_type(p2))
@@ -5162,14 +5170,14 @@ propval_equals(nvpair_t *p1, nvpair_t *p2)
 	if (nvpair_type(p1) == DATA_TYPE_STRING) {
 		const char *valstr1, *valstr2;
 
-		VERIFY(nvpair_value_string(p1, &valstr1) == 0);
-		VERIFY(nvpair_value_string(p2, &valstr2) == 0);
+		VERIFY0(nvpair_value_string(p1, &valstr1));
+		VERIFY0(nvpair_value_string(p2, &valstr2));
 		return (strcmp(valstr1, valstr2) == 0);
 	} else {
 		uint64_t intval1, intval2;
 
-		VERIFY(nvpair_value_uint64(p1, &intval1) == 0);
-		VERIFY(nvpair_value_uint64(p2, &intval2) == 0);
+		VERIFY0(nvpair_value_uint64(p1, &intval1));
+		VERIFY0(nvpair_value_uint64(p2, &intval2));
 		return (intval1 == intval2);
 	}
 }
@@ -5237,7 +5245,7 @@ extract_delay_props(nvlist_t *props)
 	};
 	int i;
 
-	VERIFY(nvlist_alloc(&delayprops, NV_UNIQUE_NAME, KM_SLEEP) == 0);
+	VERIFY0(nvlist_alloc(&delayprops, NV_UNIQUE_NAME, KM_SLEEP));
 
 	for (nvp = nvlist_next_nvpair(props, NULL); nvp != NULL;
 	    nvp = nvlist_next_nvpair(props, nvp)) {
@@ -5253,8 +5261,8 @@ extract_delay_props(nvlist_t *props)
 		}
 		if (delayable[i] != 0) {
 			tmp = nvlist_prev_nvpair(props, nvp);
-			VERIFY(nvlist_add_nvpair(delayprops, nvp) == 0);
-			VERIFY(nvlist_remove_nvpair(props, nvp) == 0);
+			VERIFY0(nvlist_add_nvpair(delayprops, nvp));
+			VERIFY0(nvlist_remove_nvpair(props, nvp));
 			nvp = tmp;
 		}
 	}
@@ -5453,7 +5461,7 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, const char *origin,
 			}
 			error = error ? error : end_err;
 			zfs_vfs_rele(zfsvfs);
-		} else if ((zv = zvol_suspend(tofs)) != NULL) {
+		} else if (zvol_suspend(tofs, &zv) == 0) {
 			error = dmu_recv_end(&drc, zvol_tag(zv));
 			zvol_resume(zv);
 		} else {
@@ -5485,15 +5493,15 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, const char *origin,
 	 * using ASSERT() will be just like a VERIFY.
 	 */
 	if (recv_delayprops != NULL) {
-		ASSERT(nvlist_merge(recvprops, recv_delayprops, 0) == 0);
+		ASSERT0(nvlist_merge(recvprops, recv_delayprops, 0));
 		nvlist_free(recv_delayprops);
 	}
 	if (local_delayprops != NULL) {
-		ASSERT(nvlist_merge(localprops, local_delayprops, 0) == 0);
+		ASSERT0(nvlist_merge(localprops, local_delayprops, 0));
 		nvlist_free(local_delayprops);
 	}
 	if (inherited_delayprops != NULL) {
-		ASSERT(nvlist_merge(localprops, inherited_delayprops, 0) == 0);
+		ASSERT0(nvlist_merge(localprops, inherited_delayprops, 0));
 		nvlist_free(inherited_delayprops);
 	}
 	*read_bytes = off - noff;
@@ -7342,8 +7350,8 @@ zfs_ioctl_register_legacy(zfs_ioc_t ioc, zfs_ioc_legacy_func_t *func,
 
 	ASSERT3U(ioc, >=, ZFS_IOC_FIRST);
 	ASSERT3U(ioc, <, ZFS_IOC_LAST);
-	ASSERT3P(vec->zvec_legacy_func, ==, NULL);
-	ASSERT3P(vec->zvec_func, ==, NULL);
+	ASSERT0P(vec->zvec_legacy_func);
+	ASSERT0P(vec->zvec_func);
 
 	vec->zvec_legacy_func = func;
 	vec->zvec_secpolicy = secpolicy;
@@ -7366,8 +7374,8 @@ zfs_ioctl_register(const char *name, zfs_ioc_t ioc, zfs_ioc_func_t *func,
 
 	ASSERT3U(ioc, >=, ZFS_IOC_FIRST);
 	ASSERT3U(ioc, <, ZFS_IOC_LAST);
-	ASSERT3P(vec->zvec_legacy_func, ==, NULL);
-	ASSERT3P(vec->zvec_func, ==, NULL);
+	ASSERT0P(vec->zvec_legacy_func);
+	ASSERT0P(vec->zvec_func);
 
 	/* if we are logging, the name must be valid */
 	ASSERT(!allow_log || namecheck != NO_NAME);
@@ -7624,7 +7632,7 @@ zfs_ioctl_init(void)
 
 	zfs_ioctl_register("scrub", ZFS_IOC_POOL_SCRUB,
 	    zfs_ioc_pool_scrub, zfs_secpolicy_config, POOL_NAME,
-	    POOL_CHECK_NONE, B_TRUE, B_TRUE,
+	    POOL_CHECK_SUSPENDED | POOL_CHECK_READONLY, B_TRUE, B_TRUE,
 	    zfs_keys_pool_scrub, ARRAY_SIZE(zfs_keys_pool_scrub));
 
 	zfs_ioctl_register("get_props", ZFS_IOC_POOL_GET_PROPS,
@@ -8148,7 +8156,7 @@ zfsdev_ioctl_common(uint_t vecnum, zfs_cmd_t *zc, int flag)
 		spa_t *spa;
 		nvlist_t *lognv = NULL;
 
-		ASSERT(vec->zvec_legacy_func == NULL);
+		ASSERT0P(vec->zvec_legacy_func);
 
 		/*
 		 * Add the innvl to the lognv before calling the func,
diff --git a/sys/contrib/openzfs/module/zfs/zfs_log.c b/sys/contrib/openzfs/module/zfs/zfs_log.c
index 2f61ecfd9b3b..ea17e049279f 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_log.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_log.c
@@ -620,7 +620,7 @@ zfs_log_write(zilog_t *zilog, dmu_tx_t *tx, int txtype,
 	if (zil_replaying(zilog, tx) || zp->z_unlinked ||
 	    zfs_xattr_owner_unlinked(zp)) {
 		if (callback != NULL)
-			callback(callback_data);
+			callback(callback_data, 0);
 		return;
 	}
 
@@ -663,7 +663,7 @@ zfs_log_write(zilog_t *zilog, dmu_tx_t *tx, int txtype,
 			    DMU_KEEP_CACHING);
 			DB_DNODE_EXIT(db);
 			if (err != 0) {
-				zil_itx_destroy(itx);
+				zil_itx_destroy(itx, 0);
 				itx = zil_itx_create(txtype, sizeof (*lr));
 				lr = (lr_write_t *)&itx->itx_lr;
 				wr_state = WR_NEED_COPY;
diff --git a/sys/contrib/openzfs/module/zfs/zfs_quota.c b/sys/contrib/openzfs/module/zfs/zfs_quota.c
index b8fe512d4f09..2e91ccc27d6d 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_quota.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_quota.c
@@ -374,7 +374,7 @@ zfs_set_userquota(zfsvfs_t *zfsvfs, zfs_userquota_prop_t type,
 	if (*objp == 0) {
 		*objp = zap_create(zfsvfs->z_os, DMU_OT_USERGROUP_QUOTA,
 		    DMU_OT_NONE, 0, tx);
-		VERIFY(0 == zap_add(zfsvfs->z_os, MASTER_NODE_OBJ,
+		VERIFY0(zap_add(zfsvfs->z_os, MASTER_NODE_OBJ,
 		    zfs_userquota_prop_prefixes[type], 8, 1, objp, tx));
 	}
 	mutex_exit(&zfsvfs->z_lock);
@@ -386,7 +386,7 @@ zfs_set_userquota(zfsvfs_t *zfsvfs, zfs_userquota_prop_t type,
 	} else {
 		err = zap_update(zfsvfs->z_os, *objp, buf, 8, 1, &quota, tx);
 	}
-	ASSERT(err == 0);
+	ASSERT0(err);
 	if (fuid_dirtied)
 		zfs_fuid_sync(zfsvfs, tx);
 	dmu_tx_commit(tx);
diff --git a/sys/contrib/openzfs/module/zfs/zfs_rlock.c b/sys/contrib/openzfs/module/zfs/zfs_rlock.c
index 53eb3ef1b66e..4035baff77d6 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_rlock.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_rlock.c
@@ -666,7 +666,7 @@ zfs_rangelock_reduce(zfs_locked_range_t *lr, uint64_t off, uint64_t len)
 
 	/* Ensure there are no other locks */
 	ASSERT3U(avl_numnodes(&rl->rl_tree), ==, 1);
-	ASSERT3U(lr->lr_offset, ==, 0);
+	ASSERT0(lr->lr_offset);
 	ASSERT3U(lr->lr_type, ==, RL_WRITER);
 	ASSERT(!lr->lr_proxy);
 	ASSERT3U(lr->lr_length, ==, UINT64_MAX);
diff --git a/sys/contrib/openzfs/module/zfs/zfs_sa.c b/sys/contrib/openzfs/module/zfs/zfs_sa.c
index 59b6ae4e4203..8b4fc6fd7fbd 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_sa.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_sa.c
@@ -169,7 +169,7 @@ zfs_sa_set_scanstamp(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx)
 	ASSERT(MUTEX_HELD(&zp->z_lock));
 	VERIFY((xoap = xva_getxoptattr(xvap)) != NULL);
 	if (zp->z_is_sa)
-		VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_SCANSTAMP(zfsvfs),
+		VERIFY0(sa_update(zp->z_sa_hdl, SA_ZPL_SCANSTAMP(zfsvfs),
 		    &xoap->xoa_av_scanstamp,
 		    sizeof (xoap->xoa_av_scanstamp), tx));
 	else {
@@ -181,12 +181,12 @@ zfs_sa_set_scanstamp(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx)
 		len = sizeof (xoap->xoa_av_scanstamp) +
 		    ZFS_OLD_ZNODE_PHYS_SIZE;
 		if (len > doi.doi_bonus_size)
-			VERIFY(dmu_set_bonus(db, len, tx) == 0);
+			VERIFY0(dmu_set_bonus(db, len, tx));
 		(void) memcpy((caddr_t)db->db_data + ZFS_OLD_ZNODE_PHYS_SIZE,
 		    xoap->xoa_av_scanstamp, sizeof (xoap->xoa_av_scanstamp));
 
 		zp->z_pflags |= ZFS_BONUS_SCANSTAMP;
-		VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_FLAGS(zfsvfs),
+		VERIFY0(sa_update(zp->z_sa_hdl, SA_ZPL_FLAGS(zfsvfs),
 		    &zp->z_pflags, sizeof (uint64_t), tx));
 	}
 }
@@ -286,7 +286,7 @@ zfs_sa_set_xattr(znode_t *zp, const char *name, const void *value, size_t vsize)
 
 		dmu_tx_commit(tx);
 		if (logsaxattr && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-			zil_commit(zilog, 0);
+			error = zil_commit(zilog, 0);
 	}
 out_free:
 	vmem_free(obj, size);
@@ -427,11 +427,10 @@ zfs_sa_upgrade(sa_handle_t *hdl, dmu_tx_t *tx)
 		zp->z_pflags &= ~ZFS_BONUS_SCANSTAMP;
 	}
 
-	VERIFY(dmu_set_bonustype(db, DMU_OT_SA, tx) == 0);
-	VERIFY(sa_replace_all_by_template_locked(hdl, sa_attrs,
-	    count, tx) == 0);
+	VERIFY0(dmu_set_bonustype(db, DMU_OT_SA, tx));
+	VERIFY0(sa_replace_all_by_template_locked(hdl, sa_attrs, count, tx));
 	if (znode_acl.z_acl_extern_obj)
-		VERIFY(0 == dmu_object_free(zfsvfs->z_os,
+		VERIFY0(dmu_object_free(zfsvfs->z_os,
 		    znode_acl.z_acl_extern_obj, tx));
 
 	zp->z_is_sa = B_TRUE;
diff --git a/sys/contrib/openzfs/module/zfs/zfs_vnops.c b/sys/contrib/openzfs/module/zfs/zfs_vnops.c
index 74aa91a4f2eb..7bb9ba57c69e 100644
--- a/sys/contrib/openzfs/module/zfs/zfs_vnops.c
+++ b/sys/contrib/openzfs/module/zfs/zfs_vnops.c
@@ -27,6 +27,7 @@
  * Copyright 2017 Nexenta Systems, Inc.
  * Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
  * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
+ * Copyright (c) 2025, Klara, Inc.
  */
 
 /* Portions Copyright 2007 Jeremy Teo */
@@ -116,7 +117,7 @@ zfs_fsync(znode_t *zp, int syncflag, cred_t *cr)
 	if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
 		if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
 			return (error);
-		zil_commit(zfsvfs->z_log, zp->z_id);
+		error = zil_commit(zfsvfs->z_log, zp->z_id);
 		zfs_exit(zfsvfs, FTAG);
 	}
 	return (error);
@@ -375,8 +376,13 @@ zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
 	frsync = !!(ioflag & FRSYNC);
 #endif
 	if (zfsvfs->z_log &&
-	    (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS))
-		zil_commit(zfsvfs->z_log, zp->z_id);
+	    (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)) {
+		error = zil_commit(zfsvfs->z_log, zp->z_id);
+		if (error != 0) {
+			zfs_exit(zfsvfs, FTAG);
+			return (error);
+		}
+	}
 
 	/*
 	 * Lock the range against changes.
@@ -1074,8 +1080,13 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
 		return (error);
 	}
 
-	if (commit)
-		zil_commit(zilog, zp->z_id);
+	if (commit) {
+		error = zil_commit(zilog, zp->z_id);
+		if (error != 0) {
+			zfs_exit(zfsvfs, FTAG);
+			return (error);
+		}
+	}
 
 	int64_t nwritten = start_resid - zfs_uio_resid(uio);
 	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
@@ -1260,8 +1271,8 @@ zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
 	zilog = zfsvfs->z_log;
 	error = zfs_setacl(zp, vsecp, skipaclchk, cr);
 
-	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
-		zil_commit(zilog, 0);
+	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
+		error = zil_commit(zilog, 0);
 
 	zfs_exit(zfsvfs, FTAG);
 	return (error);
@@ -1946,7 +1957,7 @@ unlock:
 		ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp);
 
 		if (outos->os_sync == ZFS_SYNC_ALWAYS) {
-			zil_commit(zilog, outzp->z_id);
+			error = zil_commit(zilog, outzp->z_id);
 		}
 
 		*inoffp += done;
diff --git a/sys/contrib/openzfs/module/zfs/zil.c b/sys/contrib/openzfs/module/zfs/zil.c
index 6e4f84257407..0307df55aa21 100644
--- a/sys/contrib/openzfs/module/zfs/zil.c
+++ b/sys/contrib/openzfs/module/zfs/zil.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2014 Integros [integros.com]
  * Copyright (c) 2018 Datto Inc.
+ * Copyright (c) 2025, Klara, Inc.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -103,6 +104,7 @@ static zil_kstat_values_t zil_stats = {
 	{ "zil_commit_error_count",		KSTAT_DATA_UINT64 },
 	{ "zil_commit_stall_count",		KSTAT_DATA_UINT64 },
 	{ "zil_commit_suspend_count",		KSTAT_DATA_UINT64 },
+	{ "zil_commit_crash_count",		KSTAT_DATA_UINT64 },
 	{ "zil_itx_count",			KSTAT_DATA_UINT64 },
 	{ "zil_itx_indirect_count",		KSTAT_DATA_UINT64 },
 	{ "zil_itx_indirect_bytes",		KSTAT_DATA_UINT64 },
@@ -145,7 +147,7 @@ static uint64_t zil_slog_bulk = 64 * 1024 * 1024;
 static kmem_cache_t *zil_lwb_cache;
 static kmem_cache_t *zil_zcw_cache;
 
-static void zil_lwb_commit(zilog_t *zilog, lwb_t *lwb, itx_t *itx);
+static int zil_lwb_commit(zilog_t *zilog, lwb_t *lwb, itx_t *itx);
 static itx_t *zil_itx_clone(itx_t *oitx);
 static uint64_t zil_max_waste_space(zilog_t *zilog);
 
@@ -367,6 +369,7 @@ zil_sums_init(zil_sums_t *zs)
 	wmsum_init(&zs->zil_commit_error_count, 0);
 	wmsum_init(&zs->zil_commit_stall_count, 0);
 	wmsum_init(&zs->zil_commit_suspend_count, 0);
+	wmsum_init(&zs->zil_commit_crash_count, 0);
 	wmsum_init(&zs->zil_itx_count, 0);
 	wmsum_init(&zs->zil_itx_indirect_count, 0);
 	wmsum_init(&zs->zil_itx_indirect_bytes, 0);
@@ -392,6 +395,7 @@ zil_sums_fini(zil_sums_t *zs)
 	wmsum_fini(&zs->zil_commit_error_count);
 	wmsum_fini(&zs->zil_commit_stall_count);
 	wmsum_fini(&zs->zil_commit_suspend_count);
+	wmsum_fini(&zs->zil_commit_crash_count);
 	wmsum_fini(&zs->zil_itx_count);
 	wmsum_fini(&zs->zil_itx_indirect_count);
 	wmsum_fini(&zs->zil_itx_indirect_bytes);
@@ -422,6 +426,8 @@ zil_kstat_values_update(zil_kstat_values_t *zs, zil_sums_t *zil_sums)
 	    wmsum_value(&zil_sums->zil_commit_stall_count);
 	zs->zil_commit_suspend_count.value.ui64 =
 	    wmsum_value(&zil_sums->zil_commit_suspend_count);
+	zs->zil_commit_crash_count.value.ui64 =
+	    wmsum_value(&zil_sums->zil_commit_crash_count);
 	zs->zil_itx_count.value.ui64 =
 	    wmsum_value(&zil_sums->zil_itx_count);
 	zs->zil_itx_indirect_count.value.ui64 =
@@ -813,34 +819,37 @@ zil_lwb_vdev_compare(const void *x1, const void *x2)
  * we choose them here and later make the block allocation match.
  */
 static lwb_t *
-zil_alloc_lwb(zilog_t *zilog, int sz, blkptr_t *bp, boolean_t slog,
-    uint64_t txg, lwb_state_t state)
+zil_alloc_lwb(zilog_t *zilog, blkptr_t *bp, int min_sz, int sz,
+    boolean_t slog, uint64_t txg)
 {
 	lwb_t *lwb;
 
 	lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP);
+	lwb->lwb_flags = 0;
 	lwb->lwb_zilog = zilog;
 	if (bp) {
 		lwb->lwb_blk = *bp;
-		lwb->lwb_slim = (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2);
+		if (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2)
+			lwb->lwb_flags |= LWB_FLAG_SLIM;
 		sz = BP_GET_LSIZE(bp);
+		lwb->lwb_min_sz = sz;
 	} else {
 		BP_ZERO(&lwb->lwb_blk);
-		lwb->lwb_slim = (spa_version(zilog->zl_spa) >=
-		    SPA_VERSION_SLIM_ZIL);
+		if (spa_version(zilog->zl_spa) >= SPA_VERSION_SLIM_ZIL)
+			lwb->lwb_flags |= LWB_FLAG_SLIM;
+		lwb->lwb_min_sz = min_sz;
 	}
-	lwb->lwb_slog = slog;
+	if (slog)
+		lwb->lwb_flags |= LWB_FLAG_SLOG;
 	lwb->lwb_error = 0;
-	if (lwb->lwb_slim) {
-		lwb->lwb_nmax = sz;
-		lwb->lwb_nused = lwb->lwb_nfilled = sizeof (zil_chain_t);
-	} else {
-		lwb->lwb_nmax = sz - sizeof (zil_chain_t);
-		lwb->lwb_nused = lwb->lwb_nfilled = 0;
-	}
+	/*
+	 * Buffer allocation and capacity setup will be done in
+	 * zil_lwb_write_open() when the LWB is opened for ITX assignment.
+	 */
+	lwb->lwb_nmax = lwb->lwb_nused = lwb->lwb_nfilled = 0;
 	lwb->lwb_sz = sz;
-	lwb->lwb_state = state;
-	lwb->lwb_buf = zio_buf_alloc(sz);
+	lwb->lwb_buf = NULL;
+	lwb->lwb_state = LWB_STATE_NEW;
 	lwb->lwb_child_zio = NULL;
 	lwb->lwb_write_zio = NULL;
 	lwb->lwb_root_zio = NULL;
@@ -851,8 +860,6 @@ zil_alloc_lwb(zilog_t *zilog, int sz, blkptr_t *bp, boolean_t slog,
 
 	mutex_enter(&zilog->zl_lock);
 	list_insert_tail(&zilog->zl_lwb_list, lwb);
-	if (state != LWB_STATE_NEW)
-		zilog->zl_last_lwb_opened = lwb;
 	mutex_exit(&zilog->zl_lock);
 
 	return (lwb);
@@ -864,15 +871,15 @@ zil_free_lwb(zilog_t *zilog, lwb_t *lwb)
 	ASSERT(MUTEX_HELD(&zilog->zl_lock));
 	ASSERT(lwb->lwb_state == LWB_STATE_NEW ||
 	    lwb->lwb_state == LWB_STATE_FLUSH_DONE);
-	ASSERT3P(lwb->lwb_child_zio, ==, NULL);
-	ASSERT3P(lwb->lwb_write_zio, ==, NULL);
-	ASSERT3P(lwb->lwb_root_zio, ==, NULL);
+	ASSERT0P(lwb->lwb_child_zio);
+	ASSERT0P(lwb->lwb_write_zio);
+	ASSERT0P(lwb->lwb_root_zio);
 	ASSERT3U(lwb->lwb_alloc_txg, <=, spa_syncing_txg(zilog->zl_spa));
 	ASSERT3U(lwb->lwb_max_txg, <=, spa_syncing_txg(zilog->zl_spa));
 	VERIFY(list_is_empty(&lwb->lwb_itxs));
 	VERIFY(list_is_empty(&lwb->lwb_waiters));
 	ASSERT(avl_is_empty(&lwb->lwb_vdev_tree));
-	ASSERT(!MUTEX_HELD(&lwb->lwb_vdev_lock));
+	ASSERT(!MUTEX_HELD(&lwb->lwb_lock));
 
 	/*
 	 * Clear the zilog's field to indicate this lwb is no longer
@@ -991,8 +998,8 @@ zil_create(zilog_t *zilog)
 	 */
 	txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg);
 
-	ASSERT(zh->zh_claim_txg == 0);
-	ASSERT(zh->zh_replay_seq == 0);
+	ASSERT0(zh->zh_claim_txg);
+	ASSERT0(zh->zh_replay_seq);
 
 	blk = zh->zh_log;
 
@@ -1013,7 +1020,7 @@ zil_create(zilog_t *zilog)
 		}
 
 		error = zio_alloc_zil(zilog->zl_spa, zilog->zl_os, txg, &blk,
-		    ZIL_MIN_BLKSZ, &slog);
+		    ZIL_MIN_BLKSZ, ZIL_MIN_BLKSZ, &slog, B_TRUE);
 		if (error == 0)
 			zil_init_log_chain(zilog, &blk);
 	}
@@ -1022,7 +1029,7 @@ zil_create(zilog_t *zilog)
 	 * Allocate a log write block (lwb) for the first log block.
 	 */
 	if (error == 0)
-		lwb = zil_alloc_lwb(zilog, 0, &blk, slog, txg, LWB_STATE_NEW);
+		lwb = zil_alloc_lwb(zilog, &blk, 0, 0, slog, txg);
 
 	/*
 	 * If we just allocated the first log block, commit our transaction
@@ -1104,7 +1111,7 @@ zil_destroy(zilog_t *zilog, boolean_t keep_first)
 	zilog->zl_keep_first = keep_first;
 
 	if (!list_is_empty(&zilog->zl_lwb_list)) {
-		ASSERT(zh->zh_claim_txg == 0);
+		ASSERT0(zh->zh_claim_txg);
 		VERIFY(!keep_first);
 		while ((lwb = list_remove_head(&zilog->zl_lwb_list)) != NULL) {
 			if (lwb->lwb_buf != NULL)
@@ -1250,7 +1257,7 @@ zil_check_log_chain(dsl_pool_t *dp, dsl_dataset_t *ds, void *tx)
 	blkptr_t *bp;
 	int error;
 
-	ASSERT(tx == NULL);
+	ASSERT0P(tx);
 
 	error = dmu_objset_from_ds(ds, &os);
 	if (error != 0) {
@@ -1318,10 +1325,12 @@ zil_check_log_chain(dsl_pool_t *dp, dsl_dataset_t *ds, void *tx)
  * zil_commit() is racing with spa_sync().
  */
 static void
-zil_commit_waiter_skip(zil_commit_waiter_t *zcw)
+zil_commit_waiter_done(zil_commit_waiter_t *zcw, int err)
 {
 	mutex_enter(&zcw->zcw_lock);
 	ASSERT3B(zcw->zcw_done, ==, B_FALSE);
+	zcw->zcw_lwb = NULL;
+	zcw->zcw_error = err;
 	zcw->zcw_done = B_TRUE;
 	cv_broadcast(&zcw->zcw_cv);
 	mutex_exit(&zcw->zcw_lock);
@@ -1351,7 +1360,7 @@ zil_commit_waiter_link_lwb(zil_commit_waiter_t *zcw, lwb_t *lwb)
 
 	ASSERT(!list_link_active(&zcw->zcw_node));
 	list_insert_tail(&lwb->lwb_waiters, zcw);
-	ASSERT3P(zcw->zcw_lwb, ==, NULL);
+	ASSERT0P(zcw->zcw_lwb);
 	zcw->zcw_lwb = lwb;
 }
 
@@ -1365,7 +1374,7 @@ zil_commit_waiter_link_nolwb(zil_commit_waiter_t *zcw, list_t *nolwb)
 {
 	ASSERT(!list_link_active(&zcw->zcw_node));
 	list_insert_tail(nolwb, zcw);
-	ASSERT3P(zcw->zcw_lwb, ==, NULL);
+	ASSERT0P(zcw->zcw_lwb);
 }
 
 void
@@ -1383,7 +1392,7 @@ zil_lwb_add_block(lwb_t *lwb, const blkptr_t *bp)
 	if (zil_nocacheflush)
 		return;
 
-	mutex_enter(&lwb->lwb_vdev_lock);
+	mutex_enter(&lwb->lwb_lock);
 	for (i = 0; i < ndvas; i++) {
 		zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]);
 		if (avl_find(t, &zvsearch, &where) == NULL) {
@@ -1392,7 +1401,7 @@ zil_lwb_add_block(lwb_t *lwb, const blkptr_t *bp)
 			avl_insert(t, zv, where);
 		}
 	}
-	mutex_exit(&lwb->lwb_vdev_lock);
+	mutex_exit(&lwb->lwb_lock);
 }
 
 static void
@@ -1409,12 +1418,12 @@ zil_lwb_flush_defer(lwb_t *lwb, lwb_t *nlwb)
 
 	/*
 	 * While 'lwb' is at a point in its lifetime where lwb_vdev_tree does
-	 * not need the protection of lwb_vdev_lock (it will only be modified
+	 * not need the protection of lwb_lock (it will only be modified
 	 * while holding zilog->zl_lock) as its writes and those of its
 	 * children have all completed.  The younger 'nlwb' may be waiting on
 	 * future writes to additional vdevs.
 	 */
-	mutex_enter(&nlwb->lwb_vdev_lock);
+	mutex_enter(&nlwb->lwb_lock);
 	/*
 	 * Tear down the 'lwb' vdev tree, ensuring that entries which do not
 	 * exist in 'nlwb' are moved to it, freeing any would-be duplicates.
@@ -1428,7 +1437,7 @@ zil_lwb_flush_defer(lwb_t *lwb, lwb_t *nlwb)
 			kmem_free(zv, sizeof (*zv));
 		}
 	}
-	mutex_exit(&nlwb->lwb_vdev_lock);
+	mutex_exit(&nlwb->lwb_lock);
 }
 
 void
@@ -1482,13 +1491,9 @@ zil_lwb_flush_vdevs_done(zio_t *zio)
 	}
 
 	while ((itx = list_remove_head(&lwb->lwb_itxs)) != NULL)
-		zil_itx_destroy(itx);
+		zil_itx_destroy(itx, 0);
 
 	while ((zcw = list_remove_head(&lwb->lwb_waiters)) != NULL) {
-		mutex_enter(&zcw->zcw_lock);
-
-		ASSERT3P(zcw->zcw_lwb, ==, lwb);
-		zcw->zcw_lwb = NULL;
 		/*
 		 * We expect any ZIO errors from child ZIOs to have been
 		 * propagated "up" to this specific LWB's root ZIO, in
@@ -1503,14 +1508,7 @@ zil_lwb_flush_vdevs_done(zio_t *zio)
 		 * errors not being handled correctly here. See the
 		 * comment above the call to "zio_flush" for details.
 		 */
-
-		zcw->zcw_zio_error = zio->io_error;
-
-		ASSERT3B(zcw->zcw_done, ==, B_FALSE);
-		zcw->zcw_done = B_TRUE;
-		cv_broadcast(&zcw->zcw_cv);
-
-		mutex_exit(&zcw->zcw_lock);
+		zil_commit_waiter_done(zcw, zio->io_error);
 	}
 
 	uint64_t txg = lwb->lwb_issued_txg;
@@ -1582,7 +1580,7 @@ zil_lwb_write_done(zio_t *zio)
 	avl_tree_t *t = &lwb->lwb_vdev_tree;
 	void *cookie = NULL;
 	zil_vdev_node_t *zv;
-	lwb_t *nlwb;
+	lwb_t *nlwb = NULL;
 
 	ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), !=, 0);
 
@@ -1602,9 +1600,11 @@ zil_lwb_write_done(zio_t *zio)
 	 * its write ZIO a parent this ZIO.  In such case we can not defer
 	 * our flushes or below may be a race between the done callbacks.
 	 */
-	nlwb = list_next(&zilog->zl_lwb_list, lwb);
-	if (nlwb && nlwb->lwb_state != LWB_STATE_ISSUED)
-		nlwb = NULL;
+	if (!(lwb->lwb_flags & LWB_FLAG_CRASHED)) {
+		nlwb = list_next(&zilog->zl_lwb_list, lwb);
+		if (nlwb && nlwb->lwb_state != LWB_STATE_ISSUED)
+			nlwb = NULL;
+	}
 	mutex_exit(&zilog->zl_lock);
 
 	if (avl_numnodes(t) == 0)
@@ -1618,12 +1618,17 @@ zil_lwb_write_done(zio_t *zio)
 	 * written out.
 	 *
 	 * Additionally, we don't perform any further error handling at
-	 * this point (e.g. setting "zcw_zio_error" appropriately), as
-	 * we expect that to occur in "zil_lwb_flush_vdevs_done" (thus,
-	 * we expect any error seen here, to have been propagated to
-	 * that function).
+	 * this point (e.g. setting "zcw_error" appropriately), as we
+	 * expect that to occur in "zil_lwb_flush_vdevs_done" (thus, we
+	 * expect any error seen here, to have been propagated to that
+	 * function).
+	 *
+	 * Note that we treat a "crashed" LWB as though it was in error,
+	 * even if it did appear to succeed, because we've already
+	 * signaled error and cleaned up waiters and committers in
+	 * zil_crash(); we just want to clean up and get out of here.
 	 */
-	if (zio->io_error != 0) {
+	if (zio->io_error != 0 || (lwb->lwb_flags & LWB_FLAG_CRASHED)) {
 		while ((zv = avl_destroy_nodes(t, &cookie)) != NULL)
 			kmem_free(zv, sizeof (*zv));
 		return;
@@ -1736,10 +1741,26 @@ zil_lwb_write_open(zilog_t *zilog, lwb_t *lwb)
 		return;
 	}
 
+	mutex_enter(&lwb->lwb_lock);
 	mutex_enter(&zilog->zl_lock);
 	lwb->lwb_state = LWB_STATE_OPENED;
 	zilog->zl_last_lwb_opened = lwb;
 	mutex_exit(&zilog->zl_lock);
+	mutex_exit(&lwb->lwb_lock);
+
+	/*
+	 * Allocate buffer and set up LWB capacities.
+	 */
+	ASSERT0P(lwb->lwb_buf);
+	ASSERT3U(lwb->lwb_sz, >, 0);
+	lwb->lwb_buf = zio_buf_alloc(lwb->lwb_sz);
+	if (lwb->lwb_flags & LWB_FLAG_SLIM) {
+		lwb->lwb_nmax = lwb->lwb_sz;
+		lwb->lwb_nused = lwb->lwb_nfilled = sizeof (zil_chain_t);
+	} else {
+		lwb->lwb_nmax = lwb->lwb_sz - sizeof (zil_chain_t);
+		lwb->lwb_nused = lwb->lwb_nfilled = 0;
+	}
 }
 
 /*
@@ -1756,6 +1777,8 @@ static uint_t
 zil_lwb_plan(zilog_t *zilog, uint64_t size, uint_t *minsize)
 {
 	uint_t md = zilog->zl_max_block_size - sizeof (zil_chain_t);
+	uint_t waste = zil_max_waste_space(zilog);
+	waste = MAX(waste, zilog->zl_cur_max);
 
 	if (size <= md) {
 		/*
@@ -1766,9 +1789,10 @@ zil_lwb_plan(zilog_t *zilog, uint64_t size, uint_t *minsize)
 	} else if (size > 8 * md) {
 		/*
 		 * Big bursts use maximum blocks.  The first block size
-		 * is hard to predict, but it does not really matter.
+		 * is hard to predict, but we need at least enough space
+		 * to make reasonable progress.
 		 */
-		*minsize = 0;
+		*minsize = waste;
 		return (md);
 	}
 
@@ -1781,57 +1805,52 @@ zil_lwb_plan(zilog_t *zilog, uint64_t size, uint_t *minsize)
 	uint_t s = size;
 	uint_t n = DIV_ROUND_UP(s, md - sizeof (lr_write_t));
 	uint_t chunk = DIV_ROUND_UP(s, n);
-	uint_t waste = zil_max_waste_space(zilog);
-	waste = MAX(waste, zilog->zl_cur_max);
 	if (chunk <= md - waste) {
 		*minsize = MAX(s - (md - waste) * (n - 1), waste);
 		return (chunk);
 	} else {
-		*minsize = 0;
+		*minsize = waste;
 		return (md);
 	}
 }
 
 /*
  * Try to predict next block size based on previous history.  Make prediction
- * sufficient for 7 of 8 previous bursts.  Don't try to save if the saving is
- * less then 50%, extra writes may cost more, but we don't want single spike
- * to badly affect our predictions.
+ * sufficient for 7 of 8 previous bursts, but don't try to save if the saving
+ * is less then 50%.  Extra writes may cost more, but we don't want single
+ * spike to badly affect our predictions.
  */
-static uint_t
-zil_lwb_predict(zilog_t *zilog)
+static void
+zil_lwb_predict(zilog_t *zilog, uint64_t *min_predict, uint64_t *max_predict)
 {
-	uint_t m, o;
+	uint_t m1 = 0, m2 = 0, o;
 
-	/* If we are in the middle of a burst, take it into account also. */
-	if (zilog->zl_cur_size > 0) {
-		o = zil_lwb_plan(zilog, zilog->zl_cur_size, &m);
-	} else {
+	/* If we are in the middle of a burst, take it as another data point. */
+	if (zilog->zl_cur_size > 0)
+		o = zil_lwb_plan(zilog, zilog->zl_cur_size, &m1);
+	else
 		o = UINT_MAX;
-		m = 0;
-	}
 
-	/* Find minimum optimal size.  We don't need to go below that. */
-	for (int i = 0; i < ZIL_BURSTS; i++)
-		o = MIN(o, zilog->zl_prev_opt[i]);
-
-	/* Find two biggest minimal first block sizes above the optimal. */
-	uint_t m1 = MAX(m, o), m2 = o;
+	/* Find two largest minimal first block sizes. */
 	for (int i = 0; i < ZIL_BURSTS; i++) {
-		m = zilog->zl_prev_min[i];
-		if (m >= m1) {
+		uint_t cur = zilog->zl_prev_min[i];
+		if (cur >= m1) {
 			m2 = m1;
-			m1 = m;
-		} else if (m > m2) {
-			m2 = m;
+			m1 = cur;
+		} else if (cur > m2) {
+			m2 = cur;
 		}
 	}
 
-	/*
-	 * If second minimum size gives 50% saving -- use it.  It may cost us
-	 * one additional write later, but the space saving is just too big.
-	 */
-	return ((m1 < m2 * 2) ? m1 : m2);
+	/* Minimum should guarantee progress in most cases. */
+	*min_predict = (m1 < m2 * 2) ? m1 : m2;
+
+	/* Maximum doesn't need to go below the minimum optimal size. */
+	for (int i = 0; i < ZIL_BURSTS; i++)
+		o = MIN(o, zilog->zl_prev_opt[i]);
+	m1 = MAX(m1, o);
+	m2 = MAX(m2, o);
+	*max_predict = (m1 < m2 * 2) ? m1 : m2;
 }
 
 /*
@@ -1839,12 +1858,13 @@ zil_lwb_predict(zilog_t *zilog)
  * Has to be called under zl_issuer_lock to chain more lwbs.
  */
 static lwb_t *
-zil_lwb_write_close(zilog_t *zilog, lwb_t *lwb, lwb_state_t state)
+zil_lwb_write_close(zilog_t *zilog, lwb_t *lwb)
 {
-	uint64_t blksz, plan, plan2;
+	uint64_t minbs, maxbs;
 
 	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));
 	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_OPENED);
+	membar_producer();
 	lwb->lwb_state = LWB_STATE_CLOSED;
 
 	/*
@@ -1869,33 +1889,40 @@ zil_lwb_write_close(zilog_t *zilog, lwb_t *lwb, lwb_state_t state)
 		 * Try to predict what can it be and plan for the worst case.
 		 */
 		uint_t m;
-		plan = zil_lwb_plan(zilog, zilog->zl_cur_left, &m);
+		maxbs = zil_lwb_plan(zilog, zilog->zl_cur_left, &m);
+		minbs = m;
 		if (zilog->zl_parallel) {
-			plan2 = zil_lwb_plan(zilog, zilog->zl_cur_left +
-			    zil_lwb_predict(zilog), &m);
-			if (plan < plan2)
-				plan = plan2;
+			uint64_t minp, maxp;
+			zil_lwb_predict(zilog, &minp, &maxp);
+			maxp = zil_lwb_plan(zilog, zilog->zl_cur_left + maxp,
+			    &m);
+			if (maxbs < maxp)
+				maxbs = maxp;
 		}
 	} else {
 		/*
 		 * The previous burst is done and we can only predict what
 		 * will come next.
 		 */
-		plan = zil_lwb_predict(zilog);
+		zil_lwb_predict(zilog, &minbs, &maxbs);
 	}
-	blksz = plan + sizeof (zil_chain_t);
-	blksz = P2ROUNDUP_TYPED(blksz, ZIL_MIN_BLKSZ, uint64_t);
-	blksz = MIN(blksz, zilog->zl_max_block_size);
-	DTRACE_PROBE3(zil__block__size, zilog_t *, zilog, uint64_t, blksz,
-	    uint64_t, plan);
 
-	return (zil_alloc_lwb(zilog, blksz, NULL, 0, 0, state));
+	minbs += sizeof (zil_chain_t);
+	maxbs += sizeof (zil_chain_t);
+	minbs = P2ROUNDUP_TYPED(minbs, ZIL_MIN_BLKSZ, uint64_t);
+	maxbs = P2ROUNDUP_TYPED(maxbs, ZIL_MIN_BLKSZ, uint64_t);
+	maxbs = MIN(maxbs, zilog->zl_max_block_size);
+	minbs = MIN(minbs, maxbs);
+	DTRACE_PROBE3(zil__block__size, zilog_t *, zilog, uint64_t, minbs,
+	    uint64_t, maxbs);
+
+	return (zil_alloc_lwb(zilog, NULL, minbs, maxbs, 0, 0));
 }
 
 /*
  * Finalize previously closed block and issue the write zio.
  */
-static void
+static int
 zil_lwb_write_issue(zilog_t *zilog, lwb_t *lwb)
 {
 	spa_t *spa = zilog->zl_spa;
@@ -1909,8 +1936,13 @@ zil_lwb_write_issue(zilog_t *zilog, lwb_t *lwb)
 
 	/* Actually fill the lwb with the data. */
 	for (itx_t *itx = list_head(&lwb->lwb_itxs); itx;
-	    itx = list_next(&lwb->lwb_itxs, itx))
-		zil_lwb_commit(zilog, lwb, itx);
+	    itx = list_next(&lwb->lwb_itxs, itx)) {
+		error = zil_lwb_commit(zilog, lwb, itx);
+		if (error != 0) {
+			ASSERT3U(error, ==, ESHUTDOWN);
+			return (error);
+		}
+	}
 	lwb->lwb_nused = lwb->lwb_nfilled;
 	ASSERT3U(lwb->lwb_nused, <=, lwb->lwb_nmax);
 
@@ -1928,19 +1960,21 @@ zil_lwb_write_issue(zilog_t *zilog, lwb_t *lwb)
 	lwb->lwb_state = LWB_STATE_READY;
 	if (BP_IS_HOLE(&lwb->lwb_blk) && lwb->lwb_error == 0) {
 		mutex_exit(&zilog->zl_lock);
-		return;
+		return (0);
 	}
 	mutex_exit(&zilog->zl_lock);
 
 next_lwb:
-	if (lwb->lwb_slim)
+	if (lwb->lwb_flags & LWB_FLAG_SLIM)
 		zilc = (zil_chain_t *)lwb->lwb_buf;
 	else
 		zilc = (zil_chain_t *)(lwb->lwb_buf + lwb->lwb_nmax);
-	int wsz = lwb->lwb_sz;
+	uint64_t alloc_size = BP_GET_LSIZE(&lwb->lwb_blk);
+	int wsz = alloc_size;
 	if (lwb->lwb_error == 0) {
 		abd_t *lwb_abd = abd_get_from_buf(lwb->lwb_buf, lwb->lwb_sz);
-		if (!lwb->lwb_slog || zilog->zl_cur_size <= zil_slog_bulk)
+		if (!(lwb->lwb_flags & LWB_FLAG_SLOG) ||
+		    zilog->zl_cur_size <= zil_slog_bulk)
 			prio = ZIO_PRIORITY_SYNC_WRITE;
 		else
 			prio = ZIO_PRIORITY_ASYNC_WRITE;
@@ -1948,16 +1982,17 @@ next_lwb:
 		    ZB_ZIL_OBJECT, ZB_ZIL_LEVEL,
 		    lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]);
 		lwb->lwb_write_zio = zio_rewrite(lwb->lwb_root_zio, spa, 0,
-		    &lwb->lwb_blk, lwb_abd, lwb->lwb_sz, zil_lwb_write_done,
+		    &lwb->lwb_blk, lwb_abd, alloc_size, zil_lwb_write_done,
 		    lwb, prio, ZIO_FLAG_CANFAIL, &zb);
 		zil_lwb_add_block(lwb, &lwb->lwb_blk);
 
-		if (lwb->lwb_slim) {
+		if (lwb->lwb_flags & LWB_FLAG_SLIM) {
 			/* For Slim ZIL only write what is used. */
 			wsz = P2ROUNDUP_TYPED(lwb->lwb_nused, ZIL_MIN_BLKSZ,
 			    int);
-			ASSERT3S(wsz, <=, lwb->lwb_sz);
-			zio_shrink(lwb->lwb_write_zio, wsz);
+			ASSERT3S(wsz, <=, alloc_size);
+			if (wsz < alloc_size)
+				zio_shrink(lwb->lwb_write_zio, wsz);
 			wsz = lwb->lwb_write_zio->io_size;
 		}
 		memset(lwb->lwb_buf + lwb->lwb_nused, 0, wsz - lwb->lwb_nused);
@@ -1993,13 +2028,53 @@ next_lwb:
 	BP_ZERO(bp);
 	error = lwb->lwb_error;
 	if (error == 0) {
-		error = zio_alloc_zil(spa, zilog->zl_os, txg, bp, nlwb->lwb_sz,
-		    &slog);
+		/*
+		 * Allocation flexibility depends on LWB state:
+		 * if NEW: allow range allocation and larger sizes;
+		 * if OPENED: use fixed predetermined allocation size;
+		 * if CLOSED + Slim: allocate precisely for actual usage.
+		 */
+		boolean_t flexible = (nlwb->lwb_state == LWB_STATE_NEW);
+		if (flexible) {
+			/* We need to prevent opening till we update lwb_sz. */
+			mutex_enter(&nlwb->lwb_lock);
+			flexible = (nlwb->lwb_state == LWB_STATE_NEW);
+			if (!flexible)
+				mutex_exit(&nlwb->lwb_lock); /* We lost. */
+		}
+		boolean_t closed_slim = (nlwb->lwb_state == LWB_STATE_CLOSED &&
+		    (lwb->lwb_flags & LWB_FLAG_SLIM));
+
+		uint64_t min_size, max_size;
+		if (closed_slim) {
+			/* This transition is racy, but only one way. */
+			membar_consumer();
+			min_size = max_size = P2ROUNDUP_TYPED(nlwb->lwb_nused,
+			    ZIL_MIN_BLKSZ, uint64_t);
+		} else if (flexible) {
+			min_size = nlwb->lwb_min_sz;
+			max_size = nlwb->lwb_sz;
+		} else {
+			min_size = max_size = nlwb->lwb_sz;
+		}
+
+		error = zio_alloc_zil(spa, zilog->zl_os, txg, bp,
+		    min_size, max_size, &slog, flexible);
+		if (error == 0) {
+			if (closed_slim)
+				ASSERT3U(BP_GET_LSIZE(bp), ==, max_size);
+			else if (flexible)
+				nlwb->lwb_sz = BP_GET_LSIZE(bp);
+			else
+				ASSERT3U(BP_GET_LSIZE(bp), ==, nlwb->lwb_sz);
+		}
+		if (flexible)
+			mutex_exit(&nlwb->lwb_lock);
 	}
 	if (error == 0) {
 		ASSERT3U(BP_GET_BIRTH(bp), ==, txg);
-		BP_SET_CHECKSUM(bp, nlwb->lwb_slim ? ZIO_CHECKSUM_ZILOG2 :
-		    ZIO_CHECKSUM_ZILOG);
+		BP_SET_CHECKSUM(bp, (nlwb->lwb_flags & LWB_FLAG_SLIM) ?
+		    ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG);
 		bp->blk_cksum = lwb->lwb_blk.blk_cksum;
 		bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++;
 	}
@@ -2028,14 +2103,15 @@ next_lwb:
 	if (nlwb) {
 		nlwb->lwb_blk = *bp;
 		nlwb->lwb_error = error;
-		nlwb->lwb_slog = slog;
+		if (slog)
+			nlwb->lwb_flags |= LWB_FLAG_SLOG;
 		nlwb->lwb_alloc_txg = txg;
 		if (nlwb->lwb_state != LWB_STATE_READY)
 			nlwb = NULL;
 	}
 	mutex_exit(&zilog->zl_lock);
 
-	if (lwb->lwb_slog) {
+	if (lwb->lwb_flags & LWB_FLAG_SLOG) {
 		ZIL_STAT_BUMP(zilog, zil_itx_metaslab_slog_count);
 		ZIL_STAT_INCR(zilog, zil_itx_metaslab_slog_bytes,
 		    lwb->lwb_nused);
@@ -2065,6 +2141,8 @@ next_lwb:
 	lwb = nlwb;
 	if (lwb)
 		goto next_lwb;
+
+	return (0);
 }
 
 /*
@@ -2207,7 +2285,6 @@ zil_lwb_assign(zilog_t *zilog, lwb_t *lwb, itx_t *itx, list_t *ilwbs)
 
 	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));
 	ASSERT3P(lwb, !=, NULL);
-	ASSERT3P(lwb->lwb_buf, !=, NULL);
 
 	zil_lwb_write_open(zilog, lwb);
 
@@ -2249,9 +2326,10 @@ cont:
 	    (dlen % max_log_data == 0 ||
 	    lwb_sp < reclen + dlen % max_log_data))) {
 		list_insert_tail(ilwbs, lwb);
-		lwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_OPENED);
+		lwb = zil_lwb_write_close(zilog, lwb);
 		if (lwb == NULL)
 			return (NULL);
+		zil_lwb_write_open(zilog, lwb);
 		lwb_sp = lwb->lwb_nmax - lwb->lwb_nused;
 	}
 
@@ -2308,11 +2386,13 @@ cont:
 	return (lwb);
 }
 
+static void zil_crash(zilog_t *zilog);
+
 /*
  * Fill the actual transaction data into the lwb, following zil_lwb_assign().
  * Does not require locking.
  */
-static void
+static int
 zil_lwb_commit(zilog_t *zilog, lwb_t *lwb, itx_t *itx)
 {
 	lr_t *lr, *lrb;
@@ -2324,7 +2404,7 @@ zil_lwb_commit(zilog_t *zilog, lwb_t *lwb, itx_t *itx)
 	lrw = (lr_write_t *)lr;
 
 	if (lr->lrc_txtype == TX_COMMIT)
-		return;
+		return (0);
 
 	reclen = lr->lrc_reclen;
 	dlen = zil_itx_data_size(itx);
@@ -2410,16 +2490,35 @@ zil_lwb_commit(zilog_t *zilog, lwb_t *lwb, itx_t *itx)
 				    ". Falling back to txg_wait_synced().",
 				    error);
 				zfs_fallthrough;
-			case EIO:
-				txg_wait_synced(zilog->zl_dmu_pool,
-				    lr->lrc_txg);
+			case EIO: {
+				int error = txg_wait_synced_flags(
+				    zilog->zl_dmu_pool,
+				    lr->lrc_txg, TXG_WAIT_SUSPEND);
+				if (error != 0) {
+					ASSERT3U(error, ==, ESHUTDOWN);
+					/*
+					 * zil_lwb_commit() is called from a
+					 * loop over a list of itxs at the
+					 * top of zil_lwb_write_issue(), which
+					 * itself is called from a loop over a
+					 * list of lwbs in various places.
+					 * zil_crash() will free those itxs
+					 * and sometimes the lwbs, so they
+					 * are invalid when zil_crash() returns.
+					 * Callers must pretty much abort
+					 * immediately.
+					 */
+					zil_crash(zilog);
+					return (error);
+				}
 				zfs_fallthrough;
+			}
 			case ENOENT:
 				zfs_fallthrough;
 			case EEXIST:
 				zfs_fallthrough;
 			case EALREADY:
-				return;
+				return (0);
 			}
 		}
 	}
@@ -2427,6 +2526,8 @@ zil_lwb_commit(zilog_t *zilog, lwb_t *lwb, itx_t *itx)
 	lwb->lwb_nfilled += reclen + dlen;
 	ASSERT3S(lwb->lwb_nfilled, <=, lwb->lwb_nused);
 	ASSERT0(P2PHASE(lwb->lwb_nfilled, sizeof (uint64_t)));
+
+	return (0);
 }
 
 itx_t *
@@ -2468,7 +2569,7 @@ zil_itx_clone(itx_t *oitx)
 }
 
 void
-zil_itx_destroy(itx_t *itx)
+zil_itx_destroy(itx_t *itx, int err)
 {
 	ASSERT3U(itx->itx_size, >=, sizeof (itx_t));
 	ASSERT3U(itx->itx_lr.lrc_reclen, ==,
@@ -2477,7 +2578,7 @@ zil_itx_destroy(itx_t *itx)
 	IMPLY(itx->itx_callback != NULL, itx->itx_lr.lrc_txtype != TX_COMMIT);
 
 	if (itx->itx_callback != NULL)
-		itx->itx_callback(itx->itx_callback_data);
+		itx->itx_callback(itx->itx_callback_data, err);
 
 	zio_data_buf_free(itx, itx->itx_size);
 }
@@ -2518,9 +2619,9 @@ zil_itxg_clean(void *arg)
 		 * called) we will hit this case.
 		 */
 		if (itx->itx_lr.lrc_txtype == TX_COMMIT)
-			zil_commit_waiter_skip(itx->itx_private);
+			zil_commit_waiter_done(itx->itx_private, 0);
 
-		zil_itx_destroy(itx);
+		zil_itx_destroy(itx, 0);
 	}
 
 	cookie = NULL;
@@ -2530,7 +2631,7 @@ zil_itxg_clean(void *arg)
 		while ((itx = list_remove_head(list)) != NULL) {
 			/* commit itxs should never be on the async lists. */
 			ASSERT3U(itx->itx_lr.lrc_txtype, !=, TX_COMMIT);
-			zil_itx_destroy(itx);
+			zil_itx_destroy(itx, 0);
 		}
 		list_destroy(list);
 		kmem_free(ian, sizeof (itx_async_node_t));
@@ -2592,7 +2693,7 @@ zil_remove_async(zilog_t *zilog, uint64_t oid)
 	while ((itx = list_remove_head(&clean_list)) != NULL) {
 		/* commit itxs should never be on the async lists. */
 		ASSERT3U(itx->itx_lr.lrc_txtype, !=, TX_COMMIT);
-		zil_itx_destroy(itx);
+		zil_itx_destroy(itx, 0);
 	}
 	list_destroy(&clean_list);
 }
@@ -2677,6 +2778,68 @@ zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx)
 }
 
 /*
+ * Post-crash cleanup. This is called from zil_clean() because it needs to
+ * do cleanup after every txg until the ZIL is restarted, and zilog_dirty()
+ * can arrange that easily, unlike zil_sync() which is more complicated to
+ * get a call to without actual dirty data.
+ */
+static void
+zil_crash_clean(zilog_t *zilog, uint64_t synced_txg)
+{
+	ASSERT(MUTEX_HELD(&zilog->zl_lock));
+	ASSERT3U(zilog->zl_restart_txg, >, 0);
+
+	/* Clean up anything on the crash list from earlier txgs */
+	lwb_t *lwb;
+	while ((lwb = list_head(&zilog->zl_lwb_crash_list)) != NULL) {
+		if (lwb->lwb_alloc_txg >= synced_txg ||
+		    lwb->lwb_max_txg >= synced_txg) {
+			/*
+			 * This lwb was allocated or updated on this txg, or
+			 * in the future. We stop processing here, to avoid
+			 * the strange situation of freeing a ZIL block on
+			 * on the same or earlier txg than what it was
+			 * allocated for.
+			 *
+			 * We'll take care of it on the next txg.
+			 */
+			break;
+		}
+
+		/* This LWB is from the past, so we can clean it up now. */
+		ASSERT(lwb->lwb_flags & LWB_FLAG_CRASHED);
+		list_remove(&zilog->zl_lwb_crash_list, lwb);
+		if (lwb->lwb_buf != NULL)
+			zio_buf_free(lwb->lwb_buf, lwb->lwb_sz);
+		if (!BP_IS_HOLE(&lwb->lwb_blk))
+			/*
+			 * Free on the next txg, since zil_clean() is called
+			 * once synced_txg has already been completed.
+			 */
+			zio_free(zilog->zl_spa, synced_txg+1, &lwb->lwb_blk);
+		zil_free_lwb(zilog, lwb);
+	}
+
+	if (zilog->zl_restart_txg > synced_txg) {
+		/*
+		 * Not reached the restart txg yet, so mark the ZIL dirty for
+		 * the next txg and we'll consider it all again then.
+		 */
+		zilog_dirty(zilog, synced_txg+1);
+		return;
+	}
+
+	/*
+	 * Reached the restart txg, so we can allow new calls to zil_commit().
+	 * All ZIL txgs have long past so there should be no IO waiting.
+	 */
+	ASSERT(list_is_empty(&zilog->zl_lwb_list));
+	ASSERT(list_is_empty(&zilog->zl_lwb_crash_list));
+
+	zilog->zl_restart_txg = 0;
+}
+
+/*
  * If there are any in-memory intent log transactions which have now been
  * synced then start up a taskq to free them. We should only do this after we
  * have written out the uberblocks (i.e. txg has been committed) so that
@@ -2691,6 +2854,15 @@ zil_clean(zilog_t *zilog, uint64_t synced_txg)
 
 	ASSERT3U(synced_txg, <, ZILTEST_TXG);
 
+	/* Do cleanup and restart after crash. */
+	if (zilog->zl_restart_txg > 0) {
+		mutex_enter(&zilog->zl_lock);
+		/* Make sure we didn't lose a race. */
+		if (zilog->zl_restart_txg > 0)
+			zil_crash_clean(zilog, synced_txg);
+		mutex_exit(&zilog->zl_lock);
+	}
+
 	mutex_enter(&itxg->itxg_lock);
 	if (itxg->itxg_itxs == NULL || itxg->itxg_txg == ZILTEST_TXG) {
 		mutex_exit(&itxg->itxg_lock);
@@ -2875,7 +3047,7 @@ zil_prune_commit_list(zilog_t *zilog)
 			 * never any itx's for it to wait on), so it's
 			 * safe to skip this waiter and mark it done.
 			 */
-			zil_commit_waiter_skip(itx->itx_private);
+			zil_commit_waiter_done(itx->itx_private, 0);
 		} else {
 			zil_commit_waiter_link_lwb(itx->itx_private, last_lwb);
 		}
@@ -2883,13 +3055,13 @@ zil_prune_commit_list(zilog_t *zilog)
 		mutex_exit(&zilog->zl_lock);
 
 		list_remove(&zilog->zl_itx_commit_list, itx);
-		zil_itx_destroy(itx);
+		zil_itx_destroy(itx, 0);
 	}
 
 	IMPLY(itx != NULL, itx->itx_lr.lrc_txtype != TX_COMMIT);
 }
 
-static void
+static int
 zil_commit_writer_stall(zilog_t *zilog)
 {
 	/*
@@ -2914,8 +3086,22 @@ zil_commit_writer_stall(zilog_t *zilog)
 	 */
 	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));
 	ZIL_STAT_BUMP(zilog, zil_commit_stall_count);
-	txg_wait_synced(zilog->zl_dmu_pool, 0);
+
+	int err = txg_wait_synced_flags(zilog->zl_dmu_pool, 0,
+	    TXG_WAIT_SUSPEND);
+	if (err != 0) {
+		ASSERT3U(err, ==, ESHUTDOWN);
+		zil_crash(zilog);
+	}
+
+	/*
+	 * Either zil_sync() has been called to wait for and clean up any
+	 * in-flight LWBs, or zil_crash() has emptied out the list and arranged
+	 * for them to be cleaned up later.
+	 */
 	ASSERT(list_is_empty(&zilog->zl_lwb_list));
+
+	return (err);
 }
 
 static void
@@ -3082,7 +3268,7 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs)
 		} else {
 			ASSERT3S(lrc->lrc_txtype, !=, TX_COMMIT);
 			zilog->zl_cur_left -= zil_itx_full_size(itx);
-			zil_itx_destroy(itx);
+			zil_itx_destroy(itx, 0);
 		}
 	}
 
@@ -3092,10 +3278,21 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs)
 		 * "next" lwb on-disk. When this happens, we must stall
 		 * the ZIL write pipeline; see the comment within
 		 * zil_commit_writer_stall() for more details.
+		 *
+		 * ESHUTDOWN has to be handled carefully here. If we get it,
+		 * then the pool suspended and zil_crash() was called, so we
+		 * need to stop trying and just get an error back to the
+		 * callers.
 		 */
-		while ((lwb = list_remove_head(ilwbs)) != NULL)
-			zil_lwb_write_issue(zilog, lwb);
-		zil_commit_writer_stall(zilog);
+		int err = 0;
+		while ((lwb = list_remove_head(ilwbs)) != NULL) {
+			if (err == 0)
+				err = zil_lwb_write_issue(zilog, lwb);
+		}
+		if (err != ESHUTDOWN)
+			err = zil_commit_writer_stall(zilog);
+		if (err == ESHUTDOWN)
+			err = SET_ERROR(EIO);
 
 		/*
 		 * Additionally, we have to signal and mark the "nolwb"
@@ -3105,7 +3302,7 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs)
 		 */
 		zil_commit_waiter_t *zcw;
 		while ((zcw = list_remove_head(&nolwb_waiters)) != NULL)
-			zil_commit_waiter_skip(zcw);
+			zil_commit_waiter_done(zcw, err);
 
 		/*
 		 * And finally, we have to destroy the itx's that
@@ -3113,7 +3310,7 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs)
 		 * the itx's callback if one exists for the itx.
 		 */
 		while ((itx = list_remove_head(&nolwb_itxs)) != NULL)
-			zil_itx_destroy(itx);
+			zil_itx_destroy(itx, err);
 	} else {
 		ASSERT(list_is_empty(&nolwb_waiters));
 		ASSERT3P(lwb, !=, NULL);
@@ -3167,11 +3364,17 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs)
 		    (!zilog->zl_parallel || zilog->zl_suspend > 0)) {
 			zil_burst_done(zilog);
 			list_insert_tail(ilwbs, lwb);
-			lwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_NEW);
+			lwb = zil_lwb_write_close(zilog, lwb);
 			if (lwb == NULL) {
-				while ((lwb = list_remove_head(ilwbs)) != NULL)
-					zil_lwb_write_issue(zilog, lwb);
-				zil_commit_writer_stall(zilog);
+				int err = 0;
+				while ((lwb =
+				    list_remove_head(ilwbs)) != NULL) {
+					if (err == 0)
+						err = zil_lwb_write_issue(
+						    zilog, lwb);
+				}
+				if (err != ESHUTDOWN)
+					(void) zil_commit_writer_stall(zilog);
 			}
 		}
 	}
@@ -3230,10 +3433,23 @@ zil_commit_writer(zilog_t *zilog, zil_commit_waiter_t *zcw)
 	zil_prune_commit_list(zilog);
 	zil_process_commit_list(zilog, zcw, &ilwbs);
 
+	/*
+	 * If the ZIL failed somewhere inside zil_process_commit_list(), it's
+	 * will be because a fallback to txg_wait_sync_flags() happened at some
+	 * point (eg zil_commit_writer_stall()). All cases should issue and
+	 * empty ilwbs, so there will be nothing to in the issue loop below.
+	 * That's why we don't have to plumb the error value back from
+	 * zil_process_commit_list(), and don't have to skip it.
+	 */
+	IMPLY(zilog->zl_restart_txg > 0, list_is_empty(&ilwbs));
+
 out:
 	mutex_exit(&zilog->zl_issuer_lock);
-	while ((lwb = list_remove_head(&ilwbs)) != NULL)
-		zil_lwb_write_issue(zilog, lwb);
+	int err = 0;
+	while ((lwb = list_remove_head(&ilwbs)) != NULL) {
+		if (err == 0)
+			err = zil_lwb_write_issue(zilog, lwb);
+	}
 	list_destroy(&ilwbs);
 	return (wtxg);
 }
@@ -3326,7 +3542,7 @@ zil_commit_waiter_timeout(zilog_t *zilog, zil_commit_waiter_t *zcw)
 	 * hasn't been issued.
 	 */
 	zil_burst_done(zilog);
-	lwb_t *nlwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_NEW);
+	lwb_t *nlwb = zil_lwb_write_close(zilog, lwb);
 
 	ASSERT3S(lwb->lwb_state, ==, LWB_STATE_CLOSED);
 
@@ -3402,7 +3618,7 @@ zil_commit_waiter(zilog_t *zilog, zil_commit_waiter_t *zcw)
 		 * commit itxs. When this occurs, the commit waiters linked
 		 * off of these commit itxs will not be committed to an
 		 * lwb.  Additionally, these commit waiters will not be
-		 * marked done until zil_commit_waiter_skip() is called via
+		 * marked done until zil_commit_waiter_done() is called via
 		 * zil_itxg_clean().
 		 *
 		 * Thus, it's possible for this commit waiter (i.e. the
@@ -3480,7 +3696,7 @@ zil_alloc_commit_waiter(void)
 	list_link_init(&zcw->zcw_node);
 	zcw->zcw_lwb = NULL;
 	zcw->zcw_done = B_FALSE;
-	zcw->zcw_zio_error = 0;
+	zcw->zcw_error = 0;
 
 	return (zcw);
 }
@@ -3489,7 +3705,7 @@ static void
 zil_free_commit_waiter(zil_commit_waiter_t *zcw)
 {
 	ASSERT(!list_link_active(&zcw->zcw_node));
-	ASSERT3P(zcw->zcw_lwb, ==, NULL);
+	ASSERT0P(zcw->zcw_lwb);
 	ASSERT3B(zcw->zcw_done, ==, B_TRUE);
 	mutex_destroy(&zcw->zcw_lock);
 	cv_destroy(&zcw->zcw_cv);
@@ -3526,6 +3742,99 @@ zil_commit_itx_assign(zilog_t *zilog, zil_commit_waiter_t *zcw)
 }
 
 /*
+ * Crash the ZIL. This is something like suspending, but abandons the ZIL
+ * without further IO until the wanted txg completes. No effort is made to
+ * close the on-disk chain or do any other on-disk work, as the pool may
+ * have suspended. zil_sync() will handle cleanup as normal and restart the
+ * ZIL once enough txgs have passed.
+ */
+static void
+zil_crash(zilog_t *zilog)
+{
+	mutex_enter(&zilog->zl_lock);
+
+	uint64_t txg = spa_syncing_txg(zilog->zl_spa);
+	uint64_t restart_txg =
+	    spa_syncing_txg(zilog->zl_spa) + TXG_CONCURRENT_STATES;
+
+	if (zilog->zl_restart_txg > 0) {
+		/*
+		 * If the ZIL is already crashed, it's almost certainly because
+		 * we lost a race involving multiple callers from
+		 * zil_commit_impl().
+		 */
+
+		/*
+		 * This sanity check is to support my understanding that in the
+		 * event of multiple callers to zil_crash(), only one of them
+		 * can possibly be in the codepath to issue lwbs; the rest
+		 * should be calling from zil_commit_impl() after their waiters
+		 * have completed. As I understand it, a second thread trying
+		 * to issue will eventually wait on zl_issuer_lock, and then
+		 * have no work to do and leave.
+		 *
+		 * If more lwbs had been created an issued between zil_crash()
+		 * calls, then we probably just need to take those too, add
+		 * them to the crash list and clean them up, but it complicates
+		 * this function and I don't think it can happend.
+		 */
+		ASSERT(list_is_empty(&zilog->zl_lwb_list));
+
+		mutex_exit(&zilog->zl_lock);
+		return;
+	}
+
+	zilog->zl_restart_txg = restart_txg;
+
+	/*
+	 * Capture any live LWBs. Depending on the state of the pool they may
+	 * represent in-flight IO that won't return for some time, and we want
+	 * to make sure they don't get in the way of normal ZIL operation.
+	 */
+	ASSERT(list_is_empty(&zilog->zl_lwb_crash_list));
+	list_move_tail(&zilog->zl_lwb_crash_list, &zilog->zl_lwb_list);
+
+	/*
+	 * Run through the LWB list; erroring all itxes and signalling error
+	 * to all waiters.
+	 */
+	for (lwb_t *lwb = list_head(&zilog->zl_lwb_crash_list); lwb != NULL;
+	    lwb = list_next(&zilog->zl_lwb_crash_list, lwb)) {
+		ASSERT(!(lwb->lwb_flags & LWB_FLAG_CRASHED));
+		lwb->lwb_flags |= LWB_FLAG_CRASHED;
+
+		itx_t *itx;
+		while ((itx = list_remove_head(&lwb->lwb_itxs)) != NULL)
+			zil_itx_destroy(itx, EIO);
+
+		zil_commit_waiter_t *zcw;
+		while ((zcw = list_remove_head(&lwb->lwb_waiters)) != NULL) {
+			mutex_enter(&zcw->zcw_lock);
+			zcw->zcw_lwb = NULL;
+			zcw->zcw_error = EIO;
+			zcw->zcw_done = B_TRUE;
+			cv_broadcast(&zcw->zcw_cv);
+			mutex_exit(&zcw->zcw_lock);
+		}
+	}
+
+	/*
+	 * Zero the ZIL header bp after the ZIL restarts. We'll free it in
+	 * zil_clean() when we clean up the lwbs.
+	 */
+	zil_header_t *zh = zil_header_in_syncing_context(zilog);
+	BP_ZERO(&zh->zh_log);
+
+	/*
+	 * Mark this ZIL dirty on the next txg, so that zil_clean() will be
+	 * called for cleanup.
+	 */
+	zilog_dirty(zilog, txg+1);
+
+	mutex_exit(&zilog->zl_lock);
+}
+
+/*
  * Commit ZFS Intent Log transactions (itxs) to stable storage.
  *
  * When writing ZIL transactions to the on-disk representation of the
@@ -3640,9 +3949,17 @@ zil_commit_itx_assign(zilog_t *zilog, zil_commit_waiter_t *zcw)
  *      but the order in which they complete will be the same order in
  *      which they were created.
  */
-void
+static int zil_commit_impl(zilog_t *zilog, uint64_t foid);
+
+int
 zil_commit(zilog_t *zilog, uint64_t foid)
 {
+	return (zil_commit_flags(zilog, foid, ZIL_COMMIT_FAILMODE));
+}
+
+int
+zil_commit_flags(zilog_t *zilog, uint64_t foid, zil_commit_flag_t flags)
+{
 	/*
 	 * We should never attempt to call zil_commit on a snapshot for
 	 * a couple of reasons:
@@ -3659,7 +3976,7 @@ zil_commit(zilog_t *zilog, uint64_t foid)
 	ASSERT3B(dmu_objset_is_snapshot(zilog->zl_os), ==, B_FALSE);
 
 	if (zilog->zl_sync == ZFS_SYNC_DISABLED)
-		return;
+		return (0);
 
 	if (!spa_writeable(zilog->zl_spa)) {
 		/*
@@ -3670,10 +3987,23 @@ zil_commit(zilog_t *zilog, uint64_t foid)
 		 * verifying that truth before we return to the caller.
 		 */
 		ASSERT(list_is_empty(&zilog->zl_lwb_list));
-		ASSERT3P(zilog->zl_last_lwb_opened, ==, NULL);
+		ASSERT0P(zilog->zl_last_lwb_opened);
 		for (int i = 0; i < TXG_SIZE; i++)
-			ASSERT3P(zilog->zl_itxg[i].itxg_itxs, ==, NULL);
-		return;
+			ASSERT0P(zilog->zl_itxg[i].itxg_itxs);
+		return (0);
+	}
+
+	int err = 0;
+
+	/*
+	 * If the ZIL crashed, bypass it entirely, and rely on txg_wait_sync()
+	 * to get the data out to disk.
+	 */
+	if (zilog->zl_restart_txg > 0) {
+		ZIL_STAT_BUMP(zilog, zil_commit_crash_count);
+		err = txg_wait_synced_flags(zilog->zl_dmu_pool, 0,
+		    TXG_WAIT_SUSPEND);
+		goto out;
 	}
 
 	/*
@@ -3685,14 +4015,43 @@ zil_commit(zilog_t *zilog, uint64_t foid)
 	 */
 	if (zilog->zl_suspend > 0) {
 		ZIL_STAT_BUMP(zilog, zil_commit_suspend_count);
-		txg_wait_synced(zilog->zl_dmu_pool, 0);
-		return;
+		err = txg_wait_synced_flags(zilog->zl_dmu_pool, 0,
+		    TXG_WAIT_SUSPEND);
+		if (err != 0) {
+			ASSERT3U(err, ==, ESHUTDOWN);
+			zil_crash(zilog);
+		}
+		goto out;
 	}
 
-	zil_commit_impl(zilog, foid);
+	err = zil_commit_impl(zilog, foid);
+
+out:
+	if (err == 0)
+		return (0);
+
+	/*
+	 * The ZIL write failed and the pool is suspended. There's nothing else
+	 * we can do except return or block.
+	 */
+	ASSERT3U(err, ==, ESHUTDOWN);
+
+	/*
+	 * Return error if failmode=continue or caller will handle directly.
+	 */
+	if (!(flags & ZIL_COMMIT_FAILMODE) ||
+	    spa_get_failmode(zilog->zl_spa) == ZIO_FAILURE_MODE_CONTINUE)
+		return (SET_ERROR(EIO));
+
+	/*
+	 * Block until the pool returns. We assume that the data will make
+	 * it out to disk in the end, and so return success.
+	 */
+	txg_wait_synced(zilog->zl_dmu_pool, 0);
+	return (0);
 }
 
-void
+static int
 zil_commit_impl(zilog_t *zilog, uint64_t foid)
 {
 	ZIL_STAT_BUMP(zilog, zil_commit_count);
@@ -3729,7 +4088,8 @@ zil_commit_impl(zilog_t *zilog, uint64_t foid)
 	uint64_t wtxg = zil_commit_writer(zilog, zcw);
 	zil_commit_waiter(zilog, zcw);
 
-	if (zcw->zcw_zio_error != 0) {
+	int err = 0;
+	if (zcw->zcw_error != 0) {
 		/*
 		 * If there was an error writing out the ZIL blocks that
 		 * this thread is waiting on, then we fallback to
@@ -3741,13 +4101,29 @@ zil_commit_impl(zilog_t *zilog, uint64_t foid)
 		ZIL_STAT_BUMP(zilog, zil_commit_error_count);
 		DTRACE_PROBE2(zil__commit__io__error,
 		    zilog_t *, zilog, zil_commit_waiter_t *, zcw);
-		txg_wait_synced(zilog->zl_dmu_pool, 0);
+		err = txg_wait_synced_flags(zilog->zl_dmu_pool, 0,
+		    TXG_WAIT_SUSPEND);
 	} else if (wtxg != 0) {
 		ZIL_STAT_BUMP(zilog, zil_commit_suspend_count);
-		txg_wait_synced(zilog->zl_dmu_pool, wtxg);
+		err = txg_wait_synced_flags(zilog->zl_dmu_pool, wtxg,
+		    TXG_WAIT_SUSPEND);
 	}
 
 	zil_free_commit_waiter(zcw);
+
+	if (err == 0)
+		return (0);
+
+	/*
+	 * ZIL write failed and pool failed in the fallback to
+	 * txg_wait_synced_flags(). Right now we have no idea if the data is on
+	 * disk and the pool is probably suspended so we have no idea when it's
+	 * coming back. All we can do is shut down and return error to the
+	 * caller.
+	 */
+	ASSERT3U(err, ==, ESHUTDOWN);
+	zil_crash(zilog);
+	return (err);
 }
 
 /*
@@ -3773,7 +4149,7 @@ zil_sync(zilog_t *zilog, dmu_tx_t *tx)
 
 	mutex_enter(&zilog->zl_lock);
 
-	ASSERT(zilog->zl_stop_sync == 0);
+	ASSERT0(zilog->zl_stop_sync);
 
 	if (*replayed_seq != 0) {
 		ASSERT(zh->zh_replay_seq < *replayed_seq);
@@ -3848,7 +4224,7 @@ zil_lwb_cons(void *vbuf, void *unused, int kmflag)
 	    offsetof(zil_commit_waiter_t, zcw_node));
 	avl_create(&lwb->lwb_vdev_tree, zil_lwb_vdev_compare,
 	    sizeof (zil_vdev_node_t), offsetof(zil_vdev_node_t, zv_node));
-	mutex_init(&lwb->lwb_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
+	mutex_init(&lwb->lwb_lock, NULL, MUTEX_DEFAULT, NULL);
 	return (0);
 }
 
@@ -3857,7 +4233,7 @@ zil_lwb_dest(void *vbuf, void *unused)
 {
 	(void) unused;
 	lwb_t *lwb = vbuf;
-	mutex_destroy(&lwb->lwb_vdev_lock);
+	mutex_destroy(&lwb->lwb_lock);
 	avl_destroy(&lwb->lwb_vdev_tree);
 	list_destroy(&lwb->lwb_waiters);
 	list_destroy(&lwb->lwb_itxs);
@@ -3943,6 +4319,8 @@ zil_alloc(objset_t *os, zil_header_t *zh_phys)
 
 	list_create(&zilog->zl_lwb_list, sizeof (lwb_t),
 	    offsetof(lwb_t, lwb_node));
+	list_create(&zilog->zl_lwb_crash_list, sizeof (lwb_t),
+	    offsetof(lwb_t, lwb_node));
 
 	list_create(&zilog->zl_itx_commit_list, sizeof (itx_t),
 	    offsetof(itx_t, itx_node));
@@ -3967,9 +4345,12 @@ zil_free(zilog_t *zilog)
 
 	ASSERT0(zilog->zl_suspend);
 	ASSERT0(zilog->zl_suspending);
+	ASSERT0(zilog->zl_restart_txg);
 
 	ASSERT(list_is_empty(&zilog->zl_lwb_list));
 	list_destroy(&zilog->zl_lwb_list);
+	ASSERT(list_is_empty(&zilog->zl_lwb_crash_list));
+	list_destroy(&zilog->zl_lwb_crash_list);
 
 	ASSERT(list_is_empty(&zilog->zl_itx_commit_list));
 	list_destroy(&zilog->zl_itx_commit_list);
@@ -4005,8 +4386,8 @@ zil_open(objset_t *os, zil_get_data_t *get_data, zil_sums_t *zil_sums)
 {
 	zilog_t *zilog = dmu_objset_zil(os);
 
-	ASSERT3P(zilog->zl_get_data, ==, NULL);
-	ASSERT3P(zilog->zl_last_lwb_opened, ==, NULL);
+	ASSERT0P(zilog->zl_get_data);
+	ASSERT0P(zilog->zl_last_lwb_opened);
 	ASSERT(list_is_empty(&zilog->zl_lwb_list));
 
 	zilog->zl_get_data = get_data;
@@ -4025,7 +4406,8 @@ zil_close(zilog_t *zilog)
 	uint64_t txg;
 
 	if (!dmu_objset_is_snapshot(zilog->zl_os)) {
-		zil_commit(zilog, 0);
+		if (zil_commit_flags(zilog, 0, ZIL_COMMIT_NOW) != 0)
+			txg_wait_synced(zilog->zl_dmu_pool, 0);
 	} else {
 		ASSERT(list_is_empty(&zilog->zl_lwb_list));
 		ASSERT0(zilog->zl_dirty_max_txg);
@@ -4074,7 +4456,7 @@ zil_close(zilog_t *zilog)
 	if (lwb != NULL) {
 		ASSERT(list_is_empty(&zilog->zl_lwb_list));
 		ASSERT3S(lwb->lwb_state, ==, LWB_STATE_NEW);
-		zio_buf_free(lwb->lwb_buf, lwb->lwb_sz);
+		ASSERT0P(lwb->lwb_buf);
 		zil_free_lwb(zilog, lwb);
 	}
 	mutex_exit(&zilog->zl_lock);
@@ -4126,6 +4508,17 @@ zil_suspend(const char *osname, void **cookiep)
 		return (SET_ERROR(EBUSY));
 	}
 
+	if (zilog->zl_restart_txg > 0) {
+		/*
+		 * ZIL crashed. It effectively _is_ suspended, but callers
+		 * are usually trying to make sure it's empty on-disk, which
+		 * we can't guarantee right now.
+		 */
+		mutex_exit(&zilog->zl_lock);
+		dmu_objset_rele(os, suspend_tag);
+		return (SET_ERROR(EBUSY));
+	}
+
 	/*
 	 * Don't put a long hold in the cases where we can avoid it.  This
 	 * is when there is no cookie so we are doing a suspend & resume
@@ -4154,11 +4547,16 @@ zil_suspend(const char *osname, void **cookiep)
 			cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock);
 		mutex_exit(&zilog->zl_lock);
 
-		if (cookiep == NULL)
+		if (zilog->zl_restart_txg > 0) {
+			/* ZIL crashed while we were waiting. */
+			zil_resume(os);
+			error = SET_ERROR(EBUSY);
+		} else if (cookiep == NULL)
 			zil_resume(os);
 		else
 			*cookiep = os;
-		return (0);
+
+		return (error);
 	}
 
 	/*
@@ -4199,17 +4597,34 @@ zil_suspend(const char *osname, void **cookiep)
 	 * would just call txg_wait_synced(), because zl_suspend is set.
 	 * txg_wait_synced() doesn't wait for these lwb's to be
 	 * LWB_STATE_FLUSH_DONE before returning.
+	 *
+	 * However, zil_commit_impl() itself can return an error if any of the
+	 * lwbs fail, or the pool suspends in the fallback
+	 * txg_wait_sync_flushed(), which affects what we do next, so we
+	 * capture that error.
 	 */
-	zil_commit_impl(zilog, 0);
+	error = zil_commit_impl(zilog, 0);
+	if (error == ESHUTDOWN)
+		/* zil_commit_impl() has called zil_crash() already */
+		error = SET_ERROR(EBUSY);
 
 	/*
 	 * Now that we've ensured all lwb's are LWB_STATE_FLUSH_DONE, we
 	 * use txg_wait_synced() to ensure the data from the zilog has
 	 * migrated to the main pool before calling zil_destroy().
 	 */
-	txg_wait_synced(zilog->zl_dmu_pool, 0);
+	if (error == 0) {
+		error = txg_wait_synced_flags(zilog->zl_dmu_pool, 0,
+		    TXG_WAIT_SUSPEND);
+		if (error != 0) {
+			ASSERT3U(error, ==, ESHUTDOWN);
+			zil_crash(zilog);
+			error = SET_ERROR(EBUSY);
+		}
+	}
 
-	zil_destroy(zilog, B_FALSE);
+	if (error == 0)
+		zil_destroy(zilog, B_FALSE);
 
 	mutex_enter(&zilog->zl_lock);
 	zilog->zl_suspending = B_FALSE;
@@ -4223,7 +4638,8 @@ zil_suspend(const char *osname, void **cookiep)
 		zil_resume(os);
 	else
 		*cookiep = os;
-	return (0);
+
+	return (error);
 }
 
 void
@@ -4386,7 +4802,7 @@ zil_replay(objset_t *os, void *arg,
 
 	zilog->zl_replay = B_TRUE;
 	zilog->zl_replay_time = ddi_get_lbolt();
-	ASSERT(zilog->zl_replay_blks == 0);
+	ASSERT0(zilog->zl_replay_blks);
 	(void) zil_parse(zilog, zil_incr_blks, zil_replay_log_record, &zr,
 	    zh->zh_claim_txg, B_TRUE);
 	vmem_free(zr.zr_lr, 2 * SPA_MAXBLOCKSIZE);
diff --git a/sys/contrib/openzfs/module/zfs/zio.c b/sys/contrib/openzfs/module/zfs/zio.c
index 218aec6093e2..aeea58bedfe4 100644
--- a/sys/contrib/openzfs/module/zfs/zio.c
+++ b/sys/contrib/openzfs/module/zfs/zio.c
@@ -339,8 +339,8 @@ zio_fini(void)
 	}
 
 	for (size_t i = 0; i < n; i++) {
-		VERIFY3P(zio_buf_cache[i], ==, NULL);
-		VERIFY3P(zio_data_buf_cache[i], ==, NULL);
+		VERIFY0P(zio_buf_cache[i]);
+		VERIFY0P(zio_data_buf_cache[i]);
 	}
 
 	if (zio_ksp != NULL) {
@@ -771,7 +771,7 @@ zio_add_child_impl(zio_t *pio, zio_t *cio, boolean_t first)
 	else
 		mutex_enter(&cio->io_lock);
 
-	ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);
+	ASSERT0(pio->io_state[ZIO_WAIT_DONE]);
 
 	uint64_t *countp = pio->io_children[cio->io_child_type];
 	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
@@ -821,7 +821,7 @@ zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
 	boolean_t waiting = B_FALSE;
 
 	mutex_enter(&zio->io_lock);
-	ASSERT(zio->io_stall == NULL);
+	ASSERT0P(zio->io_stall);
 	for (int c = 0; c < ZIO_CHILD_TYPES; c++) {
 		if (!(ZIO_CHILD_BIT_IS_SET(childbits, c)))
 			continue;
@@ -955,8 +955,8 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
 	zio_t *zio;
 
 	IMPLY(type != ZIO_TYPE_TRIM, psize <= SPA_MAXBLOCKSIZE);
-	ASSERT(P2PHASE(psize, SPA_MINBLOCKSIZE) == 0);
-	ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0);
+	ASSERT0(P2PHASE(psize, SPA_MINBLOCKSIZE));
+	ASSERT0(P2PHASE(offset, SPA_MINBLOCKSIZE));
 
 	ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER));
 	ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER));
@@ -1451,7 +1451,7 @@ zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
 		metaslab_check_free(spa, bp);
 		bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);
 	} else {
-		VERIFY3P(zio_free_sync(NULL, spa, txg, bp, 0), ==, NULL);
+		VERIFY0P(zio_free_sync(NULL, spa, txg, bp, 0));
 	}
 }
 
@@ -1559,7 +1559,7 @@ zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
 {
 	zio_t *zio;
 
-	ASSERT(vd->vdev_children == 0);
+	ASSERT0(vd->vdev_children);
 	ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
 	    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
 	ASSERT3U(offset + size, <=, vd->vdev_psize);
@@ -1580,7 +1580,7 @@ zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
 {
 	zio_t *zio;
 
-	ASSERT(vd->vdev_children == 0);
+	ASSERT0(vd->vdev_children);
 	ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
 	    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
 	ASSERT3U(offset + size, <=, vd->vdev_psize);
@@ -1747,7 +1747,7 @@ zio_flush(zio_t *pio, vdev_t *vd)
 void
 zio_shrink(zio_t *zio, uint64_t size)
 {
-	ASSERT3P(zio->io_executor, ==, NULL);
+	ASSERT0P(zio->io_executor);
 	ASSERT3U(zio->io_orig_size, ==, zio->io_size);
 	ASSERT3U(size, <=, zio->io_size);
 
@@ -1941,7 +1941,7 @@ zio_write_compress(zio_t *zio)
 	}
 
 	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
-	ASSERT(zio->io_bp_override == NULL);
+	ASSERT0P(zio->io_bp_override);
 
 	if (!BP_IS_HOLE(bp) && BP_GET_BIRTH(bp) == zio->io_txg) {
 		/*
@@ -2436,7 +2436,7 @@ __zio_execute(zio_t *zio)
 
 		ASSERT(!MUTEX_HELD(&zio->io_lock));
 		ASSERT(ISP2(stage));
-		ASSERT(zio->io_stall == NULL);
+		ASSERT0P(zio->io_stall);
 
 		do {
 			stage <<= 1;
@@ -2509,7 +2509,7 @@ zio_wait(zio_t *zio)
 	int error;
 
 	ASSERT3S(zio->io_stage, ==, ZIO_STAGE_OPEN);
-	ASSERT3P(zio->io_executor, ==, NULL);
+	ASSERT0P(zio->io_executor);
 
 	zio->io_waiter = curthread;
 	ASSERT0(zio->io_queued_timestamp);
@@ -2551,7 +2551,7 @@ zio_nowait(zio_t *zio)
 	if (zio == NULL)
 		return;
 
-	ASSERT3P(zio->io_executor, ==, NULL);
+	ASSERT0P(zio->io_executor);
 
 	if (zio->io_child_type == ZIO_CHILD_LOGICAL &&
 	    list_is_empty(&zio->io_parent_list)) {
@@ -2590,8 +2590,8 @@ zio_reexecute(void *arg)
 
 	ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL);
 	ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);
-	ASSERT(pio->io_gang_leader == NULL);
-	ASSERT(pio->io_gang_tree == NULL);
+	ASSERT0P(pio->io_gang_leader);
+	ASSERT0P(pio->io_gang_tree);
 
 	mutex_enter(&pio->io_lock);
 	pio->io_flags = pio->io_orig_flags;
@@ -2689,7 +2689,7 @@ zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t reason)
 		ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
 		ASSERT(zio != spa->spa_suspend_zio_root);
 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
-		ASSERT(zio_unique_parent(zio) == NULL);
+		ASSERT0P(zio_unique_parent(zio));
 		ASSERT(zio->io_stage == ZIO_STAGE_DONE);
 		zio_add_child(spa->spa_suspend_zio_root, zio);
 	}
@@ -2908,7 +2908,7 @@ zio_gang_node_alloc(zio_gang_node_t **gnpp, uint64_t gangblocksize)
 {
 	zio_gang_node_t *gn;
 
-	ASSERT(*gnpp == NULL);
+	ASSERT0P(*gnpp);
 
 	gn = kmem_zalloc(sizeof (*gn) +
 	    (gbh_nblkptrs(gangblocksize) * sizeof (gn)), KM_SLEEP);
@@ -2925,7 +2925,7 @@ zio_gang_node_free(zio_gang_node_t **gnpp)
 	zio_gang_node_t *gn = *gnpp;
 
 	for (int g = 0; g < gbh_nblkptrs(gn->gn_allocsize); g++)
-		ASSERT(gn->gn_child[g] == NULL);
+		ASSERT0P(gn->gn_child[g]);
 
 	zio_buf_free(gn->gn_gbh, gn->gn_allocsize);
 	kmem_free(gn, sizeof (*gn) +
@@ -3362,11 +3362,11 @@ zio_nop_write(zio_t *zio)
 	zio_prop_t *zp = &zio->io_prop;
 
 	ASSERT(BP_IS_HOLE(bp));
-	ASSERT(BP_GET_LEVEL(bp) == 0);
+	ASSERT0(BP_GET_LEVEL(bp));
 	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
 	ASSERT(zp->zp_nopwrite);
 	ASSERT(!zp->zp_dedup);
-	ASSERT(zio->io_bp_override == NULL);
+	ASSERT0P(zio->io_bp_override);
 	ASSERT(IO_IS_ALLOCATING(zio));
 
 	/*
@@ -3495,7 +3495,7 @@ zio_ddt_read_start(zio_t *zio)
 		ddt_univ_phys_t *ddp = dde->dde_phys;
 		blkptr_t blk;
 
-		ASSERT(zio->io_vsd == NULL);
+		ASSERT0P(zio->io_vsd);
 		zio->io_vsd = dde;
 
 		if (v_self == DDT_PHYS_NONE)
@@ -3560,7 +3560,7 @@ zio_ddt_read_done(zio_t *zio)
 		zio->io_vsd = NULL;
 	}
 
-	ASSERT(zio->io_vsd == NULL);
+	ASSERT0P(zio->io_vsd);
 
 	return (zio);
 }
@@ -4415,7 +4415,7 @@ static void
 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp)
 {
 	ASSERT(BP_GET_BIRTH(bp) == zio->io_txg || BP_IS_HOLE(bp));
-	ASSERT(zio->io_bp_override == NULL);
+	ASSERT0P(zio->io_bp_override);
 
 	if (!BP_IS_HOLE(bp)) {
 		metaslab_free(zio->io_spa, bp, BP_GET_BIRTH(bp), B_TRUE);
@@ -4434,12 +4434,15 @@ zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp)
  */
 int
 zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
-    uint64_t size, boolean_t *slog)
+    uint64_t min_size, uint64_t max_size, boolean_t *slog,
+    boolean_t allow_larger)
 {
 	int error;
 	zio_alloc_list_t io_alloc_list;
+	uint64_t alloc_size = 0;
 
 	ASSERT(txg > spa_syncing_txg(spa));
+	ASSERT3U(min_size, <=, max_size);
 
 	metaslab_trace_init(&io_alloc_list);
 
@@ -4448,7 +4451,7 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
 	 * Fill in the obvious ones before calling into metaslab_alloc().
 	 */
 	BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
-	BP_SET_PSIZE(new_bp, size);
+	BP_SET_PSIZE(new_bp, max_size);
 	BP_SET_LEVEL(new_bp, 0);
 
 	/*
@@ -4463,43 +4466,51 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
 	ZIOSTAT_BUMP(ziostat_total_allocations);
 
 	/* Try log class (dedicated slog devices) first */
-	error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1,
-	    txg, NULL, flags, &io_alloc_list, allocator, NULL);
+	error = metaslab_alloc_range(spa, spa_log_class(spa), min_size,
+	    max_size, new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
+	    NULL, &alloc_size);
 	*slog = (error == 0);
 
 	/* Try special_embedded_log class (reserved on special vdevs) */
 	if (error != 0) {
-		error = metaslab_alloc(spa, spa_special_embedded_log_class(spa),
-		    size, new_bp, 1, txg, NULL, flags, &io_alloc_list,
-		    allocator, NULL);
+		error = metaslab_alloc_range(spa,
+		    spa_special_embedded_log_class(spa), min_size, max_size,
+		    new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
+		    NULL, &alloc_size);
 	}
 
 	/* Try special class (general special vdev allocation) */
 	if (error != 0) {
-		error = metaslab_alloc(spa, spa_special_class(spa), size,
-		    new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
-		    NULL);
+		error = metaslab_alloc_range(spa, spa_special_class(spa),
+		    min_size, max_size, new_bp, 1, txg, NULL, flags,
+		    &io_alloc_list, allocator, NULL, &alloc_size);
 	}
 
 	/* Try embedded_log class (reserved on normal vdevs) */
 	if (error != 0) {
-		error = metaslab_alloc(spa, spa_embedded_log_class(spa), size,
-		    new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
-		    NULL);
+		error = metaslab_alloc_range(spa, spa_embedded_log_class(spa),
+		    min_size, max_size, new_bp, 1, txg, NULL, flags,
+		    &io_alloc_list, allocator, NULL, &alloc_size);
 	}
 
 	/* Finally fall back to normal class */
 	if (error != 0) {
 		ZIOSTAT_BUMP(ziostat_alloc_class_fallbacks);
-		error = metaslab_alloc(spa, spa_normal_class(spa), size,
-		    new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
-		    NULL);
+		error = metaslab_alloc_range(spa, spa_normal_class(spa),
+		    min_size, max_size, new_bp, 1, txg, NULL, flags,
+		    &io_alloc_list, allocator, NULL, &alloc_size);
 	}
 	metaslab_trace_fini(&io_alloc_list);
 
 	if (error == 0) {
-		BP_SET_LSIZE(new_bp, size);
-		BP_SET_PSIZE(new_bp, size);
+		if (!allow_larger)
+			alloc_size = MIN(alloc_size, max_size);
+		else if (max_size <= SPA_OLD_MAXBLOCKSIZE)
+			alloc_size = MIN(alloc_size, SPA_OLD_MAXBLOCKSIZE);
+		alloc_size = P2ALIGN_TYPED(alloc_size, ZIL_MIN_BLKSZ, uint64_t);
+
+		BP_SET_LSIZE(new_bp, alloc_size);
+		BP_SET_PSIZE(new_bp, alloc_size);
 		BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF);
 		BP_SET_CHECKSUM(new_bp,
 		    spa_version(spa) >= SPA_VERSION_SLIM_ZIL
@@ -4527,8 +4538,8 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
 		}
 	} else {
 		zfs_dbgmsg("%s: zil block allocation failure: "
-		    "size %llu, error %d", spa_name(spa), (u_longlong_t)size,
-		    error);
+		    "min_size %llu, max_size %llu, error %d", spa_name(spa),
+		    (u_longlong_t)min_size, (u_longlong_t)max_size, error);
 	}
 
 	return (error);
@@ -4559,12 +4570,33 @@ zio_vdev_io_start(zio_t *zio)
 
 	zio->io_delay = 0;
 
-	ASSERT(zio->io_error == 0);
-	ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0);
+	ASSERT0(zio->io_error);
+	ASSERT0(zio->io_child_error[ZIO_CHILD_VDEV]);
 
 	if (vd == NULL) {
-		if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
-			spa_config_enter(spa, SCL_ZIO, zio, RW_READER);
+		if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) {
+			/*
+			 * A deadlock workaround. The ddt_prune_unique_entries()
+			 * -> prune_candidates_sync() code path takes the
+			 * SCL_ZIO reader lock and may request it again here.
+			 * If there is another thread who wants the SCL_ZIO
+			 * writer lock, then scl_write_wanted will be set.
+			 * Thus, the spa_config_enter_priority() is used to
+			 * ignore pending writer requests.
+			 *
+			 * The locking should be revised to remove the need
+			 * for this workaround.  If that's not workable then
+			 * it should only be applied to the zios involved in
+			 * the pruning process.  This impacts the read/write
+			 * I/O balance while pruning.
+			 */
+			if (spa->spa_active_ddt_prune)
+				spa_config_enter_priority(spa, SCL_ZIO, zio,
+				    RW_READER);
+			else
+				spa_config_enter(spa, SCL_ZIO, zio,
+				    RW_READER);
+		}
 
 		/*
 		 * The mirror_ops handle multiple DVAs in a single BP.
@@ -4751,7 +4783,7 @@ zio_vdev_io_done(zio_t *zio)
 	ops->vdev_op_io_done(zio);
 
 	if (unexpected_error && vd->vdev_remove_wanted == B_FALSE)
-		VERIFY(vdev_probe(vd, zio) == NULL);
+		VERIFY0P(vdev_probe(vd, zio));
 
 	return (zio);
 }
@@ -4903,7 +4935,7 @@ void
 zio_vdev_io_reissue(zio_t *zio)
 {
 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
-	ASSERT(zio->io_error == 0);
+	ASSERT0(zio->io_error);
 
 	zio->io_stage >>= 1;
 }
@@ -4920,7 +4952,7 @@ void
 zio_vdev_io_bypass(zio_t *zio)
 {
 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
-	ASSERT(zio->io_error == 0);
+	ASSERT0(zio->io_error);
 
 	zio->io_flags |= ZIO_FLAG_IO_BYPASS;
 	zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1;
@@ -5294,11 +5326,21 @@ zio_ready(zio_t *zio)
 		return (NULL);
 	}
 
+	if (zio_injection_enabled) {
+		hrtime_t target = zio_handle_ready_delay(zio);
+		if (target != 0 && zio->io_target_timestamp == 0) {
+			zio->io_stage >>= 1;
+			zio->io_target_timestamp = target;
+			zio_delay_interrupt(zio);
+			return (NULL);
+		}
+	}
+
 	if (zio->io_ready) {
 		ASSERT(IO_IS_ALLOCATING(zio));
 		ASSERT(BP_GET_BIRTH(bp) == zio->io_txg ||
 		    BP_IS_HOLE(bp) || (zio->io_flags & ZIO_FLAG_NOPWRITE));
-		ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);
+		ASSERT0(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY]);
 
 		zio->io_ready(zio);
 	}
@@ -5448,7 +5490,7 @@ zio_done(zio_t *zio)
 
 	for (int c = 0; c < ZIO_CHILD_TYPES; c++)
 		for (int w = 0; w < ZIO_WAIT_TYPES; w++)
-			ASSERT(zio->io_children[c][w] == 0);
+			ASSERT0(zio->io_children[c][w]);
 
 	if (zio->io_bp != NULL && !BP_IS_EMBEDDED(zio->io_bp)) {
 		ASSERT(memcmp(zio->io_bp, &zio->io_bp_copy,
diff --git a/sys/contrib/openzfs/module/zfs/zio_checksum.c b/sys/contrib/openzfs/module/zfs/zio_checksum.c
index 63d0c6dadd46..1d0646a61185 100644
--- a/sys/contrib/openzfs/module/zfs/zio_checksum.c
+++ b/sys/contrib/openzfs/module/zfs/zio_checksum.c
@@ -215,7 +215,7 @@ zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
 spa_feature_t
 zio_checksum_to_feature(enum zio_checksum cksum)
 {
-	VERIFY((cksum & ~ZIO_CHECKSUM_MASK) == 0);
+	VERIFY0((cksum & ~ZIO_CHECKSUM_MASK));
 
 	switch (cksum) {
 	case ZIO_CHECKSUM_BLAKE3:
diff --git a/sys/contrib/openzfs/module/zfs/zio_compress.c b/sys/contrib/openzfs/module/zfs/zio_compress.c
index 9f0ac1b63146..89ceeb58ad91 100644
--- a/sys/contrib/openzfs/module/zfs/zio_compress.c
+++ b/sys/contrib/openzfs/module/zfs/zio_compress.c
@@ -38,12 +38,6 @@
 #include <sys/zstd/zstd.h>
 
 /*
- * If nonzero, every 1/X decompression attempts will fail, simulating
- * an undetected memory error.
- */
-static unsigned long zio_decompress_fail_fraction = 0;
-
-/*
  * Compression vectors.
  */
 zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = {
@@ -171,15 +165,6 @@ zio_decompress_data(enum zio_compress c, abd_t *src, abd_t *dst,
 	else
 		err = ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level);
 
-	/*
-	 * Decompression shouldn't fail, because we've already verified
-	 * the checksum.  However, for extra protection (e.g. against bitflips
-	 * in non-ECC RAM), we handle this error (and test it).
-	 */
-	if (zio_decompress_fail_fraction != 0 &&
-	    random_in_range(zio_decompress_fail_fraction) == 0)
-		err = SET_ERROR(EINVAL);
-
 	return (err);
 }
 
diff --git a/sys/contrib/openzfs/module/zfs/zio_inject.c b/sys/contrib/openzfs/module/zfs/zio_inject.c
index df7b01ba879e..287577018ed1 100644
--- a/sys/contrib/openzfs/module/zfs/zio_inject.c
+++ b/sys/contrib/openzfs/module/zfs/zio_inject.c
@@ -827,6 +827,44 @@ zio_handle_export_delay(spa_t *spa, hrtime_t elapsed)
 	zio_handle_pool_delay(spa, elapsed, ZINJECT_DELAY_EXPORT);
 }
 
+/*
+ * For testing, inject a delay before ready state.
+ */
+hrtime_t
+zio_handle_ready_delay(zio_t *zio)
+{
+	inject_handler_t *handler;
+	hrtime_t now = gethrtime();
+	hrtime_t target = 0;
+
+	/*
+	 * Ignore I/O not associated with any logical data.
+	 */
+	if (zio->io_logical == NULL)
+		return (0);
+
+	rw_enter(&inject_lock, RW_READER);
+
+	for (handler = list_head(&inject_handlers); handler != NULL;
+	    handler = list_next(&inject_handlers, handler)) {
+		if (zio->io_spa != handler->zi_spa ||
+		    handler->zi_record.zi_cmd != ZINJECT_DELAY_READY)
+			continue;
+
+		/* If this handler matches, inject the delay */
+		if (zio_match_iotype(zio, handler->zi_record.zi_iotype) &&
+		    zio_match_handler(&zio->io_logical->io_bookmark,
+		    zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE,
+		    zio_match_dva(zio), &handler->zi_record, zio->io_error)) {
+			target = now + (hrtime_t)handler->zi_record.zi_timer;
+			break;
+		}
+	}
+
+	rw_exit(&inject_lock);
+	return (target);
+}
+
 static int
 zio_calculate_range(const char *pool, zinject_record_t *record)
 {
@@ -1119,7 +1157,7 @@ zio_clear_fault(int id)
 		kmem_free(handler->zi_lanes, sizeof (*handler->zi_lanes) *
 		    handler->zi_record.zi_nlanes);
 	} else {
-		ASSERT3P(handler->zi_lanes, ==, NULL);
+		ASSERT0P(handler->zi_lanes);
 	}
 
 	if (handler->zi_spa_name != NULL)
diff --git a/sys/contrib/openzfs/module/zfs/zrlock.c b/sys/contrib/openzfs/module/zfs/zrlock.c
index 3c0f1b7bbbc1..09c110945c97 100644
--- a/sys/contrib/openzfs/module/zfs/zrlock.c
+++ b/sys/contrib/openzfs/module/zfs/zrlock.c
@@ -129,7 +129,7 @@ zrl_tryenter(zrlock_t *zrl)
 		    (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED);
 		if (cas == 0) {
 #ifdef	ZFS_DEBUG
-			ASSERT3P(zrl->zr_owner, ==, NULL);
+			ASSERT0P(zrl->zr_owner);
 			zrl->zr_owner = curthread;
 #endif
 			return (1);
diff --git a/sys/contrib/openzfs/module/zfs/zthr.c b/sys/contrib/openzfs/module/zfs/zthr.c
index 597a510528ea..d245ce4946e0 100644
--- a/sys/contrib/openzfs/module/zfs/zthr.c
+++ b/sys/contrib/openzfs/module/zfs/zthr.c
@@ -316,7 +316,7 @@ zthr_destroy(zthr_t *t)
 {
 	ASSERT(!MUTEX_HELD(&t->zthr_state_lock));
 	ASSERT(!MUTEX_HELD(&t->zthr_request_lock));
-	VERIFY3P(t->zthr_thread, ==, NULL);
+	VERIFY0P(t->zthr_thread);
 	mutex_destroy(&t->zthr_request_lock);
 	mutex_destroy(&t->zthr_state_lock);
 	cv_destroy(&t->zthr_cv);
diff --git a/sys/contrib/openzfs/module/zfs/zvol.c b/sys/contrib/openzfs/module/zfs/zvol.c
index 7e264f308cf2..00f98168d3d8 100644
--- a/sys/contrib/openzfs/module/zfs/zvol.c
+++ b/sys/contrib/openzfs/module/zfs/zvol.c
@@ -38,25 +38,36 @@
  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
- * Copyright (c) 2024, Klara, Inc.
+ * Copyright (c) 2024, 2025, Klara, Inc.
  */
 
 /*
  * Note on locking of zvol state structures.
  *
- * These structures are used to maintain internal state used to emulate block
- * devices on top of zvols. In particular, management of device minor number
- * operations - create, remove, rename, and set_snapdev - involves access to
- * these structures. The zvol_state_lock is primarily used to protect the
- * zvol_state_list. The zv->zv_state_lock is used to protect the contents
- * of the zvol_state_t structures, as well as to make sure that when the
- * time comes to remove the structure from the list, it is not in use, and
- * therefore, it can be taken off zvol_state_list and freed.
+ * zvol_state_t represents the connection between a single dataset
+ * (DMU_OST_ZVOL) and the device "minor" (some OS-specific representation of a
+ * "disk" or "device" or "volume", eg, a /dev/zdXX node, a GEOM object, etc).
  *
- * The zv_suspend_lock was introduced to allow for suspending I/O to a zvol,
- * e.g. for the duration of receive and rollback operations. This lock can be
- * held for significant periods of time. Given that it is undesirable to hold
- * mutexes for long periods of time, the following lock ordering applies:
+ * The global zvol_state_lock is used to protect access to zvol_state_list and
+ * zvol_htable, which are the primary way to obtain a zvol_state_t from a name.
+ * It should not be used for anything not name-relateds, and you should avoid
+ * sleeping or waiting while its held. See zvol_find_by_name(), zvol_insert(),
+ * zvol_remove().
+ *
+ * The zv_state_lock is used to protect the contents of the associated
+ * zvol_state_t. Most of the zvol_state_t is dedicated to control and
+ * configuration; almost none of it is needed for data operations (that is,
+ * read, write, flush) so this lock is rarely taken during general IO. It
+ * should be released quickly; you should avoid sleeping or waiting while its
+ * held.
+ *
+ * zv_suspend_lock is used to suspend IO/data operations to a zvol. The read
+ * half should held for the duration of an IO operation. The write half should
+ * be taken when something to wait for IO to complete and the block further IO,
+ * eg for the duration of receive and rollback operations. This lock can be
+ * held for long periods of time.
+ *
+ * Thus, the following lock ordering appies.
  * - take zvol_state_lock if necessary, to protect zvol_state_list
  * - take zv_suspend_lock if necessary, by the code path in question
  * - take zv_state_lock to protect zvol_state_t
@@ -67,9 +78,8 @@
  * these operations are serialized per pool. Consequently, we can be certain
  * that for a given zvol, there is only one operation at a time in progress.
  * That is why one can be sure that first, zvol_state_t for a given zvol is
- * allocated and placed on zvol_state_list, and then other minor operations
- * for this zvol are going to proceed in the order of issue.
- *
+ * allocated and placed on zvol_state_list, and then other minor operations for
+ * this zvol are going to proceed in the order of issue.
  */
 
 #include <sys/dataset_kstats.h>
@@ -215,8 +225,8 @@ zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
 	int error;
 	uint64_t volblocksize, volsize;
 
-	VERIFY(nvlist_lookup_uint64(nvprops,
-	    zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize) == 0);
+	VERIFY0(nvlist_lookup_uint64(nvprops,
+	    zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize));
 	if (nvlist_lookup_uint64(nvprops,
 	    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0)
 		volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE);
@@ -225,21 +235,20 @@ zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
 	 * These properties must be removed from the list so the generic
 	 * property setting step won't apply to them.
 	 */
-	VERIFY(nvlist_remove_all(nvprops,
-	    zfs_prop_to_name(ZFS_PROP_VOLSIZE)) == 0);
+	VERIFY0(nvlist_remove_all(nvprops, zfs_prop_to_name(ZFS_PROP_VOLSIZE)));
 	(void) nvlist_remove_all(nvprops,
 	    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE));
 
 	error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize,
 	    DMU_OT_NONE, 0, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP,
 	    DMU_OT_NONE, 0, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 
 	error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx);
-	ASSERT(error == 0);
+	ASSERT0(error);
 }
 
 /*
@@ -254,7 +263,7 @@ zvol_get_stats(objset_t *os, nvlist_t *nv)
 
 	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val);
 	if (error)
-		return (SET_ERROR(error));
+		return (error);
 
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val);
 	doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);
@@ -267,7 +276,7 @@ zvol_get_stats(objset_t *os, nvlist_t *nv)
 
 	kmem_free(doi, sizeof (dmu_object_info_t));
 
-	return (SET_ERROR(error));
+	return (error);
 }
 
 /*
@@ -305,7 +314,7 @@ zvol_update_volsize(uint64_t volsize, objset_t *os)
 	error = dmu_tx_assign(tx, DMU_TX_WAIT);
 	if (error) {
 		dmu_tx_abort(tx);
-		return (SET_ERROR(error));
+		return (error);
 	}
 	txg = dmu_tx_get_txg(tx);
 
@@ -337,7 +346,7 @@ zvol_set_volsize(const char *name, uint64_t volsize)
 	error = dsl_prop_get_integer(name,
 	    zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL);
 	if (error != 0)
-		return (SET_ERROR(error));
+		return (error);
 	if (readonly)
 		return (SET_ERROR(EROFS));
 
@@ -353,7 +362,7 @@ zvol_set_volsize(const char *name, uint64_t volsize)
 		    FTAG, &os)) != 0) {
 			if (zv != NULL)
 				mutex_exit(&zv->zv_state_lock);
-			return (SET_ERROR(error));
+			return (error);
 		}
 		owned = B_TRUE;
 		if (zv != NULL)
@@ -390,7 +399,7 @@ out:
 	if (error == 0 && zv != NULL)
 		zvol_os_update_volsize(zv, volsize);
 
-	return (SET_ERROR(error));
+	return (error);
 }
 
 /*
@@ -401,7 +410,7 @@ zvol_set_volthreading(const char *name, boolean_t value)
 {
 	zvol_state_t *zv = zvol_find_by_name(name, RW_NONE);
 	if (zv == NULL)
-		return (ENOENT);
+		return (-1);
 	zv->zv_threading = value;
 	mutex_exit(&zv->zv_state_lock);
 	return (0);
@@ -450,8 +459,10 @@ zvol_check_volblocksize(const char *name, uint64_t volblocksize)
 		 * We don't allow setting the property above 1MB,
 		 * unless the tunable has been changed.
 		 */
-		if (volblocksize > zfs_max_recordsize)
+		if (volblocksize > zfs_max_recordsize) {
+			spa_close(spa, FTAG);
 			return (SET_ERROR(EDOM));
+		}
 
 		spa_close(spa, FTAG);
 	}
@@ -618,7 +629,7 @@ zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst,
 	dmu_tx_t *tx;
 	blkptr_t *bps;
 	size_t maxblocks;
-	int error = EINVAL;
+	int error = 0;
 
 	rw_enter(&zv_dst->zv_suspend_lock, RW_READER);
 	if (zv_dst->zv_zilog == NULL) {
@@ -644,23 +655,22 @@ zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst,
 	 */
 	if (!spa_feature_is_enabled(dmu_objset_spa(outos),
 	    SPA_FEATURE_BLOCK_CLONING)) {
-		error = EOPNOTSUPP;
+		error = SET_ERROR(EOPNOTSUPP);
 		goto out;
 	}
 	if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
-		error = EXDEV;
+		error = SET_ERROR(EXDEV);
 		goto out;
 	}
 	if (inos->os_encrypted != outos->os_encrypted) {
-		error = EXDEV;
+		error = SET_ERROR(EXDEV);
 		goto out;
 	}
 	if (zv_src->zv_volblocksize != zv_dst->zv_volblocksize) {
-		error = EINVAL;
+		error = SET_ERROR(EINVAL);
 		goto out;
 	}
 	if (inoff >= zv_src->zv_volsize || outoff >= zv_dst->zv_volsize) {
-		error = 0;
 		goto out;
 	}
 
@@ -671,17 +681,15 @@ zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst,
 		len = zv_src->zv_volsize - inoff;
 	if (len > zv_dst->zv_volsize - outoff)
 		len = zv_dst->zv_volsize - outoff;
-	if (len == 0) {
-		error = 0;
+	if (len == 0)
 		goto out;
-	}
 
 	/*
 	 * No overlapping if we are cloning within the same file
 	 */
 	if (zv_src == zv_dst) {
 		if (inoff < outoff + len && outoff < inoff + len) {
-			error = EINVAL;
+			error = SET_ERROR(EINVAL);
 			goto out;
 		}
 	}
@@ -691,7 +699,7 @@ zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst,
 	 */
 	if ((inoff % zv_src->zv_volblocksize) != 0 ||
 	    (outoff % zv_dst->zv_volblocksize) != 0) {
-		error = EINVAL;
+		error = SET_ERROR(EINVAL);
 		goto out;
 	}
 
@@ -699,7 +707,7 @@ zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst,
 	 * Length must be multiple of block size
 	 */
 	if ((len % zv_src->zv_volblocksize) != 0) {
-		error = EINVAL;
+		error = SET_ERROR(EINVAL);
 		goto out;
 	}
 
@@ -771,13 +779,13 @@ zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst,
 	zfs_rangelock_exit(outlr);
 	zfs_rangelock_exit(inlr);
 	if (error == 0 && zv_dst->zv_objset->os_sync == ZFS_SYNC_ALWAYS) {
-		zil_commit(zilog_dst, ZVOL_OBJ);
+		error = zil_commit(zilog_dst, ZVOL_OBJ);
 	}
 out:
 	if (zv_src != zv_dst)
 		rw_exit(&zv_src->zv_suspend_lock);
 	rw_exit(&zv_dst->zv_suspend_lock);
-	return (SET_ERROR(error));
+	return (error);
 }
 
 /*
@@ -897,7 +905,7 @@ zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset,
 		if (wr_state == WR_COPIED &&
 		    dmu_read_by_dnode(zv->zv_dn, offset, len, lr + 1,
 		    DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING) != 0) {
-			zil_itx_destroy(itx);
+			zil_itx_destroy(itx, 0);
 			itx = zil_itx_create(TX_WRITE, sizeof (*lr));
 			lr = (lr_write_t *)&itx->itx_lr;
 			wr_state = WR_NEED_COPY;
@@ -916,7 +924,7 @@ zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset,
 
 		itx->itx_private = zv;
 
-		(void) zil_itx_assign(zilog, itx, tx);
+		zil_itx_assign(zilog, itx, tx);
 
 		offset += len;
 		size -= len;
@@ -1026,7 +1034,7 @@ zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
 
 	zvol_get_done(zgd, error);
 
-	return (SET_ERROR(error));
+	return (error);
 }
 
 /*
@@ -1071,15 +1079,15 @@ zvol_setup_zv(zvol_state_t *zv)
 
 	error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL);
 	if (error)
-		return (SET_ERROR(error));
+		return (error);
 
 	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
 	if (error)
-		return (SET_ERROR(error));
+		return (error);
 
 	error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn);
 	if (error)
-		return (SET_ERROR(error));
+		return (error);
 
 	zvol_os_set_capacity(zv, volsize >> 9);
 	zv->zv_volsize = volsize;
@@ -1121,7 +1129,7 @@ zvol_shutdown_zv(zvol_state_t *zv)
 	 */
 	if (zv->zv_flags & ZVOL_WRITTEN_TO)
 		txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
-	(void) dmu_objset_evict_dbufs(zv->zv_objset);
+	dmu_objset_evict_dbufs(zv->zv_objset);
 }
 
 /*
@@ -1137,20 +1145,34 @@ zvol_tag(zvol_state_t *zv)
 /*
  * Suspend the zvol for recv and rollback.
  */
-zvol_state_t *
-zvol_suspend(const char *name)
+int
+zvol_suspend(const char *name, zvol_state_t **zvp)
 {
 	zvol_state_t *zv;
 
 	zv = zvol_find_by_name(name, RW_WRITER);
 
 	if (zv == NULL)
-		return (NULL);
+		return (SET_ERROR(ENOENT));
 
 	/* block all I/O, release in zvol_resume. */
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 	ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
 
+	/*
+	 * If it's being removed, unlock and return error. It doesn't make any
+	 * sense to try to suspend a zvol being removed, but being here also
+	 * means that zvol_remove_minors_impl() is about to call zvol_remove()
+	 * and then destroy the zvol_state_t, so returning a pointer to it for
+	 * the caller to mess with would be a disaster anyway.
+	 */
+	if (zv->zv_flags & ZVOL_REMOVING) {
+		mutex_exit(&zv->zv_state_lock);
+		rw_exit(&zv->zv_suspend_lock);
+		/* NB: Returning EIO here to match zfsvfs_teardown() */
+		return (SET_ERROR(EIO));
+	}
+
 	atomic_inc(&zv->zv_suspend_ref);
 
 	if (zv->zv_open_count > 0)
@@ -1163,7 +1185,8 @@ zvol_suspend(const char *name)
 	mutex_exit(&zv->zv_state_lock);
 
 	/* zv_suspend_lock is released in zvol_resume() */
-	return (zv);
+	*zvp = zv;
+	return (0);
 }
 
 int
@@ -1198,7 +1221,7 @@ zvol_resume(zvol_state_t *zv)
 	if (zv->zv_flags & ZVOL_REMOVING)
 		cv_broadcast(&zv->zv_removing_cv);
 
-	return (SET_ERROR(error));
+	return (error);
 }
 
 int
@@ -1214,7 +1237,7 @@ zvol_first_open(zvol_state_t *zv, boolean_t readonly)
 	boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL));
 	error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os);
 	if (error)
-		return (SET_ERROR(error));
+		return (error);
 
 	zv->zv_objset = os;
 
@@ -1440,41 +1463,32 @@ zvol_task_update_status(zvol_task_t *task, uint64_t total, uint64_t done,
 	}
 }
 
-static const char *
-zvol_task_op_msg(zvol_async_op_t op)
-{
-	switch (op) {
-	case ZVOL_ASYNC_CREATE_MINORS:
-		return ("create");
-	case ZVOL_ASYNC_REMOVE_MINORS:
-		return ("remove");
-	case ZVOL_ASYNC_RENAME_MINORS:
-		return ("rename");
-	case ZVOL_ASYNC_SET_SNAPDEV:
-	case ZVOL_ASYNC_SET_VOLMODE:
-		return ("set property");
-	default:
-		return ("unknown");
-	}
-
-	__builtin_unreachable();
-	return (NULL);
-}
-
 static void
 zvol_task_report_status(zvol_task_t *task)
 {
+#ifdef ZFS_DEBUG
+	static const char *const msg[] = {
+		"create",
+		"remove",
+		"rename",
+		"set snapdev",
+		"set volmode",
+		"unknown",
+	};
 
 	if (task->zt_status == 0)
 		return;
 
+	zvol_async_op_t op = MIN(task->zt_op, ZVOL_ASYNC_MAX);
 	if (task->zt_error) {
 		dprintf("The %s minors zvol task was not ok, last error %d\n",
-		    zvol_task_op_msg(task->zt_op), task->zt_error);
+		    msg[op], task->zt_error);
 	} else {
-		dprintf("The %s minors zvol task was not ok\n",
-		    zvol_task_op_msg(task->zt_op));
+		dprintf("The %s minors zvol task was not ok\n", msg[op]);
 	}
+#else
+	(void) task;
+#endif
 }
 
 /*
@@ -1581,184 +1595,156 @@ zvol_create_minors_impl(zvol_task_t *task)
 }
 
 /*
- * Remove minors for specified dataset including children and snapshots.
- */
-
-/*
- * Remove the minor for a given zvol. This will do it all:
- *  - flag the zvol for removal, so new requests are rejected
- *  - wait until outstanding requests are completed
- *  - remove it from lists
- *  - free it
- * It's also usable as a taskq task, and smells nice too.
+ * Remove minors for specified dataset and, optionally, its children and
+ * snapshots.
  */
 static void
-zvol_remove_minor_task(void *arg)
-{
-	zvol_state_t *zv = (zvol_state_t *)arg;
-
-	ASSERT(!RW_LOCK_HELD(&zvol_state_lock));
-	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
-
-	mutex_enter(&zv->zv_state_lock);
-	while (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) {
-		zv->zv_flags |= ZVOL_REMOVING;
-		cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock);
-	}
-	mutex_exit(&zv->zv_state_lock);
-
-	rw_enter(&zvol_state_lock, RW_WRITER);
-	mutex_enter(&zv->zv_state_lock);
-
-	zvol_remove(zv);
-	zvol_os_clear_private(zv);
-
-	mutex_exit(&zv->zv_state_lock);
-	rw_exit(&zvol_state_lock);
-
-	zvol_os_free(zv);
-}
-
-static void
-zvol_free_task(void *arg)
-{
-	zvol_os_free(arg);
-}
-
-static void
 zvol_remove_minors_impl(zvol_task_t *task)
 {
 	zvol_state_t *zv, *zv_next;
 	const char *name = task ? task->zt_name1 : NULL;
 	int namelen = ((name) ? strlen(name) : 0);
-	taskqid_t t;
-	list_t delay_list, free_list;
+	boolean_t children = task ? !!task->zt_value : B_TRUE;
 
 	if (zvol_inhibit_dev)
 		return;
 
-	list_create(&delay_list, sizeof (zvol_state_t),
-	    offsetof(zvol_state_t, zv_next));
-	list_create(&free_list, sizeof (zvol_state_t),
-	    offsetof(zvol_state_t, zv_next));
+	/*
+	 * We collect up zvols that we want to remove on a separate list, so
+	 * that we don't have to hold zvol_state_lock for the whole time.
+	 *
+	 * We can't remove them from the global lists until we're completely
+	 * done with them, because that would make them appear to ZFS-side ops
+	 * that they don't exist, and the name might be reused, which can't be
+	 * good.
+	 */
+	list_t remove_list;
+	list_create(&remove_list, sizeof (zvol_state_t),
+	    offsetof(zvol_state_t, zv_remove_node));
 
-	rw_enter(&zvol_state_lock, RW_WRITER);
+	rw_enter(&zvol_state_lock, RW_READER);
 
 	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
 		zv_next = list_next(&zvol_state_list, zv);
 
 		mutex_enter(&zv->zv_state_lock);
+		if (zv->zv_flags & ZVOL_REMOVING) {
+			/* Another thread is handling shutdown, skip it. */
+			mutex_exit(&zv->zv_state_lock);
+			continue;
+		}
+
+		/*
+		 * This zvol should be removed if:
+		 * - no name was offered (ie removing all at shutdown); or
+		 * - name matches exactly; or
+		 * - we were asked to remove children, and
+		 *   - the start of the name matches, and
+		 *   - there is a '/' immediately after the matched name; or
+		 *   - there is a '@' immediately after the matched name
+		 */
 		if (name == NULL || strcmp(zv->zv_name, name) == 0 ||
-		    (strncmp(zv->zv_name, name, namelen) == 0 &&
+		    (children && strncmp(zv->zv_name, name, namelen) == 0 &&
 		    (zv->zv_name[namelen] == '/' ||
 		    zv->zv_name[namelen] == '@'))) {
-			/*
-			 * By holding zv_state_lock here, we guarantee that no
-			 * one is currently using this zv
-			 */
 
 			/*
-			 * If in use, try to throw everyone off and try again
-			 * later.
+			 * Matched, so mark it removal. We want to take the
+			 * write half of the suspend lock to make sure that
+			 * the zvol is not suspended, and give any data ops
+			 * chance to finish.
 			 */
-			if (zv->zv_open_count > 0 ||
-			    atomic_read(&zv->zv_suspend_ref)) {
-				zv->zv_flags |= ZVOL_REMOVING;
-				t = taskq_dispatch(
-				    zv->zv_objset->os_spa->spa_zvol_taskq,
-				    zvol_remove_minor_task, zv, TQ_SLEEP);
-				if (t == TASKQID_INVALID) {
-					/*
-					 * Couldn't create the task, so we'll
-					 * do it in place once the loop is
-					 * finished.
-					 */
-					list_insert_head(&delay_list, zv);
-				}
+			mutex_exit(&zv->zv_state_lock);
+			rw_enter(&zv->zv_suspend_lock, RW_WRITER);
+			mutex_enter(&zv->zv_state_lock);
+
+			if (zv->zv_flags & ZVOL_REMOVING) {
+				/* Another thread has taken it, let them. */
 				mutex_exit(&zv->zv_state_lock);
+				rw_exit(&zv->zv_suspend_lock);
 				continue;
 			}
 
-			zvol_remove(zv);
-
 			/*
-			 * Cleared while holding zvol_state_lock as a writer
-			 * which will prevent zvol_open() from opening it.
+			 * Mark it and unlock. New entries will see the flag
+			 * and return ENXIO.
 			 */
-			zvol_os_clear_private(zv);
-
-			/* Drop zv_state_lock before zvol_free() */
+			zv->zv_flags |= ZVOL_REMOVING;
 			mutex_exit(&zv->zv_state_lock);
+			rw_exit(&zv->zv_suspend_lock);
 
-			/* Try parallel zv_free, if failed do it in place */
-			t = taskq_dispatch(system_taskq, zvol_free_task, zv,
-			    TQ_SLEEP);
-			if (t == TASKQID_INVALID)
-				list_insert_head(&free_list, zv);
-		} else {
+			/* Put it on the list for the next stage. */
+			list_insert_head(&remove_list, zv);
+		} else
 			mutex_exit(&zv->zv_state_lock);
-		}
 	}
-	rw_exit(&zvol_state_lock);
-
-	/* Wait for zvols that we couldn't create a remove task for */
-	while ((zv = list_remove_head(&delay_list)) != NULL)
-		zvol_remove_minor_task(zv);
 
-	/* Free any that we couldn't free in parallel earlier */
-	while ((zv = list_remove_head(&free_list)) != NULL)
-		zvol_os_free(zv);
-}
-
-/* Remove minor for this specific volume only */
-static int
-zvol_remove_minor_impl(const char *name)
-{
-	zvol_state_t *zv = NULL, *zv_next;
-
-	if (zvol_inhibit_dev)
-		return (0);
+	rw_exit(&zvol_state_lock);
 
-	rw_enter(&zvol_state_lock, RW_WRITER);
+	/* Didn't match any, nothing to do! */
+	if (list_is_empty(&remove_list)) {
+		if (task)
+			task->zt_error = SET_ERROR(ENOENT);
+		return;
+	}
 
-	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
-		zv_next = list_next(&zvol_state_list, zv);
+	/* Actually shut them all down. */
+	for (zv = list_head(&remove_list); zv != NULL; zv = zv_next) {
+		zv_next = list_next(&remove_list, zv);
 
 		mutex_enter(&zv->zv_state_lock);
-		if (strcmp(zv->zv_name, name) == 0)
-			/* Found, leave the the loop with zv_lock held */
-			break;
-		mutex_exit(&zv->zv_state_lock);
-	}
-
-	if (zv == NULL) {
-		rw_exit(&zvol_state_lock);
-		return (ENOENT);
-	}
 
-	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
+		/*
+		 * Still open or suspended, just wait. This can happen if, for
+		 * example, we managed to acquire zv_state_lock in the moments
+		 * where zvol_open() or zvol_release() are trading locks to
+		 * call zvol_first_open() or zvol_last_close().
+		 */
+		while (zv->zv_open_count > 0 ||
+		    atomic_read(&zv->zv_suspend_ref))
+			cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock);
 
-	if (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) {
 		/*
-		 * In use, so try to throw everyone off, then wait
-		 * until finished.
+		 * No users, shut down the OS side. This may not remove the
+		 * minor from view immediately, depending on the kernel
+		 * specifics, but it will ensure that it is unusable and that
+		 * this zvol_state_t can never again be reached from an OS-side
+		 * operation.
 		 */
-		zv->zv_flags |= ZVOL_REMOVING;
+		zvol_os_remove_minor(zv);
 		mutex_exit(&zv->zv_state_lock);
+
+		/* Remove it from the name lookup lists */
+		rw_enter(&zvol_state_lock, RW_WRITER);
+		zvol_remove(zv);
 		rw_exit(&zvol_state_lock);
-		zvol_remove_minor_task(zv);
-		return (0);
 	}
 
-	zvol_remove(zv);
-	zvol_os_clear_private(zv);
+	/*
+	 * Our own references on remove_list is the last one, free them and
+	 * we're done.
+	 */
+	while ((zv = list_remove_head(&remove_list)) != NULL)
+		zvol_os_free(zv);
 
-	mutex_exit(&zv->zv_state_lock);
-	rw_exit(&zvol_state_lock);
+	list_destroy(&remove_list);
+}
+
+/* Remove minor for this specific volume only */
+static int
+zvol_remove_minor_impl(const char *name)
+{
+	if (zvol_inhibit_dev)
+		return (0);
 
-	zvol_os_free(zv);
+	zvol_task_t task;
+	memset(&task, 0, sizeof (zvol_task_t));
+	strlcpy(task.zt_name1, name, sizeof (task.zt_name1));
+	task.zt_value = B_FALSE;
 
-	return (0);
+	zvol_remove_minors_impl(&task);
+
+	return (task.zt_error);
 }
 
 /*
@@ -2078,6 +2064,7 @@ zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
 	task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
 	task->zt_op = ZVOL_ASYNC_REMOVE_MINORS;
 	strlcpy(task->zt_name1, name, sizeof (task->zt_name1));
+	task->zt_value = B_TRUE;
 	id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
 	if ((async == B_FALSE) && (id != TASKQID_INVALID))
 		taskq_wait_id(spa->spa_zvol_taskq, id);
@@ -2199,20 +2186,12 @@ zvol_fini_impl(void)
 
 	zvol_remove_minors_impl(NULL);
 
-	/*
-	 * The call to "zvol_remove_minors_impl" may dispatch entries to
-	 * the system_taskq, but it doesn't wait for those entries to
-	 * complete before it returns. Thus, we must wait for all of the
-	 * removals to finish, before we can continue.
-	 */
-	taskq_wait_outstanding(system_taskq, 0);
-
 	kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head));
 	list_destroy(&zvol_state_list);
 	rw_destroy(&zvol_state_lock);
 
 	if (ztqs->tqs_taskq == NULL) {
-		ASSERT3U(ztqs->tqs_cnt, ==, 0);
+		ASSERT0(ztqs->tqs_cnt);
 	} else {
 		for (uint_t i = 0; i < ztqs->tqs_cnt; i++) {
 			ASSERT3P(ztqs->tqs_taskq[i], !=, NULL);
diff --git a/sys/contrib/openzfs/module/zstd/zfs_zstd.c b/sys/contrib/openzfs/module/zstd/zfs_zstd.c
index 391216d6e263..c403c001086a 100644
--- a/sys/contrib/openzfs/module/zstd/zfs_zstd.c
+++ b/sys/contrib/openzfs/module/zstd/zfs_zstd.c
@@ -441,64 +441,6 @@ zstd_enum_to_level(enum zio_zstd_levels level, int16_t *zstd_level)
 }
 
 #ifndef IN_LIBSA
-static size_t
-zfs_zstd_compress_wrap(void *s_start, void *d_start, size_t s_len, size_t d_len,
-    int level)
-{
-	int16_t zstd_level;
-	if (zstd_enum_to_level(level, &zstd_level)) {
-		ZSTDSTAT_BUMP(zstd_stat_com_inval);
-		return (s_len);
-	}
-	/*
-	 * A zstd early abort heuristic.
-	 *
-	 * - Zeroth, if this is <= zstd-3, or < zstd_abort_size (currently
-	 *   128k), don't try any of this, just go.
-	 *   (because experimentally that was a reasonable cutoff for a perf win
-	 *   with tiny ratio change)
-	 * - First, we try LZ4 compression, and if it doesn't early abort, we
-	 *   jump directly to whatever compression level we intended to try.
-	 * - Second, we try zstd-1 - if that errors out (usually, but not
-	 *   exclusively, if it would overflow), we give up early.
-	 *
-	 *   If it works, instead we go on and compress anyway.
-	 *
-	 * Why two passes? LZ4 alone gets you a lot of the way, but on highly
-	 * compressible data, it was losing up to 8.5% of the compressed
-	 * savings versus no early abort, and all the zstd-fast levels are
-	 * worse indications on their own than LZ4, and don't improve the LZ4
-	 * pass noticably if stacked like this.
-	 */
-	size_t actual_abort_size = zstd_abort_size;
-	if (zstd_earlyabort_pass > 0 && zstd_level >= zstd_cutoff_level &&
-	    s_len >= actual_abort_size) {
-		int pass_len = 1;
-		pass_len = zfs_lz4_compress(s_start, d_start, s_len, d_len, 0);
-		if (pass_len < d_len) {
-			ZSTDSTAT_BUMP(zstd_stat_lz4pass_allowed);
-			goto keep_trying;
-		}
-		ZSTDSTAT_BUMP(zstd_stat_lz4pass_rejected);
-
-		pass_len = zfs_zstd_compress(s_start, d_start, s_len, d_len,
-		    ZIO_ZSTD_LEVEL_1);
-		if (pass_len == s_len || pass_len <= 0 || pass_len > d_len) {
-			ZSTDSTAT_BUMP(zstd_stat_zstdpass_rejected);
-			return (s_len);
-		}
-		ZSTDSTAT_BUMP(zstd_stat_zstdpass_allowed);
-	} else {
-		ZSTDSTAT_BUMP(zstd_stat_passignored);
-		if (s_len < actual_abort_size) {
-			ZSTDSTAT_BUMP(zstd_stat_passignored_size);
-		}
-	}
-keep_trying:
-	return (zfs_zstd_compress(s_start, d_start, s_len, d_len, level));
-
-}
-
 /* Compress block using zstd */
 static size_t
 zfs_zstd_compress_impl(void *s_start, void *d_start, size_t s_len, size_t d_len,
@@ -876,9 +818,9 @@ static void __init
 zstd_mempool_init(void)
 {
 	zstd_mempool_cctx =
-	    kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
+	    vmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
 	zstd_mempool_dctx =
-	    kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
+	    vmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
 
 	for (int i = 0; i < ZSTD_POOL_MAX; i++) {
 		mutex_init(&zstd_mempool_cctx[i].barrier, NULL,
@@ -924,8 +866,8 @@ zstd_mempool_deinit(void)
 		release_pool(&zstd_mempool_dctx[i]);
 	}
 
-	kmem_free(zstd_mempool_dctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
-	kmem_free(zstd_mempool_cctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
+	vmem_free(zstd_mempool_dctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
+	vmem_free(zstd_mempool_cctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
 	zstd_mempool_dctx = NULL;
 	zstd_mempool_cctx = NULL;
 }
diff --git a/sys/contrib/openzfs/rpm/generic/zfs.spec.in b/sys/contrib/openzfs/rpm/generic/zfs.spec.in
index dddc0a6c8f02..8986e29eb7fb 100644
--- a/sys/contrib/openzfs/rpm/generic/zfs.spec.in
+++ b/sys/contrib/openzfs/rpm/generic/zfs.spec.in
@@ -433,7 +433,7 @@ make install DESTDIR=%{?buildroot}
 find %{?buildroot}%{_libdir} -name '*.la' -exec rm -f {} \;
 %if 0%{!?__brp_mangle_shebangs:1}
 find %{?buildroot}%{_bindir} \
-    \( -name arc_summary -or -name arcstat -or -name dbufstat \
+    \( -name zarcsummary -or -name zarcstat -or -name dbufstat \
     -or -name zilstat \) \
     -exec %{__sed} -i 's|^#!.*|#!%{__python}|' {} \;
 find %{?buildroot}%{_datadir} \
@@ -506,11 +506,10 @@ systemctl --system daemon-reload >/dev/null || true
 # Core utilities
 %{_sbindir}/*
 %{_bindir}/raidz_test
-%{_sbindir}/zgenhostid
 %{_bindir}/zvol_wait
 # Optional Python 3 scripts
-%{_bindir}/arc_summary
-%{_bindir}/arcstat
+%{_bindir}/zarcsummary
+%{_bindir}/zarcstat
 %{_bindir}/dbufstat
 %{_bindir}/zilstat
 # Man pages
diff --git a/sys/contrib/openzfs/scripts/mancheck.sh b/sys/contrib/openzfs/scripts/mancheck.sh
index 364ad1b76286..33d7d3c7155f 100755
--- a/sys/contrib/openzfs/scripts/mancheck.sh
+++ b/sys/contrib/openzfs/scripts/mancheck.sh
@@ -11,12 +11,12 @@
 # AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
 # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #
-# shellcheck disable=SC2086
+# shellcheck disable=SC2068,SC2086
 
 trap 'rm -f "$stdout_file" "$stderr_file" "$result_file"' EXIT
 
 if [ "$#" -eq 0 ]; then
-    echo "Usage: $0 manpage-directory..."
+    echo "Usage: $0 <manpage-directory|manpage-file>..."
     exit 1
 fi
 
@@ -27,7 +27,16 @@ fi
 
 IFS="
 "
-files="$(find "$@" -type f -name '*[1-9]*' -not -name '.*')" || exit 1
+files="$(
+  for path in $@ ; do
+    find -L $path -type f -name '*[1-9]*' -not -name '.*'
+  done | sort | uniq
+)"
+
+if [ "$files" = "" ] ; then
+    echo no files to process! 1>&2
+    exit 1
+fi
 
 add_excl="$(awk '
     /^.\\" lint-ok:/ {
@@ -48,6 +57,4 @@ grep -vhE -e 'mandoc: outdated mandoc.db' -e 'STYLE: referenced manual not found
 if [ -s "$result_file" ]; then
     cat "$result_file"
     exit 1
-else
-    echo "no errors found"
 fi
diff --git a/sys/contrib/openzfs/scripts/spdxcheck.pl b/sys/contrib/openzfs/scripts/spdxcheck.pl
index 88f5a235d70c..4d4e14368beb 100755
--- a/sys/contrib/openzfs/scripts/spdxcheck.pl
+++ b/sys/contrib/openzfs/scripts/spdxcheck.pl
@@ -83,8 +83,8 @@ my $tagged_patterns = q(
 	man/man?/*.?.in
 
 	# Unsuffixed programs (or generated of same)
-	cmd/arcstat.in
-	cmd/arc_summary
+	cmd/zarcstat.in
+	cmd/zarcsummary
 	cmd/dbufstat.in
 	cmd/zilstat.in
 	cmd/zpool/zpool.d/*
@@ -190,6 +190,7 @@ my @path_license_tags = (
 	    ['BSD-2-Clause OR GPL-2.0-only', 'CDDL-1.0'],
 
 	'module/icp' => ['Apache-2.0', 'CDDL-1.0'],
+	'contrib/icp' => ['Apache-2.0', 'CDDL-1.0'],
 
 	# Python bindings are always Apache-2.0
 	'contrib/pyzfs' => ['Apache-2.0'],
diff --git a/sys/contrib/openzfs/scripts/zfs-helpers.sh b/sys/contrib/openzfs/scripts/zfs-helpers.sh
index b45384a9aa52..2e97d40db1c1 100755
--- a/sys/contrib/openzfs/scripts/zfs-helpers.sh
+++ b/sys/contrib/openzfs/scripts/zfs-helpers.sh
@@ -122,6 +122,13 @@ install() {
 	src=$1
 	dst=$2
 
+	# We may have an old symlink pointing to different ZFS workspace.
+	# Remove the old symlink if it doesn't point to our workspace.
+	if [ -h "$dst" ] && [ "$(readlink -f """$dst""")" != "$src" ] ; then
+		echo "Removing old symlink: $dst -> $(readlink """$dst""")"
+		rm "$dst"
+	fi
+
 	if [ -h "$dst" ]; then
 		echo "Symlink exists: $dst"
 	elif [ -e "$dst" ]; then
diff --git a/sys/contrib/openzfs/scripts/zfs-tests.sh b/sys/contrib/openzfs/scripts/zfs-tests.sh
index 04f3b6f32cb8..09a15bafc27e 100755
--- a/sys/contrib/openzfs/scripts/zfs-tests.sh
+++ b/sys/contrib/openzfs/scripts/zfs-tests.sh
@@ -38,6 +38,7 @@ DEBUG=""
 CLEANUP="yes"
 CLEANUPALL="no"
 KMSG=""
+TIMEOUT_DEBUG=""
 LOOPBACK="yes"
 STACK_TRACER="no"
 FILESIZE="4G"
@@ -364,6 +365,7 @@ OPTIONS:
 	-k          Disable cleanup after test failure
 	-K          Log test names to /dev/kmsg
 	-f          Use files only, disables block device tests
+	-O          Dump debugging info to /dev/kmsg on test timeout
 	-S          Enable stack tracer (negative performance impact)
 	-c          Only create and populate constrained path
 	-R          Automatically rerun failing tests
@@ -402,7 +404,7 @@ $0 -x
 EOF
 }
 
-while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do
+while getopts 'hvqxkKfScRmOn:d:Ds:r:?t:T:u:I:' OPTION; do
 	case $OPTION in
 	h)
 		usage
@@ -445,6 +447,9 @@ while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do
 		export NFS=1
 		. "$nfsfile"
 		;;
+	O)
+		TIMEOUT_DEBUG="yes"
+		;;
 	d)
 		FILEDIR="$OPTARG"
 		;;
@@ -773,6 +778,7 @@ msg "${TEST_RUNNER}" \
     "${DEBUG:+-D}" \
     "${KMEMLEAK:+-m}" \
     "${KMSG:+-K}" \
+    "${TIMEOUT_DEBUG:+-O}" \
     "-c \"${RUNFILES}\"" \
     "-T \"${TAGS}\"" \
     "-i \"${STF_SUITE}\"" \
@@ -783,6 +789,7 @@ msg "${TEST_RUNNER}" \
     ${DEBUG:+-D} \
     ${KMEMLEAK:+-m} \
     ${KMSG:+-K} \
+    ${TIMEOUT_DEBUG:+-O} \
     -c "${RUNFILES}" \
     -T "${TAGS}" \
     -i "${STF_SUITE}" \
@@ -790,6 +797,10 @@ msg "${TEST_RUNNER}" \
     2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 read -r RUNRESULT <"$REPORT_FILE"
 
+if [[ "$RUNRESULT" -eq "255" ]] ; then
+    fail "$TEST_RUNNER failed, test aborted."
+fi 
+
 #
 # Analyze the results.
 #
diff --git a/sys/contrib/openzfs/tests/runfiles/common.run b/sys/contrib/openzfs/tests/runfiles/common.run
index 9fad8946f4f3..9f531411fbe1 100644
--- a/sys/contrib/openzfs/tests/runfiles/common.run
+++ b/sys/contrib/openzfs/tests/runfiles/common.run
@@ -168,10 +168,10 @@ tags = ['functional', 'cli_root', 'zinject']
 tests = ['zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos', 'zdb_005_pos',
     'zdb_006_pos', 'zdb_args_neg', 'zdb_args_pos',
     'zdb_block_size_histogram', 'zdb_checksum', 'zdb_decompress',
-    'zdb_display_block', 'zdb_encrypted', 'zdb_label_checksum',
-    'zdb_object_range_neg', 'zdb_object_range_pos', 'zdb_objset_id',
-    'zdb_decompress_zstd', 'zdb_recover', 'zdb_recover_2', 'zdb_backup',
-    'zdb_tunables']
+    'zdb_display_block', 'zdb_encrypted', 'zdb_encrypted_raw',
+    'zdb_label_checksum', 'zdb_object_range_neg', 'zdb_object_range_pos',
+    'zdb_objset_id', 'zdb_decompress_zstd', 'zdb_recover', 'zdb_recover_2',
+    'zdb_backup', 'zdb_tunables']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zdb']
@@ -323,6 +323,10 @@ tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos',
     'zfs_send_raw', 'zfs_send_sparse', 'zfs_send-b', 'zfs_send_skip_missing']
 tags = ['functional', 'cli_root', 'zfs_send']
 
+[tests/functional/cli_root/zfs_send_delegation]
+tests = ['zfs_send_test']
+tags = ['functional', 'cli_root', 'zfs_send_delegation']
+
 [tests/functional/cli_root/zfs_set]
 tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos',
     'canmount_002_pos', 'canmount_003_pos', 'canmount_004_pos',
@@ -379,7 +383,7 @@ tags = ['functional', 'cli_root', 'zfs_wait']
 
 [tests/functional/cli_root/zhack]
 tests = ['zhack_label_repair_001', 'zhack_label_repair_002',
-    'zhack_label_repair_003', 'zhack_label_repair_004']
+    'zhack_label_repair_003', 'zhack_label_repair_004', 'zhack_metaslab_leak']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zhack']
@@ -391,8 +395,9 @@ tags = ['functional', 'cli_root', 'zpool']
 [tests/functional/cli_root/zpool_add]
 tests = ['zpool_add_001_pos', 'zpool_add_002_pos', 'zpool_add_003_pos',
     'zpool_add_004_pos', 'zpool_add_006_pos', 'zpool_add_007_neg',
-    'zpool_add_008_neg', 'zpool_add_009_neg', 'zpool_add_010_pos',
-    'add-o_ashift', 'add_prop_ashift', 'zpool_add_dryrun_output']
+    'zpool_add_008_neg', 'zpool_add_009_neg', 'zpool_add_warn_create',
+    'zpool_add_warn_degraded', 'zpool_add_warn_removal', 'add-o_ashift',
+    'add_prop_ashift', 'zpool_add_dryrun_output']
 tags = ['functional', 'cli_root', 'zpool_add']
 
 [tests/functional/cli_root/zpool_attach]
@@ -486,6 +491,10 @@ tests = ['zpool_import_001_pos', 'zpool_import_002_pos',
 tags = ['functional', 'cli_root', 'zpool_import']
 timeout = 1200
 
+[tests/functional/cli_root/zpool_iostat]
+tests = ['zpool_iostat_interval_all', 'zpool_iostat_interval_some']
+tags = ['functional', 'cli_root', 'zpool_iostat']
+
 [tests/functional/cli_root/zpool_labelclear]
 tests = ['zpool_labelclear_active', 'zpool_labelclear_exported',
     'zpool_labelclear_removed', 'zpool_labelclear_valid']
@@ -546,7 +555,7 @@ tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos',
     'zpool_scrub_multiple_pools',
     'zpool_error_scrub_001_pos', 'zpool_error_scrub_002_pos',
     'zpool_error_scrub_003_pos', 'zpool_error_scrub_004_pos',
-    'zpool_scrub_date_range_001']
+    'zpool_scrub_date_range_001', 'zpool_scrub_date_range_002']
 tags = ['functional', 'cli_root', 'zpool_scrub']
 
 [tests/functional/cli_root/zpool_set]
@@ -624,8 +633,8 @@ tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg',
     'zpool_history_001_neg', 'zpool_import_001_neg', 'zpool_import_002_neg',
     'zpool_offline_001_neg', 'zpool_online_001_neg', 'zpool_remove_001_neg',
     'zpool_replace_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg',
-    'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'arcstat_001_pos',
-    'arc_summary_001_pos', 'arc_summary_002_neg', 'zpool_wait_privilege',
+    'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'zarcstat_001_pos',
+    'zarcsummary_001_pos', 'zarcsummary_002_neg', 'zpool_wait_privilege',
     'zilstat_001_pos']
 user =
 tags = ['functional', 'cli_user', 'misc']
@@ -637,6 +646,10 @@ tests = ['zfs_list_001_pos', 'zfs_list_002_pos', 'zfs_list_003_pos',
 user =
 tags = ['functional', 'cli_user', 'zfs_list']
 
+[tests/functional/cli_user/zfs_send_delegation_user]
+tests = ['zfs_send_usertest']
+tags = ['functional', 'cli_user', 'zfs_send_delegation_user']
+
 [tests/functional/cli_user/zpool_iostat]
 tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos',
     'zpool_iostat_003_neg', 'zpool_iostat_004_pos',
@@ -725,7 +738,11 @@ tests = ['fadvise_willneed']
 tags = ['functional', 'fadvise']
 
 [tests/functional/failmode]
-tests = ['failmode_dmu_tx_wait', 'failmode_dmu_tx_continue']
+tests = ['failmode_dmu_tx_wait', 'failmode_dmu_tx_continue',
+    'failmode_fsync_wait', 'failmode_fsync_continue',
+    'failmode_msync_wait', 'failmode_msync_continue',
+    'failmode_osync_wait', 'failmode_osync_continue',
+    'failmode_syncalways_wait', 'failmode_syncalways_continue']
 tags = ['functional', 'failmode']
 
 [tests/functional/fallocate]
@@ -936,10 +953,11 @@ tags = ['functional', 'rename_dirs']
 
 [tests/functional/replacement]
 tests = ['attach_import', 'attach_multiple', 'attach_rebuild',
-    'attach_resilver', 'detach', 'rebuild_disabled_feature',
-    'rebuild_multiple', 'rebuild_raidz', 'replace_import', 'replace_rebuild',
-    'replace_resilver', 'resilver_restart_001', 'resilver_restart_002',
-    'scrub_cancel']
+    'attach_resilver', 'attach_resilver_sit_out', 'detach',
+    'rebuild_disabled_feature', 'rebuild_multiple', 'rebuild_raidz',
+    'replace_import', 'replace_rebuild', 'replace_resilver',
+    'replace_resilver_sit_out', 'resilver_restart_001',
+    'resilver_restart_002', 'scrub_cancel']
 tags = ['functional', 'replacement']
 
 [tests/functional/reservation]
@@ -1072,7 +1090,8 @@ tags = ['functional', 'write_dirs']
 [tests/functional/xattr]
 tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos',
     'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg',
-    'xattr_011_pos', 'xattr_012_pos', 'xattr_013_pos', 'xattr_compat']
+    'xattr_011_pos', 'xattr_012_pos', 'xattr_013_pos', 'xattr_014_pos',
+    'xattr_compat']
 tags = ['functional', 'xattr']
 
 [tests/functional/zvol/zvol_ENOSPC]
@@ -1089,7 +1108,7 @@ tests = ['zvol_misc_002_pos', 'zvol_misc_hierarchy', 'zvol_misc_rename_inuse',
 tags = ['functional', 'zvol', 'zvol_misc']
 
 [tests/functional/zvol/zvol_stress]
-tests = ['zvol_stress']
+tests = ['zvol_stress', 'zvol_stress_destroy']
 tags = ['functional', 'zvol', 'zvol_stress']
 
 [tests/functional/zvol/zvol_swap]
diff --git a/sys/contrib/openzfs/tests/runfiles/linux.run b/sys/contrib/openzfs/tests/runfiles/linux.run
index f3d56acffde0..339361cc2762 100644
--- a/sys/contrib/openzfs/tests/runfiles/linux.run
+++ b/sys/contrib/openzfs/tests/runfiles/linux.run
@@ -109,7 +109,9 @@ tags = ['functional', 'direct']
 [tests/functional/events:Linux]
 tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter', 'zed_fd_spill',
     'zed_cksum_reported', 'zed_cksum_config', 'zed_io_config',
-    'zed_slow_io', 'zed_slow_io_many_vdevs', 'zed_diagnose_multiple']
+    'zed_slow_io', 'zed_slow_io_many_vdevs', 'zed_diagnose_multiple',
+    'zed_synchronous_zedlet', 'slow_vdev_sit_out', 'slow_vdev_sit_out_neg',
+    'slow_vdev_degraded_sit_out']
 tags = ['functional', 'events']
 
 [tests/functional/fallocate:Linux]
@@ -161,7 +163,7 @@ tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
 tags = ['functional', 'mmp']
 
 [tests/functional/mount:Linux]
-tests = ['umount_unlinked_drain']
+tests = ['umount_unlinked_drain', 'mount_loopback']
 tags = ['functional', 'mount']
 
 [tests/functional/pam:Linux]
diff --git a/sys/contrib/openzfs/tests/runfiles/sanity.run b/sys/contrib/openzfs/tests/runfiles/sanity.run
index 7767c0c2d535..249b415029c4 100644
--- a/sys/contrib/openzfs/tests/runfiles/sanity.run
+++ b/sys/contrib/openzfs/tests/runfiles/sanity.run
@@ -400,8 +400,8 @@ tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg',
     'zpool_detach_001_neg', 'zpool_export_001_neg', 'zpool_get_001_neg',
     'zpool_history_001_neg', 'zpool_offline_001_neg', 'zpool_online_001_neg',
     'zpool_remove_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg',
-    'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'arcstat_001_pos',
-    'arc_summary_001_pos', 'arc_summary_002_neg', 'zpool_wait_privilege',
+    'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'zarcstat_001_pos',
+    'zarcsummary_001_pos', 'zarcsummary_002_neg', 'zpool_wait_privilege',
     'zilstat_001_pos']
 user =
 tags = ['functional', 'cli_user', 'misc']
@@ -622,7 +622,7 @@ tags = ['functional', 'vdev_zaps']
 [tests/functional/xattr]
 tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos',
     'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg',
-    'xattr_011_pos', 'xattr_013_pos', 'xattr_compat']
+    'xattr_011_pos', 'xattr_013_pos', 'xattr_014_pos', 'xattr_compat']
 tags = ['functional', 'xattr']
 
 [tests/functional/zvol/zvol_ENOSPC]
diff --git a/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in b/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in
index 2158208be6e5..6688b6c4beb6 100755
--- a/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in
+++ b/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in
@@ -25,6 +25,7 @@ import sys
 import ctypes
 import re
 import configparser
+import traceback
 
 from datetime import datetime
 from optparse import OptionParser
@@ -34,6 +35,7 @@ from select import select
 from subprocess import PIPE
 from subprocess import Popen
 from subprocess import check_output
+from subprocess import run
 from threading import Timer
 from time import time, CLOCK_MONOTONIC
 from os.path import exists
@@ -187,6 +189,63 @@ User: %s
 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
 
     def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False):
+
+        """
+        We're about to kill a command due to a timeout.
+        If we're running with the -O option, then dump debug info about the
+        process with the highest CPU usage to /dev/kmsg (Linux only).  This can
+        help debug the timeout.
+
+        Debug info includes:
+        - 30 lines from 'top'
+        - /proc/<PID>/stack output of process with highest CPU usage
+        - Last lines strace-ing process with highest CPU usage
+        """
+        if exists("/dev/kmsg"):
+            c = """
+TOP_OUT="$(COLUMNS=160 top -b -n 1 | head -n 30)"
+read -r PID CMD <<< $(echo "$TOP_OUT" | /usr/bin/awk \
+"/COMMAND/{
+    print_next=1
+    next
+}
+{
+    if (print_next == 1) {
+        print \\$1\\" \\"\\$12
+        exit
+    }
+}")
+echo "##### ZTS timeout debug #####"
+echo "----- top -----"
+echo "$TOP_OUT"
+echo "----- /proc/$PID/stack ($CMD)) -----"
+cat /proc/$PID/stack
+echo "----- strace ($CMD) -----"
+TMPFILE="$(mktemp --suffix=ZTS)"
+/usr/bin/strace -k --stack-traces -p $PID &> "$TMPFILE" &
+sleep 0.1
+killall strace
+tail -n 30 $TMPFILE
+rm "$TMPFILE"
+echo "##### /proc/sysrq-trigger stack #####"
+"""
+            c = "sudo bash -c '" + c + "'"
+            data = run(c, capture_output=True, shell=True, text=True)
+            out = data.stdout
+            try:
+                kp = Popen([SUDO, "sh", "-c",
+                            "echo '" + out + "' > /dev/kmsg"])
+                kp.wait()
+
+                """
+                Trigger kernel stack traces
+                """
+                kp = Popen([SUDO, "sh", "-c",
+                            "echo l > /proc/sysrq-trigger"])
+                kp.wait()
+            except Exception:
+                pass
+
         """
         Kill a running command due to timeout, or ^C from the keyboard. If
         sudo is required, this user was verified previously.
@@ -1080,7 +1139,7 @@ def filter_tests(testrun, options):
     testrun.filter(failed)
 
 
-def fail(retstr, ret=1):
+def fail(retstr, ret=255):
     print('%s: %s' % (sys.argv[0], retstr))
     exit(ret)
 
@@ -1129,6 +1188,9 @@ def parse_args():
     parser.add_option('-o', action='callback', callback=options_cb,
                       default=BASEDIR, dest='outputdir', type='string',
                       metavar='outputdir', help='Specify an output directory.')
+    parser.add_option('-O', action='store_true', default=False,
+                      dest='timeout_debug',
+                      help='Dump debugging info to /dev/kmsg on test timeout')
     parser.add_option('-i', action='callback', callback=options_cb,
                       default=TESTDIR, dest='testdir', type='string',
                       metavar='testdir', help='Specify a test directory.')
@@ -1186,23 +1248,27 @@ def parse_args():
 def main():
     options = parse_args()
 
-    testrun = TestRun(options)
+    try:
+        testrun = TestRun(options)
 
-    if options.runfiles:
-        testrun.read(options)
-    else:
-        find_tests(testrun, options)
+        if options.runfiles:
+            testrun.read(options)
+        else:
+            find_tests(testrun, options)
+
+        if options.logfile:
+            filter_tests(testrun, options)
 
-    if options.logfile:
-        filter_tests(testrun, options)
+        if options.template:
+            testrun.write(options)
+            exit(0)
 
-    if options.template:
-        testrun.write(options)
-        exit(0)
+        testrun.complete_outputdirs()
+        testrun.run(options)
+        exit(testrun.summary())
 
-    testrun.complete_outputdirs()
-    testrun.run(options)
-    exit(testrun.summary())
+    except Exception:
+        fail("Uncaught exception in test runner:\n" + traceback.format_exc())
 
 
 if __name__ == '__main__':
diff --git a/sys/contrib/openzfs/tests/test-runner/bin/zts-report.py.in b/sys/contrib/openzfs/tests/test-runner/bin/zts-report.py.in
index 001970120148..5bc65f993734 100755
--- a/sys/contrib/openzfs/tests/test-runner/bin/zts-report.py.in
+++ b/sys/contrib/openzfs/tests/test-runner/bin/zts-report.py.in
@@ -232,7 +232,7 @@ maybe = {
     'cli_root/zpool_trim/zpool_trim_fault_export_import_online':
         ['FAIL', known_reason],
     'cli_root/zpool_upgrade/zpool_upgrade_004_pos': ['FAIL', 6141],
-    'cli_user/misc/arc_summary_001_pos': ['FAIL', known_reason],
+    'cli_user/misc/zarcsummary_001_pos': ['FAIL', known_reason],
     'delegate/setup': ['SKIP', exec_reason],
     'events/zed_cksum_config': ['FAIL', known_reason],
     'fault/auto_replace_002_pos': ['FAIL', known_reason],
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/.gitignore b/sys/contrib/openzfs/tests/zfs-tests/cmd/.gitignore
index 1cd90024e94d..62f1684acfb4 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/cmd/.gitignore
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/.gitignore
@@ -28,6 +28,7 @@
 /mmap_seek
 /mmap_sync
 /mmapwrite
+/mmap_write_sync
 /nvlist_to_lua
 /randfree_file
 /randwritecomp
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/cmd/Makefile.am
index d5448055a1e1..85c3cf3c35a8 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/cmd/Makefile.am
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/Makefile.am
@@ -74,7 +74,7 @@ scripts_zfs_tests_bin_PROGRAMS += %D%/mkbusy %D%/mkfile %D%/mkfiles %D%/mktree
 
 scripts_zfs_tests_bin_PROGRAMS += \
 	%D%/mmap_exec %D%/mmap_ftruncate %D%/mmap_seek \
-	%D%/mmap_sync %D%/mmapwrite %D%/readmmap
+	%D%/mmap_sync %D%/mmapwrite %D%/readmmap %D%/mmap_write_sync
 %C%_mmapwrite_LDADD = -lpthread
 
 if WANT_MMAP_LIBAIO
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/btree_test.c b/sys/contrib/openzfs/tests/zfs-tests/cmd/btree_test.c
index e7b80d01efaa..f8948a61833d 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/cmd/btree_test.c
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/btree_test.c
@@ -207,7 +207,7 @@ insert_find_remove(zfs_btree_t *bt, char *why)
 		    "Found removed value (%llu)\n", *p);
 		return (1);
 	}
-	ASSERT3S(zfs_btree_numnodes(bt), ==, 0);
+	ASSERT0(zfs_btree_numnodes(bt));
 	zfs_btree_verify(bt);
 
 	return (0);
@@ -279,7 +279,7 @@ drain_tree(zfs_btree_t *bt, char *why)
 		node = avl_last(&avl);
 		ASSERT3U(node->data, ==, *(uint64_t *)zfs_btree_last(bt, NULL));
 	}
-	ASSERT3S(zfs_btree_numnodes(bt), ==, 0);
+	ASSERT0(zfs_btree_numnodes(bt));
 
 	void *avl_cookie = NULL;
 	while ((node = avl_destroy_nodes(&avl, &avl_cookie)) != NULL)
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c b/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c
index e08003f80464..c8d8622c7571 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c
@@ -529,6 +529,8 @@ static const char *aes_gcm_impl[][2] = {
 	{ "aesni",   "pclmulqdq" },
 	{ "x86_64",  "avx" },
 	{ "aesni",   "avx" },
+	{ "x86_64",  "avx2" },
+	{ "aesni",   "avx2" },
 };
 
 /* signature of function to call after setting implementation params */
@@ -861,7 +863,8 @@ test_result(const crypto_test_t *test, int encrypt_rv, uint8_t *encrypt_buf,
 		return (pass);
 
 	/* print summary of test result */
-	printf("%s[%lu]: encrypt=%s decrypt=%s\n", test->fileloc, test->id,
+	printf("%s[%ju]: encrypt=%s decrypt=%s\n", test->fileloc,
+	    (uintmax_t)test->id,
 	    encrypt_pass ? "PASS" : "FAIL",
 	    decrypt_pass ? "PASS" : "FAIL");
 
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/mmap_seek.c b/sys/contrib/openzfs/tests/zfs-tests/cmd/mmap_seek.c
index 45ba17e36c35..f46980cad111 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/cmd/mmap_seek.c
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/mmap_seek.c
@@ -47,25 +47,34 @@
 #endif
 
 static void
+seek_expect(int fd, off_t offset, int whence, off_t expect_offset)
+{
+	errno = 0;
+	off_t seek_offset = lseek(fd, offset, whence);
+	if (seek_offset == expect_offset)
+		return;
+
+	int err = errno;
+	fprintf(stderr, "lseek(fd, %ld, SEEK_%s) = %ld (expected %ld)",
+	    offset, (whence == SEEK_DATA ? "DATA" : "HOLE"),
+	    seek_offset, expect_offset);
+	if (err != 0)
+		fprintf(stderr, " (errno %d [%s])\n", err, strerror(err));
+	else
+		fputc('\n', stderr);
+	exit(2);
+}
+
+static inline void
 seek_data(int fd, off_t offset, off_t expected)
 {
-	off_t data_offset = lseek(fd, offset, SEEK_DATA);
-	if (data_offset != expected) {
-		fprintf(stderr, "lseek(fd, %d, SEEK_DATA) = %d (expected %d)\n",
-		    (int)offset, (int)data_offset, (int)expected);
-		exit(2);
-	}
+	seek_expect(fd, offset, SEEK_DATA, expected);
 }
 
-static void
+static inline void
 seek_hole(int fd, off_t offset, off_t expected)
 {
-	off_t hole_offset = lseek(fd, offset, SEEK_HOLE);
-	if (hole_offset != expected) {
-		fprintf(stderr, "lseek(fd, %d, SEEK_HOLE) = %d (expected %d)\n",
-		    (int)offset, (int)hole_offset, (int)expected);
-		exit(2);
-	}
+	seek_expect(fd, offset, SEEK_HOLE, expected);
 }
 
 int
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/mmap_write_sync.c b/sys/contrib/openzfs/tests/zfs-tests/cmd/mmap_write_sync.c
new file mode 100644
index 000000000000..ad5e37f24960
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/mmap_write_sync.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2025, Klara, Inc.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#define	PAGES	(8)
+
+int
+main(int argc, char **argv)
+{
+	if (argc != 2) {
+		fprintf(stderr, "usage: %s <filename>\n", argv[0]);
+		exit(1);
+	}
+
+	long page_size = sysconf(_SC_PAGESIZE);
+	if (page_size < 0) {
+		perror("sysconf");
+		exit(2);
+	}
+	size_t map_size = page_size * PAGES;
+
+	int fd = open(argv[1], O_CREAT|O_RDWR, S_IRWXU|S_IRWXG|S_IRWXO);
+	if (fd < 0) {
+		perror("open");
+		exit(2);
+	}
+
+	if (ftruncate(fd, map_size) < 0) {
+		perror("ftruncate");
+		close(fd);
+		exit(2);
+	}
+
+	uint64_t *p =
+	    mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+	if (p == MAP_FAILED) {
+		perror("mmap");
+		close(fd);
+		exit(2);
+	}
+
+	for (int i = 0; i < (map_size / sizeof (uint64_t)); i++)
+		p[i] = 0x0123456789abcdef;
+
+	if (msync(p, map_size, MS_SYNC) < 0) {
+		perror("msync");
+		munmap(p, map_size);
+		close(fd);
+		exit(3);
+	}
+
+	munmap(p, map_size);
+	close(fd);
+	exit(0);
+}
diff --git a/sys/contrib/openzfs/tests/zfs-tests/include/commands.cfg b/sys/contrib/openzfs/tests/zfs-tests/include/commands.cfg
index bbaa8665ecc8..1c4d25e152a7 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/include/commands.cfg
+++ b/sys/contrib/openzfs/tests/zfs-tests/include/commands.cfg
@@ -100,6 +100,7 @@ export SYSTEM_FILES_COMMON='awk
     uniq
     vmstat
     wc
+    which
     xargs
     xxh128sum'
 
@@ -146,6 +147,7 @@ export SYSTEM_FILES_LINUX='attr
     lscpu
     lsmod
     lsscsi
+    mkfs.xfs
     mkswap
     modprobe
     mountpoint
@@ -169,8 +171,8 @@ export ZFS_FILES='zdb
     zpool
     ztest
     raidz_test
-    arc_summary
-    arcstat
+    zarcsummary
+    zarcstat
     zilstat
     dbufstat
     mount.zfs
@@ -210,6 +212,7 @@ export ZFSTEST_FILES='badsend
     mmap_seek
     mmap_sync
     mmapwrite
+    mmap_write_sync
     nvlist_to_lua
     randfree_file
     randwritecomp
diff --git a/sys/contrib/openzfs/tests/zfs-tests/include/libtest.shlib b/sys/contrib/openzfs/tests/zfs-tests/include/libtest.shlib
index 23e89599cae0..8b30b9b91641 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/include/libtest.shlib
+++ b/sys/contrib/openzfs/tests/zfs-tests/include/libtest.shlib
@@ -1085,7 +1085,7 @@ function fill_fs # destdir dirnum filenum bytes num_writes data
 	typeset -i filenum=${3:-50}
 	typeset -i bytes=${4:-8192}
 	typeset -i num_writes=${5:-10240}
-	typeset data=${6:-0}
+	typeset data=${6:-"R"}
 
 	mkdir -p $destdir/{1..$dirnum}
 	for f in $destdir/{1..$dirnum}/$TESTFILE{1..$filenum}; do
@@ -1112,6 +1112,16 @@ function get_pool_prop # property pool
 	zpool get -Hpo value "$prop" "$pool" || log_fail "zpool get $prop $pool"
 }
 
+# Get the specified vdev property in parsable format or fail
+function get_vdev_prop
+{
+	typeset prop="$1"
+	typeset pool="$2"
+	typeset vdev="$3"
+
+	zpool get -Hpo value "$prop" "$pool" "$vdev" || log_fail "zpool get $prop $pool $vdev"
+}
+
 # Return 0 if a pool exists; $? otherwise
 #
 # $1 - pool name
@@ -1971,6 +1981,28 @@ function wait_vdev_state # pool disk state timeout
 }
 
 #
+# Wait for vdev 'sit_out' property to be cleared.
+#
+# $1 pool name
+# $2 vdev name
+# $3 timeout
+#
+function wait_sit_out #pool vdev timeout
+{
+	typeset pool=${1:-$TESTPOOL}
+	typeset vdev="$2"
+	typeset timeout=${3:-300}
+	for (( timer = 0; timer < $timeout; timer++ )); do
+		if [ "$(get_vdev_prop sit_out "$pool" "$vdev")" = "off" ]; then
+			return 0
+		fi
+		sleep 1;
+	done
+
+	return 1
+}
+
+#
 # Check the output of 'zpool status -v <pool>',
 # and to see if the content of <token> contain the <keyword> specified.
 #
@@ -2881,6 +2913,28 @@ function user_run
 	log_note "user: $user"
 	log_note "cmd: $*"
 
+	if ! sudo -Eu $user test -x $PATH ; then
+		log_note "-------------------------------------------------"
+		log_note "Warning: $user doesn't have permissions on $PATH"
+		log_note ""
+		log_note "This usually happens when you're running ZTS locally"
+		log_note "from inside the ZFS source dir, and are attempting to"
+		log_note "run a test that calls user_run.  The ephemeral user"
+		log_note "($user) that ZTS is creating does not have permission"
+		log_note "to traverse to $PATH, or the binaries in $PATH are"
+		log_note "not the right permissions."
+		log_note ""
+		log_note "To get around this, copy your ZFS source directory"
+		log_note "to a world-accessible location (like /tmp), and "
+		log_note "change the permissions on your ZFS source dir "
+		log_note "to allow access."
+		log_note ""
+		log_note "Also, verify that /dev/zfs is RW for others:"
+		log_note ""
+		log_note "    sudo chmod o+rw /dev/zfs"
+		log_note "-------------------------------------------------"
+	fi
+
 	typeset out=$TEST_BASE_DIR/out
 	typeset err=$TEST_BASE_DIR/err
 
diff --git a/sys/contrib/openzfs/tests/zfs-tests/include/tunables.cfg b/sys/contrib/openzfs/tests/zfs-tests/include/tunables.cfg
index e273c9f85c28..127ea188f17f 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/include/tunables.cfg
+++ b/sys/contrib/openzfs/tests/zfs-tests/include/tunables.cfg
@@ -72,6 +72,9 @@ MULTIHOST_INTERVAL		multihost.interval		zfs_multihost_interval
 OVERRIDE_ESTIMATE_RECORDSIZE	send.override_estimate_recordsize	zfs_override_estimate_recordsize
 PREFETCH_DISABLE		prefetch.disable		zfs_prefetch_disable
 RAIDZ_EXPAND_MAX_REFLOW_BYTES	vdev.expand_max_reflow_bytes	raidz_expand_max_reflow_bytes
+READ_SIT_OUT_SECS		vdev.read_sit_out_secs		vdev_read_sit_out_secs
+SIT_OUT_CHECK_INTERVAL		vdev.raidz_outlier_check_interval_ms	vdev_raidz_outlier_check_interval_ms
+SIT_OUT_INSENSITIVITY		vdev.raidz_outlier_insensitivity	vdev_raidz_outlier_insensitivity
 REBUILD_SCRUB_ENABLED		rebuild_scrub_enabled		zfs_rebuild_scrub_enabled
 REMOVAL_SUSPEND_PROGRESS	removal_suspend_progress	zfs_removal_suspend_progress
 REMOVE_MAX_SEGMENT		remove_max_segment		zfs_remove_max_segment
@@ -88,6 +91,7 @@ SPA_DISCARD_MEMORY_LIMIT	spa.discard_memory_limit	zfs_spa_discard_memory_limit
 SPA_LOAD_VERIFY_DATA		spa.load_verify_data		spa_load_verify_data
 SPA_LOAD_VERIFY_METADATA	spa.load_verify_metadata	spa_load_verify_metadata
 SPA_NOTE_TXG_TIME		spa.note_txg_time		spa_note_txg_time
+SPA_FLUSH_TXG_TIME		spa.flush_txg_time		spa_flush_txg_time
 TRIM_EXTENT_BYTES_MIN		trim.extent_bytes_min		zfs_trim_extent_bytes_min
 TRIM_METASLAB_SKIP		trim.metaslab_skip		zfs_trim_metaslab_skip
 TRIM_TXG_BATCH			trim.txg_batch			zfs_trim_txg_batch
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
index c2542287c1d7..678c01b58f94 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
@@ -197,6 +197,7 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \
 	functional/cli_root/zpool_import/blockfiles/unclean_export.dat.bz2 \
 	functional/cli_root/zpool_import/zpool_import.cfg \
 	functional/cli_root/zpool_import/zpool_import.kshlib \
+	functional/cli_root/zpool_iostat/zpool_iostat.kshlib \
 	functional/cli_root/zpool_initialize/zpool_initialize.kshlib \
 	functional/cli_root/zpool_labelclear/labelclear.cfg \
 	functional/cli_root/zpool_remove/zpool_remove.cfg \
@@ -276,6 +277,7 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \
 	functional/direct/dio.kshlib \
 	functional/events/events.cfg \
 	functional/events/events_common.kshlib \
+	functional/failmode/failmode.kshlib \
 	functional/fault/fault.cfg \
 	functional/gang_blocks/gang_blocks.kshlib \
 	functional/grow/grow.cfg \
@@ -639,6 +641,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zdb/zdb_decompress_zstd.ksh \
 	functional/cli_root/zdb/zdb_display_block.ksh \
 	functional/cli_root/zdb/zdb_encrypted.ksh \
+	functional/cli_root/zdb/zdb_encrypted_raw.ksh \
 	functional/cli_root/zdb/zdb_label_checksum.ksh \
 	functional/cli_root/zdb/zdb_object_range_neg.ksh \
 	functional/cli_root/zdb/zdb_object_range_pos.ksh \
@@ -891,6 +894,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zfs_send/zfs_send_raw.ksh \
 	functional/cli_root/zfs_send/zfs_send_skip_missing.ksh \
 	functional/cli_root/zfs_send/zfs_send_sparse.ksh \
+	functional/cli_root/zfs_send_delegation/cleanup.ksh \
+	functional/cli_root/zfs_send_delegation/setup.ksh \
+	functional/cli_root/zfs_send_delegation/zfs_send_test.ksh \
 	functional/cli_root/zfs_set/cache_001_pos.ksh \
 	functional/cli_root/zfs_set/cache_002_neg.ksh \
 	functional/cli_root/zfs_set/canmount_001_pos.ksh \
@@ -1008,6 +1014,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zhack/zhack_label_repair_002.ksh \
 	functional/cli_root/zhack/zhack_label_repair_003.ksh \
 	functional/cli_root/zhack/zhack_label_repair_004.ksh \
+	functional/cli_root/zhack/zhack_metaslab_leak.ksh \
 	functional/cli_root/zpool_add/add_nested_replacing_spare.ksh \
 	functional/cli_root/zpool_add/add-o_ashift.ksh \
 	functional/cli_root/zpool_add/add_prop_ashift.ksh \
@@ -1022,7 +1029,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool_add/zpool_add_007_neg.ksh \
 	functional/cli_root/zpool_add/zpool_add_008_neg.ksh \
 	functional/cli_root/zpool_add/zpool_add_009_neg.ksh \
-	functional/cli_root/zpool_add/zpool_add_010_pos.ksh \
+	functional/cli_root/zpool_add/zpool_add_warn_create.ksh \
+	functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh \
+	functional/cli_root/zpool_add/zpool_add_warn_removal.ksh \
 	functional/cli_root/zpool_add/zpool_add_dryrun_output.ksh \
 	functional/cli_root/zpool_attach/attach-o_ashift.ksh \
 	functional/cli_root/zpool_attach/cleanup.ksh \
@@ -1173,6 +1182,10 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool_import/zpool_import_parallel_admin.ksh \
 	functional/cli_root/zpool_import/zpool_import_parallel_neg.ksh \
 	functional/cli_root/zpool_import/zpool_import_parallel_pos.ksh \
+	functional/cli_root/zpool_iostat/setup.ksh \
+	functional/cli_root/zpool_iostat/cleanup.ksh \
+	functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh \
+	functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh \
 	functional/cli_root/zpool_initialize/cleanup.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_attach_detach_add_remove.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_fault_export_import_online.ksh \
@@ -1246,6 +1259,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool_scrub/zpool_scrub_print_repairing.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_txg_continue_from_last.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_date_range_001.ksh \
+	functional/cli_root/zpool_scrub/zpool_scrub_date_range_002.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_001_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_002_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_003_pos.ksh \
@@ -1350,9 +1364,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool/zpool_002_pos.ksh \
 	functional/cli_root/zpool/zpool_003_pos.ksh \
 	functional/cli_root/zpool/zpool_colors.ksh \
-	functional/cli_user/misc/arcstat_001_pos.ksh \
-	functional/cli_user/misc/arc_summary_001_pos.ksh \
-	functional/cli_user/misc/arc_summary_002_neg.ksh \
+	functional/cli_user/misc/zarcstat_001_pos.ksh \
+	functional/cli_user/misc/zarcsummary_001_pos.ksh \
+	functional/cli_user/misc/zarcsummary_002_neg.ksh \
 	functional/cli_user/misc/zilstat_001_pos.ksh \
 	functional/cli_user/misc/cleanup.ksh \
 	functional/cli_user/misc/setup.ksh \
@@ -1407,6 +1421,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_user/zfs_list/zfs_list_005_neg.ksh \
 	functional/cli_user/zfs_list/zfs_list_007_pos.ksh \
 	functional/cli_user/zfs_list/zfs_list_008_neg.ksh \
+	functional/cli_user/zfs_send_delegation_user/cleanup.ksh \
+	functional/cli_user/zfs_send_delegation_user/setup.ksh \
+	functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh \
 	functional/cli_user/zpool_iostat/cleanup.ksh \
 	functional/cli_user/zpool_iostat/setup.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_001_neg.ksh \
@@ -1523,6 +1540,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/events/events_001_pos.ksh \
 	functional/events/events_002_pos.ksh \
 	functional/events/setup.ksh \
+	functional/events/slow_vdev_degraded_sit_out.ksh \
+	functional/events/slow_vdev_sit_out.ksh \
+	functional/events/slow_vdev_sit_out_neg.ksh \
 	functional/events/zed_cksum_config.ksh \
 	functional/events/zed_cksum_reported.ksh \
 	functional/events/zed_diagnose_multiple.ksh \
@@ -1531,6 +1551,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/events/zed_rc_filter.ksh \
 	functional/events/zed_slow_io.ksh \
 	functional/events/zed_slow_io_many_vdevs.ksh \
+	functional/events/zed_synchronous_zedlet.ksh \
 	functional/exec/cleanup.ksh \
 	functional/exec/exec_001_pos.ksh \
 	functional/exec/exec_002_neg.ksh \
@@ -1541,6 +1562,14 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/failmode/cleanup.ksh \
 	functional/failmode/failmode_dmu_tx_wait.ksh \
 	functional/failmode/failmode_dmu_tx_continue.ksh \
+	functional/failmode/failmode_fsync_wait.ksh \
+	functional/failmode/failmode_fsync_continue.ksh \
+	functional/failmode/failmode_msync_wait.ksh \
+	functional/failmode/failmode_msync_continue.ksh \
+	functional/failmode/failmode_osync_wait.ksh \
+	functional/failmode/failmode_osync_continue.ksh \
+	functional/failmode/failmode_syncalways_wait.ksh \
+	functional/failmode/failmode_syncalways_continue.ksh \
 	functional/failmode/setup.ksh \
 	functional/fallocate/cleanup.ksh \
 	functional/fallocate/fallocate_prealloc.ksh \
@@ -1697,6 +1726,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/mmp/setup.ksh \
 	functional/mount/cleanup.ksh \
 	functional/mount/setup.ksh \
+	functional/mount/mount_loopback.ksh \
 	functional/mount/umount_001.ksh \
 	functional/mount/umountall_001.ksh \
 	functional/mount/umount_unlinked_drain.ksh \
@@ -1926,6 +1956,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/replacement/attach_multiple.ksh \
 	functional/replacement/attach_rebuild.ksh \
 	functional/replacement/attach_resilver.ksh \
+	functional/replacement/attach_resilver_sit_out.ksh \
 	functional/replacement/cleanup.ksh \
 	functional/replacement/detach.ksh \
 	functional/replacement/rebuild_disabled_feature.ksh \
@@ -1934,6 +1965,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/replacement/replace_import.ksh \
 	functional/replacement/replace_rebuild.ksh \
 	functional/replacement/replace_resilver.ksh \
+	functional/replacement/replace_resilver_sit_out.ksh \
 	functional/replacement/resilver_restart_001.ksh \
 	functional/replacement/resilver_restart_002.ksh \
 	functional/replacement/scrub_cancel.ksh \
@@ -2202,6 +2234,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/xattr/xattr_011_pos.ksh \
 	functional/xattr/xattr_012_pos.ksh \
 	functional/xattr/xattr_013_pos.ksh \
+	functional/xattr/xattr_014_pos.ksh \
 	functional/xattr/xattr_compat.ksh \
 	functional/zap_shrink/cleanup.ksh \
 	functional/zap_shrink/zap_shrink_001_pos.ksh \
@@ -2235,6 +2268,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/zvol/zvol_stress/cleanup.ksh \
 	functional/zvol/zvol_stress/setup.ksh \
 	functional/zvol/zvol_stress/zvol_stress.ksh \
+	functional/zvol/zvol_stress/zvol_stress_destroy.ksh \
 	functional/zvol/zvol_swap/cleanup.ksh \
 	functional/zvol/zvol_swap/setup.ksh \
 	functional/zvol/zvol_swap/zvol_swap_001_pos.ksh \
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted_raw.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted_raw.ksh
new file mode 100755
index 000000000000..85d267d5402f
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted_raw.ksh
@@ -0,0 +1,75 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2023, Klara Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+
+#
+# DESCRIPTION:
+# 'zdb -K ...' should enable reading from a raw-encrypted dataset
+#
+# STRATEGY:
+# 1. Create an encrypted dataset
+# 2. Write some data to a file
+# 3. Run zdb -dddd on the file, confirm it can't be read
+# 4. Run zdb -K ... -ddddd on the file, confirm it can be read
+#
+
+verify_runnable "both"
+
+dataset="$TESTPOOL/$TESTFS2"
+file="$TESTDIR2/somefile"
+keyfile="$TEST_BASE_DIR/keyfile"
+
+function cleanup
+{
+	datasetexists "$dataset" && destroy_dataset "$dataset" -f
+	rm -f "$keyfile"
+	default_cleanup_noexit
+}
+
+log_onexit cleanup
+
+log_must default_setup_noexit $DISKS
+
+log_assert "'zdb -K' should enable reading from a raw-encrypted dataset"
+
+# The key must be 32 bytes long.
+echo -n "$RAWKEY" > "$keyfile"
+
+log_must zfs create -o mountpoint="$TESTDIR2" \
+    -o encryption=on -o keyformat=raw -o keylocation="file://$keyfile" \
+    "$dataset"
+
+echo 'my great encrypted text' > "$file"
+
+typeset -i obj=$(ls -i "$file" | cut -d' ' -f1)
+typeset -i size=$(wc -c < "$file")
+
+log_note "test file $file is objid $obj, size $size"
+
+sync_pool "$TESTPOOL" true
+
+log_must eval "zdb -dddd $dataset $obj | grep -q 'object encrypted'"
+
+log_must eval "zdb -K $keyfile -dddd $dataset $obj | grep -q 'size\s$size$'"
+
+log_pass "'zdb -K' enables reading from a raw-encrypted dataset"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_tunables.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_tunables.ksh
index 46965aa7cc37..b89790d5d525 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_tunables.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_tunables.ksh
@@ -63,7 +63,7 @@ log_mustnot_expect 'no such tunable: 0' zdb -o show=0
 log_mustnot_expect 'no such tunable: 1' zdb -o info=1
 
 # can set multiple in same command
-log_must eval 'zdb -o zfs_recover=1 -o zfs_flags=512 | xargs | grep -qE "^zfs_recover: 0 -> 1 zfs_flags: 4294965758 -> 512$"'
+log_must eval 'zdb -o zfs_recover=1 -o zfs_flags=512 | xargs | grep -qE "^zfs_recover: 0 -> 1 zfs_flags: 4294932990 -> 512$"'
 
 # can set and show in same command
 log_must eval 'zdb -o zfs_recover=1 -o zfs_recover -o zfs_recover=0 | xargs | grep -qE "^zfs_recover: 0 -> 1 zfs_recover: 1 zfs_recover: 1 -> 0$"'
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/cleanup.ksh
new file mode 100755
index 000000000000..4a59e15cc693
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/cleanup.ksh
@@ -0,0 +1,43 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib
+
+
+poolexists $TESTPOOL1 && \
+	destroy_pool $TESTPOOL1
+
+del_user $STAFF1
+del_user $STAFF2
+del_group $STAFF_GROUP
+
+del_user $OTHER1
+del_user $OTHER2
+del_group $OTHER_GROUP
+
+default_cleanup
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/setup.ksh
new file mode 100755
index 000000000000..0978193eddc4
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/setup.ksh
@@ -0,0 +1,50 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib
+
+# Create staff group and add two user to it
+log_must add_group $STAFF_GROUP
+if ! id $STAFF1 > /dev/null 2>&1; then
+	log_must add_user $STAFF_GROUP $STAFF1
+fi
+if ! id $STAFF2 > /dev/null 2>&1; then
+	log_must add_user $STAFF_GROUP $STAFF2
+fi
+
+# Create other group and add two user to it
+log_must add_group $OTHER_GROUP
+if ! id $OTHER1 > /dev/null 2>&1; then
+	log_must add_user $OTHER_GROUP $OTHER1
+fi
+if ! id $OTHER2 > /dev/null 2>&1; then
+	log_must add_user $OTHER_GROUP $OTHER2
+fi
+DISK=${DISKS%% *}
+
+default_raidz_setup $DISKS
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/zfs_send_test.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/zfs_send_test.ksh
new file mode 100755
index 000000000000..d018f313fae1
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/zfs_send_test.ksh
@@ -0,0 +1,111 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara Inc.
+#
+
+# STRATEGY:
+# 1. Create a pool (this is done by the test framework)
+# 2. Create an encrypted dataset
+# 3. Write random data to the encrypted dataset
+# 4. Snapshot the dataset
+# 5. As root: attempt a send and raw send (both should succeed)
+# 6. Create a delegation (zfs allow -u user send testpool/encrypted_dataset)
+# 7. As root: attempt a send and raw send (both should succeed)
+# 8. Create a delegation (zfs allow -u user send:raw testpool/encrypted_dataset)
+# 9. As root: attempt a send and raw send (both should succeed)
+# 10. Disable delegation (zfs unallow)
+# 11. As root: attempt a send and raw send (both should succeed)
+# 12. Clean up (handled by framework)
+#
+# Tested as a user under ../cli_user/zfs_send_delegation_user/
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_create/zfs_create_common.kshlib
+. $STF_SUITE/tests/functional/cli_root/zfs_create/properties.kshlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+. $STF_SUITE/tests/functional/delegate/delegate.cfg
+
+# create encrypted dataset
+
+log_must eval "echo $PASSPHRASE | zfs create -o encryption=on -o keyformat=passphrase $TESTPOOL/$TESTFS1"
+
+# create target dataset for receives
+if ! zfs list | grep testfs2 >/dev/null 2>&1; then
+    dataset_created="TRUE"
+    log_must zfs create $TESTPOOL/$TESTFS2
+fi
+
+# create user and group
+typeset perms="snapshot,reservation,compression,checksum,userprop,receive"
+
+log_note "Added permissions to the $OTHER1 user."
+log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS1
+log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS2
+
+# create random data
+log_must fill_fs $TESTPOOL/$TESTFS1/child 1 2047 1024 1 R
+
+# snapshot
+log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
+
+
+# check baseline send abilities (should pass)
+log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv0_datastream.$$"
+log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv0raw_datastream.$$"
+
+
+# create delegation
+log_must zfs allow $OTHER1 send $TESTPOOL/$TESTFS1
+
+# attempt send with full allow
+
+log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv1_datastream.$$"
+log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv1raw_datastream.$$"
+
+# create raw delegation
+log_must zfs allow $OTHER1 send:raw $TESTPOOL/$TESTFS1
+log_must zfs unallow $OTHER1 send $TESTPOOL/$TESTFS1
+
+# test new send abilities (should pass)
+log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv2_datastream.$$"
+log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv2raw_datastream.$$"
+
+
+# disable raw delegation
+zfs unallow $OTHER1 send:raw $TESTPOOL/$TESTFS1
+zfs allow $OTHER1 send $TESTPOOL/$TESTFS1
+
+# verify original send abilities (should pass)
+log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv3_datastream.$$"
+log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv3raw_datastream.$$"
+
+
+function cleanup
+{
+    datasetexists $TESTPOOL/$TESTFS1 && \
+	    destroy_dataset $TESTPOOL/$TESTFS1 -r \
+            destroy_dataset $TESTPOOL/$TESTFS2 -r
+
+}
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/library.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/library.kshlib
index 0f5f6198daf2..0d07b1fd1952 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/library.kshlib
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/library.kshlib
@@ -33,13 +33,16 @@
 # Test one:
 #
 # 1. Create pool on a loopback device with some test data
-# 2. Export the pool.
-# 3. Corrupt all label checksums in the pool
-# 4. Check that pool cannot be imported
-# 5. Verify that it cannot be imported after using zhack label repair -u
+# 2. Checksum repair should work with a valid TXG. Repeatedly write and
+#    sync the pool so there are enough transactions for every uberblock
+#    to have a TXG
+# 3. Export the pool.
+# 4. Corrupt all label checksums in the pool
+# 5. Check that pool cannot be imported
+# 6. Verify that it cannot be imported after using zhack label repair -u
 #    to ensure that the -u option will quit on corrupted checksums.
-# 6. Use zhack label repair -c on device
-# 7. Check that pool can be imported and that data is intact
+# 7. Use zhack label repair -c on device
+# 8. Check that pool can be imported and that data is intact
 #
 # Test two:
 #
@@ -97,7 +100,7 @@ VIRTUAL_MIRROR_DEVICE=
 
 function cleanup_lo
 {
-	L_DEVICE="$1"
+	typeset L_DEVICE="$1"
 
 	if [[ -e $L_DEVICE ]]; then
 		if is_linux; then
@@ -133,9 +136,9 @@ function get_devsize
 
 function pick_logop
 {
-	L_SHOULD_SUCCEED="$1"
+	typeset L_SHOULD_SUCCEED="$1"
 
-	l_logop="log_mustnot"
+	typeset l_logop="log_mustnot"
 	if [ "$L_SHOULD_SUCCEED" == true ]; then
 		l_logop="log_must"
 	fi
@@ -145,7 +148,9 @@ function pick_logop
 
 function check_dataset
 {
-	L_SHOULD_SUCCEED="$1"
+	typeset L_SHOULD_SUCCEED="$1"
+
+	typeset L_LOGOP=
 	L_LOGOP="$(pick_logop "$L_SHOULD_SUCCEED")"
 
 	"$L_LOGOP" mounted "$TESTPOOL"/"$TESTFS"
@@ -170,9 +175,21 @@ function setup_dataset
 	check_dataset true
 }
 
+function force_transactions
+{
+	typeset L_TIMES="$1"
+	typeset i=
+	for ((i=0; i < L_TIMES; i++))
+	do
+		touch "$TESTDIR"/"test" || return $?
+		zpool sync -f "$TESTPOOL" || return $?
+	done
+	return 0
+}
+
 function get_practical_size
 {
-	L_SIZE="$1"
+	typeset L_SIZE="$1"
 
 	if [ "$((L_SIZE % LABEL_SIZE))" -ne 0 ]; then
 		echo "$(((L_SIZE / LABEL_SIZE) * LABEL_SIZE))"
@@ -183,10 +200,11 @@ function get_practical_size
 
 function corrupt_sized_label_checksum
 {
-	L_SIZE="$1"
-	L_LABEL="$2"
-	L_DEVICE="$3"
+	typeset L_SIZE="$1"
+	typeset L_LABEL="$2"
+	typeset L_DEVICE="$3"
 
+	typeset L_PRACTICAL_SIZE=
 	L_PRACTICAL_SIZE="$(get_practical_size "$L_SIZE")"
 
 	typeset -a L_OFFSETS=("$LABEL_CKSUM_START" \
@@ -201,8 +219,8 @@ function corrupt_sized_label_checksum
 
 function corrupt_labels
 {
-	L_SIZE="$1"
-	L_DISK="$2"
+	typeset L_SIZE="$1"
+	typeset L_DISK="$2"
 
 	corrupt_sized_label_checksum "$L_SIZE" 0 "$L_DISK"
 	corrupt_sized_label_checksum "$L_SIZE" 1 "$L_DISK"
@@ -212,11 +230,14 @@ function corrupt_labels
 
 function try_import_and_repair
 {
-	L_REPAIR_SHOULD_SUCCEED="$1"
-	L_IMPORT_SHOULD_SUCCEED="$2"
-	L_OP="$3"
-	L_POOLDISK="$4"
+	typeset L_REPAIR_SHOULD_SUCCEED="$1"
+	typeset L_IMPORT_SHOULD_SUCCEED="$2"
+	typeset L_OP="$3"
+	typeset L_POOLDISK="$4"
+
+	typeset L_REPAIR_LOGOP=
 	L_REPAIR_LOGOP="$(pick_logop "$L_REPAIR_SHOULD_SUCCEED")"
+	typeset L_IMPORT_LOGOP=
 	L_IMPORT_LOGOP="$(pick_logop "$L_IMPORT_SHOULD_SUCCEED")"
 
 	log_mustnot zpool import "$TESTPOOL" -d "$L_POOLDISK"
@@ -230,10 +251,10 @@ function try_import_and_repair
 
 function prepare_vdev
 {
-	L_SIZE="$1"
-	L_BACKFILE="$2"
+	typeset L_SIZE="$1"
+	typeset L_BACKFILE="$2"
 
-	l_devname=
+	typeset l_devname=
 	if truncate -s "$L_SIZE" "$L_BACKFILE"; then
 		if is_linux; then
 			l_devname="$(losetup -f "$L_BACKFILE" --show)"
@@ -248,7 +269,7 @@ function prepare_vdev
 
 function run_test_one
 {
-	L_SIZE="$1"
+	typeset L_SIZE="$1"
 
 	VIRTUAL_DEVICE="$(prepare_vdev "$L_SIZE" "$VIRTUAL_DISK")"
 	log_must test -e "$VIRTUAL_DEVICE"
@@ -257,6 +278,9 @@ function run_test_one
 
 	setup_dataset
 
+	# Force 256 extra transactions to ensure all uberblocks are assigned a TXG
+	log_must force_transactions 256
+
 	log_must zpool export "$TESTPOOL"
 
 	corrupt_labels "$L_SIZE" "$VIRTUAL_DISK"
@@ -272,7 +296,7 @@ function run_test_one
 
 function make_mirrored_pool
 {
-	L_SIZE="$1"
+	typeset L_SIZE="$1"
 
 	VIRTUAL_DEVICE="$(prepare_vdev "$L_SIZE" "$VIRTUAL_DISK")"
 	log_must test -e "$VIRTUAL_DEVICE"
@@ -296,7 +320,7 @@ function export_and_cleanup_vdisk
 
 function run_test_two
 {
-	L_SIZE="$1"
+	typeset L_SIZE="$1"
 
 	make_mirrored_pool "$L_SIZE"
 
@@ -317,7 +341,7 @@ function run_test_two
 
 function run_test_three
 {
-	L_SIZE="$1"
+	typeset L_SIZE="$1"
 
 	make_mirrored_pool "$L_SIZE"
 
@@ -342,7 +366,7 @@ function run_test_three
 
 function run_test_four
 {
-	L_SIZE="$1"
+	typeset L_SIZE="$1"
 
 	make_mirrored_pool "$L_SIZE"
 
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_label_repair_001.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_label_repair_001.ksh
index ce159b555d20..b5b24322f882 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_label_repair_001.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_label_repair_001.ksh
@@ -18,13 +18,16 @@
 # Strategy:
 #
 # 1. Create pool on a loopback device with some test data
-# 2. Export the pool.
-# 3. Corrupt all label checksums in the pool
-# 4. Check that pool cannot be imported
-# 5. Verify that it cannot be imported after using zhack label repair -u
+# 2. Checksum repair should work with a valid TXG. Repeatedly write and
+#    sync the pool so there are enough transactions for every uberblock
+#    to have a TXG
+# 3. Export the pool.
+# 4. Corrupt all label checksums in the pool
+# 5. Check that pool cannot be imported
+# 6. Verify that it cannot be imported after using zhack label repair -u
 #    to ensure that the -u option will quit on corrupted checksums.
-# 6. Use zhack label repair -c on device
-# 7. Check that pool can be imported and that data is intact
+# 7. Use zhack label repair -c on device
+# 8. Check that pool can be imported and that data is intact
 
 . "$STF_SUITE"/tests/functional/cli_root/zhack/library.kshlib
 
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_leak.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_leak.ksh
new file mode 100755
index 000000000000..0d2a39be6b5a
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_leak.ksh
@@ -0,0 +1,70 @@
+#!/bin/ksh
+# SPDX-License-Identifier: CDDL-1.0
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+
+#
+# Description:
+#
+# Test whether zhack metaslab leak functions correctly
+#
+# Strategy:
+#
+# 1. Create pool on a loopback device with some test data
+# 2. Gather pool capacity stats
+# 3. Generate fragmentation data with zdb
+# 4. Destroy the pool
+# 5. Create a new pool with the same configuration
+# 6. Export the pool
+# 7. Apply the fragmentation information with zhack metaslab leak
+# 8. Import the pool
+# 9. Verify that pool capacity stats match
+
+. "$STF_SUITE"/include/libtest.shlib
+
+verify_runnable "global"
+
+function cleanup
+{
+	zpool destroy $TESTPOOL
+	rm $tmp
+}
+
+log_onexit cleanup
+log_assert "zhack metaslab leak leaks the right amount of space"
+
+typeset tmp=$(mktemp)
+
+log_must zpool create $TESTPOOL $DISKS
+for i in `seq 1 16`; do
+	log_must dd if=/dev/urandom of=/$TESTPOOL/f$i bs=1M count=16
+	log_must zpool sync $TESTPOOL
+done
+for i in `seq 2 2 16`; do
+	log_must rm /$TESTPOOL/f$i
+done
+for i in `seq 1 16`; do
+	log_must touch /$TESTPOOL/g$i
+	log_must zpool sync $TESTPOOL
+done
+
+alloc=$(zpool get -Hpo value alloc $TESTPOOL)
+log_must eval "zdb -m --allocated-map $TESTPOOL > $tmp"
+log_must zpool destroy $TESTPOOL
+
+log_must zpool create $TESTPOOL $DISKS
+log_must zpool export $TESTPOOL
+log_must eval "zhack metaslab leak $TESTPOOL < $tmp"
+log_must zpool import $TESTPOOL
+
+alloc2=$(zpool get -Hpo value alloc $TESTPOOL)
+
+within_percent $alloc $alloc2 98 || 
+	log_fail "space usage changed too much: $alloc to $alloc2"
+
+log_pass "zhack metaslab leak behaved correctly"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib
index 091d65bb4f33..74780bb02141 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib
@@ -27,6 +27,7 @@
 
 #
 # Copyright (c) 2012, 2016 by Delphix. All rights reserved.
+# Copyright 2025 by Lawrence Livermore National Security, LLC.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -89,3 +90,44 @@ function save_dump_dev
 	fi
 	echo $dumpdev
 }
+
+function zpool_create_add_setup
+{
+	typeset -i i=0
+
+	while ((i < 10)); do
+		log_must truncate -s $MINVDEVSIZE $TEST_BASE_DIR/vdev$i
+
+		eval vdev$i=$TEST_BASE_DIR/vdev$i
+		((i += 1))
+	done
+
+	if is_linux; then
+	        vdev_lo="$(losetup -f "$vdev4" --show)"
+	elif is_freebsd; then
+	        vdev_lo=/dev/"$(mdconfig -a -t vnode -f "$vdev4")"
+	else
+	        vdev_lo="$(lofiadm -a "$vdev4")"
+	fi
+}
+
+function zpool_create_add_cleanup
+{
+	datasetexists $TESTPOOL1 && destroy_pool $TESTPOOL1
+
+	if [[ -e $vdev_lo ]]; then
+		if is_linux; then
+			log_must losetup -d "$vdev_lo"
+		elif is_freebsd; then
+			log_must mdconfig -d -u "$vdev_lo"
+		else
+			log_must lofiadm -d "$vdev_lo"
+		fi
+	fi
+
+	typeset -i i=0
+	while ((i < 10)); do
+		rm -f $TEST_BASE_DIR/vdev$i
+		((i += 1))
+	done
+}
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_create.ksh
index df085a2ec746..661e55998d8d 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_create.ksh
@@ -23,67 +23,51 @@
 
 #
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
-# Use is subject to license terms.
-#
-
-#
-# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
+# Copyright 2012, 2016 by Delphix. All rights reserved.
+# Copyright 2025 by Lawrence Livermore National Security, LLC.
 #
 
 . $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/cli_root/zpool_create/zpool_create.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
 
 #
 # DESCRIPTION:
-#	Verify zpool add succeed when adding vdevs with matching redundancy.
+#	Verify zpool add succeeds when adding vdevs with matching redundancy
+#	and warns with differing redundancy for a healthy pool.
 #
 # STRATEGY:
 #	1. Create several files == $MINVDEVSIZE.
 #	2. Verify 'zpool add' succeeds with matching redundancy.
 #	3. Verify 'zpool add' warns with differing redundancy.
-#	4. Verify 'zpool add' warns with differing redundancy after removal.
 #
 
 verify_runnable "global"
 
-function cleanup
-{
-	datasetexists $TESTPOOL1 && destroy_pool $TESTPOOL1
-
-	typeset -i i=0
-	while ((i < 10)); do
-		rm -f $TEST_BASE_DIR/vdev$i
-		((i += 1))
-	done
-}
-
+log_assert "Verify 'zpool add' warns for differing redundancy."
+log_onexit zpool_create_add_cleanup
 
-log_assert "Verify 'zpool add' succeed with keywords combination."
-log_onexit cleanup
+zpool_create_add_setup
 
-# 1. Create several files == $MINVDEVSIZE.
 typeset -i i=0
-while ((i < 10)); do
-	log_must truncate -s $MINVDEVSIZE $TEST_BASE_DIR/vdev$i
-
-	eval vdev$i=$TEST_BASE_DIR/vdev$i
-	((i += 1))
-done
+typeset -i j=0
 
 set -A redundancy0_create_args \
 	"$vdev0"
 
 set -A redundancy1_create_args \
 	"mirror $vdev0 $vdev1" \
-	"raidz1 $vdev0 $vdev1"
+	"raidz1 $vdev0 $vdev1" \
+	"draid1:1s $vdev0 $vdev1 $vdev9"
 
 set -A redundancy2_create_args \
 	"mirror $vdev0 $vdev1 $vdev2" \
-	"raidz2 $vdev0 $vdev1 $vdev2"
+	"raidz2 $vdev0 $vdev1 $vdev2" \
+	"draid2:1s $vdev0 $vdev1 $vdev2 $vdev9"
 
 set -A redundancy3_create_args \
 	"mirror $vdev0 $vdev1 $vdev2 $vdev3" \
-	"raidz3 $vdev0 $vdev1 $vdev2 $vdev3"
+	"raidz3 $vdev0 $vdev1 $vdev2 $vdev3" \
+	"draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9"
 
 set -A redundancy0_add_args \
 	"$vdev5" \
@@ -93,21 +77,19 @@ set -A redundancy1_add_args \
 	"mirror $vdev5 $vdev6" \
 	"raidz1 $vdev5 $vdev6" \
 	"raidz1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \
-	"mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8"
+	"mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8" \
+	"draid1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \
+	"mirror $vdev5 $vdev6 draid1 $vdev7 $vdev8"
 
 set -A redundancy2_add_args \
 	"mirror $vdev5 $vdev6 $vdev7" \
-	"raidz2 $vdev5 $vdev6 $vdev7"
+	"raidz2 $vdev5 $vdev6 $vdev7" \
+	"draid2 $vdev5 $vdev6 $vdev7"
 
 set -A redundancy3_add_args \
 	"mirror $vdev5 $vdev6 $vdev7 $vdev8" \
-	"raidz3 $vdev5 $vdev6 $vdev7 $vdev8"
-
-set -A log_args "log" "$vdev4"
-set -A cache_args "cache" "$vdev4"
-set -A spare_args "spare" "$vdev4"
-
-typeset -i j=0
+	"raidz3 $vdev5 $vdev6 $vdev7 $vdev8" \
+	"draid3 $vdev5 $vdev6 $vdev7 $vdev8"
 
 function zpool_create_add
 {
@@ -148,30 +130,6 @@ function zpool_create_forced_add
 	done
 }
 
-function zpool_create_rm_add
-{
-	typeset -n create_args=$1
-	typeset -n add_args=$2
-	typeset -n rm_args=$3
-
-	i=0
-	while ((i < ${#create_args[@]})); do
-		j=0
-		while ((j < ${#add_args[@]})); do
-			log_must zpool create $TESTPOOL1 ${create_args[$i]}
-			log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
-			log_must zpool add $TESTPOOL1 ${add_args[$j]}
-			log_must zpool remove $TESTPOOL1 ${rm_args[1]}
-			log_mustnot zpool add $TESTPOOL1 ${rm_args[1]}
-			log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
-			log_must zpool destroy -f $TESTPOOL1
-
-			((j += 1))
-		done
-		((i += 1))
-	done
-}
-
 # 2. Verify 'zpool add' succeeds with matching redundancy.
 zpool_create_add redundancy0_create_args redundancy0_add_args
 zpool_create_add redundancy1_create_args redundancy1_add_args
@@ -195,17 +153,4 @@ zpool_create_forced_add redundancy3_create_args redundancy0_add_args
 zpool_create_forced_add redundancy3_create_args redundancy1_add_args
 zpool_create_forced_add redundancy3_create_args redundancy2_add_args
 
-# 4. Verify 'zpool add' warns with differing redundancy after removal.
-zpool_create_rm_add redundancy1_create_args redundancy1_add_args log_args
-zpool_create_rm_add redundancy2_create_args redundancy2_add_args log_args
-zpool_create_rm_add redundancy3_create_args redundancy3_add_args log_args
-
-zpool_create_rm_add redundancy1_create_args redundancy1_add_args cache_args
-zpool_create_rm_add redundancy2_create_args redundancy2_add_args cache_args
-zpool_create_rm_add redundancy3_create_args redundancy3_add_args cache_args
-
-zpool_create_rm_add redundancy1_create_args redundancy1_add_args spare_args
-zpool_create_rm_add redundancy2_create_args redundancy2_add_args spare_args
-zpool_create_rm_add redundancy3_create_args redundancy3_add_args spare_args
-
-log_pass "'zpool add' succeed with keywords combination."
+log_pass "Verify 'zpool add' warns for differing redundancy."
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh
new file mode 100755
index 000000000000..313eb3666f27
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh
@@ -0,0 +1,204 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2012, 2016 by Delphix. All rights reserved.
+# Copyright 2025 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
+
+#
+# DESCRIPTION:
+#	Verify zpool add succeeds when adding vdevs with matching redundancy
+#	and warns with differing redundancy for a degraded pool.
+#
+# STRATEGY:
+#	1. Create several files == $MINVDEVSIZE.
+#	2. Verify 'zpool add' succeeds with matching redundancy
+#	3. Verify 'zpool add' warns with differing redundancy when
+#	  a. Degraded pool with replaced mismatch vdev (file vs disk)
+#	  b. Degraded pool dRAID distributed spare active
+#	  c. Degraded pool hot spare active
+#
+
+verify_runnable "global"
+
+log_assert "Verify 'zpool add' warns for differing redundancy."
+log_onexit zpool_create_add_cleanup
+
+zpool_create_add_setup
+
+set -A redundancy1_create_args \
+	"mirror $vdev0 $vdev1" \
+	"raidz1 $vdev0 $vdev1" \
+	"draid1:1s $vdev0 $vdev1 $vdev9"
+
+set -A redundancy2_create_args \
+	"mirror $vdev0 $vdev1 $vdev2" \
+	"raidz2 $vdev0 $vdev1 $vdev2" \
+	"draid2:1s $vdev0 $vdev1 $vdev2 $vdev9"
+
+set -A redundancy3_create_args \
+	"mirror $vdev0 $vdev1 $vdev2 $vdev3" \
+	"raidz3 $vdev0 $vdev1 $vdev2 $vdev3" \
+	"draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9"
+
+set -A redundancy1_add_args \
+	"mirror $vdev5 $vdev6" \
+	"raidz1 $vdev5 $vdev6" \
+	"raidz1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \
+	"mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8" \
+	"draid1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \
+	"mirror $vdev5 $vdev6 draid1 $vdev7 $vdev8"
+
+set -A redundancy2_add_args \
+	"mirror $vdev5 $vdev6 $vdev7" \
+	"raidz2 $vdev5 $vdev6 $vdev7" \
+	"draid2 $vdev5 $vdev6 $vdev7"
+
+set -A redundancy3_add_args \
+	"mirror $vdev5 $vdev6 $vdev7 $vdev8" \
+	"raidz3 $vdev5 $vdev6 $vdev7 $vdev8" \
+	"draid3 $vdev5 $vdev6 $vdev7 $vdev8"
+
+set -A redundancy1_create_draid_args \
+	"draid1:1s $vdev0 $vdev1 $vdev2"
+
+set -A redundancy2_create_draid_args \
+	"draid2:1s $vdev0 $vdev1 $vdev2 $vdev3"
+
+set -A redundancy3_create_draid_args \
+	"draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9"
+
+set -A redundancy1_create_spare_args \
+	"mirror $vdev0 $vdev1 spare $vdev_lo" \
+	"raidz1 $vdev0 $vdev1 spare $vdev_lo" \
+	"draid1 $vdev0 $vdev1 spare $vdev_lo"
+
+set -A redundancy2_create_spare_args \
+	"mirror $vdev0 $vdev1 $vdev2 spare $vdev_lo" \
+	"raidz2 $vdev0 $vdev1 $vdev2 spare $vdev_lo" \
+	"draid2 $vdev0 $vdev1 $vdev2 spare $vdev_lo"
+
+set -A redundancy3_create_spare_args \
+	"mirror $vdev0 $vdev1 $vdev2 $vdev3 spare $vdev_lo" \
+	"raidz3 $vdev0 $vdev1 $vdev2 $vdev3 spare $vdev_lo" \
+	"draid3 $vdev0 $vdev1 $vdev2 $vdev3 spare $vdev_lo"
+
+set -A replace_args "$vdev1" "$vdev_lo"
+set -A draid1_args "$vdev1" "draid1-0-0"
+set -A draid2_args "$vdev1" "draid2-0-0"
+set -A draid3_args "$vdev1" "draid3-0-0"
+
+typeset -i i=0
+typeset -i j=0
+
+function zpool_create_degraded_add
+{
+	typeset -n create_args=$1
+	typeset -n add_args=$2
+	typeset -n rm_args=$3
+
+	i=0
+	while ((i < ${#create_args[@]})); do
+		j=0
+		while ((j < ${#add_args[@]})); do
+			log_must zpool create $TESTPOOL1 ${create_args[$i]}
+			log_must zpool offline -f $TESTPOOL1 ${rm_args[0]}
+			log_must zpool replace -w $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
+			log_must zpool add $TESTPOOL1 ${add_args[$j]}
+			log_must zpool destroy -f $TESTPOOL1
+			log_must zpool labelclear -f ${rm_args[0]}
+
+			((j += 1))
+		done
+		((i += 1))
+	done
+}
+
+function zpool_create_forced_degraded_add
+{
+	typeset -n create_args=$1
+	typeset -n add_args=$2
+	typeset -n rm_args=$3
+
+	i=0
+	while ((i < ${#create_args[@]})); do
+		j=0
+		while ((j < ${#add_args[@]})); do
+			log_must zpool create $TESTPOOL1 ${create_args[$i]}
+			log_must zpool offline -f $TESTPOOL1 ${rm_args[0]}
+			log_must zpool replace -w $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
+			log_mustnot zpool add $TESTPOOL1 ${add_args[$j]}
+			log_must zpool add --allow-replication-mismatch $TESTPOOL1 ${add_args[$j]}
+			log_must zpool destroy -f $TESTPOOL1
+			log_must zpool labelclear -f ${rm_args[0]}
+
+			((j += 1))
+		done
+		((i += 1))
+	done
+}
+
+# 2. Verify 'zpool add' succeeds with matching redundancy and a degraded pool.
+zpool_create_degraded_add redundancy1_create_args redundancy1_add_args replace_args
+zpool_create_degraded_add redundancy2_create_args redundancy2_add_args replace_args
+zpool_create_degraded_add redundancy3_create_args redundancy3_add_args replace_args
+
+# 3. Verify 'zpool add' warns with differing redundancy and a degraded pool.
+#
+# a. Degraded pool with replaced mismatch vdev (file vs disk)
+zpool_create_forced_degraded_add redundancy1_create_args redundancy2_add_args replace_args
+zpool_create_forced_degraded_add redundancy1_create_args redundancy3_add_args replace_args
+
+zpool_create_forced_degraded_add redundancy2_create_args redundancy1_add_args replace_args
+zpool_create_forced_degraded_add redundancy2_create_args redundancy3_add_args replace_args
+
+zpool_create_forced_degraded_add redundancy3_create_args redundancy1_add_args replace_args
+zpool_create_forced_degraded_add redundancy3_create_args redundancy2_add_args replace_args
+
+# b. Degraded pool dRAID distributed spare active
+
+zpool_create_forced_degraded_add redundancy1_create_draid_args redundancy2_add_args draid1_args
+zpool_create_forced_degraded_add redundancy1_create_draid_args redundancy3_add_args draid1_args
+
+zpool_create_forced_degraded_add redundancy2_create_draid_args redundancy1_add_args draid2_args
+zpool_create_forced_degraded_add redundancy2_create_draid_args redundancy3_add_args draid2_args
+
+zpool_create_forced_degraded_add redundancy3_create_draid_args redundancy1_add_args draid3_args
+zpool_create_forced_degraded_add redundancy3_create_draid_args redundancy2_add_args draid3_args
+
+# c. Degraded pool hot spare active
+zpool_create_forced_degraded_add redundancy1_create_spare_args redundancy2_add_args replace_args
+zpool_create_forced_degraded_add redundancy1_create_spare_args redundancy3_add_args replace_args
+
+zpool_create_forced_degraded_add redundancy2_create_spare_args redundancy1_add_args replace_args
+zpool_create_forced_degraded_add redundancy2_create_spare_args redundancy3_add_args replace_args
+
+zpool_create_forced_degraded_add redundancy3_create_spare_args redundancy1_add_args replace_args
+zpool_create_forced_degraded_add redundancy3_create_spare_args redundancy2_add_args replace_args
+
+log_pass "Verify 'zpool add' warns for differing redundancy."
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_removal.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_removal.ksh
new file mode 100755
index 000000000000..782858e301ac
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_removal.ksh
@@ -0,0 +1,126 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2012, 2016 by Delphix. All rights reserved.
+# Copyright 2025 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
+
+#
+# DESCRIPTION:
+#	Verify zpool add succeeds when adding vdevs with matching redundancy
+#	and warns with differing redundancy after removal.
+#
+# STRATEGY:
+#	1. Create several files == $MINVDEVSIZE.
+#	2. Verify 'zpool add' warns with differing redundancy after removal.
+#
+
+verify_runnable "global"
+
+log_assert "Verify 'zpool add' warns for differing redundancy."
+log_onexit zpool_create_add_cleanup
+
+zpool_create_add_setup
+
+typeset -i i=0
+typeset -i j=0
+
+set -A redundancy1_create_args \
+	"mirror $vdev0 $vdev1" \
+	"raidz1 $vdev0 $vdev1" \
+	"draid1:1s $vdev0 $vdev1 $vdev9"
+
+set -A redundancy2_create_args \
+	"mirror $vdev0 $vdev1 $vdev2" \
+	"raidz2 $vdev0 $vdev1 $vdev2" \
+	"draid2:1s $vdev0 $vdev1 $vdev2 $vdev9"
+
+set -A redundancy3_create_args \
+	"mirror $vdev0 $vdev1 $vdev2 $vdev3" \
+	"raidz3 $vdev0 $vdev1 $vdev2 $vdev3" \
+	"draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9"
+
+set -A redundancy1_add_args \
+	"mirror $vdev5 $vdev6" \
+	"raidz1 $vdev5 $vdev6" \
+	"raidz1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \
+	"mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8" \
+	"draid1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \
+	"mirror $vdev5 $vdev6 draid1 $vdev7 $vdev8"
+
+set -A redundancy2_add_args \
+	"mirror $vdev5 $vdev6 $vdev7" \
+	"raidz2 $vdev5 $vdev6 $vdev7" \
+	"draid2 $vdev5 $vdev6 $vdev7"
+
+set -A redundancy3_add_args \
+	"mirror $vdev5 $vdev6 $vdev7 $vdev8" \
+	"raidz3 $vdev5 $vdev6 $vdev7 $vdev8" \
+	"draid3 $vdev5 $vdev6 $vdev7 $vdev8"
+
+set -A log_args "log" "$vdev_lo"
+set -A cache_args "cache" "$vdev_lo"
+set -A spare_args "spare" "$vdev_lo"
+
+
+function zpool_create_rm_add
+{
+	typeset -n create_args=$1
+	typeset -n add_args=$2
+	typeset -n rm_args=$3
+
+	i=0
+	while ((i < ${#create_args[@]})); do
+		j=0
+		while ((j < ${#add_args[@]})); do
+			log_must zpool create $TESTPOOL1 ${create_args[$i]}
+			log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
+			log_must zpool add $TESTPOOL1 ${add_args[$j]}
+			log_must zpool remove $TESTPOOL1 ${rm_args[1]}
+			log_mustnot zpool add $TESTPOOL1 ${rm_args[1]}
+			log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]}
+			log_must zpool destroy -f $TESTPOOL1
+
+			((j += 1))
+		done
+		((i += 1))
+	done
+}
+
+# 2. Verify 'zpool add' warns with differing redundancy after removal.
+zpool_create_rm_add redundancy1_create_args redundancy1_add_args log_args
+zpool_create_rm_add redundancy2_create_args redundancy2_add_args log_args
+zpool_create_rm_add redundancy3_create_args redundancy3_add_args log_args
+
+zpool_create_rm_add redundancy1_create_args redundancy1_add_args cache_args
+zpool_create_rm_add redundancy2_create_args redundancy2_add_args cache_args
+zpool_create_rm_add redundancy3_create_args redundancy3_add_args cache_args
+
+zpool_create_rm_add redundancy1_create_args redundancy1_add_args spare_args
+zpool_create_rm_add redundancy2_create_args redundancy2_add_args spare_args
+zpool_create_rm_add redundancy3_create_args redundancy3_add_args spare_args
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/cleanup.ksh
new file mode 100755
index 000000000000..099b5426031d
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/cleanup.ksh
@@ -0,0 +1,30 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+#
+. $STF_SUITE/include/libtest.shlib
+
+log_pass
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/setup.ksh
new file mode 100755
index 000000000000..3529a0ccc015
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/setup.ksh
@@ -0,0 +1,32 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+#
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_pass
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib
new file mode 100644
index 000000000000..ea4b0bd2756d
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib
@@ -0,0 +1,235 @@
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+# Since we want to make sure that iostat responds correctly as pools appear and
+# disappear, we run it in the background and capture its output to a file.
+# Once we're done, we parse the output and ensure it matches what we'd expect
+# from the operations we performed.
+#
+# Because iostat is producing output every interval, it may produce the "same"
+# output for each step of the change; in fact, we want that to make sure we
+# don't miss anything. So, we describe what we expect as a series of "chunks".
+# Each chunk is a particular kind of output, which may repeat. Current known
+# chunk types are:
+#
+# NOPOOL: the text "no pools available"
+# HEADER: three lines, starting with "capacity", "pool" and "----" respectively.
+#         (the rough shape of the normal iostat header).
+# POOL1: a line starting with "pool1" (stats line for a pool of that name)
+# POOL2: a line starting with "pool2"
+# POOLBOTH: three lines, starting with "pool1", "pool2" (either order) and
+#           "-----" respectively. (the pool stat output for multiple pools)
+#
+# (the parser may produce other chunks in a failed parse to assist with
+#  debugging, but they should never be part of the "wanted" output See the
+#  parser commentary below).
+#
+# To help recognise the start of a new interval output, we run iostat with the
+# -T u option, which will output a numeric timestamp before each header or
+# second-or-later pool stat after the header.
+#
+# To keep the test run shorter, we use a subsecond interval, but to make sure
+# nothing is missed, we sleep for three intervals after each change.
+
+typeset _iostat_out=$(mktemp)
+typeset _iostat_pid=""
+
+function cleanup_iostat {
+	if [[ -n $_iostat_pid ]] ; then
+		kill -KILL $_iostat_pid || true
+	fi
+	rm -f $_iostat_out
+}
+
+function start_iostat {
+	zpool iostat -T u $@ 0.1 > $_iostat_out 2>&1 &
+	_iostat_pid=$!
+}
+
+function stop_iostat {
+	kill -TERM $_iostat_pid
+	wait $_iostat_pid
+	_iostat_pid=""
+}
+
+function delay_iostat {
+	sleep 0.3
+}
+
+typeset -a _iostat_expect
+function expect_iostat {
+	typeset chunk=$1
+	_iostat_expect+=($chunk)
+}
+
+# Parse the output The `state` var is used to track state across
+# multiple lines. The `last` var and the `_got_iostat` function are used
+# to record the completed chunks, and to collapse repetitions.
+typeset -a _iostat_got
+typeset _iostat_last=""
+typeset _iostat_state=""
+
+function _got_iostat {
+	typeset chunk=$1
+	if [[ -n $chunk && $_iostat_last != $chunk ]] ; then
+		_iostat_last=$chunk
+		_iostat_got+=($chunk)
+	fi
+	_iostat_state=""
+}
+
+function verify_iostat {
+
+	cat $_iostat_out | while read line ; do
+
+		# The "no pools available" text has no timestamp or other
+		# header, and should never appear in the middle of multiline
+		# chunk, so we can close any in-flight state.
+		if [[ $line = "no pools available" ]] ; then
+			_got_iostat $_iostat_state
+			_got_iostat "NOPOOL"
+			continue
+		fi
+
+		# A run of digits alone on the line is a timestamp (the `-T u`
+		# switch to `iostat`). It closes any in-flight state as a
+		# complete chunk, and indicates the start of a new chunk.
+		if [[ -z ${line/#+([0-9])/} ]] ; then
+			_got_iostat $_iostat_state
+			_iostat_state="TIMESTAMP"
+			continue
+		fi
+
+		# For this test, the first word of each line should be unique,
+		# so we extract it and use it for simplicity.
+		typeset first=${line%% *}
+
+		# Header is emitted whenever the pool list changes. It has
+		# three lines:
+		#
+		#                   capacity     operations     bandwidth
+		#     pool        alloc   free   read  write   read  write
+		#     ----------  -----  -----  -----  -----  -----  -----
+		#
+		# Each line moves the state; when we get to a run of dashes, we
+		# commit.  Note that we check for one-or-more dashes, because
+		# the width can vary depending on the length of pool name.
+		#
+		if [[ $_iostat_state = "TIMESTAMP" &&
+		    $first = "capacity" ]] ; then
+			_iostat_state="INHEADER1"
+			continue
+		fi
+		if [[ $_iostat_state = "INHEADER1" &&
+		    $first = "pool" ]] ; then
+			_iostat_state="INHEADER2"
+			continue
+		fi
+		if [[ $_iostat_state = "INHEADER2" &&
+		    -z ${first/#+(-)/} ]] ; then
+			# Headers never repeat, so if the last committed chunk
+			# was a header, we commit this one as EXTRAHEADER so we
+			# can see it in the error output.
+			if [[ $_iostat_last = "HEADER" ]] ; then
+				_got_iostat "EXTRAHEADER"
+			elif [[ $_iostat_last != "EXTRAHEADER" ]] ; then
+				_got_iostat "HEADER"
+			fi
+			_iostat_state="HEADER"
+			continue
+		fi
+
+		# A pool stat line looks like:
+		#
+		#     pool1        147K   240M      0      0      0      0
+		#
+		# If there are multiple pools, iostat follows them with a
+		# separator of dashed lines:
+		#
+		#     pool1        147K   240M      0      0      0      0
+		#     pool2        147K   240M      0      0      0      0
+		#     ----------  -----  -----  -----  -----  -----  -----
+		#
+		# Stats rows always start after a timestamp or a header. If the
+		# header was emitted, we won't see a timestamp here (it goes
+		# before the header).
+		#
+		# Because our test exercises both pools on their own and
+		# together, we allow pools in either order. In practice they
+		# are sorted, but that's a side-effect of the implementation
+		# (see zpool_compare()), so we're not going to rely on it here.
+		if [[ $first = "pool1" ]] || [[ $first = "pool2" ]] ; then
+
+			# First line, track which one we saw. If it's a
+			# standalone line, it will be committed by the next
+			# NOPOOL or TIMESTAMP above (or the `_got_iostat` after
+			# the loop if this is the last line).
+			if [[ $_iostat_state == "TIMESTAMP" ||
+			    $_iostat_state == "HEADER" ]] ; then
+				if [[ $first = "pool1" ]] ; then
+					_iostat_state="POOL1"
+				elif [[ $first = "pool2" ]] ; then
+					_iostat_state="POOL2"
+				fi
+				continue
+			fi
+
+			# If this is the second pool, we're in a multi-pool
+			# block, and need to look for the separator to close it
+			# out.
+			if [[ $_iostat_state = "POOL1" && $first = "pool2" ]] ||
+			    [[ $_iostat_state = "POOL2" && $first = "pool1" ]] ;
+			    then
+				_iostat_state="INPOOLBOTH"
+				continue
+			fi
+		fi
+
+		# Separator after the stats block.
+		if [[ $_iostat_state = "INPOOLBOTH" &&
+		    -z ${first/#+(-)/} ]] ; then
+			_got_iostat "POOLBOTH"
+			continue
+		fi
+
+		# Anything else will fall through to here. We commit any
+		# in-flight state, then "UNKNOWN", all to help with debugging..
+		if [[ $_iostat_state != "UNKNOWN" ]] ; then
+			_got_iostat $_iostat_state
+			_got_iostat "UNKNOWN"
+		fi
+	done
+
+	# Close out any remaining state.
+	_got_iostat $_iostat_state
+
+	# Compare what we wanted with what we got, and pass/fail the test!
+	if [[ "${_iostat_expect[*]}" != "${_iostat_got[*]}" ]] ; then
+		log_note "expected: ${_iostat_expect[*]}"
+		log_note "     got: ${_iostat_got[*]}"
+		log_fail "zpool iostat did not produce expected output"
+	fi
+}
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh
new file mode 100755
index 000000000000..8e040058ec3e
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh
@@ -0,0 +1,90 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+# `zpool iostat <N>` should keep running and update the pools it displays as
+# pools are created/destroyed/imported/export.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib
+
+typeset vdev1=$(mktemp)
+typeset vdev2=$(mktemp)
+
+function cleanup {
+	cleanup_iostat
+
+	poolexists pool1 && destroy_pool pool1
+	poolexists pool2 && destroy_pool pool2
+	rm -f $vdev1 $vdev2
+}
+
+log_must mkfile $MINVDEVSIZE $vdev1 $vdev2
+
+expect_iostat "NOPOOL"
+
+start_iostat
+
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL1"
+log_must zpool create pool1 $vdev1
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOLBOTH"
+log_must zpool create pool2 $vdev2
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool export -a
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL2"
+log_must zpool import -d $vdev2 pool2
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOLBOTH"
+log_must zpool import -d $vdev1 pool1
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL2"
+log_must zpool destroy pool1
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool destroy pool2
+delay_iostat
+
+stop_iostat
+
+verify_iostat
+
+log_pass "zpool iostat in interval mode follows pool updates"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh
new file mode 100755
index 000000000000..ab1f258aa1cd
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh
@@ -0,0 +1,80 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+# `zpool iostat <pools> <N>` should keep running and only show the listed pools.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib
+
+typeset vdev1=$(mktemp)
+typeset vdev2=$(mktemp)
+
+function cleanup {
+	cleanup_iostat
+
+	poolexists pool1 && destroy_pool pool1
+	poolexists pool2 && destroy_pool pool2
+	rm -f $vdev1 $vdev2
+}
+
+log_must mkfile $MINVDEVSIZE $vdev1 $vdev2
+
+log_must zpool create pool1 $vdev1
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL1"
+start_iostat pool1
+delay_iostat
+
+log_must zpool create pool2 $vdev2
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool export -a
+delay_iostat
+
+log_must zpool import -d $vdev2 pool2
+delay_iostat
+
+expect_iostat "HEADER"
+expect_iostat "POOL1"
+log_must zpool import -d $vdev1 pool1
+delay_iostat
+
+expect_iostat "NOPOOL"
+log_must zpool destroy pool1
+delay_iostat
+
+log_must zpool destroy pool2
+delay_iostat
+
+stop_iostat
+
+verify_iostat
+
+log_pass "zpool iostat in interval mode with pools follows listed pool updates"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_004_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_004_pos.ksh
index 2bd9f616170b..1a9774331ba1 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_004_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_004_pos.ksh
@@ -47,6 +47,8 @@ function cleanup
 	# bring back removed disk online for further tests
 	insert_disk $REMOVED_DISK $scsi_host
 	poolexists $TESTPOOL && destroy_pool $TESTPOOL
+	# Since the disk was offline during destroy, remove the label
+	zpool labelclear $DISK2 -f
 }
 
 log_assert "Testing zpool reopen with pool name as argument"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_date_range_002.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_date_range_002.ksh
new file mode 100755
index 000000000000..9327df81a5c5
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_date_range_002.ksh
@@ -0,0 +1,76 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+# Copyright 2025 Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_scrub/zpool_scrub.cfg
+
+#
+# DESCRIPTION:
+#       Verify that the timestamp database updates all the tables as expected.
+#
+# STRATEGY:
+#     1. Decrease the note and flush frequency of the txg database.
+#     2. Force the pool to sync several txgs
+#     3. Verify that there are entries in each of the "month", "day", and
+#        "minute" tables.
+#
+
+verify_runnable "global"
+
+function cleanup
+{
+	log_must restore_tunable SPA_NOTE_TXG_TIME
+	log_must restore_tunable SPA_FLUSH_TXG_TIME
+	rm /$TESTPOOL/f1
+}
+
+log_onexit cleanup
+
+log_assert "Verifiy timestamp databases all update as expected."
+
+log_must save_tunable SPA_NOTE_TXG_TIME
+log_must set_tunable64 SPA_NOTE_TXG_TIME 1
+log_must save_tunable SPA_FLUSH_TXG_TIME
+log_must set_tunable64 SPA_FLUSH_TXG_TIME 1
+
+log_must touch /$TESTPOOL/f1
+log_must zpool sync $TESTPOOL
+sleep 1
+log_must touch /$TESTPOOL/f1
+log_must zpool sync $TESTPOOL
+sleep 1
+log_must touch /$TESTPOOL/f1
+log_must zpool sync $TESTPOOL
+
+mos_zap="$(zdb -dddd $TESTPOOL 1)"
+minutes_entries=$(echo "$mos_zap" | grep "txg_log_time:minutes" | awk '{print $5}')
+days_entries=$(echo "$mos_zap" | grep "txg_log_time:days" | awk '{print $5}')
+months_entries=$(echo "$mos_zap" | grep "txg_log_time:months" | awk '{print $5}')
+
+[[ "$minutes_entries" -ne "0" ]] || log_fail "0 entries in the minutes table"
+[[ "$days_entries" -ne "0" ]] || log_fail "0 entries in the days table"
+[[ "$months_entries" -ne "0" ]] || log_fail "0 entries in the months table"
+
+log_pass "Verified all timestamp databases had entries as expected."
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/zarcstat_001_pos.ksh
index 700bd9a6f529..d63b72f8039d 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/zarcstat_001_pos.ksh
@@ -37,7 +37,7 @@ log_assert "arcstat generates output and doesn't return an error code"
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "arcstat ${args[i]} > /dev/null"
+        log_must eval "zarcstat ${args[i]} > /dev/null"
         ((i = i + 1))
 done
 log_pass "arcstat generates output and doesn't return an error code"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/zarcsummary_001_pos.ksh
index 0840878fdb0d..b7faac5243c9 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/zarcsummary_001_pos.ksh
@@ -30,16 +30,16 @@
 
 is_freebsd && ! python3 -c 'import sysctl' 2>/dev/null && log_unsupported "python3 sysctl module missing"
 
-log_assert "arc_summary generates output and doesn't return an error code"
+log_assert "zarcsummary generates output and doesn't return an error code"
 
 # Without this, the below checks aren't going to work the way we hope...
 set -o pipefail
 
 for arg in "" "-a" "-d" "-p 1" "-g" "-s arc" "-r"; do
-	log_must eval "arc_summary $arg > /dev/null"
+	log_must eval "zarcsummary $arg > /dev/null"
 done
 
-log_must eval "arc_summary | head > /dev/null"
-log_must eval "arc_summary | head -1 > /dev/null"
+log_must eval "zarcsummary | head > /dev/null"
+log_must eval "zarcsummary | head -1 > /dev/null"
 
-log_pass "arc_summary generates output and doesn't return an error code"
+log_pass "zarcsummary generates output and doesn't return an error code"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/zarcsummary_002_neg.ksh
index ec4abe35409d..227646777ba0 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/misc/zarcsummary_002_neg.ksh
@@ -30,10 +30,10 @@
 
 is_freebsd && ! python3 -c 'import sysctl' 2>/dev/null && log_unsupported "python3 sysctl module missing"
 
-log_assert "arc_summary generates an error code with invalid options"
+log_assert "zarcsummary generates an error code with invalid options"
 
 for arg in "-x" "-5" "-p 7" "--err" "-@"; do
-        log_mustnot eval "arc_summary $arg > /dev/null"
+        log_mustnot eval "zarcsummary $arg > /dev/null"
 done
 
-log_pass "arc_summary generates an error code with invalid options"
+log_pass "zarcsummary generates an error code with invalid options"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/cleanup.ksh
new file mode 100755
index 000000000000..4a59e15cc693
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/cleanup.ksh
@@ -0,0 +1,43 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib
+
+
+poolexists $TESTPOOL1 && \
+	destroy_pool $TESTPOOL1
+
+del_user $STAFF1
+del_user $STAFF2
+del_group $STAFF_GROUP
+
+del_user $OTHER1
+del_user $OTHER2
+del_group $OTHER_GROUP
+
+default_cleanup
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/setup.ksh
new file mode 100755
index 000000000000..0978193eddc4
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/setup.ksh
@@ -0,0 +1,50 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib
+
+# Create staff group and add two user to it
+log_must add_group $STAFF_GROUP
+if ! id $STAFF1 > /dev/null 2>&1; then
+	log_must add_user $STAFF_GROUP $STAFF1
+fi
+if ! id $STAFF2 > /dev/null 2>&1; then
+	log_must add_user $STAFF_GROUP $STAFF2
+fi
+
+# Create other group and add two user to it
+log_must add_group $OTHER_GROUP
+if ! id $OTHER1 > /dev/null 2>&1; then
+	log_must add_user $OTHER_GROUP $OTHER1
+fi
+if ! id $OTHER2 > /dev/null 2>&1; then
+	log_must add_user $OTHER_GROUP $OTHER2
+fi
+DISK=${DISKS%% *}
+
+default_raidz_setup $DISKS
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh
new file mode 100755
index 000000000000..f62f2b07929c
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh
@@ -0,0 +1,145 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara Inc.
+#
+
+# STRATEGY:
+# 1. Create a pool (this is done by the test framework)
+# 2. Create a user
+# 3. Create an encrypted dataset
+# 4. Write random data to the encrypted dataset
+# 5. Snapshot the dataset
+# 6. As root: attempt a send and raw send (both should succeed)
+# 7. As user: attempt a send and raw send (both should fail, no permission)
+# 8. Create a delegation (zfs allow -u user send testpool/encrypted_dataset)
+# 9. As root: attempt a send and raw send (both should succeed)
+# 10. As user: attempt a send and raw send (both should succeed)
+# 11. Create a delegation (zfs allow -u user sendraw testpool/encrypted_dataset)
+# 12. As root: attempt a send and raw send (both should succeed)
+# 13. As user: attempt a send and raw send (send should fail, raw send should succeed)
+# 14. Disable delegation (zfs unallow)
+# 15. As root: attempt a send and raw send (both should succeed)
+# 16. As user: attempt a send and raw send (both should fail, no permission)
+# 17. Clean up (handled by framework)
+# root tests to verify this doesnt affect root user under ../cli_root/zfs_send_delegation/
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_create/zfs_create_common.kshlib
+. $STF_SUITE/tests/functional/cli_root/zfs_create/properties.kshlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+. $STF_SUITE/tests/functional/delegate/delegate.cfg
+
+# create encrypted dataset
+
+log_must eval "echo $PASSPHRASE | zfs create -o encryption=on -o keyformat=passphrase $TESTPOOL/$TESTFS1"
+
+# create target dataset for receives
+log_must zfs create $TESTPOOL/$TESTFS2
+
+# set user perms
+# need to run chown for fs permissions for $OTHER1
+typeset perms="snapshot,reservation,compression,checksum,userprop,receive,mount,create"
+
+log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS1
+log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS2
+log_must chown ${OTHER1}:${OTHER_GROUP} /$TESTPOOL/$TESTFS2
+
+# create random data
+log_must fill_fs $TESTPOOL/$TESTFS1/child 1 2047 1024 1 R
+
+# snapshot
+log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
+
+# note
+# we need to use `sh -c` here becuase the quoting on <<<"$*" in the user_run wrapper is broken once pipes and redirects get involved
+
+# check baseline send abilities (should fail)
+log_mustnot user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv0_user_datastream.$$'"
+# verify nothing went through
+if [ -s $TESTPOOL/$TESTFS2/zfsrecv0_user_datastream.$$ ]
+then
+	log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv0_user_datastream !"
+fi
+log_mustnot user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv0raw_user_datastream.$$'"
+# verify nothing went through
+if [ -s $TESTPOOL/$TESTFS2/zfsrecv0raw_user_datastream.$$ ]
+then
+	log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv0raw_user_datastream !"
+fi
+
+# create delegation
+log_must zfs allow $OTHER1 send $TESTPOOL/$TESTFS1
+
+# attempt send with full allow (should pass)
+log_must user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv1_user_datastream.$$'"
+log_must user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv1raw_user_datastream.$$'"
+
+
+# create raw delegation
+log_must zfs allow $OTHER1 send:raw $TESTPOOL/$TESTFS1
+# We have to remove 'send' to confirm 'send raw' only allows what we want
+log_must zfs unallow -u $OTHER1 send $TESTPOOL/$TESTFS1
+
+# test new sendraw abilities (send should fail, sendraw should pass)
+log_mustnot user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv2_user_datastream.$$'"
+ verify nothing went through
+if [ -s $TESTPOOL/$TESTFS2/zfsrecv2_user_datastream.$$ ]
+then
+	log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv2_user_datastream !"
+fi
+log_must user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv2raw_user_datastream.$$'"
+
+# disable raw delegation
+log_must zfs unallow -u $OTHER1 send:raw $TESTPOOL/$TESTFS1
+log_must zfs allow $OTHER1 send $TESTPOOL/$TESTFS1
+
+# test with raw taken away (should pass)
+log_must user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv3_user_datastream.$$'"
+log_must user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv3raw_user_datastream.$$'"
+
+# disable send abilities
+log_must zfs unallow -u $OTHER1 send $TESTPOOL/$TESTFS1
+
+# verify original send abilities (should fail)
+log_mustnot user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv4_user_datastream.$$'"
+ verify nothing went through
+if [ -s $TESTPOOL/$TESTFS2/zfsrecv4_user_datastream.$$ ]
+then
+        log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv4_user_datastream !"
+fi
+log_mustnot user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv4raw_user_datastream.$$'"
+ verify nothing went through
+if [ -s $TESTPOOL/$TESTFS2/zfsrecv4raw_user_datastream.$$ ]
+then
+        log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv4raw_user_datastream !"
+fi
+
+
+function cleanup
+{
+    datasetexists $TESTPOOL/$TESTFS1 && \
+	    destroy_dataset $TESTPOOL/$TESTFS1 -r \
+            destroy_dataset $TESTPOOL/$TESTFS2 -r
+
+}
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib
index 0a402e71ee68..345239b88680 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib
@@ -1234,10 +1234,10 @@ function verify_fs_aedsx
 	typeset oldval
 	set -A modes "on" "off"
 	oldval=$(get_prop $perm $fs)
-	if [[ $oldval == "on" ]]; then
-		n=1
-	elif [[ $oldval == "off" ]]; then
+	if [[ $oldval == "off" ]]; then
 		n=0
+	else
+		n=1
 	fi
 	log_note "$user zfs set $perm=${modes[$n]} $fs"
 	user_run $user zfs set $perm=${modes[$n]} $fs
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_degraded_sit_out.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_degraded_sit_out.ksh
new file mode 100755
index 000000000000..d5feb6936b4b
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_degraded_sit_out.ksh
@@ -0,0 +1,106 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+# Copyright (c) 2024 by Lawrence Livermore National Security, LLC.
+# Copyright (c) 2025 by Klara, Inc.
+
+# DESCRIPTION:
+#	Verify that vdevs 'sit out' when they are slow
+#
+# STRATEGY:
+#	1. Create various raidz/draid pools
+#	2. Degrade/fault one of the disks.
+#	3. Inject delays into one of the disks
+#	4. Verify disk is set to 'sit out' for awhile.
+#	5. Wait for READ_SIT_OUT_SECS and verify sit out state is lifted.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+function cleanup
+{
+	restore_tunable READ_SIT_OUT_SECS
+	restore_tunable SIT_OUT_CHECK_INTERVAL
+	log_must zinject -c all
+	log_must zpool events -c
+	destroy_pool $TESTPOOL2
+	log_must rm -f $TEST_BASE_DIR/vdev.$$.*
+}
+
+log_assert "Verify sit_out works"
+
+log_onexit cleanup
+
+# shorten sit out period for testing
+save_tunable READ_SIT_OUT_SECS
+set_tunable32 READ_SIT_OUT_SECS 5
+
+save_tunable SIT_OUT_CHECK_INTERVAL
+set_tunable64 SIT_OUT_CHECK_INTERVAL 20
+
+log_must truncate -s 150M $TEST_BASE_DIR/vdev.$$.{0..9}
+
+for raidtype in raidz2 raidz3 draid2 draid3 ; do
+	log_must zpool create $TESTPOOL2 $raidtype $TEST_BASE_DIR/vdev.$$.{0..9}
+	log_must zpool set autosit=on $TESTPOOL2 "${raidtype}-0"
+	log_must dd if=/dev/urandom of=/$TESTPOOL2/bigfile bs=1M count=400
+	log_must zpool export $TESTPOOL2
+	log_must zpool import -d $TEST_BASE_DIR $TESTPOOL2
+
+	BAD_VDEV=$TEST_BASE_DIR/vdev.$$.9
+	SLOW_VDEV=$TEST_BASE_DIR/vdev.$$.8
+
+	# Initial state should not be sitting out
+	log_must eval [[ "$(get_vdev_prop sit_out $TESTPOOL2 $SLOW_VDEV)" == "off" ]]
+
+	# Delay our reads 200ms to trigger sit out
+	log_must zinject -d $SLOW_VDEV -D200:1 -T read $TESTPOOL2
+	type=$((RANDOM % 2))
+	[[ "$type" -eq "0" ]] && action="degrade" || action="fault"
+	log_must zinject -d $BAD_VDEV -A $action -T read $TESTPOOL2
+
+	# Do some reads and wait for us to sit out
+	for i in {0..99} ; do
+		dd if=/$TESTPOOL2/bigfile skip=$i bs=2M count=1 of=/dev/null &
+		dd if=/$TESTPOOL2/bigfile skip=$((i + 100)) bs=2M count=1 of=/dev/null
+
+		sit_out=$(get_vdev_prop sit_out $TESTPOOL2 $SLOW_VDEV)
+		if [[ "$sit_out" == "on" ]] ; then
+			break
+		fi
+	done
+
+	log_must test "$(get_vdev_prop sit_out $TESTPOOL2 $SLOW_VDEV)" == "on"
+
+	# Clear fault injection
+	log_must zinject -c all
+
+	# Wait for us to exit our sit out period
+	log_must wait_sit_out $TESTPOOL2 $SLOW_VDEV 10
+
+	log_must test "$(get_vdev_prop sit_out $TESTPOOL2 $SLOW_VDEV)" == "off"
+	destroy_pool $TESTPOOL2
+	log_must zpool labelclear -f $BAD_VDEV
+done
+
+log_pass "sit_out works correctly"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_sit_out.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_sit_out.ksh
new file mode 100755
index 000000000000..37f616cf56ee
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_sit_out.ksh
@@ -0,0 +1,102 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+# Copyright (c) 2024 by Lawrence Livermore National Security, LLC.
+
+# DESCRIPTION:
+#	Verify that vdevs 'sit out' when they are slow
+#
+# STRATEGY:
+#	1. Create various raidz/draid pools
+#	2. Inject delays into one of the disks
+#	3. Verify disk is set to 'sit out' for awhile.
+#	4. Wait for READ_SIT_OUT_SECS and verify sit out state is lifted.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+function cleanup
+{
+	restore_tunable READ_SIT_OUT_SECS
+	restore_tunable SIT_OUT_CHECK_INTERVAL
+	log_must zinject -c all
+	log_must zpool events -c
+	destroy_pool $TESTPOOL2
+	log_must rm -f $TEST_BASE_DIR/vdev.$$.*
+}
+
+log_assert "Verify sit_out works"
+
+log_onexit cleanup
+
+# shorten sit out period for testing
+save_tunable READ_SIT_OUT_SECS
+set_tunable32 READ_SIT_OUT_SECS 5
+
+save_tunable SIT_OUT_CHECK_INTERVAL
+set_tunable64 SIT_OUT_CHECK_INTERVAL 20
+
+log_must truncate -s200M $TEST_BASE_DIR/vdev.$$.{0..9}
+
+for raidtype in raidz raidz2 raidz3 draid1 draid2 draid3 ; do
+	log_must zpool create $TESTPOOL2 $raidtype $TEST_BASE_DIR/vdev.$$.{0..9}
+	log_must zpool set autosit=on $TESTPOOL2 "${raidtype}-0"
+	log_must dd if=/dev/urandom of=/$TESTPOOL2/bigfile bs=1M count=600
+	log_must zpool export $TESTPOOL2
+	log_must zpool import -d $TEST_BASE_DIR $TESTPOOL2
+
+	BAD_VDEV=$TEST_BASE_DIR/vdev.$$.9
+
+	# Initial state should not be sitting out
+	log_must eval [[ "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV)" == "off" ]]
+
+	# Delay our reads 200ms to trigger sit out
+	log_must zinject -d $BAD_VDEV -D200:1 -T read $TESTPOOL2
+
+	# Do some reads and wait for us to sit out
+	for i in {0..99} ; do
+		dd if=/$TESTPOOL2/bigfile skip=$i bs=2M count=1 of=/dev/null &
+		dd if=/$TESTPOOL2/bigfile skip=$((i + 100)) bs=2M count=1 of=/dev/null &
+		dd if=/$TESTPOOL2/bigfile skip=$((i + 200)) bs=2M count=1 of=/dev/null
+
+		sit_out=$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV)
+		if [[ "$sit_out" == "on" ]] ; then
+			break
+		fi
+	done
+
+	log_must test "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV)" == "on"
+
+	# Clear fault injection
+	log_must zinject -c all
+
+	# Wait for us to exit our sit out period
+	log_must wait_sit_out $TESTPOOL2 $BAD_VDEV 10
+
+	# Verify sit_out was cleared during wait_sit_out
+	log_must test "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV)" == "off"
+
+	destroy_pool $TESTPOOL2
+done
+
+log_pass "sit_out works correctly"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_sit_out_neg.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_sit_out_neg.ksh
new file mode 100755
index 000000000000..457105a66453
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/slow_vdev_sit_out_neg.ksh
@@ -0,0 +1,116 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+# Copyright (c) 2024 by Lawrence Livermore National Security, LLC.
+# Copyright (c) 2025 by Klara, Inc.
+
+# DESCRIPTION:
+#	Verify that we don't sit out too many vdevs
+#
+# STRATEGY:
+#	1. Create draid2 pool
+#	2. Inject delays into three of the disks
+#	3. Do reads to trigger sit-outs
+#	4. Verify exactly 2 disks sit out
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+function cleanup
+{
+	restore_tunable READ_SIT_OUT_SECS
+	restore_tunable SIT_OUT_CHECK_INTERVAL
+	log_must zinject -c all
+	log_must zpool events -c
+	destroy_pool $TESTPOOL2
+	log_must rm -f $TEST_BASE_DIR/vdev.$$.*
+}
+
+log_assert "Verify sit_out works"
+
+log_onexit cleanup
+
+save_tunable SIT_OUT_CHECK_INTERVAL
+set_tunable64 SIT_OUT_CHECK_INTERVAL 20
+
+log_must truncate -s 150M $TEST_BASE_DIR/vdev.$$.{0..9}
+
+log_must zpool create $TESTPOOL2 draid2 $TEST_BASE_DIR/vdev.$$.{0..9}
+log_must zpool set autosit=on $TESTPOOL2 draid2-0
+log_must dd if=/dev/urandom of=/$TESTPOOL2/bigfile bs=1M count=400
+log_must zpool export $TESTPOOL2
+log_must zpool import -d $TEST_BASE_DIR $TESTPOOL2
+
+BAD_VDEV1=$TEST_BASE_DIR/vdev.$$.7
+BAD_VDEV2=$TEST_BASE_DIR/vdev.$$.8
+BAD_VDEV3=$TEST_BASE_DIR/vdev.$$.9
+
+# Initial state should not be sitting out
+log_must eval [[ "$(get_vdev_prop autosit $TESTPOOL2 draid2-0)" == "on" ]]
+log_must eval [[ "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV1)" == "off" ]]
+log_must eval [[ "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV2)" == "off" ]]
+log_must eval [[ "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV3)" == "off" ]]
+
+# Delay our reads 200ms to trigger sit out
+log_must zinject -d $BAD_VDEV1 -D200:1 -T read $TESTPOOL2
+
+# Do some reads and wait for us to sit out
+for i in {0..99} ; do
+	dd if=/$TESTPOOL2/bigfile skip=$i bs=2M count=1 of=/dev/null &
+	dd if=/$TESTPOOL2/bigfile skip=$((i + 100)) bs=2M count=1 of=/dev/null
+
+	sit_out=$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV1)
+	if [[ "$sit_out" == "on" ]] ; then
+		break
+	fi
+done
+log_must test "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV1)" == "on"
+
+log_must zinject -d $BAD_VDEV2 -D200:1 -T read $TESTPOOL2
+# Do some reads and wait for us to sit out
+for i in {0..99} ; do
+	dd if=/$TESTPOOL2/bigfile skip=$i bs=2M count=1 of=/dev/null &
+	dd if=/$TESTPOOL2/bigfile skip=$((i + 100)) bs=2M count=1 of=/dev/null
+
+	sit_out=$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV2)
+	if [[ "$sit_out" == "on" ]] ; then
+		break
+	fi
+done
+log_must test "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV2)" == "on"
+
+log_must zinject -d $BAD_VDEV3 -D200:1 -T read $TESTPOOL2
+# Do some reads and wait for us to sit out
+for i in {0..99} ; do
+	dd if=/$TESTPOOL2/bigfile skip=$i bs=2M count=1 of=/dev/null &
+	dd if=/$TESTPOOL2/bigfile skip=$((i + 100)) bs=2M count=1 of=/dev/null
+
+	sit_out=$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV3)
+	if [[ "$sit_out" == "on" ]] ; then
+		break
+	fi
+done
+log_must test "$(get_vdev_prop sit_out $TESTPOOL2 $BAD_VDEV3)" == "off"
+
+
+log_pass "sit_out works correctly"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_synchronous_zedlet.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_synchronous_zedlet.ksh
new file mode 100755
index 000000000000..6b732ea96d0c
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/zed_synchronous_zedlet.ksh
@@ -0,0 +1,149 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025 by Lawrence Livermore National Security, LLC.
+#
+
+# DESCRIPTION:
+# Verify ZED synchronous zedlets work as expected
+#
+# STRATEGY:
+# 1. Create a scrub_start zedlet that runs quickly
+# 2. Create a scrub_start zedlet that runs slowly (takes seconds)
+# 3. Create a scrub_finish zedlet that is synchronous and runs slowly
+# 4. Create a trim_start zedlet that runs quickly
+# 4. Scrub the pool
+# 5. Trim the pool
+# 6. Verify the synchronous scrub_finish zedlet waited for the scrub_start
+#    zedlets to finish (including the slow one).  If the scrub_finish zedlet
+#    was not synchronous, it would have completed before the slow scrub_start
+#    zedlet.
+# 7. Verify the trim_start zedlet waited for the slow synchronous scrub_finish
+#    zedlet to complete.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+OUR_ZEDLETS="scrub_start-async.sh scrub_start-slow.sh scrub_finish-sync-slow.sh trim_start-async.sh"
+
+OUTFILE="$TEST_BASE_DIR/zed_synchronous_zedlet_lines"
+TESTPOOL2=testpool2
+
+function cleanup
+{
+	zed_stop
+
+	for i in $OUR_ZEDLETS ; do
+		log_must rm -f $ZEDLET_DIR/$i
+	done
+	destroy_pool $TESTPOOL2
+	log_must rm -f $TEST_BASE_DIR/vdev-file-sync-zedlet
+	log_must rm -f $OUTFILE
+}
+
+log_assert "Verify ZED synchronous zedlets work as expected"
+
+log_onexit cleanup
+
+# Make a pool
+log_must truncate -s 100M $TEST_BASE_DIR/vdev-file-sync-zedlet
+log_must zpool create $TESTPOOL2 $TEST_BASE_DIR/vdev-file-sync-zedlet
+
+# Do an initial scrub
+log_must zpool scrub -w $TESTPOOL2
+
+log_must zpool events -c
+
+mkdir -p $ZEDLET_DIR
+
+# Create zedlets
+cat << EOF > $ZEDLET_DIR/scrub_start-async.sh
+#!/bin/ksh -p
+echo "\$(date) \$(basename \$0)"  >> $OUTFILE
+EOF
+
+cat << EOF > $ZEDLET_DIR/scrub_start-slow.sh
+#!/bin/ksh -p
+sleep 3
+echo "\$(date) \$(basename \$0)"  >> $OUTFILE
+EOF
+
+cat << EOF > $ZEDLET_DIR/scrub_finish-sync-slow.sh
+#!/bin/ksh -p
+sleep 3
+echo "\$(date) \$(basename \$0)"  >> $OUTFILE
+EOF
+
+cat << EOF > $ZEDLET_DIR/trim_start-async.sh
+#!/bin/ksh -p
+echo "\$(date) \$(basename \$0)"  >> $OUTFILE
+EOF
+
+for i in $OUR_ZEDLETS ; do
+	log_must chmod +x $ZEDLET_DIR/$i
+done
+
+log_must zed_start
+
+# Do a scrub - it should be instantaneous.
+log_must zpool scrub -w $TESTPOOL2
+
+# Start off a trim immediately after scrubiung.  The trim should be
+# instantaneous and generate a trimp_start event.  This will happen in parallel
+# with the slow 'scrub_finish-sync-slow.sh' zedlet still running.
+log_must zpool trim -w $TESTPOOL2
+
+# Wait for scrub_finish event to happen for sanity. This is the *event*, not
+# the completion of zedlets for the event.
+log_must file_wait_event $ZED_DEBUG_LOG 'sysevent\.fs\.zfs\.trim_finish' 10
+
+# At a minimum, scrub_start-slow.sh + scrub_finish-sync-slow.sh will take a
+# total of 6 seconds to run, so wait 7 sec to be sure.
+sleep 7
+
+# If our zedlets were run in the right order, with sync correctly honored, you
+# will see this ordering in $OUTFILE:
+#
+# Fri May 16 12:04:23 PDT 2025 scrub_start-async.sh
+# Fri May 16 12:04:26 PDT 2025 scrub_start-slow.sh
+# Fri May 16 12:04:31 PDT 2025 scrub_finish-sync-slow.sh
+# Fri May 16 12:04:31 PDT 2025 trim_start-async.sh
+#
+# Check for this ordering
+
+# Get a list of just the script names in the order they were executed
+# from OUTFILE
+lines="$(echo $(grep -Eo '(scrub|trim)_.+\.sh$' $OUTFILE))"
+
+# Compare it to the ordering we expect
+expected="\
+scrub_start-async.sh \
+scrub_start-slow.sh \
+scrub_finish-sync-slow.sh \
+trim_start-async.sh"
+log_must test "$lines" == "$expected"
+
+log_pass "Verified synchronous zedlets"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode.kshlib
new file mode 100644
index 000000000000..d0b7404557ab
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode.kshlib
@@ -0,0 +1,149 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+typeset -A failmode_sync_helper_cmd=(
+    ["fsync"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1 conv=fsync'
+    ["msync"]='mmap_write_sync DATAFILE'
+    ["osync"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1 oflag=sync'
+    ["syncalways"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1'
+)
+
+typeset -A failmode_sync_helper_dsopts=(
+    ["syncalways"]="-o sync=always"
+)
+
+function failmode_sync_cleanup
+{
+	zinject -c all || true
+	zpool clear $TESTPOOL || true
+	destroy_pool $TESTPOOL
+}
+
+#
+# failmode_sync_test <failmode> <helper>
+#
+# run a failmode sync test:
+# - failmode: wait|continue
+# - helper: fsync|msync|osync|syncalways
+#
+function failmode_sync_test
+{
+	typeset failmode=$1
+	typeset helper=$2
+
+	# we'll need two disks, one for the main pool, one for the log
+	read -r DISK1 DISK2 _ <<<"$DISKS"
+
+	# file to write to the pool
+	typeset datafile="/$TESTPOOL/$TESTFS/datafile"
+
+	# create a single-disk pool with a separate log and the wanted failmode
+	log_must zpool create \
+	    -f -o failmode=$failmode $TESTPOOL $DISK1 log $DISK2
+
+	# create the test dataset. we bias the ZIL towards the log device to
+	# try to ensure that the sync write never involves the main device
+	log_must zfs create \
+	    -o recordsize=128k -o logbias=latency \
+	    ${failmode_sync_helper_dsopts[$helper]} \
+	    $TESTPOOL/$TESTFS
+
+	# create the target file. the ZIL head structure is created on first
+	# use, and does a full txg wait to finish, which we want to avoid
+	log_must dd if=/dev/zero of=$datafile bs=128k count=1 conv=fsync
+	log_must zpool sync
+
+	# inject errors. writes will fail, as will the followup probes
+	zinject -d $DISK1 -e io -T write $TESTPOOL
+	zinject -d $DISK1 -e nxio -T probe $TESTPOOL
+	zinject -d $DISK2 -e io -T write $TESTPOOL
+	zinject -d $DISK2 -e nxio -T probe $TESTPOOL
+
+	# run the helper program in the background. the pool should immediately
+	# suspend, and the sync op block or fail based on the failmode
+	typeset helper_cmd=${failmode_sync_helper_cmd[$helper]/DATAFILE/$datafile}
+	log_note "running failmode sync helper: $helper_cmd"
+	$helper_cmd &
+	typeset -i pid=$!
+
+	# should only take a moment, but give it a chance
+	log_note "waiting for pool to suspend"
+	typeset -i tries=10
+	until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do
+		if ((tries-- == 0)); then
+			log_fail "pool didn't suspend"
+		fi
+		sleep 1
+	done
+
+	# zil_commit() should have noticed the suspend by now
+	typeset -i zilerr=$(kstat zil.zil_commit_error_count)
+
+	# see if the helper program blocked
+	typeset -i blocked
+	if kill -0 $pid ; then
+		blocked=1
+		log_note "$helper: blocked in the kernel"
+	else
+		blocked=0
+		log_note "$helper: exited while pool suspended"
+	fi
+
+	# bring the pool back online
+	zinject -c all
+	zpool clear $TESTPOOL
+
+	# program definitely exited now, get its return code
+	wait $pid
+	typeset -i rc=$?
+
+	failmode_sync_cleanup
+
+	log_note "$helper: zilerr=$zilerr blocked=$blocked rc=$rc"
+
+	# confirm expected results for the failmode
+	if [[ $failmode = "wait" ]] ; then
+		# - the ZIL saw an error, and fell back to a txg sync
+		# - sync op blocked when the pool suspended
+		# - after resume, sync op succeeded, helper returned success
+		log_must test $zilerr -ne 0
+		log_must test $blocked -eq 1
+		log_must test $rc -eq 0
+	elif [[ $failmode = "continue" ]] ; then
+		# confirm expected results:
+		# - the ZIL saw an error, and fell back to a txg sync
+		# - helper exited when the pool suspended
+		# - sync op returned an error, so helper returned failure
+		log_must test $zilerr -ne 0
+		log_must test $blocked -eq 0
+		log_must test $rc -ne 0
+	else
+		log_fail "impossible failmode: $failmode"
+	fi
+}
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_fsync_continue.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_fsync_continue.ksh
new file mode 100755
index 000000000000..7b145d3a2b4c
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_fsync_continue.ksh
@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="fsync() returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue fsync
+log_pass $desc
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_fsync_wait.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_fsync_wait.ksh
new file mode 100755
index 000000000000..677d226b5481
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_fsync_wait.ksh
@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="fsync() blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait fsync
+log_pass $desc
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_msync_continue.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_msync_continue.ksh
new file mode 100755
index 000000000000..0c79ee15a1ba
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_msync_continue.ksh
@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="msync() returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue msync
+log_pass $desc
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_msync_wait.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_msync_wait.ksh
new file mode 100755
index 000000000000..a59d8cc50d61
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_msync_wait.ksh
@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="msync() blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait msync
+log_pass $desc
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_osync_continue.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_osync_continue.ksh
new file mode 100755
index 000000000000..c4fa0c8f042c
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_osync_continue.ksh
@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="O_SYNC returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue osync
+log_pass $desc
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_osync_wait.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_osync_wait.ksh
new file mode 100755
index 000000000000..5f65cf92ad33
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_osync_wait.ksh
@@ -0,0 +1,37 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="O_SYNC blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait osync
+log_pass $desc
+
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_syncalways_continue.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_syncalways_continue.ksh
new file mode 100755
index 000000000000..b80d776224a0
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_syncalways_continue.ksh
@@ -0,0 +1,37 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="write()+sync=always returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue syncalways
+log_pass $desc
+
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_syncalways_wait.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_syncalways_wait.ksh
new file mode 100755
index 000000000000..4fcb167b5c77
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/failmode/failmode_syncalways_wait.ksh
@@ -0,0 +1,37 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="write()+sync=always blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait syncalways
+log_pass $desc
+
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh
index ef49a5d50f6c..45848bec1f5a 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh
@@ -86,7 +86,7 @@ log_must zpool set autoreplace=on $TESTPOOL
 
 # Add some data to the pool
 log_must zfs create $TESTPOOL/fs
-log_must fill_fs /$TESTPOOL/fs 4 100 4096 512 Z
+log_must fill_fs /$TESTPOOL/fs 4 100 4096 512 R
 log_must zpool export $TESTPOOL
 
 # Record the partition UUID for later comparison
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_002_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_002_pos.ksh
index a77957f32255..878b4e450340 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_002_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_replace_002_pos.ksh
@@ -119,7 +119,7 @@ log_must zpool set autoreplace=on $TESTPOOL
 
 # Add some data to the pool
 log_must zfs create $TESTPOOL/fs
-log_must fill_fs /$TESTPOOL/fs 4 100 4096 512 Z
+log_must fill_fs /$TESTPOOL/fs 4 100 4096 512 R
 log_must zpool export $TESTPOOL
 
 # Record the partition UUID for later comparison
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/fault_limits.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/fault_limits.ksh
index 1b3310edb98b..45b041503e22 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/fault_limits.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/fault_limits.ksh
@@ -67,7 +67,7 @@ log_must zpool create -f ${TESTPOOL} raidz${PARITY} ${disks[1..$((VDEV_CNT - 1))
 # Add some data to the pool
 log_must zfs create $TESTPOOL/fs
 MNTPOINT="$(get_prop mountpoint $TESTPOOL/fs)"
-log_must fill_fs $MNTPOINT $PARITY 200 32768 1000 Z
+log_must fill_fs $MNTPOINT $PARITY 200 32768 100 R
 sync_pool $TESTPOOL
 
 # Replace the last child vdev to form a replacing vdev
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/suspend_on_probe_errors.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/suspend_on_probe_errors.ksh
index 340994bb60c5..b5df1c7e37f8 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/suspend_on_probe_errors.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/suspend_on_probe_errors.ksh
@@ -101,7 +101,7 @@ sync_pool $TESTPOOL
 log_must zfs create $TESTPOOL/fs
 MNTPOINT="$(get_prop mountpoint $TESTPOOL/fs)"
 SECONDS=0
-log_must fill_fs $MNTPOINT 1 200 4096 10 Z
+log_must fill_fs $MNTPOINT 1 200 4096 10 R
 log_note "fill_fs took $SECONDS seconds"
 sync_pool $TESTPOOL
 
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/mount/mount_loopback.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/mount/mount_loopback.ksh
new file mode 100755
index 000000000000..86adef7ea032
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/mount/mount_loopback.ksh
@@ -0,0 +1,111 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+# Copyright (c) 2025 by Lawrence Livermore National Security, LLC.
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# Verify that we can make an xfs filesystem on a ZFS-backed loopback device.
+#
+# See:
+# https://github.com/openzfs/zfs/pull/17298
+# https://github.com/openzfs/zfs/issues/17277
+#
+# STRATEGY:
+# 1. Make a pool
+# 2. Make a file on the pool or create zvol
+# 3. Mount the file/zvol behind a loopback device
+# 4. Create & mount an xfs filesystem on the loopback device
+
+function cleanup
+{
+	if [ -d $TEST_BASE_DIR/mnt ] ; then
+		umount $TEST_BASE_DIR/mnt
+		log_must rmdir $TEST_BASE_DIR/mnt
+	fi
+	if [ -n "$DEV" ] ; then
+		log_must losetup -d $DEV
+	fi
+	destroy_pool $TESTPOOL2
+	log_must rm -f $TEST_BASE_DIR/file1
+}
+
+if [ ! -x "$(which mkfs.xfs)" ] ; then
+	log_unsupported "No mkfs.xfs binary"
+fi
+
+if [ ! -d /lib/modules/$(uname -r)/kernel/fs/xfs ] && \
+     ! grep -qE '\sxfs$' /proc/filesystems ; then
+	log_unsupported "No XFS kernel support"
+fi
+
+log_assert "Make an xfs filesystem on a ZFS-backed loopback device"
+log_onexit cleanup
+
+# fio options
+export NUMJOBS=2
+export RUNTIME=3
+export PERF_RANDSEED=1234
+export PERF_COMPPERCENT=66
+export PERF_COMPCHUNK=0
+export BLOCKSIZE=128K
+export SYNC_TYPE=0
+export FILE_SIZE=$(( 1024 * 1024 ))
+
+function do_test
+{
+	imgfile=$1
+	log_note "Running test on $imgfile"
+	log_must losetup -f $imgfile
+	DEV=$(losetup --associated $imgfile | grep -Eo '^/dev/loop[0-9]+')
+	log_must mkfs.xfs $DEV
+	mkdir $TEST_BASE_DIR/mnt
+	log_must mount $DEV $TEST_BASE_DIR/mnt
+	export DIRECTORY=$TEST_BASE_DIR/mnt
+
+	for d in 0 1 ; do
+		# fio options
+		export DIRECT=$d
+		log_must fio $FIO_SCRIPTS/mkfiles.fio
+		log_must fio $FIO_SCRIPTS/random_reads.fio
+	done
+	log_must umount $TEST_BASE_DIR/mnt
+	log_must rmdir $TEST_BASE_DIR/mnt
+	log_must losetup -d $DEV
+	DEV=""
+}
+
+log_must truncate -s 1G $TEST_BASE_DIR/file1
+log_must zpool create $TESTPOOL2 $TEST_BASE_DIR/file1
+log_must truncate -s 512M /$TESTPOOL2/img
+do_test /$TESTPOOL2/img
+log_must rm /$TESTPOOL2/img
+log_must zfs create -V 512M $TESTPOOL2/vol
+
+blkdev="$ZVOL_DEVDIR/$TESTPOOL2/vol"
+block_device_wait $blkdev
+do_test $blkdev
+
+log_pass "Verified xfs filesystem on a ZFS-backed loopback device"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh
index 3249bd93d5ce..a1da4a8631e1 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh
@@ -17,6 +17,7 @@
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/refreserv/refreserv.cfg
+. $STF_SUITE/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
 
 #
 # DESCRIPTION:
@@ -24,7 +25,7 @@
 #
 # STRATEGY:
 #	1. Create a pool with a single raidz vdev
-#	2. For each block size [512b, 1k, 128k] or [4k, 8k, 128k]
+#	2. For each block size [4k, 8k, 128k]
 #	    - create a volume
 #	    - fully overwrite it
 #	    - verify that referenced is less than or equal to reservation
@@ -38,6 +39,7 @@
 #	1. This test will use up to 14 disks but can cover the key concepts with
 #	   5 disks.
 #	2. If the disks are a mixture of 4Kn and 512n/512e, failures are likely.
+#	   Therefore, when creating the pool we specify 4Kn sectors.
 #
 
 verify_runnable "global"
@@ -60,29 +62,10 @@ log_onexit cleanup
 
 poolexists "$TESTPOOL" && log_must_busy zpool destroy "$TESTPOOL"
 
-# Testing tiny block sizes on ashift=12 pools causes so much size inflation
-# that small test disks may fill before creating small volumes.  However,
-# testing 512b and 1K blocks on ashift=9 pools is an ok approximation for
-# testing the problems that arise from 4K and 8K blocks on ashift=12 pools.
-if is_freebsd; then
-	bps=$(diskinfo -v ${alldisks[0]} | awk '/sectorsize/ { print $1 }')
-elif is_linux; then
-	bps=$(lsblk -nrdo min-io /dev/${alldisks[0]})
-fi
-log_must test "$bps" -eq 512 -o "$bps" -eq 4096
-case "$bps" in
-512)
-	allshifts=(9 10 17)
-	maxpct=151
-	;;
-4096)
-	allshifts=(12 13 17)
-	maxpct=110
-	;;
-*)
-	log_fail "bytes/sector: $bps != (512|4096)"
-	;;
-esac
+ashift=12
+allshifts=(12 13 17)
+maxpct=110
+
 log_note "Testing in ashift=${allshifts[0]} mode"
 
 # This loop handles all iterations of steps 1 through 4 described in strategy
@@ -99,18 +82,21 @@ for parity in 1 2 3; do
 			continue
 		fi
 
-		log_must zpool create -O compression=off "$TESTPOOL" "$raid" "${disks[@]}"
+		log_must zpool create -o ashift=$ashift "$TESTPOOL" "$raid" "${disks[@]}"
 
 		for bits in "${allshifts[@]}"; do
 			vbs=$((1 << bits))
 			log_note "Testing $raid-$ndisks volblocksize=$vbs"
 
-			vol=$TESTPOOL/$TESTVOL
+			vol=$TESTPOOL/$TESTVOL-$vbs
+			zdev=$ZVOL_DEVDIR/$vol
 			log_must zfs create -V ${volsize}m \
 			    -o volblocksize=$vbs "$vol"
-			block_device_wait "/dev/zvol/$vol"
-			log_must dd if=/dev/zero of=/dev/zvol/$vol \
-			    bs=1024k count=$volsize
+			block_device_wait $zdev
+			blockdev_exists $zdev
+
+			log_must timeout 120 dd if=/dev/urandom of=$zdev \
+			    bs=1024k count=$volsize status=progress
 			sync_pool $TESTPOOL
 
 			ref=$(zfs get -Hpo value referenced "$vol")
@@ -126,7 +112,7 @@ for parity in 1 2 3; do
 			log_must test "$deltapct" -le $maxpct
 
 			log_must_busy zfs destroy "$vol"
-			block_device_wait
+			blockdev_missing $zdev
 		done
 
 		log_must_busy zpool destroy "$TESTPOOL"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/replacement/attach_resilver_sit_out.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/replacement/attach_resilver_sit_out.ksh
new file mode 100755
index 000000000000..6820aba184b7
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/replacement/attach_resilver_sit_out.ksh
@@ -0,0 +1,189 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# DESCRIPTION:
+# 	Attaching disks while a disk is sitting out reads should pass
+#
+# STRATEGY:
+#	1. Create raidz pools
+#	2. Make one disk slower and trigger a read sit out for that disk
+#	3. Start some random I/O
+#	4. Attach a disk to the pool.
+#	5. Verify the integrity of the file system and the resilvering.
+
+verify_runnable "global"
+
+save_tunable READ_SIT_OUT_SECS
+set_tunable32 READ_SIT_OUT_SECS 120
+save_tunable SIT_OUT_CHECK_INTERVAL
+set_tunable64 SIT_OUT_CHECK_INTERVAL 20
+
+function cleanup
+{
+	restore_tunable READ_SIT_OUT_SECS
+	restore_tunable SIT_OUT_CHECK_INTERVAL
+	log_must zinject -c all
+	log_must zpool events -c
+
+	if [[ -n "$child_pids" ]]; then
+		for wait_pid in $child_pids; do
+		        kill $wait_pid
+		done
+	fi
+
+	if poolexists $TESTPOOL1; then
+		destroy_pool $TESTPOOL1
+	fi
+
+	[[ -e $TESTDIR ]] && log_must rm -rf $TESTDIR/*
+}
+
+log_assert "Replacing a disk during I/O with a sit out completes."
+
+options=""
+options_display="default options"
+
+log_onexit cleanup
+
+[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE "
+
+[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE "
+
+[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT "
+
+[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED "
+
+[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET "
+
+options="$options -r "
+
+[[ -n "$options" ]] && options_display=$options
+
+child_pids=""
+
+function attach_test
+{
+	typeset vdev=$1
+	typeset disk=$2
+
+	typeset i=0
+	while [[ $i -lt $iters ]]; do
+		log_note "Invoking file_trunc with: $options_display on $TESTFILE.$i"
+		file_trunc $options $TESTDIR/$TESTFILE.$i &
+		typeset pid=$!
+
+		sleep 1
+
+		child_pids="$child_pids $pid"
+		((i = i + 1))
+	done
+
+	# attach disk with a slow drive still present
+	SECONDS=0
+	log_must zpool attach -w $TESTPOOL1 $vdev $disk
+	log_note took $SECONDS seconds to attach disk
+
+	for wait_pid in $child_pids
+	do
+		kill $wait_pid
+	done
+	child_pids=""
+
+	log_must zinject -c all
+        log_must zpool export $TESTPOOL1
+        log_must zpool import -d $TESTDIR $TESTPOOL1
+        log_must zfs umount $TESTPOOL1/$TESTFS1
+        log_must zdb -cdui $TESTPOOL1/$TESTFS1
+        log_must zfs mount $TESTPOOL1/$TESTFS1
+	verify_pool $TESTPOOL1
+}
+
+DEVSIZE="150M"
+specials_list=""
+i=0
+while [[ $i != 10 ]]; do
+	truncate -s $DEVSIZE $TESTDIR/$TESTFILE1.$i
+	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
+
+	((i = i + 1))
+done
+
+slow_disk=$TESTDIR/$TESTFILE1.3
+log_must truncate -s $DEVSIZE $TESTDIR/$REPLACEFILE
+
+# Test file size in MB
+count=200
+
+for type in "raidz1" "raidz2" "raidz3" ; do
+	create_pool $TESTPOOL1 $type $specials_list
+	log_must zpool set autosit=on $TESTPOOL1 "${type}-0"
+	log_must zfs create -o primarycache=none -o recordsize=512K \
+	    $TESTPOOL1/$TESTFS1
+	log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
+
+	log_must dd if=/dev/urandom of=/$TESTDIR1/bigfile bs=1M count=$count
+
+	# Make one disk 100ms slower to trigger a sit out
+	log_must zinject -d $slow_disk -D100:1 -T read $TESTPOOL1
+
+	# Do some reads and wait for sit out on slow disk
+	SECONDS=0
+	typeset -i size=0
+	for i in $(seq 1 $count) ; do
+		dd if=/$TESTDIR1/bigfile skip=$i bs=1M count=1 of=/dev/null
+		size=$i
+
+		sit_out=$(get_vdev_prop sit_out $TESTPOOL1 $slow_disk)
+		if [[ "$sit_out" == "on" ]] ; then
+			break
+		fi
+	done
+
+	log_must test "$(get_vdev_prop sit_out $TESTPOOL1 $slow_disk)" == "on"
+	log_note took $SECONDS seconds to reach sit out reading ${size}M
+	log_must zpool status -s $TESTPOOL1
+
+	typeset top=$(zpool status -j | jq -r ".pools.$TESTPOOL1.vdevs[].vdevs[].name")
+	attach_test $top $TESTDIR/$REPLACEFILE
+
+	log_must eval "zpool iostat -v $TESTPOOL1 | grep \"$REPLACEFILE\""
+
+	destroy_pool $TESTPOOL1
+	log_must rm -rf /$TESTPOOL1
+done
+
+log_pass
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/replacement/replace_resilver_sit_out.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/replacement/replace_resilver_sit_out.ksh
new file mode 100755
index 000000000000..4109dbaf45ac
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/replacement/replace_resilver_sit_out.ksh
@@ -0,0 +1,199 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# DESCRIPTION:
+# 	Replacing disks while a disk is sitting out reads should pass
+#
+# STRATEGY:
+#	1. Create raidz and draid pools
+#	2. Make one disk slower and trigger a read sit out for that disk
+#	3. Start some random I/O
+#	4. Replace a disk in the pool with another disk.
+#	5. Verify the integrity of the file system and the resilvering.
+#
+
+verify_runnable "global"
+
+save_tunable READ_SIT_OUT_SECS
+set_tunable32 READ_SIT_OUT_SECS 120
+save_tunable SIT_OUT_CHECK_INTERVAL
+set_tunable64 SIT_OUT_CHECK_INTERVAL 20
+
+function cleanup
+{
+	restore_tunable READ_SIT_OUT_SECS
+	restore_tunable SIT_OUT_CHECK_INTERVAL
+	log_must zinject -c all
+	log_must zpool events -c
+
+	if [[ -n "$child_pids" ]]; then
+		for wait_pid in $child_pids
+		do
+			kill $wait_pid
+		done
+	fi
+
+	if poolexists $TESTPOOL1; then
+		destroy_pool $TESTPOOL1
+	fi
+
+	[[ -e $TESTDIR ]] && log_must rm -rf $TESTDIR/*
+}
+
+log_assert "Replacing a disk during I/O with a sit out completes."
+
+options=""
+options_display="default options"
+
+log_onexit cleanup
+
+[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE "
+
+[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE "
+
+[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT "
+
+[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED "
+
+[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET "
+
+options="$options -r "
+
+[[ -n "$options" ]] && options_display=$options
+
+child_pids=""
+
+function replace_test
+{
+	typeset -i iters=2
+	typeset disk1=$1
+	typeset disk2=$2
+	typeset repl_type=$3
+
+	typeset i=0
+	while [[ $i -lt $iters ]]; do
+		log_note "Invoking file_trunc with: $options_display on $TESTFILE.$i"
+		file_trunc $options $TESTDIR/$TESTFILE.$i &
+		typeset pid=$!
+
+		sleep 1
+
+		child_pids="$child_pids $pid"
+		((i = i + 1))
+	done
+
+	typeset repl_flag="-w"
+	if [[ "$repl_type" == "seq" ]]; then
+		repl_flag="-ws"
+	fi
+	# replace disk with a slow drive still present
+	SECONDS=0
+	log_must zpool replace $repl_flag $TESTPOOL1 $disk1 $disk2
+	log_note took $SECONDS seconds to replace disk
+
+	for wait_pid in $child_pids
+	do
+		kill $wait_pid
+	done
+	child_pids=""
+
+	log_must zinject -c all
+	log_must zpool export $TESTPOOL1
+	log_must zpool import -d $TESTDIR $TESTPOOL1
+	log_must zfs umount $TESTPOOL1/$TESTFS1
+	log_must zdb -cdui $TESTPOOL1/$TESTFS1
+	log_must zfs mount $TESTPOOL1/$TESTFS1
+	verify_pool $TESTPOOL1
+}
+
+DEVSIZE="150M"
+specials_list=""
+i=0
+while [[ $i != 10 ]]; do
+	log_must truncate -s $DEVSIZE $TESTDIR/$TESTFILE1.$i
+	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
+
+	((i = i + 1))
+done
+
+slow_disk=$TESTDIR/$TESTFILE1.3
+log_must truncate -s $DEVSIZE $TESTDIR/$REPLACEFILE
+
+# Test file size in MB
+count=400
+
+for type in "raidz2" "raidz3" "draid2"; do
+	create_pool $TESTPOOL1 $type $specials_list
+	log_must zpool set autosit=on $TESTPOOL1 "${type}-0"
+	log_must zfs create -o primarycache=none -o recordsize=512K \
+	    $TESTPOOL1/$TESTFS1
+	log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
+
+	log_must dd if=/dev/urandom of=/$TESTDIR1/bigfile bs=1M count=$count
+
+	# Make one disk 100ms slower to trigger a sit out
+	log_must zinject -d $slow_disk -D100:1 -T read $TESTPOOL1
+
+	# Do some reads and wait for sit out on slow disk
+	SECONDS=0
+	typeset -i size=0
+	for i in $(seq 1 $count) ; do
+		dd if=/$TESTDIR1/bigfile skip=$i bs=1M count=1 of=/dev/null
+		size=$i
+
+		sit_out=$(get_vdev_prop sit_out $TESTPOOL1 $slow_disk)
+		if [[ "$sit_out" == "on" ]] ; then
+			break
+		fi
+	done
+	log_must test "$(get_vdev_prop sit_out $TESTPOOL1 $slow_disk)" == "on"
+	log_note took $SECONDS seconds to reach sit out reading ${size}M
+	log_must zpool status -s $TESTPOOL1
+
+	typeset repl_type="replace"
+	if [[ "$type" == "draid2" && $((RANDOM % 2)) -eq 0 ]]; then
+		repl_type="seq"
+	fi
+	replace_test $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE $repl_type
+
+	log_must eval "zpool iostat -v $TESTPOOL1 | grep \"$REPLACEFILE\""
+
+	destroy_pool $TESTPOOL1
+	log_must rm -rf /$TESTPOOL1
+done
+
+log_pass
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh
index 26153aafbc02..0e79e9b8b70c 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh
@@ -39,6 +39,6 @@
 verify_runnable "global"
 
 # create a pool without any features
-log_must mkfile 128m $TMPDEV
+log_must truncate -s $MINVDEVSIZE $TMPDEV
 
 log_pass
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh
index d6bd69b7e134..e81d07794689 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh
@@ -35,17 +35,19 @@
 
 verify_runnable "global"
 
-TESTFILE="$TESTDIR/file.bin"
-
 log_assert "User accounting upgrade should not be executed on readonly pool"
 log_onexit cleanup_upgrade
 
 # 1. Create a pool with the feature@userobj_accounting disabled to simulate
 #    a legacy pool from a previous ZFS version.
-log_must zpool create -d -m $TESTDIR $TESTPOOL $TMPDEV
+log_must zpool create -d $TESTPOOL $TMPDEV
+log_must zfs create $TESTPOOL/$TESTFS
+
+MNTPNT=$(get_prop mountpoint $TESTPOOL/$TESTFS)
+TESTFILE="$MNTPNT/file.bin"
 
 # 2. Create a file on the "legecy" dataset
-log_must touch $TESTDIR/file.bin
+log_must touch $TESTFILE
 
 # 3. Enable feature@userobj_accounting on the pool and verify it is only
 #    "enabled" and not "active": upgrading starts when the filesystem is mounted
@@ -54,12 +56,12 @@ log_must test "enabled" == "$(get_pool_prop 'feature@userobj_accounting' $TESTPO
 
 # 4. Export the pool and re-import is readonly, without mounting any filesystem
 log_must zpool export $TESTPOOL
-log_must zpool import -o readonly=on -N -d "$(dirname $TMPDEV)" $TESTPOOL
+log_must zpool import -o readonly=on -N -d $TEST_BASE_DIR $TESTPOOL
 
 # 5. Try to mount the root dataset manually without the "ro" option, then verify
 #    filesystem status and the pool feature status (not "active") to ensure the
 #    pool "readonly" status is enforced.
-log_must mount -t zfs -o zfsutil $TESTPOOL $TESTDIR
+log_must zfs mount -R $TESTPOOL
 log_must stat "$TESTFILE"
 log_mustnot touch "$TESTFILE"
 log_must test "enabled" == "$(get_pool_prop 'feature@userobj_accounting' $TESTPOOL)"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/xattr/xattr_014_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/xattr/xattr_014_pos.ksh
new file mode 100755
index 000000000000..d4c9a0a41816
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/xattr/xattr_014_pos.ksh
@@ -0,0 +1,53 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025 by Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/xattr/xattr_common.kshlib
+
+#
+# DESCRIPTION:
+# The default xattr should be shown as 'sa', not 'on', for clarity.
+#
+# STRATEGY:
+#	1. Create a filesystem.
+#	2. Verify that the xattra is shown as 'sa'.
+#	3. Manually set the value to 'dir', 'sa', 'on', and 'off'.
+#	4. Verify that it is shown as 'dir', 'sa', 'sa', and 'off.
+#
+
+log_assert "The default and specific xattr values are displayed correctly."
+
+set -A args "dir" "sa" "on" "off"
+set -A display "dir" "sa" "sa" "off"
+
+log_must eval "[[ 'sa' == '$(zfs get -Hpo value xattr $TESTPOOL)' ]]"
+
+for i in `seq 0 3`; do
+	log_must zfs set xattr="${args[$i]}" $TESTPOOL
+	log_must eval "[[ '${display[$i]}' == '$(zfs get -Hpo value xattr $TESTPOOL)' ]]"
+done
+log_pass "The default and specific xattr values are displayed correctly."
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh
index 571a698eb63a..502ebada22dc 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh
@@ -50,17 +50,53 @@ fi
 
 typeset datafile1="$(mktemp -t zvol_misc_fua1.XXXXXX)"
 typeset datafile2="$(mktemp -t zvol_misc_fua2.XXXXXX)"
+typeset datafile3="$(mktemp -t zvol_misc_fua3_log.XXXXXX)"
 typeset zvolpath=${ZVOL_DEVDIR}/$TESTPOOL/$TESTVOL
 
+typeset DISK1=${DISKS%% *}
 function cleanup
 {
-       rm "$datafile1" "$datafile2"
+	log_must zpool remove $TESTPOOL $datafile3
+	rm "$datafile1" "$datafile2" "$datafile2"
+}
+
+# Prints the total number of sync writes for a vdev
+# $1: vdev
+function get_sync
+{
+	zpool iostat -p -H -v -r $TESTPOOL $1 | \
+	    awk '/[0-9]+$/{s+=$4+$5} END{print s}'
 }
 
 function do_test {
 	# Wait for udev to create symlinks to our zvol
 	block_device_wait $zvolpath
 
+	# Write using sync (creates FLUSH calls after writes, but not FUA)
+	old_vdev_writes=$(get_sync $DISK1)
+	old_log_writes=$(get_sync $datafile3)
+
+	log_must fio --name=write_iops --size=5M \
+		--ioengine=libaio --verify=0 --bs=4K \
+		--iodepth=1 --rw=randwrite --group_reporting=1 \
+		--filename=$zvolpath --sync=1
+
+	vdev_writes=$(( $(get_sync $DISK1) - $old_vdev_writes))
+	log_writes=$(( $(get_sync $datafile3) - $old_log_writes))
+
+	# When we're doing sync writes, we should see many more writes go to
+	# the log vs the first vdev.  Experiments show anywhere from a 160-320x
+	# ratio of writes to the log vs the first vdev (due to some straggler
+	# writes to the first vdev).
+	#
+	# Check that we have a large ratio (100x) of sync writes going to the
+	# log device
+	ratio=$(($log_writes / $vdev_writes))
+	log_note "Got $log_writes log writes, $vdev_writes vdev writes."
+	if [ $ratio -lt 100 ] ; then
+		log_fail "Expected > 100x more log writes than vdev writes. "
+	fi
+
 	# Create a data file
 	log_must dd if=/dev/urandom of="$datafile1" bs=1M count=5
 
@@ -81,6 +117,8 @@ log_assert "Verify that a ZFS volume can do Force Unit Access (FUA)"
 log_onexit cleanup
 
 log_must zfs set compression=off $TESTPOOL/$TESTVOL
+log_must truncate -s 100M $datafile3
+log_must zpool add $TESTPOOL log $datafile3
 
 log_note "Testing without blk-mq"
 
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh
index bb697b76a9f3..9de308c4a11a 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh
@@ -41,6 +41,7 @@
 # 5. TRIM the first 1MB and last 2MB of the 5MB block of data.
 # 6. Observe 2MB of used space on the zvol
 # 7. Verify the trimmed regions are zero'd on the zvol
+# 8. Verify Secure Erase does not work on zvols (Linux only)
 
 verify_runnable "global"
 
@@ -56,6 +57,7 @@ if is_linux ; then
 	else
 		trimcmd='blkdiscard'
 	fi
+	secure_trimcmd="$trimcmd --secure"
 else
 	# By default, FreeBSD 'trim' always does a dry-run.  '-f' makes
 	# it perform the actual operation.
@@ -114,6 +116,11 @@ function do_test {
 	log_must diff $datafile1 $datafile2
 
 	log_must rm $datafile1 $datafile2
+
+	# Secure erase should not work (Linux check only).
+	if [ -n "$secure_trimcmd" ] ; then
+		log_mustnot $secure_trimcmd $zvolpath
+	fi
 }
 
 log_assert "Verify that a ZFS volume can be TRIMed"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_stress/zvol_stress_destroy.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_stress/zvol_stress_destroy.ksh
new file mode 100755
index 000000000000..669b59fac01f
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_stress/zvol_stress_destroy.ksh
@@ -0,0 +1,66 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+typeset -i nzvols=1000
+typeset -i parallel=$(( $(get_num_cpus) * 2 ))
+
+function cleanup {
+  for zvol in $(zfs list -Ho name -t vol) ; do
+    log_must_busy zfs destroy $zvol
+  done
+}
+
+log_onexit cleanup
+
+log_assert "stress test concurrent zvol create/destroy"
+
+function destroy_zvols_until {
+  typeset cond=$1
+  while true ; do
+    IFS='' zfs list -Ho name -t vol | read -r -d '' zvols
+    if [[ -n $zvols ]] ; then
+      echo $zvols | xargs -n 1 -P $parallel zfs destroy
+    fi
+    if ! $cond ; then
+      break
+    fi
+  done
+}
+
+( seq $nzvols | \
+    xargs -P $parallel -I % zfs create -s -V 1G $TESTPOOL/testvol% ) &
+cpid=$!
+sleep 1
+
+destroy_zvols_until "kill -0 $cpid"
+destroy_zvols_until "false"
+
+log_pass "stress test done"
diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c
index ab8981e25cb2..0150ce72f0a4 100644
--- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c
+++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c
@@ -464,7 +464,8 @@ create_pagelist(char __user *buf, size_t count, unsigned short type,
 	    (type == PAGELIST_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, pages, num_pages);
 
 	if (actual_pages != num_pages) {
-		vm_page_unhold_pages(pages, actual_pages);
+		if (actual_pages > 0)
+			vm_page_unhold_pages(pages, actual_pages);
 		free(pagelist, M_VCPAGELIST);
 		return (-ENOMEM);
 	}
diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c
index 2e30dd7dc3de..e7459a5553e4 100644
--- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c
+++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c
@@ -392,9 +392,9 @@ make_service_callback(VCHIQ_SERVICE_T *service, VCHIQ_REASON_T reason,
 	VCHIQ_HEADER_T *header, void *bulk_userdata)
 {
 	VCHIQ_STATUS_T status;
-	vchiq_log_trace(vchiq_core_log_level, "%d: callback:%d (%s, %x, %x)",
+	vchiq_log_trace(vchiq_core_log_level, "%d: callback:%d (%s, %p, %p)",
 		service->state->id, service->localport, reason_names[reason],
-		(unsigned int)header, (unsigned int)bulk_userdata);
+		header, bulk_userdata);
 	status = service->base.callback(reason, header, service->handle,
 		bulk_userdata);
 	if (status == VCHIQ_ERROR) {
@@ -640,8 +640,8 @@ process_free_queue(VCHIQ_STATE_T *state)
 
 		rmb();
 
-		vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%x %x %x",
-			state->id, slot_index, (unsigned int)data,
+		vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%p %x %x",
+			state->id, slot_index, data,
 			local->slot_queue_recycle, slot_queue_available);
 
 		/* Initialise the bitmask for services which have used this
@@ -675,13 +675,13 @@ process_free_queue(VCHIQ_STATE_T *state)
 					vchiq_log_error(vchiq_core_log_level,
 						"service %d "
 						"message_use_count=%d "
-						"(header %x, msgid %x, "
+						"(header %p, msgid %x, "
 						"header->msgid %x, "
 						"header->size %x)",
 						port,
 						service_quota->
 							message_use_count,
-						(unsigned int)header, msgid,
+						header, msgid,
 						header->msgid,
 						header->size);
 					WARN(1, "invalid message use count\n");
@@ -704,24 +704,24 @@ process_free_queue(VCHIQ_STATE_T *state)
 						up(&service_quota->quota_event);
 						vchiq_log_trace(
 							vchiq_core_log_level,
-							"%d: pfq:%d %x@%x - "
+							"%d: pfq:%d %x@%p - "
 							"slot_use->%d",
 							state->id, port,
 							header->size,
-							(unsigned int)header,
+							header,
 							count - 1);
 					} else {
 						vchiq_log_error(
 							vchiq_core_log_level,
 								"service %d "
 								"slot_use_count"
-								"=%d (header %x"
+								"=%d (header %p"
 								", msgid %x, "
 								"header->msgid"
 								" %x, header->"
 								"size %x)",
 							port, count,
-							(unsigned int)header,
+							header,
 							msgid,
 							header->msgid,
 							header->size);
@@ -735,9 +735,9 @@ process_free_queue(VCHIQ_STATE_T *state)
 			pos += calc_stride(header->size);
 			if (pos > VCHIQ_SLOT_SIZE) {
 				vchiq_log_error(vchiq_core_log_level,
-					"pfq - pos %x: header %x, msgid %x, "
+					"pfq - pos %x: header %p, msgid %x, "
 					"header->msgid %x, header->size %x",
-					pos, (unsigned int)header, msgid,
+					pos, header, msgid,
 					header->msgid, header->size);
 				WARN(1, "invalid slot position\n");
 			}
@@ -885,17 +885,16 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 		int slot_use_count;
 
 		vchiq_log_info(vchiq_core_log_level,
-			"%d: qm %s@%x,%x (%d->%d)",
+			"%d: qm %s@%p,%x (%d->%d)",
 			state->id,
 			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
+			header, size,
 			VCHIQ_MSG_SRCPORT(msgid),
 			VCHIQ_MSG_DSTPORT(msgid));
 
 		BUG_ON(!service);
 		BUG_ON((flags & (QMFLAGS_NO_MUTEX_LOCK |
 				 QMFLAGS_NO_MUTEX_UNLOCK)) != 0);
-
 		for (i = 0, pos = 0; i < (unsigned int)count;
 			pos += elements[i++].size)
 			if (elements[i].size) {
@@ -951,9 +950,9 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 		VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);
 	} else {
 		vchiq_log_info(vchiq_core_log_level,
-			"%d: qm %s@%x,%x (%d->%d)", state->id,
+			"%d: qm %s@%p,%x (%d->%d)", state->id,
 			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
+			header, size,
 			VCHIQ_MSG_SRCPORT(msgid),
 			VCHIQ_MSG_DSTPORT(msgid));
 		if (size != 0) {
@@ -1036,9 +1035,9 @@ queue_message_sync(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 		int i, pos;
 
 		vchiq_log_info(vchiq_sync_log_level,
-			"%d: qms %s@%x,%x (%d->%d)", state->id,
+			"%d: qms %s@%p,%x (%d->%d)", state->id,
 			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
+			header, size,
 			VCHIQ_MSG_SRCPORT(msgid),
 			VCHIQ_MSG_DSTPORT(msgid));
 
@@ -1065,9 +1064,9 @@ queue_message_sync(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 		VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);
 	} else {
 		vchiq_log_info(vchiq_sync_log_level,
-			"%d: qms %s@%x,%x (%d->%d)", state->id,
+			"%d: qms %s@%p,%x (%d->%d)", state->id,
 			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
+			header, size,
 			VCHIQ_MSG_SRCPORT(msgid),
 			VCHIQ_MSG_DSTPORT(msgid));
 		if (size != 0) {
@@ -1368,26 +1367,26 @@ resolve_bulks(VCHIQ_SERVICE_T *service, VCHIQ_BULK_QUEUE_T *queue)
 				"Send Bulk to" : "Recv Bulk from";
 			if (bulk->actual != VCHIQ_BULK_ACTUAL_ABORTED)
 				vchiq_log_info(SRVTRACE_LEVEL(service),
-					"%s %c%c%c%c d:%d len:%d %x<->%x",
+					"%s %c%c%c%c d:%d len:%d %p<->%p",
 					header,
 					VCHIQ_FOURCC_AS_4CHARS(
 						service->base.fourcc),
 					service->remoteport,
 					bulk->size,
-					(unsigned int)bulk->data,
-					(unsigned int)bulk->remote_data);
+					bulk->data,
+					bulk->remote_data);
 			else
 				vchiq_log_info(SRVTRACE_LEVEL(service),
 					"%s %c%c%c%c d:%d ABORTED - tx len:%d,"
-					" rx len:%d %x<->%x",
+					" rx len:%d %p<->%p",
 					header,
 					VCHIQ_FOURCC_AS_4CHARS(
 						service->base.fourcc),
 					service->remoteport,
 					bulk->size,
 					bulk->remote_size,
-					(unsigned int)bulk->data,
-					(unsigned int)bulk->remote_data);
+					bulk->data,
+					bulk->remote_data);
 		}
 
 		vchiq_complete_bulk(bulk);
@@ -1522,8 +1521,8 @@ parse_open(VCHIQ_STATE_T *state, VCHIQ_HEADER_T *header)
 
 		fourcc = payload->fourcc;
 		vchiq_log_info(vchiq_core_log_level,
-			"%d: prs OPEN@%x (%d->'%c%c%c%c')",
-			state->id, (unsigned int)header,
+			"%d: prs OPEN@%p (%d->'%c%c%c%c')",
+			state->id, header,
 			localport,
 			VCHIQ_FOURCC_AS_4CHARS(fourcc));
 
@@ -1661,7 +1660,7 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 
 		header = (VCHIQ_HEADER_T *)(state->rx_data +
 			(state->rx_pos & VCHIQ_SLOT_MASK));
-		DEBUG_VALUE(PARSE_HEADER, (int)header);
+		DEBUG_VALUE(PARSE_HEADER, (size_t)header);
 		msgid = header->msgid;
 		DEBUG_VALUE(PARSE_MSGID, msgid);
 		size = header->size;
@@ -1695,20 +1694,20 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 					remoteport);
 				if (service)
 					vchiq_log_warning(vchiq_core_log_level,
-						"%d: prs %s@%x (%d->%d) - "
+						"%d: prs %s@%p (%d->%d) - "
 						"found connected service %d",
 						state->id, msg_type_str(type),
-						(unsigned int)header,
+						header,
 						remoteport, localport,
 						service->localport);
 			}
 
 			if (!service) {
 				vchiq_log_error(vchiq_core_log_level,
-					"%d: prs %s@%x (%d->%d) - "
+					"%d: prs %s@%p (%d->%d) - " /* XXX */
 					"invalid/closed service %d",
 					state->id, msg_type_str(type),
-					(unsigned int)header,
+					header,
 					remoteport, localport, localport);
 				goto skip_message;
 			}
@@ -1734,12 +1733,12 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 					min(16, size));
 		}
 
-		if (((unsigned int)header & VCHIQ_SLOT_MASK) + calc_stride(size)
+		if (((size_t)header & VCHIQ_SLOT_MASK) + calc_stride(size)
 			> VCHIQ_SLOT_SIZE) {
 			vchiq_log_error(vchiq_core_log_level,
-				"header %x (msgid %x) - size %x too big for "
+				"header %p (msgid %x) - size %x too big for "
 				"slot",
-				(unsigned int)header, (unsigned int)msgid,
+				header, (unsigned int)msgid,
 				(unsigned int)size);
 			WARN(1, "oversized for slot\n");
 		}
@@ -1758,8 +1757,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				service->peer_version = payload->version;
 			}
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs OPENACK@%x,%x (%d->%d) v:%d",
-				state->id, (unsigned int)header, size,
+				"%d: prs OPENACK@%p,%x (%d->%d) v:%d",
+				state->id, header, size,
 				remoteport, localport, service->peer_version);
 			if (service->srvstate ==
 				VCHIQ_SRVSTATE_OPENING) {
@@ -1776,8 +1775,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			WARN_ON(size != 0); /* There should be no data */
 
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs CLOSE@%x (%d->%d)",
-				state->id, (unsigned int)header,
+				"%d: prs CLOSE@%p (%d->%d)",
+				state->id, header,
 				remoteport, localport);
 
 			mark_service_closing_internal(service, 1);
@@ -1794,8 +1793,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_DATA:
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs DATA@%x,%x (%d->%d)",
-				state->id, (unsigned int)header, size,
+				"%d: prs DATA@%p,%x (%d->%d)",
+				state->id, header, size,
 				remoteport, localport);
 
 			if ((service->remoteport == remoteport)
@@ -1819,8 +1818,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_CONNECT:
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs CONNECT@%x",
-				state->id, (unsigned int)header);
+				"%d: prs CONNECT@%p",
+				state->id, header);
 			state->version_common = ((VCHIQ_SLOT_ZERO_T *)
 						 state->slot_data)->version;
 			up(&state->connect);
@@ -1854,12 +1853,12 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				wmb();
 
 				vchiq_log_info(vchiq_core_log_level,
-					"%d: prs %s@%x (%d->%d) %x@%x",
+					"%d: prs %s@%p (%d->%d) %x@%p",
 					state->id, msg_type_str(type),
-					(unsigned int)header,
+					header,
 					remoteport, localport,
 					bulk->remote_size,
-					(unsigned int)bulk->remote_data);
+					bulk->remote_data);
 
 				queue->remote_insert++;
 
@@ -1912,10 +1911,10 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				if ((int)(queue->remote_insert -
 					queue->local_insert) >= 0) {
 					vchiq_log_error(vchiq_core_log_level,
-						"%d: prs %s@%x (%d->%d) "
+						"%d: prs %s@%p (%d->%d) "
 						"unexpected (ri=%d,li=%d)",
 						state->id, msg_type_str(type),
-						(unsigned int)header,
+						header,
 						remoteport, localport,
 						queue->remote_insert,
 						queue->local_insert);
@@ -1932,11 +1931,11 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				queue->remote_insert++;
 
 				vchiq_log_info(vchiq_core_log_level,
-					"%d: prs %s@%x (%d->%d) %x@%x",
+					"%d: prs %s@%p (%d->%d) %x@%p",
 					state->id, msg_type_str(type),
-					(unsigned int)header,
+					header,
 					remoteport, localport,
-					bulk->actual, (unsigned int)bulk->data);
+					bulk->actual, bulk->data);
 
 				vchiq_log_trace(vchiq_core_log_level,
 					"%d: prs:%d %cx li=%x ri=%x p=%x",
@@ -1958,14 +1957,14 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_PADDING:
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: prs PADDING@%x,%x",
-				state->id, (unsigned int)header, size);
+				"%d: prs PADDING@%p,%x",
+				state->id, header, size);
 			break;
 		case VCHIQ_MSG_PAUSE:
 			/* If initiated, signal the application thread */
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: prs PAUSE@%x,%x",
-				state->id, (unsigned int)header, size);
+				"%d: prs PAUSE@%p,%x",
+				state->id, header, size);
 			if (state->conn_state == VCHIQ_CONNSTATE_PAUSED) {
 				vchiq_log_error(vchiq_core_log_level,
 					"%d: PAUSE received in state PAUSED",
@@ -1988,8 +1987,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_RESUME:
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: prs RESUME@%x,%x",
-				state->id, (unsigned int)header, size);
+				"%d: prs RESUME@%p,%x",
+				state->id, header, size);
 			/* Release the slot mutex */
 			lmutex_unlock(&state->slot_mutex);
 			if (state->is_master)
@@ -2010,8 +2009,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 
 		default:
 			vchiq_log_error(vchiq_core_log_level,
-				"%d: prs invalid msgid %x@%x,%x",
-				state->id, msgid, (unsigned int)header, size);
+				"%d: prs invalid msgid %x@%p,%x",
+				state->id, msgid, header, size);
 			WARN(1, "invalid message\n");
 			break;
 		}
@@ -2179,10 +2178,10 @@ sync_func(void *v)
 
 		if (!service) {
 			vchiq_log_error(vchiq_sync_log_level,
-				"%d: sf %s@%x (%d->%d) - "
+				"%d: sf %s@%p (%d->%d) - "
 				"invalid/closed service %d",
 				state->id, msg_type_str(type),
-				(unsigned int)header,
+				header,
 				remoteport, localport, localport);
 			release_message_sync(state, header);
 			continue;
@@ -2213,8 +2212,8 @@ sync_func(void *v)
 				service->peer_version = payload->version;
 			}
 			vchiq_log_info(vchiq_sync_log_level,
-				"%d: sf OPENACK@%x,%x (%d->%d) v:%d",
-				state->id, (unsigned int)header, size,
+				"%d: sf OPENACK@%p,%x (%d->%d) v:%d",
+				state->id, header, size,
 				remoteport, localport, service->peer_version);
 			if (service->srvstate == VCHIQ_SRVSTATE_OPENING) {
 				service->remoteport = remoteport;
@@ -2228,8 +2227,8 @@ sync_func(void *v)
 
 		case VCHIQ_MSG_DATA:
 			vchiq_log_trace(vchiq_sync_log_level,
-				"%d: sf DATA@%x,%x (%d->%d)",
-				state->id, (unsigned int)header, size,
+				"%d: sf DATA@%p,%x (%d->%d)",
+				state->id, header, size,
 				remoteport, localport);
 
 			if ((service->remoteport == remoteport) &&
@@ -2248,8 +2247,8 @@ sync_func(void *v)
 
 		default:
 			vchiq_log_error(vchiq_sync_log_level,
-				"%d: sf unexpected msgid %x@%x,%x",
-				state->id, msgid, (unsigned int)header, size);
+				"%d: sf unexpected msgid %x@%p,%x",
+				state->id, msgid, header, size);
 			release_message_sync(state, header);
 			break;
 		}
@@ -2334,8 +2333,8 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	if (slot_zero->magic != VCHIQ_MAGIC) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("Invalid VCHIQ magic value found.");
-		vchiq_loud_error("slot_zero=%x: magic=%x (expected %x)",
-			(unsigned int)slot_zero, slot_zero->magic, VCHIQ_MAGIC);
+		vchiq_loud_error("slot_zero=%p: magic=%x (expected %x)",
+			slot_zero, slot_zero->magic, VCHIQ_MAGIC);
 		vchiq_loud_error_footer();
 		return VCHIQ_ERROR;
 	}
@@ -2348,9 +2347,9 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	if (slot_zero->version < VCHIQ_VERSION_MIN) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("Incompatible VCHIQ versions found.");
-		vchiq_loud_error("slot_zero=%x: VideoCore version=%d "
+		vchiq_loud_error("slot_zero=%p: VideoCore version=%d "
 			"(minimum %d)",
-			(unsigned int)slot_zero, slot_zero->version,
+			slot_zero, slot_zero->version,
 			VCHIQ_VERSION_MIN);
 		vchiq_loud_error("Restart with a newer VideoCore image.");
 		vchiq_loud_error_footer();
@@ -2360,9 +2359,9 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	if (VCHIQ_VERSION < slot_zero->version_min) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("Incompatible VCHIQ versions found.");
-		vchiq_loud_error("slot_zero=%x: version=%d (VideoCore "
+		vchiq_loud_error("slot_zero=%p: version=%d (VideoCore "
 			"minimum %d)",
-			(unsigned int)slot_zero, VCHIQ_VERSION,
+			slot_zero, VCHIQ_VERSION,
 			slot_zero->version_min);
 		vchiq_loud_error("Restart with a newer kernel.");
 		vchiq_loud_error_footer();
@@ -2375,25 +2374,25 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 		 (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE)) {
 		vchiq_loud_error_header();
 		if (slot_zero->slot_zero_size != sizeof(VCHIQ_SLOT_ZERO_T))
-			vchiq_loud_error("slot_zero=%x: slot_zero_size=%x "
+			vchiq_loud_error("slot_zero=%p: slot_zero_size=%x "
 				"(expected %zx)",
-				(unsigned int)slot_zero,
+				slot_zero,
 				slot_zero->slot_zero_size,
 				sizeof(VCHIQ_SLOT_ZERO_T));
 		if (slot_zero->slot_size != VCHIQ_SLOT_SIZE)
-			vchiq_loud_error("slot_zero=%x: slot_size=%d "
+			vchiq_loud_error("slot_zero=%p: slot_size=%d "
 				"(expected %d",
-				(unsigned int)slot_zero, slot_zero->slot_size,
+				slot_zero, slot_zero->slot_size,
 				VCHIQ_SLOT_SIZE);
 		if (slot_zero->max_slots != VCHIQ_MAX_SLOTS)
-			vchiq_loud_error("slot_zero=%x: max_slots=%d "
+			vchiq_loud_error("slot_zero=%p: max_slots=%d "
 				"(expected %d)",
-				(unsigned int)slot_zero, slot_zero->max_slots,
+				slot_zero, slot_zero->max_slots,
 				VCHIQ_MAX_SLOTS);
 		if (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE)
-			vchiq_loud_error("slot_zero=%x: max_slots_per_side=%d "
+			vchiq_loud_error("slot_zero=%p: max_slots_per_side=%d "
 				"(expected %d)",
-				(unsigned int)slot_zero,
+				slot_zero,
 				slot_zero->max_slots_per_side,
 				VCHIQ_MAX_SLOTS_PER_SIDE);
 		vchiq_loud_error_footer();
@@ -2775,18 +2774,18 @@ release_service_messages(VCHIQ_SERVICE_T *service)
 				if ((port == service->localport) &&
 					(msgid & VCHIQ_MSGID_CLAIMED)) {
 					vchiq_log_info(vchiq_core_log_level,
-						"  fsi - hdr %x",
-						(unsigned int)header);
+						"  fsi - hdr %p",
+						header);
 					release_slot(state, slot_info, header,
 						NULL);
 				}
 				pos += calc_stride(header->size);
 				if (pos > VCHIQ_SLOT_SIZE) {
 					vchiq_log_error(vchiq_core_log_level,
-						"fsi - pos %x: header %x, "
+						"fsi - pos %x: header %p, "
 						"msgid %x, header->msgid %x, "
 						"header->size %x",
-						pos, (unsigned int)header,
+						pos, header,
 						msgid, header->msgid,
 						header->size);
 					WARN(1, "invalid slot position\n");
@@ -3360,10 +3359,10 @@ vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
 	wmb();
 
 	vchiq_log_info(vchiq_core_log_level,
-		"%d: bt (%d->%d) %cx %x@%x %x",
+		"%d: bt (%d->%d) %cx %x@%p %p",
 		state->id,
 		service->localport, service->remoteport, dir_char,
-		size, (unsigned int)bulk->data, (unsigned int)userdata);
+		size, bulk->data, userdata);
 
 	/* The slot mutex must be held when the service is being closed, so
 	   claim it here to ensure that isn't happening */
@@ -3710,12 +3709,12 @@ vchiq_dump_state(void *dump_context, VCHIQ_STATE_T *state)
 	vchiq_dump(dump_context, buf, len + 1);
 
 	len = snprintf(buf, sizeof(buf),
-		"  tx_pos=%x(@%x), rx_pos=%x(@%x)",
+		"  tx_pos=%x(@%p), rx_pos=%x(@%p)",
 		state->local->tx_pos,
-		(uint32_t)state->tx_data +
+		state->tx_data +
 			(state->local_tx_pos & VCHIQ_SLOT_MASK),
 		state->rx_pos,
-		(uint32_t)state->rx_data +
+		state->rx_data +
 			(state->rx_pos & VCHIQ_SLOT_MASK));
 	vchiq_dump(dump_context, buf, len + 1);
 
@@ -3817,21 +3816,21 @@ vchiq_dump_service_state(void *dump_context, VCHIQ_SERVICE_T *service)
 			vchiq_dump(dump_context, buf, len + 1);
 
 			len = snprintf(buf, sizeof(buf),
-				"  Ctrl: tx_count=%d, tx_bytes=%llu, "
-				"rx_count=%d, rx_bytes=%llu",
+				"  Ctrl: tx_count=%d, tx_bytes=%ju, "
+				"rx_count=%d, rx_bytes=%ju",
 				service->stats.ctrl_tx_count,
-				service->stats.ctrl_tx_bytes,
+				(uintmax_t) service->stats.ctrl_tx_bytes,
 				service->stats.ctrl_rx_count,
-				service->stats.ctrl_rx_bytes);
+				(uintmax_t) service->stats.ctrl_rx_bytes);
 			vchiq_dump(dump_context, buf, len + 1);
 
 			len = snprintf(buf, sizeof(buf),
-				"  Bulk: tx_count=%d, tx_bytes=%llu, "
-				"rx_count=%d, rx_bytes=%llu",
+				"  Bulk: tx_count=%d, tx_bytes=%ju, "
+				"rx_count=%d, rx_bytes=%ju",
 				service->stats.bulk_tx_count,
-				service->stats.bulk_tx_bytes,
+				(uintmax_t) service->stats.bulk_tx_bytes,
 				service->stats.bulk_rx_count,
-				service->stats.bulk_rx_bytes);
+				(uintmax_t) service->stats.bulk_rx_bytes);
 			vchiq_dump(dump_context, buf, len + 1);
 
 			len = snprintf(buf, sizeof(buf),
diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c
index 1f849a09d854..4eddcf3b43b2 100644
--- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c
+++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c
@@ -151,9 +151,9 @@ VCHIQ_STATUS_T vchiq_shutdown(VCHIQ_INSTANCE_T instance)
 					list);
 			list_del(pos);
 			vchiq_log_info(vchiq_arm_log_level,
-					"bulk_waiter - cleaned up %x "
+					"bulk_waiter - cleaned up %p "
 					"for pid %d",
-					(unsigned int)waiter, waiter->pid);
+					waiter, waiter->pid);
 			_sema_destroy(&waiter->bulk_waiter.event);
 
 			kfree(waiter);
@@ -454,8 +454,8 @@ vchiq_blocking_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *data,
 		list_add(&waiter->list, &instance->bulk_waiter_list);
 		lmutex_unlock(&instance->bulk_waiter_list_mutex);
 		vchiq_log_info(vchiq_arm_log_level,
-				"saved bulk_waiter %x for pid %d",
-				(unsigned int)waiter, current->p_pid);
+				"saved bulk_waiter %p for pid %d",
+				waiter, current->p_pid);
 	}
 
 	return status;