1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
|
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2013-2015, 2024 The FreeBSD Foundation
*
* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
* under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef __X86_IOMMU_X86_IOMMU_H
#define __X86_IOMMU_X86_IOMMU_H
/* Both Intel and AMD are not too crazy to have different sizes. */
typedef struct iommu_pte {
uint64_t pte;
} iommu_pte_t;
#define IOMMU_PAGE_SIZE PAGE_SIZE
#define IOMMU_PAGE_MASK (IOMMU_PAGE_SIZE - 1)
#define IOMMU_PAGE_SHIFT PAGE_SHIFT
#define IOMMU_NPTEPG (IOMMU_PAGE_SIZE / sizeof(iommu_pte_t))
#define IOMMU_NPTEPGSHIFT 9
#define IOMMU_PTEMASK (IOMMU_NPTEPG - 1)
struct sf_buf;
struct vm_object;
struct vm_page *iommu_pgalloc(struct vm_object *obj, vm_pindex_t idx,
int flags);
void iommu_pgfree(struct vm_object *obj, vm_pindex_t idx, int flags,
struct iommu_map_entry *entry);
void *iommu_map_pgtbl(struct vm_object *obj, vm_pindex_t idx, int flags,
struct sf_buf **sf);
void iommu_unmap_pgtbl(struct sf_buf *sf);
extern iommu_haddr_t iommu_high;
extern int iommu_tbl_pagecnt;
extern int iommu_qi_batch_coalesce;
SYSCTL_DECL(_hw_iommu);
struct x86_unit_common;
struct x86_iommu {
struct x86_unit_common *(*get_x86_common)(struct
iommu_unit *iommu);
void (*unit_pre_instantiate_ctx)(struct iommu_unit *iommu);
void (*qi_ensure)(struct iommu_unit *unit, int descr_count);
void (*qi_emit_wait_descr)(struct iommu_unit *unit, uint32_t seq,
bool, bool, bool);
void (*qi_advance_tail)(struct iommu_unit *unit);
void (*qi_invalidate_emit)(struct iommu_domain *idomain,
iommu_gaddr_t base, iommu_gaddr_t size, struct iommu_qi_genseq *
pseq, bool emit_wait);
void (*domain_unload_entry)(struct iommu_map_entry *entry, bool free,
bool cansleep);
void (*domain_unload)(struct iommu_domain *iodom,
struct iommu_map_entries_tailq *entries, bool cansleep);
struct iommu_ctx *(*get_ctx)(struct iommu_unit *iommu,
device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init);
void (*free_ctx_locked)(struct iommu_unit *iommu,
struct iommu_ctx *context);
struct iommu_unit *(*find)(device_t dev, bool verbose);
int (*alloc_msi_intr)(device_t src, u_int *cookies, u_int count);
int (*map_msi_intr)(device_t src, u_int cpu, u_int vector,
u_int cookie, uint64_t *addr, uint32_t *data);
int (*unmap_msi_intr)(device_t src, u_int cookie);
int (*map_ioapic_intr)(u_int ioapic_id, u_int cpu, u_int vector,
bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi,
uint32_t *lo);
int (*unmap_ioapic_intr)(u_int ioapic_id, u_int *cookie);
};
void set_x86_iommu(struct x86_iommu *);
struct x86_iommu *get_x86_iommu(void);
struct iommu_msi_data {
int irq;
int irq_rid;
struct resource *irq_res;
void *intr_handle;
int (*handler)(void *);
int msi_data_reg;
int msi_addr_reg;
int msi_uaddr_reg;
uint64_t msi_addr;
uint32_t msi_data;
void (*enable_intr)(struct iommu_unit *);
void (*disable_intr)(struct iommu_unit *);
const char *name;
};
#define IOMMU_MAX_MSI 3
struct x86_unit_common {
uint32_t qi_buf_maxsz;
uint32_t qi_cmd_sz;
char *inv_queue;
vm_size_t inv_queue_size;
uint32_t inv_queue_avail;
uint32_t inv_queue_tail;
/*
* Hw writes there on completion of wait descriptor
* processing. Intel writes 4 bytes, while AMD does the
* 8-bytes write. Due to little-endian, and use of 4-byte
* sequence numbers, the difference does not matter for us.
*/
volatile uint64_t inv_waitd_seq_hw;
uint64_t inv_waitd_seq_hw_phys;
uint32_t inv_waitd_seq; /* next sequence number to use for wait descr */
u_int inv_waitd_gen; /* seq number generation AKA seq overflows */
u_int inv_seq_waiters; /* count of waiters for seq */
u_int inv_queue_full; /* informational counter */
/*
* Delayed freeing of map entries queue processing:
*
* tlb_flush_head and tlb_flush_tail are used to implement a FIFO
* queue that supports concurrent dequeues and enqueues. However,
* there can only be a single dequeuer (accessing tlb_flush_head) and
* a single enqueuer (accessing tlb_flush_tail) at a time. Since the
* unit's qi_task is the only dequeuer, it can access tlb_flush_head
* without any locking. In contrast, there may be multiple enqueuers,
* so the enqueuers acquire the iommu unit lock to serialize their
* accesses to tlb_flush_tail.
*
* In this FIFO queue implementation, the key to enabling concurrent
* dequeues and enqueues is that the dequeuer never needs to access
* tlb_flush_tail and the enqueuer never needs to access
* tlb_flush_head. In particular, tlb_flush_head and tlb_flush_tail
* are never NULL, so neither a dequeuer nor an enqueuer ever needs to
* update both. Instead, tlb_flush_head always points to a "zombie"
* struct, which previously held the last dequeued item. Thus, the
* zombie's next field actually points to the struct holding the first
* item in the queue. When an item is dequeued, the current zombie is
* finally freed, and the struct that held the just dequeued item
* becomes the new zombie. When the queue is empty, tlb_flush_tail
* also points to the zombie.
*/
struct iommu_map_entry *tlb_flush_head;
struct iommu_map_entry *tlb_flush_tail;
struct task qi_task;
struct taskqueue *qi_taskqueue;
struct iommu_msi_data intrs[IOMMU_MAX_MSI];
};
void iommu_domain_free_entry(struct iommu_map_entry *entry, bool free);
void iommu_qi_emit_wait_seq(struct iommu_unit *unit, struct iommu_qi_genseq *
pseq, bool emit_wait);
void iommu_qi_wait_for_seq(struct iommu_unit *unit, const struct
iommu_qi_genseq *gseq, bool nowait);
void iommu_qi_drain_tlb_flush(struct iommu_unit *unit);
void iommu_qi_invalidate_locked(struct iommu_domain *domain,
struct iommu_map_entry *entry, bool emit_wait);
void iommu_qi_invalidate_sync(struct iommu_domain *domain, iommu_gaddr_t base,
iommu_gaddr_t size, bool cansleep);
void iommu_qi_common_init(struct iommu_unit *unit, task_fn_t taskfunc);
void iommu_qi_common_fini(struct iommu_unit *unit, void (*disable_qi)(
struct iommu_unit *));
int iommu_alloc_irq(struct iommu_unit *unit, int idx);
void iommu_release_intr(struct iommu_unit *unit, int idx);
void iommu_device_tag_init(struct iommu_ctx *ctx, device_t dev);
void iommu_device_set_iommu_prop(device_t dev, device_t iommu);
int pglvl_pgtbl_pte_off(int pglvl, iommu_gaddr_t base, int lvl);
vm_pindex_t pglvl_pgtbl_get_pindex(int pglvl, iommu_gaddr_t base, int lvl);
vm_pindex_t pglvl_max_pages(int pglvl);
iommu_gaddr_t pglvl_page_size(int total_pglvl, int lvl);
void iommu_db_print_domain_entry(const struct iommu_map_entry *entry);
void iommu_db_print_ctx(struct iommu_ctx *ctx);
void iommu_db_domain_print_contexts(struct iommu_domain *iodom);
void iommu_db_domain_print_mappings(struct iommu_domain *iodom);
#endif
|
