aboutsummaryrefslogtreecommitdiff
path: root/sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c
blob: ab82b2aaeb78b4a0f5e511fe620aff7bc9348887 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
 * Copyright (c) 2016 by Delphix. All rights reserved.
 */

/*
 * See abd.c for a general overview of the arc buffered data (ABD).
 *
 * Using a large proportion of scattered ABDs decreases ARC fragmentation since
 * when we are at the limit of allocatable space, using equal-size chunks will
 * allow us to quickly reclaim enough space for a new large allocation (assuming
 * it is also scattered).
 *
 * ABDs are allocated scattered by default unless the caller uses
 * abd_alloc_linear() or zfs_abd_scatter_enabled is disabled.
 */

#include <sys/abd_impl.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/zio.h>
#include <sys/zfs_context.h>
#include <sys/zfs_znode.h>

typedef struct abd_stats {
	kstat_named_t abdstat_struct_size;
	kstat_named_t abdstat_scatter_cnt;
	kstat_named_t abdstat_scatter_data_size;
	kstat_named_t abdstat_scatter_chunk_waste;
	kstat_named_t abdstat_linear_cnt;
	kstat_named_t abdstat_linear_data_size;
} abd_stats_t;

static abd_stats_t abd_stats = {
	/* Amount of memory occupied by all of the abd_t struct allocations */
	{ "struct_size",			KSTAT_DATA_UINT64 },
	/*
	 * The number of scatter ABDs which are currently allocated, excluding
	 * ABDs which don't own their data (for instance the ones which were
	 * allocated through abd_get_offset()).
	 */
	{ "scatter_cnt",			KSTAT_DATA_UINT64 },
	/* Amount of data stored in all scatter ABDs tracked by scatter_cnt */
	{ "scatter_data_size",			KSTAT_DATA_UINT64 },
	/*
	 * The amount of space wasted at the end of the last chunk across all
	 * scatter ABDs tracked by scatter_cnt.
	 */
	{ "scatter_chunk_waste",		KSTAT_DATA_UINT64 },
	/*
	 * The number of linear ABDs which are currently allocated, excluding
	 * ABDs which don't own their data (for instance the ones which were
	 * allocated through abd_get_offset() and abd_get_from_buf()). If an
	 * ABD takes ownership of its buf then it will become tracked.
	 */
	{ "linear_cnt",				KSTAT_DATA_UINT64 },
	/* Amount of data stored in all linear ABDs tracked by linear_cnt */
	{ "linear_data_size",			KSTAT_DATA_UINT64 },
};

/*
 * The size of the chunks ABD allocates. Because the sizes allocated from the
 * kmem_cache can't change, this tunable can only be modified at boot. Changing
 * it at runtime would cause ABD iteration to work incorrectly for ABDs which
 * were allocated with the old size, so a safeguard has been put in place which
 * will cause the machine to panic if you change it and try to access the data
 * within a scattered ABD.
 */
size_t zfs_abd_chunk_size = 4096;

#if defined(_KERNEL)
SYSCTL_DECL(_vfs_zfs);

SYSCTL_INT(_vfs_zfs, OID_AUTO, abd_scatter_enabled, CTLFLAG_RWTUN,
	&zfs_abd_scatter_enabled, 0, "Enable scattered ARC data buffers");
SYSCTL_ULONG(_vfs_zfs, OID_AUTO, abd_chunk_size, CTLFLAG_RDTUN,
	&zfs_abd_chunk_size, 0, "The size of the chunks ABD allocates");
#endif

kmem_cache_t *abd_chunk_cache;
static kstat_t *abd_ksp;

/*
 * We use a scattered SPA_MAXBLOCKSIZE sized ABD whose chunks are
 * just a single zero'd sized zfs_abd_chunk_size buffer. This
 * allows us to conserve memory by only using a single zero buffer
 * for the scatter chunks.
 */
abd_t *abd_zero_scatter = NULL;
static char *abd_zero_buf = NULL;

static void
abd_free_chunk(void *c)
{
	kmem_cache_free(abd_chunk_cache, c);
}

static uint_t
abd_chunkcnt_for_bytes(size_t size)
{
	return (P2ROUNDUP(size, zfs_abd_chunk_size) / zfs_abd_chunk_size);
}

static inline uint_t
abd_scatter_chunkcnt(abd_t *abd)
{
	ASSERT(!abd_is_linear(abd));
	return (abd_chunkcnt_for_bytes(
	    ABD_SCATTER(abd).abd_offset + abd->abd_size));
}

boolean_t
abd_size_alloc_linear(size_t size)
{
	return (size <= zfs_abd_chunk_size ? B_TRUE : B_FALSE);
}

void
abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op)
{
	uint_t n = abd_scatter_chunkcnt(abd);
	ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
	int waste = n * zfs_abd_chunk_size - abd->abd_size;
	if (op == ABDSTAT_INCR) {
		ABDSTAT_BUMP(abdstat_scatter_cnt);
		ABDSTAT_INCR(abdstat_scatter_data_size, abd->abd_size);
		ABDSTAT_INCR(abdstat_scatter_chunk_waste, waste);
		arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE);
	} else {
		ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
		ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size);
		ABDSTAT_INCR(abdstat_scatter_chunk_waste, -waste);
		arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE);
	}
}

void
abd_update_linear_stats(abd_t *abd, abd_stats_op_t op)
{
	ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
	if (op == ABDSTAT_INCR) {
		ABDSTAT_BUMP(abdstat_linear_cnt);
		ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size);
	} else {
		ABDSTAT_BUMPDOWN(abdstat_linear_cnt);
		ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
	}
}

void
abd_verify_scatter(abd_t *abd)
{
	uint_t i, n;

	/*
	 * There is no scatter linear pages in FreeBSD so there is an
	 * if an error if the ABD has been marked as a linear page.
	 */
	ASSERT(!abd_is_linear_page(abd));
	ASSERT3U(ABD_SCATTER(abd).abd_offset, <,
	    zfs_abd_chunk_size);
	n = abd_scatter_chunkcnt(abd);
	for (i = 0; i < n; i++) {
		ASSERT3P(ABD_SCATTER(abd).abd_chunks[i], !=, NULL);
	}
}

void
abd_alloc_chunks(abd_t *abd, size_t size)
{
	uint_t i, n;

	n = abd_chunkcnt_for_bytes(size);
	for (i = 0; i < n; i++) {
		void *c = kmem_cache_alloc(abd_chunk_cache, KM_PUSHPAGE);
		ASSERT3P(c, !=, NULL);
		ABD_SCATTER(abd).abd_chunks[i] = c;
	}
	ABD_SCATTER(abd).abd_chunk_size = zfs_abd_chunk_size;
}

void
abd_free_chunks(abd_t *abd)
{
	uint_t i, n;

	n = abd_scatter_chunkcnt(abd);
	for (i = 0; i < n; i++) {
		abd_free_chunk(ABD_SCATTER(abd).abd_chunks[i]);
	}
}

abd_t *
abd_alloc_struct_impl(size_t size)
{
	uint_t chunkcnt = abd_chunkcnt_for_bytes(size);
	/*
	 * In the event we are allocating a gang ABD, the size passed in
	 * will be 0. We must make sure to set abd_size to the size of an
	 * ABD struct as opposed to an ABD scatter with 0 chunks. The gang
	 * ABD struct allocation accounts for an additional 24 bytes over
	 * a scatter ABD with 0 chunks.
	 */
	size_t abd_size = MAX(sizeof (abd_t),
	    offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));
	abd_t *abd = kmem_alloc(abd_size, KM_PUSHPAGE);
	ASSERT3P(abd, !=, NULL);
	ABDSTAT_INCR(abdstat_struct_size, abd_size);

	return (abd);
}

void
abd_free_struct_impl(abd_t *abd)
{
	uint_t chunkcnt = abd_is_linear(abd) || abd_is_gang(abd) ? 0 :
	    abd_scatter_chunkcnt(abd);
	ssize_t size = MAX(sizeof (abd_t),
	    offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));
	kmem_free(abd, size);
	ABDSTAT_INCR(abdstat_struct_size, -size);
}

/*
 * Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where
 * each chunk in the scatterlist will be set to abd_zero_buf.
 */
static void
abd_alloc_zero_scatter(void)
{
	uint_t i, n;

	n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
	abd_zero_buf = kmem_zalloc(zfs_abd_chunk_size, KM_SLEEP);
	abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);

	abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER | ABD_FLAG_ZEROS;
	abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;

	ABD_SCATTER(abd_zero_scatter).abd_offset = 0;
	ABD_SCATTER(abd_zero_scatter).abd_chunk_size =
	    zfs_abd_chunk_size;

	for (i = 0; i < n; i++) {
		ABD_SCATTER(abd_zero_scatter).abd_chunks[i] =
		    abd_zero_buf;
	}

	ABDSTAT_BUMP(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, zfs_abd_chunk_size);
}

static void
abd_free_zero_scatter(void)
{
	ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
	ABDSTAT_INCR(abdstat_scatter_data_size, -(int)zfs_abd_chunk_size);

	abd_free_struct(abd_zero_scatter);
	abd_zero_scatter = NULL;
	kmem_free(abd_zero_buf, zfs_abd_chunk_size);
}

void
abd_init(void)
{
	abd_chunk_cache = kmem_cache_create("abd_chunk", zfs_abd_chunk_size, 0,
	    NULL, NULL, NULL, NULL, 0, KMC_NODEBUG);

	abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
	    sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
	if (abd_ksp != NULL) {
		abd_ksp->ks_data = &abd_stats;
		kstat_install(abd_ksp);
	}

	abd_alloc_zero_scatter();
}

void
abd_fini(void)
{
	abd_free_zero_scatter();

	if (abd_ksp != NULL) {
		kstat_delete(abd_ksp);
		abd_ksp = NULL;
	}

	kmem_cache_destroy(abd_chunk_cache);
	abd_chunk_cache = NULL;
}

void
abd_free_linear_page(abd_t *abd)
{
	/*
	 * FreeBSD does not have have scatter linear pages
	 * so there is an error.
	 */
	VERIFY(0);
}

/*
 * If we're going to use this ABD for doing I/O using the block layer, the
 * consumer of the ABD data doesn't care if it's scattered or not, and we don't
 * plan to store this ABD in memory for a long period of time, we should
 * allocate the ABD type that requires the least data copying to do the I/O.
 *
 * Currently this is linear ABDs, however if ldi_strategy() can ever issue I/Os
 * using a scatter/gather list we should switch to that and replace this call
 * with vanilla abd_alloc().
 */
abd_t *
abd_alloc_for_io(size_t size, boolean_t is_metadata)
{
	return (abd_alloc_linear(size, is_metadata));
}

/*
 * This is just a helper function to abd_get_offset_scatter() to alloc a
 * scatter ABD using the calculated chunkcnt based on the offset within the
 * parent ABD.
 */
static abd_t *
abd_alloc_scatter_offset_chunkcnt(size_t chunkcnt)
{
	size_t abd_size = offsetof(abd_t,
	    abd_u.abd_scatter.abd_chunks[chunkcnt]);
	abd_t *abd = kmem_alloc(abd_size, KM_PUSHPAGE);
	ASSERT3P(abd, !=, NULL);
	list_link_init(&abd->abd_gang_link);
	mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
	ABDSTAT_INCR(abdstat_struct_size, abd_size);

	return (abd);
}

abd_t *
abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off)
{
	abd_verify(sabd);
	ASSERT3U(off, <=, sabd->abd_size);

	size_t new_offset = ABD_SCATTER(sabd).abd_offset + off;
	uint_t chunkcnt = abd_scatter_chunkcnt(sabd) -
	    (new_offset / zfs_abd_chunk_size);

	/*
	 * If an abd struct is provided, it is only the minimum size.  If we
	 * need additional chunks, we need to allocate a new struct.
	 */
	if (abd != NULL &&
	    offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]) >
	    sizeof (abd_t)) {
		abd = NULL;
	}

	if (abd == NULL)
		abd = abd_alloc_struct(chunkcnt * zfs_abd_chunk_size);

	/*
	 * Even if this buf is filesystem metadata, we only track that
	 * if we own the underlying data buffer, which is not true in
	 * this case. Therefore, we don't ever use ABD_FLAG_META here.
	 */

	ABD_SCATTER(abd).abd_offset = new_offset % zfs_abd_chunk_size;
	ABD_SCATTER(abd).abd_chunk_size = zfs_abd_chunk_size;

	/* Copy the scatterlist starting at the correct offset */
	(void) memcpy(&ABD_SCATTER(abd).abd_chunks,
	    &ABD_SCATTER(sabd).abd_chunks[new_offset /
	    zfs_abd_chunk_size],
	    chunkcnt * sizeof (void *));

	return (abd);
}

static inline size_t
abd_iter_scatter_chunk_offset(struct abd_iter *aiter)
{
	ASSERT(!abd_is_linear(aiter->iter_abd));
	return ((ABD_SCATTER(aiter->iter_abd).abd_offset +
	    aiter->iter_pos) % zfs_abd_chunk_size);
}

static inline size_t
abd_iter_scatter_chunk_index(struct abd_iter *aiter)
{
	ASSERT(!abd_is_linear(aiter->iter_abd));
	return ((ABD_SCATTER(aiter->iter_abd).abd_offset +
	    aiter->iter_pos) / zfs_abd_chunk_size);
}

/*
 * Initialize the abd_iter.
 */
void
abd_iter_init(struct abd_iter *aiter, abd_t *abd)
{
	ASSERT(!abd_is_gang(abd));
	abd_verify(abd);
	aiter->iter_abd = abd;
	aiter->iter_pos = 0;
	aiter->iter_mapaddr = NULL;
	aiter->iter_mapsize = 0;
}

/*
 * This is just a helper function to see if we have exhausted the
 * abd_iter and reached the end.
 */
boolean_t
abd_iter_at_end(struct abd_iter *aiter)
{
	return (aiter->iter_pos == aiter->iter_abd->abd_size);
}

/*
 * Advance the iterator by a certain amount. Cannot be called when a chunk is
 * in use. This can be safely called when the aiter has already exhausted, in
 * which case this does nothing.
 */
void
abd_iter_advance(struct abd_iter *aiter, size_t amount)
{
	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
	ASSERT0(aiter->iter_mapsize);

	/* There's nothing left to advance to, so do nothing */
	if (abd_iter_at_end(aiter))
		return;

	aiter->iter_pos += amount;
}

/*
 * Map the current chunk into aiter. This can be safely called when the aiter
 * has already exhausted, in which case this does nothing.
 */
void
abd_iter_map(struct abd_iter *aiter)
{
	void *paddr;
	size_t offset = 0;

	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
	ASSERT0(aiter->iter_mapsize);

	/* Panic if someone has changed zfs_abd_chunk_size */
	IMPLY(!abd_is_linear(aiter->iter_abd), zfs_abd_chunk_size ==
	    ABD_SCATTER(aiter->iter_abd).abd_chunk_size);

	/* There's nothing left to iterate over, so do nothing */
	if (abd_iter_at_end(aiter))
		return;

	if (abd_is_linear(aiter->iter_abd)) {
		offset = aiter->iter_pos;
		aiter->iter_mapsize = aiter->iter_abd->abd_size - offset;
		paddr = ABD_LINEAR_BUF(aiter->iter_abd);
	} else {
		size_t index = abd_iter_scatter_chunk_index(aiter);
		offset = abd_iter_scatter_chunk_offset(aiter);
		aiter->iter_mapsize = MIN(zfs_abd_chunk_size - offset,
		    aiter->iter_abd->abd_size - aiter->iter_pos);
		paddr = ABD_SCATTER(aiter->iter_abd).abd_chunks[index];
	}
	aiter->iter_mapaddr = (char *)paddr + offset;
}

/*
 * Unmap the current chunk from aiter. This can be safely called when the aiter
 * has already exhausted, in which case this does nothing.
 */
void
abd_iter_unmap(struct abd_iter *aiter)
{
	/* There's nothing left to unmap, so do nothing */
	if (abd_iter_at_end(aiter))
		return;

	ASSERT3P(aiter->iter_mapaddr, !=, NULL);
	ASSERT3U(aiter->iter_mapsize, >, 0);

	aiter->iter_mapaddr = NULL;
	aiter->iter_mapsize = 0;
}

void
abd_cache_reap_now(void)
{
	kmem_cache_reap_soon(abd_chunk_cache);
}