xfs_rtalloc.c 36.9 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2
/*
3
4
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
Linus Torvalds's avatar
Linus Torvalds committed
5
6
 */
#include "xfs.h"
7
#include "xfs_fs.h"
8
#include "xfs_shared.h"
9
10
11
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
12
#include "xfs_bit.h"
Linus Torvalds's avatar
Linus Torvalds committed
13
14
15
#include "xfs_mount.h"
#include "xfs_inode.h"
#include "xfs_bmap.h"
16
#include "xfs_bmap_btree.h"
17
#include "xfs_trans.h"
Linus Torvalds's avatar
Linus Torvalds committed
18
#include "xfs_trans_space.h"
Dave Chinner's avatar
Dave Chinner committed
19
#include "xfs_icache.h"
20
#include "xfs_rtalloc.h"
21
#include "xfs_sb.h"
Linus Torvalds's avatar
Linus Torvalds committed
22
23

/*
24
25
26
27
 * Read and return the summary information for a given extent size,
 * bitmap block combination.
 * Keeps track of a current summary block, so we don't keep reading
 * it from the buffer cache.
Linus Torvalds's avatar
Linus Torvalds committed
28
 */
29
static int
30
31
32
33
34
xfs_rtget_summary(
	xfs_mount_t	*mp,		/* file system mount structure */
	xfs_trans_t	*tp,		/* transaction pointer */
	int		log,		/* log2 of extent size */
	xfs_rtblock_t	bbno,		/* bitmap block number */
Dave Chinner's avatar
Dave Chinner committed
35
	struct xfs_buf	**rbpp,		/* in/out: summary block buffer */
36
37
	xfs_fsblock_t	*rsb,		/* in/out: summary block number */
	xfs_suminfo_t	*sum)		/* out: summary info for this block */
Linus Torvalds's avatar
Linus Torvalds committed
38
{
39
	return xfs_rtmodify_summary_int(mp, tp, log, bbno, 0, rbpp, rsb, sum);
Linus Torvalds's avatar
Linus Torvalds committed
40
41
42
}

/*
43
44
 * Return whether there are any free extents in the size range given
 * by low and high, for the bitmap block bbno.
Linus Torvalds's avatar
Linus Torvalds committed
45
46
 */
STATIC int				/* error */
47
48
xfs_rtany_summary(
	xfs_mount_t	*mp,		/* file system mount structure */
Linus Torvalds's avatar
Linus Torvalds committed
49
	xfs_trans_t	*tp,		/* transaction pointer */
50
51
	int		low,		/* low log2 extent size */
	int		high,		/* high log2 extent size */
Linus Torvalds's avatar
Linus Torvalds committed
52
	xfs_rtblock_t	bbno,		/* bitmap block number */
Dave Chinner's avatar
Dave Chinner committed
53
	struct xfs_buf	**rbpp,		/* in/out: summary block buffer */
Linus Torvalds's avatar
Linus Torvalds committed
54
	xfs_fsblock_t	*rsb,		/* in/out: summary block number */
55
	int		*stat)		/* out: any good extents here? */
Linus Torvalds's avatar
Linus Torvalds committed
56
57
{
	int		error;		/* error value */
58
59
	int		log;		/* loop counter, log2 of ext. size */
	xfs_suminfo_t	sum;		/* summary data */
Linus Torvalds's avatar
Linus Torvalds committed
60

61
62
63
64
	/* There are no extents at levels < m_rsum_cache[bbno]. */
	if (mp->m_rsum_cache && low < mp->m_rsum_cache[bbno])
		low = mp->m_rsum_cache[bbno];

Linus Torvalds's avatar
Linus Torvalds committed
65
	/*
66
	 * Loop over logs of extent sizes.
Linus Torvalds's avatar
Linus Torvalds committed
67
	 */
68
	for (log = low; log <= high; log++) {
Linus Torvalds's avatar
Linus Torvalds committed
69
		/*
70
		 * Get one summary datum.
Linus Torvalds's avatar
Linus Torvalds committed
71
		 */
72
		error = xfs_rtget_summary(mp, tp, log, bbno, rbpp, rsb, &sum);
Linus Torvalds's avatar
Linus Torvalds committed
73
74
75
76
		if (error) {
			return error;
		}
		/*
77
		 * If there are any, return success.
Linus Torvalds's avatar
Linus Torvalds committed
78
		 */
79
80
		if (sum) {
			*stat = 1;
81
			goto out;
Linus Torvalds's avatar
Linus Torvalds committed
82
83
84
		}
	}
	/*
85
	 * Found nothing, return failure.
Linus Torvalds's avatar
Linus Torvalds committed
86
	 */
87
	*stat = 0;
88
89
90
91
out:
	/* There were no extents at levels < log. */
	if (mp->m_rsum_cache && log > mp->m_rsum_cache[bbno])
		mp->m_rsum_cache[bbno] = log;
92
93
	return 0;
}
Linus Torvalds's avatar
Linus Torvalds committed
94

95
96
97
98
99
100
101
102
103
104
105
106

/*
 * Copy and transform the summary file, given the old and new
 * parameters in the mount structures.
 */
STATIC int				/* error */
xfs_rtcopy_summary(
	xfs_mount_t	*omp,		/* old file system mount point */
	xfs_mount_t	*nmp,		/* new file system mount point */
	xfs_trans_t	*tp)		/* transaction pointer */
{
	xfs_rtblock_t	bbno;		/* bitmap block number */
Dave Chinner's avatar
Dave Chinner committed
107
	struct xfs_buf	*bp;		/* summary buffer */
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
	int		error;		/* error return value */
	int		log;		/* summary level number (log length) */
	xfs_suminfo_t	sum;		/* summary data */
	xfs_fsblock_t	sumbno;		/* summary block number */

	bp = NULL;
	for (log = omp->m_rsumlevels - 1; log >= 0; log--) {
		for (bbno = omp->m_sb.sb_rbmblocks - 1;
		     (xfs_srtblock_t)bbno >= 0;
		     bbno--) {
			error = xfs_rtget_summary(omp, tp, log, bbno, &bp,
				&sumbno, &sum);
			if (error)
				return error;
			if (sum == 0)
				continue;
			error = xfs_rtmodify_summary(omp, tp, log, bbno, -sum,
				&bp, &sumbno);
			if (error)
				return error;
			error = xfs_rtmodify_summary(nmp, tp, log, bbno, sum,
				&bp, &sumbno);
			if (error)
				return error;
			ASSERT(sum > 0);
		}
	}
	return 0;
}
/*
 * Mark an extent specified by start and len allocated.
 * Updates all the summary information as well as the bitmap.
 */
STATIC int				/* error */
xfs_rtallocate_range(
	xfs_mount_t	*mp,		/* file system mount point */
	xfs_trans_t	*tp,		/* transaction pointer */
	xfs_rtblock_t	start,		/* start block to allocate */
	xfs_extlen_t	len,		/* length to allocate */
Dave Chinner's avatar
Dave Chinner committed
147
	struct xfs_buf	**rbpp,		/* in/out: summary block buffer */
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
	xfs_fsblock_t	*rsb)		/* in/out: summary block number */
{
	xfs_rtblock_t	end;		/* end of the allocated extent */
	int		error;		/* error value */
	xfs_rtblock_t	postblock = 0;	/* first block allocated > end */
	xfs_rtblock_t	preblock = 0;	/* first block allocated < start */

	end = start + len - 1;
	/*
	 * Assume we're allocating out of the middle of a free extent.
	 * We need to find the beginning and end of the extent so we can
	 * properly update the summary.
	 */
	error = xfs_rtfind_back(mp, tp, start, 0, &preblock);
	if (error) {
		return error;
	}
	/*
	 * Find the next allocated block (end of free extent).
	 */
	error = xfs_rtfind_forw(mp, tp, end, mp->m_sb.sb_rextents - 1,
		&postblock);
	if (error) {
		return error;
	}
	/*
	 * Decrement the summary information corresponding to the entire
	 * (old) free extent.
	 */
	error = xfs_rtmodify_summary(mp, tp,
		XFS_RTBLOCKLOG(postblock + 1 - preblock),
		XFS_BITTOBLOCK(mp, preblock), -1, rbpp, rsb);
	if (error) {
		return error;
	}
	/*
	 * If there are blocks not being allocated at the front of the
	 * old extent, add summary data for them to be free.
	 */
	if (preblock < start) {
		error = xfs_rtmodify_summary(mp, tp,
			XFS_RTBLOCKLOG(start - preblock),
			XFS_BITTOBLOCK(mp, preblock), 1, rbpp, rsb);
		if (error) {
			return error;
		}
	}
	/*
	 * If there are blocks not being allocated at the end of the
	 * old extent, add summary data for them to be free.
	 */
	if (postblock > end) {
		error = xfs_rtmodify_summary(mp, tp,
			XFS_RTBLOCKLOG(postblock - end),
			XFS_BITTOBLOCK(mp, end + 1), 1, rbpp, rsb);
		if (error) {
			return error;
		}
	}
	/*
	 * Modify the bitmap to mark this extent allocated.
	 */
	error = xfs_rtmodify_range(mp, tp, start, len, 0);
	return error;
}

/*
 * Attempt to allocate an extent minlen<=len<=maxlen starting from
 * bitmap block bbno.  If we don't get maxlen then use prod to trim
 * the length, if given.  Returns error; returns starting block in *rtblock.
 * The lengths are all in rtextents.
 */
STATIC int				/* error */
xfs_rtallocate_extent_block(
	xfs_mount_t	*mp,		/* file system mount point */
	xfs_trans_t	*tp,		/* transaction pointer */
	xfs_rtblock_t	bbno,		/* bitmap block number */
	xfs_extlen_t	minlen,		/* minimum length to allocate */
	xfs_extlen_t	maxlen,		/* maximum length to allocate */
	xfs_extlen_t	*len,		/* out: actual length allocated */
	xfs_rtblock_t	*nextp,		/* out: next block to try */
Dave Chinner's avatar
Dave Chinner committed
229
	struct xfs_buf	**rbpp,		/* in/out: summary block buffer */
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
	xfs_fsblock_t	*rsb,		/* in/out: summary block number */
	xfs_extlen_t	prod,		/* extent product factor */
	xfs_rtblock_t	*rtblock)	/* out: start block allocated */
{
	xfs_rtblock_t	besti;		/* best rtblock found so far */
	xfs_rtblock_t	bestlen;	/* best length found so far */
	xfs_rtblock_t	end;		/* last rtblock in chunk */
	int		error;		/* error value */
	xfs_rtblock_t	i;		/* current rtblock trying */
	xfs_rtblock_t	next;		/* next rtblock to try */
	int		stat;		/* status from internal calls */

	/*
	 * Loop over all the extents starting in this bitmap block,
	 * looking for one that's long enough.
	 */
	for (i = XFS_BLOCKTOBIT(mp, bbno), besti = -1, bestlen = 0,
		end = XFS_BLOCKTOBIT(mp, bbno + 1) - 1;
	     i <= end;
	     i++) {
250
251
252
		/* Make sure we don't scan off the end of the rt volume. */
		maxlen = min(mp->m_sb.sb_rextents, i + maxlen) - i;

253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
		/*
		 * See if there's a free extent of maxlen starting at i.
		 * If it's not so then next will contain the first non-free.
		 */
		error = xfs_rtcheck_range(mp, tp, i, maxlen, 1, &next, &stat);
		if (error) {
			return error;
		}
		if (stat) {
			/*
			 * i for maxlen is all free, allocate and return that.
			 */
			error = xfs_rtallocate_range(mp, tp, i, maxlen, rbpp,
				rsb);
			if (error) {
				return error;
			}
			*len = maxlen;
			*rtblock = i;
			return 0;
		}
		/*
		 * In the case where we have a variable-sized allocation
		 * request, figure out how big this free piece is,
		 * and if it's big enough for the minimum, and the best
		 * so far, remember it.
		 */
		if (minlen < maxlen) {
			xfs_rtblock_t	thislen;	/* this extent size */

			thislen = next - i;
			if (thislen >= minlen && thislen > bestlen) {
				besti = i;
				bestlen = thislen;
			}
		}
		/*
		 * If not done yet, find the start of the next free space.
		 */
		if (next < end) {
			error = xfs_rtfind_forw(mp, tp, next, end, &i);
			if (error) {
				return error;
			}
		} else
			break;
	}
	/*
	 * Searched the whole thing & didn't find a maxlen free extent.
	 */
	if (minlen < maxlen && besti != -1) {
		xfs_extlen_t	p;	/* amount to trim length by */

		/*
		 * If size should be a multiple of prod, make that so.
		 */
309
310
311
312
313
314
		if (prod > 1) {
			div_u64_rem(bestlen, prod, &p);
			if (p)
				bestlen -= p;
		}

Linus Torvalds's avatar
Linus Torvalds committed
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
		/*
		 * Allocate besti for bestlen & return that.
		 */
		error = xfs_rtallocate_range(mp, tp, besti, bestlen, rbpp, rsb);
		if (error) {
			return error;
		}
		*len = bestlen;
		*rtblock = besti;
		return 0;
	}
	/*
	 * Allocation failed.  Set *nextp to the next block to try.
	 */
	*nextp = next;
	*rtblock = NULLRTBLOCK;
	return 0;
}

/*
 * Allocate an extent of length minlen<=len<=maxlen, starting at block
 * bno.  If we don't get maxlen then use prod to trim the length, if given.
 * Returns error; returns starting block in *rtblock.
 * The lengths are all in rtextents.
 */
STATIC int				/* error */
xfs_rtallocate_extent_exact(
	xfs_mount_t	*mp,		/* file system mount point */
	xfs_trans_t	*tp,		/* transaction pointer */
	xfs_rtblock_t	bno,		/* starting block number to allocate */
	xfs_extlen_t	minlen,		/* minimum length to allocate */
	xfs_extlen_t	maxlen,		/* maximum length to allocate */
	xfs_extlen_t	*len,		/* out: actual length allocated */
Dave Chinner's avatar
Dave Chinner committed
348
	struct xfs_buf	**rbpp,		/* in/out: summary block buffer */
Linus Torvalds's avatar
Linus Torvalds committed
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
	xfs_fsblock_t	*rsb,		/* in/out: summary block number */
	xfs_extlen_t	prod,		/* extent product factor */
	xfs_rtblock_t	*rtblock)	/* out: start block allocated */
{
	int		error;		/* error value */
	xfs_extlen_t	i;		/* extent length trimmed due to prod */
	int		isfree;		/* extent is free */
	xfs_rtblock_t	next;		/* next block to try (dummy) */

	ASSERT(minlen % prod == 0 && maxlen % prod == 0);
	/*
	 * Check if the range in question (for maxlen) is free.
	 */
	error = xfs_rtcheck_range(mp, tp, bno, maxlen, 1, &next, &isfree);
	if (error) {
		return error;
	}
	if (isfree) {
		/*
		 * If it is, allocate it and return success.
		 */
		error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);
		if (error) {
			return error;
		}
		*len = maxlen;
		*rtblock = bno;
		return 0;
	}
	/*
	 * If not, allocate what there is, if it's at least minlen.
	 */
	maxlen = next - bno;
	if (maxlen < minlen) {
		/*
		 * Failed, return failure status.
		 */
		*rtblock = NULLRTBLOCK;
		return 0;
	}
	/*
	 * Trim off tail of extent, if prod is specified.
	 */
	if (prod > 1 && (i = maxlen % prod)) {
		maxlen -= i;
		if (maxlen < minlen) {
			/*
			 * Now we can't do it, return failure status.
			 */
			*rtblock = NULLRTBLOCK;
			return 0;
		}
	}
	/*
	 * Allocate what we can and return it.
	 */
	error = xfs_rtallocate_range(mp, tp, bno, maxlen, rbpp, rsb);
	if (error) {
		return error;
	}
	*len = maxlen;
	*rtblock = bno;
	return 0;
}

/*
 * Allocate an extent of length minlen<=len<=maxlen, starting as near
 * to bno as possible.  If we don't get maxlen then use prod to trim
 * the length, if given.  The lengths are all in rtextents.
 */
STATIC int				/* error */
xfs_rtallocate_extent_near(
	xfs_mount_t	*mp,		/* file system mount point */
	xfs_trans_t	*tp,		/* transaction pointer */
	xfs_rtblock_t	bno,		/* starting block number to allocate */
	xfs_extlen_t	minlen,		/* minimum length to allocate */
	xfs_extlen_t	maxlen,		/* maximum length to allocate */
	xfs_extlen_t	*len,		/* out: actual length allocated */
Dave Chinner's avatar
Dave Chinner committed
427
	struct xfs_buf	**rbpp,		/* in/out: summary block buffer */
Linus Torvalds's avatar
Linus Torvalds committed
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
	xfs_fsblock_t	*rsb,		/* in/out: summary block number */
	xfs_extlen_t	prod,		/* extent product factor */
	xfs_rtblock_t	*rtblock)	/* out: start block allocated */
{
	int		any;		/* any useful extents from summary */
	xfs_rtblock_t	bbno;		/* bitmap block number */
	int		error;		/* error value */
	int		i;		/* bitmap block offset (loop control) */
	int		j;		/* secondary loop control */
	int		log2len;	/* log2 of minlen */
	xfs_rtblock_t	n;		/* next block to try */
	xfs_rtblock_t	r;		/* result block */

	ASSERT(minlen % prod == 0 && maxlen % prod == 0);
	/*
	 * If the block number given is off the end, silently set it to
	 * the last block.
	 */
	if (bno >= mp->m_sb.sb_rextents)
		bno = mp->m_sb.sb_rextents - 1;
448
449
450
451
452
453
454
455

	/* Make sure we don't run off the end of the rt volume. */
	maxlen = min(mp->m_sb.sb_rextents, bno + maxlen) - bno;
	if (maxlen < minlen) {
		*rtblock = NULLRTBLOCK;
		return 0;
	}

Linus Torvalds's avatar
Linus Torvalds committed
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
	/*
	 * Try the exact allocation first.
	 */
	error = xfs_rtallocate_extent_exact(mp, tp, bno, minlen, maxlen, len,
		rbpp, rsb, prod, &r);
	if (error) {
		return error;
	}
	/*
	 * If the exact allocation worked, return that.
	 */
	if (r != NULLRTBLOCK) {
		*rtblock = r;
		return 0;
	}
	bbno = XFS_BITTOBLOCK(mp, bno);
	i = 0;
473
	ASSERT(minlen != 0);
Linus Torvalds's avatar
Linus Torvalds committed
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
	log2len = xfs_highbit32(minlen);
	/*
	 * Loop over all bitmap blocks (bbno + i is current block).
	 */
	for (;;) {
		/*
		 * Get summary information of extents of all useful levels
		 * starting in this bitmap block.
		 */
		error = xfs_rtany_summary(mp, tp, log2len, mp->m_rsumlevels - 1,
			bbno + i, rbpp, rsb, &any);
		if (error) {
			return error;
		}
		/*
		 * If there are any useful extents starting here, try
		 * allocating one.
		 */
		if (any) {
			/*
			 * On the positive side of the starting location.
			 */
			if (i >= 0) {
				/*
				 * Try to allocate an extent starting in
				 * this block.
				 */
				error = xfs_rtallocate_extent_block(mp, tp,
					bbno + i, minlen, maxlen, len, &n, rbpp,
					rsb, prod, &r);
				if (error) {
					return error;
				}
				/*
				 * If it worked, return it.
				 */
				if (r != NULLRTBLOCK) {
					*rtblock = r;
					return 0;
				}
			}
			/*
			 * On the negative side of the starting location.
			 */
			else {		/* i < 0 */
				/*
				 * Loop backwards through the bitmap blocks from
				 * the starting point-1 up to where we are now.
				 * There should be an extent which ends in this
				 * bitmap block and is long enough.
				 */
				for (j = -1; j > i; j--) {
					/*
					 * Grab the summary information for
					 * this bitmap block.
					 */
					error = xfs_rtany_summary(mp, tp,
						log2len, mp->m_rsumlevels - 1,
						bbno + j, rbpp, rsb, &any);
					if (error) {
						return error;
					}
					/*
					 * If there's no extent given in the
					 * summary that means the extent we
					 * found must carry over from an
					 * earlier block.  If there is an
					 * extent given, we've already tried
					 * that allocation, don't do it again.
					 */
					if (any)
						continue;
					error = xfs_rtallocate_extent_block(mp,
						tp, bbno + j, minlen, maxlen,
						len, &n, rbpp, rsb, prod, &r);
					if (error) {
						return error;
					}
					/*
					 * If it works, return the extent.
					 */
					if (r != NULLRTBLOCK) {
						*rtblock = r;
						return 0;
					}
				}
				/*
				 * There weren't intervening bitmap blocks
				 * with a long enough extent, or the
				 * allocation didn't work for some reason
				 * (i.e. it's a little * too short).
				 * Try to allocate from the summary block
				 * that we found.
				 */
				error = xfs_rtallocate_extent_block(mp, tp,
					bbno + i, minlen, maxlen, len, &n, rbpp,
					rsb, prod, &r);
				if (error) {
					return error;
				}
				/*
				 * If it works, return the extent.
				 */
				if (r != NULLRTBLOCK) {
					*rtblock = r;
					return 0;
				}
			}
		}
		/*
		 * Loop control.  If we were on the positive side, and there's
		 * still more blocks on the negative side, go there.
		 */
		if (i > 0 && (int)bbno - i >= 0)
			i = -i;
		/*
		 * If positive, and no more negative, but there are more
		 * positive, go there.
		 */
		else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1)
			i++;
		/*
		 * If negative or 0 (just started), and there are positive
		 * blocks to go, go there.  The 0 case moves to block 1.
		 */
		else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1)
			i = 1 - i;
		/*
		 * If negative or 0 and there are more negative blocks,
		 * go there.
		 */
		else if (i <= 0 && (int)bbno + i > 0)
			i--;
		/*
		 * Must be done.  Return failure.
		 */
		else
			break;
	}
	*rtblock = NULLRTBLOCK;
	return 0;
}

/*
 * Allocate an extent of length minlen<=len<=maxlen, with no position
 * specified.  If we don't get maxlen then use prod to trim
 * the length, if given.  The lengths are all in rtextents.
 */
STATIC int				/* error */
xfs_rtallocate_extent_size(
	xfs_mount_t	*mp,		/* file system mount point */
	xfs_trans_t	*tp,		/* transaction pointer */
	xfs_extlen_t	minlen,		/* minimum length to allocate */
	xfs_extlen_t	maxlen,		/* maximum length to allocate */
	xfs_extlen_t	*len,		/* out: actual length allocated */
Dave Chinner's avatar
Dave Chinner committed
629
	struct xfs_buf	**rbpp,		/* in/out: summary block buffer */
Linus Torvalds's avatar
Linus Torvalds committed
630
631
632
633
634
635
636
637
638
639
640
641
	xfs_fsblock_t	*rsb,		/* in/out: summary block number */
	xfs_extlen_t	prod,		/* extent product factor */
	xfs_rtblock_t	*rtblock)	/* out: start block allocated */
{
	int		error;		/* error value */
	int		i;		/* bitmap block number */
	int		l;		/* level number (loop control) */
	xfs_rtblock_t	n;		/* next block to be tried */
	xfs_rtblock_t	r;		/* result block number */
	xfs_suminfo_t	sum;		/* summary information for extents */

	ASSERT(minlen % prod == 0 && maxlen % prod == 0);
642
643
	ASSERT(maxlen != 0);

Linus Torvalds's avatar
Linus Torvalds committed
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
	/*
	 * Loop over all the levels starting with maxlen.
	 * At each level, look at all the bitmap blocks, to see if there
	 * are extents starting there that are long enough (>= maxlen).
	 * Note, only on the initial level can the allocation fail if
	 * the summary says there's an extent.
	 */
	for (l = xfs_highbit32(maxlen); l < mp->m_rsumlevels; l++) {
		/*
		 * Loop over all the bitmap blocks.
		 */
		for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {
			/*
			 * Get the summary for this level/block.
			 */
			error = xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,
				&sum);
			if (error) {
				return error;
			}
			/*
			 * Nothing there, on to the next block.
			 */
			if (!sum)
				continue;
			/*
			 * Try allocating the extent.
			 */
			error = xfs_rtallocate_extent_block(mp, tp, i, maxlen,
				maxlen, len, &n, rbpp, rsb, prod, &r);
			if (error) {
				return error;
			}
			/*
			 * If it worked, return that.
			 */
			if (r != NULLRTBLOCK) {
				*rtblock = r;
682
683
684
685
686
687
688
689
690
691
				return 0;
			}
			/*
			 * If the "next block to try" returned from the
			 * allocator is beyond the next bitmap block,
			 * skip to that bitmap block.
			 */
			if (XFS_BITTOBLOCK(mp, n) > i + 1)
				i = XFS_BITTOBLOCK(mp, n) - 1;
		}
Linus Torvalds's avatar
Linus Torvalds committed
692
693
	}
	/*
694
695
	 * Didn't find any maxlen blocks.  Try smaller ones, unless
	 * we're asking for a fixed size extent.
Linus Torvalds's avatar
Linus Torvalds committed
696
	 */
697
698
699
700
701
702
703
	if (minlen > --maxlen) {
		*rtblock = NULLRTBLOCK;
		return 0;
	}
	ASSERT(minlen != 0);
	ASSERT(maxlen != 0);

Linus Torvalds's avatar
Linus Torvalds committed
704
	/*
705
706
707
	 * Loop over sizes, from maxlen down to minlen.
	 * This time, when we do the allocations, allow smaller ones
	 * to succeed.
Linus Torvalds's avatar
Linus Torvalds committed
708
	 */
709
	for (l = xfs_highbit32(maxlen); l >= xfs_highbit32(minlen); l--) {
Linus Torvalds's avatar
Linus Torvalds committed
710
		/*
711
712
		 * Loop over all the bitmap blocks, try an allocation
		 * starting in that block.
Linus Torvalds's avatar
Linus Torvalds committed
713
		 */
714
		for (i = 0; i < mp->m_sb.sb_rbmblocks; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
715
			/*
716
			 * Get the summary information for this level/block.
Linus Torvalds's avatar
Linus Torvalds committed
717
			 */
718
719
			error =	xfs_rtget_summary(mp, tp, l, i, rbpp, rsb,
						  &sum);
Linus Torvalds's avatar
Linus Torvalds committed
720
721
722
723
			if (error) {
				return error;
			}
			/*
724
			 * If nothing there, go on to next.
Linus Torvalds's avatar
Linus Torvalds committed
725
			 */
726
727
			if (!sum)
				continue;
Linus Torvalds's avatar
Linus Torvalds committed
728
			/*
729
730
731
			 * Try the allocation.  Make sure the specified
			 * minlen/maxlen are in the possible range for
			 * this summary level.
Linus Torvalds's avatar
Linus Torvalds committed
732
			 */
733
734
735
736
			error = xfs_rtallocate_extent_block(mp, tp, i,
					XFS_RTMAX(minlen, 1 << l),
					XFS_RTMIN(maxlen, (1 << (l + 1)) - 1),
					len, &n, rbpp, rsb, prod, &r);
Linus Torvalds's avatar
Linus Torvalds committed
737
738
739
740
			if (error) {
				return error;
			}
			/*
741
742
743
744
745
746
747
748
749
750
			 * If it worked, return that extent.
			 */
			if (r != NULLRTBLOCK) {
				*rtblock = r;
				return 0;
			}
			/*
			 * If the "next block to try" returned from the
			 * allocator is beyond the next bitmap block,
			 * skip to that bitmap block.
Linus Torvalds's avatar
Linus Torvalds committed
751
			 */
752
753
			if (XFS_BITTOBLOCK(mp, n) > i + 1)
				i = XFS_BITTOBLOCK(mp, n) - 1;
Linus Torvalds's avatar
Linus Torvalds committed
754
755
756
		}
	}
	/*
757
	 * Got nothing, return failure.
Linus Torvalds's avatar
Linus Torvalds committed
758
	 */
759
	*rtblock = NULLRTBLOCK;
Linus Torvalds's avatar
Linus Torvalds committed
760
761
762
763
	return 0;
}

/*
764
 * Allocate space to the bitmap or summary file, and zero it, for growfs.
Linus Torvalds's avatar
Linus Torvalds committed
765
 */
766
STATIC int
767
xfs_growfs_rt_alloc(
768
769
770
771
	struct xfs_mount	*mp,		/* file system mount point */
	xfs_extlen_t		oblocks,	/* old count of blocks */
	xfs_extlen_t		nblocks,	/* new count of blocks */
	struct xfs_inode	*ip)		/* inode (bitmap/summary) */
Linus Torvalds's avatar
Linus Torvalds committed
772
{
773
774
775
776
777
778
779
780
	xfs_fileoff_t		bno;		/* block number in file */
	struct xfs_buf		*bp;	/* temporary buffer for zeroing */
	xfs_daddr_t		d;		/* disk block address */
	int			error;		/* error return value */
	xfs_fsblock_t		fsbno;		/* filesystem block for bno */
	struct xfs_bmbt_irec	map;		/* block map output */
	int			nmap;		/* number of block maps */
	int			resblks;	/* space reservation */
781
	enum xfs_blft		buf_type;
782
	struct xfs_trans	*tp;
Linus Torvalds's avatar
Linus Torvalds committed
783

784
785
786
787
788
	if (ip == mp->m_rsumip)
		buf_type = XFS_BLFT_RTSUMMARY_BUF;
	else
		buf_type = XFS_BLFT_RTBITMAP_BUF;

Linus Torvalds's avatar
Linus Torvalds committed
789
	/*
790
	 * Allocate space to the file, as necessary.
Linus Torvalds's avatar
Linus Torvalds committed
791
	 */
792
793
	while (oblocks < nblocks) {
		resblks = XFS_GROWFSRT_SPACE_RES(mp, nblocks - oblocks);
Linus Torvalds's avatar
Linus Torvalds committed
794
		/*
795
		 * Reserve space & log for one extent added to the file.
Linus Torvalds's avatar
Linus Torvalds committed
796
		 */
797
798
		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtalloc, resblks,
				0, 0, &tp);
799
		if (error)
800
			return error;
Linus Torvalds's avatar
Linus Torvalds committed
801
		/*
802
		 * Lock the inode.
Linus Torvalds's avatar
Linus Torvalds committed
803
		 */
804
805
806
		xfs_ilock(ip, XFS_ILOCK_EXCL);
		xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);

807
808
809
810
811
		error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK,
				XFS_IEXT_ADD_NOSPLIT_CNT);
		if (error)
			goto out_trans_cancel;

812
813
814
815
816
		/*
		 * Allocate blocks to the bitmap file.
		 */
		nmap = 1;
		error = xfs_bmapi_write(tp, ip, oblocks, nblocks - oblocks,
817
					XFS_BMAPI_METADATA, 0, &map, &nmap);
818
		if (!error && nmap < 1)
819
			error = -ENOSPC;
820
		if (error)
821
			goto out_trans_cancel;
822
823
824
		/*
		 * Free any blocks freed up in the transaction, then commit.
		 */
825
		error = xfs_trans_commit(tp);
826
		if (error)
827
			return error;
828
829
830
831
832
833
834
835
836
837
		/*
		 * Now we need to clear the allocated blocks.
		 * Do this one block per transaction, to keep it simple.
		 */
		for (bno = map.br_startoff, fsbno = map.br_startblock;
		     bno < map.br_startoff + map.br_blockcount;
		     bno++, fsbno++) {
			/*
			 * Reserve log for one block zeroing.
			 */
838
839
			error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtzero,
					0, 0, 0, &tp);
840
			if (error)
841
				return error;
842
843
844
845
846
847
848
849
850
			/*
			 * Lock the bitmap inode.
			 */
			xfs_ilock(ip, XFS_ILOCK_EXCL);
			xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
			/*
			 * Get a buffer for the block.
			 */
			d = XFS_FSB_TO_DADDR(mp, fsbno);
851
852
853
			error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d,
					mp->m_bsize, 0, &bp);
			if (error)
854
				goto out_trans_cancel;
855
856

			xfs_trans_buf_set_type(tp, bp, buf_type);
857
			bp->b_ops = &xfs_rtbuf_ops;
858
859
860
861
862
			memset(bp->b_addr, 0, mp->m_sb.sb_blocksize);
			xfs_trans_log_buf(tp, bp, 0, mp->m_sb.sb_blocksize - 1);
			/*
			 * Commit the transaction.
			 */
863
			error = xfs_trans_commit(tp);
864
			if (error)
865
				return error;
Linus Torvalds's avatar
Linus Torvalds committed
866
		}
867
868
869
870
		/*
		 * Go on to the next extent, if any.
		 */
		oblocks = map.br_startoff + map.br_blockcount;
Linus Torvalds's avatar
Linus Torvalds committed
871
	}
872

Linus Torvalds's avatar
Linus Torvalds committed
873
	return 0;
874

875
876
out_trans_cancel:
	xfs_trans_cancel(tp);
877
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
878
879
}

880
881
882
883
884
885
886
887
888
889
static void
xfs_alloc_rsum_cache(
	xfs_mount_t	*mp,		/* file system mount structure */
	xfs_extlen_t	rbmblocks)	/* number of rt bitmap blocks */
{
	/*
	 * The rsum cache is initialized to all zeroes, which is trivially a
	 * lower bound on the minimum level with any free extents. We can
	 * continue without the cache if it couldn't be allocated.
	 */
890
	mp->m_rsum_cache = kvzalloc(rbmblocks, GFP_KERNEL);
891
892
893
894
	if (!mp->m_rsum_cache)
		xfs_warn(mp, "could not allocate realtime summary cache");
}

Linus Torvalds's avatar
Linus Torvalds committed
895
896
897
898
899
900
901
902
903
904
905
906
907
/*
 * Visible (exported) functions.
 */

/*
 * Grow the realtime area of the filesystem.
 */
int
xfs_growfs_rt(
	xfs_mount_t	*mp,		/* mount point for filesystem */
	xfs_growfs_rt_t	*in)		/* growfs rt input struct */
{
	xfs_rtblock_t	bmbno;		/* bitmap block number */
Dave Chinner's avatar
Dave Chinner committed
908
	struct xfs_buf	*bp;		/* temporary buffer */
Linus Torvalds's avatar
Linus Torvalds committed
909
910
	int		error;		/* error return value */
	xfs_mount_t	*nmp;		/* new (fake) mount structure */
911
	xfs_rfsblock_t	nrblocks;	/* new number of realtime blocks */
Linus Torvalds's avatar
Linus Torvalds committed
912
	xfs_extlen_t	nrbmblocks;	/* new number of rt bitmap blocks */
913
	xfs_rtblock_t	nrextents;	/* new number of realtime extents */
Linus Torvalds's avatar
Linus Torvalds committed
914
915
916
917
918
919
920
921
922
	uint8_t		nrextslog;	/* new log2 of sb_rextents */
	xfs_extlen_t	nrsumblocks;	/* new number of summary blocks */
	uint		nrsumlevels;	/* new rt summary levels */
	uint		nrsumsize;	/* new size of rt summary, bytes */
	xfs_sb_t	*nsbp;		/* new superblock */
	xfs_extlen_t	rbmblocks;	/* current number of rt bitmap blocks */
	xfs_extlen_t	rsumblocks;	/* current number of rt summary blks */
	xfs_sb_t	*sbp;		/* old superblock */
	xfs_fsblock_t	sumbno;		/* summary block number */
923
	uint8_t		*rsum_cache;	/* old summary cache */
Linus Torvalds's avatar
Linus Torvalds committed
924
925

	sbp = &mp->m_sb;
926

927
	if (!capable(CAP_SYS_ADMIN))
928
		return -EPERM;
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950

	/* Needs to have been mounted with an rt device. */
	if (!XFS_IS_REALTIME_MOUNT(mp))
		return -EINVAL;
	/*
	 * Mount should fail if the rt bitmap/summary files don't load, but
	 * we'll check anyway.
	 */
	if (!mp->m_rbmip || !mp->m_rsumip)
		return -EINVAL;

	/* Shrink not supported. */
	if (in->newblocks <= sbp->sb_rblocks)
		return -EINVAL;

	/* Can only change rt extent size when adding rt volume. */
	if (sbp->sb_rblocks > 0 && in->extsize != sbp->sb_rextsize)
		return -EINVAL;

	/* Range check the extent size. */
	if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE ||
	    XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE)
951
		return -EINVAL;
952
953
954
955
956
957
958
959
960

	/* Unsupported realtime features. */
	if (xfs_sb_version_hasrmapbt(&mp->m_sb) ||
	    xfs_sb_version_hasreflink(&mp->m_sb))
		return -EOPNOTSUPP;

	nrblocks = in->newblocks;
	error = xfs_sb_validate_fsb_count(sbp, nrblocks);
	if (error)
961
		return error;
Linus Torvalds's avatar
Linus Torvalds committed
962
963
964
	/*
	 * Read in the last block of the device, make sure it exists.
	 */
965
	error = xfs_buf_read_uncached(mp->m_rtdev_targp,
966
				XFS_FSB_TO_BB(mp, nrblocks - 1),
967
968
				XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);
	if (error)
969
		return error;
Linus Torvalds's avatar
Linus Torvalds committed
970
	xfs_buf_relse(bp);
971

Linus Torvalds's avatar
Linus Torvalds committed
972
973
974
975
976
	/*
	 * Calculate new parameters.  These are the final values to be reached.
	 */
	nrextents = nrblocks;
	do_div(nrextents, in->extsize);
977
	nrbmblocks = howmany_64(nrextents, NBBY * sbp->sb_blocksize);
Linus Torvalds's avatar
Linus Torvalds committed
978
979
980
981
982
983
984
985
986
987
988
	nrextslog = xfs_highbit32(nrextents);
	nrsumlevels = nrextslog + 1;
	nrsumsize = (uint)sizeof(xfs_suminfo_t) * nrsumlevels * nrbmblocks;
	nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);
	nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);
	/*
	 * New summary size can't be more than half the size of
	 * the log.  This prevents us from getting a log overflow,
	 * since we'll log basically the whole summary file at once.
	 */
	if (nrsumblocks > (mp->m_sb.sb_logblocks >> 1))
989
		return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
990
991
992
993
	/*
	 * Get the old block counts for bitmap and summary inodes.
	 * These can't change since other growfs callers are locked out.
	 */
994
995
	rbmblocks = XFS_B_TO_FSB(mp, mp->m_rbmip->i_disk_size);
	rsumblocks = XFS_B_TO_FSB(mp, mp->m_rsumip->i_disk_size);
Linus Torvalds's avatar
Linus Torvalds committed
996
997
998
	/*
	 * Allocate space to the bitmap and summary files, as necessary.
	 */
999
1000
	error = xfs_growfs_rt_alloc(mp, rbmblocks, nrbmblocks, mp->m_rbmip);
	if (error)
Linus Torvalds's avatar
Linus Torvalds committed
1001
		return error;
1002
1003
	error = xfs_growfs_rt_alloc(mp, rsumblocks, nrsumblocks, mp->m_rsumip);
	if (error)
Linus Torvalds's avatar
Linus Torvalds committed
1004
		return error;
1005
1006
1007
1008
1009

	rsum_cache = mp->m_rsum_cache;
	if (nrbmblocks != sbp->sb_rbmblocks)
		xfs_alloc_rsum_cache(mp, nrbmblocks);

1010
1011
1012
	/*
	 * Allocate a new (fake) mount/sb.
	 */
1013
	nmp = kmem_alloc(sizeof(*nmp), 0);
Linus Torvalds's avatar
Linus Torvalds committed
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
	/*
	 * Loop over the bitmap blocks.
	 * We will do everything one bitmap block at a time.
	 * Skip the current block if it is exactly full.
	 * This also deals with the case where there were no rtextents before.
	 */
	for (bmbno = sbp->sb_rbmblocks -
		     ((sbp->sb_rextents & ((1 << mp->m_blkbit_log) - 1)) != 0);
	     bmbno < nrbmblocks;
	     bmbno++) {
1024
1025
		struct xfs_trans	*tp;
		xfs_rfsblock_t		nrblocks_step;
1026

Linus Torvalds's avatar
Linus Torvalds committed
1027
1028
1029
1030
1031
1032
1033
		*nmp = *mp;
		nsbp = &nmp->m_sb;
		/*
		 * Calculate new sb and mount fields for this round.
		 */
		nsbp->sb_rextsize = in->extsize;
		nsbp->sb_rbmblocks = bmbno + 1;
1034
1035
1036
		nrblocks_step = (bmbno + 1) * NBBY * nsbp->sb_blocksize *
				nsbp->sb_rextsize;
		nsbp->sb_rblocks = min(nrblocks, nrblocks_step);
Linus Torvalds's avatar
Linus Torvalds committed
1037
1038
		nsbp->sb_rextents = nsbp->sb_rblocks;
		do_div(nsbp->sb_rextents, nsbp->sb_rextsize);
1039
		ASSERT(nsbp->sb_rextents != 0);
Linus Torvalds's avatar
Linus Torvalds committed
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
		nsbp->sb_rextslog = xfs_highbit32(nsbp->sb_rextents);
		nrsumlevels = nmp->m_rsumlevels = nsbp->sb_rextslog + 1;
		nrsumsize =
			(uint)sizeof(xfs_suminfo_t) * nrsumlevels *
			nsbp->sb_rbmblocks;
		nrsumblocks = XFS_B_TO_FSB(mp, nrsumsize);
		nmp->m_rsumsize = nrsumsize = XFS_FSB_TO_B(mp, nrsumblocks);
		/*
		 * Start a transaction, get the log reservation.
		 */
1050
1051
		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtfree, 0, 0, 0,
				&tp);
1052
		if (error)
1053
			break;
Linus Torvalds's avatar
Linus Torvalds committed
1054
1055
1056
		/*
		 * Lock out other callers by grabbing the bitmap inode lock.
		 */
1057
		xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP);
1058
		xfs_trans_ijoin(tp, mp->m_rbmip, XFS_ILOCK_EXCL);
Linus Torvalds's avatar
Linus Torvalds committed
1059
		/*
1060
1061
1062
		 * Update the bitmap inode's size ondisk and incore.  We need
		 * to update the incore size so that inode inactivation won't
		 * punch what it thinks are "posteof" blocks.
Linus Torvalds's avatar
Linus Torvalds committed
1063
		 */
1064
		mp->m_rbmip->i_disk_size =
Linus Torvalds's avatar
Linus Torvalds committed
1065
			nsbp->sb_rbmblocks * nsbp->sb_blocksize;
1066
		i_size_write(VFS_I(mp->m_rbmip), mp->m_rbmip->i_disk_size);
Linus Torvalds's avatar
Linus Torvalds committed
1067
1068
1069
1070
		xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);
		/*
		 * Get the summary inode into the transaction.
		 */
1071
		xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM);
1072
		xfs_trans_ijoin(tp, mp->m_rsumip, XFS_ILOCK_EXCL);
Linus Torvalds's avatar
Linus Torvalds committed
1073
		/*
1074
1075
1076
		 * Update the summary inode's size.  We need to update the
		 * incore size so that inode inactivation won't punch what it
		 * thinks are "posteof" blocks.
Linus Torvalds's avatar
Linus Torvalds committed
1077
		 */
1078
1079
		mp->m_rsumip->i_disk_size = nmp->m_rsumsize;
		i_size_write(VFS_I(mp->m_rsumip), mp->m_rsumip->i_disk_size);
Linus Torvalds's avatar
Linus Torvalds committed
1080
1081
1082
1083
1084
1085
1086
1087
1088
		xfs_trans_log_inode(tp, mp->m_rsumip, XFS_ILOG_CORE);
		/*
		 * Copy summary data from old to new sizes.
		 * Do this when the real size (not block-aligned) changes.
		 */
		if (sbp->sb_rbmblocks != nsbp->sb_rbmblocks ||
		    mp->m_rsumlevels != nmp->m_rsumlevels) {
			error = xfs_rtcopy_summary(mp, nmp, tp);
			if (error)
1089
				goto error_cancel;
Linus Torvalds's avatar
Linus Torvalds committed
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
		}
		/*
		 * Update superblock fields.
		 */
		if (nsbp->sb_rextsize != sbp->sb_rextsize)
			xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSIZE,
				nsbp->sb_rextsize - sbp->sb_rextsize);
		if (nsbp->sb_rbmblocks != sbp->sb_rbmblocks)
			xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS,
				nsbp->sb_rbmblocks - sbp->sb_rbmblocks);
		if (nsbp->sb_rblocks != sbp->sb_rblocks)
			xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBLOCKS,
				nsbp->sb_rblocks - sbp->sb_rblocks);
		if (nsbp->sb_rextents != sbp->sb_rextents)
			xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTENTS,
				nsbp->sb_rextents - sbp->sb_rextents);
		if (nsbp->sb_rextslog != sbp->sb_rextslog)
			xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG,
				nsbp->sb_rextslog - sbp->sb_rextslog);
		/*
		 * Free new extent.
		 */
		bp = NULL;
		error = xfs_rtfree_range(nmp, tp, sbp->sb_rextents,
			nsbp->sb_rextents - sbp->sb_rextents, &bp, &sumbno);
1115
1116
		if (error) {
error_cancel:
1117
			xfs_trans_cancel(tp);
1118
			break;
1119
		}
Linus Torvalds's avatar
Linus Torvalds committed
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
		/*
		 * Mark more blocks free in the superblock.
		 */
		xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS,
			nsbp->sb_rextents - sbp->sb_rextents);
		/*
		 * Update mp values into the real mp structure.
		 */
		mp->m_rsumlevels = nrsumlevels;
		mp->m_rsumsize = nrsumsize;
1130

1131
		error = xfs_trans_commit(tp);
1132
		if (error)
1133
			break;
Linus Torvalds's avatar
Linus Torvalds committed
1134
	}
1135
1136
1137
1138
1139
	if (error)
		goto out_free;

	/* Update secondary superblocks now the physical grow has completed */
	error = xfs_update_secondary_sbs(mp);
1140

1141
out_free:
Linus Torvalds's avatar
Linus Torvalds committed
1142
	/*
1143
	 * Free the fake mp structure.
Linus Torvalds's avatar
Linus Torvalds committed
1144
	 */
1145
	kmem_free(nmp);
1146

1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
	/*
	 * If we had to allocate a new rsum_cache, we either need to free the
	 * old one (if we succeeded) or free the new one and restore the old one
	 * (if there was an error).
	 */
	if (rsum_cache != mp->m_rsum_cache) {
		if (error) {
			kmem_free(mp->m_rsum_cache);
			mp->m_rsum_cache = rsum_cache;
		} else {
			kmem_free(rsum_cache);
		}
	}

Linus Torvalds's avatar
Linus Torvalds committed
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
	return error;
}

/*
 * Allocate an extent in the realtime subvolume, with the usual allocation
 * parameters.  The length units are all in realtime extents, as is the
 * result block number.
 */
int					/* error */
xfs_rtallocate_extent(
	xfs_trans_t	*tp,		/* transaction pointer */
	xfs_rtblock_t	bno,		/* starting block number to allocate */
	xfs_extlen_t	minlen,		/* minimum length to allocate */
	xfs_extlen_t	maxlen,		/* maximum length to allocate */
	xfs_extlen_t	*len,		/* out: actual length allocated */
	int		wasdel,		/* was a delayed allocation extent */
	xfs_extlen_t	prod,		/* extent product factor */
	xfs_rtblock_t	*rtblock)	/* out: start block allocated */
{
1180
	xfs_mount_t	*mp = tp->t_mountp;
Linus Torvalds's avatar
Linus Torvalds committed
1181
1182
1183
	int		error;		/* error value */
	xfs_rtblock_t	r;		/* result allocated block */
	xfs_fsblock_t	sb;		/* summary file block number */
Dave Chinner's avatar
Dave Chinner committed
1184
	struct xfs_buf	*sumbp;		/* summary file block buffer */
Linus Torvalds's avatar
Linus Torvalds committed
1185

1186
	ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));
Linus Torvalds's avatar
Linus Torvalds committed
1187
	ASSERT(minlen > 0 && minlen <= maxlen);
1188

Linus Torvalds's avatar
Linus Torvalds committed
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
	/*
	 * If prod is set then figure out what to do to minlen and maxlen.
	 */
	if (prod > 1) {
		xfs_extlen_t	i;

		if ((i = maxlen % prod))
			maxlen -= i;
		if ((i = minlen % prod))
			minlen += prod - i;
		if (maxlen < minlen) {
			*rtblock = NULLRTBLOCK;
			return 0;
		}
	}
1204

1205
retry:
Linus Torvalds's avatar
Linus Torvalds committed
1206
	sumbp = NULL;
1207
	if (bno == 0) {
Linus Torvalds's avatar
Linus Torvalds committed
1208
1209
		error = xfs_rtallocate_extent_size(mp, tp, minlen, maxlen, len,
				&sumbp,	&sb, prod, &r);
1210
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
1211
1212
1213
		error = xfs_rtallocate_extent_near(mp, tp, bno, minlen, maxlen,
				len, &sumbp, &sb, prod, &r);
	}
1214

1215
	if (error)
Linus Torvalds's avatar
Linus Torvalds committed
1216
		return error;
1217

Linus Torvalds's avatar
Linus Torvalds committed
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
	/*
	 * If it worked, update the superblock.
	 */
	if (r != NULLRTBLOCK) {
		long	slen = (long)*len;

		ASSERT(*len >= minlen && *len <= maxlen);
		if (wasdel)
			xfs_trans_mod_sb(tp, XFS_TRANS_SB_RES_FREXTENTS, -slen);
		else
			xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, -slen);
1229
1230
1231
	} else if (prod > 1) {
		prod = 1;
		goto retry;
Linus Torvalds's avatar
Linus Torvalds committed
1232
	}
1233

Linus Torvalds's avatar
Linus Torvalds committed
1234
1235
1236
1237
1238
1239
1240
1241
1242
	*rtblock = r;
	return 0;
}

/*
 * Initialize realtime fields in the mount structure.
 */
int				/* error */
xfs_rtmount_init(
1243
	struct xfs_mount	*mp)	/* file system mount structure */
Linus Torvalds's avatar
Linus Torvalds committed
1244
{
1245
1246
1247
1248
	struct xfs_buf		*bp;	/* buffer for last block of subvolume */
	struct xfs_sb		*sbp;	/* filesystem superblock copy in mount */
	xfs_daddr_t		d;	/* address of last block of subvolume */
	int			error;
Linus Torvalds's avatar
Linus Torvalds committed
1249
1250
1251
1252
1253

	sbp = &mp->m_sb;
	if (sbp->sb_rblocks == 0)
		return 0;
	if (mp->m_rtdev_targp == NULL) {
1254
1255
		xfs_warn(mp,
	"Filesystem has a realtime volume, use rtdev=device option");
1256
		return -ENODEV;
Linus Torvalds's avatar
Linus Torvalds committed
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
	}
	mp->m_rsumlevels = sbp->sb_rextslog + 1;
	mp->m_rsumsize =
		(uint)sizeof(xfs_suminfo_t) * mp->m_rsumlevels *
		sbp->sb_rbmblocks;
	mp->m_rsumsize = roundup(mp->m_rsumsize, sbp->sb_blocksize);
	mp->m_rbmip = mp->m_rsumip = NULL;
	/*
	 * Check that the realtime section is an ok size.
	 */
	d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);
	if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_rblocks) {
1269
		xfs_warn(mp, "realtime mount -- %llu != %llu",
Linus Torvalds's avatar
Linus Torvalds committed
1270
1271
			(unsigned long long) XFS_BB_TO_FSB(mp, d),
			(unsigned long long) mp->m_sb.sb_rblocks);
1272
		return -EFBIG;
Linus Torvalds's avatar
Linus Torvalds committed
1273
	}
1274
	error = xfs_buf_read_uncached(mp->m_rtdev_targp,
1275
					d - XFS_FSB_TO_BB(mp, 1),
1276
1277
					XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);
	if (error) {
1278
		xfs_warn(mp, "realtime device size check failed");
1279
		return error;
Linus Torvalds's avatar
Linus Torvalds committed
1280
1281
1282
1283
1284
1285
	}
	xfs_buf_relse(bp);
	return 0;
}

/*
1286
1287
 * Get the bitmap and summary inodes and the summary cache into the mount
 * structure at mount time.
Linus Torvalds's avatar
Linus Torvalds committed
1288
1289
1290
1291
1292
1293
1294
1295
1296
 */
int					/* error */
xfs_rtmount_inodes(
	xfs_mount_t	*mp)		/* file system mount structure */
{
	int		error;		/* error return value */
	xfs_sb_t	*sbp;

	sbp = &mp->m_sb;
1297
	error = xfs_iget(mp, NULL, sbp->sb_rbmino, 0, 0, &mp->m_rbmip);
Linus Torvalds's avatar
Linus Torvalds committed
1298
1299
1300
	if (error)
		return error;
	ASSERT(mp->m_rbmip != NULL);
1301

1302
	error = xfs_iget(mp, NULL, sbp->sb_rsumino, 0, 0, &mp->m_rsumip);
Linus Torvalds's avatar
Linus Torvalds committed
1303
	if (error) {
1304
		xfs_irele(mp->m_rbmip);
Linus Torvalds's avatar
Linus Torvalds committed
1305
1306
1307
		return error;
	}
	ASSERT(mp->m_rsumip != NULL);
1308
	xfs_alloc_rsum_cache(mp, sbp->sb_rbmblocks);
Linus Torvalds's avatar
Linus Torvalds committed
1309
1310
1311
	return 0;
}

1312
1313
1314
1315
void
xfs_rtunmount_inodes(
	struct xfs_mount	*mp)
{
1316
	kmem_free(mp->m_rsum_cache);
1317
	if (mp->m_rbmip)
1318
		xfs_irele(mp->m_rbmip);
1319
	if (mp->m_rsumip)
1320
		xfs_irele(mp->m_rsumip);
1321
1322
}

Linus Torvalds's avatar
Linus Torvalds committed
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
/*
 * Pick an extent for allocation at the start of a new realtime file.
 * Use the sequence number stored in the atime field of the bitmap inode.
 * Translate this to a fraction of the rtextents, and return the product
 * of rtextents and the fraction.
 * The fraction sequence is 0, 1/2, 1/4, 3/4, 1/8, ..., 7/8, 1/16, ...
 */
int					/* error */
xfs_rtpick_extent(
	xfs_mount_t	*mp,		/* file system mount point */
	xfs_trans_t	*tp,		/* transaction pointer */
	xfs_extlen_t	len,		/* allocation length (rtextents) */
	xfs_rtblock_t	*pick)		/* result rt extent */
{
	xfs_rtblock_t	b;		/* result block */
	int		log2;		/* log of sequence number */
1339
1340
1341
	uint64_t	resid;		/* residual after log removed */
	uint64_t	seq;		/* sequence number of file creation */
	uint64_t	*seqp;		/* pointer to seqno in inode */
Linus Torvalds's avatar
Linus Torvalds committed
1342

1343
1344
	ASSERT(xfs_isilocked(mp->m_rbmip, XFS_ILOCK_EXCL));

1345
	seqp = (uint64_t *)&VFS_I(mp->m_rbmip)->i_atime;
1346
1347
	if (!(mp->m_rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) {
		mp->m_rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;
Linus Torvalds's avatar
Linus Torvalds committed
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
		*seqp = 0;
	}
	seq = *seqp;
	if ((log2 = xfs_highbit64(seq)) == -1)
		b = 0;
	else {
		resid = seq - (1ULL << log2);
		b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >>
		    (log2 + 1);
		if (b >= mp->m_sb.sb_rextents)
1358
			div64_u64_rem(b, mp->m_sb.sb_rextents, &b);
Linus Torvalds's avatar
Linus Torvalds committed
1359
1360
1361
1362
		if (b + len > mp->m_sb.sb_rextents)
			b = mp->m_sb.sb_rextents - len;
	}
	*seqp = seq + 1;
1363
	xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);
Linus Torvalds's avatar
Linus Torvalds committed
1364
1365
1366
	*pick = b;
	return 0;
}