• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
3  *
4  * bitmap_create  - sets up the bitmap structure
5  * bitmap_destroy - destroys the bitmap structure
6  *
7  * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
8  * - added disk storage for bitmap
9  * - changes to allow various bitmap chunk sizes
10  */
11 
12 /*
13  * Still to do:
14  *
15  * flush after percent set rather than just time based. (maybe both).
16  */
17 
18 #include <linux/blkdev.h>
19 #include <linux/module.h>
20 #include <linux/errno.h>
21 #include <linux/slab.h>
22 #include <linux/init.h>
23 #include <linux/timer.h>
24 #include <linux/sched.h>
25 #include <linux/list.h>
26 #include <linux/file.h>
27 #include <linux/mount.h>
28 #include <linux/buffer_head.h>
29 #include <linux/seq_file.h>
30 #include "md.h"
31 #include "bitmap.h"
32 
bmname(struct bitmap * bitmap)33 static inline char *bmname(struct bitmap *bitmap)
34 {
35 	return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
36 }
37 
38 /*
39  * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
40  *
41  * 1) check to see if this page is allocated, if it's not then try to alloc
42  * 2) if the alloc fails, set the page's hijacked flag so we'll use the
43  *    page pointer directly as a counter
44  *
45  * if we find our page, we increment the page's refcount so that it stays
46  * allocated while we're using it
47  */
bitmap_checkpage(struct bitmap_counts * bitmap,unsigned long page,int create)48 static int bitmap_checkpage(struct bitmap_counts *bitmap,
49 			    unsigned long page, int create)
50 __releases(bitmap->lock)
51 __acquires(bitmap->lock)
52 {
53 	unsigned char *mappage;
54 
55 	if (page >= bitmap->pages) {
56 		/* This can happen if bitmap_start_sync goes beyond
57 		 * End-of-device while looking for a whole page.
58 		 * It is harmless.
59 		 */
60 		return -EINVAL;
61 	}
62 
63 	if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
64 		return 0;
65 
66 	if (bitmap->bp[page].map) /* page is already allocated, just return */
67 		return 0;
68 
69 	if (!create)
70 		return -ENOENT;
71 
72 	/* this page has not been allocated yet */
73 
74 	spin_unlock_irq(&bitmap->lock);
75 	mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
76 	spin_lock_irq(&bitmap->lock);
77 
78 	if (mappage == NULL) {
79 		pr_debug("md/bitmap: map page allocation failed, hijacking\n");
80 		/* failed - set the hijacked flag so that we can use the
81 		 * pointer as a counter */
82 		if (!bitmap->bp[page].map)
83 			bitmap->bp[page].hijacked = 1;
84 	} else if (bitmap->bp[page].map ||
85 		   bitmap->bp[page].hijacked) {
86 		/* somebody beat us to getting the page */
87 		kfree(mappage);
88 		return 0;
89 	} else {
90 
91 		/* no page was in place and we have one, so install it */
92 
93 		bitmap->bp[page].map = mappage;
94 		bitmap->missing_pages--;
95 	}
96 	return 0;
97 }
98 
99 /* if page is completely empty, put it back on the free list, or dealloc it */
100 /* if page was hijacked, unmark the flag so it might get alloced next time */
101 /* Note: lock should be held when calling this */
bitmap_checkfree(struct bitmap_counts * bitmap,unsigned long page)102 static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
103 {
104 	char *ptr;
105 
106 	if (bitmap->bp[page].count) /* page is still busy */
107 		return;
108 
109 	/* page is no longer in use, it can be released */
110 
111 	if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
112 		bitmap->bp[page].hijacked = 0;
113 		bitmap->bp[page].map = NULL;
114 	} else {
115 		/* normal case, free the page */
116 		ptr = bitmap->bp[page].map;
117 		bitmap->bp[page].map = NULL;
118 		bitmap->missing_pages++;
119 		kfree(ptr);
120 	}
121 }
122 
123 /*
124  * bitmap file handling - read and write the bitmap file and its superblock
125  */
126 
127 /*
128  * basic page I/O operations
129  */
130 
131 /* IO operations when bitmap is stored near all superblocks */
read_sb_page(struct mddev * mddev,loff_t offset,struct page * page,unsigned long index,int size)132 static int read_sb_page(struct mddev *mddev, loff_t offset,
133 			struct page *page,
134 			unsigned long index, int size)
135 {
136 	/* choose a good rdev and read the page from there */
137 
138 	struct md_rdev *rdev;
139 	sector_t target;
140 
141 	rdev_for_each(rdev, mddev) {
142 		if (! test_bit(In_sync, &rdev->flags)
143 		    || test_bit(Faulty, &rdev->flags))
144 			continue;
145 
146 		target = offset + index * (PAGE_SIZE/512);
147 
148 		if (sync_page_io(rdev, target,
149 				 roundup(size, bdev_logical_block_size(rdev->bdev)),
150 				 page, READ, true)) {
151 			page->index = index;
152 			return 0;
153 		}
154 	}
155 	return -EIO;
156 }
157 
next_active_rdev(struct md_rdev * rdev,struct mddev * mddev)158 static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
159 {
160 	/* Iterate the disks of an mddev, using rcu to protect access to the
161 	 * linked list, and raising the refcount of devices we return to ensure
162 	 * they don't disappear while in use.
163 	 * As devices are only added or removed when raid_disk is < 0 and
164 	 * nr_pending is 0 and In_sync is clear, the entries we return will
165 	 * still be in the same position on the list when we re-enter
166 	 * list_for_each_entry_continue_rcu.
167 	 */
168 	rcu_read_lock();
169 	if (rdev == NULL)
170 		/* start at the beginning */
171 		rdev = list_entry_rcu(&mddev->disks, struct md_rdev, same_set);
172 	else {
173 		/* release the previous rdev and start from there. */
174 		rdev_dec_pending(rdev, mddev);
175 	}
176 	list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
177 		if (rdev->raid_disk >= 0 &&
178 		    !test_bit(Faulty, &rdev->flags)) {
179 			/* this is a usable devices */
180 			atomic_inc(&rdev->nr_pending);
181 			rcu_read_unlock();
182 			return rdev;
183 		}
184 	}
185 	rcu_read_unlock();
186 	return NULL;
187 }
188 
write_sb_page(struct bitmap * bitmap,struct page * page,int wait)189 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
190 {
191 	struct md_rdev *rdev = NULL;
192 	struct block_device *bdev;
193 	struct mddev *mddev = bitmap->mddev;
194 	struct bitmap_storage *store = &bitmap->storage;
195 
196 	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
197 		int size = PAGE_SIZE;
198 		loff_t offset = mddev->bitmap_info.offset;
199 
200 		bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
201 
202 		if (page->index == store->file_pages-1) {
203 			int last_page_size = store->bytes & (PAGE_SIZE-1);
204 			if (last_page_size == 0)
205 				last_page_size = PAGE_SIZE;
206 			size = roundup(last_page_size,
207 				       bdev_logical_block_size(bdev));
208 		}
209 		/* Just make sure we aren't corrupting data or
210 		 * metadata
211 		 */
212 		if (mddev->external) {
213 			/* Bitmap could be anywhere. */
214 			if (rdev->sb_start + offset + (page->index
215 						       * (PAGE_SIZE/512))
216 			    > rdev->data_offset
217 			    &&
218 			    rdev->sb_start + offset
219 			    < (rdev->data_offset + mddev->dev_sectors
220 			     + (PAGE_SIZE/512)))
221 				goto bad_alignment;
222 		} else if (offset < 0) {
223 			/* DATA  BITMAP METADATA  */
224 			if (offset
225 			    + (long)(page->index * (PAGE_SIZE/512))
226 			    + size/512 > 0)
227 				/* bitmap runs in to metadata */
228 				goto bad_alignment;
229 			if (rdev->data_offset + mddev->dev_sectors
230 			    > rdev->sb_start + offset)
231 				/* data runs in to bitmap */
232 				goto bad_alignment;
233 		} else if (rdev->sb_start < rdev->data_offset) {
234 			/* METADATA BITMAP DATA */
235 			if (rdev->sb_start
236 			    + offset
237 			    + page->index*(PAGE_SIZE/512) + size/512
238 			    > rdev->data_offset)
239 				/* bitmap runs in to data */
240 				goto bad_alignment;
241 		} else {
242 			/* DATA METADATA BITMAP - no problems */
243 		}
244 		md_super_write(mddev, rdev,
245 			       rdev->sb_start + offset
246 			       + page->index * (PAGE_SIZE/512),
247 			       size,
248 			       page);
249 	}
250 
251 	if (wait)
252 		md_super_wait(mddev);
253 	return 0;
254 
255  bad_alignment:
256 	return -EINVAL;
257 }
258 
259 static void bitmap_file_kick(struct bitmap *bitmap);
260 /*
261  * write out a page to a file
262  */
write_page(struct bitmap * bitmap,struct page * page,int wait)263 static void write_page(struct bitmap *bitmap, struct page *page, int wait)
264 {
265 	struct buffer_head *bh;
266 
267 	if (bitmap->storage.file == NULL) {
268 		switch (write_sb_page(bitmap, page, wait)) {
269 		case -EINVAL:
270 			set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
271 		}
272 	} else {
273 
274 		bh = page_buffers(page);
275 
276 		while (bh && bh->b_blocknr) {
277 			atomic_inc(&bitmap->pending_writes);
278 			set_buffer_locked(bh);
279 			set_buffer_mapped(bh);
280 			submit_bh(WRITE | REQ_SYNC, bh);
281 			bh = bh->b_this_page;
282 		}
283 
284 		if (wait)
285 			wait_event(bitmap->write_wait,
286 				   atomic_read(&bitmap->pending_writes)==0);
287 	}
288 	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
289 		bitmap_file_kick(bitmap);
290 }
291 
end_bitmap_write(struct buffer_head * bh,int uptodate)292 static void end_bitmap_write(struct buffer_head *bh, int uptodate)
293 {
294 	struct bitmap *bitmap = bh->b_private;
295 
296 	if (!uptodate)
297 		set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
298 	if (atomic_dec_and_test(&bitmap->pending_writes))
299 		wake_up(&bitmap->write_wait);
300 }
301 
302 /* copied from buffer.c */
303 static void
__clear_page_buffers(struct page * page)304 __clear_page_buffers(struct page *page)
305 {
306 	ClearPagePrivate(page);
307 	set_page_private(page, 0);
308 	page_cache_release(page);
309 }
free_buffers(struct page * page)310 static void free_buffers(struct page *page)
311 {
312 	struct buffer_head *bh;
313 
314 	if (!PagePrivate(page))
315 		return;
316 
317 	bh = page_buffers(page);
318 	while (bh) {
319 		struct buffer_head *next = bh->b_this_page;
320 		free_buffer_head(bh);
321 		bh = next;
322 	}
323 	__clear_page_buffers(page);
324 	put_page(page);
325 }
326 
327 /* read a page from a file.
328  * We both read the page, and attach buffers to the page to record the
329  * address of each block (using bmap).  These addresses will be used
330  * to write the block later, completely bypassing the filesystem.
331  * This usage is similar to how swap files are handled, and allows us
332  * to write to a file with no concerns of memory allocation failing.
333  */
read_page(struct file * file,unsigned long index,struct bitmap * bitmap,unsigned long count,struct page * page)334 static int read_page(struct file *file, unsigned long index,
335 		     struct bitmap *bitmap,
336 		     unsigned long count,
337 		     struct page *page)
338 {
339 	int ret = 0;
340 	struct inode *inode = file_inode(file);
341 	struct buffer_head *bh;
342 	sector_t block;
343 
344 	pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
345 		 (unsigned long long)index << PAGE_SHIFT);
346 
347 	bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
348 	if (!bh) {
349 		ret = -ENOMEM;
350 		goto out;
351 	}
352 	attach_page_buffers(page, bh);
353 	block = index << (PAGE_SHIFT - inode->i_blkbits);
354 	while (bh) {
355 		if (count == 0)
356 			bh->b_blocknr = 0;
357 		else {
358 			bh->b_blocknr = bmap(inode, block);
359 			if (bh->b_blocknr == 0) {
360 				/* Cannot use this file! */
361 				ret = -EINVAL;
362 				goto out;
363 			}
364 			bh->b_bdev = inode->i_sb->s_bdev;
365 			if (count < (1<<inode->i_blkbits))
366 				count = 0;
367 			else
368 				count -= (1<<inode->i_blkbits);
369 
370 			bh->b_end_io = end_bitmap_write;
371 			bh->b_private = bitmap;
372 			atomic_inc(&bitmap->pending_writes);
373 			set_buffer_locked(bh);
374 			set_buffer_mapped(bh);
375 			submit_bh(READ, bh);
376 		}
377 		block++;
378 		bh = bh->b_this_page;
379 	}
380 	page->index = index;
381 
382 	wait_event(bitmap->write_wait,
383 		   atomic_read(&bitmap->pending_writes)==0);
384 	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
385 		ret = -EIO;
386 out:
387 	if (ret)
388 		printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
389 			(int)PAGE_SIZE,
390 			(unsigned long long)index << PAGE_SHIFT,
391 			ret);
392 	return ret;
393 }
394 
395 /*
396  * bitmap file superblock operations
397  */
398 
399 /* update the event counter and sync the superblock to disk */
bitmap_update_sb(struct bitmap * bitmap)400 void bitmap_update_sb(struct bitmap *bitmap)
401 {
402 	bitmap_super_t *sb;
403 
404 	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
405 		return;
406 	if (bitmap->mddev->bitmap_info.external)
407 		return;
408 	if (!bitmap->storage.sb_page) /* no superblock */
409 		return;
410 	sb = kmap_atomic(bitmap->storage.sb_page);
411 	sb->events = cpu_to_le64(bitmap->mddev->events);
412 	if (bitmap->mddev->events < bitmap->events_cleared)
413 		/* rocking back to read-only */
414 		bitmap->events_cleared = bitmap->mddev->events;
415 	sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
416 	sb->state = cpu_to_le32(bitmap->flags);
417 	/* Just in case these have been changed via sysfs: */
418 	sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
419 	sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
420 	/* This might have been changed by a reshape */
421 	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
422 	sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
423 	sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
424 					   bitmap_info.space);
425 	kunmap_atomic(sb);
426 	write_page(bitmap, bitmap->storage.sb_page, 1);
427 }
428 
429 /* print out the bitmap file superblock */
bitmap_print_sb(struct bitmap * bitmap)430 void bitmap_print_sb(struct bitmap *bitmap)
431 {
432 	bitmap_super_t *sb;
433 
434 	if (!bitmap || !bitmap->storage.sb_page)
435 		return;
436 	sb = kmap_atomic(bitmap->storage.sb_page);
437 	printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
438 	printk(KERN_DEBUG "         magic: %08x\n", le32_to_cpu(sb->magic));
439 	printk(KERN_DEBUG "       version: %d\n", le32_to_cpu(sb->version));
440 	printk(KERN_DEBUG "          uuid: %08x.%08x.%08x.%08x\n",
441 					*(__u32 *)(sb->uuid+0),
442 					*(__u32 *)(sb->uuid+4),
443 					*(__u32 *)(sb->uuid+8),
444 					*(__u32 *)(sb->uuid+12));
445 	printk(KERN_DEBUG "        events: %llu\n",
446 			(unsigned long long) le64_to_cpu(sb->events));
447 	printk(KERN_DEBUG "events cleared: %llu\n",
448 			(unsigned long long) le64_to_cpu(sb->events_cleared));
449 	printk(KERN_DEBUG "         state: %08x\n", le32_to_cpu(sb->state));
450 	printk(KERN_DEBUG "     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
451 	printk(KERN_DEBUG "  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
452 	printk(KERN_DEBUG "     sync size: %llu KB\n",
453 			(unsigned long long)le64_to_cpu(sb->sync_size)/2);
454 	printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
455 	kunmap_atomic(sb);
456 }
457 
458 /*
459  * bitmap_new_disk_sb
460  * @bitmap
461  *
462  * This function is somewhat the reverse of bitmap_read_sb.  bitmap_read_sb
463  * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
464  * This function verifies 'bitmap_info' and populates the on-disk bitmap
465  * structure, which is to be written to disk.
466  *
467  * Returns: 0 on success, -Exxx on error
468  */
bitmap_new_disk_sb(struct bitmap * bitmap)469 static int bitmap_new_disk_sb(struct bitmap *bitmap)
470 {
471 	bitmap_super_t *sb;
472 	unsigned long chunksize, daemon_sleep, write_behind;
473 
474 	bitmap->storage.sb_page = alloc_page(GFP_KERNEL);
475 	if (bitmap->storage.sb_page == NULL)
476 		return -ENOMEM;
477 	bitmap->storage.sb_page->index = 0;
478 
479 	sb = kmap_atomic(bitmap->storage.sb_page);
480 
481 	sb->magic = cpu_to_le32(BITMAP_MAGIC);
482 	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
483 
484 	chunksize = bitmap->mddev->bitmap_info.chunksize;
485 	BUG_ON(!chunksize);
486 	if (!is_power_of_2(chunksize)) {
487 		kunmap_atomic(sb);
488 		printk(KERN_ERR "bitmap chunksize not a power of 2\n");
489 		return -EINVAL;
490 	}
491 	sb->chunksize = cpu_to_le32(chunksize);
492 
493 	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
494 	if (!daemon_sleep ||
495 	    (daemon_sleep < 1) || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
496 		printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
497 		daemon_sleep = 5 * HZ;
498 	}
499 	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
500 	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
501 
502 	/*
503 	 * FIXME: write_behind for RAID1.  If not specified, what
504 	 * is a good choice?  We choose COUNTER_MAX / 2 arbitrarily.
505 	 */
506 	write_behind = bitmap->mddev->bitmap_info.max_write_behind;
507 	if (write_behind > COUNTER_MAX)
508 		write_behind = COUNTER_MAX / 2;
509 	sb->write_behind = cpu_to_le32(write_behind);
510 	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
511 
512 	/* keep the array size field of the bitmap superblock up to date */
513 	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
514 
515 	memcpy(sb->uuid, bitmap->mddev->uuid, 16);
516 
517 	set_bit(BITMAP_STALE, &bitmap->flags);
518 	sb->state = cpu_to_le32(bitmap->flags);
519 	bitmap->events_cleared = bitmap->mddev->events;
520 	sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
521 
522 	kunmap_atomic(sb);
523 
524 	return 0;
525 }
526 
527 /* read the superblock from the bitmap file and initialize some bitmap fields */
bitmap_read_sb(struct bitmap * bitmap)528 static int bitmap_read_sb(struct bitmap *bitmap)
529 {
530 	char *reason = NULL;
531 	bitmap_super_t *sb;
532 	unsigned long chunksize, daemon_sleep, write_behind;
533 	unsigned long long events;
534 	unsigned long sectors_reserved = 0;
535 	int err = -EINVAL;
536 	struct page *sb_page;
537 
538 	if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
539 		chunksize = 128 * 1024 * 1024;
540 		daemon_sleep = 5 * HZ;
541 		write_behind = 0;
542 		set_bit(BITMAP_STALE, &bitmap->flags);
543 		err = 0;
544 		goto out_no_sb;
545 	}
546 	/* page 0 is the superblock, read it... */
547 	sb_page = alloc_page(GFP_KERNEL);
548 	if (!sb_page)
549 		return -ENOMEM;
550 	bitmap->storage.sb_page = sb_page;
551 
552 	if (bitmap->storage.file) {
553 		loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
554 		int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
555 
556 		err = read_page(bitmap->storage.file, 0,
557 				bitmap, bytes, sb_page);
558 	} else {
559 		err = read_sb_page(bitmap->mddev,
560 				   bitmap->mddev->bitmap_info.offset,
561 				   sb_page,
562 				   0, sizeof(bitmap_super_t));
563 	}
564 	if (err)
565 		return err;
566 
567 	err = -EINVAL;
568 
569 	sb = kmap_atomic(sb_page);
570 
571 	chunksize = le32_to_cpu(sb->chunksize);
572 	daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
573 	write_behind = le32_to_cpu(sb->write_behind);
574 	sectors_reserved = le32_to_cpu(sb->sectors_reserved);
575 
576 	/* verify that the bitmap-specific fields are valid */
577 	if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
578 		reason = "bad magic";
579 	else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
580 		 le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
581 		reason = "unrecognized superblock version";
582 	else if (chunksize < 512)
583 		reason = "bitmap chunksize too small";
584 	else if (!is_power_of_2(chunksize))
585 		reason = "bitmap chunksize not a power of 2";
586 	else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
587 		reason = "daemon sleep period out of range";
588 	else if (write_behind > COUNTER_MAX)
589 		reason = "write-behind limit out of range (0 - 16383)";
590 	if (reason) {
591 		printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
592 			bmname(bitmap), reason);
593 		goto out;
594 	}
595 
596 	/* keep the array size field of the bitmap superblock up to date */
597 	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
598 
599 	if (bitmap->mddev->persistent) {
600 		/*
601 		 * We have a persistent array superblock, so compare the
602 		 * bitmap's UUID and event counter to the mddev's
603 		 */
604 		if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
605 			printk(KERN_INFO
606 			       "%s: bitmap superblock UUID mismatch\n",
607 			       bmname(bitmap));
608 			goto out;
609 		}
610 		events = le64_to_cpu(sb->events);
611 		if (events < bitmap->mddev->events) {
612 			printk(KERN_INFO
613 			       "%s: bitmap file is out of date (%llu < %llu) "
614 			       "-- forcing full recovery\n",
615 			       bmname(bitmap), events,
616 			       (unsigned long long) bitmap->mddev->events);
617 			set_bit(BITMAP_STALE, &bitmap->flags);
618 		}
619 	}
620 
621 	/* assign fields using values from superblock */
622 	bitmap->flags |= le32_to_cpu(sb->state);
623 	if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
624 		set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
625 	bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
626 	err = 0;
627 out:
628 	kunmap_atomic(sb);
629 out_no_sb:
630 	if (test_bit(BITMAP_STALE, &bitmap->flags))
631 		bitmap->events_cleared = bitmap->mddev->events;
632 	bitmap->mddev->bitmap_info.chunksize = chunksize;
633 	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
634 	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
635 	if (bitmap->mddev->bitmap_info.space == 0 ||
636 	    bitmap->mddev->bitmap_info.space > sectors_reserved)
637 		bitmap->mddev->bitmap_info.space = sectors_reserved;
638 	if (err)
639 		bitmap_print_sb(bitmap);
640 	return err;
641 }
642 
643 /*
644  * general bitmap file operations
645  */
646 
647 /*
648  * on-disk bitmap:
649  *
650  * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
651  * file a page at a time. There's a superblock at the start of the file.
652  */
653 /* calculate the index of the page that contains this bit */
file_page_index(struct bitmap_storage * store,unsigned long chunk)654 static inline unsigned long file_page_index(struct bitmap_storage *store,
655 					    unsigned long chunk)
656 {
657 	if (store->sb_page)
658 		chunk += sizeof(bitmap_super_t) << 3;
659 	return chunk >> PAGE_BIT_SHIFT;
660 }
661 
662 /* calculate the (bit) offset of this bit within a page */
file_page_offset(struct bitmap_storage * store,unsigned long chunk)663 static inline unsigned long file_page_offset(struct bitmap_storage *store,
664 					     unsigned long chunk)
665 {
666 	if (store->sb_page)
667 		chunk += sizeof(bitmap_super_t) << 3;
668 	return chunk & (PAGE_BITS - 1);
669 }
670 
671 /*
672  * return a pointer to the page in the filemap that contains the given bit
673  *
674  */
filemap_get_page(struct bitmap_storage * store,unsigned long chunk)675 static inline struct page *filemap_get_page(struct bitmap_storage *store,
676 					    unsigned long chunk)
677 {
678 	if (file_page_index(store, chunk) >= store->file_pages)
679 		return NULL;
680 	return store->filemap[file_page_index(store, chunk)];
681 }
682 
bitmap_storage_alloc(struct bitmap_storage * store,unsigned long chunks,int with_super)683 static int bitmap_storage_alloc(struct bitmap_storage *store,
684 				unsigned long chunks, int with_super)
685 {
686 	int pnum;
687 	unsigned long num_pages;
688 	unsigned long bytes;
689 
690 	bytes = DIV_ROUND_UP(chunks, 8);
691 	if (with_super)
692 		bytes += sizeof(bitmap_super_t);
693 
694 	num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
695 
696 	store->filemap = kmalloc(sizeof(struct page *)
697 				 * num_pages, GFP_KERNEL);
698 	if (!store->filemap)
699 		return -ENOMEM;
700 
701 	if (with_super && !store->sb_page) {
702 		store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
703 		if (store->sb_page == NULL)
704 			return -ENOMEM;
705 		store->sb_page->index = 0;
706 	}
707 	pnum = 0;
708 	if (store->sb_page) {
709 		store->filemap[0] = store->sb_page;
710 		pnum = 1;
711 	}
712 	for ( ; pnum < num_pages; pnum++) {
713 		store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
714 		if (!store->filemap[pnum]) {
715 			store->file_pages = pnum;
716 			return -ENOMEM;
717 		}
718 		store->filemap[pnum]->index = pnum;
719 	}
720 	store->file_pages = pnum;
721 
722 	/* We need 4 bits per page, rounded up to a multiple
723 	 * of sizeof(unsigned long) */
724 	store->filemap_attr = kzalloc(
725 		roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
726 		GFP_KERNEL);
727 	if (!store->filemap_attr)
728 		return -ENOMEM;
729 
730 	store->bytes = bytes;
731 
732 	return 0;
733 }
734 
bitmap_file_unmap(struct bitmap_storage * store)735 static void bitmap_file_unmap(struct bitmap_storage *store)
736 {
737 	struct page **map, *sb_page;
738 	int pages;
739 	struct file *file;
740 
741 	file = store->file;
742 	map = store->filemap;
743 	pages = store->file_pages;
744 	sb_page = store->sb_page;
745 
746 	while (pages--)
747 		if (map[pages] != sb_page) /* 0 is sb_page, release it below */
748 			free_buffers(map[pages]);
749 	kfree(map);
750 	kfree(store->filemap_attr);
751 
752 	if (sb_page)
753 		free_buffers(sb_page);
754 
755 	if (file) {
756 		struct inode *inode = file_inode(file);
757 		invalidate_mapping_pages(inode->i_mapping, 0, -1);
758 		fput(file);
759 	}
760 }
761 
762 /*
763  * bitmap_file_kick - if an error occurs while manipulating the bitmap file
764  * then it is no longer reliable, so we stop using it and we mark the file
765  * as failed in the superblock
766  */
bitmap_file_kick(struct bitmap * bitmap)767 static void bitmap_file_kick(struct bitmap *bitmap)
768 {
769 	char *path, *ptr = NULL;
770 
771 	if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
772 		bitmap_update_sb(bitmap);
773 
774 		if (bitmap->storage.file) {
775 			path = kmalloc(PAGE_SIZE, GFP_KERNEL);
776 			if (path)
777 				ptr = d_path(&bitmap->storage.file->f_path,
778 					     path, PAGE_SIZE);
779 
780 			printk(KERN_ALERT
781 			      "%s: kicking failed bitmap file %s from array!\n",
782 			      bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
783 
784 			kfree(path);
785 		} else
786 			printk(KERN_ALERT
787 			       "%s: disabling internal bitmap due to errors\n",
788 			       bmname(bitmap));
789 	}
790 }
791 
792 enum bitmap_page_attr {
793 	BITMAP_PAGE_DIRTY = 0,     /* there are set bits that need to be synced */
794 	BITMAP_PAGE_PENDING = 1,   /* there are bits that are being cleaned.
795 				    * i.e. counter is 1 or 2. */
796 	BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
797 };
798 
set_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)799 static inline void set_page_attr(struct bitmap *bitmap, int pnum,
800 				 enum bitmap_page_attr attr)
801 {
802 	set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
803 }
804 
clear_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)805 static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
806 				   enum bitmap_page_attr attr)
807 {
808 	clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
809 }
810 
test_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)811 static inline int test_page_attr(struct bitmap *bitmap, int pnum,
812 				 enum bitmap_page_attr attr)
813 {
814 	return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
815 }
816 
test_and_clear_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)817 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
818 					   enum bitmap_page_attr attr)
819 {
820 	return test_and_clear_bit((pnum<<2) + attr,
821 				  bitmap->storage.filemap_attr);
822 }
823 /*
824  * bitmap_file_set_bit -- called before performing a write to the md device
825  * to set (and eventually sync) a particular bit in the bitmap file
826  *
827  * we set the bit immediately, then we record the page number so that
828  * when an unplug occurs, we can flush the dirty pages out to disk
829  */
bitmap_file_set_bit(struct bitmap * bitmap,sector_t block)830 static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
831 {
832 	unsigned long bit;
833 	struct page *page;
834 	void *kaddr;
835 	unsigned long chunk = block >> bitmap->counts.chunkshift;
836 
837 	page = filemap_get_page(&bitmap->storage, chunk);
838 	if (!page)
839 		return;
840 	bit = file_page_offset(&bitmap->storage, chunk);
841 
842 	/* set the bit */
843 	kaddr = kmap_atomic(page);
844 	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
845 		set_bit(bit, kaddr);
846 	else
847 		set_bit_le(bit, kaddr);
848 	kunmap_atomic(kaddr);
849 	pr_debug("set file bit %lu page %lu\n", bit, page->index);
850 	/* record page number so it gets flushed to disk when unplug occurs */
851 	set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY);
852 }
853 
bitmap_file_clear_bit(struct bitmap * bitmap,sector_t block)854 static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
855 {
856 	unsigned long bit;
857 	struct page *page;
858 	void *paddr;
859 	unsigned long chunk = block >> bitmap->counts.chunkshift;
860 
861 	page = filemap_get_page(&bitmap->storage, chunk);
862 	if (!page)
863 		return;
864 	bit = file_page_offset(&bitmap->storage, chunk);
865 	paddr = kmap_atomic(page);
866 	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
867 		clear_bit(bit, paddr);
868 	else
869 		clear_bit_le(bit, paddr);
870 	kunmap_atomic(paddr);
871 	if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) {
872 		set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING);
873 		bitmap->allclean = 0;
874 	}
875 }
876 
877 /* this gets called when the md device is ready to unplug its underlying
878  * (slave) device queues -- before we let any writes go down, we need to
879  * sync the dirty pages of the bitmap file to disk */
bitmap_unplug(struct bitmap * bitmap)880 void bitmap_unplug(struct bitmap *bitmap)
881 {
882 	unsigned long i;
883 	int dirty, need_write;
884 
885 	if (!bitmap || !bitmap->storage.filemap ||
886 	    test_bit(BITMAP_STALE, &bitmap->flags))
887 		return;
888 
889 	/* look at each page to see if there are any set bits that need to be
890 	 * flushed out to disk */
891 	for (i = 0; i < bitmap->storage.file_pages; i++) {
892 		if (!bitmap->storage.filemap)
893 			return;
894 		dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
895 		need_write = test_and_clear_page_attr(bitmap, i,
896 						      BITMAP_PAGE_NEEDWRITE);
897 		if (dirty || need_write) {
898 			clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
899 			write_page(bitmap, bitmap->storage.filemap[i], 0);
900 		}
901 	}
902 	if (bitmap->storage.file)
903 		wait_event(bitmap->write_wait,
904 			   atomic_read(&bitmap->pending_writes)==0);
905 	else
906 		md_super_wait(bitmap->mddev);
907 
908 	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
909 		bitmap_file_kick(bitmap);
910 }
911 EXPORT_SYMBOL(bitmap_unplug);
912 
913 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
914 /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
915  * the in-memory bitmap from the on-disk bitmap -- also, sets up the
916  * memory mapping of the bitmap file
917  * Special cases:
918  *   if there's no bitmap file, or if the bitmap file had been
919  *   previously kicked from the array, we mark all the bits as
920  *   1's in order to cause a full resync.
921  *
922  * We ignore all bits for sectors that end earlier than 'start'.
923  * This is used when reading an out-of-date bitmap...
924  */
bitmap_init_from_disk(struct bitmap * bitmap,sector_t start)925 static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
926 {
927 	unsigned long i, chunks, index, oldindex, bit;
928 	struct page *page = NULL;
929 	unsigned long bit_cnt = 0;
930 	struct file *file;
931 	unsigned long offset;
932 	int outofdate;
933 	int ret = -ENOSPC;
934 	void *paddr;
935 	struct bitmap_storage *store = &bitmap->storage;
936 
937 	chunks = bitmap->counts.chunks;
938 	file = store->file;
939 
940 	if (!file && !bitmap->mddev->bitmap_info.offset) {
941 		/* No permanent bitmap - fill with '1s'. */
942 		store->filemap = NULL;
943 		store->file_pages = 0;
944 		for (i = 0; i < chunks ; i++) {
945 			/* if the disk bit is set, set the memory bit */
946 			int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
947 				      >= start);
948 			bitmap_set_memory_bits(bitmap,
949 					       (sector_t)i << bitmap->counts.chunkshift,
950 					       needed);
951 		}
952 		return 0;
953 	}
954 
955 	outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
956 	if (outofdate)
957 		printk(KERN_INFO "%s: bitmap file is out of date, doing full "
958 			"recovery\n", bmname(bitmap));
959 
960 	if (file && i_size_read(file->f_mapping->host) < store->bytes) {
961 		printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
962 		       bmname(bitmap),
963 		       (unsigned long) i_size_read(file->f_mapping->host),
964 		       store->bytes);
965 		goto err;
966 	}
967 
968 	oldindex = ~0L;
969 	offset = 0;
970 	if (!bitmap->mddev->bitmap_info.external)
971 		offset = sizeof(bitmap_super_t);
972 
973 	for (i = 0; i < chunks; i++) {
974 		int b;
975 		index = file_page_index(&bitmap->storage, i);
976 		bit = file_page_offset(&bitmap->storage, i);
977 		if (index != oldindex) { /* this is a new page, read it in */
978 			int count;
979 			/* unmap the old page, we're done with it */
980 			if (index == store->file_pages-1)
981 				count = store->bytes - index * PAGE_SIZE;
982 			else
983 				count = PAGE_SIZE;
984 			page = store->filemap[index];
985 			if (file)
986 				ret = read_page(file, index, bitmap,
987 						count, page);
988 			else
989 				ret = read_sb_page(
990 					bitmap->mddev,
991 					bitmap->mddev->bitmap_info.offset,
992 					page,
993 					index, count);
994 
995 			if (ret)
996 				goto err;
997 
998 			oldindex = index;
999 
1000 			if (outofdate) {
1001 				/*
1002 				 * if bitmap is out of date, dirty the
1003 				 * whole page and write it out
1004 				 */
1005 				paddr = kmap_atomic(page);
1006 				memset(paddr + offset, 0xff,
1007 				       PAGE_SIZE - offset);
1008 				kunmap_atomic(paddr);
1009 				write_page(bitmap, page, 1);
1010 
1011 				ret = -EIO;
1012 				if (test_bit(BITMAP_WRITE_ERROR,
1013 					     &bitmap->flags))
1014 					goto err;
1015 			}
1016 		}
1017 		paddr = kmap_atomic(page);
1018 		if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1019 			b = test_bit(bit, paddr);
1020 		else
1021 			b = test_bit_le(bit, paddr);
1022 		kunmap_atomic(paddr);
1023 		if (b) {
1024 			/* if the disk bit is set, set the memory bit */
1025 			int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1026 				      >= start);
1027 			bitmap_set_memory_bits(bitmap,
1028 					       (sector_t)i << bitmap->counts.chunkshift,
1029 					       needed);
1030 			bit_cnt++;
1031 		}
1032 		offset = 0;
1033 	}
1034 
1035 	printk(KERN_INFO "%s: bitmap initialized from disk: "
1036 	       "read %lu pages, set %lu of %lu bits\n",
1037 	       bmname(bitmap), store->file_pages,
1038 	       bit_cnt, chunks);
1039 
1040 	return 0;
1041 
1042  err:
1043 	printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
1044 	       bmname(bitmap), ret);
1045 	return ret;
1046 }
1047 
bitmap_write_all(struct bitmap * bitmap)1048 void bitmap_write_all(struct bitmap *bitmap)
1049 {
1050 	/* We don't actually write all bitmap blocks here,
1051 	 * just flag them as needing to be written
1052 	 */
1053 	int i;
1054 
1055 	if (!bitmap || !bitmap->storage.filemap)
1056 		return;
1057 	if (bitmap->storage.file)
1058 		/* Only one copy, so nothing needed */
1059 		return;
1060 
1061 	for (i = 0; i < bitmap->storage.file_pages; i++)
1062 		set_page_attr(bitmap, i,
1063 			      BITMAP_PAGE_NEEDWRITE);
1064 	bitmap->allclean = 0;
1065 }
1066 
bitmap_count_page(struct bitmap_counts * bitmap,sector_t offset,int inc)1067 static void bitmap_count_page(struct bitmap_counts *bitmap,
1068 			      sector_t offset, int inc)
1069 {
1070 	sector_t chunk = offset >> bitmap->chunkshift;
1071 	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1072 	bitmap->bp[page].count += inc;
1073 	bitmap_checkfree(bitmap, page);
1074 }
1075 
bitmap_set_pending(struct bitmap_counts * bitmap,sector_t offset)1076 static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1077 {
1078 	sector_t chunk = offset >> bitmap->chunkshift;
1079 	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1080 	struct bitmap_page *bp = &bitmap->bp[page];
1081 
1082 	if (!bp->pending)
1083 		bp->pending = 1;
1084 }
1085 
1086 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1087 					    sector_t offset, sector_t *blocks,
1088 					    int create);
1089 
1090 /*
1091  * bitmap daemon -- periodically wakes up to clean bits and flush pages
1092  *			out to disk
1093  */
1094 
bitmap_daemon_work(struct mddev * mddev)1095 void bitmap_daemon_work(struct mddev *mddev)
1096 {
1097 	struct bitmap *bitmap;
1098 	unsigned long j;
1099 	unsigned long nextpage;
1100 	sector_t blocks;
1101 	struct bitmap_counts *counts;
1102 
1103 	/* Use a mutex to guard daemon_work against
1104 	 * bitmap_destroy.
1105 	 */
1106 	mutex_lock(&mddev->bitmap_info.mutex);
1107 	bitmap = mddev->bitmap;
1108 	if (bitmap == NULL) {
1109 		mutex_unlock(&mddev->bitmap_info.mutex);
1110 		return;
1111 	}
1112 	if (time_before(jiffies, bitmap->daemon_lastrun
1113 			+ mddev->bitmap_info.daemon_sleep))
1114 		goto done;
1115 
1116 	bitmap->daemon_lastrun = jiffies;
1117 	if (bitmap->allclean) {
1118 		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1119 		goto done;
1120 	}
1121 	bitmap->allclean = 1;
1122 
1123 	/* Any file-page which is PENDING now needs to be written.
1124 	 * So set NEEDWRITE now, then after we make any last-minute changes
1125 	 * we will write it.
1126 	 */
1127 	for (j = 0; j < bitmap->storage.file_pages; j++)
1128 		if (test_and_clear_page_attr(bitmap, j,
1129 					     BITMAP_PAGE_PENDING))
1130 			set_page_attr(bitmap, j,
1131 				      BITMAP_PAGE_NEEDWRITE);
1132 
1133 	if (bitmap->need_sync &&
1134 	    mddev->bitmap_info.external == 0) {
1135 		/* Arrange for superblock update as well as
1136 		 * other changes */
1137 		bitmap_super_t *sb;
1138 		bitmap->need_sync = 0;
1139 		if (bitmap->storage.filemap) {
1140 			sb = kmap_atomic(bitmap->storage.sb_page);
1141 			sb->events_cleared =
1142 				cpu_to_le64(bitmap->events_cleared);
1143 			kunmap_atomic(sb);
1144 			set_page_attr(bitmap, 0,
1145 				      BITMAP_PAGE_NEEDWRITE);
1146 		}
1147 	}
1148 	/* Now look at the bitmap counters and if any are '2' or '1',
1149 	 * decrement and handle accordingly.
1150 	 */
1151 	counts = &bitmap->counts;
1152 	spin_lock_irq(&counts->lock);
1153 	nextpage = 0;
1154 	for (j = 0; j < counts->chunks; j++) {
1155 		bitmap_counter_t *bmc;
1156 		sector_t  block = (sector_t)j << counts->chunkshift;
1157 
1158 		if (j == nextpage) {
1159 			nextpage += PAGE_COUNTER_RATIO;
1160 			if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1161 				j |= PAGE_COUNTER_MASK;
1162 				continue;
1163 			}
1164 			counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1165 		}
1166 		bmc = bitmap_get_counter(counts,
1167 					 block,
1168 					 &blocks, 0);
1169 
1170 		if (!bmc) {
1171 			j |= PAGE_COUNTER_MASK;
1172 			continue;
1173 		}
1174 		if (*bmc == 1 && !bitmap->need_sync) {
1175 			/* We can clear the bit */
1176 			*bmc = 0;
1177 			bitmap_count_page(counts, block, -1);
1178 			bitmap_file_clear_bit(bitmap, block);
1179 		} else if (*bmc && *bmc <= 2) {
1180 			*bmc = 1;
1181 			bitmap_set_pending(counts, block);
1182 			bitmap->allclean = 0;
1183 		}
1184 	}
1185 	spin_unlock_irq(&counts->lock);
1186 
1187 	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1188 	 * DIRTY pages need to be written by bitmap_unplug so it can wait
1189 	 * for them.
1190 	 * If we find any DIRTY page we stop there and let bitmap_unplug
1191 	 * handle all the rest.  This is important in the case where
1192 	 * the first blocking holds the superblock and it has been updated.
1193 	 * We mustn't write any other blocks before the superblock.
1194 	 */
1195 	for (j = 0;
1196 	     j < bitmap->storage.file_pages
1197 		     && !test_bit(BITMAP_STALE, &bitmap->flags);
1198 	     j++) {
1199 
1200 		if (test_page_attr(bitmap, j,
1201 				   BITMAP_PAGE_DIRTY))
1202 			/* bitmap_unplug will handle the rest */
1203 			break;
1204 		if (test_and_clear_page_attr(bitmap, j,
1205 					     BITMAP_PAGE_NEEDWRITE)) {
1206 			write_page(bitmap, bitmap->storage.filemap[j], 0);
1207 		}
1208 	}
1209 
1210  done:
1211 	if (bitmap->allclean == 0)
1212 		mddev->thread->timeout =
1213 			mddev->bitmap_info.daemon_sleep;
1214 	mutex_unlock(&mddev->bitmap_info.mutex);
1215 }
1216 
bitmap_get_counter(struct bitmap_counts * bitmap,sector_t offset,sector_t * blocks,int create)1217 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1218 					    sector_t offset, sector_t *blocks,
1219 					    int create)
1220 __releases(bitmap->lock)
1221 __acquires(bitmap->lock)
1222 {
1223 	/* If 'create', we might release the lock and reclaim it.
1224 	 * The lock must have been taken with interrupts enabled.
1225 	 * If !create, we don't release the lock.
1226 	 */
1227 	sector_t chunk = offset >> bitmap->chunkshift;
1228 	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1229 	unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1230 	sector_t csize;
1231 	int err;
1232 
1233 	err = bitmap_checkpage(bitmap, page, create);
1234 
1235 	if (bitmap->bp[page].hijacked ||
1236 	    bitmap->bp[page].map == NULL)
1237 		csize = ((sector_t)1) << (bitmap->chunkshift +
1238 					  PAGE_COUNTER_SHIFT - 1);
1239 	else
1240 		csize = ((sector_t)1) << bitmap->chunkshift;
1241 	*blocks = csize - (offset & (csize - 1));
1242 
1243 	if (err < 0)
1244 		return NULL;
1245 
1246 	/* now locked ... */
1247 
1248 	if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1249 		/* should we use the first or second counter field
1250 		 * of the hijacked pointer? */
1251 		int hi = (pageoff > PAGE_COUNTER_MASK);
1252 		return  &((bitmap_counter_t *)
1253 			  &bitmap->bp[page].map)[hi];
1254 	} else /* page is allocated */
1255 		return (bitmap_counter_t *)
1256 			&(bitmap->bp[page].map[pageoff]);
1257 }
1258 
bitmap_startwrite(struct bitmap * bitmap,sector_t offset,unsigned long sectors,int behind)1259 int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1260 {
1261 	if (!bitmap)
1262 		return 0;
1263 
1264 	if (behind) {
1265 		int bw;
1266 		atomic_inc(&bitmap->behind_writes);
1267 		bw = atomic_read(&bitmap->behind_writes);
1268 		if (bw > bitmap->behind_writes_used)
1269 			bitmap->behind_writes_used = bw;
1270 
1271 		pr_debug("inc write-behind count %d/%lu\n",
1272 			 bw, bitmap->mddev->bitmap_info.max_write_behind);
1273 	}
1274 
1275 	while (sectors) {
1276 		sector_t blocks;
1277 		bitmap_counter_t *bmc;
1278 
1279 		spin_lock_irq(&bitmap->counts.lock);
1280 		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1281 		if (!bmc) {
1282 			spin_unlock_irq(&bitmap->counts.lock);
1283 			return 0;
1284 		}
1285 
1286 		if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1287 			DEFINE_WAIT(__wait);
1288 			/* note that it is safe to do the prepare_to_wait
1289 			 * after the test as long as we do it before dropping
1290 			 * the spinlock.
1291 			 */
1292 			prepare_to_wait(&bitmap->overflow_wait, &__wait,
1293 					TASK_UNINTERRUPTIBLE);
1294 			spin_unlock_irq(&bitmap->counts.lock);
1295 			schedule();
1296 			finish_wait(&bitmap->overflow_wait, &__wait);
1297 			continue;
1298 		}
1299 
1300 		switch (*bmc) {
1301 		case 0:
1302 			bitmap_file_set_bit(bitmap, offset);
1303 			bitmap_count_page(&bitmap->counts, offset, 1);
1304 			/* fall through */
1305 		case 1:
1306 			*bmc = 2;
1307 		}
1308 
1309 		(*bmc)++;
1310 
1311 		spin_unlock_irq(&bitmap->counts.lock);
1312 
1313 		offset += blocks;
1314 		if (sectors > blocks)
1315 			sectors -= blocks;
1316 		else
1317 			sectors = 0;
1318 	}
1319 	return 0;
1320 }
1321 EXPORT_SYMBOL(bitmap_startwrite);
1322 
bitmap_endwrite(struct bitmap * bitmap,sector_t offset,unsigned long sectors,int success,int behind)1323 void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1324 		     int success, int behind)
1325 {
1326 	if (!bitmap)
1327 		return;
1328 	if (behind) {
1329 		if (atomic_dec_and_test(&bitmap->behind_writes))
1330 			wake_up(&bitmap->behind_wait);
1331 		pr_debug("dec write-behind count %d/%lu\n",
1332 			 atomic_read(&bitmap->behind_writes),
1333 			 bitmap->mddev->bitmap_info.max_write_behind);
1334 	}
1335 
1336 	while (sectors) {
1337 		sector_t blocks;
1338 		unsigned long flags;
1339 		bitmap_counter_t *bmc;
1340 
1341 		spin_lock_irqsave(&bitmap->counts.lock, flags);
1342 		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1343 		if (!bmc) {
1344 			spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1345 			return;
1346 		}
1347 
1348 		if (success && !bitmap->mddev->degraded &&
1349 		    bitmap->events_cleared < bitmap->mddev->events) {
1350 			bitmap->events_cleared = bitmap->mddev->events;
1351 			bitmap->need_sync = 1;
1352 			sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1353 		}
1354 
1355 		if (!success && !NEEDED(*bmc))
1356 			*bmc |= NEEDED_MASK;
1357 
1358 		if (COUNTER(*bmc) == COUNTER_MAX)
1359 			wake_up(&bitmap->overflow_wait);
1360 
1361 		(*bmc)--;
1362 		if (*bmc <= 2) {
1363 			bitmap_set_pending(&bitmap->counts, offset);
1364 			bitmap->allclean = 0;
1365 		}
1366 		spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1367 		offset += blocks;
1368 		if (sectors > blocks)
1369 			sectors -= blocks;
1370 		else
1371 			sectors = 0;
1372 	}
1373 }
1374 EXPORT_SYMBOL(bitmap_endwrite);
1375 
__bitmap_start_sync(struct bitmap * bitmap,sector_t offset,sector_t * blocks,int degraded)1376 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1377 			       int degraded)
1378 {
1379 	bitmap_counter_t *bmc;
1380 	int rv;
1381 	if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1382 		*blocks = 1024;
1383 		return 1; /* always resync if no bitmap */
1384 	}
1385 	spin_lock_irq(&bitmap->counts.lock);
1386 	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1387 	rv = 0;
1388 	if (bmc) {
1389 		/* locked */
1390 		if (RESYNC(*bmc))
1391 			rv = 1;
1392 		else if (NEEDED(*bmc)) {
1393 			rv = 1;
1394 			if (!degraded) { /* don't set/clear bits if degraded */
1395 				*bmc |= RESYNC_MASK;
1396 				*bmc &= ~NEEDED_MASK;
1397 			}
1398 		}
1399 	}
1400 	spin_unlock_irq(&bitmap->counts.lock);
1401 	return rv;
1402 }
1403 
bitmap_start_sync(struct bitmap * bitmap,sector_t offset,sector_t * blocks,int degraded)1404 int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1405 		      int degraded)
1406 {
1407 	/* bitmap_start_sync must always report on multiples of whole
1408 	 * pages, otherwise resync (which is very PAGE_SIZE based) will
1409 	 * get confused.
1410 	 * So call __bitmap_start_sync repeatedly (if needed) until
1411 	 * At least PAGE_SIZE>>9 blocks are covered.
1412 	 * Return the 'or' of the result.
1413 	 */
1414 	int rv = 0;
1415 	sector_t blocks1;
1416 
1417 	*blocks = 0;
1418 	while (*blocks < (PAGE_SIZE>>9)) {
1419 		rv |= __bitmap_start_sync(bitmap, offset,
1420 					  &blocks1, degraded);
1421 		offset += blocks1;
1422 		*blocks += blocks1;
1423 	}
1424 	return rv;
1425 }
1426 EXPORT_SYMBOL(bitmap_start_sync);
1427 
bitmap_end_sync(struct bitmap * bitmap,sector_t offset,sector_t * blocks,int aborted)1428 void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1429 {
1430 	bitmap_counter_t *bmc;
1431 	unsigned long flags;
1432 
1433 	if (bitmap == NULL) {
1434 		*blocks = 1024;
1435 		return;
1436 	}
1437 	spin_lock_irqsave(&bitmap->counts.lock, flags);
1438 	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1439 	if (bmc == NULL)
1440 		goto unlock;
1441 	/* locked */
1442 	if (RESYNC(*bmc)) {
1443 		*bmc &= ~RESYNC_MASK;
1444 
1445 		if (!NEEDED(*bmc) && aborted)
1446 			*bmc |= NEEDED_MASK;
1447 		else {
1448 			if (*bmc <= 2) {
1449 				bitmap_set_pending(&bitmap->counts, offset);
1450 				bitmap->allclean = 0;
1451 			}
1452 		}
1453 	}
1454  unlock:
1455 	spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1456 }
1457 EXPORT_SYMBOL(bitmap_end_sync);
1458 
bitmap_close_sync(struct bitmap * bitmap)1459 void bitmap_close_sync(struct bitmap *bitmap)
1460 {
1461 	/* Sync has finished, and any bitmap chunks that weren't synced
1462 	 * properly have been aborted.  It remains to us to clear the
1463 	 * RESYNC bit wherever it is still on
1464 	 */
1465 	sector_t sector = 0;
1466 	sector_t blocks;
1467 	if (!bitmap)
1468 		return;
1469 	while (sector < bitmap->mddev->resync_max_sectors) {
1470 		bitmap_end_sync(bitmap, sector, &blocks, 0);
1471 		sector += blocks;
1472 	}
1473 }
1474 EXPORT_SYMBOL(bitmap_close_sync);
1475 
bitmap_cond_end_sync(struct bitmap * bitmap,sector_t sector)1476 void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1477 {
1478 	sector_t s = 0;
1479 	sector_t blocks;
1480 
1481 	if (!bitmap)
1482 		return;
1483 	if (sector == 0) {
1484 		bitmap->last_end_sync = jiffies;
1485 		return;
1486 	}
1487 	if (time_before(jiffies, (bitmap->last_end_sync
1488 				  + bitmap->mddev->bitmap_info.daemon_sleep)))
1489 		return;
1490 	wait_event(bitmap->mddev->recovery_wait,
1491 		   atomic_read(&bitmap->mddev->recovery_active) == 0);
1492 
1493 	bitmap->mddev->curr_resync_completed = sector;
1494 	set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1495 	sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1496 	s = 0;
1497 	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1498 		bitmap_end_sync(bitmap, s, &blocks, 0);
1499 		s += blocks;
1500 	}
1501 	bitmap->last_end_sync = jiffies;
1502 	sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1503 }
1504 EXPORT_SYMBOL(bitmap_cond_end_sync);
1505 
bitmap_set_memory_bits(struct bitmap * bitmap,sector_t offset,int needed)1506 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1507 {
1508 	/* For each chunk covered by any of these sectors, set the
1509 	 * counter to 2 and possibly set resync_needed.  They should all
1510 	 * be 0 at this point
1511 	 */
1512 
1513 	sector_t secs;
1514 	bitmap_counter_t *bmc;
1515 	spin_lock_irq(&bitmap->counts.lock);
1516 	bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1517 	if (!bmc) {
1518 		spin_unlock_irq(&bitmap->counts.lock);
1519 		return;
1520 	}
1521 	if (!*bmc) {
1522 		*bmc = 2 | (needed ? NEEDED_MASK : 0);
1523 		bitmap_count_page(&bitmap->counts, offset, 1);
1524 		bitmap_set_pending(&bitmap->counts, offset);
1525 		bitmap->allclean = 0;
1526 	}
1527 	spin_unlock_irq(&bitmap->counts.lock);
1528 }
1529 
1530 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
bitmap_dirty_bits(struct bitmap * bitmap,unsigned long s,unsigned long e)1531 void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1532 {
1533 	unsigned long chunk;
1534 
1535 	for (chunk = s; chunk <= e; chunk++) {
1536 		sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1537 		bitmap_set_memory_bits(bitmap, sec, 1);
1538 		bitmap_file_set_bit(bitmap, sec);
1539 		if (sec < bitmap->mddev->recovery_cp)
1540 			/* We are asserting that the array is dirty,
1541 			 * so move the recovery_cp address back so
1542 			 * that it is obvious that it is dirty
1543 			 */
1544 			bitmap->mddev->recovery_cp = sec;
1545 	}
1546 }
1547 
1548 /*
1549  * flush out any pending updates
1550  */
bitmap_flush(struct mddev * mddev)1551 void bitmap_flush(struct mddev *mddev)
1552 {
1553 	struct bitmap *bitmap = mddev->bitmap;
1554 	long sleep;
1555 
1556 	if (!bitmap) /* there was no bitmap */
1557 		return;
1558 
1559 	/* run the daemon_work three time to ensure everything is flushed
1560 	 * that can be
1561 	 */
1562 	sleep = mddev->bitmap_info.daemon_sleep * 2;
1563 	bitmap->daemon_lastrun -= sleep;
1564 	bitmap_daemon_work(mddev);
1565 	bitmap->daemon_lastrun -= sleep;
1566 	bitmap_daemon_work(mddev);
1567 	bitmap->daemon_lastrun -= sleep;
1568 	bitmap_daemon_work(mddev);
1569 	bitmap_update_sb(bitmap);
1570 }
1571 
1572 /*
1573  * free memory that was allocated
1574  */
bitmap_free(struct bitmap * bitmap)1575 static void bitmap_free(struct bitmap *bitmap)
1576 {
1577 	unsigned long k, pages;
1578 	struct bitmap_page *bp;
1579 
1580 	if (!bitmap) /* there was no bitmap */
1581 		return;
1582 
1583 	/* Shouldn't be needed - but just in case.... */
1584 	wait_event(bitmap->write_wait,
1585 		   atomic_read(&bitmap->pending_writes) == 0);
1586 
1587 	/* release the bitmap file  */
1588 	bitmap_file_unmap(&bitmap->storage);
1589 
1590 	bp = bitmap->counts.bp;
1591 	pages = bitmap->counts.pages;
1592 
1593 	/* free all allocated memory */
1594 
1595 	if (bp) /* deallocate the page memory */
1596 		for (k = 0; k < pages; k++)
1597 			if (bp[k].map && !bp[k].hijacked)
1598 				kfree(bp[k].map);
1599 	kfree(bp);
1600 	kfree(bitmap);
1601 }
1602 
bitmap_destroy(struct mddev * mddev)1603 void bitmap_destroy(struct mddev *mddev)
1604 {
1605 	struct bitmap *bitmap = mddev->bitmap;
1606 
1607 	if (!bitmap) /* there was no bitmap */
1608 		return;
1609 
1610 	mutex_lock(&mddev->bitmap_info.mutex);
1611 	mddev->bitmap = NULL; /* disconnect from the md device */
1612 	mutex_unlock(&mddev->bitmap_info.mutex);
1613 	if (mddev->thread)
1614 		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1615 
1616 	if (bitmap->sysfs_can_clear)
1617 		sysfs_put(bitmap->sysfs_can_clear);
1618 
1619 	bitmap_free(bitmap);
1620 }
1621 
1622 /*
1623  * initialize the bitmap structure
1624  * if this returns an error, bitmap_destroy must be called to do clean up
1625  */
bitmap_create(struct mddev * mddev)1626 int bitmap_create(struct mddev *mddev)
1627 {
1628 	struct bitmap *bitmap;
1629 	sector_t blocks = mddev->resync_max_sectors;
1630 	struct file *file = mddev->bitmap_info.file;
1631 	int err;
1632 	struct kernfs_node *bm = NULL;
1633 
1634 	BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1635 
1636 	BUG_ON(file && mddev->bitmap_info.offset);
1637 
1638 	bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1639 	if (!bitmap)
1640 		return -ENOMEM;
1641 
1642 	spin_lock_init(&bitmap->counts.lock);
1643 	atomic_set(&bitmap->pending_writes, 0);
1644 	init_waitqueue_head(&bitmap->write_wait);
1645 	init_waitqueue_head(&bitmap->overflow_wait);
1646 	init_waitqueue_head(&bitmap->behind_wait);
1647 
1648 	bitmap->mddev = mddev;
1649 
1650 	if (mddev->kobj.sd)
1651 		bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1652 	if (bm) {
1653 		bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1654 		sysfs_put(bm);
1655 	} else
1656 		bitmap->sysfs_can_clear = NULL;
1657 
1658 	bitmap->storage.file = file;
1659 	if (file) {
1660 		get_file(file);
1661 		/* As future accesses to this file will use bmap,
1662 		 * and bypass the page cache, we must sync the file
1663 		 * first.
1664 		 */
1665 		vfs_fsync(file, 1);
1666 	}
1667 	/* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1668 	if (!mddev->bitmap_info.external) {
1669 		/*
1670 		 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1671 		 * instructing us to create a new on-disk bitmap instance.
1672 		 */
1673 		if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1674 			err = bitmap_new_disk_sb(bitmap);
1675 		else
1676 			err = bitmap_read_sb(bitmap);
1677 	} else {
1678 		err = 0;
1679 		if (mddev->bitmap_info.chunksize == 0 ||
1680 		    mddev->bitmap_info.daemon_sleep == 0)
1681 			/* chunksize and time_base need to be
1682 			 * set first. */
1683 			err = -EINVAL;
1684 	}
1685 	if (err)
1686 		goto error;
1687 
1688 	bitmap->daemon_lastrun = jiffies;
1689 	err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1690 	if (err)
1691 		goto error;
1692 
1693 	printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1694 	       bitmap->counts.pages, bmname(bitmap));
1695 
1696 	mddev->bitmap = bitmap;
1697 	return test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1698 
1699  error:
1700 	bitmap_free(bitmap);
1701 	return err;
1702 }
1703 
bitmap_load(struct mddev * mddev)1704 int bitmap_load(struct mddev *mddev)
1705 {
1706 	int err = 0;
1707 	sector_t start = 0;
1708 	sector_t sector = 0;
1709 	struct bitmap *bitmap = mddev->bitmap;
1710 
1711 	if (!bitmap)
1712 		goto out;
1713 
1714 	/* Clear out old bitmap info first:  Either there is none, or we
1715 	 * are resuming after someone else has possibly changed things,
1716 	 * so we should forget old cached info.
1717 	 * All chunks should be clean, but some might need_sync.
1718 	 */
1719 	while (sector < mddev->resync_max_sectors) {
1720 		sector_t blocks;
1721 		bitmap_start_sync(bitmap, sector, &blocks, 0);
1722 		sector += blocks;
1723 	}
1724 	bitmap_close_sync(bitmap);
1725 
1726 	if (mddev->degraded == 0
1727 	    || bitmap->events_cleared == mddev->events)
1728 		/* no need to keep dirty bits to optimise a
1729 		 * re-add of a missing device */
1730 		start = mddev->recovery_cp;
1731 
1732 	mutex_lock(&mddev->bitmap_info.mutex);
1733 	err = bitmap_init_from_disk(bitmap, start);
1734 	mutex_unlock(&mddev->bitmap_info.mutex);
1735 
1736 	if (err)
1737 		goto out;
1738 	clear_bit(BITMAP_STALE, &bitmap->flags);
1739 
1740 	/* Kick recovery in case any bits were set */
1741 	set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1742 
1743 	mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1744 	md_wakeup_thread(mddev->thread);
1745 
1746 	bitmap_update_sb(bitmap);
1747 
1748 	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1749 		err = -EIO;
1750 out:
1751 	return err;
1752 }
1753 EXPORT_SYMBOL_GPL(bitmap_load);
1754 
bitmap_status(struct seq_file * seq,struct bitmap * bitmap)1755 void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
1756 {
1757 	unsigned long chunk_kb;
1758 	struct bitmap_counts *counts;
1759 
1760 	if (!bitmap)
1761 		return;
1762 
1763 	counts = &bitmap->counts;
1764 
1765 	chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
1766 	seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
1767 		   "%lu%s chunk",
1768 		   counts->pages - counts->missing_pages,
1769 		   counts->pages,
1770 		   (counts->pages - counts->missing_pages)
1771 		   << (PAGE_SHIFT - 10),
1772 		   chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
1773 		   chunk_kb ? "KB" : "B");
1774 	if (bitmap->storage.file) {
1775 		seq_printf(seq, ", file: ");
1776 		seq_path(seq, &bitmap->storage.file->f_path, " \t\n");
1777 	}
1778 
1779 	seq_printf(seq, "\n");
1780 }
1781 
bitmap_resize(struct bitmap * bitmap,sector_t blocks,int chunksize,int init)1782 int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
1783 		  int chunksize, int init)
1784 {
1785 	/* If chunk_size is 0, choose an appropriate chunk size.
1786 	 * Then possibly allocate new storage space.
1787 	 * Then quiesce, copy bits, replace bitmap, and re-start
1788 	 *
1789 	 * This function is called both to set up the initial bitmap
1790 	 * and to resize the bitmap while the array is active.
1791 	 * If this happens as a result of the array being resized,
1792 	 * chunksize will be zero, and we need to choose a suitable
1793 	 * chunksize, otherwise we use what we are given.
1794 	 */
1795 	struct bitmap_storage store;
1796 	struct bitmap_counts old_counts;
1797 	unsigned long chunks;
1798 	sector_t block;
1799 	sector_t old_blocks, new_blocks;
1800 	int chunkshift;
1801 	int ret = 0;
1802 	long pages;
1803 	struct bitmap_page *new_bp;
1804 
1805 	if (bitmap->storage.file && !init) {
1806 		pr_info("md: cannot resize file-based bitmap\n");
1807 		return -EINVAL;
1808 	}
1809 
1810 	if (chunksize == 0) {
1811 		/* If there is enough space, leave the chunk size unchanged,
1812 		 * else increase by factor of two until there is enough space.
1813 		 */
1814 		long bytes;
1815 		long space = bitmap->mddev->bitmap_info.space;
1816 
1817 		if (space == 0) {
1818 			/* We don't know how much space there is, so limit
1819 			 * to current size - in sectors.
1820 			 */
1821 			bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
1822 			if (!bitmap->mddev->bitmap_info.external)
1823 				bytes += sizeof(bitmap_super_t);
1824 			space = DIV_ROUND_UP(bytes, 512);
1825 			bitmap->mddev->bitmap_info.space = space;
1826 		}
1827 		chunkshift = bitmap->counts.chunkshift;
1828 		chunkshift--;
1829 		do {
1830 			/* 'chunkshift' is shift from block size to chunk size */
1831 			chunkshift++;
1832 			chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1833 			bytes = DIV_ROUND_UP(chunks, 8);
1834 			if (!bitmap->mddev->bitmap_info.external)
1835 				bytes += sizeof(bitmap_super_t);
1836 		} while (bytes > (space << 9));
1837 	} else
1838 		chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
1839 
1840 	chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1841 	memset(&store, 0, sizeof(store));
1842 	if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
1843 		ret = bitmap_storage_alloc(&store, chunks,
1844 					   !bitmap->mddev->bitmap_info.external);
1845 	if (ret)
1846 		goto err;
1847 
1848 	pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
1849 
1850 	new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
1851 	ret = -ENOMEM;
1852 	if (!new_bp) {
1853 		bitmap_file_unmap(&store);
1854 		goto err;
1855 	}
1856 
1857 	if (!init)
1858 		bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
1859 
1860 	store.file = bitmap->storage.file;
1861 	bitmap->storage.file = NULL;
1862 
1863 	if (store.sb_page && bitmap->storage.sb_page)
1864 		memcpy(page_address(store.sb_page),
1865 		       page_address(bitmap->storage.sb_page),
1866 		       sizeof(bitmap_super_t));
1867 	bitmap_file_unmap(&bitmap->storage);
1868 	bitmap->storage = store;
1869 
1870 	old_counts = bitmap->counts;
1871 	bitmap->counts.bp = new_bp;
1872 	bitmap->counts.pages = pages;
1873 	bitmap->counts.missing_pages = pages;
1874 	bitmap->counts.chunkshift = chunkshift;
1875 	bitmap->counts.chunks = chunks;
1876 	bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
1877 						     BITMAP_BLOCK_SHIFT);
1878 
1879 	blocks = min(old_counts.chunks << old_counts.chunkshift,
1880 		     chunks << chunkshift);
1881 
1882 	spin_lock_irq(&bitmap->counts.lock);
1883 	for (block = 0; block < blocks; ) {
1884 		bitmap_counter_t *bmc_old, *bmc_new;
1885 		int set;
1886 
1887 		bmc_old = bitmap_get_counter(&old_counts, block,
1888 					     &old_blocks, 0);
1889 		set = bmc_old && NEEDED(*bmc_old);
1890 
1891 		if (set) {
1892 			bmc_new = bitmap_get_counter(&bitmap->counts, block,
1893 						     &new_blocks, 1);
1894 			if (*bmc_new == 0) {
1895 				/* need to set on-disk bits too. */
1896 				sector_t end = block + new_blocks;
1897 				sector_t start = block >> chunkshift;
1898 				start <<= chunkshift;
1899 				while (start < end) {
1900 					bitmap_file_set_bit(bitmap, block);
1901 					start += 1 << chunkshift;
1902 				}
1903 				*bmc_new = 2;
1904 				bitmap_count_page(&bitmap->counts,
1905 						  block, 1);
1906 				bitmap_set_pending(&bitmap->counts,
1907 						   block);
1908 			}
1909 			*bmc_new |= NEEDED_MASK;
1910 			if (new_blocks < old_blocks)
1911 				old_blocks = new_blocks;
1912 		}
1913 		block += old_blocks;
1914 	}
1915 
1916 	if (!init) {
1917 		int i;
1918 		while (block < (chunks << chunkshift)) {
1919 			bitmap_counter_t *bmc;
1920 			bmc = bitmap_get_counter(&bitmap->counts, block,
1921 						 &new_blocks, 1);
1922 			if (bmc) {
1923 				/* new space.  It needs to be resynced, so
1924 				 * we set NEEDED_MASK.
1925 				 */
1926 				if (*bmc == 0) {
1927 					*bmc = NEEDED_MASK | 2;
1928 					bitmap_count_page(&bitmap->counts,
1929 							  block, 1);
1930 					bitmap_set_pending(&bitmap->counts,
1931 							   block);
1932 				}
1933 			}
1934 			block += new_blocks;
1935 		}
1936 		for (i = 0; i < bitmap->storage.file_pages; i++)
1937 			set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1938 	}
1939 	spin_unlock_irq(&bitmap->counts.lock);
1940 
1941 	if (!init) {
1942 		bitmap_unplug(bitmap);
1943 		bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
1944 	}
1945 	ret = 0;
1946 err:
1947 	return ret;
1948 }
1949 EXPORT_SYMBOL_GPL(bitmap_resize);
1950 
1951 static ssize_t
location_show(struct mddev * mddev,char * page)1952 location_show(struct mddev *mddev, char *page)
1953 {
1954 	ssize_t len;
1955 	if (mddev->bitmap_info.file)
1956 		len = sprintf(page, "file");
1957 	else if (mddev->bitmap_info.offset)
1958 		len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
1959 	else
1960 		len = sprintf(page, "none");
1961 	len += sprintf(page+len, "\n");
1962 	return len;
1963 }
1964 
1965 static ssize_t
location_store(struct mddev * mddev,const char * buf,size_t len)1966 location_store(struct mddev *mddev, const char *buf, size_t len)
1967 {
1968 
1969 	if (mddev->pers) {
1970 		if (!mddev->pers->quiesce)
1971 			return -EBUSY;
1972 		if (mddev->recovery || mddev->sync_thread)
1973 			return -EBUSY;
1974 	}
1975 
1976 	if (mddev->bitmap || mddev->bitmap_info.file ||
1977 	    mddev->bitmap_info.offset) {
1978 		/* bitmap already configured.  Only option is to clear it */
1979 		if (strncmp(buf, "none", 4) != 0)
1980 			return -EBUSY;
1981 		if (mddev->pers) {
1982 			mddev->pers->quiesce(mddev, 1);
1983 			bitmap_destroy(mddev);
1984 			mddev->pers->quiesce(mddev, 0);
1985 		}
1986 		mddev->bitmap_info.offset = 0;
1987 		if (mddev->bitmap_info.file) {
1988 			struct file *f = mddev->bitmap_info.file;
1989 			mddev->bitmap_info.file = NULL;
1990 			fput(f);
1991 		}
1992 	} else {
1993 		/* No bitmap, OK to set a location */
1994 		long long offset;
1995 		if (strncmp(buf, "none", 4) == 0)
1996 			/* nothing to be done */;
1997 		else if (strncmp(buf, "file:", 5) == 0) {
1998 			/* Not supported yet */
1999 			return -EINVAL;
2000 		} else {
2001 			int rv;
2002 			if (buf[0] == '+')
2003 				rv = kstrtoll(buf+1, 10, &offset);
2004 			else
2005 				rv = kstrtoll(buf, 10, &offset);
2006 			if (rv)
2007 				return rv;
2008 			if (offset == 0)
2009 				return -EINVAL;
2010 			if (mddev->bitmap_info.external == 0 &&
2011 			    mddev->major_version == 0 &&
2012 			    offset != mddev->bitmap_info.default_offset)
2013 				return -EINVAL;
2014 			mddev->bitmap_info.offset = offset;
2015 			if (mddev->pers) {
2016 				mddev->pers->quiesce(mddev, 1);
2017 				rv = bitmap_create(mddev);
2018 				if (!rv)
2019 					rv = bitmap_load(mddev);
2020 				if (rv) {
2021 					bitmap_destroy(mddev);
2022 					mddev->bitmap_info.offset = 0;
2023 				}
2024 				mddev->pers->quiesce(mddev, 0);
2025 				if (rv)
2026 					return rv;
2027 			}
2028 		}
2029 	}
2030 	if (!mddev->external) {
2031 		/* Ensure new bitmap info is stored in
2032 		 * metadata promptly.
2033 		 */
2034 		set_bit(MD_CHANGE_DEVS, &mddev->flags);
2035 		md_wakeup_thread(mddev->thread);
2036 	}
2037 	return len;
2038 }
2039 
2040 static struct md_sysfs_entry bitmap_location =
2041 __ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2042 
2043 /* 'bitmap/space' is the space available at 'location' for the
2044  * bitmap.  This allows the kernel to know when it is safe to
2045  * resize the bitmap to match a resized array.
2046  */
2047 static ssize_t
space_show(struct mddev * mddev,char * page)2048 space_show(struct mddev *mddev, char *page)
2049 {
2050 	return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2051 }
2052 
2053 static ssize_t
space_store(struct mddev * mddev,const char * buf,size_t len)2054 space_store(struct mddev *mddev, const char *buf, size_t len)
2055 {
2056 	unsigned long sectors;
2057 	int rv;
2058 
2059 	rv = kstrtoul(buf, 10, &sectors);
2060 	if (rv)
2061 		return rv;
2062 
2063 	if (sectors == 0)
2064 		return -EINVAL;
2065 
2066 	if (mddev->bitmap &&
2067 	    sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2068 		return -EFBIG; /* Bitmap is too big for this small space */
2069 
2070 	/* could make sure it isn't too big, but that isn't really
2071 	 * needed - user-space should be careful.
2072 	 */
2073 	mddev->bitmap_info.space = sectors;
2074 	return len;
2075 }
2076 
2077 static struct md_sysfs_entry bitmap_space =
2078 __ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2079 
2080 static ssize_t
timeout_show(struct mddev * mddev,char * page)2081 timeout_show(struct mddev *mddev, char *page)
2082 {
2083 	ssize_t len;
2084 	unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2085 	unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2086 
2087 	len = sprintf(page, "%lu", secs);
2088 	if (jifs)
2089 		len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2090 	len += sprintf(page+len, "\n");
2091 	return len;
2092 }
2093 
2094 static ssize_t
timeout_store(struct mddev * mddev,const char * buf,size_t len)2095 timeout_store(struct mddev *mddev, const char *buf, size_t len)
2096 {
2097 	/* timeout can be set at any time */
2098 	unsigned long timeout;
2099 	int rv = strict_strtoul_scaled(buf, &timeout, 4);
2100 	if (rv)
2101 		return rv;
2102 
2103 	/* just to make sure we don't overflow... */
2104 	if (timeout >= LONG_MAX / HZ)
2105 		return -EINVAL;
2106 
2107 	timeout = timeout * HZ / 10000;
2108 
2109 	if (timeout >= MAX_SCHEDULE_TIMEOUT)
2110 		timeout = MAX_SCHEDULE_TIMEOUT-1;
2111 	if (timeout < 1)
2112 		timeout = 1;
2113 	mddev->bitmap_info.daemon_sleep = timeout;
2114 	if (mddev->thread) {
2115 		/* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
2116 		 * the bitmap is all clean and we don't need to
2117 		 * adjust the timeout right now
2118 		 */
2119 		if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2120 			mddev->thread->timeout = timeout;
2121 			md_wakeup_thread(mddev->thread);
2122 		}
2123 	}
2124 	return len;
2125 }
2126 
2127 static struct md_sysfs_entry bitmap_timeout =
2128 __ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2129 
2130 static ssize_t
backlog_show(struct mddev * mddev,char * page)2131 backlog_show(struct mddev *mddev, char *page)
2132 {
2133 	return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2134 }
2135 
2136 static ssize_t
backlog_store(struct mddev * mddev,const char * buf,size_t len)2137 backlog_store(struct mddev *mddev, const char *buf, size_t len)
2138 {
2139 	unsigned long backlog;
2140 	int rv = kstrtoul(buf, 10, &backlog);
2141 	if (rv)
2142 		return rv;
2143 	if (backlog > COUNTER_MAX)
2144 		return -EINVAL;
2145 	mddev->bitmap_info.max_write_behind = backlog;
2146 	return len;
2147 }
2148 
2149 static struct md_sysfs_entry bitmap_backlog =
2150 __ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2151 
2152 static ssize_t
chunksize_show(struct mddev * mddev,char * page)2153 chunksize_show(struct mddev *mddev, char *page)
2154 {
2155 	return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2156 }
2157 
2158 static ssize_t
chunksize_store(struct mddev * mddev,const char * buf,size_t len)2159 chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2160 {
2161 	/* Can only be changed when no bitmap is active */
2162 	int rv;
2163 	unsigned long csize;
2164 	if (mddev->bitmap)
2165 		return -EBUSY;
2166 	rv = kstrtoul(buf, 10, &csize);
2167 	if (rv)
2168 		return rv;
2169 	if (csize < 512 ||
2170 	    !is_power_of_2(csize))
2171 		return -EINVAL;
2172 	mddev->bitmap_info.chunksize = csize;
2173 	return len;
2174 }
2175 
2176 static struct md_sysfs_entry bitmap_chunksize =
2177 __ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2178 
metadata_show(struct mddev * mddev,char * page)2179 static ssize_t metadata_show(struct mddev *mddev, char *page)
2180 {
2181 	return sprintf(page, "%s\n", (mddev->bitmap_info.external
2182 				      ? "external" : "internal"));
2183 }
2184 
metadata_store(struct mddev * mddev,const char * buf,size_t len)2185 static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2186 {
2187 	if (mddev->bitmap ||
2188 	    mddev->bitmap_info.file ||
2189 	    mddev->bitmap_info.offset)
2190 		return -EBUSY;
2191 	if (strncmp(buf, "external", 8) == 0)
2192 		mddev->bitmap_info.external = 1;
2193 	else if (strncmp(buf, "internal", 8) == 0)
2194 		mddev->bitmap_info.external = 0;
2195 	else
2196 		return -EINVAL;
2197 	return len;
2198 }
2199 
2200 static struct md_sysfs_entry bitmap_metadata =
2201 __ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2202 
can_clear_show(struct mddev * mddev,char * page)2203 static ssize_t can_clear_show(struct mddev *mddev, char *page)
2204 {
2205 	int len;
2206 	if (mddev->bitmap)
2207 		len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2208 					     "false" : "true"));
2209 	else
2210 		len = sprintf(page, "\n");
2211 	return len;
2212 }
2213 
can_clear_store(struct mddev * mddev,const char * buf,size_t len)2214 static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2215 {
2216 	if (mddev->bitmap == NULL)
2217 		return -ENOENT;
2218 	if (strncmp(buf, "false", 5) == 0)
2219 		mddev->bitmap->need_sync = 1;
2220 	else if (strncmp(buf, "true", 4) == 0) {
2221 		if (mddev->degraded)
2222 			return -EBUSY;
2223 		mddev->bitmap->need_sync = 0;
2224 	} else
2225 		return -EINVAL;
2226 	return len;
2227 }
2228 
2229 static struct md_sysfs_entry bitmap_can_clear =
2230 __ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2231 
2232 static ssize_t
behind_writes_used_show(struct mddev * mddev,char * page)2233 behind_writes_used_show(struct mddev *mddev, char *page)
2234 {
2235 	if (mddev->bitmap == NULL)
2236 		return sprintf(page, "0\n");
2237 	return sprintf(page, "%lu\n",
2238 		       mddev->bitmap->behind_writes_used);
2239 }
2240 
2241 static ssize_t
behind_writes_used_reset(struct mddev * mddev,const char * buf,size_t len)2242 behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2243 {
2244 	if (mddev->bitmap)
2245 		mddev->bitmap->behind_writes_used = 0;
2246 	return len;
2247 }
2248 
2249 static struct md_sysfs_entry max_backlog_used =
2250 __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2251        behind_writes_used_show, behind_writes_used_reset);
2252 
2253 static struct attribute *md_bitmap_attrs[] = {
2254 	&bitmap_location.attr,
2255 	&bitmap_space.attr,
2256 	&bitmap_timeout.attr,
2257 	&bitmap_backlog.attr,
2258 	&bitmap_chunksize.attr,
2259 	&bitmap_metadata.attr,
2260 	&bitmap_can_clear.attr,
2261 	&max_backlog_used.attr,
2262 	NULL
2263 };
2264 struct attribute_group md_bitmap_group = {
2265 	.name = "bitmap",
2266 	.attrs = md_bitmap_attrs,
2267 };
2268 
2269