• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  /*
2   * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
3   *
4   * bitmap_create  - sets up the bitmap structure
5   * bitmap_destroy - destroys the bitmap structure
6   *
7   * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
8   * - added disk storage for bitmap
9   * - changes to allow various bitmap chunk sizes
10   */
11  
12  /*
13   * Still to do:
14   *
15   * flush after percent set rather than just time based. (maybe both).
16   */
17  
18  #include <linux/blkdev.h>
19  #include <linux/module.h>
20  #include <linux/errno.h>
21  #include <linux/slab.h>
22  #include <linux/init.h>
23  #include <linux/timer.h>
24  #include <linux/sched.h>
25  #include <linux/list.h>
26  #include <linux/file.h>
27  #include <linux/mount.h>
28  #include <linux/buffer_head.h>
29  #include <linux/seq_file.h>
30  #include "md.h"
31  #include "bitmap.h"
32  
bmname(struct bitmap * bitmap)33  static inline char *bmname(struct bitmap *bitmap)
34  {
35  	return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
36  }
37  
38  /*
39   * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
40   *
41   * 1) check to see if this page is allocated, if it's not then try to alloc
42   * 2) if the alloc fails, set the page's hijacked flag so we'll use the
43   *    page pointer directly as a counter
44   *
45   * if we find our page, we increment the page's refcount so that it stays
46   * allocated while we're using it
47   */
bitmap_checkpage(struct bitmap_counts * bitmap,unsigned long page,int create)48  static int bitmap_checkpage(struct bitmap_counts *bitmap,
49  			    unsigned long page, int create)
50  __releases(bitmap->lock)
51  __acquires(bitmap->lock)
52  {
53  	unsigned char *mappage;
54  
55  	if (page >= bitmap->pages) {
56  		/* This can happen if bitmap_start_sync goes beyond
57  		 * End-of-device while looking for a whole page.
58  		 * It is harmless.
59  		 */
60  		return -EINVAL;
61  	}
62  
63  	if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
64  		return 0;
65  
66  	if (bitmap->bp[page].map) /* page is already allocated, just return */
67  		return 0;
68  
69  	if (!create)
70  		return -ENOENT;
71  
72  	/* this page has not been allocated yet */
73  
74  	spin_unlock_irq(&bitmap->lock);
75  	mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
76  	spin_lock_irq(&bitmap->lock);
77  
78  	if (mappage == NULL) {
79  		pr_debug("md/bitmap: map page allocation failed, hijacking\n");
80  		/* failed - set the hijacked flag so that we can use the
81  		 * pointer as a counter */
82  		if (!bitmap->bp[page].map)
83  			bitmap->bp[page].hijacked = 1;
84  	} else if (bitmap->bp[page].map ||
85  		   bitmap->bp[page].hijacked) {
86  		/* somebody beat us to getting the page */
87  		kfree(mappage);
88  		return 0;
89  	} else {
90  
91  		/* no page was in place and we have one, so install it */
92  
93  		bitmap->bp[page].map = mappage;
94  		bitmap->missing_pages--;
95  	}
96  	return 0;
97  }
98  
99  /* if page is completely empty, put it back on the free list, or dealloc it */
100  /* if page was hijacked, unmark the flag so it might get alloced next time */
101  /* Note: lock should be held when calling this */
bitmap_checkfree(struct bitmap_counts * bitmap,unsigned long page)102  static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
103  {
104  	char *ptr;
105  
106  	if (bitmap->bp[page].count) /* page is still busy */
107  		return;
108  
109  	/* page is no longer in use, it can be released */
110  
111  	if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
112  		bitmap->bp[page].hijacked = 0;
113  		bitmap->bp[page].map = NULL;
114  	} else {
115  		/* normal case, free the page */
116  		ptr = bitmap->bp[page].map;
117  		bitmap->bp[page].map = NULL;
118  		bitmap->missing_pages++;
119  		kfree(ptr);
120  	}
121  }
122  
123  /*
124   * bitmap file handling - read and write the bitmap file and its superblock
125   */
126  
127  /*
128   * basic page I/O operations
129   */
130  
131  /* IO operations when bitmap is stored near all superblocks */
read_sb_page(struct mddev * mddev,loff_t offset,struct page * page,unsigned long index,int size)132  static int read_sb_page(struct mddev *mddev, loff_t offset,
133  			struct page *page,
134  			unsigned long index, int size)
135  {
136  	/* choose a good rdev and read the page from there */
137  
138  	struct md_rdev *rdev;
139  	sector_t target;
140  
141  	rdev_for_each(rdev, mddev) {
142  		if (! test_bit(In_sync, &rdev->flags)
143  		    || test_bit(Faulty, &rdev->flags))
144  			continue;
145  
146  		target = offset + index * (PAGE_SIZE/512);
147  
148  		if (sync_page_io(rdev, target,
149  				 roundup(size, bdev_logical_block_size(rdev->bdev)),
150  				 page, READ, true)) {
151  			page->index = index;
152  			return 0;
153  		}
154  	}
155  	return -EIO;
156  }
157  
next_active_rdev(struct md_rdev * rdev,struct mddev * mddev)158  static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
159  {
160  	/* Iterate the disks of an mddev, using rcu to protect access to the
161  	 * linked list, and raising the refcount of devices we return to ensure
162  	 * they don't disappear while in use.
163  	 * As devices are only added or removed when raid_disk is < 0 and
164  	 * nr_pending is 0 and In_sync is clear, the entries we return will
165  	 * still be in the same position on the list when we re-enter
166  	 * list_for_each_entry_continue_rcu.
167  	 */
168  	rcu_read_lock();
169  	if (rdev == NULL)
170  		/* start at the beginning */
171  		rdev = list_entry_rcu(&mddev->disks, struct md_rdev, same_set);
172  	else {
173  		/* release the previous rdev and start from there. */
174  		rdev_dec_pending(rdev, mddev);
175  	}
176  	list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
177  		if (rdev->raid_disk >= 0 &&
178  		    !test_bit(Faulty, &rdev->flags)) {
179  			/* this is a usable devices */
180  			atomic_inc(&rdev->nr_pending);
181  			rcu_read_unlock();
182  			return rdev;
183  		}
184  	}
185  	rcu_read_unlock();
186  	return NULL;
187  }
188  
write_sb_page(struct bitmap * bitmap,struct page * page,int wait)189  static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
190  {
191  	struct md_rdev *rdev = NULL;
192  	struct block_device *bdev;
193  	struct mddev *mddev = bitmap->mddev;
194  	struct bitmap_storage *store = &bitmap->storage;
195  
196  	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
197  		int size = PAGE_SIZE;
198  		loff_t offset = mddev->bitmap_info.offset;
199  
200  		bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
201  
202  		if (page->index == store->file_pages-1) {
203  			int last_page_size = store->bytes & (PAGE_SIZE-1);
204  			if (last_page_size == 0)
205  				last_page_size = PAGE_SIZE;
206  			size = roundup(last_page_size,
207  				       bdev_logical_block_size(bdev));
208  		}
209  		/* Just make sure we aren't corrupting data or
210  		 * metadata
211  		 */
212  		if (mddev->external) {
213  			/* Bitmap could be anywhere. */
214  			if (rdev->sb_start + offset + (page->index
215  						       * (PAGE_SIZE/512))
216  			    > rdev->data_offset
217  			    &&
218  			    rdev->sb_start + offset
219  			    < (rdev->data_offset + mddev->dev_sectors
220  			     + (PAGE_SIZE/512)))
221  				goto bad_alignment;
222  		} else if (offset < 0) {
223  			/* DATA  BITMAP METADATA  */
224  			if (offset
225  			    + (long)(page->index * (PAGE_SIZE/512))
226  			    + size/512 > 0)
227  				/* bitmap runs in to metadata */
228  				goto bad_alignment;
229  			if (rdev->data_offset + mddev->dev_sectors
230  			    > rdev->sb_start + offset)
231  				/* data runs in to bitmap */
232  				goto bad_alignment;
233  		} else if (rdev->sb_start < rdev->data_offset) {
234  			/* METADATA BITMAP DATA */
235  			if (rdev->sb_start
236  			    + offset
237  			    + page->index*(PAGE_SIZE/512) + size/512
238  			    > rdev->data_offset)
239  				/* bitmap runs in to data */
240  				goto bad_alignment;
241  		} else {
242  			/* DATA METADATA BITMAP - no problems */
243  		}
244  		md_super_write(mddev, rdev,
245  			       rdev->sb_start + offset
246  			       + page->index * (PAGE_SIZE/512),
247  			       size,
248  			       page);
249  	}
250  
251  	if (wait)
252  		md_super_wait(mddev);
253  	return 0;
254  
255   bad_alignment:
256  	return -EINVAL;
257  }
258  
259  static void bitmap_file_kick(struct bitmap *bitmap);
260  /*
261   * write out a page to a file
262   */
write_page(struct bitmap * bitmap,struct page * page,int wait)263  static void write_page(struct bitmap *bitmap, struct page *page, int wait)
264  {
265  	struct buffer_head *bh;
266  
267  	if (bitmap->storage.file == NULL) {
268  		switch (write_sb_page(bitmap, page, wait)) {
269  		case -EINVAL:
270  			set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
271  		}
272  	} else {
273  
274  		bh = page_buffers(page);
275  
276  		while (bh && bh->b_blocknr) {
277  			atomic_inc(&bitmap->pending_writes);
278  			set_buffer_locked(bh);
279  			set_buffer_mapped(bh);
280  			submit_bh(WRITE | REQ_SYNC, bh);
281  			bh = bh->b_this_page;
282  		}
283  
284  		if (wait)
285  			wait_event(bitmap->write_wait,
286  				   atomic_read(&bitmap->pending_writes)==0);
287  	}
288  	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
289  		bitmap_file_kick(bitmap);
290  }
291  
end_bitmap_write(struct buffer_head * bh,int uptodate)292  static void end_bitmap_write(struct buffer_head *bh, int uptodate)
293  {
294  	struct bitmap *bitmap = bh->b_private;
295  
296  	if (!uptodate)
297  		set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
298  	if (atomic_dec_and_test(&bitmap->pending_writes))
299  		wake_up(&bitmap->write_wait);
300  }
301  
302  /* copied from buffer.c */
303  static void
__clear_page_buffers(struct page * page)304  __clear_page_buffers(struct page *page)
305  {
306  	ClearPagePrivate(page);
307  	set_page_private(page, 0);
308  	page_cache_release(page);
309  }
free_buffers(struct page * page)310  static void free_buffers(struct page *page)
311  {
312  	struct buffer_head *bh;
313  
314  	if (!PagePrivate(page))
315  		return;
316  
317  	bh = page_buffers(page);
318  	while (bh) {
319  		struct buffer_head *next = bh->b_this_page;
320  		free_buffer_head(bh);
321  		bh = next;
322  	}
323  	__clear_page_buffers(page);
324  	put_page(page);
325  }
326  
327  /* read a page from a file.
328   * We both read the page, and attach buffers to the page to record the
329   * address of each block (using bmap).  These addresses will be used
330   * to write the block later, completely bypassing the filesystem.
331   * This usage is similar to how swap files are handled, and allows us
332   * to write to a file with no concerns of memory allocation failing.
333   */
read_page(struct file * file,unsigned long index,struct bitmap * bitmap,unsigned long count,struct page * page)334  static int read_page(struct file *file, unsigned long index,
335  		     struct bitmap *bitmap,
336  		     unsigned long count,
337  		     struct page *page)
338  {
339  	int ret = 0;
340  	struct inode *inode = file_inode(file);
341  	struct buffer_head *bh;
342  	sector_t block;
343  
344  	pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
345  		 (unsigned long long)index << PAGE_SHIFT);
346  
347  	bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
348  	if (!bh) {
349  		ret = -ENOMEM;
350  		goto out;
351  	}
352  	attach_page_buffers(page, bh);
353  	block = index << (PAGE_SHIFT - inode->i_blkbits);
354  	while (bh) {
355  		if (count == 0)
356  			bh->b_blocknr = 0;
357  		else {
358  			bh->b_blocknr = bmap(inode, block);
359  			if (bh->b_blocknr == 0) {
360  				/* Cannot use this file! */
361  				ret = -EINVAL;
362  				goto out;
363  			}
364  			bh->b_bdev = inode->i_sb->s_bdev;
365  			if (count < (1<<inode->i_blkbits))
366  				count = 0;
367  			else
368  				count -= (1<<inode->i_blkbits);
369  
370  			bh->b_end_io = end_bitmap_write;
371  			bh->b_private = bitmap;
372  			atomic_inc(&bitmap->pending_writes);
373  			set_buffer_locked(bh);
374  			set_buffer_mapped(bh);
375  			submit_bh(READ, bh);
376  		}
377  		block++;
378  		bh = bh->b_this_page;
379  	}
380  	page->index = index;
381  
382  	wait_event(bitmap->write_wait,
383  		   atomic_read(&bitmap->pending_writes)==0);
384  	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
385  		ret = -EIO;
386  out:
387  	if (ret)
388  		printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
389  			(int)PAGE_SIZE,
390  			(unsigned long long)index << PAGE_SHIFT,
391  			ret);
392  	return ret;
393  }
394  
395  /*
396   * bitmap file superblock operations
397   */
398  
399  /* update the event counter and sync the superblock to disk */
bitmap_update_sb(struct bitmap * bitmap)400  void bitmap_update_sb(struct bitmap *bitmap)
401  {
402  	bitmap_super_t *sb;
403  
404  	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
405  		return;
406  	if (bitmap->mddev->bitmap_info.external)
407  		return;
408  	if (!bitmap->storage.sb_page) /* no superblock */
409  		return;
410  	sb = kmap_atomic(bitmap->storage.sb_page);
411  	sb->events = cpu_to_le64(bitmap->mddev->events);
412  	if (bitmap->mddev->events < bitmap->events_cleared)
413  		/* rocking back to read-only */
414  		bitmap->events_cleared = bitmap->mddev->events;
415  	sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
416  	sb->state = cpu_to_le32(bitmap->flags);
417  	/* Just in case these have been changed via sysfs: */
418  	sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
419  	sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
420  	/* This might have been changed by a reshape */
421  	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
422  	sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
423  	sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
424  					   bitmap_info.space);
425  	kunmap_atomic(sb);
426  	write_page(bitmap, bitmap->storage.sb_page, 1);
427  }
428  
429  /* print out the bitmap file superblock */
bitmap_print_sb(struct bitmap * bitmap)430  void bitmap_print_sb(struct bitmap *bitmap)
431  {
432  	bitmap_super_t *sb;
433  
434  	if (!bitmap || !bitmap->storage.sb_page)
435  		return;
436  	sb = kmap_atomic(bitmap->storage.sb_page);
437  	printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
438  	printk(KERN_DEBUG "         magic: %08x\n", le32_to_cpu(sb->magic));
439  	printk(KERN_DEBUG "       version: %d\n", le32_to_cpu(sb->version));
440  	printk(KERN_DEBUG "          uuid: %08x.%08x.%08x.%08x\n",
441  					*(__u32 *)(sb->uuid+0),
442  					*(__u32 *)(sb->uuid+4),
443  					*(__u32 *)(sb->uuid+8),
444  					*(__u32 *)(sb->uuid+12));
445  	printk(KERN_DEBUG "        events: %llu\n",
446  			(unsigned long long) le64_to_cpu(sb->events));
447  	printk(KERN_DEBUG "events cleared: %llu\n",
448  			(unsigned long long) le64_to_cpu(sb->events_cleared));
449  	printk(KERN_DEBUG "         state: %08x\n", le32_to_cpu(sb->state));
450  	printk(KERN_DEBUG "     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
451  	printk(KERN_DEBUG "  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
452  	printk(KERN_DEBUG "     sync size: %llu KB\n",
453  			(unsigned long long)le64_to_cpu(sb->sync_size)/2);
454  	printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
455  	kunmap_atomic(sb);
456  }
457  
458  /*
459   * bitmap_new_disk_sb
460   * @bitmap
461   *
462   * This function is somewhat the reverse of bitmap_read_sb.  bitmap_read_sb
463   * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
464   * This function verifies 'bitmap_info' and populates the on-disk bitmap
465   * structure, which is to be written to disk.
466   *
467   * Returns: 0 on success, -Exxx on error
468   */
bitmap_new_disk_sb(struct bitmap * bitmap)469  static int bitmap_new_disk_sb(struct bitmap *bitmap)
470  {
471  	bitmap_super_t *sb;
472  	unsigned long chunksize, daemon_sleep, write_behind;
473  
474  	bitmap->storage.sb_page = alloc_page(GFP_KERNEL);
475  	if (bitmap->storage.sb_page == NULL)
476  		return -ENOMEM;
477  	bitmap->storage.sb_page->index = 0;
478  
479  	sb = kmap_atomic(bitmap->storage.sb_page);
480  
481  	sb->magic = cpu_to_le32(BITMAP_MAGIC);
482  	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
483  
484  	chunksize = bitmap->mddev->bitmap_info.chunksize;
485  	BUG_ON(!chunksize);
486  	if (!is_power_of_2(chunksize)) {
487  		kunmap_atomic(sb);
488  		printk(KERN_ERR "bitmap chunksize not a power of 2\n");
489  		return -EINVAL;
490  	}
491  	sb->chunksize = cpu_to_le32(chunksize);
492  
493  	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
494  	if (!daemon_sleep ||
495  	    (daemon_sleep < 1) || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
496  		printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
497  		daemon_sleep = 5 * HZ;
498  	}
499  	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
500  	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
501  
502  	/*
503  	 * FIXME: write_behind for RAID1.  If not specified, what
504  	 * is a good choice?  We choose COUNTER_MAX / 2 arbitrarily.
505  	 */
506  	write_behind = bitmap->mddev->bitmap_info.max_write_behind;
507  	if (write_behind > COUNTER_MAX)
508  		write_behind = COUNTER_MAX / 2;
509  	sb->write_behind = cpu_to_le32(write_behind);
510  	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
511  
512  	/* keep the array size field of the bitmap superblock up to date */
513  	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
514  
515  	memcpy(sb->uuid, bitmap->mddev->uuid, 16);
516  
517  	set_bit(BITMAP_STALE, &bitmap->flags);
518  	sb->state = cpu_to_le32(bitmap->flags);
519  	bitmap->events_cleared = bitmap->mddev->events;
520  	sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
521  
522  	kunmap_atomic(sb);
523  
524  	return 0;
525  }
526  
527  /* read the superblock from the bitmap file and initialize some bitmap fields */
bitmap_read_sb(struct bitmap * bitmap)528  static int bitmap_read_sb(struct bitmap *bitmap)
529  {
530  	char *reason = NULL;
531  	bitmap_super_t *sb;
532  	unsigned long chunksize, daemon_sleep, write_behind;
533  	unsigned long long events;
534  	unsigned long sectors_reserved = 0;
535  	int err = -EINVAL;
536  	struct page *sb_page;
537  
538  	if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
539  		chunksize = 128 * 1024 * 1024;
540  		daemon_sleep = 5 * HZ;
541  		write_behind = 0;
542  		set_bit(BITMAP_STALE, &bitmap->flags);
543  		err = 0;
544  		goto out_no_sb;
545  	}
546  	/* page 0 is the superblock, read it... */
547  	sb_page = alloc_page(GFP_KERNEL);
548  	if (!sb_page)
549  		return -ENOMEM;
550  	bitmap->storage.sb_page = sb_page;
551  
552  	if (bitmap->storage.file) {
553  		loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
554  		int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
555  
556  		err = read_page(bitmap->storage.file, 0,
557  				bitmap, bytes, sb_page);
558  	} else {
559  		err = read_sb_page(bitmap->mddev,
560  				   bitmap->mddev->bitmap_info.offset,
561  				   sb_page,
562  				   0, sizeof(bitmap_super_t));
563  	}
564  	if (err)
565  		return err;
566  
567  	err = -EINVAL;
568  
569  	sb = kmap_atomic(sb_page);
570  
571  	chunksize = le32_to_cpu(sb->chunksize);
572  	daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
573  	write_behind = le32_to_cpu(sb->write_behind);
574  	sectors_reserved = le32_to_cpu(sb->sectors_reserved);
575  
576  	/* verify that the bitmap-specific fields are valid */
577  	if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
578  		reason = "bad magic";
579  	else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
580  		 le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
581  		reason = "unrecognized superblock version";
582  	else if (chunksize < 512)
583  		reason = "bitmap chunksize too small";
584  	else if (!is_power_of_2(chunksize))
585  		reason = "bitmap chunksize not a power of 2";
586  	else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
587  		reason = "daemon sleep period out of range";
588  	else if (write_behind > COUNTER_MAX)
589  		reason = "write-behind limit out of range (0 - 16383)";
590  	if (reason) {
591  		printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
592  			bmname(bitmap), reason);
593  		goto out;
594  	}
595  
596  	/* keep the array size field of the bitmap superblock up to date */
597  	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
598  
599  	if (bitmap->mddev->persistent) {
600  		/*
601  		 * We have a persistent array superblock, so compare the
602  		 * bitmap's UUID and event counter to the mddev's
603  		 */
604  		if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
605  			printk(KERN_INFO
606  			       "%s: bitmap superblock UUID mismatch\n",
607  			       bmname(bitmap));
608  			goto out;
609  		}
610  		events = le64_to_cpu(sb->events);
611  		if (events < bitmap->mddev->events) {
612  			printk(KERN_INFO
613  			       "%s: bitmap file is out of date (%llu < %llu) "
614  			       "-- forcing full recovery\n",
615  			       bmname(bitmap), events,
616  			       (unsigned long long) bitmap->mddev->events);
617  			set_bit(BITMAP_STALE, &bitmap->flags);
618  		}
619  	}
620  
621  	/* assign fields using values from superblock */
622  	bitmap->flags |= le32_to_cpu(sb->state);
623  	if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
624  		set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
625  	bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
626  	err = 0;
627  out:
628  	kunmap_atomic(sb);
629  out_no_sb:
630  	if (test_bit(BITMAP_STALE, &bitmap->flags))
631  		bitmap->events_cleared = bitmap->mddev->events;
632  	bitmap->mddev->bitmap_info.chunksize = chunksize;
633  	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
634  	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
635  	if (bitmap->mddev->bitmap_info.space == 0 ||
636  	    bitmap->mddev->bitmap_info.space > sectors_reserved)
637  		bitmap->mddev->bitmap_info.space = sectors_reserved;
638  	if (err)
639  		bitmap_print_sb(bitmap);
640  	return err;
641  }
642  
643  /*
644   * general bitmap file operations
645   */
646  
647  /*
648   * on-disk bitmap:
649   *
650   * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
651   * file a page at a time. There's a superblock at the start of the file.
652   */
653  /* calculate the index of the page that contains this bit */
file_page_index(struct bitmap_storage * store,unsigned long chunk)654  static inline unsigned long file_page_index(struct bitmap_storage *store,
655  					    unsigned long chunk)
656  {
657  	if (store->sb_page)
658  		chunk += sizeof(bitmap_super_t) << 3;
659  	return chunk >> PAGE_BIT_SHIFT;
660  }
661  
662  /* calculate the (bit) offset of this bit within a page */
file_page_offset(struct bitmap_storage * store,unsigned long chunk)663  static inline unsigned long file_page_offset(struct bitmap_storage *store,
664  					     unsigned long chunk)
665  {
666  	if (store->sb_page)
667  		chunk += sizeof(bitmap_super_t) << 3;
668  	return chunk & (PAGE_BITS - 1);
669  }
670  
671  /*
672   * return a pointer to the page in the filemap that contains the given bit
673   *
674   */
filemap_get_page(struct bitmap_storage * store,unsigned long chunk)675  static inline struct page *filemap_get_page(struct bitmap_storage *store,
676  					    unsigned long chunk)
677  {
678  	if (file_page_index(store, chunk) >= store->file_pages)
679  		return NULL;
680  	return store->filemap[file_page_index(store, chunk)];
681  }
682  
bitmap_storage_alloc(struct bitmap_storage * store,unsigned long chunks,int with_super)683  static int bitmap_storage_alloc(struct bitmap_storage *store,
684  				unsigned long chunks, int with_super)
685  {
686  	int pnum;
687  	unsigned long num_pages;
688  	unsigned long bytes;
689  
690  	bytes = DIV_ROUND_UP(chunks, 8);
691  	if (with_super)
692  		bytes += sizeof(bitmap_super_t);
693  
694  	num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
695  
696  	store->filemap = kmalloc(sizeof(struct page *)
697  				 * num_pages, GFP_KERNEL);
698  	if (!store->filemap)
699  		return -ENOMEM;
700  
701  	if (with_super && !store->sb_page) {
702  		store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
703  		if (store->sb_page == NULL)
704  			return -ENOMEM;
705  		store->sb_page->index = 0;
706  	}
707  	pnum = 0;
708  	if (store->sb_page) {
709  		store->filemap[0] = store->sb_page;
710  		pnum = 1;
711  	}
712  	for ( ; pnum < num_pages; pnum++) {
713  		store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
714  		if (!store->filemap[pnum]) {
715  			store->file_pages = pnum;
716  			return -ENOMEM;
717  		}
718  		store->filemap[pnum]->index = pnum;
719  	}
720  	store->file_pages = pnum;
721  
722  	/* We need 4 bits per page, rounded up to a multiple
723  	 * of sizeof(unsigned long) */
724  	store->filemap_attr = kzalloc(
725  		roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
726  		GFP_KERNEL);
727  	if (!store->filemap_attr)
728  		return -ENOMEM;
729  
730  	store->bytes = bytes;
731  
732  	return 0;
733  }
734  
bitmap_file_unmap(struct bitmap_storage * store)735  static void bitmap_file_unmap(struct bitmap_storage *store)
736  {
737  	struct page **map, *sb_page;
738  	int pages;
739  	struct file *file;
740  
741  	file = store->file;
742  	map = store->filemap;
743  	pages = store->file_pages;
744  	sb_page = store->sb_page;
745  
746  	while (pages--)
747  		if (map[pages] != sb_page) /* 0 is sb_page, release it below */
748  			free_buffers(map[pages]);
749  	kfree(map);
750  	kfree(store->filemap_attr);
751  
752  	if (sb_page)
753  		free_buffers(sb_page);
754  
755  	if (file) {
756  		struct inode *inode = file_inode(file);
757  		invalidate_mapping_pages(inode->i_mapping, 0, -1);
758  		fput(file);
759  	}
760  }
761  
762  /*
763   * bitmap_file_kick - if an error occurs while manipulating the bitmap file
764   * then it is no longer reliable, so we stop using it and we mark the file
765   * as failed in the superblock
766   */
bitmap_file_kick(struct bitmap * bitmap)767  static void bitmap_file_kick(struct bitmap *bitmap)
768  {
769  	char *path, *ptr = NULL;
770  
771  	if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
772  		bitmap_update_sb(bitmap);
773  
774  		if (bitmap->storage.file) {
775  			path = kmalloc(PAGE_SIZE, GFP_KERNEL);
776  			if (path)
777  				ptr = d_path(&bitmap->storage.file->f_path,
778  					     path, PAGE_SIZE);
779  
780  			printk(KERN_ALERT
781  			      "%s: kicking failed bitmap file %s from array!\n",
782  			      bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
783  
784  			kfree(path);
785  		} else
786  			printk(KERN_ALERT
787  			       "%s: disabling internal bitmap due to errors\n",
788  			       bmname(bitmap));
789  	}
790  }
791  
792  enum bitmap_page_attr {
793  	BITMAP_PAGE_DIRTY = 0,     /* there are set bits that need to be synced */
794  	BITMAP_PAGE_PENDING = 1,   /* there are bits that are being cleaned.
795  				    * i.e. counter is 1 or 2. */
796  	BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
797  };
798  
set_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)799  static inline void set_page_attr(struct bitmap *bitmap, int pnum,
800  				 enum bitmap_page_attr attr)
801  {
802  	set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
803  }
804  
clear_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)805  static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
806  				   enum bitmap_page_attr attr)
807  {
808  	clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
809  }
810  
test_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)811  static inline int test_page_attr(struct bitmap *bitmap, int pnum,
812  				 enum bitmap_page_attr attr)
813  {
814  	return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
815  }
816  
test_and_clear_page_attr(struct bitmap * bitmap,int pnum,enum bitmap_page_attr attr)817  static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
818  					   enum bitmap_page_attr attr)
819  {
820  	return test_and_clear_bit((pnum<<2) + attr,
821  				  bitmap->storage.filemap_attr);
822  }
823  /*
824   * bitmap_file_set_bit -- called before performing a write to the md device
825   * to set (and eventually sync) a particular bit in the bitmap file
826   *
827   * we set the bit immediately, then we record the page number so that
828   * when an unplug occurs, we can flush the dirty pages out to disk
829   */
bitmap_file_set_bit(struct bitmap * bitmap,sector_t block)830  static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
831  {
832  	unsigned long bit;
833  	struct page *page;
834  	void *kaddr;
835  	unsigned long chunk = block >> bitmap->counts.chunkshift;
836  
837  	page = filemap_get_page(&bitmap->storage, chunk);
838  	if (!page)
839  		return;
840  	bit = file_page_offset(&bitmap->storage, chunk);
841  
842  	/* set the bit */
843  	kaddr = kmap_atomic(page);
844  	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
845  		set_bit(bit, kaddr);
846  	else
847  		set_bit_le(bit, kaddr);
848  	kunmap_atomic(kaddr);
849  	pr_debug("set file bit %lu page %lu\n", bit, page->index);
850  	/* record page number so it gets flushed to disk when unplug occurs */
851  	set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY);
852  }
853  
bitmap_file_clear_bit(struct bitmap * bitmap,sector_t block)854  static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
855  {
856  	unsigned long bit;
857  	struct page *page;
858  	void *paddr;
859  	unsigned long chunk = block >> bitmap->counts.chunkshift;
860  
861  	page = filemap_get_page(&bitmap->storage, chunk);
862  	if (!page)
863  		return;
864  	bit = file_page_offset(&bitmap->storage, chunk);
865  	paddr = kmap_atomic(page);
866  	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
867  		clear_bit(bit, paddr);
868  	else
869  		clear_bit_le(bit, paddr);
870  	kunmap_atomic(paddr);
871  	if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) {
872  		set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING);
873  		bitmap->allclean = 0;
874  	}
875  }
876  
877  /* this gets called when the md device is ready to unplug its underlying
878   * (slave) device queues -- before we let any writes go down, we need to
879   * sync the dirty pages of the bitmap file to disk */
bitmap_unplug(struct bitmap * bitmap)880  void bitmap_unplug(struct bitmap *bitmap)
881  {
882  	unsigned long i;
883  	int dirty, need_write;
884  
885  	if (!bitmap || !bitmap->storage.filemap ||
886  	    test_bit(BITMAP_STALE, &bitmap->flags))
887  		return;
888  
889  	/* look at each page to see if there are any set bits that need to be
890  	 * flushed out to disk */
891  	for (i = 0; i < bitmap->storage.file_pages; i++) {
892  		if (!bitmap->storage.filemap)
893  			return;
894  		dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
895  		need_write = test_and_clear_page_attr(bitmap, i,
896  						      BITMAP_PAGE_NEEDWRITE);
897  		if (dirty || need_write) {
898  			clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
899  			write_page(bitmap, bitmap->storage.filemap[i], 0);
900  		}
901  	}
902  	if (bitmap->storage.file)
903  		wait_event(bitmap->write_wait,
904  			   atomic_read(&bitmap->pending_writes)==0);
905  	else
906  		md_super_wait(bitmap->mddev);
907  
908  	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
909  		bitmap_file_kick(bitmap);
910  }
911  EXPORT_SYMBOL(bitmap_unplug);
912  
913  static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
914  /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
915   * the in-memory bitmap from the on-disk bitmap -- also, sets up the
916   * memory mapping of the bitmap file
917   * Special cases:
918   *   if there's no bitmap file, or if the bitmap file had been
919   *   previously kicked from the array, we mark all the bits as
920   *   1's in order to cause a full resync.
921   *
922   * We ignore all bits for sectors that end earlier than 'start'.
923   * This is used when reading an out-of-date bitmap...
924   */
bitmap_init_from_disk(struct bitmap * bitmap,sector_t start)925  static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
926  {
927  	unsigned long i, chunks, index, oldindex, bit;
928  	struct page *page = NULL;
929  	unsigned long bit_cnt = 0;
930  	struct file *file;
931  	unsigned long offset;
932  	int outofdate;
933  	int ret = -ENOSPC;
934  	void *paddr;
935  	struct bitmap_storage *store = &bitmap->storage;
936  
937  	chunks = bitmap->counts.chunks;
938  	file = store->file;
939  
940  	if (!file && !bitmap->mddev->bitmap_info.offset) {
941  		/* No permanent bitmap - fill with '1s'. */
942  		store->filemap = NULL;
943  		store->file_pages = 0;
944  		for (i = 0; i < chunks ; i++) {
945  			/* if the disk bit is set, set the memory bit */
946  			int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
947  				      >= start);
948  			bitmap_set_memory_bits(bitmap,
949  					       (sector_t)i << bitmap->counts.chunkshift,
950  					       needed);
951  		}
952  		return 0;
953  	}
954  
955  	outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
956  	if (outofdate)
957  		printk(KERN_INFO "%s: bitmap file is out of date, doing full "
958  			"recovery\n", bmname(bitmap));
959  
960  	if (file && i_size_read(file->f_mapping->host) < store->bytes) {
961  		printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
962  		       bmname(bitmap),
963  		       (unsigned long) i_size_read(file->f_mapping->host),
964  		       store->bytes);
965  		goto err;
966  	}
967  
968  	oldindex = ~0L;
969  	offset = 0;
970  	if (!bitmap->mddev->bitmap_info.external)
971  		offset = sizeof(bitmap_super_t);
972  
973  	for (i = 0; i < chunks; i++) {
974  		int b;
975  		index = file_page_index(&bitmap->storage, i);
976  		bit = file_page_offset(&bitmap->storage, i);
977  		if (index != oldindex) { /* this is a new page, read it in */
978  			int count;
979  			/* unmap the old page, we're done with it */
980  			if (index == store->file_pages-1)
981  				count = store->bytes - index * PAGE_SIZE;
982  			else
983  				count = PAGE_SIZE;
984  			page = store->filemap[index];
985  			if (file)
986  				ret = read_page(file, index, bitmap,
987  						count, page);
988  			else
989  				ret = read_sb_page(
990  					bitmap->mddev,
991  					bitmap->mddev->bitmap_info.offset,
992  					page,
993  					index, count);
994  
995  			if (ret)
996  				goto err;
997  
998  			oldindex = index;
999  
1000  			if (outofdate) {
1001  				/*
1002  				 * if bitmap is out of date, dirty the
1003  				 * whole page and write it out
1004  				 */
1005  				paddr = kmap_atomic(page);
1006  				memset(paddr + offset, 0xff,
1007  				       PAGE_SIZE - offset);
1008  				kunmap_atomic(paddr);
1009  				write_page(bitmap, page, 1);
1010  
1011  				ret = -EIO;
1012  				if (test_bit(BITMAP_WRITE_ERROR,
1013  					     &bitmap->flags))
1014  					goto err;
1015  			}
1016  		}
1017  		paddr = kmap_atomic(page);
1018  		if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1019  			b = test_bit(bit, paddr);
1020  		else
1021  			b = test_bit_le(bit, paddr);
1022  		kunmap_atomic(paddr);
1023  		if (b) {
1024  			/* if the disk bit is set, set the memory bit */
1025  			int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1026  				      >= start);
1027  			bitmap_set_memory_bits(bitmap,
1028  					       (sector_t)i << bitmap->counts.chunkshift,
1029  					       needed);
1030  			bit_cnt++;
1031  		}
1032  		offset = 0;
1033  	}
1034  
1035  	printk(KERN_INFO "%s: bitmap initialized from disk: "
1036  	       "read %lu pages, set %lu of %lu bits\n",
1037  	       bmname(bitmap), store->file_pages,
1038  	       bit_cnt, chunks);
1039  
1040  	return 0;
1041  
1042   err:
1043  	printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
1044  	       bmname(bitmap), ret);
1045  	return ret;
1046  }
1047  
bitmap_write_all(struct bitmap * bitmap)1048  void bitmap_write_all(struct bitmap *bitmap)
1049  {
1050  	/* We don't actually write all bitmap blocks here,
1051  	 * just flag them as needing to be written
1052  	 */
1053  	int i;
1054  
1055  	if (!bitmap || !bitmap->storage.filemap)
1056  		return;
1057  	if (bitmap->storage.file)
1058  		/* Only one copy, so nothing needed */
1059  		return;
1060  
1061  	for (i = 0; i < bitmap->storage.file_pages; i++)
1062  		set_page_attr(bitmap, i,
1063  			      BITMAP_PAGE_NEEDWRITE);
1064  	bitmap->allclean = 0;
1065  }
1066  
bitmap_count_page(struct bitmap_counts * bitmap,sector_t offset,int inc)1067  static void bitmap_count_page(struct bitmap_counts *bitmap,
1068  			      sector_t offset, int inc)
1069  {
1070  	sector_t chunk = offset >> bitmap->chunkshift;
1071  	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1072  	bitmap->bp[page].count += inc;
1073  	bitmap_checkfree(bitmap, page);
1074  }
1075  
bitmap_set_pending(struct bitmap_counts * bitmap,sector_t offset)1076  static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1077  {
1078  	sector_t chunk = offset >> bitmap->chunkshift;
1079  	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1080  	struct bitmap_page *bp = &bitmap->bp[page];
1081  
1082  	if (!bp->pending)
1083  		bp->pending = 1;
1084  }
1085  
1086  static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1087  					    sector_t offset, sector_t *blocks,
1088  					    int create);
1089  
1090  /*
1091   * bitmap daemon -- periodically wakes up to clean bits and flush pages
1092   *			out to disk
1093   */
1094  
bitmap_daemon_work(struct mddev * mddev)1095  void bitmap_daemon_work(struct mddev *mddev)
1096  {
1097  	struct bitmap *bitmap;
1098  	unsigned long j;
1099  	unsigned long nextpage;
1100  	sector_t blocks;
1101  	struct bitmap_counts *counts;
1102  
1103  	/* Use a mutex to guard daemon_work against
1104  	 * bitmap_destroy.
1105  	 */
1106  	mutex_lock(&mddev->bitmap_info.mutex);
1107  	bitmap = mddev->bitmap;
1108  	if (bitmap == NULL) {
1109  		mutex_unlock(&mddev->bitmap_info.mutex);
1110  		return;
1111  	}
1112  	if (time_before(jiffies, bitmap->daemon_lastrun
1113  			+ mddev->bitmap_info.daemon_sleep))
1114  		goto done;
1115  
1116  	bitmap->daemon_lastrun = jiffies;
1117  	if (bitmap->allclean) {
1118  		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1119  		goto done;
1120  	}
1121  	bitmap->allclean = 1;
1122  
1123  	/* Any file-page which is PENDING now needs to be written.
1124  	 * So set NEEDWRITE now, then after we make any last-minute changes
1125  	 * we will write it.
1126  	 */
1127  	for (j = 0; j < bitmap->storage.file_pages; j++)
1128  		if (test_and_clear_page_attr(bitmap, j,
1129  					     BITMAP_PAGE_PENDING))
1130  			set_page_attr(bitmap, j,
1131  				      BITMAP_PAGE_NEEDWRITE);
1132  
1133  	if (bitmap->need_sync &&
1134  	    mddev->bitmap_info.external == 0) {
1135  		/* Arrange for superblock update as well as
1136  		 * other changes */
1137  		bitmap_super_t *sb;
1138  		bitmap->need_sync = 0;
1139  		if (bitmap->storage.filemap) {
1140  			sb = kmap_atomic(bitmap->storage.sb_page);
1141  			sb->events_cleared =
1142  				cpu_to_le64(bitmap->events_cleared);
1143  			kunmap_atomic(sb);
1144  			set_page_attr(bitmap, 0,
1145  				      BITMAP_PAGE_NEEDWRITE);
1146  		}
1147  	}
1148  	/* Now look at the bitmap counters and if any are '2' or '1',
1149  	 * decrement and handle accordingly.
1150  	 */
1151  	counts = &bitmap->counts;
1152  	spin_lock_irq(&counts->lock);
1153  	nextpage = 0;
1154  	for (j = 0; j < counts->chunks; j++) {
1155  		bitmap_counter_t *bmc;
1156  		sector_t  block = (sector_t)j << counts->chunkshift;
1157  
1158  		if (j == nextpage) {
1159  			nextpage += PAGE_COUNTER_RATIO;
1160  			if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1161  				j |= PAGE_COUNTER_MASK;
1162  				continue;
1163  			}
1164  			counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1165  		}
1166  		bmc = bitmap_get_counter(counts,
1167  					 block,
1168  					 &blocks, 0);
1169  
1170  		if (!bmc) {
1171  			j |= PAGE_COUNTER_MASK;
1172  			continue;
1173  		}
1174  		if (*bmc == 1 && !bitmap->need_sync) {
1175  			/* We can clear the bit */
1176  			*bmc = 0;
1177  			bitmap_count_page(counts, block, -1);
1178  			bitmap_file_clear_bit(bitmap, block);
1179  		} else if (*bmc && *bmc <= 2) {
1180  			*bmc = 1;
1181  			bitmap_set_pending(counts, block);
1182  			bitmap->allclean = 0;
1183  		}
1184  	}
1185  	spin_unlock_irq(&counts->lock);
1186  
1187  	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1188  	 * DIRTY pages need to be written by bitmap_unplug so it can wait
1189  	 * for them.
1190  	 * If we find any DIRTY page we stop there and let bitmap_unplug
1191  	 * handle all the rest.  This is important in the case where
1192  	 * the first blocking holds the superblock and it has been updated.
1193  	 * We mustn't write any other blocks before the superblock.
1194  	 */
1195  	for (j = 0;
1196  	     j < bitmap->storage.file_pages
1197  		     && !test_bit(BITMAP_STALE, &bitmap->flags);
1198  	     j++) {
1199  
1200  		if (test_page_attr(bitmap, j,
1201  				   BITMAP_PAGE_DIRTY))
1202  			/* bitmap_unplug will handle the rest */
1203  			break;
1204  		if (test_and_clear_page_attr(bitmap, j,
1205  					     BITMAP_PAGE_NEEDWRITE)) {
1206  			write_page(bitmap, bitmap->storage.filemap[j], 0);
1207  		}
1208  	}
1209  
1210   done:
1211  	if (bitmap->allclean == 0)
1212  		mddev->thread->timeout =
1213  			mddev->bitmap_info.daemon_sleep;
1214  	mutex_unlock(&mddev->bitmap_info.mutex);
1215  }
1216  
bitmap_get_counter(struct bitmap_counts * bitmap,sector_t offset,sector_t * blocks,int create)1217  static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1218  					    sector_t offset, sector_t *blocks,
1219  					    int create)
1220  __releases(bitmap->lock)
1221  __acquires(bitmap->lock)
1222  {
1223  	/* If 'create', we might release the lock and reclaim it.
1224  	 * The lock must have been taken with interrupts enabled.
1225  	 * If !create, we don't release the lock.
1226  	 */
1227  	sector_t chunk = offset >> bitmap->chunkshift;
1228  	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1229  	unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1230  	sector_t csize;
1231  	int err;
1232  
1233  	err = bitmap_checkpage(bitmap, page, create);
1234  
1235  	if (bitmap->bp[page].hijacked ||
1236  	    bitmap->bp[page].map == NULL)
1237  		csize = ((sector_t)1) << (bitmap->chunkshift +
1238  					  PAGE_COUNTER_SHIFT - 1);
1239  	else
1240  		csize = ((sector_t)1) << bitmap->chunkshift;
1241  	*blocks = csize - (offset & (csize - 1));
1242  
1243  	if (err < 0)
1244  		return NULL;
1245  
1246  	/* now locked ... */
1247  
1248  	if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1249  		/* should we use the first or second counter field
1250  		 * of the hijacked pointer? */
1251  		int hi = (pageoff > PAGE_COUNTER_MASK);
1252  		return  &((bitmap_counter_t *)
1253  			  &bitmap->bp[page].map)[hi];
1254  	} else /* page is allocated */
1255  		return (bitmap_counter_t *)
1256  			&(bitmap->bp[page].map[pageoff]);
1257  }
1258  
bitmap_startwrite(struct bitmap * bitmap,sector_t offset,unsigned long sectors,int behind)1259  int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1260  {
1261  	if (!bitmap)
1262  		return 0;
1263  
1264  	if (behind) {
1265  		int bw;
1266  		atomic_inc(&bitmap->behind_writes);
1267  		bw = atomic_read(&bitmap->behind_writes);
1268  		if (bw > bitmap->behind_writes_used)
1269  			bitmap->behind_writes_used = bw;
1270  
1271  		pr_debug("inc write-behind count %d/%lu\n",
1272  			 bw, bitmap->mddev->bitmap_info.max_write_behind);
1273  	}
1274  
1275  	while (sectors) {
1276  		sector_t blocks;
1277  		bitmap_counter_t *bmc;
1278  
1279  		spin_lock_irq(&bitmap->counts.lock);
1280  		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1281  		if (!bmc) {
1282  			spin_unlock_irq(&bitmap->counts.lock);
1283  			return 0;
1284  		}
1285  
1286  		if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1287  			DEFINE_WAIT(__wait);
1288  			/* note that it is safe to do the prepare_to_wait
1289  			 * after the test as long as we do it before dropping
1290  			 * the spinlock.
1291  			 */
1292  			prepare_to_wait(&bitmap->overflow_wait, &__wait,
1293  					TASK_UNINTERRUPTIBLE);
1294  			spin_unlock_irq(&bitmap->counts.lock);
1295  			schedule();
1296  			finish_wait(&bitmap->overflow_wait, &__wait);
1297  			continue;
1298  		}
1299  
1300  		switch (*bmc) {
1301  		case 0:
1302  			bitmap_file_set_bit(bitmap, offset);
1303  			bitmap_count_page(&bitmap->counts, offset, 1);
1304  			/* fall through */
1305  		case 1:
1306  			*bmc = 2;
1307  		}
1308  
1309  		(*bmc)++;
1310  
1311  		spin_unlock_irq(&bitmap->counts.lock);
1312  
1313  		offset += blocks;
1314  		if (sectors > blocks)
1315  			sectors -= blocks;
1316  		else
1317  			sectors = 0;
1318  	}
1319  	return 0;
1320  }
1321  EXPORT_SYMBOL(bitmap_startwrite);
1322  
bitmap_endwrite(struct bitmap * bitmap,sector_t offset,unsigned long sectors,int success,int behind)1323  void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1324  		     int success, int behind)
1325  {
1326  	if (!bitmap)
1327  		return;
1328  	if (behind) {
1329  		if (atomic_dec_and_test(&bitmap->behind_writes))
1330  			wake_up(&bitmap->behind_wait);
1331  		pr_debug("dec write-behind count %d/%lu\n",
1332  			 atomic_read(&bitmap->behind_writes),
1333  			 bitmap->mddev->bitmap_info.max_write_behind);
1334  	}
1335  
1336  	while (sectors) {
1337  		sector_t blocks;
1338  		unsigned long flags;
1339  		bitmap_counter_t *bmc;
1340  
1341  		spin_lock_irqsave(&bitmap->counts.lock, flags);
1342  		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1343  		if (!bmc) {
1344  			spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1345  			return;
1346  		}
1347  
1348  		if (success && !bitmap->mddev->degraded &&
1349  		    bitmap->events_cleared < bitmap->mddev->events) {
1350  			bitmap->events_cleared = bitmap->mddev->events;
1351  			bitmap->need_sync = 1;
1352  			sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1353  		}
1354  
1355  		if (!success && !NEEDED(*bmc))
1356  			*bmc |= NEEDED_MASK;
1357  
1358  		if (COUNTER(*bmc) == COUNTER_MAX)
1359  			wake_up(&bitmap->overflow_wait);
1360  
1361  		(*bmc)--;
1362  		if (*bmc <= 2) {
1363  			bitmap_set_pending(&bitmap->counts, offset);
1364  			bitmap->allclean = 0;
1365  		}
1366  		spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1367  		offset += blocks;
1368  		if (sectors > blocks)
1369  			sectors -= blocks;
1370  		else
1371  			sectors = 0;
1372  	}
1373  }
1374  EXPORT_SYMBOL(bitmap_endwrite);
1375  
__bitmap_start_sync(struct bitmap * bitmap,sector_t offset,sector_t * blocks,int degraded)1376  static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1377  			       int degraded)
1378  {
1379  	bitmap_counter_t *bmc;
1380  	int rv;
1381  	if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1382  		*blocks = 1024;
1383  		return 1; /* always resync if no bitmap */
1384  	}
1385  	spin_lock_irq(&bitmap->counts.lock);
1386  	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1387  	rv = 0;
1388  	if (bmc) {
1389  		/* locked */
1390  		if (RESYNC(*bmc))
1391  			rv = 1;
1392  		else if (NEEDED(*bmc)) {
1393  			rv = 1;
1394  			if (!degraded) { /* don't set/clear bits if degraded */
1395  				*bmc |= RESYNC_MASK;
1396  				*bmc &= ~NEEDED_MASK;
1397  			}
1398  		}
1399  	}
1400  	spin_unlock_irq(&bitmap->counts.lock);
1401  	return rv;
1402  }
1403  
bitmap_start_sync(struct bitmap * bitmap,sector_t offset,sector_t * blocks,int degraded)1404  int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1405  		      int degraded)
1406  {
1407  	/* bitmap_start_sync must always report on multiples of whole
1408  	 * pages, otherwise resync (which is very PAGE_SIZE based) will
1409  	 * get confused.
1410  	 * So call __bitmap_start_sync repeatedly (if needed) until
1411  	 * At least PAGE_SIZE>>9 blocks are covered.
1412  	 * Return the 'or' of the result.
1413  	 */
1414  	int rv = 0;
1415  	sector_t blocks1;
1416  
1417  	*blocks = 0;
1418  	while (*blocks < (PAGE_SIZE>>9)) {
1419  		rv |= __bitmap_start_sync(bitmap, offset,
1420  					  &blocks1, degraded);
1421  		offset += blocks1;
1422  		*blocks += blocks1;
1423  	}
1424  	return rv;
1425  }
1426  EXPORT_SYMBOL(bitmap_start_sync);
1427  
bitmap_end_sync(struct bitmap * bitmap,sector_t offset,sector_t * blocks,int aborted)1428  void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1429  {
1430  	bitmap_counter_t *bmc;
1431  	unsigned long flags;
1432  
1433  	if (bitmap == NULL) {
1434  		*blocks = 1024;
1435  		return;
1436  	}
1437  	spin_lock_irqsave(&bitmap->counts.lock, flags);
1438  	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1439  	if (bmc == NULL)
1440  		goto unlock;
1441  	/* locked */
1442  	if (RESYNC(*bmc)) {
1443  		*bmc &= ~RESYNC_MASK;
1444  
1445  		if (!NEEDED(*bmc) && aborted)
1446  			*bmc |= NEEDED_MASK;
1447  		else {
1448  			if (*bmc <= 2) {
1449  				bitmap_set_pending(&bitmap->counts, offset);
1450  				bitmap->allclean = 0;
1451  			}
1452  		}
1453  	}
1454   unlock:
1455  	spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1456  }
1457  EXPORT_SYMBOL(bitmap_end_sync);
1458  
bitmap_close_sync(struct bitmap * bitmap)1459  void bitmap_close_sync(struct bitmap *bitmap)
1460  {
1461  	/* Sync has finished, and any bitmap chunks that weren't synced
1462  	 * properly have been aborted.  It remains to us to clear the
1463  	 * RESYNC bit wherever it is still on
1464  	 */
1465  	sector_t sector = 0;
1466  	sector_t blocks;
1467  	if (!bitmap)
1468  		return;
1469  	while (sector < bitmap->mddev->resync_max_sectors) {
1470  		bitmap_end_sync(bitmap, sector, &blocks, 0);
1471  		sector += blocks;
1472  	}
1473  }
1474  EXPORT_SYMBOL(bitmap_close_sync);
1475  
bitmap_cond_end_sync(struct bitmap * bitmap,sector_t sector)1476  void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1477  {
1478  	sector_t s = 0;
1479  	sector_t blocks;
1480  
1481  	if (!bitmap)
1482  		return;
1483  	if (sector == 0) {
1484  		bitmap->last_end_sync = jiffies;
1485  		return;
1486  	}
1487  	if (time_before(jiffies, (bitmap->last_end_sync
1488  				  + bitmap->mddev->bitmap_info.daemon_sleep)))
1489  		return;
1490  	wait_event(bitmap->mddev->recovery_wait,
1491  		   atomic_read(&bitmap->mddev->recovery_active) == 0);
1492  
1493  	bitmap->mddev->curr_resync_completed = sector;
1494  	set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1495  	sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1496  	s = 0;
1497  	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1498  		bitmap_end_sync(bitmap, s, &blocks, 0);
1499  		s += blocks;
1500  	}
1501  	bitmap->last_end_sync = jiffies;
1502  	sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1503  }
1504  EXPORT_SYMBOL(bitmap_cond_end_sync);
1505  
bitmap_set_memory_bits(struct bitmap * bitmap,sector_t offset,int needed)1506  static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1507  {
1508  	/* For each chunk covered by any of these sectors, set the
1509  	 * counter to 2 and possibly set resync_needed.  They should all
1510  	 * be 0 at this point
1511  	 */
1512  
1513  	sector_t secs;
1514  	bitmap_counter_t *bmc;
1515  	spin_lock_irq(&bitmap->counts.lock);
1516  	bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1517  	if (!bmc) {
1518  		spin_unlock_irq(&bitmap->counts.lock);
1519  		return;
1520  	}
1521  	if (!*bmc) {
1522  		*bmc = 2 | (needed ? NEEDED_MASK : 0);
1523  		bitmap_count_page(&bitmap->counts, offset, 1);
1524  		bitmap_set_pending(&bitmap->counts, offset);
1525  		bitmap->allclean = 0;
1526  	}
1527  	spin_unlock_irq(&bitmap->counts.lock);
1528  }
1529  
1530  /* dirty the memory and file bits for bitmap chunks "s" to "e" */
bitmap_dirty_bits(struct bitmap * bitmap,unsigned long s,unsigned long e)1531  void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1532  {
1533  	unsigned long chunk;
1534  
1535  	for (chunk = s; chunk <= e; chunk++) {
1536  		sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1537  		bitmap_set_memory_bits(bitmap, sec, 1);
1538  		bitmap_file_set_bit(bitmap, sec);
1539  		if (sec < bitmap->mddev->recovery_cp)
1540  			/* We are asserting that the array is dirty,
1541  			 * so move the recovery_cp address back so
1542  			 * that it is obvious that it is dirty
1543  			 */
1544  			bitmap->mddev->recovery_cp = sec;
1545  	}
1546  }
1547  
1548  /*
1549   * flush out any pending updates
1550   */
bitmap_flush(struct mddev * mddev)1551  void bitmap_flush(struct mddev *mddev)
1552  {
1553  	struct bitmap *bitmap = mddev->bitmap;
1554  	long sleep;
1555  
1556  	if (!bitmap) /* there was no bitmap */
1557  		return;
1558  
1559  	/* run the daemon_work three time to ensure everything is flushed
1560  	 * that can be
1561  	 */
1562  	sleep = mddev->bitmap_info.daemon_sleep * 2;
1563  	bitmap->daemon_lastrun -= sleep;
1564  	bitmap_daemon_work(mddev);
1565  	bitmap->daemon_lastrun -= sleep;
1566  	bitmap_daemon_work(mddev);
1567  	bitmap->daemon_lastrun -= sleep;
1568  	bitmap_daemon_work(mddev);
1569  	bitmap_update_sb(bitmap);
1570  }
1571  
1572  /*
1573   * free memory that was allocated
1574   */
bitmap_free(struct bitmap * bitmap)1575  static void bitmap_free(struct bitmap *bitmap)
1576  {
1577  	unsigned long k, pages;
1578  	struct bitmap_page *bp;
1579  
1580  	if (!bitmap) /* there was no bitmap */
1581  		return;
1582  
1583  	/* Shouldn't be needed - but just in case.... */
1584  	wait_event(bitmap->write_wait,
1585  		   atomic_read(&bitmap->pending_writes) == 0);
1586  
1587  	/* release the bitmap file  */
1588  	bitmap_file_unmap(&bitmap->storage);
1589  
1590  	bp = bitmap->counts.bp;
1591  	pages = bitmap->counts.pages;
1592  
1593  	/* free all allocated memory */
1594  
1595  	if (bp) /* deallocate the page memory */
1596  		for (k = 0; k < pages; k++)
1597  			if (bp[k].map && !bp[k].hijacked)
1598  				kfree(bp[k].map);
1599  	kfree(bp);
1600  	kfree(bitmap);
1601  }
1602  
bitmap_destroy(struct mddev * mddev)1603  void bitmap_destroy(struct mddev *mddev)
1604  {
1605  	struct bitmap *bitmap = mddev->bitmap;
1606  
1607  	if (!bitmap) /* there was no bitmap */
1608  		return;
1609  
1610  	mutex_lock(&mddev->bitmap_info.mutex);
1611  	mddev->bitmap = NULL; /* disconnect from the md device */
1612  	mutex_unlock(&mddev->bitmap_info.mutex);
1613  	if (mddev->thread)
1614  		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1615  
1616  	if (bitmap->sysfs_can_clear)
1617  		sysfs_put(bitmap->sysfs_can_clear);
1618  
1619  	bitmap_free(bitmap);
1620  }
1621  
1622  /*
1623   * initialize the bitmap structure
1624   * if this returns an error, bitmap_destroy must be called to do clean up
1625   */
bitmap_create(struct mddev * mddev)1626  int bitmap_create(struct mddev *mddev)
1627  {
1628  	struct bitmap *bitmap;
1629  	sector_t blocks = mddev->resync_max_sectors;
1630  	struct file *file = mddev->bitmap_info.file;
1631  	int err;
1632  	struct kernfs_node *bm = NULL;
1633  
1634  	BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1635  
1636  	BUG_ON(file && mddev->bitmap_info.offset);
1637  
1638  	bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1639  	if (!bitmap)
1640  		return -ENOMEM;
1641  
1642  	spin_lock_init(&bitmap->counts.lock);
1643  	atomic_set(&bitmap->pending_writes, 0);
1644  	init_waitqueue_head(&bitmap->write_wait);
1645  	init_waitqueue_head(&bitmap->overflow_wait);
1646  	init_waitqueue_head(&bitmap->behind_wait);
1647  
1648  	bitmap->mddev = mddev;
1649  
1650  	if (mddev->kobj.sd)
1651  		bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1652  	if (bm) {
1653  		bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1654  		sysfs_put(bm);
1655  	} else
1656  		bitmap->sysfs_can_clear = NULL;
1657  
1658  	bitmap->storage.file = file;
1659  	if (file) {
1660  		get_file(file);
1661  		/* As future accesses to this file will use bmap,
1662  		 * and bypass the page cache, we must sync the file
1663  		 * first.
1664  		 */
1665  		vfs_fsync(file, 1);
1666  	}
1667  	/* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1668  	if (!mddev->bitmap_info.external) {
1669  		/*
1670  		 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1671  		 * instructing us to create a new on-disk bitmap instance.
1672  		 */
1673  		if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1674  			err = bitmap_new_disk_sb(bitmap);
1675  		else
1676  			err = bitmap_read_sb(bitmap);
1677  	} else {
1678  		err = 0;
1679  		if (mddev->bitmap_info.chunksize == 0 ||
1680  		    mddev->bitmap_info.daemon_sleep == 0)
1681  			/* chunksize and time_base need to be
1682  			 * set first. */
1683  			err = -EINVAL;
1684  	}
1685  	if (err)
1686  		goto error;
1687  
1688  	bitmap->daemon_lastrun = jiffies;
1689  	err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1690  	if (err)
1691  		goto error;
1692  
1693  	printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1694  	       bitmap->counts.pages, bmname(bitmap));
1695  
1696  	mddev->bitmap = bitmap;
1697  	return test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1698  
1699   error:
1700  	bitmap_free(bitmap);
1701  	return err;
1702  }
1703  
bitmap_load(struct mddev * mddev)1704  int bitmap_load(struct mddev *mddev)
1705  {
1706  	int err = 0;
1707  	sector_t start = 0;
1708  	sector_t sector = 0;
1709  	struct bitmap *bitmap = mddev->bitmap;
1710  
1711  	if (!bitmap)
1712  		goto out;
1713  
1714  	/* Clear out old bitmap info first:  Either there is none, or we
1715  	 * are resuming after someone else has possibly changed things,
1716  	 * so we should forget old cached info.
1717  	 * All chunks should be clean, but some might need_sync.
1718  	 */
1719  	while (sector < mddev->resync_max_sectors) {
1720  		sector_t blocks;
1721  		bitmap_start_sync(bitmap, sector, &blocks, 0);
1722  		sector += blocks;
1723  	}
1724  	bitmap_close_sync(bitmap);
1725  
1726  	if (mddev->degraded == 0
1727  	    || bitmap->events_cleared == mddev->events)
1728  		/* no need to keep dirty bits to optimise a
1729  		 * re-add of a missing device */
1730  		start = mddev->recovery_cp;
1731  
1732  	mutex_lock(&mddev->bitmap_info.mutex);
1733  	err = bitmap_init_from_disk(bitmap, start);
1734  	mutex_unlock(&mddev->bitmap_info.mutex);
1735  
1736  	if (err)
1737  		goto out;
1738  	clear_bit(BITMAP_STALE, &bitmap->flags);
1739  
1740  	/* Kick recovery in case any bits were set */
1741  	set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1742  
1743  	mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1744  	md_wakeup_thread(mddev->thread);
1745  
1746  	bitmap_update_sb(bitmap);
1747  
1748  	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1749  		err = -EIO;
1750  out:
1751  	return err;
1752  }
1753  EXPORT_SYMBOL_GPL(bitmap_load);
1754  
bitmap_status(struct seq_file * seq,struct bitmap * bitmap)1755  void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
1756  {
1757  	unsigned long chunk_kb;
1758  	struct bitmap_counts *counts;
1759  
1760  	if (!bitmap)
1761  		return;
1762  
1763  	counts = &bitmap->counts;
1764  
1765  	chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
1766  	seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
1767  		   "%lu%s chunk",
1768  		   counts->pages - counts->missing_pages,
1769  		   counts->pages,
1770  		   (counts->pages - counts->missing_pages)
1771  		   << (PAGE_SHIFT - 10),
1772  		   chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
1773  		   chunk_kb ? "KB" : "B");
1774  	if (bitmap->storage.file) {
1775  		seq_printf(seq, ", file: ");
1776  		seq_path(seq, &bitmap->storage.file->f_path, " \t\n");
1777  	}
1778  
1779  	seq_printf(seq, "\n");
1780  }
1781  
bitmap_resize(struct bitmap * bitmap,sector_t blocks,int chunksize,int init)1782  int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
1783  		  int chunksize, int init)
1784  {
1785  	/* If chunk_size is 0, choose an appropriate chunk size.
1786  	 * Then possibly allocate new storage space.
1787  	 * Then quiesce, copy bits, replace bitmap, and re-start
1788  	 *
1789  	 * This function is called both to set up the initial bitmap
1790  	 * and to resize the bitmap while the array is active.
1791  	 * If this happens as a result of the array being resized,
1792  	 * chunksize will be zero, and we need to choose a suitable
1793  	 * chunksize, otherwise we use what we are given.
1794  	 */
1795  	struct bitmap_storage store;
1796  	struct bitmap_counts old_counts;
1797  	unsigned long chunks;
1798  	sector_t block;
1799  	sector_t old_blocks, new_blocks;
1800  	int chunkshift;
1801  	int ret = 0;
1802  	long pages;
1803  	struct bitmap_page *new_bp;
1804  
1805  	if (bitmap->storage.file && !init) {
1806  		pr_info("md: cannot resize file-based bitmap\n");
1807  		return -EINVAL;
1808  	}
1809  
1810  	if (chunksize == 0) {
1811  		/* If there is enough space, leave the chunk size unchanged,
1812  		 * else increase by factor of two until there is enough space.
1813  		 */
1814  		long bytes;
1815  		long space = bitmap->mddev->bitmap_info.space;
1816  
1817  		if (space == 0) {
1818  			/* We don't know how much space there is, so limit
1819  			 * to current size - in sectors.
1820  			 */
1821  			bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
1822  			if (!bitmap->mddev->bitmap_info.external)
1823  				bytes += sizeof(bitmap_super_t);
1824  			space = DIV_ROUND_UP(bytes, 512);
1825  			bitmap->mddev->bitmap_info.space = space;
1826  		}
1827  		chunkshift = bitmap->counts.chunkshift;
1828  		chunkshift--;
1829  		do {
1830  			/* 'chunkshift' is shift from block size to chunk size */
1831  			chunkshift++;
1832  			chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1833  			bytes = DIV_ROUND_UP(chunks, 8);
1834  			if (!bitmap->mddev->bitmap_info.external)
1835  				bytes += sizeof(bitmap_super_t);
1836  		} while (bytes > (space << 9));
1837  	} else
1838  		chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
1839  
1840  	chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1841  	memset(&store, 0, sizeof(store));
1842  	if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
1843  		ret = bitmap_storage_alloc(&store, chunks,
1844  					   !bitmap->mddev->bitmap_info.external);
1845  	if (ret)
1846  		goto err;
1847  
1848  	pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
1849  
1850  	new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
1851  	ret = -ENOMEM;
1852  	if (!new_bp) {
1853  		bitmap_file_unmap(&store);
1854  		goto err;
1855  	}
1856  
1857  	if (!init)
1858  		bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
1859  
1860  	store.file = bitmap->storage.file;
1861  	bitmap->storage.file = NULL;
1862  
1863  	if (store.sb_page && bitmap->storage.sb_page)
1864  		memcpy(page_address(store.sb_page),
1865  		       page_address(bitmap->storage.sb_page),
1866  		       sizeof(bitmap_super_t));
1867  	bitmap_file_unmap(&bitmap->storage);
1868  	bitmap->storage = store;
1869  
1870  	old_counts = bitmap->counts;
1871  	bitmap->counts.bp = new_bp;
1872  	bitmap->counts.pages = pages;
1873  	bitmap->counts.missing_pages = pages;
1874  	bitmap->counts.chunkshift = chunkshift;
1875  	bitmap->counts.chunks = chunks;
1876  	bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
1877  						     BITMAP_BLOCK_SHIFT);
1878  
1879  	blocks = min(old_counts.chunks << old_counts.chunkshift,
1880  		     chunks << chunkshift);
1881  
1882  	spin_lock_irq(&bitmap->counts.lock);
1883  	for (block = 0; block < blocks; ) {
1884  		bitmap_counter_t *bmc_old, *bmc_new;
1885  		int set;
1886  
1887  		bmc_old = bitmap_get_counter(&old_counts, block,
1888  					     &old_blocks, 0);
1889  		set = bmc_old && NEEDED(*bmc_old);
1890  
1891  		if (set) {
1892  			bmc_new = bitmap_get_counter(&bitmap->counts, block,
1893  						     &new_blocks, 1);
1894  			if (*bmc_new == 0) {
1895  				/* need to set on-disk bits too. */
1896  				sector_t end = block + new_blocks;
1897  				sector_t start = block >> chunkshift;
1898  				start <<= chunkshift;
1899  				while (start < end) {
1900  					bitmap_file_set_bit(bitmap, block);
1901  					start += 1 << chunkshift;
1902  				}
1903  				*bmc_new = 2;
1904  				bitmap_count_page(&bitmap->counts,
1905  						  block, 1);
1906  				bitmap_set_pending(&bitmap->counts,
1907  						   block);
1908  			}
1909  			*bmc_new |= NEEDED_MASK;
1910  			if (new_blocks < old_blocks)
1911  				old_blocks = new_blocks;
1912  		}
1913  		block += old_blocks;
1914  	}
1915  
1916  	if (!init) {
1917  		int i;
1918  		while (block < (chunks << chunkshift)) {
1919  			bitmap_counter_t *bmc;
1920  			bmc = bitmap_get_counter(&bitmap->counts, block,
1921  						 &new_blocks, 1);
1922  			if (bmc) {
1923  				/* new space.  It needs to be resynced, so
1924  				 * we set NEEDED_MASK.
1925  				 */
1926  				if (*bmc == 0) {
1927  					*bmc = NEEDED_MASK | 2;
1928  					bitmap_count_page(&bitmap->counts,
1929  							  block, 1);
1930  					bitmap_set_pending(&bitmap->counts,
1931  							   block);
1932  				}
1933  			}
1934  			block += new_blocks;
1935  		}
1936  		for (i = 0; i < bitmap->storage.file_pages; i++)
1937  			set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1938  	}
1939  	spin_unlock_irq(&bitmap->counts.lock);
1940  
1941  	if (!init) {
1942  		bitmap_unplug(bitmap);
1943  		bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
1944  	}
1945  	ret = 0;
1946  err:
1947  	return ret;
1948  }
1949  EXPORT_SYMBOL_GPL(bitmap_resize);
1950  
1951  static ssize_t
location_show(struct mddev * mddev,char * page)1952  location_show(struct mddev *mddev, char *page)
1953  {
1954  	ssize_t len;
1955  	if (mddev->bitmap_info.file)
1956  		len = sprintf(page, "file");
1957  	else if (mddev->bitmap_info.offset)
1958  		len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
1959  	else
1960  		len = sprintf(page, "none");
1961  	len += sprintf(page+len, "\n");
1962  	return len;
1963  }
1964  
1965  static ssize_t
location_store(struct mddev * mddev,const char * buf,size_t len)1966  location_store(struct mddev *mddev, const char *buf, size_t len)
1967  {
1968  
1969  	if (mddev->pers) {
1970  		if (!mddev->pers->quiesce)
1971  			return -EBUSY;
1972  		if (mddev->recovery || mddev->sync_thread)
1973  			return -EBUSY;
1974  	}
1975  
1976  	if (mddev->bitmap || mddev->bitmap_info.file ||
1977  	    mddev->bitmap_info.offset) {
1978  		/* bitmap already configured.  Only option is to clear it */
1979  		if (strncmp(buf, "none", 4) != 0)
1980  			return -EBUSY;
1981  		if (mddev->pers) {
1982  			mddev->pers->quiesce(mddev, 1);
1983  			bitmap_destroy(mddev);
1984  			mddev->pers->quiesce(mddev, 0);
1985  		}
1986  		mddev->bitmap_info.offset = 0;
1987  		if (mddev->bitmap_info.file) {
1988  			struct file *f = mddev->bitmap_info.file;
1989  			mddev->bitmap_info.file = NULL;
1990  			fput(f);
1991  		}
1992  	} else {
1993  		/* No bitmap, OK to set a location */
1994  		long long offset;
1995  		if (strncmp(buf, "none", 4) == 0)
1996  			/* nothing to be done */;
1997  		else if (strncmp(buf, "file:", 5) == 0) {
1998  			/* Not supported yet */
1999  			return -EINVAL;
2000  		} else {
2001  			int rv;
2002  			if (buf[0] == '+')
2003  				rv = kstrtoll(buf+1, 10, &offset);
2004  			else
2005  				rv = kstrtoll(buf, 10, &offset);
2006  			if (rv)
2007  				return rv;
2008  			if (offset == 0)
2009  				return -EINVAL;
2010  			if (mddev->bitmap_info.external == 0 &&
2011  			    mddev->major_version == 0 &&
2012  			    offset != mddev->bitmap_info.default_offset)
2013  				return -EINVAL;
2014  			mddev->bitmap_info.offset = offset;
2015  			if (mddev->pers) {
2016  				mddev->pers->quiesce(mddev, 1);
2017  				rv = bitmap_create(mddev);
2018  				if (!rv)
2019  					rv = bitmap_load(mddev);
2020  				if (rv) {
2021  					bitmap_destroy(mddev);
2022  					mddev->bitmap_info.offset = 0;
2023  				}
2024  				mddev->pers->quiesce(mddev, 0);
2025  				if (rv)
2026  					return rv;
2027  			}
2028  		}
2029  	}
2030  	if (!mddev->external) {
2031  		/* Ensure new bitmap info is stored in
2032  		 * metadata promptly.
2033  		 */
2034  		set_bit(MD_CHANGE_DEVS, &mddev->flags);
2035  		md_wakeup_thread(mddev->thread);
2036  	}
2037  	return len;
2038  }
2039  
2040  static struct md_sysfs_entry bitmap_location =
2041  __ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2042  
2043  /* 'bitmap/space' is the space available at 'location' for the
2044   * bitmap.  This allows the kernel to know when it is safe to
2045   * resize the bitmap to match a resized array.
2046   */
2047  static ssize_t
space_show(struct mddev * mddev,char * page)2048  space_show(struct mddev *mddev, char *page)
2049  {
2050  	return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2051  }
2052  
2053  static ssize_t
space_store(struct mddev * mddev,const char * buf,size_t len)2054  space_store(struct mddev *mddev, const char *buf, size_t len)
2055  {
2056  	unsigned long sectors;
2057  	int rv;
2058  
2059  	rv = kstrtoul(buf, 10, &sectors);
2060  	if (rv)
2061  		return rv;
2062  
2063  	if (sectors == 0)
2064  		return -EINVAL;
2065  
2066  	if (mddev->bitmap &&
2067  	    sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2068  		return -EFBIG; /* Bitmap is too big for this small space */
2069  
2070  	/* could make sure it isn't too big, but that isn't really
2071  	 * needed - user-space should be careful.
2072  	 */
2073  	mddev->bitmap_info.space = sectors;
2074  	return len;
2075  }
2076  
2077  static struct md_sysfs_entry bitmap_space =
2078  __ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2079  
2080  static ssize_t
timeout_show(struct mddev * mddev,char * page)2081  timeout_show(struct mddev *mddev, char *page)
2082  {
2083  	ssize_t len;
2084  	unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2085  	unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2086  
2087  	len = sprintf(page, "%lu", secs);
2088  	if (jifs)
2089  		len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2090  	len += sprintf(page+len, "\n");
2091  	return len;
2092  }
2093  
2094  static ssize_t
timeout_store(struct mddev * mddev,const char * buf,size_t len)2095  timeout_store(struct mddev *mddev, const char *buf, size_t len)
2096  {
2097  	/* timeout can be set at any time */
2098  	unsigned long timeout;
2099  	int rv = strict_strtoul_scaled(buf, &timeout, 4);
2100  	if (rv)
2101  		return rv;
2102  
2103  	/* just to make sure we don't overflow... */
2104  	if (timeout >= LONG_MAX / HZ)
2105  		return -EINVAL;
2106  
2107  	timeout = timeout * HZ / 10000;
2108  
2109  	if (timeout >= MAX_SCHEDULE_TIMEOUT)
2110  		timeout = MAX_SCHEDULE_TIMEOUT-1;
2111  	if (timeout < 1)
2112  		timeout = 1;
2113  	mddev->bitmap_info.daemon_sleep = timeout;
2114  	if (mddev->thread) {
2115  		/* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
2116  		 * the bitmap is all clean and we don't need to
2117  		 * adjust the timeout right now
2118  		 */
2119  		if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2120  			mddev->thread->timeout = timeout;
2121  			md_wakeup_thread(mddev->thread);
2122  		}
2123  	}
2124  	return len;
2125  }
2126  
2127  static struct md_sysfs_entry bitmap_timeout =
2128  __ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2129  
2130  static ssize_t
backlog_show(struct mddev * mddev,char * page)2131  backlog_show(struct mddev *mddev, char *page)
2132  {
2133  	return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2134  }
2135  
2136  static ssize_t
backlog_store(struct mddev * mddev,const char * buf,size_t len)2137  backlog_store(struct mddev *mddev, const char *buf, size_t len)
2138  {
2139  	unsigned long backlog;
2140  	int rv = kstrtoul(buf, 10, &backlog);
2141  	if (rv)
2142  		return rv;
2143  	if (backlog > COUNTER_MAX)
2144  		return -EINVAL;
2145  	mddev->bitmap_info.max_write_behind = backlog;
2146  	return len;
2147  }
2148  
2149  static struct md_sysfs_entry bitmap_backlog =
2150  __ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2151  
2152  static ssize_t
chunksize_show(struct mddev * mddev,char * page)2153  chunksize_show(struct mddev *mddev, char *page)
2154  {
2155  	return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2156  }
2157  
2158  static ssize_t
chunksize_store(struct mddev * mddev,const char * buf,size_t len)2159  chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2160  {
2161  	/* Can only be changed when no bitmap is active */
2162  	int rv;
2163  	unsigned long csize;
2164  	if (mddev->bitmap)
2165  		return -EBUSY;
2166  	rv = kstrtoul(buf, 10, &csize);
2167  	if (rv)
2168  		return rv;
2169  	if (csize < 512 ||
2170  	    !is_power_of_2(csize))
2171  		return -EINVAL;
2172  	mddev->bitmap_info.chunksize = csize;
2173  	return len;
2174  }
2175  
2176  static struct md_sysfs_entry bitmap_chunksize =
2177  __ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2178  
metadata_show(struct mddev * mddev,char * page)2179  static ssize_t metadata_show(struct mddev *mddev, char *page)
2180  {
2181  	return sprintf(page, "%s\n", (mddev->bitmap_info.external
2182  				      ? "external" : "internal"));
2183  }
2184  
metadata_store(struct mddev * mddev,const char * buf,size_t len)2185  static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2186  {
2187  	if (mddev->bitmap ||
2188  	    mddev->bitmap_info.file ||
2189  	    mddev->bitmap_info.offset)
2190  		return -EBUSY;
2191  	if (strncmp(buf, "external", 8) == 0)
2192  		mddev->bitmap_info.external = 1;
2193  	else if (strncmp(buf, "internal", 8) == 0)
2194  		mddev->bitmap_info.external = 0;
2195  	else
2196  		return -EINVAL;
2197  	return len;
2198  }
2199  
2200  static struct md_sysfs_entry bitmap_metadata =
2201  __ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2202  
can_clear_show(struct mddev * mddev,char * page)2203  static ssize_t can_clear_show(struct mddev *mddev, char *page)
2204  {
2205  	int len;
2206  	if (mddev->bitmap)
2207  		len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2208  					     "false" : "true"));
2209  	else
2210  		len = sprintf(page, "\n");
2211  	return len;
2212  }
2213  
can_clear_store(struct mddev * mddev,const char * buf,size_t len)2214  static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2215  {
2216  	if (mddev->bitmap == NULL)
2217  		return -ENOENT;
2218  	if (strncmp(buf, "false", 5) == 0)
2219  		mddev->bitmap->need_sync = 1;
2220  	else if (strncmp(buf, "true", 4) == 0) {
2221  		if (mddev->degraded)
2222  			return -EBUSY;
2223  		mddev->bitmap->need_sync = 0;
2224  	} else
2225  		return -EINVAL;
2226  	return len;
2227  }
2228  
2229  static struct md_sysfs_entry bitmap_can_clear =
2230  __ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2231  
2232  static ssize_t
behind_writes_used_show(struct mddev * mddev,char * page)2233  behind_writes_used_show(struct mddev *mddev, char *page)
2234  {
2235  	if (mddev->bitmap == NULL)
2236  		return sprintf(page, "0\n");
2237  	return sprintf(page, "%lu\n",
2238  		       mddev->bitmap->behind_writes_used);
2239  }
2240  
2241  static ssize_t
behind_writes_used_reset(struct mddev * mddev,const char * buf,size_t len)2242  behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2243  {
2244  	if (mddev->bitmap)
2245  		mddev->bitmap->behind_writes_used = 0;
2246  	return len;
2247  }
2248  
2249  static struct md_sysfs_entry max_backlog_used =
2250  __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2251         behind_writes_used_show, behind_writes_used_reset);
2252  
2253  static struct attribute *md_bitmap_attrs[] = {
2254  	&bitmap_location.attr,
2255  	&bitmap_space.attr,
2256  	&bitmap_timeout.attr,
2257  	&bitmap_backlog.attr,
2258  	&bitmap_chunksize.attr,
2259  	&bitmap_metadata.attr,
2260  	&bitmap_can_clear.attr,
2261  	&max_backlog_used.attr,
2262  	NULL
2263  };
2264  struct attribute_group md_bitmap_group = {
2265  	.name = "bitmap",
2266  	.attrs = md_bitmap_attrs,
2267  };
2268  
2269