1 #include <linux/swap_cgroup.h>
2 #include <linux/vmalloc.h>
3 #include <linux/mm.h>
4
5 #include <linux/swapops.h> /* depends on mm.h include */
6
7 static DEFINE_MUTEX(swap_cgroup_mutex);
8 struct swap_cgroup_ctrl {
9 struct page **map;
10 unsigned long length;
11 spinlock_t lock;
12 };
13
14 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
15
16 struct swap_cgroup {
17 unsigned short id;
18 };
19 #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
20
21 /*
22 * SwapCgroup implements "lookup" and "exchange" operations.
23 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
24 * against SwapCache. At swap_free(), this is accessed directly from swap.
25 *
26 * This means,
27 * - we have no race in "exchange" when we're accessed via SwapCache because
28 * SwapCache(and its swp_entry) is under lock.
29 * - When called via swap_free(), there is no user of this entry and no race.
30 * Then, we don't need lock around "exchange".
31 *
32 * TODO: we can push these buffers out to HIGHMEM.
33 */
34
35 /*
36 * allocate buffer for swap_cgroup.
37 */
swap_cgroup_prepare(int type)38 static int swap_cgroup_prepare(int type)
39 {
40 struct page *page;
41 struct swap_cgroup_ctrl *ctrl;
42 unsigned long idx, max;
43
44 ctrl = &swap_cgroup_ctrl[type];
45
46 for (idx = 0; idx < ctrl->length; idx++) {
47 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
48 if (!page)
49 goto not_enough_page;
50 ctrl->map[idx] = page;
51
52 if (!(idx % SWAP_CLUSTER_MAX))
53 cond_resched();
54 }
55 return 0;
56 not_enough_page:
57 max = idx;
58 for (idx = 0; idx < max; idx++)
59 __free_page(ctrl->map[idx]);
60
61 return -ENOMEM;
62 }
63
lookup_swap_cgroup(swp_entry_t ent,struct swap_cgroup_ctrl ** ctrlp)64 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
65 struct swap_cgroup_ctrl **ctrlp)
66 {
67 pgoff_t offset = swp_offset(ent);
68 struct swap_cgroup_ctrl *ctrl;
69 struct page *mappage;
70 struct swap_cgroup *sc;
71
72 ctrl = &swap_cgroup_ctrl[swp_type(ent)];
73 if (ctrlp)
74 *ctrlp = ctrl;
75
76 mappage = ctrl->map[offset / SC_PER_PAGE];
77 sc = page_address(mappage);
78 return sc + offset % SC_PER_PAGE;
79 }
80
81 /**
82 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
83 * @ent: swap entry to be cmpxchged
84 * @old: old id
85 * @new: new id
86 *
87 * Returns old id at success, 0 at failure.
88 * (There is no mem_cgroup using 0 as its id)
89 */
swap_cgroup_cmpxchg(swp_entry_t ent,unsigned short old,unsigned short new)90 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
91 unsigned short old, unsigned short new)
92 {
93 struct swap_cgroup_ctrl *ctrl;
94 struct swap_cgroup *sc;
95 unsigned long flags;
96 unsigned short retval;
97
98 sc = lookup_swap_cgroup(ent, &ctrl);
99
100 spin_lock_irqsave(&ctrl->lock, flags);
101 retval = sc->id;
102 if (retval == old)
103 sc->id = new;
104 else
105 retval = 0;
106 spin_unlock_irqrestore(&ctrl->lock, flags);
107 return retval;
108 }
109
110 /**
111 * swap_cgroup_record - record mem_cgroup for this swp_entry.
112 * @ent: swap entry to be recorded into
113 * @id: mem_cgroup to be recorded
114 *
115 * Returns old value at success, 0 at failure.
116 * (Of course, old value can be 0.)
117 */
swap_cgroup_record(swp_entry_t ent,unsigned short id)118 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
119 {
120 struct swap_cgroup_ctrl *ctrl;
121 struct swap_cgroup *sc;
122 unsigned short old;
123 unsigned long flags;
124
125 sc = lookup_swap_cgroup(ent, &ctrl);
126
127 spin_lock_irqsave(&ctrl->lock, flags);
128 old = sc->id;
129 sc->id = id;
130 spin_unlock_irqrestore(&ctrl->lock, flags);
131
132 return old;
133 }
134
135 /**
136 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
137 * @ent: swap entry to be looked up.
138 *
139 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
140 */
lookup_swap_cgroup_id(swp_entry_t ent)141 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
142 {
143 return lookup_swap_cgroup(ent, NULL)->id;
144 }
145
swap_cgroup_swapon(int type,unsigned long max_pages)146 int swap_cgroup_swapon(int type, unsigned long max_pages)
147 {
148 void *array;
149 unsigned long array_size;
150 unsigned long length;
151 struct swap_cgroup_ctrl *ctrl;
152
153 if (!do_swap_account)
154 return 0;
155
156 length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
157 array_size = length * sizeof(void *);
158
159 array = vzalloc(array_size);
160 if (!array)
161 goto nomem;
162
163 ctrl = &swap_cgroup_ctrl[type];
164 mutex_lock(&swap_cgroup_mutex);
165 ctrl->length = length;
166 ctrl->map = array;
167 spin_lock_init(&ctrl->lock);
168 if (swap_cgroup_prepare(type)) {
169 /* memory shortage */
170 ctrl->map = NULL;
171 ctrl->length = 0;
172 mutex_unlock(&swap_cgroup_mutex);
173 vfree(array);
174 goto nomem;
175 }
176 mutex_unlock(&swap_cgroup_mutex);
177
178 return 0;
179 nomem:
180 printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
181 printk(KERN_INFO
182 "swap_cgroup can be disabled by swapaccount=0 boot option\n");
183 return -ENOMEM;
184 }
185
swap_cgroup_swapoff(int type)186 void swap_cgroup_swapoff(int type)
187 {
188 struct page **map;
189 unsigned long i, length;
190 struct swap_cgroup_ctrl *ctrl;
191
192 if (!do_swap_account)
193 return;
194
195 mutex_lock(&swap_cgroup_mutex);
196 ctrl = &swap_cgroup_ctrl[type];
197 map = ctrl->map;
198 length = ctrl->length;
199 ctrl->map = NULL;
200 ctrl->length = 0;
201 mutex_unlock(&swap_cgroup_mutex);
202
203 if (map) {
204 for (i = 0; i < length; i++) {
205 struct page *page = map[i];
206 if (page)
207 __free_page(page);
208 if (!(i % SWAP_CLUSTER_MAX))
209 cond_resched();
210 }
211 vfree(map);
212 }
213 }
214