12 static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,  in pcpu_chunk_page()  argument
16 	WARN_ON(chunk->immutable);  in pcpu_chunk_page()
18 	return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));  in pcpu_chunk_page()
53 static void pcpu_free_pages(struct pcpu_chunk *chunk,  in pcpu_free_pages()  argument
81 static int pcpu_alloc_pages(struct pcpu_chunk *chunk,  in pcpu_alloc_pages()  argument
126 static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,  in pcpu_pre_unmap_flush()  argument
130 		pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),  in pcpu_pre_unmap_flush()
131 		pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));  in pcpu_pre_unmap_flush()
152 static void pcpu_unmap_pages(struct pcpu_chunk *chunk,  in pcpu_unmap_pages()  argument
162 			page = pcpu_chunk_page(chunk, cpu, i);  in pcpu_unmap_pages()
166 		__pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),  in pcpu_unmap_pages()
184 static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,  in pcpu_post_unmap_tlb_flush()  argument
188 		pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),  in pcpu_post_unmap_tlb_flush()
189 		pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));  in pcpu_post_unmap_tlb_flush()
213 static int pcpu_map_pages(struct pcpu_chunk *chunk,  in pcpu_map_pages()  argument
220 		err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),  in pcpu_map_pages()
228 					    chunk);  in pcpu_map_pages()
235 		__pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),  in pcpu_map_pages()
238 	pcpu_post_unmap_tlb_flush(chunk, page_start, page_end);  in pcpu_map_pages()
254 static void pcpu_post_map_flush(struct pcpu_chunk *chunk,  in pcpu_post_map_flush()  argument
258 		pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),  in pcpu_post_map_flush()
259 		pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));  in pcpu_post_map_flush()
275 static int pcpu_populate_chunk(struct pcpu_chunk *chunk,  in pcpu_populate_chunk()  argument
284 	if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp))  in pcpu_populate_chunk()
287 	if (pcpu_map_pages(chunk, pages, page_start, page_end)) {  in pcpu_populate_chunk()
288 		pcpu_free_pages(chunk, pages, page_start, page_end);  in pcpu_populate_chunk()
291 	pcpu_post_map_flush(chunk, page_start, page_end);  in pcpu_populate_chunk()
308 static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,  in pcpu_depopulate_chunk()  argument
322 	pcpu_pre_unmap_flush(chunk, page_start, page_end);  in pcpu_depopulate_chunk()
324 	pcpu_unmap_pages(chunk, pages, page_start, page_end);  in pcpu_depopulate_chunk()
328 	pcpu_free_pages(chunk, pages, page_start, page_end);  in pcpu_depopulate_chunk()
334 	struct pcpu_chunk *chunk;  in pcpu_create_chunk()  local
337 	chunk = pcpu_alloc_chunk(type, gfp);  in pcpu_create_chunk()
338 	if (!chunk)  in pcpu_create_chunk()
344 		pcpu_free_chunk(chunk);  in pcpu_create_chunk()
348 	chunk->data = vms;  in pcpu_create_chunk()
349 	chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];  in pcpu_create_chunk()
352 	trace_percpu_create_chunk(chunk->base_addr);  in pcpu_create_chunk()
354 	return chunk;  in pcpu_create_chunk()
357 static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)  in pcpu_destroy_chunk()  argument
359 	if (!chunk)  in pcpu_destroy_chunk()
363 	trace_percpu_destroy_chunk(chunk->base_addr);  in pcpu_destroy_chunk()
365 	if (chunk->data)  in pcpu_destroy_chunk()
366 		pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);  in pcpu_destroy_chunk()
367 	pcpu_free_chunk(chunk);  in pcpu_destroy_chunk()