Lines Matching refs:ai
1586 static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
2442 struct pcpu_alloc_info *ai; in pcpu_alloc_alloc_info() local
2447 base_size = ALIGN(struct_size(ai, groups, nr_groups), in pcpu_alloc_alloc_info()
2448 __alignof__(ai->groups[0].cpu_map[0])); in pcpu_alloc_alloc_info()
2449 ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]); in pcpu_alloc_alloc_info()
2454 ai = ptr; in pcpu_alloc_alloc_info()
2457 ai->groups[0].cpu_map = ptr; in pcpu_alloc_alloc_info()
2460 ai->groups[0].cpu_map[unit] = NR_CPUS; in pcpu_alloc_alloc_info()
2462 ai->nr_groups = nr_groups; in pcpu_alloc_alloc_info()
2463 ai->__ai_size = PFN_ALIGN(ai_size); in pcpu_alloc_alloc_info()
2465 return ai; in pcpu_alloc_alloc_info()
2474 void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai) in pcpu_free_alloc_info() argument
2476 memblock_free_early(__pa(ai), ai->__ai_size); in pcpu_free_alloc_info()
2487 const struct pcpu_alloc_info *ai) in pcpu_dump_alloc_info() argument
2495 v = ai->nr_groups; in pcpu_dump_alloc_info()
2504 upa = ai->alloc_size / ai->unit_size; in pcpu_dump_alloc_info()
2509 lvl, ai->static_size, ai->reserved_size, ai->dyn_size, in pcpu_dump_alloc_info()
2510 ai->unit_size, ai->alloc_size / ai->atom_size, ai->atom_size); in pcpu_dump_alloc_info()
2512 for (group = 0; group < ai->nr_groups; group++) { in pcpu_dump_alloc_info()
2513 const struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_dump_alloc_info()
2590 void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, in pcpu_setup_first_chunk() argument
2593 size_t size_sum = ai->static_size + ai->reserved_size + ai->dyn_size; in pcpu_setup_first_chunk()
2611 pcpu_dump_alloc_info(KERN_EMERG, ai); \ in pcpu_setup_first_chunk()
2617 PCPU_SETUP_BUG_ON(ai->nr_groups <= 0); in pcpu_setup_first_chunk()
2619 PCPU_SETUP_BUG_ON(!ai->static_size); in pcpu_setup_first_chunk()
2624 PCPU_SETUP_BUG_ON(ai->unit_size < size_sum); in pcpu_setup_first_chunk()
2625 PCPU_SETUP_BUG_ON(offset_in_page(ai->unit_size)); in pcpu_setup_first_chunk()
2626 PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); in pcpu_setup_first_chunk()
2627 PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->unit_size, PCPU_BITMAP_BLOCK_SIZE)); in pcpu_setup_first_chunk()
2628 PCPU_SETUP_BUG_ON(ai->dyn_size < PERCPU_DYNAMIC_EARLY_SIZE); in pcpu_setup_first_chunk()
2629 PCPU_SETUP_BUG_ON(!ai->dyn_size); in pcpu_setup_first_chunk()
2630 PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->reserved_size, PCPU_MIN_ALLOC_SIZE)); in pcpu_setup_first_chunk()
2633 PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0); in pcpu_setup_first_chunk()
2636 alloc_size = ai->nr_groups * sizeof(group_offsets[0]); in pcpu_setup_first_chunk()
2642 alloc_size = ai->nr_groups * sizeof(group_sizes[0]); in pcpu_setup_first_chunk()
2666 for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) { in pcpu_setup_first_chunk()
2667 const struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_setup_first_chunk()
2670 group_sizes[group] = gi->nr_units * ai->unit_size; in pcpu_setup_first_chunk()
2682 unit_off[cpu] = gi->base_offset + i * ai->unit_size; in pcpu_setup_first_chunk()
2700 pcpu_dump_alloc_info(KERN_DEBUG, ai); in pcpu_setup_first_chunk()
2702 pcpu_nr_groups = ai->nr_groups; in pcpu_setup_first_chunk()
2709 pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT; in pcpu_setup_first_chunk()
2711 pcpu_atom_size = ai->atom_size; in pcpu_setup_first_chunk()
2715 pcpu_stats_save_ai(ai); in pcpu_setup_first_chunk()
2745 static_size = ALIGN(ai->static_size, PCPU_MIN_ALLOC_SIZE); in pcpu_setup_first_chunk()
2746 dyn_size = ai->dyn_size - (static_size - ai->static_size); in pcpu_setup_first_chunk()
2757 map_size = ai->reserved_size ?: dyn_size; in pcpu_setup_first_chunk()
2761 if (ai->reserved_size) { in pcpu_setup_first_chunk()
2765 ai->reserved_size; in pcpu_setup_first_chunk()
2869 struct pcpu_alloc_info *ai; in pcpu_build_alloc_info() local
2959 ai = pcpu_alloc_alloc_info(nr_groups, nr_units); in pcpu_build_alloc_info()
2960 if (!ai) in pcpu_build_alloc_info()
2962 cpu_map = ai->groups[0].cpu_map; in pcpu_build_alloc_info()
2965 ai->groups[group].cpu_map = cpu_map; in pcpu_build_alloc_info()
2969 ai->static_size = static_size; in pcpu_build_alloc_info()
2970 ai->reserved_size = reserved_size; in pcpu_build_alloc_info()
2971 ai->dyn_size = dyn_size; in pcpu_build_alloc_info()
2972 ai->unit_size = alloc_size / upa; in pcpu_build_alloc_info()
2973 ai->atom_size = atom_size; in pcpu_build_alloc_info()
2974 ai->alloc_size = alloc_size; in pcpu_build_alloc_info()
2977 struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_build_alloc_info()
2984 gi->base_offset = unit * ai->unit_size; in pcpu_build_alloc_info()
2994 return ai; in pcpu_build_alloc_info()
3039 struct pcpu_alloc_info *ai; in pcpu_embed_first_chunk() local
3044 ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size, in pcpu_embed_first_chunk()
3046 if (IS_ERR(ai)) in pcpu_embed_first_chunk()
3047 return PTR_ERR(ai); in pcpu_embed_first_chunk()
3049 size_sum = ai->static_size + ai->reserved_size + ai->dyn_size; in pcpu_embed_first_chunk()
3050 areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *)); in pcpu_embed_first_chunk()
3060 for (group = 0; group < ai->nr_groups; group++) { in pcpu_embed_first_chunk()
3061 struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_embed_first_chunk()
3070 ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size); in pcpu_embed_first_chunk()
3084 max_distance += ai->unit_size * ai->groups[highest_group].nr_units; in pcpu_embed_first_chunk()
3102 for (group = 0; group < ai->nr_groups; group++) { in pcpu_embed_first_chunk()
3103 struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_embed_first_chunk()
3106 for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) { in pcpu_embed_first_chunk()
3109 free_fn(ptr, ai->unit_size); in pcpu_embed_first_chunk()
3113 memcpy(ptr, __per_cpu_load, ai->static_size); in pcpu_embed_first_chunk()
3114 free_fn(ptr + size_sum, ai->unit_size - size_sum); in pcpu_embed_first_chunk()
3119 for (group = 0; group < ai->nr_groups; group++) { in pcpu_embed_first_chunk()
3120 ai->groups[group].base_offset = areas[group] - base; in pcpu_embed_first_chunk()
3124 PFN_DOWN(size_sum), ai->static_size, ai->reserved_size, in pcpu_embed_first_chunk()
3125 ai->dyn_size, ai->unit_size); in pcpu_embed_first_chunk()
3127 pcpu_setup_first_chunk(ai, base); in pcpu_embed_first_chunk()
3131 for (group = 0; group < ai->nr_groups; group++) in pcpu_embed_first_chunk()
3134 ai->groups[group].nr_units * ai->unit_size); in pcpu_embed_first_chunk()
3136 pcpu_free_alloc_info(ai); in pcpu_embed_first_chunk()
3166 struct pcpu_alloc_info *ai; in pcpu_page_first_chunk() local
3177 ai = pcpu_build_alloc_info(reserved_size, 0, PAGE_SIZE, NULL); in pcpu_page_first_chunk()
3178 if (IS_ERR(ai)) in pcpu_page_first_chunk()
3179 return PTR_ERR(ai); in pcpu_page_first_chunk()
3180 BUG_ON(ai->nr_groups != 1); in pcpu_page_first_chunk()
3181 upa = ai->alloc_size/ai->unit_size; in pcpu_page_first_chunk()
3183 if (WARN_ON(ai->groups[0].nr_units != nr_g0_units)) { in pcpu_page_first_chunk()
3184 pcpu_free_alloc_info(ai); in pcpu_page_first_chunk()
3188 unit_pages = ai->unit_size >> PAGE_SHIFT; in pcpu_page_first_chunk()
3201 unsigned int cpu = ai->groups[0].cpu_map[unit]; in pcpu_page_first_chunk()
3219 vm.size = num_possible_cpus() * ai->unit_size; in pcpu_page_first_chunk()
3224 (unsigned long)vm.addr + unit * ai->unit_size; in pcpu_page_first_chunk()
3244 memcpy((void *)unit_addr, __per_cpu_load, ai->static_size); in pcpu_page_first_chunk()
3249 unit_pages, psize_str, ai->static_size, in pcpu_page_first_chunk()
3250 ai->reserved_size, ai->dyn_size); in pcpu_page_first_chunk()
3252 pcpu_setup_first_chunk(ai, vm.addr); in pcpu_page_first_chunk()
3261 pcpu_free_alloc_info(ai); in pcpu_page_first_chunk()
3329 struct pcpu_alloc_info *ai; in setup_per_cpu_areas() local
3332 ai = pcpu_alloc_alloc_info(1, 1); in setup_per_cpu_areas()
3334 if (!ai || !fc) in setup_per_cpu_areas()
3339 ai->dyn_size = unit_size; in setup_per_cpu_areas()
3340 ai->unit_size = unit_size; in setup_per_cpu_areas()
3341 ai->atom_size = unit_size; in setup_per_cpu_areas()
3342 ai->alloc_size = unit_size; in setup_per_cpu_areas()
3343 ai->groups[0].nr_units = 1; in setup_per_cpu_areas()
3344 ai->groups[0].cpu_map[0] = 0; in setup_per_cpu_areas()
3346 pcpu_setup_first_chunk(ai, fc); in setup_per_cpu_areas()
3347 pcpu_free_alloc_info(ai); in setup_per_cpu_areas()