33 	struct nd_region *nd_region = to_nd_region(dev->parent);  in namespace_pmem_release()  local
36 		ida_simple_remove(&nd_region->ns_ida, nspm->id);  in namespace_pmem_release()
45 	struct nd_region *nd_region = to_nd_region(dev->parent);  in namespace_blk_release()  local
48 		ida_simple_remove(&nd_region->ns_ida, nsblk->id);  in namespace_blk_release()
140 	struct nd_region *nd_region = to_nd_region(dev->parent);  in pmem_should_map_pages()  local
147 	if (!test_bit(ND_REGION_PAGEMAP, &nd_region->flags))  in pmem_should_map_pages()
192 	struct nd_region *nd_region = to_nd_region(ndns->dev.parent);  in nvdimm_namespace_disk_name()  local
209 			sprintf(name, "pmem%d.%d%s", nd_region->id, nsidx,  in nvdimm_namespace_disk_name()
212 			sprintf(name, "pmem%d%s", nd_region->id,  in nvdimm_namespace_disk_name()
218 		sprintf(name, "ndblk%d.%d%s", nd_region->id, nsblk->id,  in nvdimm_namespace_disk_name()
251 	struct nd_region *nd_region = to_nd_region(dev->parent);  in nstype_show()  local
253 	return sprintf(buf, "%d\n", nd_region_to_nstype(nd_region));  in nstype_show()
305 	struct nd_region *nd_region = to_nd_region(nsblk->common.dev.parent);  in nd_namespace_blk_size()  local
306 	struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in nd_namespace_blk_size()
323 	struct nd_region *nd_region = to_nd_region(nsblk->common.dev.parent);  in __nd_namespace_blk_validate()  local
324 	struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in __nd_namespace_blk_validate()
380 static int nd_namespace_label_update(struct nd_region *nd_region,  in nd_namespace_label_update()  argument
401 		return nd_pmem_namespace_label_update(nd_region, nspm, size);  in nd_namespace_label_update()
411 		return nd_blk_namespace_label_update(nd_region, nsblk, size);  in nd_namespace_label_update()
419 	struct nd_region *nd_region = to_nd_region(dev->parent);  in alt_name_store()  local
427 		rc = nd_namespace_label_update(nd_region, dev);  in alt_name_store()
455 static int scan_free(struct nd_region *nd_region,  in scan_free()  argument
477 			nd_dbg_dpa(nd_region, ndd, res, "delete %d\n", rc);  in scan_free()
495 		nd_dbg_dpa(nd_region, ndd, res, "shrink %d\n", rc);  in scan_free()
513 static int shrink_dpa_allocation(struct nd_region *nd_region,  in shrink_dpa_allocation()  argument
518 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in shrink_dpa_allocation()
519 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in shrink_dpa_allocation()
522 		rc = scan_free(nd_region, nd_mapping, label_id, n);  in shrink_dpa_allocation()
531 		struct nd_region *nd_region, struct nd_mapping *nd_mapping,  in init_dpa_allocation()  argument
551 	nd_dbg_dpa(nd_region, ndd, res, "init %d\n", rc);  in init_dpa_allocation()
572 static void space_valid(struct nd_region *nd_region, struct nvdimm_drvdata *ndd,  in space_valid()  argument
587 		struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in space_valid()
595 		WARN_ON(!is_nd_blk(&nd_region->dev));  in space_valid()
596 		nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);  in space_valid()
623 static resource_size_t scan_allocate(struct nd_region *nd_region,  in scan_allocate()  argument
660 			space_valid(nd_region, ndd, label_id, NULL, next, exist,  in scan_allocate()
671 			space_valid(nd_region, ndd, label_id, res, next, exist,  in scan_allocate()
682 			space_valid(nd_region, ndd, label_id, res, next, exist,  in scan_allocate()
745 		nd_dbg_dpa(nd_region, ndd, new_res, "%s(%d) %d\n",  in scan_allocate()
771 		return init_dpa_allocation(label_id, nd_region, nd_mapping, n);  in scan_allocate()
775 static int merge_dpa(struct nd_region *nd_region,  in merge_dpa()  argument
796 		nd_dbg_dpa(nd_region, ndd, res, "merge %d\n", rc);  in merge_dpa()
809 	struct nd_region *nd_region;  in __reserve_free_pmem()  local
816 	nd_region = to_nd_region(dev);  in __reserve_free_pmem()
817 	if (nd_region->ndr_mappings == 0)  in __reserve_free_pmem()
822 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in __reserve_free_pmem()
823 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in __reserve_free_pmem()
829 		n = nd_pmem_available_dpa(nd_region, nd_mapping, &rem);  in __reserve_free_pmem()
832 		rem = scan_allocate(nd_region, nd_mapping, &label_id, n);  in __reserve_free_pmem()
833 		dev_WARN_ONCE(&nd_region->dev, rem,  in __reserve_free_pmem()
880 static int grow_dpa_allocation(struct nd_region *nd_region,  in grow_dpa_allocation()  argument
883 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);  in grow_dpa_allocation()
887 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in grow_dpa_allocation()
888 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in grow_dpa_allocation()
904 			rem = scan_allocate(nd_region, nd_mapping,  in grow_dpa_allocation()
914 		dev_WARN_ONCE(&nd_region->dev, rem,  in grow_dpa_allocation()
921 		rc = merge_dpa(nd_region, nd_mapping, label_id);  in grow_dpa_allocation()
929 static void nd_namespace_pmem_set_resource(struct nd_region *nd_region,  in nd_namespace_pmem_set_resource()  argument
941 		struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in nd_namespace_pmem_set_resource()
957 					* nd_region->ndr_mappings;  in nd_namespace_pmem_set_resource()
966 	res->start = nd_region->ndr_start + offset;  in nd_namespace_pmem_set_resource()
982 	struct nd_region *nd_region = to_nd_region(dev->parent);  in __size_store()  local
1013 	if (nd_region->ndr_mappings == 0) {  in __size_store()
1018 	div_u64_rem(val, SZ_4K * nd_region->ndr_mappings, &remainder);  in __size_store()
1021 				(SZ_4K * nd_region->ndr_mappings) / SZ_1K);  in __size_store()
1026 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in __size_store()
1027 		nd_mapping = &nd_region->mapping[i];  in __size_store()
1039 	available = nd_region_available_dpa(nd_region);  in __size_store()
1047 	val = div_u64(val, nd_region->ndr_mappings);  in __size_store()
1048 	allocated = div_u64(allocated, nd_region->ndr_mappings);  in __size_store()
1050 		rc = shrink_dpa_allocation(nd_region, &label_id,  in __size_store()
1053 		rc = grow_dpa_allocation(nd_region, &label_id, val - allocated);  in __size_store()
1061 		nd_namespace_pmem_set_resource(nd_region, nspm,  in __size_store()
1062 				val * nd_region->ndr_mappings);  in __size_store()
1071 	if (val == 0 && id != 0 && nd_region->ns_seed != dev && !ndns->claim)  in __size_store()
1080 	struct nd_region *nd_region = to_nd_region(dev->parent);  in size_store()  local
1094 		rc = nd_namespace_label_update(nd_region, dev);  in size_store()
1193 static int namespace_update_uuid(struct nd_region *nd_region,  in namespace_update_uuid()  argument
1213 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in namespace_update_uuid()
1214 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in namespace_update_uuid()
1229 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in namespace_update_uuid()
1230 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in namespace_update_uuid()
1263 	struct nd_region *nd_region = to_nd_region(dev->parent);  in uuid_store()  local
1287 		rc = namespace_update_uuid(nd_region, dev, uuid, ns_uuid);  in uuid_store()
1289 		rc = nd_namespace_label_update(nd_region, dev);  in uuid_store()
1351 	struct nd_region *nd_region = to_nd_region(dev->parent);  in sector_size_store()  local
1376 		rc = nd_namespace_label_update(nd_region, dev);  in sector_size_store()
1390 	struct nd_region *nd_region = to_nd_region(dev->parent);  in dpa_extents_show()  local
1413 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in dpa_extents_show()
1414 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in dpa_extents_show()
1431 	struct nd_region *nd_region = to_nd_region(dev->parent);  in btt_claim_class()  local
1434 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in btt_claim_class()
1435 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in btt_claim_class()
1532 	struct nd_region *nd_region = to_nd_region(dev->parent);  in holder_class_store()  local
1540 		rc = nd_namespace_label_update(nd_region, dev);  in holder_class_store()
1751 static struct device **create_namespace_io(struct nd_region *nd_region)  in create_namespace_io()  argument
1769 	dev->parent = &nd_region->dev;  in create_namespace_io()
1771 	res->name = dev_name(&nd_region->dev);  in create_namespace_io()
1773 	res->start = nd_region->ndr_start;  in create_namespace_io()
1774 	res->end = res->start + nd_region->ndr_size - 1;  in create_namespace_io()
1780 static bool has_uuid_at_pos(struct nd_region *nd_region, u8 *uuid,  in has_uuid_at_pos()  argument
1786 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in has_uuid_at_pos()
1787 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in has_uuid_at_pos()
1788 		struct nd_interleave_set *nd_set = nd_region->nd_set;  in has_uuid_at_pos()
1826 			if (nlabel != nd_region->ndr_mappings)  in has_uuid_at_pos()
1839 static int select_pmem_id(struct nd_region *nd_region, u8 *pmem_id)  in select_pmem_id()  argument
1846 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in select_pmem_id()
1847 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in select_pmem_id()
1880 			dev_dbg(&nd_region->dev, "%s invalid label for %pUb\n",  in select_pmem_id()
1897 struct device *create_namespace_pmem(struct nd_region *nd_region,  in create_namespace_pmem()  argument
1901 	u64 cookie = nd_region_interleave_set_cookie(nd_region, nsindex);  in create_namespace_pmem()
1902 	u64 altcookie = nd_region_interleave_set_altcookie(nd_region);  in create_namespace_pmem()
1913 		dev_dbg(&nd_region->dev, "invalid interleave-set-cookie\n");  in create_namespace_pmem()
1918 		dev_dbg(&nd_region->dev, "invalid cookie in label: %pUb\n",  in create_namespace_pmem()
1923 		dev_dbg(&nd_region->dev, "valid altcookie in label: %pUb\n",  in create_namespace_pmem()
1934 	dev->parent = &nd_region->dev;  in create_namespace_pmem()
1936 	res->name = dev_name(&nd_region->dev);  in create_namespace_pmem()
1939 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in create_namespace_pmem()
1940 		if (has_uuid_at_pos(nd_region, nd_label->uuid, cookie, i))  in create_namespace_pmem()
1942 		if (has_uuid_at_pos(nd_region, nd_label->uuid, altcookie, i))  in create_namespace_pmem()
1947 	if (i < nd_region->ndr_mappings) {  in create_namespace_pmem()
1948 		struct nvdimm *nvdimm = nd_region->mapping[i].nvdimm;  in create_namespace_pmem()
1955 		dev_err(&nd_region->dev, "%s missing label for %pUb\n",  in create_namespace_pmem()
1969 	rc = select_pmem_id(nd_region, nd_label->uuid);  in create_namespace_pmem()
1974 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in create_namespace_pmem()
1978 		nd_mapping = &nd_region->mapping[i];  in create_namespace_pmem()
2009 	nd_namespace_pmem_set_resource(nd_region, nspm, size);  in create_namespace_pmem()
2016 		dev_dbg(&nd_region->dev, "%s: invalid label(s)\n", __func__);  in create_namespace_pmem()
2019 		dev_dbg(&nd_region->dev, "%s: label not found\n", __func__);  in create_namespace_pmem()
2022 		dev_dbg(&nd_region->dev, "%s: unexpected err: %d\n",  in create_namespace_pmem()
2029 struct resource *nsblk_add_resource(struct nd_region *nd_region,  in nsblk_add_resource()  argument
2052 static struct device *nd_namespace_blk_create(struct nd_region *nd_region)  in nd_namespace_blk_create()  argument
2057 	if (!is_nd_blk(&nd_region->dev))  in nd_namespace_blk_create()
2066 	nsblk->id = ida_simple_get(&nd_region->ns_ida, 0, 0, GFP_KERNEL);  in nd_namespace_blk_create()
2071 	dev_set_name(dev, "namespace%d.%d", nd_region->id, nsblk->id);  in nd_namespace_blk_create()
2072 	dev->parent = &nd_region->dev;  in nd_namespace_blk_create()
2078 static struct device *nd_namespace_pmem_create(struct nd_region *nd_region)  in nd_namespace_pmem_create()  argument
2084 	if (!is_memory(&nd_region->dev))  in nd_namespace_pmem_create()
2093 	dev->parent = &nd_region->dev;  in nd_namespace_pmem_create()
2095 	res->name = dev_name(&nd_region->dev);  in nd_namespace_pmem_create()
2098 	nspm->id = ida_simple_get(&nd_region->ns_ida, 0, 0, GFP_KERNEL);  in nd_namespace_pmem_create()
2103 	dev_set_name(dev, "namespace%d.%d", nd_region->id, nspm->id);  in nd_namespace_pmem_create()
2104 	dev->parent = &nd_region->dev;  in nd_namespace_pmem_create()
2106 	nd_namespace_pmem_set_resource(nd_region, nspm, 0);  in nd_namespace_pmem_create()
2111 void nd_region_create_ns_seed(struct nd_region *nd_region)  in nd_region_create_ns_seed()  argument
2113 	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));  in nd_region_create_ns_seed()
2115 	if (nd_region_to_nstype(nd_region) == ND_DEVICE_NAMESPACE_IO)  in nd_region_create_ns_seed()
2118 	if (is_nd_blk(&nd_region->dev))  in nd_region_create_ns_seed()
2119 		nd_region->ns_seed = nd_namespace_blk_create(nd_region);  in nd_region_create_ns_seed()
2121 		nd_region->ns_seed = nd_namespace_pmem_create(nd_region);  in nd_region_create_ns_seed()
2127 	if (!nd_region->ns_seed)  in nd_region_create_ns_seed()
2128 		dev_err(&nd_region->dev, "failed to create %s namespace\n",  in nd_region_create_ns_seed()
2129 				is_nd_blk(&nd_region->dev) ? "blk" : "pmem");  in nd_region_create_ns_seed()
2131 		nd_device_register(nd_region->ns_seed);  in nd_region_create_ns_seed()
2134 void nd_region_create_dax_seed(struct nd_region *nd_region)  in nd_region_create_dax_seed()  argument
2136 	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));  in nd_region_create_dax_seed()
2137 	nd_region->dax_seed = nd_dax_create(nd_region);  in nd_region_create_dax_seed()
2142 	if (!nd_region->dax_seed)  in nd_region_create_dax_seed()
2143 		dev_err(&nd_region->dev, "failed to create dax namespace\n");  in nd_region_create_dax_seed()
2146 void nd_region_create_pfn_seed(struct nd_region *nd_region)  in nd_region_create_pfn_seed()  argument
2148 	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));  in nd_region_create_pfn_seed()
2149 	nd_region->pfn_seed = nd_pfn_create(nd_region);  in nd_region_create_pfn_seed()
2154 	if (!nd_region->pfn_seed)  in nd_region_create_pfn_seed()
2155 		dev_err(&nd_region->dev, "failed to create pfn namespace\n");  in nd_region_create_pfn_seed()
2158 void nd_region_create_btt_seed(struct nd_region *nd_region)  in nd_region_create_btt_seed()  argument
2160 	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));  in nd_region_create_btt_seed()
2161 	nd_region->btt_seed = nd_btt_create(nd_region);  in nd_region_create_btt_seed()
2166 	if (!nd_region->btt_seed)  in nd_region_create_btt_seed()
2167 		dev_err(&nd_region->dev, "failed to create btt namespace\n");  in nd_region_create_btt_seed()
2170 static int add_namespace_resource(struct nd_region *nd_region,  in add_namespace_resource()  argument
2174 	struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in add_namespace_resource()
2190 			res = nsblk_add_resource(nd_region, ndd,  in add_namespace_resource()
2195 			nd_dbg_dpa(nd_region, ndd, res, "%d assign\n", count);  in add_namespace_resource()
2197 			dev_err(&nd_region->dev,  in add_namespace_resource()
2208 struct device *create_namespace_blk(struct nd_region *nd_region,  in create_namespace_blk()  argument
2212 	struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in create_namespace_blk()
2213 	struct nd_interleave_set *nd_set = nd_region->nd_set;  in create_namespace_blk()
2241 	dev->parent = &nd_region->dev;  in create_namespace_blk()
2258 	res = nsblk_add_resource(nd_region, ndd, nsblk,  in create_namespace_blk()
2262 	nd_dbg_dpa(nd_region, ndd, res, "%d: assign\n", count);  in create_namespace_blk()
2294 static struct device **scan_labels(struct nd_region *nd_region)  in scan_labels()  argument
2299 	struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in scan_labels()
2311 		if (is_nd_blk(&nd_region->dev)  in scan_labels()
2321 		i = add_namespace_resource(nd_region, nd_label, devs, count);  in scan_labels()
2333 		if (is_nd_blk(&nd_region->dev))  in scan_labels()
2334 			dev = create_namespace_blk(nd_region, nd_label, count);  in scan_labels()
2340 			dev = create_namespace_pmem(nd_region, nsindex, nd_label);  in scan_labels()
2359 	dev_dbg(&nd_region->dev, "%s: discovered %d %s namespace%s\n",  in scan_labels()
2360 			__func__, count, is_nd_blk(&nd_region->dev)  in scan_labels()
2370 		if (is_nd_blk(&nd_region->dev)) {  in scan_labels()
2386 			nd_namespace_pmem_set_resource(nd_region, nspm, 0);  in scan_labels()
2388 		dev->parent = &nd_region->dev;  in scan_labels()
2390 	} else if (is_memory(&nd_region->dev)) {  in scan_labels()
2392 		for (i = 0; i < nd_region->ndr_mappings; i++) {  in scan_labels()
2397 			nd_mapping = &nd_region->mapping[i];  in scan_labels()
2422 			if (is_nd_blk(&nd_region->dev))  in scan_labels()
2431 static struct device **create_namespaces(struct nd_region *nd_region)  in create_namespaces()  argument
2433 	struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in create_namespaces()
2437 	if (nd_region->ndr_mappings == 0)  in create_namespaces()
2441 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in create_namespaces()
2442 		nd_mapping = &nd_region->mapping[i];  in create_namespaces()
2446 	devs = scan_labels(nd_region);  in create_namespaces()
2448 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in create_namespaces()
2449 		int reverse = nd_region->ndr_mappings - 1 - i;  in create_namespaces()
2451 		nd_mapping = &nd_region->mapping[reverse];  in create_namespaces()
2458 static int init_active_labels(struct nd_region *nd_region)  in init_active_labels()  argument
2462 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in init_active_labels()
2463 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in init_active_labels()
2481 			dev_err(&nd_region->dev, "%s: is %s, failing probe\n",  in init_active_labels()
2521 int nd_region_register_namespaces(struct nd_region *nd_region, int *err)  in nd_region_register_namespaces()  argument
2527 	nvdimm_bus_lock(&nd_region->dev);  in nd_region_register_namespaces()
2528 	rc = init_active_labels(nd_region);  in nd_region_register_namespaces()
2530 		nvdimm_bus_unlock(&nd_region->dev);  in nd_region_register_namespaces()
2534 	type = nd_region_to_nstype(nd_region);  in nd_region_register_namespaces()
2537 		devs = create_namespace_io(nd_region);  in nd_region_register_namespaces()
2541 		devs = create_namespaces(nd_region);  in nd_region_register_namespaces()
2546 	nvdimm_bus_unlock(&nd_region->dev);  in nd_region_register_namespaces()
2559 			id = ida_simple_get(&nd_region->ns_ida, 0, 0,  in nd_region_register_namespaces()
2566 			id = ida_simple_get(&nd_region->ns_ida, 0, 0,  in nd_region_register_namespaces()
2574 		dev_set_name(dev, "namespace%d.%d", nd_region->id, id);  in nd_region_register_namespaces()
2579 		nd_region->ns_seed = devs[0];  in nd_region_register_namespaces()