241 	struct gmap *sg, *next;  in gmap_remove()  local
247 		list_for_each_entry_safe(sg, next, &gmap->children, list) {  in gmap_remove()
248 			list_del(&sg->list);  in gmap_remove()
249 			gmap_put(sg);  in gmap_remove()
957  * Expected to be called with sg->mm->mmap_lock in read and
1003  * Expected to be called with sg->mm->mmap_lock in read
1039  * Called with sg->mm->mmap_lock in read.
1172  * @sg: pointer to the shadow guest address space structure
1176  * Called with the sg->guest_table_lock
1178 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,  in gmap_insert_rmap()  argument
1183 	BUG_ON(!gmap_is_shadow(sg));  in gmap_insert_rmap()
1184 	slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_insert_rmap()
1187 							&sg->guest_table_lock);  in gmap_insert_rmap()
1188 		radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap);  in gmap_insert_rmap()
1191 		radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,  in gmap_insert_rmap()
1198  * @sg: pointer to the shadow guest address space structure
1206 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,  in gmap_protect_rmap()  argument
1216 	BUG_ON(!gmap_is_shadow(sg));  in gmap_protect_rmap()
1217 	parent = sg->parent;  in gmap_protect_rmap()
1234 			spin_lock(&sg->guest_table_lock);  in gmap_protect_rmap()
1238 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_protect_rmap()
1239 			spin_unlock(&sg->guest_table_lock);  in gmap_protect_rmap()
1282  * @sg: pointer to the shadow guest address space structure
1285  * Called with the sg->guest_table_lock
1287 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_page()  argument
1291 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_page()
1292 	table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */  in gmap_unshadow_page()
1295 	gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1);  in gmap_unshadow_page()
1296 	ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);  in gmap_unshadow_page()
1301  * @sg: pointer to the shadow guest address space structure
1305  * Called with the sg->guest_table_lock
1307 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_pgt()  argument
1312 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_pgt()
1319  * @sg: pointer to the shadow guest address space structure
1322  * Called with the sg->guest_table_lock
1324 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_pgt()  argument
1329 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_pgt()
1330 	ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */  in gmap_unshadow_pgt()
1333 	gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1);  in gmap_unshadow_pgt()
1338 	__gmap_unshadow_pgt(sg, raddr, pgt);  in gmap_unshadow_pgt()
1347  * @sg: pointer to the shadow guest address space structure
1351  * Called with the sg->guest_table_lock
1353 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_sgt()  argument
1360 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_sgt()
1366 		__gmap_unshadow_pgt(sg, raddr, pgt);  in __gmap_unshadow_sgt()
1376  * @sg: pointer to the shadow guest address space structure
1381 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_sgt()  argument
1386 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_sgt()
1387 	r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */  in gmap_unshadow_sgt()
1390 	gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1);  in gmap_unshadow_sgt()
1395 	__gmap_unshadow_sgt(sg, raddr, sgt);  in gmap_unshadow_sgt()
1404  * @sg: pointer to the shadow guest address space structure
1408  * Called with the sg->guest_table_lock
1410 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r3t()  argument
1417 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r3t()
1423 		__gmap_unshadow_sgt(sg, raddr, sgt);  in __gmap_unshadow_r3t()
1433  * @sg: pointer to the shadow guest address space structure
1436  * Called with the sg->guest_table_lock
1438 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_r3t()  argument
1443 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_r3t()
1444 	r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */  in gmap_unshadow_r3t()
1447 	gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1);  in gmap_unshadow_r3t()
1452 	__gmap_unshadow_r3t(sg, raddr, r3t);  in gmap_unshadow_r3t()
1461  * @sg: pointer to the shadow guest address space structure
1465  * Called with the sg->guest_table_lock
1467 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r2t()  argument
1474 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r2t()
1480 		__gmap_unshadow_r3t(sg, raddr, r3t);  in __gmap_unshadow_r2t()
1490  * @sg: pointer to the shadow guest address space structure
1493  * Called with the sg->guest_table_lock
1495 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_r2t()  argument
1500 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_r2t()
1501 	r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */  in gmap_unshadow_r2t()
1504 	gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1);  in gmap_unshadow_r2t()
1509 	__gmap_unshadow_r2t(sg, raddr, r2t);  in gmap_unshadow_r2t()
1518  * @sg: pointer to the shadow guest address space structure
1524 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r1t()  argument
1531 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r1t()
1537 		__gmap_unshadow_r2t(sg, raddr, r2t);  in __gmap_unshadow_r1t()
1550  * @sg: pointer to the shadow guest address space structure
1552  * Called with sg->guest_table_lock
1554 static void gmap_unshadow(struct gmap *sg)  in gmap_unshadow()  argument
1558 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow()
1559 	if (sg->removed)  in gmap_unshadow()
1561 	sg->removed = 1;  in gmap_unshadow()
1562 	gmap_call_notifier(sg, 0, -1UL);  in gmap_unshadow()
1563 	gmap_flush_tlb(sg);  in gmap_unshadow()
1564 	table = (unsigned long *)(sg->asce & _ASCE_ORIGIN);  in gmap_unshadow()
1565 	switch (sg->asce & _ASCE_TYPE_MASK) {  in gmap_unshadow()
1567 		__gmap_unshadow_r1t(sg, 0, table);  in gmap_unshadow()
1570 		__gmap_unshadow_r2t(sg, 0, table);  in gmap_unshadow()
1573 		__gmap_unshadow_r3t(sg, 0, table);  in gmap_unshadow()
1576 		__gmap_unshadow_sgt(sg, 0, table);  in gmap_unshadow()
1594 	struct gmap *sg;  in gmap_find_shadow()  local
1596 	list_for_each_entry(sg, &parent->children, list) {  in gmap_find_shadow()
1597 		if (sg->orig_asce != asce || sg->edat_level != edat_level ||  in gmap_find_shadow()
1598 		    sg->removed)  in gmap_find_shadow()
1600 		if (!sg->initialized)  in gmap_find_shadow()
1602 		refcount_inc(&sg->ref_count);  in gmap_find_shadow()
1603 		return sg;  in gmap_find_shadow()
1611  * @sg: pointer to the shadow guest address space structure
1620 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level)  in gmap_shadow_valid()  argument
1622 	if (sg->removed)  in gmap_shadow_valid()
1624 	return sg->orig_asce == asce && sg->edat_level == edat_level;  in gmap_shadow_valid()
1646 	struct gmap *sg, *new;  in gmap_shadow()  local
1653 	sg = gmap_find_shadow(parent, asce, edat_level);  in gmap_shadow()
1655 	if (sg)  in gmap_shadow()
1656 		return sg;  in gmap_shadow()
1671 	sg = gmap_find_shadow(parent, asce, edat_level);  in gmap_shadow()
1672 	if (sg) {  in gmap_shadow()
1675 		return sg;  in gmap_shadow()
1679 		list_for_each_entry(sg, &parent->children, list) {  in gmap_shadow()
1680 			if (sg->orig_asce & _ASCE_REAL_SPACE) {  in gmap_shadow()
1681 				spin_lock(&sg->guest_table_lock);  in gmap_shadow()
1682 				gmap_unshadow(sg);  in gmap_shadow()
1683 				spin_unlock(&sg->guest_table_lock);  in gmap_shadow()
1684 				list_del(&sg->list);  in gmap_shadow()
1685 				gmap_put(sg);  in gmap_shadow()
1719  * @sg: pointer to the shadow guest address space structure
1733  * Called with sg->mm->mmap_lock in read.
1735 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,  in gmap_shadow_r2t()  argument
1743 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_r2t()
1753 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1754 	table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */  in gmap_shadow_r2t()
1770 	if (sg->edat_level >= 1)  in gmap_shadow_r2t()
1772 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_r2t()
1776 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1779 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1785 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_r2t()
1786 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1788 		table = gmap_table_walk(sg, saddr, 4);  in gmap_shadow_r2t()
1795 		gmap_unshadow_r2t(sg, raddr);  in gmap_shadow_r2t()
1797 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1800 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1808  * @sg: pointer to the shadow guest address space structure
1817  * Called with sg->mm->mmap_lock in read.
1819 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,  in gmap_shadow_r3t()  argument
1827 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_r3t()
1837 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1838 	table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */  in gmap_shadow_r3t()
1854 	if (sg->edat_level >= 1)  in gmap_shadow_r3t()
1856 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_r3t()
1860 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1863 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1869 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_r3t()
1870 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1872 		table = gmap_table_walk(sg, saddr, 3);  in gmap_shadow_r3t()
1879 		gmap_unshadow_r3t(sg, raddr);  in gmap_shadow_r3t()
1881 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1884 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1892  * @sg: pointer to the shadow guest address space structure
1901  * Called with sg->mm->mmap_lock in read.
1903 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,  in gmap_shadow_sgt()  argument
1911 	BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE));  in gmap_shadow_sgt()
1921 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1922 	table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */  in gmap_shadow_sgt()
1938 	if (sg->edat_level >= 1)  in gmap_shadow_sgt()
1940 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_sgt()
1944 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1947 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1953 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_sgt()
1954 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1956 		table = gmap_table_walk(sg, saddr, 2);  in gmap_shadow_sgt()
1963 		gmap_unshadow_sgt(sg, raddr);  in gmap_shadow_sgt()
1965 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1968 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1976  * @sg: pointer to the shadow guest address space structure
1985  * Called with sg->mm->mmap_lock in read.
1987 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr,  in gmap_shadow_pgt_lookup()  argument
1995 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_pgt_lookup()
1996 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt_lookup()
1997 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt_lookup()
2008 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt_lookup()
2016  * @sg: pointer to the shadow guest address space structure
2027 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,  in gmap_shadow_pgt()  argument
2035 	BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE));  in gmap_shadow_pgt()
2037 	page = page_table_alloc_pgste(sg->mm);  in gmap_shadow_pgt()
2045 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2046 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt()
2061 	list_add(&page->lru, &sg->pt_list);  in gmap_shadow_pgt()
2065 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2068 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2072 	rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);  in gmap_shadow_pgt()
2073 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2075 		table = gmap_table_walk(sg, saddr, 1);  in gmap_shadow_pgt()
2082 		gmap_unshadow_pgt(sg, raddr);  in gmap_shadow_pgt()
2084 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2087 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2096  * @sg: pointer to the shadow guest address space structure
2104  * Called with sg->mm->mmap_lock in read.
2106 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)  in gmap_shadow_page()  argument
2116 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_page()
2117 	parent = sg->parent;  in gmap_shadow_page()
2138 			spin_lock(&sg->guest_table_lock);  in gmap_shadow_page()
2140 			tptep = (pte_t *) gmap_table_walk(sg, saddr, 0);  in gmap_shadow_page()
2142 				spin_unlock(&sg->guest_table_lock);  in gmap_shadow_page()
2147 			rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte);  in gmap_shadow_page()
2150 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_shadow_page()
2155 			spin_unlock(&sg->guest_table_lock);  in gmap_shadow_page()
2172  * Called with sg->parent->shadow_lock.
2174 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,  in gmap_shadow_notify()  argument
2180 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_notify()
2182 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_notify()
2183 	if (sg->removed) {  in gmap_shadow_notify()
2184 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2188 	start = sg->orig_asce & _ASCE_ORIGIN;  in gmap_shadow_notify()
2189 	end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE;  in gmap_shadow_notify()
2190 	if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&  in gmap_shadow_notify()
2193 		gmap_unshadow(sg);  in gmap_shadow_notify()
2194 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2195 		list_del(&sg->list);  in gmap_shadow_notify()
2196 		gmap_put(sg);  in gmap_shadow_notify()
2200 	head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_shadow_notify()
2206 			gmap_unshadow_r2t(sg, raddr);  in gmap_shadow_notify()
2209 			gmap_unshadow_r3t(sg, raddr);  in gmap_shadow_notify()
2212 			gmap_unshadow_sgt(sg, raddr);  in gmap_shadow_notify()
2215 			gmap_unshadow_pgt(sg, raddr);  in gmap_shadow_notify()
2218 			gmap_unshadow_page(sg, raddr);  in gmap_shadow_notify()
2223 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2241 	struct gmap *gmap, *sg, *next;  in ptep_notify()  local
2258 			list_for_each_entry_safe(sg, next,  in ptep_notify()
2260 				gmap_shadow_notify(sg, vmaddr, gaddr);  in ptep_notify()