15  * /fairy-tale-mode off
26 #include "intel-agp.h"
27 #include <drm/intel-gtt.h>
51 	/* This should undo anything done in ->setup() save the unmapping
56 	 * For chipsets that need to support old ums (non-gem) code, this
76 	struct page *scratch_page;
91 #define INTEL_GTT_GEN	intel_private.driver->gen
92 #define IS_G33		intel_private.driver->is_g33
93 #define IS_PINEVIEW	intel_private.driver->is_pineview
94 #define IS_IRONLAKE	intel_private.driver->is_ironlake
95 #define HAS_PGTBL_EN	intel_private.driver->has_pgtbl_enable
98 static int intel_gtt_map_memory(struct page **pages,  in intel_gtt_map_memory()
110 	for_each_sg(st->sgl, sg, num_entries, i)  in intel_gtt_map_memory()
114 			st->sgl, st->nents, PCI_DMA_BIDIRECTIONAL))  in intel_gtt_map_memory()
121 	return -ENOMEM;  in intel_gtt_map_memory()
127 	DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);  in intel_gtt_unmap_memory()
144 static struct page *i8xx_alloc_pages(void)  in i8xx_alloc_pages()
146 	struct page *page;  in i8xx_alloc_pages()  local
148 	page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);  in i8xx_alloc_pages()
149 	if (page == NULL)  in i8xx_alloc_pages()
152 	if (set_pages_uc(page, 4) < 0) {  in i8xx_alloc_pages()
153 		set_pages_wb(page, 4);  in i8xx_alloc_pages()
154 		__free_pages(page, 2);  in i8xx_alloc_pages()
157 	atomic_inc(&agp_bridge->current_memory_agp);  in i8xx_alloc_pages()
158 	return page;  in i8xx_alloc_pages()
161 static void i8xx_destroy_pages(struct page *page)  in i8xx_destroy_pages()  argument
163 	if (page == NULL)  in i8xx_destroy_pages()
166 	set_pages_wb(page, 4);  in i8xx_destroy_pages()
167 	__free_pages(page, 2);  in i8xx_destroy_pages()
168 	atomic_dec(&agp_bridge->current_memory_agp);  in i8xx_destroy_pages()
172 #define I810_GTT_ORDER 4
178 	/* i81x does not preallocate the gtt. It's always 64kb in size. */  in i810_setup()
181 		return -ENOMEM;  in i810_setup()
186 	intel_private.registers = ioremap(reg_addr, KB(64));  in i810_setup()
188 		return -ENOMEM;  in i810_setup()
197 		dev_info(&intel_private.pcidev->dev,  in i810_setup()
198 			 "detected 4MB dedicated video ram\n");  in i810_setup()
217 	if ((pg_start + mem->page_count)  in i810_insert_dcache_entries()
219 		return -EINVAL;  in i810_insert_dcache_entries()
221 	if (!mem->is_flushed)  in i810_insert_dcache_entries()
224 	for (i = pg_start; i < (pg_start + mem->page_count); i++) {  in i810_insert_dcache_entries()
226 		intel_private.driver->write_entry(addr,  in i810_insert_dcache_entries()
242 	struct page *page;  in alloc_agpphysmem_i8xx()  local
245 	case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);  in alloc_agpphysmem_i8xx()
247 	case 4:  in alloc_agpphysmem_i8xx()
248 		/* kludge to get 4 physical pages for ARGB cursor */  in alloc_agpphysmem_i8xx()
249 		page = i8xx_alloc_pages();  in alloc_agpphysmem_i8xx()
255 	if (page == NULL)  in alloc_agpphysmem_i8xx()
262 	new->pages[0] = page;  in alloc_agpphysmem_i8xx()
263 	if (pg_count == 4) {  in alloc_agpphysmem_i8xx()
264 		/* kludge to get 4 physical pages for ARGB cursor */  in alloc_agpphysmem_i8xx()
265 		new->pages[1] = new->pages[0] + 1;  in alloc_agpphysmem_i8xx()
266 		new->pages[2] = new->pages[1] + 1;  in alloc_agpphysmem_i8xx()
267 		new->pages[3] = new->pages[2] + 1;  in alloc_agpphysmem_i8xx()
269 	new->page_count = pg_count;  in alloc_agpphysmem_i8xx()
270 	new->num_scratch_pages = pg_count;  in alloc_agpphysmem_i8xx()
271 	new->type = AGP_PHYS_MEMORY;  in alloc_agpphysmem_i8xx()
272 	new->physical = page_to_phys(new->pages[0]);  in alloc_agpphysmem_i8xx()
278 	agp_free_key(curr->key);  in intel_i810_free_by_type()
279 	if (curr->type == AGP_PHYS_MEMORY) {  in intel_i810_free_by_type()
280 		if (curr->page_count == 4)  in intel_i810_free_by_type()
281 			i8xx_destroy_pages(curr->pages[0]);  in intel_i810_free_by_type()
283 			agp_bridge->driver->agp_destroy_page(curr->pages[0],  in intel_i810_free_by_type()
285 			agp_bridge->driver->agp_destroy_page(curr->pages[0],  in intel_i810_free_by_type()
296 	struct page *page;  in intel_gtt_setup_scratch_page()  local
299 	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);  in intel_gtt_setup_scratch_page()
300 	if (page == NULL)  in intel_gtt_setup_scratch_page()
301 		return -ENOMEM;  in intel_gtt_setup_scratch_page()
302 	set_pages_uc(page, 1);  in intel_gtt_setup_scratch_page()
305 		dma_addr = pci_map_page(intel_private.pcidev, page, 0,  in intel_gtt_setup_scratch_page()
308 			__free_page(page);  in intel_gtt_setup_scratch_page()
309 			return -EINVAL;  in intel_gtt_setup_scratch_page()
314 		intel_private.scratch_page_dma = page_to_phys(page);  in intel_gtt_setup_scratch_page()
316 	intel_private.scratch_page = page;  in intel_gtt_setup_scratch_page()
343 	static const int ddt[4] = { 0, 16, 32, 64 };  in intel_gtt_stolen_size()
352 	if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||  in intel_gtt_stolen_size()
353 	    intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {  in intel_gtt_stolen_size()
356 			stolen_size = KB(512);  in intel_gtt_stolen_size()
380 			stolen_size = MB(4);  in intel_gtt_stolen_size()
422 		dev_info(&intel_private.bridge_dev->dev, "detected %lluK %s memory\n",  in intel_gtt_stolen_size()
423 		       (u64)stolen_size / KB(1), local ? "local" : "stolen");  in intel_gtt_stolen_size()
425 		dev_info(&intel_private.bridge_dev->dev,  in intel_gtt_stolen_size()
426 		       "no pre-allocated video memory detected\n");  in intel_gtt_stolen_size()
478 		size = KB(128);  in i965_gtt_total_entries()
481 		size = KB(256);  in i965_gtt_total_entries()
484 		size = KB(512);  in i965_gtt_total_entries()
486 	/* GTT pagetable sizes bigger than 512KB are not possible on G33! */  in i965_gtt_total_entries()
488 		size = KB(1024);  in i965_gtt_total_entries()
491 		size = KB(2048);  in i965_gtt_total_entries()
494 		size = KB(1024 + 512);  in i965_gtt_total_entries()
497 		dev_info(&intel_private.pcidev->dev,  in i965_gtt_total_entries()
498 			 "unknown page table size, assuming 512KB\n");  in i965_gtt_total_entries()
499 		size = KB(512);  in i965_gtt_total_entries()
502 	return size/4;  in i965_gtt_total_entries()
507 	if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5)  in intel_gtt_total_entries()
562 	intel_private.driver->cleanup();  in intel_gtt_cleanup()
571  * unmapping anything from the GTT when VT-d is enabled.
576 	const unsigned short gpu_devid = intel_private.pcidev->device;  in needs_ilk_vtd_wa()
609 	ret = intel_private.driver->setup();  in intel_gtt_init()
624 	dev_info(&intel_private.bridge_dev->dev,  in intel_gtt_init()
626 			intel_private.gtt_total_entries * 4,  in intel_gtt_init()
627 			intel_private.gtt_mappable_entries * 4);  in intel_gtt_init()
629 	gtt_map_size = intel_private.gtt_total_entries * 4;  in intel_gtt_init()
639 		intel_private.driver->cleanup();  in intel_gtt_init()
641 		return -ENOMEM;  in intel_gtt_init()
670 	{64, 16384, 4},
686 			agp_bridge->current_size =  in intel_fake_agp_fetch_size()
706  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
707  * that buffer out, we just fill 1KB and clflush it out, on the assumption
762 			dev_err(&intel_private.pcidev->dev,  in intel_enable_gtt()
778 		dev_err(&intel_private.pcidev->dev,  in intel_enable_gtt()
797 	intel_private.registers = ioremap(reg_addr, KB(64));  in i830_setup()
799 		return -ENOMEM;  in i830_setup()
809 	agp_bridge->gatt_table_real = NULL;  in intel_fake_agp_create_gatt_table()
810 	agp_bridge->gatt_table = NULL;  in intel_fake_agp_create_gatt_table()
811 	agp_bridge->gatt_bus_addr = 0;  in intel_fake_agp_create_gatt_table()
824 	    return -EIO;  in intel_fake_agp_configure()
827 	agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;  in intel_fake_agp_configure()
850 	intel_private.driver->write_entry(addr, pg, flags);  in intel_gtt_insert_page()
852 	if (intel_private.driver->chipset_flush)  in intel_gtt_insert_page()
853 		intel_private.driver->chipset_flush();  in intel_gtt_insert_page()
868 	 * per-page addr for GTT */  in intel_gtt_insert_sg_entries()
869 	for_each_sg(st->sgl, sg, st->nents, i) {  in intel_gtt_insert_sg_entries()
873 			intel_private.driver->write_entry(addr, j, flags);  in intel_gtt_insert_sg_entries()
877 	readl(intel_private.gtt + j - 1);  in intel_gtt_insert_sg_entries()
878 	if (intel_private.driver->chipset_flush)  in intel_gtt_insert_sg_entries()
879 		intel_private.driver->chipset_flush();  in intel_gtt_insert_sg_entries()
886 				   struct page **pages,  in intel_gtt_insert_pages()
893 		intel_private.driver->write_entry(addr,  in intel_gtt_insert_pages()
902 	int ret = -EINVAL;  in intel_fake_agp_insert_entries()
907 		intel_gtt_clear_range(start, end - start);  in intel_fake_agp_insert_entries()
914 	if (mem->page_count == 0)  in intel_fake_agp_insert_entries()
917 	if (pg_start + mem->page_count > intel_private.gtt_total_entries)  in intel_fake_agp_insert_entries()
920 	if (type != mem->type)  in intel_fake_agp_insert_entries()
923 	if (!intel_private.driver->check_flags(type))  in intel_fake_agp_insert_entries()
926 	if (!mem->is_flushed)  in intel_fake_agp_insert_entries()
932 		ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st);  in intel_fake_agp_insert_entries()
937 		mem->sg_list = st.sgl;  in intel_fake_agp_insert_entries()
938 		mem->num_sg = st.nents;  in intel_fake_agp_insert_entries()
940 		intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages,  in intel_fake_agp_insert_entries()
946 	mem->is_flushed = true;  in intel_fake_agp_insert_entries()
956 		intel_private.driver->write_entry(intel_private.scratch_page_dma,  in intel_gtt_clear_range()
967 	if (mem->page_count == 0)  in intel_fake_agp_remove_entries()
970 	intel_gtt_clear_range(pg_start, mem->page_count);  in intel_fake_agp_remove_entries()
973 		intel_gtt_unmap_memory(mem->sg_list, mem->num_sg);  in intel_fake_agp_remove_entries()
974 		mem->sg_list = NULL;  in intel_fake_agp_remove_entries()
975 		mem->num_sg = 0;  in intel_fake_agp_remove_entries()
994 		new->type = AGP_DCACHE_MEMORY;  in intel_fake_agp_alloc_by_type()
995 		new->page_count = pg_count;  in intel_fake_agp_alloc_by_type()
996 		new->num_scratch_pages = 0;  in intel_fake_agp_alloc_by_type()
1010 	ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,  in intel_alloc_chipset_flush_resource()
1045 	pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);  in intel_i965_g33_setup_chipset_flush()
1053 		pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,  in intel_i965_g33_setup_chipset_flush()
1082 	intel_private.ifp_resource.name = "Intel Flush Page";  in intel_i9xx_setup_flush()
1086 	if (IS_G33 || INTEL_GTT_GEN >= 4) {  in intel_i9xx_setup_flush()
1095 		dev_err(&intel_private.pcidev->dev,  in intel_i9xx_setup_flush()
1096 			"can't ioremap flush page - no chipset flushing\n");  in intel_i9xx_setup_flush()
1134 	int size = KB(512);  in i9xx_setup()
1140 		return -ENOMEM;  in i9xx_setup()
1151 		intel_private.gtt_phys_addr = reg_addr + KB(512);  in i9xx_setup()
1235 	.gen = 4,
1265  * driver and gmch_driver must be non-null, and find_gmch will determine
1351 	if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {  in find_gmch()
1370 			if (gpu_pdev->device ==  in intel_gmch_probe()
1393 		bridge->driver = &intel_fake_agp_driver;  in intel_gmch_probe()
1394 		bridge->dev_private_data = &intel_private;  in intel_gmch_probe()
1395 		bridge->dev = bridge_pdev;  in intel_gmch_probe()
1410 	dev_info(&bridge_pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);  in intel_gmch_probe()
1413 		mask = intel_private.driver->dma_mask_size;  in intel_gmch_probe()
1415 			dev_err(&intel_private.pcidev->dev,  in intel_gmch_probe()
1416 				"set gfx device dma mask %d-bit failed!\n",  in intel_gmch_probe()
1445 	if (intel_private.driver->chipset_flush)  in intel_gtt_chipset_flush()
1446 		intel_private.driver->chipset_flush();  in intel_gtt_chipset_flush()
1452 	if (--intel_private.refcount)  in intel_gmch_remove()