agp: two-stage page destruction issue
[safe/jmp/linux-2.6] / drivers / char / agp / generic.c
index c321a92..b6650a6 100644 (file)
@@ -27,7 +27,6 @@
  * TODO:
  * - Allocate more than order 0 pages to avoid too much linear map splitting.
  */
-#include <linux/config.h>
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/init.h>
@@ -38,6 +37,7 @@
 #include <linux/vmalloc.h>
 #include <linux/dma-mapping.h>
 #include <linux/mm.h>
+#include <linux/sched.h>
 #include <asm/io.h>
 #include <asm/cacheflush.h>
 #include <asm/pgtable.h>
@@ -52,26 +52,6 @@ int agp_memory_reserved;
  */
 EXPORT_SYMBOL_GPL(agp_memory_reserved);
 
-#if defined(CONFIG_X86)
-int map_page_into_agp(struct page *page)
-{
-       int i;
-       i = change_page_attr(page, 1, PAGE_KERNEL_NOCACHE);
-       global_flush_tlb();
-       return i;
-}
-EXPORT_SYMBOL_GPL(map_page_into_agp);
-
-int unmap_page_from_agp(struct page *page)
-{
-       int i;
-       i = change_page_attr(page, 1, PAGE_KERNEL);
-       global_flush_tlb();
-       return i;
-}
-EXPORT_SYMBOL_GPL(unmap_page_from_agp);
-#endif
-
 /*
  * Generic routines for handling agp_memory structures -
  * They use the basic page allocation routines to do the brunt of the work.
@@ -100,24 +80,87 @@ static int agp_get_key(void)
        return -1;
 }
 
+void agp_flush_chipset(struct agp_bridge_data *bridge)
+{
+       if (bridge->driver->chipset_flush)
+               bridge->driver->chipset_flush(bridge);
+}
+EXPORT_SYMBOL(agp_flush_chipset);
 
-struct agp_memory *agp_create_memory(int scratch_pages)
+/*
+ * Use kmalloc if possible for the page list. Otherwise fall back to
+ * vmalloc. This speeds things up and also saves memory for small AGP
+ * regions.
+ */
+
+void agp_alloc_page_array(size_t size, struct agp_memory *mem)
+{
+       mem->memory = NULL;
+       mem->vmalloc_flag = 0;
+
+       if (size <= 2*PAGE_SIZE)
+               mem->memory = kmalloc(size, GFP_KERNEL | __GFP_NORETRY);
+       if (mem->memory == NULL) {
+               mem->memory = vmalloc(size);
+               mem->vmalloc_flag = 1;
+       }
+}
+EXPORT_SYMBOL(agp_alloc_page_array);
+
+void agp_free_page_array(struct agp_memory *mem)
+{
+       if (mem->vmalloc_flag) {
+               vfree(mem->memory);
+       } else {
+               kfree(mem->memory);
+       }
+}
+EXPORT_SYMBOL(agp_free_page_array);
+
+
+static struct agp_memory *agp_create_user_memory(unsigned long num_agp_pages)
 {
        struct agp_memory *new;
+       unsigned long alloc_size = num_agp_pages*sizeof(struct page *);
 
-       new = kmalloc(sizeof(struct agp_memory), GFP_KERNEL);
+       new = kzalloc(sizeof(struct agp_memory), GFP_KERNEL);
+       if (new == NULL)
+               return NULL;
 
+       new->key = agp_get_key();
+
+       if (new->key < 0) {
+               kfree(new);
+               return NULL;
+       }
+
+       agp_alloc_page_array(alloc_size, new);
+
+       if (new->memory == NULL) {
+               agp_free_key(new->key);
+               kfree(new);
+               return NULL;
+       }
+       new->num_scratch_pages = 0;
+       return new;
+}
+
+struct agp_memory *agp_create_memory(int scratch_pages)
+{
+       struct agp_memory *new;
+
+       new = kzalloc(sizeof(struct agp_memory), GFP_KERNEL);
        if (new == NULL)
                return NULL;
 
-       memset(new, 0, sizeof(struct agp_memory));
        new->key = agp_get_key();
 
        if (new->key < 0) {
                kfree(new);
                return NULL;
        }
-       new->memory = vmalloc(PAGE_SIZE * scratch_pages);
+
+       agp_alloc_page_array(PAGE_SIZE * scratch_pages, new);
 
        if (new->memory == NULL) {
                agp_free_key(new->key);
@@ -125,6 +168,7 @@ struct agp_memory *agp_create_memory(int scratch_pages)
                return NULL;
        }
        new->num_scratch_pages = scratch_pages;
+       new->type = AGP_NORMAL_MEMORY;
        return new;
 }
 EXPORT_SYMBOL(agp_create_memory);
@@ -147,17 +191,28 @@ void agp_free_memory(struct agp_memory *curr)
        if (curr->is_bound == TRUE)
                agp_unbind_memory(curr);
 
+       if (curr->type >= AGP_USER_TYPES) {
+               agp_generic_free_by_type(curr);
+               return;
+       }
+
        if (curr->type != 0) {
                curr->bridge->driver->free_by_type(curr);
                return;
        }
        if (curr->page_count != 0) {
                for (i = 0; i < curr->page_count; i++) {
-                       curr->bridge->driver->agp_destroy_page(phys_to_virt(curr->memory[i]));
+                       curr->memory[i] = (unsigned long)gart_to_virt(curr->memory[i]);
+                       curr->bridge->driver->agp_destroy_page((void *)curr->memory[i],
+                                                              AGP_PAGE_DESTROY_UNMAP);
+               }
+               for (i = 0; i < curr->page_count; i++) {
+                       curr->bridge->driver->agp_destroy_page((void *)curr->memory[i],
+                                                              AGP_PAGE_DESTROY_FREE);
                }
        }
        agp_free_key(curr->key);
-       vfree(curr->memory);
+       agp_free_page_array(curr);
        kfree(curr);
 }
 EXPORT_SYMBOL(agp_free_memory);
@@ -188,6 +243,13 @@ struct agp_memory *agp_allocate_memory(struct agp_bridge_data *bridge,
        if ((atomic_read(&bridge->current_memory_agp) + page_count) > bridge->max_memory_agp)
                return NULL;
 
+       if (type >= AGP_USER_TYPES) {
+               new = agp_generic_alloc_user(page_count, type);
+               if (new)
+                       new->bridge = bridge;
+               return new;
+       }
+
        if (type != 0) {
                new = bridge->driver->alloc_by_type(page_count, type);
                if (new)
@@ -209,12 +271,10 @@ struct agp_memory *agp_allocate_memory(struct agp_bridge_data *bridge,
                        agp_free_memory(new);
                        return NULL;
                }
-               new->memory[i] = virt_to_phys(addr);
+               new->memory[i] = virt_to_gart(addr);
                new->page_count++;
        }
-       new->bridge = bridge;
-
-       flush_agp_mappings();
+       new->bridge = bridge;
 
        return new;
 }
@@ -295,23 +355,10 @@ int agp_num_entries(void)
 EXPORT_SYMBOL_GPL(agp_num_entries);
 
 
-static int check_bridge_mode(struct pci_dev *dev)
-{
-       u32 agp3;
-       u8 cap_ptr;
-
-       cap_ptr = pci_find_capability(dev, PCI_CAP_ID_AGP);
-       pci_read_config_dword(dev, cap_ptr+AGPSTAT, &agp3);
-       if (agp3 & AGPSTAT_MODE_3_0)
-               return 1;
-       return 0;
-}
-
-
 /**
  *     agp_copy_info  -  copy bridge state information
  *
- *     @info:          agp_kern_info pointer.  The caller should insure that this pointer is valid. 
+ *     @info:          agp_kern_info pointer.  The caller should insure that this pointer is valid.
  *
  *     This function copies information about the agp bridge device and the state of
  *     the agp backend into an agp_kern_info pointer.
@@ -328,11 +375,10 @@ int agp_copy_info(struct agp_bridge_data *bridge, struct agp_kern_info *info)
        info->version.minor = bridge->version->minor;
        info->chipset = SUPPORTED;
        info->device = bridge->dev;
-       if (check_bridge_mode(bridge->dev))
+       if (bridge->mode & AGPSTAT_MODE_3_0)
                info->mode = bridge->mode & ~AGP3_RESERVED_MASK;
        else
                info->mode = bridge->mode & ~AGP2_RESERVED_MASK;
-       info->mode = bridge->mode;
        info->aper_base = bridge->gart_bus_addr;
        info->aper_size = agp_return_size();
        info->max_memory = bridge->max_memory_agp;
@@ -369,7 +415,7 @@ int agp_bind_memory(struct agp_memory *curr, off_t pg_start)
                return -EINVAL;
 
        if (curr->is_bound == TRUE) {
-               printk (KERN_INFO PFX "memory %p is already bound!\n", curr);
+               printk(KERN_INFO PFX "memory %p is already bound!\n", curr);
                return -EINVAL;
        }
        if (curr->is_flushed == FALSE) {
@@ -404,7 +450,7 @@ int agp_unbind_memory(struct agp_memory *curr)
                return -EINVAL;
 
        if (curr->is_bound != TRUE) {
-               printk (KERN_INFO PFX "memory %p was not bound!\n", curr);
+               printk(KERN_INFO PFX "memory %p was not bound!\n", curr);
                return -EINVAL;
        }
 
@@ -428,15 +474,41 @@ static void agp_v2_parse_one(u32 *requested_mode, u32 *bridge_agpstat, u32 *vga_
        u32 tmp;
 
        if (*requested_mode & AGP2_RESERVED_MASK) {
-               printk (KERN_INFO PFX "reserved bits set in mode 0x%x. Fixed.\n", *requested_mode);
+               printk(KERN_INFO PFX "reserved bits set (%x) in mode 0x%x. Fixed.\n",
+                       *requested_mode & AGP2_RESERVED_MASK, *requested_mode);
                *requested_mode &= ~AGP2_RESERVED_MASK;
        }
 
+       /*
+        * Some dumb bridges are programmed to disobey the AGP2 spec.
+        * This is likely a BIOS misprogramming rather than poweron default, or
+        * it would be a lot more common.
+        * https://bugs.freedesktop.org/show_bug.cgi?id=8816
+        * AGPv2 spec 6.1.9 states:
+        *   The RATE field indicates the data transfer rates supported by this
+        *   device. A.G.P. devices must report all that apply.
+        * Fix them up as best we can.
+        */
+       switch (*bridge_agpstat & 7) {
+       case 4:
+               *bridge_agpstat |= (AGPSTAT2_2X | AGPSTAT2_1X);
+               printk(KERN_INFO PFX "BIOS bug. AGP bridge claims to only support x4 rate"
+                       "Fixing up support for x2 & x1\n");
+               break;
+       case 2:
+               *bridge_agpstat |= AGPSTAT2_1X;
+               printk(KERN_INFO PFX "BIOS bug. AGP bridge claims to only support x2 rate"
+                       "Fixing up support for x1\n");
+               break;
+       default:
+               break;
+       }
+
        /* Check the speed bits make sense. Only one should be set. */
        tmp = *requested_mode & 7;
        switch (tmp) {
                case 0:
-                       printk (KERN_INFO PFX "%s tried to set rate=x0. Setting to x1 mode.\n", current->comm);
+                       printk(KERN_INFO PFX "%s tried to set rate=x0. Setting to x1 mode.\n", current->comm);
                        *requested_mode |= AGPSTAT2_1X;
                        break;
                case 1:
@@ -506,18 +578,19 @@ static void agp_v3_parse_one(u32 *requested_mode, u32 *bridge_agpstat, u32 *vga_
        u32 tmp;
 
        if (*requested_mode & AGP3_RESERVED_MASK) {
-               printk (KERN_INFO PFX "reserved bits set in mode 0x%x. Fixed.\n", *requested_mode);
+               printk(KERN_INFO PFX "reserved bits set (%x) in mode 0x%x. Fixed.\n",
+                       *requested_mode & AGP3_RESERVED_MASK, *requested_mode);
                *requested_mode &= ~AGP3_RESERVED_MASK;
        }
 
        /* Check the speed bits make sense. */
        tmp = *requested_mode & 7;
        if (tmp == 0) {
-               printk (KERN_INFO PFX "%s tried to set rate=x0. Setting to AGP3 x4 mode.\n", current->comm);
+               printk(KERN_INFO PFX "%s tried to set rate=x0. Setting to AGP3 x4 mode.\n", current->comm);
                *requested_mode |= AGPSTAT3_4X;
        }
        if (tmp >= 3) {
-               printk (KERN_INFO PFX "%s tried to set rate=x%d. Setting to AGP3 x8 mode.\n", current->comm, tmp * 4);
+               printk(KERN_INFO PFX "%s tried to set rate=x%d. Setting to AGP3 x8 mode.\n", current->comm, tmp * 4);
                *requested_mode = (*requested_mode & ~7) | AGPSTAT3_8X;
        }
 
@@ -546,7 +619,7 @@ static void agp_v3_parse_one(u32 *requested_mode, u32 *bridge_agpstat, u32 *vga_
                 * AGP2.x 4x -> AGP3.0 4x.
                 */
                if (*requested_mode & AGPSTAT2_4X) {
-                       printk (KERN_INFO PFX "%s passes broken AGP3 flags (%x). Fixed.\n",
+                       printk(KERN_INFO PFX "%s passes broken AGP3 flags (%x). Fixed.\n",
                                                current->comm, *requested_mode);
                        *requested_mode &= ~AGPSTAT2_4X;
                        *requested_mode |= AGPSTAT3_4X;
@@ -557,7 +630,7 @@ static void agp_v3_parse_one(u32 *requested_mode, u32 *bridge_agpstat, u32 *vga_
                 * but have been passed an AGP 2.x mode.
                 * Convert AGP 1x,2x,4x -> AGP 3.0 4x.
                 */
-               printk (KERN_INFO PFX "%s passes broken AGP2 flags (%x) in AGP3 mode. Fixed.\n",
+               printk(KERN_INFO PFX "%s passes broken AGP2 flags (%x) in AGP3 mode. Fixed.\n",
                                        current->comm, *requested_mode);
                *requested_mode &= ~(AGPSTAT2_4X | AGPSTAT2_2X | AGPSTAT2_1X);
                *requested_mode |= AGPSTAT3_4X;
@@ -567,38 +640,50 @@ static void agp_v3_parse_one(u32 *requested_mode, u32 *bridge_agpstat, u32 *vga_
                if (!(*bridge_agpstat & AGPSTAT3_8X)) {
                        *bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
                        *bridge_agpstat |= AGPSTAT3_4X;
-                       printk ("%s requested AGPx8 but bridge not capable.\n", current->comm);
+                       printk(KERN_INFO PFX "%s requested AGPx8 but bridge not capable.\n", current->comm);
                        return;
                }
                if (!(*vga_agpstat & AGPSTAT3_8X)) {
                        *bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
                        *bridge_agpstat |= AGPSTAT3_4X;
-                       printk ("%s requested AGPx8 but graphic card not capable.\n", current->comm);
+                       printk(KERN_INFO PFX "%s requested AGPx8 but graphic card not capable.\n", current->comm);
                        return;
                }
                /* All set, bridge & device can do AGP x8*/
                *bridge_agpstat &= ~(AGPSTAT3_4X | AGPSTAT3_RSVD);
                goto done;
 
+       } else if (*requested_mode & AGPSTAT3_4X) {
+               *bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
+               *bridge_agpstat |= AGPSTAT3_4X;
+               goto done;
+
        } else {
 
                /*
-                * If we didn't specify AGPx8, we can only do x4.
-                * If the hardware can't do x4, we're up shit creek, and never
-                *  should have got this far.
+                * If we didn't specify an AGP mode, we see if both
+                * the graphics card, and the bridge can do x8, and use if so.
+                * If not, we fall back to x4 mode.
                 */
-               *bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
-               if ((*bridge_agpstat & AGPSTAT3_4X) && (*vga_agpstat & AGPSTAT3_4X))
-                       *bridge_agpstat |= AGPSTAT3_4X;
-               else {
-                       printk (KERN_INFO PFX "Badness. Don't know which AGP mode to set. "
-                                                       "[bridge_agpstat:%x vga_agpstat:%x fell back to:- bridge_agpstat:%x vga_agpstat:%x]\n",
-                                                       origbridge, origvga, *bridge_agpstat, *vga_agpstat);
-                       if (!(*bridge_agpstat & AGPSTAT3_4X))
-                               printk (KERN_INFO PFX "Bridge couldn't do AGP x4.\n");
-                       if (!(*vga_agpstat & AGPSTAT3_4X))
-                               printk (KERN_INFO PFX "Graphic card couldn't do AGP x4.\n");
-                       return;
+               if ((*bridge_agpstat & AGPSTAT3_8X) && (*vga_agpstat & AGPSTAT3_8X)) {
+                       printk(KERN_INFO PFX "No AGP mode specified. Setting to highest mode "
+                               "supported by bridge & card (x8).\n");
+                       *bridge_agpstat &= ~(AGPSTAT3_4X | AGPSTAT3_RSVD);
+                       *vga_agpstat &= ~(AGPSTAT3_4X | AGPSTAT3_RSVD);
+               } else {
+                       printk(KERN_INFO PFX "Fell back to AGPx4 mode because");
+                       if (!(*bridge_agpstat & AGPSTAT3_8X)) {
+                               printk(KERN_INFO PFX "bridge couldn't do x8. bridge_agpstat:%x (orig=%x)\n",
+                                       *bridge_agpstat, origbridge);
+                               *bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
+                               *bridge_agpstat |= AGPSTAT3_4X;
+                       }
+                       if (!(*vga_agpstat & AGPSTAT3_8X)) {
+                               printk(KERN_INFO PFX "graphics card couldn't do x8. vga_agpstat:%x (orig=%x)\n",
+                                       *vga_agpstat, origvga);
+                               *vga_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
+                               *vga_agpstat |= AGPSTAT3_4X;
+                       }
                }
        }
 
@@ -635,7 +720,7 @@ u32 agp_collect_device_status(struct agp_bridge_data *bridge, u32 requested_mode
        for (;;) {
                device = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, device);
                if (!device) {
-                       printk (KERN_INFO PFX "Couldn't find an AGP VGA controller.\n");
+                       printk(KERN_INFO PFX "Couldn't find an AGP VGA controller.\n");
                        return 0;
                }
                cap_ptr = pci_find_capability(device, PCI_CAP_ID_AGP);
@@ -661,7 +746,7 @@ u32 agp_collect_device_status(struct agp_bridge_data *bridge, u32 requested_mode
                bridge_agpstat &= ~AGPSTAT_FW;
 
        /* Check to see if we are operating in 3.0 mode */
-       if (check_bridge_mode(agp_bridge->dev))
+       if (agp_bridge->mode & AGPSTAT_MODE_3_0)
                agp_v3_parse_one(&requested_mode, &bridge_agpstat, &vga_agpstat);
        else
                agp_v2_parse_one(&requested_mode, &bridge_agpstat, &vga_agpstat);
@@ -732,7 +817,7 @@ void agp_generic_enable(struct agp_bridge_data *bridge, u32 requested_mode)
 
        /* Do AGP version specific frobbing. */
        if (bridge->major_version >= 3) {
-               if (check_bridge_mode(bridge->dev)) {
+               if (bridge->mode & AGPSTAT_MODE_3_0) {
                        /* If we have 3.5, we can do the isoch stuff. */
                        if (bridge->minor_version >= 5)
                                agp_3_5_enable(bridge);
@@ -747,7 +832,7 @@ void agp_generic_enable(struct agp_bridge_data *bridge, u32 requested_mode)
                    pci_write_config_dword(bridge->dev,
                                        bridge->capndx+AGPCTRL, temp);
 
-                   printk (KERN_INFO PFX "Device is in legacy mode,"
+                   printk(KERN_INFO PFX "Device is in legacy mode,"
                                " falling back to 2.x\n");
                }
        }
@@ -806,8 +891,7 @@ int agp_generic_create_gatt_table(struct agp_bridge_data *bridge)
                                break;
                        }
 
-                       table = (char *) __get_free_pages(GFP_KERNEL,
-                                                         page_order);
+                       table = alloc_gatt_pages(page_order);
 
                        if (table == NULL) {
                                i++;
@@ -821,12 +905,10 @@ int agp_generic_create_gatt_table(struct agp_bridge_data *bridge)
                                case U32_APER_SIZE:
                                        bridge->current_size = A_IDX32(bridge);
                                        break;
-                                       /* This case will never really happen. */
+                               /* These cases will never really happen. */
                                case FIXED_APER_SIZE:
                                case LVL2_APER_SIZE:
                                default:
-                                       bridge->current_size =
-                                           bridge->current_size;
                                        break;
                                }
                                temp = bridge->current_size;
@@ -838,7 +920,7 @@ int agp_generic_create_gatt_table(struct agp_bridge_data *bridge)
                size = ((struct aper_size_info_fixed *) temp)->size;
                page_order = ((struct aper_size_info_fixed *) temp)->page_order;
                num_entries = ((struct aper_size_info_fixed *) temp)->num_entries;
-               table = (char *) __get_free_pages(GFP_KERNEL, page_order);
+               table = alloc_gatt_pages(page_order);
        }
 
        if (table == NULL)
@@ -853,19 +935,24 @@ int agp_generic_create_gatt_table(struct agp_bridge_data *bridge)
        agp_gatt_table = (void *)table;
 
        bridge->driver->cache_flush();
-       bridge->gatt_table = ioremap_nocache(virt_to_phys(table),
+#ifdef CONFIG_X86
+       set_memory_uc((unsigned long)table, 1 << page_order);
+       bridge->gatt_table = (void *)table;
+#else
+       bridge->gatt_table = ioremap_nocache(virt_to_gart(table),
                                        (PAGE_SIZE * (1 << page_order)));
        bridge->driver->cache_flush();
+#endif
 
        if (bridge->gatt_table == NULL) {
                for (page = virt_to_page(table); page <= virt_to_page(table_end); page++)
                        ClearPageReserved(page);
 
-               free_pages((unsigned long) table, page_order);
+               free_gatt_pages(table, page_order);
 
                return -ENOMEM;
        }
-       bridge->gatt_bus_addr = virt_to_phys(bridge->gatt_table_real);
+       bridge->gatt_bus_addr = virt_to_gart(bridge->gatt_table_real);
 
        /* AK: bogus, should encode addresses > 4GB */
        for (i = 0; i < num_entries; i++) {
@@ -912,14 +999,18 @@ int agp_generic_free_gatt_table(struct agp_bridge_data *bridge)
         * called, then all agp memory is deallocated and removed
         * from the table. */
 
+#ifdef CONFIG_X86
+       set_memory_wb((unsigned long)bridge->gatt_table, 1 << page_order);
+#else
        iounmap(bridge->gatt_table);
+#endif
        table = (char *) bridge->gatt_table_real;
        table_end = table + ((PAGE_SIZE * (1 << page_order)) - 1);
 
        for (page = virt_to_page(table); page <= virt_to_page(table_end); page++)
                ClearPageReserved(page);
 
-       free_pages((unsigned long) bridge->gatt_table_real, page_order);
+       free_gatt_pages(bridge->gatt_table_real, page_order);
 
        agp_gatt_table = NULL;
        bridge->gatt_table = NULL;
@@ -938,11 +1029,15 @@ int agp_generic_insert_memory(struct agp_memory * mem, off_t pg_start, int type)
        off_t j;
        void *temp;
        struct agp_bridge_data *bridge;
+       int mask_type;
 
        bridge = mem->bridge;
        if (!bridge)
                return -EINVAL;
 
+       if (mem->page_count == 0)
+               return 0;
+
        temp = bridge->current_size;
 
        switch (bridge->driver->size_type) {
@@ -970,7 +1065,11 @@ int agp_generic_insert_memory(struct agp_memory * mem, off_t pg_start, int type)
        num_entries -= agp_memory_reserved/PAGE_SIZE;
        if (num_entries < 0) num_entries = 0;
 
-       if (type != 0 || mem->type != 0) {
+       if (type != mem->type)
+               return -EINVAL;
+
+       mask_type = bridge->driver->agp_type_to_mask_type(bridge, type);
+       if (mask_type != 0) {
                /* The generic routines know nothing of memory types */
                return -EINVAL;
        }
@@ -993,9 +1092,10 @@ int agp_generic_insert_memory(struct agp_memory * mem, off_t pg_start, int type)
        }
 
        for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
-               writel(bridge->driver->mask_memory(bridge, mem->memory[i], mem->type), bridge->gatt_table+j);
-               readl(bridge->gatt_table+j);    /* PCI Posting. */
+               writel(bridge->driver->mask_memory(bridge, mem->memory[i], mask_type),
+                      bridge->gatt_table+j);
        }
+       readl(bridge->gatt_table+j-1);  /* PCI Posting. */
 
        bridge->driver->tlb_flush(mem);
        return 0;
@@ -1007,12 +1107,20 @@ int agp_generic_remove_memory(struct agp_memory *mem, off_t pg_start, int type)
 {
        size_t i;
        struct agp_bridge_data *bridge;
+       int mask_type;
 
        bridge = mem->bridge;
        if (!bridge)
                return -EINVAL;
 
-       if (type != 0 || mem->type != 0) {
+       if (mem->page_count == 0)
+               return 0;
+
+       if (type != mem->type)
+               return -EINVAL;
+
+       mask_type = bridge->driver->agp_type_to_mask_type(bridge, type);
+       if (mask_type != 0) {
                /* The generic routines know nothing of memory types */
                return -EINVAL;
        }
@@ -1020,31 +1128,48 @@ int agp_generic_remove_memory(struct agp_memory *mem, off_t pg_start, int type)
        /* AK: bogus, should encode addresses > 4GB */
        for (i = pg_start; i < (mem->page_count + pg_start); i++) {
                writel(bridge->scratch_page, bridge->gatt_table+i);
-               readl(bridge->gatt_table+i);    /* PCI Posting. */
        }
+       readl(bridge->gatt_table+i-1);  /* PCI Posting. */
 
-       global_cache_flush();
        bridge->driver->tlb_flush(mem);
        return 0;
 }
 EXPORT_SYMBOL(agp_generic_remove_memory);
 
-
 struct agp_memory *agp_generic_alloc_by_type(size_t page_count, int type)
 {
        return NULL;
 }
 EXPORT_SYMBOL(agp_generic_alloc_by_type);
 
-
 void agp_generic_free_by_type(struct agp_memory *curr)
 {
-       vfree(curr->memory);
+       agp_free_page_array(curr);
        agp_free_key(curr->key);
        kfree(curr);
 }
 EXPORT_SYMBOL(agp_generic_free_by_type);
 
+struct agp_memory *agp_generic_alloc_user(size_t page_count, int type)
+{
+       struct agp_memory *new;
+       int i;
+       int pages;
+
+       pages = (page_count + ENTRIES_PER_PAGE - 1) / ENTRIES_PER_PAGE;
+       new = agp_create_user_memory(page_count);
+       if (new == NULL)
+               return NULL;
+
+       for (i = 0; i < page_count; i++)
+               new->memory[i] = 0;
+       new->page_count = 0;
+       new->type = type;
+       new->num_scratch_pages = pages;
+
+       return new;
+}
+EXPORT_SYMBOL(agp_generic_alloc_user);
 
 /*
  * Basic Page Allocation Routines -
@@ -1057,21 +1182,20 @@ void *agp_generic_alloc_page(struct agp_bridge_data *bridge)
 {
        struct page * page;
 
-       page = alloc_page(GFP_KERNEL);
+       page = alloc_page(GFP_KERNEL | GFP_DMA32);
        if (page == NULL)
                return NULL;
 
        map_page_into_agp(page);
 
        get_page(page);
-       SetPageLocked(page);
        atomic_inc(&agp_bridge->current_memory_agp);
        return page_address(page);
 }
 EXPORT_SYMBOL(agp_generic_alloc_page);
 
 
-void agp_generic_destroy_page(void *addr)
+void agp_generic_destroy_page(void *addr, int flags)
 {
        struct page *page;
 
@@ -1079,11 +1203,14 @@ void agp_generic_destroy_page(void *addr)
                return;
 
        page = virt_to_page(addr);
-       unmap_page_from_agp(page);
-       put_page(page);
-       unlock_page(page);
-       free_page((unsigned long)addr);
-       atomic_dec(&agp_bridge->current_memory_agp);
+       if (flags & AGP_PAGE_DESTROY_UNMAP)
+               unmap_page_from_agp(page);
+
+       if (flags & AGP_PAGE_DESTROY_FREE) {
+               put_page(page);
+               free_page((unsigned long)addr);
+               atomic_dec(&agp_bridge->current_memory_agp);
+       }
 }
 EXPORT_SYMBOL(agp_generic_destroy_page);
 
@@ -1138,6 +1265,15 @@ unsigned long agp_generic_mask_memory(struct agp_bridge_data *bridge,
 }
 EXPORT_SYMBOL(agp_generic_mask_memory);
 
+int agp_generic_type_to_mask_type(struct agp_bridge_data *bridge,
+                                 int type)
+{
+       if (type >= AGP_USER_TYPES)
+               return 0;
+       return type;
+}
+EXPORT_SYMBOL(agp_generic_type_to_mask_type);
+
 /*
  * These functions are implemented according to the AGPv3 spec,
  * which covers implementation details that had previously been
@@ -1204,7 +1340,7 @@ void agp3_generic_cleanup(void)
 }
 EXPORT_SYMBOL(agp3_generic_cleanup);
 
-struct aper_size_info_16 agp3_generic_sizes[AGP_GENERIC_SIZES_ENTRIES] =
+const struct aper_size_info_16 agp3_generic_sizes[AGP_GENERIC_SIZES_ENTRIES] =
 {
        {4096, 1048576, 10,0x000},
        {2048,  524288, 9, 0x800},