mm/vmalloc.c: change void* into explict vm_struct*

[android-x86/kernel.git] / mm / vmalloc.c

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 1d34d75..3e927cc 100644 (file)
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -256,7 +256,7 @@ struct vmap_area {
        struct rb_node rb_node;         /* address sorted rbtree */
        struct list_head list;          /* address sorted list */
        struct list_head purge_list;    /* "lazy purge" list */
-       void *private;
+       struct vm_struct *vm;
        struct rcu_head rcu_head;
 };
 
@@ -732,9 +732,10 @@ static void free_unmap_vmap_area_addr(unsigned long addr)
 #define VMAP_BBMAP_BITS_MIN    (VMAP_MAX_ALLOC*2)
 #define VMAP_MIN(x, y)         ((x) < (y) ? (x) : (y)) /* can't use min() */
 #define VMAP_MAX(x, y)         ((x) > (y) ? (x) : (y)) /* can't use max() */
-#define VMAP_BBMAP_BITS                VMAP_MIN(VMAP_BBMAP_BITS_MAX,           \
-                                       VMAP_MAX(VMAP_BBMAP_BITS_MIN,   \
-                                               VMALLOC_PAGES / NR_CPUS / 16))
+#define VMAP_BBMAP_BITS                \
+               VMAP_MIN(VMAP_BBMAP_BITS_MAX,   \
+               VMAP_MAX(VMAP_BBMAP_BITS_MIN,   \
+                       VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
 
 #define VMAP_BLOCK_SIZE                (VMAP_BBMAP_BITS * PAGE_SIZE)
 
@@ -1266,18 +1267,22 @@ EXPORT_SYMBOL_GPL(map_vm_area);
 DEFINE_RWLOCK(vmlist_lock);
 struct vm_struct *vmlist;
 
-static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
+static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
                              unsigned long flags, void *caller)
 {
-       struct vm_struct *tmp, **p;
-
        vm->flags = flags;
        vm->addr = (void *)va->va_start;
        vm->size = va->va_end - va->va_start;
        vm->caller = caller;
-       va->private = vm;
+       va->vm = vm;
        va->flags |= VM_VM_AREA;
+}
 
+static void insert_vmalloc_vmlist(struct vm_struct *vm)
+{
+       struct vm_struct *tmp, **p;
+
+       vm->flags &= ~VM_UNLIST;
        write_lock(&vmlist_lock);
        for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
                if (tmp->addr >= vm->addr)
@@ -1288,6 +1293,13 @@ static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
        write_unlock(&vmlist_lock);
 }
 
+static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
+                             unsigned long flags, void *caller)
+{
+       setup_vmalloc_vm(vm, va, flags, caller);
+       insert_vmalloc_vmlist(vm);
+}
+
 static struct vm_struct *__get_vm_area_node(unsigned long size,
                unsigned long align, unsigned long flags, unsigned long start,
                unsigned long end, int node, gfp_t gfp_mask, void *caller)
@@ -1326,7 +1338,18 @@ static struct vm_struct *__get_vm_area_node(unsigned long size,
                return NULL;
        }
 
-       insert_vmalloc_vm(area, va, flags, caller);
+       /*
+        * When this function is called from __vmalloc_node_range,
+        * we do not add vm_struct to vmlist here to avoid
+        * accessing uninitialized members of vm_struct such as
+        * pages and nr_pages fields. They will be set later.
+        * To distinguish it from others, we use a VM_UNLIST flag.
+        */
+       if (flags & VM_UNLIST)
+               setup_vmalloc_vm(area, va, flags, caller);
+       else
+               insert_vmalloc_vm(area, va, flags, caller);
+
        return area;
 }
 
@@ -1374,7 +1397,7 @@ static struct vm_struct *find_vm_area(const void *addr)
 
        va = find_vmap_area((unsigned long)addr);
        if (va && va->flags & VM_VM_AREA)
-               return va->private;
+               return va->vm;
 
        return NULL;
 }
@@ -1393,18 +1416,21 @@ struct vm_struct *remove_vm_area(const void *addr)
 
        va = find_vmap_area((unsigned long)addr);
        if (va && va->flags & VM_VM_AREA) {
-               struct vm_struct *vm = va->private;
-               struct vm_struct *tmp, **p;
-               /*
-                * remove from list and disallow access to this vm_struct
-                * before unmap. (address range confliction is maintained by
-                * vmap.)
-                */
-               write_lock(&vmlist_lock);
-               for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next)
-                       ;
-               *p = tmp->next;
-               write_unlock(&vmlist_lock);
+               struct vm_struct *vm = va->vm;
+
+               if (!(vm->flags & VM_UNLIST)) {
+                       struct vm_struct *tmp, **p;
+                       /*
+                        * remove from list and disallow access to
+                        * this vm_struct before unmap. (address range
+                        * confliction is maintained by vmap.)
+                        */
+                       write_lock(&vmlist_lock);
+                       for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next)
+                               ;
+                       *p = tmp->next;
+                       write_unlock(&vmlist_lock);
+               }
 
                vmap_debug_free_range(va->va_start, va->va_end);
                free_unmap_vmap_area(va);
@@ -1615,13 +1641,21 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
        if (!size || (size >> PAGE_SHIFT) > totalram_pages)
                return NULL;
 
-       area = __get_vm_area_node(size, align, VM_ALLOC, start, end, node,
-                                 gfp_mask, caller);
+       area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNLIST,
+                                 start, end, node, gfp_mask, caller);
 
        if (!area)
                return NULL;
 
        addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
+       if (!addr)
+               return NULL;
+
+       /*
+        * In this function, newly allocated vm_struct is not added
+        * to vmlist at __get_vm_area_node(). so, it is added here.
+        */
+       insert_vmalloc_vmlist(area);
 
        /*
         * A ref_count = 3 is needed because the vm_struct and vmap_area
@@ -2153,6 +2187,14 @@ struct vm_struct *alloc_vm_area(size_t size)
                return NULL;
        }
 
+       /*
+        * If the allocated address space is passed to a hypercall
+        * before being used then we cannot rely on a page fault to
+        * trigger an update of the page tables.  So sync all the page
+        * tables here.
+        */
+       vmalloc_sync_all();
+
        return area;
 }
 EXPORT_SYMBOL_GPL(alloc_vm_area);