Merge branch 'slab/next' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg...

[linux.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index fe6d7be22ef09b30c33dff7c57665b750b0610bf..5e234f1f8853e952dceefe8c6b92201fcc3853d7 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -224,7 +224,11 @@ static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
 static inline void stat(const struct kmem_cache *s, enum stat_item si)
 {
 #ifdef CONFIG_SLUB_STATS
-       __this_cpu_inc(s->cpu_slab->stat[si]);
+       /*
+        * The rmw is racy on a preemptible kernel but this is acceptable, so
+        * avoid this_cpu_add()'s irq-disable overhead.
+        */
+       raw_cpu_inc(s->cpu_slab->stat[si]);
 #endif
 }
 
@@ -1348,11 +1352,12 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
        page = alloc_slab_page(alloc_gfp, node, oo);
        if (unlikely(!page)) {
                oo = s->min;
+               alloc_gfp = flags;
                /*
                 * Allocation may have failed due to fragmentation.
                 * Try a lower order alloc if possible
                 */
-               page = alloc_slab_page(flags, node, oo);
+               page = alloc_slab_page(alloc_gfp, node, oo);
 
                if (page)
                        stat(s, ORDER_FALLBACK);
@@ -1362,7 +1367,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
                int pages = 1 << oo_order(oo);
 
-               kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
+               kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
 
                /*
                 * Objects from caches that have a constructor don't get
@@ -1685,7 +1690,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
 
        do {
                cpuset_mems_cookie = read_mems_allowed_begin();
-               zonelist = node_zonelist(slab_node(), flags);
+               zonelist = node_zonelist(mempolicy_slab_node(), flags);
                for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
                        struct kmem_cache_node *n;
 
@@ -3685,6 +3690,9 @@ static int slab_unmergeable(struct kmem_cache *s)
        if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
                return 1;
 
+       if (!is_root_cache(s))
+               return 1;
+
        if (s->ctor)
                return 1;
 
@@ -3697,9 +3705,8 @@ static int slab_unmergeable(struct kmem_cache *s)
        return 0;
 }
 
-static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,
-               size_t align, unsigned long flags, const char *name,
-               void (*ctor)(void *))
+static struct kmem_cache *find_mergeable(size_t size, size_t align,
+               unsigned long flags, const char *name, void (*ctor)(void *))
 {
        struct kmem_cache *s;
 
@@ -3722,7 +3729,7 @@ static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,
                        continue;
 
                if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
-                               continue;
+                       continue;
                /*
                 * Check if alignment is compatible.
                 * Courtesy of Adrian Drzewiecki
@@ -3733,23 +3740,24 @@ static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,
                if (s->size - size >= sizeof(void *))
                        continue;
 
-               if (!cache_match_memcg(s, memcg))
-                       continue;
-
                return s;
        }
        return NULL;
 }
 
 struct kmem_cache *
-__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
-                  size_t align, unsigned long flags, void (*ctor)(void *))
+__kmem_cache_alias(const char *name, size_t size, size_t align,
+                  unsigned long flags, void (*ctor)(void *))
 {
        struct kmem_cache *s;
 
-       s = find_mergeable(memcg, size, align, flags, name, ctor);
+       s = find_mergeable(size, align, flags, name, ctor);
        if (s) {
+               int i;
+               struct kmem_cache *c;
+
                s->refcount++;
+
                /*
                 * Adjust the object sizes so that we clear
                 * the complete object on kzalloc.
@@ -3757,6 +3765,15 @@ __kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
                s->object_size = max(s->object_size, (int)size);
                s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
 
+               for_each_memcg_cache_index(i) {
+                       c = cache_from_memcg_idx(s, i);
+                       if (!c)
+                               continue;
+                       c->object_size = s->object_size;
+                       c->inuse = max_t(int, c->inuse,
+                                        ALIGN(size, sizeof(void *)));
+               }
+
                if (sysfs_slab_alias(s, name)) {
                        s->refcount--;
                        s = NULL;
@@ -5126,6 +5143,15 @@ static const struct kset_uevent_ops slab_uevent_ops = {
 
 static struct kset *slab_kset;
 
+static inline struct kset *cache_kset(struct kmem_cache *s)
+{
+#ifdef CONFIG_MEMCG_KMEM
+       if (!is_root_cache(s))
+               return s->memcg_params->root_cache->memcg_kset;
+#endif
+       return slab_kset;
+}
+
 #define ID_STR_LENGTH 64
 
 /* Create a unique string id for a slab cache:
@@ -5191,26 +5217,39 @@ static int sysfs_slab_add(struct kmem_cache *s)
                name = create_unique_id(s);
        }
 
-       s->kobj.kset = slab_kset;
+       s->kobj.kset = cache_kset(s);
        err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
-       if (err) {
-               kobject_put(&s->kobj);
-               return err;
-       }
+       if (err)
+               goto out_put_kobj;
 
        err = sysfs_create_group(&s->kobj, &slab_attr_group);
-       if (err) {
-               kobject_del(&s->kobj);
-               kobject_put(&s->kobj);
-               return err;
+       if (err)
+               goto out_del_kobj;
+
+#ifdef CONFIG_MEMCG_KMEM
+       if (is_root_cache(s)) {
+               s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
+               if (!s->memcg_kset) {
+                       err = -ENOMEM;
+                       goto out_del_kobj;
+               }
        }
+#endif
+
        kobject_uevent(&s->kobj, KOBJ_ADD);
        if (!unmergeable) {
                /* Setup first alias */
                sysfs_slab_alias(s, s->name);
-               kfree(name);
        }
-       return 0;
+out:
+       if (!unmergeable)
+               kfree(name);
+       return err;
+out_del_kobj:
+       kobject_del(&s->kobj);
+out_put_kobj:
+       kobject_put(&s->kobj);
+       goto out;
 }
 
 static void sysfs_slab_remove(struct kmem_cache *s)
@@ -5222,6 +5261,9 @@ static void sysfs_slab_remove(struct kmem_cache *s)
                 */
                return;
 
+#ifdef CONFIG_MEMCG_KMEM
+       kset_unregister(s->memcg_kset);
+#endif
        kobject_uevent(&s->kobj, KOBJ_REMOVE);
        kobject_del(&s->kobj);
        kobject_put(&s->kobj);