slab/slub: consider a memcg parameter in kmem_create_cache

Allow a memcg parameter to be passed during cache creation.  When the slub
allocator is being used, it will only merge caches that belong to the same
memcg.  We'll do this by scanning the global list, and then translating
the cache to a memcg-specific cache

Default function is created as a wrapper, passing NULL to the memcg
version.  We only merge caches that belong to the same memcg.

A helper is provided, memcg_css_id: because slub needs a unique cache name
for sysfs.  Since this is visible, but not the canonical location for slab
data, the cache name is not used, the css_id should suffice.

Signed-off-by: Glauber Costa <[email protected]>
Cc: Christoph Lameter <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Frederic Weisbecker <[email protected]>
Cc: Greg Thelen <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: JoonSoo Kim <[email protected]>
Cc: KAMEZAWA Hiroyuki <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Pekka Enberg <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Suleiman Souhlal <[email protected]>
Cc: Tejun Heo <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 87d61e840ddda631d185d70dcea1d582324f41dc..0b69a0470007691d05dd5fea2167f64bf333ea05 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -28,6 +28,7 @@ struct mem_cgroup;
 struct page_cgroup;
 struct page;
 struct mm_struct;
 struct page_cgroup;
 struct page;
 struct mm_struct;

+struct kmem_cache;

 
 /* Stats that can be updated by kernel. */
 enum mem_cgroup_page_stat_item {
 
 /* Stats that can be updated by kernel. */
 enum mem_cgroup_page_stat_item {


@@ -441,6 +442,11 @@ void __memcg_kmem_commit_charge(struct page *page,
                                       struct mem_cgroup *memcg, int order);
 void __memcg_kmem_uncharge_pages(struct page *page, int order);
 
                                       struct mem_cgroup *memcg, int order);
 void __memcg_kmem_uncharge_pages(struct page *page, int order);
 

+int memcg_cache_id(struct mem_cgroup *memcg);
+int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s);
+void memcg_release_cache(struct kmem_cache *cachep);
+void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep);
+

 /**
  * memcg_kmem_newpage_charge: verify if a new kmem allocation is allowed.
  * @gfp: the gfp allocation flags.
 /**
  * memcg_kmem_newpage_charge: verify if a new kmem allocation is allowed.
  * @gfp: the gfp allocation flags.


@@ -525,6 +531,26 @@ static inline void
 memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order)
 {
 }
 memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order)
 {
 }

+
+static inline int memcg_cache_id(struct mem_cgroup *memcg)
+{
+       return -1;
+}
+
+static inline int memcg_register_cache(struct mem_cgroup *memcg,
+                                      struct kmem_cache *s)
+{
+       return 0;
+}
+
+static inline void memcg_release_cache(struct kmem_cache *cachep)
+{
+}
+
+static inline void memcg_cache_list_add(struct mem_cgroup *memcg,
+                                       struct kmem_cache *s)
+{
+}

 #endif /* CONFIG_MEMCG_KMEM */
 #endif /* _LINUX_MEMCONTROL_H */
 
 #endif /* CONFIG_MEMCG_KMEM */
 #endif /* _LINUX_MEMCONTROL_H */
 

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 00efba1492220f49d9dfbdc8abdea417776b0367..c0fcf28c15b20d05b0d93e0aeaedab6ece14c33a 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -116,6 +116,7 @@ struct kmem_cache {
 };
 #endif
 
 };
 #endif
 

+struct mem_cgroup;

 /*
  * struct kmem_cache related prototypes
  */
 /*
  * struct kmem_cache related prototypes
  */


@@ -125,6 +126,9 @@ int slab_is_available(void);
 struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
                        unsigned long,
                        void (*)(void *));
 struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
                        unsigned long,
                        void (*)(void *));

+struct kmem_cache *
+kmem_cache_create_memcg(struct mem_cgroup *, const char *, size_t, size_t,
+                       unsigned long, void (*)(void *));

 void kmem_cache_destroy(struct kmem_cache *);
 int kmem_cache_shrink(struct kmem_cache *);
 void kmem_cache_free(struct kmem_cache *, void *);
 void kmem_cache_destroy(struct kmem_cache *);
 int kmem_cache_shrink(struct kmem_cache *);
 void kmem_cache_free(struct kmem_cache *, void *);


@@ -191,15 +195,23 @@ void kmem_cache_free(struct kmem_cache *, void *);
  * Child caches will hold extra metadata needed for its operation. Fields are:
  *
  * @memcg: pointer to the memcg this cache belongs to
  * Child caches will hold extra metadata needed for its operation. Fields are:
  *
  * @memcg: pointer to the memcg this cache belongs to

+ * @list: list_head for the list of all caches in this memcg
+ * @root_cache: pointer to the global, root cache, this cache was derived from

  */
 struct memcg_cache_params {
        bool is_root_cache;
        union {
                struct kmem_cache *memcg_caches[0];
  */
 struct memcg_cache_params {
        bool is_root_cache;
        union {
                struct kmem_cache *memcg_caches[0];

-               struct mem_cgroup *memcg;
+               struct {
+                       struct mem_cgroup *memcg;
+                       struct list_head list;
+                       struct kmem_cache *root_cache;
+               };

        };
 };
 
        };
 };
 

+int memcg_update_all_caches(int num_memcgs);
+

 /*
  * Common kmalloc functions provided by all allocators
  */
 /*
  * Common kmalloc functions provided by all allocators
  */

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index e16694d5e1181b90c51cfc9901d3e9456bd980df..3eafe6cf6ca4300d6af69ea918d9da915d13bca0 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -341,6 +341,14 @@ struct mem_cgroup {
 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
        struct tcp_memcontrol tcp_mem;
 #endif
 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
        struct tcp_memcontrol tcp_mem;
 #endif

+#if defined(CONFIG_MEMCG_KMEM)
+       /* analogous to slab_common's slab_caches list. per-memcg */
+       struct list_head memcg_slab_caches;
+       /* Not a spinlock, we can take a lot of time walking the list */
+       struct mutex slab_caches_mutex;
+        /* Index in the kmem_cache->memcg_params->memcg_caches array */
+       int kmemcg_id;
+#endif

 };
 
 /* internal only representation about the status of kmem accounting. */
 };
 
 /* internal only representation about the status of kmem accounting. */


@@ -2785,6 +2793,47 @@ static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
                mem_cgroup_put(memcg);
 }
 
                mem_cgroup_put(memcg);
 }
 

+void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
+{
+       if (!memcg)
+               return;
+
+       mutex_lock(&memcg->slab_caches_mutex);
+       list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
+       mutex_unlock(&memcg->slab_caches_mutex);
+}
+
+/*
+ * helper for acessing a memcg's index. It will be used as an index in the
+ * child cache array in kmem_cache, and also to derive its name. This function
+ * will return -1 when this is not a kmem-limited memcg.
+ */
+int memcg_cache_id(struct mem_cgroup *memcg)
+{
+       return memcg ? memcg->kmemcg_id : -1;
+}
+
+int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s)
+{
+       size_t size = sizeof(struct memcg_cache_params);
+
+       if (!memcg_kmem_enabled())
+               return 0;
+
+       s->memcg_params = kzalloc(size, GFP_KERNEL);
+       if (!s->memcg_params)
+               return -ENOMEM;
+
+       if (memcg)
+               s->memcg_params->memcg = memcg;
+       return 0;
+}
+
+void memcg_release_cache(struct kmem_cache *s)
+{
+       kfree(s->memcg_params);
+}
+

 /*
  * We need to verify if the allocation against current->mm->owner's memcg is
  * possible for the given order. But the page is not allocated yet, so we'll
 /*
  * We need to verify if the allocation against current->mm->owner's memcg is
  * possible for the given order. But the page is not allocated yet, so we'll


@@ -5026,7 +5075,9 @@ static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
 #ifdef CONFIG_MEMCG_KMEM
 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
 #ifdef CONFIG_MEMCG_KMEM
 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {

+       memcg->kmemcg_id = -1;

        memcg_propagate_kmem(memcg);
        memcg_propagate_kmem(memcg);

+

        return mem_cgroup_sockets_init(memcg, ss);
 };
 
        return mem_cgroup_sockets_init(memcg, ss);
 };
 

diff --git a/mm/slab.h b/mm/slab.h
index 49e7a8b1d27e84d13fde536c9603fe1ee29dcc61..abe582d20c7979b004e5ec8965baec404113c7ff 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -43,12 +43,15 @@ extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
 extern void create_boot_cache(struct kmem_cache *, const char *name,
                        size_t size, unsigned long flags);
 
 extern void create_boot_cache(struct kmem_cache *, const char *name,
                        size_t size, unsigned long flags);
 

+struct mem_cgroup;

 #ifdef CONFIG_SLUB
 #ifdef CONFIG_SLUB

-struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
-       size_t align, unsigned long flags, void (*ctor)(void *));
+struct kmem_cache *
+__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
+                  size_t align, unsigned long flags, void (*ctor)(void *));

 #else
 #else

-static inline struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
-       size_t align, unsigned long flags, void (*ctor)(void *))
+static inline struct kmem_cache *
+__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
+                  size_t align, unsigned long flags, void (*ctor)(void *))

 { return NULL; }
 #endif
 
 { return NULL; }
 #endif
 


@@ -106,11 +109,23 @@ static inline bool is_root_cache(struct kmem_cache *s)
 {
        return !s->memcg_params || s->memcg_params->is_root_cache;
 }
 {
        return !s->memcg_params || s->memcg_params->is_root_cache;
 }

+
+static inline bool cache_match_memcg(struct kmem_cache *cachep,
+                                    struct mem_cgroup *memcg)
+{
+       return (is_root_cache(cachep) && !memcg) ||
+                               (cachep->memcg_params->memcg == memcg);
+}

 #else
 static inline bool is_root_cache(struct kmem_cache *s)
 {
        return true;
 }
 
 #else
 static inline bool is_root_cache(struct kmem_cache *s)
 {
        return true;
 }
 

+static inline bool cache_match_memcg(struct kmem_cache *cachep,
+                                    struct mem_cgroup *memcg)
+{
+       return true;
+}

 #endif
 #endif
 #endif
 #endif

diff --git a/mm/slab_common.c b/mm/slab_common.c
index a8e76d79ee65628cb5071b00935202536262c490..3031badcc577d7446528330ff0c61280f1b940a7 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -18,6 +18,7 @@
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 #include <asm/page.h>
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 #include <asm/page.h>

+#include <linux/memcontrol.h>

 
 #include "slab.h"
 
 
 #include "slab.h"
 


@@ -27,7 +28,8 @@ DEFINE_MUTEX(slab_mutex);
 struct kmem_cache *kmem_cache;
 
 #ifdef CONFIG_DEBUG_VM
 struct kmem_cache *kmem_cache;
 
 #ifdef CONFIG_DEBUG_VM

-static int kmem_cache_sanity_check(const char *name, size_t size)
+static int kmem_cache_sanity_check(struct mem_cgroup *memcg, const char *name,
+                                  size_t size)

 {
        struct kmem_cache *s = NULL;
 
 {
        struct kmem_cache *s = NULL;
 


@@ -53,7 +55,13 @@ static int kmem_cache_sanity_check(const char *name, size_t size)
                        continue;
                }
 
                        continue;
                }
 

-               if (!strcmp(s->name, name)) {
+               /*
+                * For simplicity, we won't check this in the list of memcg
+                * caches. We have control over memcg naming, and if there
+                * aren't duplicates in the global list, there won't be any
+                * duplicates in the memcg lists as well.
+                */
+               if (!memcg && !strcmp(s->name, name)) {

                        pr_err("%s (%s): Cache name already exists.\n",
                               __func__, name);
                        dump_stack();
                        pr_err("%s (%s): Cache name already exists.\n",
                               __func__, name);
                        dump_stack();


@@ -66,7 +74,8 @@ static int kmem_cache_sanity_check(const char *name, size_t size)
        return 0;
 }
 #else
        return 0;
 }
 #else

-static inline int kmem_cache_sanity_check(const char *name, size_t size)
+static inline int kmem_cache_sanity_check(struct mem_cgroup *memcg,
+                                         const char *name, size_t size)

 {
        return 0;
 }
 {
        return 0;
 }


@@ -125,8 +134,9 @@ unsigned long calculate_alignment(unsigned long flags,
  * as davem.
  */
 
  * as davem.
  */
 

-struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align,
-               unsigned long flags, void (*ctor)(void *))
+struct kmem_cache *
+kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
+                       size_t align, unsigned long flags, void (*ctor)(void *))

 {
        struct kmem_cache *s = NULL;
        int err = 0;
 {
        struct kmem_cache *s = NULL;
        int err = 0;


@@ -134,7 +144,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
        get_online_cpus();
        mutex_lock(&slab_mutex);
 
        get_online_cpus();
        mutex_lock(&slab_mutex);
 

-       if (!kmem_cache_sanity_check(name, size) == 0)
+       if (!kmem_cache_sanity_check(memcg, name, size) == 0)

                goto out_locked;
 
        /*
                goto out_locked;
 
        /*


@@ -145,7 +155,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
         */
        flags &= CACHE_CREATE_MASK;
 
         */
        flags &= CACHE_CREATE_MASK;
 

-       s = __kmem_cache_alias(name, size, align, flags, ctor);
+       s = __kmem_cache_alias(memcg, name, size, align, flags, ctor);

        if (s)
                goto out_locked;
 
        if (s)
                goto out_locked;
 


@@ -154,6 +164,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
                s->object_size = s->size = size;
                s->align = calculate_alignment(flags, align, size);
                s->ctor = ctor;
                s->object_size = s->size = size;
                s->align = calculate_alignment(flags, align, size);
                s->ctor = ctor;

+
+               if (memcg_register_cache(memcg, s)) {
+                       kmem_cache_free(kmem_cache, s);
+                       err = -ENOMEM;
+                       goto out_locked;
+               }
+

                s->name = kstrdup(name, GFP_KERNEL);
                if (!s->name) {
                        kmem_cache_free(kmem_cache, s);
                s->name = kstrdup(name, GFP_KERNEL);
                if (!s->name) {
                        kmem_cache_free(kmem_cache, s);


@@ -163,10 +180,9 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
 
                err = __kmem_cache_create(s, flags);
                if (!err) {
 
                err = __kmem_cache_create(s, flags);
                if (!err) {

-

                        s->refcount = 1;
                        list_add(&s->list, &slab_caches);
                        s->refcount = 1;
                        list_add(&s->list, &slab_caches);

-
+                       memcg_cache_list_add(memcg, s);

                } else {
                        kfree(s->name);
                        kmem_cache_free(kmem_cache, s);
                } else {
                        kfree(s->name);
                        kmem_cache_free(kmem_cache, s);


@@ -194,6 +210,13 @@ out_locked:
 
        return s;
 }
 
        return s;
 }

+
+struct kmem_cache *
+kmem_cache_create(const char *name, size_t size, size_t align,
+                 unsigned long flags, void (*ctor)(void *))
+{
+       return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor);
+}

 EXPORT_SYMBOL(kmem_cache_create);
 
 void kmem_cache_destroy(struct kmem_cache *s)
 EXPORT_SYMBOL(kmem_cache_create);
 
 void kmem_cache_destroy(struct kmem_cache *s)


@@ -209,6 +232,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
                        if (s->flags & SLAB_DESTROY_BY_RCU)
                                rcu_barrier();
 
                        if (s->flags & SLAB_DESTROY_BY_RCU)
                                rcu_barrier();
 

+                       memcg_release_cache(s);

                        kfree(s->name);
                        kmem_cache_free(kmem_cache, s);
                } else {
                        kfree(s->name);
                        kmem_cache_free(kmem_cache, s);
                } else {

diff --git a/mm/slub.c b/mm/slub.c
index 87f9f32bf0cd86351ba4893bae66d64ff31e06c3..985332b38852bf309d8e88f4b7562d5272ab9173 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -31,6 +31,7 @@
 #include <linux/fault-inject.h>
 #include <linux/stacktrace.h>
 #include <linux/prefetch.h>
 #include <linux/fault-inject.h>
 #include <linux/stacktrace.h>
 #include <linux/prefetch.h>

+#include <linux/memcontrol.h>

 
 #include <trace/events/kmem.h>
 
 
 #include <trace/events/kmem.h>
 


@@ -3786,7 +3787,7 @@ static int slab_unmergeable(struct kmem_cache *s)
        return 0;
 }
 
        return 0;
 }
 

-static struct kmem_cache *find_mergeable(size_t size,
+static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,

                size_t align, unsigned long flags, const char *name,
                void (*ctor)(void *))
 {
                size_t align, unsigned long flags, const char *name,
                void (*ctor)(void *))
 {


@@ -3822,17 +3823,21 @@ static struct kmem_cache *find_mergeable(size_t size,
                if (s->size - size >= sizeof(void *))
                        continue;
 
                if (s->size - size >= sizeof(void *))
                        continue;
 

+               if (!cache_match_memcg(s, memcg))
+                       continue;
+

                return s;
        }
        return NULL;
 }
 
                return s;
        }
        return NULL;
 }
 

-struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
-               size_t align, unsigned long flags, void (*ctor)(void *))
+struct kmem_cache *
+__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
+                  size_t align, unsigned long flags, void (*ctor)(void *))

 {
        struct kmem_cache *s;
 
 {
        struct kmem_cache *s;
 

-       s = find_mergeable(size, align, flags, name, ctor);
+       s = find_mergeable(memcg, size, align, flags, name, ctor);

        if (s) {
                s->refcount++;
                /*
        if (s) {
                s->refcount++;
                /*


@@ -5156,6 +5161,12 @@ static char *create_unique_id(struct kmem_cache *s)
        if (p != name + 1)
                *p++ = '-';
        p += sprintf(p, "%07d", s->size);
        if (p != name + 1)
                *p++ = '-';
        p += sprintf(p, "%07d", s->size);

+
+#ifdef CONFIG_MEMCG_KMEM
+       if (!is_root_cache(s))
+               p += sprintf(p, "-%08d", memcg_cache_id(s->memcg_params->memcg));
+#endif
+

        BUG_ON(p > name + ID_STR_LENGTH - 1);
        return name;
 }
        BUG_ON(p > name + ID_STR_LENGTH - 1);
        return name;
 }