1、核心结构体

1.1、slab

  1. struct slab {
  2.     struct list_head list;    // slab对象本身链表，用在kmem_list3中free/full等链表中
  3.     unsigned long colouroff;  // 颜色偏移(着色)
  4.     void *s_mem;              // 指向第一个对象的地址，包括颜色偏移
  5.     unsigned int inuse;       // slab中活跃的obj
  6.     kmem_bufctl_t free;       // 本质上是一个空闲obj链表,用于描述下一个可用obj序号
  7.     unsigned short nodeid;    // numa id
  8. };
  10. /*
  11.  * struct slab_rcu
  12.  *
  13.  * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
  14.  * arrange for kmem_freepages to be called via RCU.  This is useful if
  15.  * we need to approach a kernel structure obliquely, from its address
  16.  * obtained without the usual locking.  We can lock the structure to
  17.  * stabilize it and check it's still at the given address, only if we
  18.  * can be sure that the memory has not been meanwhile reused for some
  19.  * other kind of object (which our subsystem's lock might corrupt).
  20.  *
  21.  * rcu_read_lock before reading the address, then rcu_read_unlock after
  22.  * taking the spinlock within the structure expected at that address.
  23.  *
  24.  * We assume struct slab_rcu can overlay struct slab when destroying.
  25.  */
  26. struct slab_rcu {
  27.     struct rcu_head head;
  28.     struct kmem_cache *cachep;
  29.     void *addr;
  30. };

1.2、kmem_cache

  1. struct kmem_cache {
  2.     struct array_cache *array[NR_CPUS];  // 每个CPU都有单独的缓存
  3.     unsigned int batchcount;             // 要交换到缓存或者从换成移动到slab的obj数量
  4.     unsigned int limit;                  // 每个CPU缓存的obj的数量上限
  5.     unsigned int shared;                 // 是否需要共享。共享数组保存在kmem_list3中
  6.     unsigned int buffer_size;            // 要缓存的obj的大小，会根据cache_line进行对齐
  7.     u32 reciprocal_buffer_size;          // buffer_size倒数
  8.     unsigned int flags;                     // ？？？？？？
  9.     unsigned int num;                     // 每个slab有多少obj
  10.     unsigned int gfporder;               // 用哪个order向伙伴系统申请page
  11.     gfp_t gfpflags;                      // 向伙伴系统申请page的flag
  12.     size_t colour;                         // ？？？？？？
  13.     unsigned int colour_off;             // ？？？？？？
  14.     struct kmem_cache *slabp_cache;      // 若slab处于page外，则用此kmem_cache分配slab
  15.     unsigned int slab_size;              // slab的大小(slab描述符/free_list/对齐字节)
  16.     unsigned int dflags;                 // ？？？？？？
  17.     void (*ctor)(void *obj);             // obj构造函数
  18.     const char *name;                    // kmem_cache名称
  19.     struct list_head next;               // kmem_cache链表
  20.     struct kmem_list3 *nodelists[MAX_NUMNODES]; // 放到最后，会用实际node数目进行变量替换
  21. };

1.3、kmem_list3

  1. struct kmem_list3 {
  2.     spinlock_t list_lock;           // lock
  3.     struct list_head slabs_partial; // 包含空闲的slab链表
  4.     struct list_head slabs_full;    // 不包含空闲对象的slab链表
  5.     struct list_head slabs_free;    // 还未使用的slab的链表
  6.     unsigned long free_objects;     // 还有多少空闲的对象
  7.     unsigned int free_limit;        // 最大包含多少数量的对象
  8.     unsigned int colour_next;        // 下一个slab管理对象的偏移
  9.     struct array_cache *shared;        // 同一个node的共享内存
  10.     struct array_cache **alien;        // 其他node的共享内存
  11.     unsigned long next_reap;        // 下一次要回收的时间
  12.     int free_touched;                // ？？？？？？
  13. };

2、核心全局静态变量

  1. // array_cache及全局静态初始化变量
  2. #define BOOT_CPUCACHE_ENTRIES    1
  3. // 这个给cache_cache使用
  4. static struct arraycache_init initarray_cache __initdata =
  5.     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
  6. // 这个给初始化generic kmem_cache使用，其实是给ac用
  7. // 构造l3的时候，ac已经生成好了，可以直接kmalloc
  8. static struct arraycache_init initarray_generic =
  9.     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
  10. struct arraycache_init {
  11.     struct array_cache cache;
  12.     void *entries[BOOT_CPUCACHE_ENTRIES];
  13. };
  14. struct array_cache {
  15.     unsigned int avail;
  16.     unsigned int limit;
  17.     unsigned int batchcount;
  18.     unsigned int touched;
  19.     spinlock_t lock;
  20.     // 这个地方是静态数组，因此需要封装一层，给此元素预留一个指针的地址
  21.     void *entry[];
  22. };
  24. // 初始化kmem_cache的时候使用的fake kmem_list3
  25. #define NUM_INIT_LISTS (3 * MAX_NUMNODES)
  26. struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
  28. // 静态初始化的kmem_cache
  29. // cache_cache用以生成kmem_cache结构体
  30. static struct kmem_cache cache_cache = {
  31.     .batchcount = 1,
  32.     .limit = BOOT_CPUCACHE_ENTRIES,
  33.     .shared = 1,
  34.     .buffer_size = sizeof(struct kmem_cache),
  35.     .name = "kmem_cache",
  36. };

3、slab系统初始化

  1. void __init kmem_cache_init(void)
  2. {
  3.     size_t left_over;
  4.     struct cache_sizes *sizes;
  5.     struct cache_names *names;
  6.     int i, order, node;
  8.     // 如果不包含其他numa节点，则不使用alien_caches
  9.     if (num_possible_nodes() == 1)
  10.         use_alien_caches = 0;
  12.     // 初始化 initkmem_list3
  13.     for (i = 0; i < NUM_INIT_LISTS; i++) {
  14.         // 把全局的initkmem_list3进行初始化
  15.         kmem_list3_init(&initkmem_list3[i]);
  16.         if (i < MAX_NUMNODES)
  17.             // 初始化静态kmem_cache
  18.             cache_cache.nodelists[i] = NULL;
  19.     }
  20.     // 初始化cache_cache
  21.     // 给nodelists赋值，将initkmem_list3前node_num个元素依次赋值给nodelists
  22.     // CACHE_CACHE为0
  23.     // initkmem_list3一共3 * MAX_NUMNODES条，其他2 * MAX_NUMNODES条会给
  24.     // ac/l3的kmem_cache使用
  25.     set_up_list3s(&cache_cache, CACHE_CACHE);
  27.     /* 全局变量slab_break_gfp_order为每个slab最多占用几个页面 
  28.      ，用来抑制碎片，比如大小为3360的对象 
  29.      ，如果其slab只占一个页面，碎片为736 
  30.      ，slab占用两个页面，则碎片大小也翻倍 
  31.      。只有当对象很大 
  32.      ，以至于slab中连一个对象都放不下时 
  33.      ，才可以超过这个值 
  34.      。有两个可能的取值 
  35.      ：当可用内存大于32MB时 
  36.      ，BREAK_GFP_ORDER_HI为1 
  37.      ，即每个slab最多占用2个页面 
  38.      ，只有当对象大小大于8192时 
  39.      ，才可以突破slab_break_gfp_order的限制 
  40.      。小于等于32MB时BREAK_GFP_ORDER_LO为0。*/ 
  41.     if (totalram_pages > (32 << 20) >> PAGE_SHIFT)
  42.         slab_break_gfp_order = BREAK_GFP_ORDER_HI;
  44.     // 获取当前node id
  45.     node = numa_node_id();
  47.     // 初始化cache_cache
  48.     // 1、添加到cache_chain链表里面
  49.     // 2、计算颜色偏移
  50.     // 3、填充array
  51.     // 4、设置kmem_list3
  52.     INIT_LIST_HEAD(&cache_chain);
  53.     list_add(&cache_cache.next, &cache_chain);
  54.     // 此处cache_line_size为64字节
  55.     cache_cache.colour_off = cache_line_size();
  56.     cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
  57.     // ？？？？？？
  58.     // 感觉有点多余，在set_up_list3s已经设置过了
  59.     cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
  61.     // 计算buffer_size及其倒数
  62.     // buffer_size要按照cache_line对齐
  63.     // nodelists放到最后就是方便根据实际node数量计算kmem_cache的大小
  64.     cache_cache.buffer_size = offsetof(struct kmem_cache, nodelists) +
  65.                  nr_node_ids * sizeof(struct kmem_list3 *);
  66.     cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
  67.                     cache_line_size());
  68.     cache_cache.reciprocal_buffer_size = reciprocal_value(cache_cache.buffer_size);
  70.     // 按照伙伴系统的页框order级别，找到第一个可以存放buffer_size的页框数目
  71.     // 并且计算此order级别可以存放多少个obj
  72.     for (order = 0; order < MAX_ORDER; order++) {
  73.         cache_estimate(order, cache_cache.buffer_size,
  74.             cache_line_size(), 0, &left_over, &cache_cache.num);
  75.         if (cache_cache.num)
  76.             break;
  77.     }
  78.     cache_cache.gfporder = order;
  79.     // left_over为slab_size - nr_objs * buffer_size - mgmt_size
  80.     cache_cache.colour = left_over / cache_cache.colour_off;
  81.     // 计算slab描述符+kmem_bufctl_t数组的大小
  82.     cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) +
  83.                       sizeof(struct slab), cache_line_size());
  85.     // 参见头文件，创建各种级别的kmem_cache
  86.     sizes = malloc_sizes;
  87.     names = cache_names;
  89.     // 先创建array_cache和kmem_list3可以使用的通用内存
  90.     // order相同，则使用同一个
  91.     sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name,
  92.                     sizes[INDEX_AC].cs_size,
  93.                     ARCH_KMALLOC_MINALIGN,
  94.                     ARCH_KMALLOC_FLAGS|SLAB_PANIC,
  95.                     NULL);
  96.     if (INDEX_AC != INDEX_L3) {
  97.         sizes[INDEX_L3].cs_cachep =
  98.             kmem_cache_create(names[INDEX_L3].name,
  99.                 sizes[INDEX_L3].cs_size,
  100.                 ARCH_KMALLOC_MINALIGN,
  101.                 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
  102.                 NULL);
  103.     }
  105.     slab_early_init = 0;
  107.     while (sizes->cs_size != ULONG_MAX) {
  108.         // 将sizes里面剩余的标准分别创建cache和dma cache
  109.         if (!sizes->cs_cachep) {
  110.             sizes->cs_cachep = kmem_cache_create(names->name,
  111.                     sizes->cs_size,
  112.                     ARCH_KMALLOC_MINALIGN,
  113.                     ARCH_KMALLOC_FLAGS|SLAB_PANIC,
  114.                     NULL);
  115.         }
  116.         sizes->cs_dmacachep = kmem_cache_create(
  117.                     names->name_dma,
  118.                     sizes->cs_size,
  119.                     ARCH_KMALLOC_MINALIGN,
  120.                     ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA|SLAB_PANIC,
  121.                     NULL);
  122.         sizes++;
  123.         names++;
  124.     }
  125.     {
  126.         struct array_cache *ptr;
  127.         // 分别替换cache_cache和ac的静态arraycache_init
  128.         ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
  129.         memcpy(ptr, cpu_cache_get(&cache_cache),
  130.                sizeof(struct arraycache_init));
  131.         spin_lock_init(&ptr->lock);
  132.         cache_cache.array[smp_processor_id()] = ptr;
  134.         ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
  135.         memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
  136.                sizeof(struct arraycache_init));
  137.         spin_lock_init(&ptr->lock);
  138.         malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
  139.             ptr;
  140.     }
  141.     {
  142.         int nid;
  143.         // 替换所有node的kmem_list3
  144.         for_each_online_node(nid) {
  145.             init_list(&cache_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
  146.             init_list(malloc_sizes[INDEX_AC].cs_cachep,
  147.                   &initkmem_list3[SIZE_AC + nid], nid);
  148.             if (INDEX_AC != INDEX_L3) {
  149.                 init_list(malloc_sizes[INDEX_L3].cs_cachep,
  150.                       &initkmem_list3[SIZE_L3 + nid], nid);
  151.             }
  152.         }
  153.     }
  154.     g_cpucache_up = EARLY;
  155. }

4、创建一个kmem_cache

  1. struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *))
  2. {
  3.     size_t left_over, slab_size, ralign;
  4.     struct kmem_cache *cachep = NULL, *pc;
  5.     gfp_t gfp;
  7.     // 名称为空/处于中断/对象小于4个字节/大于32个页面
  8.     if (!name || in_interrupt() || (size < BYTES_PER_WORD) || size > KMALLOC_MAX_SIZE) {
  9.         BUG();
  10.     }
  12.     if (slab_is_available()) {
  13.         get_online_cpus();
  14.         // lock
  15.         mutex_lock(&cache_chain_mutex);
  16.     }
  18.     // 此处删除了一个print log的代码，不影响主逻辑
  20.     // 检查对象的大小是不是32位对齐，如果不是则进行调整
  21.     if (size & (BYTES_PER_WORD - 1)) {
  22.         size += (BYTES_PER_WORD - 1);
  23.         size &= ~(BYTES_PER_WORD - 1);
  24.     }
  26.     // 检查Slab是不是按照硬件缓冲行对齐
  27.     // SLAB_HWCACHE_ALIGN：必须将对象按照硬件对齐
  28.     // 否则按照32对齐(4个字节)
  29.     if (flags & SLAB_HWCACHE_ALIGN) {
  30.         // 循环遍历，直到size > ralign / 2
  31.         ralign = cache_line_size();
  32.         while (size <= ralign / 2)
  33.             ralign /= 2;
  34.     } else {
  35.         ralign = BYTES_PER_WORD;
  36.     }
  38.     // 看文档注释
  39.     // SLAB_STORE_USER和SLAB_RED_ZONE用以debug
  40.     if (flags & SLAB_STORE_USER)
  41.         ralign = BYTES_PER_WORD;
  42.     if (flags & SLAB_RED_ZONE) {
  43.         ralign = REDZONE_ALIGN;
  44.         size += REDZONE_ALIGN - 1;
  45.         size &= ~(REDZONE_ALIGN - 1);
  46.     }
  48.     // ARCH_SLAB_MINALIGN为0，此处逻辑不会执行
  49.     // 不知道这个玩意有啥用
  50.     // ？？？？？？
  51.     if (ralign < ARCH_SLAB_MINALIGN) {
  52.         ralign = ARCH_SLAB_MINALIGN;
  53.     }
  55.     // 按照大的对齐
  56.     // 用户的需求覆盖实际需要对齐的大小
  57.     if (ralign < align) {
  58.         ralign = align;
  59.     }
  61.     // 如果ralign大于unsigned long long
  62.     // 关闭debug
  63.     if (ralign > __alignof__(unsigned long long))
  64.         flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
  66.     // 覆盖对齐大小
  67.     align = ralign;
  69.     if (slab_is_available())
  70.         gfp = GFP_KERNEL;
  71.     else
  72.         gfp = GFP_NOWAIT;
  74.     // 用cache_cache分配一个kmem_cache
  75.     cachep = kmem_cache_zalloc(&cache_cache, gfp);
  76.     if (!cachep)
  77.         goto oops;
  79.     // 如果对象比较大，大于512字节并且不处于初始化阶段
  80.     // 则slab描述符要处于page之外
  81.     if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init &&
  82.         !(flags & SLAB_NOLEAKTRACE))
  83.         flags |= CFLGS_OFF_SLAB;
  85.     // 计算对齐的size
  86.     size = ALIGN(size, align);
  88.     // 按照align对齐的obj_size/mgmt_size还能空余的空间
  89.     left_over = calculate_slab_order(cachep, size, align, flags);
  91.     // 这种情况出现应该是对象太大，导致分配不了
  92.     if (!cachep->num) {
  93.         printk(KERN_ERR
  94.                "kmem_cache_create: couldn't create cache %s.\n", name);
  95.         kmem_cache_free(&cache_cache, cachep);
  96.         cachep = NULL;
  97.         goto oops;
  98.     }
  99.     slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
  100.               + sizeof(struct slab), align);
  102.     // 有可能对象超过limit，导致偏移大于slab_size
  103.     // 这种情况可以将slab分配到内部，节省空间
  104.     if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
  105.         flags &= ~CFLGS_OFF_SLAB;
  106.         left_over -= slab_size;
  107.     }
  109.     // 如果位于外面，则计算slab_size
  110.     // 无需对齐
  111.     if (flags & CFLGS_OFF_SLAB) {
  112.         slab_size = cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
  113.         /* If we're going to use the generic kernel_map_pages()
  114.          * poisoning, then it's going to smash the contents of
  115.          * the redzone and userword anyhow, so switch them off.
  116.          */
  117.         if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
  118.             flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
  119.     }
  121.     cachep->colour_off = cache_line_size();
  122.     if (cachep->colour_off < align)
  123.         cachep->colour_off = align;
  124.     // 可以偏移几次
  125.     cachep->colour = left_over / cachep->colour_off;
  126.     cachep->slab_size = slab_size;
  127.     cachep->flags = flags;
  128.     cachep->gfpflags = 0;
  129.     if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
  130.         cachep->gfpflags |= GFP_DMA;
  131.     cachep->buffer_size = size;
  132.     cachep->reciprocal_buffer_size = reciprocal_value(size);
  134.     if (flags & CFLGS_OFF_SLAB) {
  135.         cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
  136.         /*
  137.          * This is a possibility for one of the malloc_sizes caches.
  138.          * But since we go off slab only for object size greater than
  139.          * PAGE_SIZE/8, and malloc_sizes gets created in ascending order,
  140.          * this should not happen at all.
  141.          * But leave a BUG_ON for some lucky dude.
  142.          */
  143.         BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));
  144.     }
  145.     cachep->ctor = ctor;
  146.     cachep->name = name;
  148.     if (setup_cpu_cache(cachep, gfp)) {
  149.         __kmem_cache_destroy(cachep);
  150.         cachep = NULL;
  151.         goto oops;
  152.     }
  154.     list_add(&cachep->next, &cache_chain);
  155. oops:
  156.     if (!cachep && (flags & SLAB_PANIC))
  157.         panic("kmem_cache_create(): failed to create slab `%s'\n",
  158.               name);
  159.     if (slab_is_available()) {
  160.         mutex_unlock(&cache_chain_mutex);
  161.         put_online_cpus();
  162.     }
  163.     return cachep;
  164. }

5、alloc相关API

5.1、wrapper API

  1. // 基于cache_cache分配一个kmem_cache
  2. // 头文件有kmem_cache_zalloc封装kmem_cache_alloc
  3. void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
  4. {
  5.     return __cache_alloc(cachep, flags, __builtin_return_address(0));
  6. }
  8. // 从给定node分配一个obj
  9. void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
  10. {
  11.     return __cache_alloc_node(cachep, flags, nodeid, __builtin_return_address(0));
  12. }
  14. // 对外暴露的malloc
  15. void *__kmalloc(size_t size, gfp_t flags)
  16. {
  17.     return __do_kmalloc(size, flags, NULL);
  18. }
  20. // 核心分配内存API
  21. static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, void *caller)
  22. {
  23.     struct kmem_cache *cachep;
  24.     void *ret;
  26.     // 寻找符合条件的cachep
  27.     cachep = __find_general_cachep(size, flags);
  28.     if (unlikely(ZERO_OR_NULL_PTR(cachep)))
  29.         return cachep;
  31.     // 分配一个obj
  32.     ret = __cache_alloc(cachep, flags, caller);
  33.     return ret;
  34. }

5.2、core API

  1. // 从给定cachep获取一个obj
  2. // 附带很多检查，不影响主逻辑，已删除
  3. static __always_inline void *__cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
  4. {
  5.     void *objp;
  7.     flags &= gfp_allowed_mask;
  9.     // 内存申请故障注入，忽略
  10.     if (slab_should_failslab(cachep, flags))
  11.         return NULL;
  13.     objp = __do_cache_alloc(cachep, flags);
  14.     prefetchw(objp);
  16.     if (unlikely((flags & __GFP_ZERO) && objp))
  17.         memset(objp, 0, obj_size(cachep));
  18.     return objp;
  19. }
  21. static __always_inline void *__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
  22. {
  23.     void *objp;
  25.     if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY))) {
  26.         objp = alternate_node_alloc(cache, flags);
  27.         if (objp)
  28.             goto out;
  29.     }
  30.     // 先从本node分配内存
  31.     objp = ____cache_alloc(cache, flags);
  33.     // 再从其他node分配内存
  34.      if (!objp)
  35.          objp = ____cache_alloc_node(cache, flags, numa_node_id());
  37.   out:
  38.     return objp;
  39. }
  41. static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
  42. {
  43.     void *objp;
  44.     struct array_cache *ac;
  46.     check_irq_off();
  48.     // 优先从ac分配数据
  49.     // ac数据没有，则进行fill
  50.     ac = cpu_cache_get(cachep);
  51.     if (likely(ac->avail)) {
  52.         ac->touched = 1;
  53.         objp = ac->entry[--ac->avail];
  54.     } else {
  55.         objp = cache_alloc_refill(cachep, flags);
  56.     }
  57.     return objp;
  58. }
  60. static __always_inline void *__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, void *caller)
  61. {
  62.     void *ptr;
  64.     flags &= gfp_allowed_mask;
  66.     if (nodeid == -1)
  67.         nodeid = numa_node_id();
  68.     if (unlikely(!cachep->nodelists[nodeid])) {
  69.         ptr = fallback_alloc(cachep, flags);
  70.         goto out;
  71.     }
  72.     if (nodeid == numa_node_id()) {
  73.         /*
  74.          * Use the locally cached objects if possible.
  75.          * However ____cache_alloc does not allow fallback
  76.          * to other nodes. It may fail while we still have
  77.          * objects on other nodes available.
  78.          */
  79.         ptr = ____cache_alloc(cachep, flags);
  80.         if (ptr)
  81.             goto out;
  82.     }
  83.     ptr = ____cache_alloc_node(cachep, flags, nodeid);
  84. out:
  85.     if (unlikely((flags & __GFP_ZERO) && ptr))
  86.         memset(ptr, 0, obj_size(cachep));
  87.     return ptr;
  88. }
  90. static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
  91. {
  92.     struct list_head *entry;
  93.     struct slab *slabp;
  94.     struct kmem_list3 *l3;
  95.     void *obj;
  96.     int x;
  98.     l3 = cachep->nodelists[nodeid];
  100. retry:
  101.     spin_lock(&l3->list_lock);
  102.     entry = l3->slabs_partial.next;
  103.     if (entry == &l3->slabs_partial) {
  104.         l3->free_touched = 1;
  105.         entry = l3->slabs_free.next;
  106.         if (entry == &l3->slabs_free)
  107.             goto must_grow;
  108.     }
  110.     slabp = list_entry(entry, struct slab, list);
  111.     check_spinlock_acquired_node(cachep, nodeid);
  112.     check_slabp(cachep, slabp);
  114.     obj = slab_get_obj(cachep, slabp, nodeid);
  115.     check_slabp(cachep, slabp);
  116.     l3->free_objects--;
  117.     list_del(&slabp->list);
  119.     if (slabp->free == BUFCTL_END)
  120.         list_add(&slabp->list, &l3->slabs_full);
  121.     else
  122.         list_add(&slabp->list, &l3->slabs_partial);
  124.     spin_unlock(&l3->list_lock);
  125.     goto done;
  127. must_grow:
  128.     spin_unlock(&l3->list_lock);
  129.     x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL);
  130.     if (x)
  131.         goto retry;
  132.     return fallback_alloc(cachep, flags);
  133. done:
  134.     return obj;
  135. }
  137. /*
  138.  * Fallback function if there was no memory available and no objects on a
  139.  * certain node and fall back is permitted. First we scan all the
  140.  * available nodelists for available objects. If that fails then we
  141.  * perform an allocation without specifying a node. This allows the page
  142.  * allocator to do its reclaim / fallback magic. We then insert the
  143.  * slab into the proper nodelist and then allocate from it.
  144.  */
  145. static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
  146. {
  147.     struct zonelist *zonelist;
  148.     gfp_t local_flags;
  149.     struct zoneref *z;
  150.     struct zone *zone;
  151.     enum zone_type high_zoneidx = gfp_zone(flags);
  152.     void *obj = NULL;
  153.     int nid;
  155.     if (flags & __GFP_THISNODE)
  156.         return NULL;
  158.     zonelist = node_zonelist(slab_node(current->mempolicy), flags);
  159.     local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
  161. retry:
  162.     /*
  163.      * Look through allowed nodes for objects available
  164.      * from existing per node queues.
  165.      */
  166.     for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
  167.         nid = zone_to_nid(zone);
  169.         if (cpuset_zone_allowed_hardwall(zone, flags) &&
  170.             cache->nodelists[nid] &&
  171.             cache->nodelists[nid]->free_objects) {
  172.                 obj = ____cache_alloc_node(cache, flags | GFP_THISNODE, nid);
  173.                 if (obj)
  174.                     break;
  175.         }
  176.     }
  178.     if (!obj) {
  179.         /*
  180.          * This allocation will be performed within the constraints
  181.          * of the current cpuset / memory policy requirements.
  182.          * We may trigger various forms of reclaim on the allowed
  183.          * set and go into memory reserves if necessary.
  184.          */
  185.         obj = kmem_getpages(cache, local_flags, numa_node_id());
  186.         if (obj) {
  187.             nid = page_to_nid(virt_to_page(obj));
  188.             if (cache_grow(cache, flags, nid, obj)) {
  189.                 obj = ____cache_alloc_node(cache, flags | GFP_THISNODE, nid);
  190.                 if (!obj)
  191.                     /*
  192.                      * Another processor may allocate the
  193.                      * objects in the slab since we are
  194.                      * not holding any locks.
  195.                      */
  196.                     goto retry;
  197.             } else {
  198.                 /* cache_grow already freed obj */
  199.                 obj = NULL;
  200.             }
  201.         }
  202.     }
  203.     return obj;
  204. }