diff -uNr linux-2.5.27-rmap/Makefile linux-2.5.27-rmap-slablru/Makefile --- linux-2.5.27-rmap/Makefile Sat Jul 20 14:26:13 2002 +++ linux-2.5.27-rmap-slablru/Makefile Sat Jul 20 15:11:55 2002 @@ -1,7 +1,7 @@ VERSION = 2 PATCHLEVEL = 5 SUBLEVEL = 27 -EXTRAVERSION = -rmap +EXTRAVERSION = -rmap-slablru # *DOCUMENTATION* # Too see a list of typical targets execute "make help" diff -uNr linux-2.5.27-rmap/fs/adfs/super.c linux-2.5.27-rmap-slablru/fs/adfs/super.c --- linux-2.5.27-rmap/fs/adfs/super.c Sat Jul 20 14:18:15 2002 +++ linux-2.5.27-rmap-slablru/fs/adfs/super.c Sat Jul 20 15:10:22 2002 @@ -237,6 +237,7 @@ init_once, NULL); if (adfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(adfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/affs/super.c linux-2.5.27-rmap-slablru/fs/affs/super.c --- linux-2.5.27-rmap/fs/affs/super.c Sat Jul 20 14:18:22 2002 +++ linux-2.5.27-rmap-slablru/fs/affs/super.c Sat Jul 20 15:10:22 2002 @@ -120,6 +120,7 @@ init_once, NULL); if (affs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(affs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/bfs/inode.c linux-2.5.27-rmap-slablru/fs/bfs/inode.c --- linux-2.5.27-rmap/fs/bfs/inode.c Sat Jul 20 14:18:15 2002 +++ linux-2.5.27-rmap-slablru/fs/bfs/inode.c Sat Jul 20 15:10:22 2002 @@ -243,6 +243,7 @@ init_once, NULL); if (bfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(bfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/coda/inode.c linux-2.5.27-rmap-slablru/fs/coda/inode.c --- linux-2.5.27-rmap/fs/coda/inode.c Sat Jul 20 14:18:11 2002 +++ linux-2.5.27-rmap-slablru/fs/coda/inode.c Sat Jul 20 15:10:22 2002 @@ -76,6 +76,8 @@ init_once, NULL); if (coda_inode_cachep == NULL) return -ENOMEM; + + kmem_set_pruner(coda_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/dcache.c linux-2.5.27-rmap-slablru/fs/dcache.c --- linux-2.5.27-rmap/fs/dcache.c Sat Jul 20 14:19:42 2002 +++ linux-2.5.27-rmap-slablru/fs/dcache.c Sat Jul 20 15:10:22 2002 @@ -123,8 +123,7 @@ return; /* dput on a free dentry? */ - if (!list_empty(&dentry->d_lru)) - BUG(); + BUG_ON(!list_empty(&dentry->d_lru)); /* * AV: ->d_delete() is _NOT_ allowed to block now. */ @@ -329,12 +328,11 @@ void prune_dcache(int count) { spin_lock(&dcache_lock); - for (;;) { + for (; count ; count--) { struct dentry *dentry; struct list_head *tmp; tmp = dentry_unused.prev; - if (tmp == &dentry_unused) break; list_del_init(tmp); @@ -349,12 +347,8 @@ dentry_stat.nr_unused--; /* Unused dentry with a count? */ - if (atomic_read(&dentry->d_count)) - BUG(); - + BUG_ON(atomic_read(&dentry->d_count)); prune_one_dentry(dentry); - if (!--count) - break; } spin_unlock(&dcache_lock); } @@ -573,19 +567,10 @@ /* * This is called from kswapd when we think we need some - * more memory, but aren't really sure how much. So we - * carefully try to free a _bit_ of our dcache, but not - * too much. - * - * Priority: - * 1 - very urgent: shrink everything - * ... - * 6 - base-level: try to shrink a bit. + * more memory. */ -int shrink_dcache_memory(int priority, unsigned int gfp_mask) +int age_dcache_memory(kmem_cache_t *cachep, int entries, int gfp_mask) { - int count = 0; - /* * Nasty deadlock avoidance. * @@ -600,10 +585,11 @@ if (!(gfp_mask & __GFP_FS)) return 0; - count = dentry_stat.nr_unused / priority; + if (entries > dentry_stat.nr_unused) + entries = dentry_stat.nr_unused; - prune_dcache(count); - return kmem_cache_shrink(dentry_cache); + prune_dcache(entries); + return entries; } #define NAME_ALLOC_LEN(len) ((len+16) & ~15) @@ -685,7 +671,7 @@ void d_instantiate(struct dentry *entry, struct inode * inode) { - if (!list_empty(&entry->d_alias)) BUG(); + BUG_ON(!list_empty(&entry->d_alias)); spin_lock(&dcache_lock); if (inode) list_add(&entry->d_alias, &inode->i_dentry); @@ -984,7 +970,7 @@ void d_rehash(struct dentry * entry) { struct list_head *list = d_hash(entry->d_parent, entry->d_name.hash); - if (!list_empty(&entry->d_hash)) BUG(); + BUG_ON(!list_empty(&entry->d_hash)); spin_lock(&dcache_lock); list_add(&entry->d_hash, list); spin_unlock(&dcache_lock); @@ -1344,6 +1330,8 @@ if (!dentry_cache) panic("Cannot create dentry cache"); + kmem_set_pruner(dentry_cache, (kmem_pruner_t)age_dcache_memory); + #if PAGE_SHIFT < 13 mempages >>= (13 - PAGE_SHIFT); #endif @@ -1416,6 +1404,9 @@ SLAB_HWCACHE_ALIGN, NULL, NULL); if (!dquot_cachep) panic("Cannot create dquot SLAB cache"); + + kmem_set_pruner(dquot_cachep, (kmem_pruner_t)age_dqcache_memory); + #endif dcache_init(mempages); diff -uNr linux-2.5.27-rmap/fs/dquot.c linux-2.5.27-rmap-slablru/fs/dquot.c --- linux-2.5.27-rmap/fs/dquot.c Sat Jul 20 14:19:42 2002 +++ linux-2.5.27-rmap-slablru/fs/dquot.c Sat Jul 20 15:10:22 2002 @@ -492,13 +492,13 @@ int shrink_dqcache_memory(int priority, unsigned int gfp_mask) { - int count = 0; + if (entries > nr_free_dquots) + entries = nr_free_dquots; lock_kernel(); - count = dqstats.free_dquots / priority; - prune_dqcache(count); + prune_dqcache(entries); unlock_kernel(); - return kmem_cache_shrink(dquot_cachep); + return entries; } /* diff -uNr linux-2.5.27-rmap/fs/efs/super.c linux-2.5.27-rmap-slablru/fs/efs/super.c --- linux-2.5.27-rmap/fs/efs/super.c Sat Jul 20 14:18:17 2002 +++ linux-2.5.27-rmap-slablru/fs/efs/super.c Sat Jul 20 15:10:22 2002 @@ -61,6 +61,7 @@ init_once, NULL); if (efs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(efs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/ext2/super.c linux-2.5.27-rmap-slablru/fs/ext2/super.c --- linux-2.5.27-rmap/fs/ext2/super.c Sat Jul 20 14:18:14 2002 +++ linux-2.5.27-rmap-slablru/fs/ext2/super.c Sat Jul 20 15:10:22 2002 @@ -184,6 +184,7 @@ init_once, NULL); if (ext2_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(ext2_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/ext3/super.c linux-2.5.27-rmap-slablru/fs/ext3/super.c --- linux-2.5.27-rmap/fs/ext3/super.c Sat Jul 20 14:18:21 2002 +++ linux-2.5.27-rmap-slablru/fs/ext3/super.c Sat Jul 20 15:10:22 2002 @@ -479,6 +479,7 @@ init_once, NULL); if (ext3_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(ext3_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/fat/inode.c linux-2.5.27-rmap-slablru/fs/fat/inode.c --- linux-2.5.27-rmap/fs/fat/inode.c Sat Jul 20 14:23:07 2002 +++ linux-2.5.27-rmap-slablru/fs/fat/inode.c Sat Jul 20 15:10:22 2002 @@ -600,6 +600,7 @@ init_once, NULL); if (fat_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(fat_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/freevxfs/vxfs_super.c linux-2.5.27-rmap-slablru/fs/freevxfs/vxfs_super.c --- linux-2.5.27-rmap/fs/freevxfs/vxfs_super.c Sat Jul 20 14:18:17 2002 +++ linux-2.5.27-rmap-slablru/fs/freevxfs/vxfs_super.c Sat Jul 20 15:10:22 2002 @@ -247,8 +247,10 @@ { vxfs_inode_cachep = kmem_cache_create("vxfs_inode", sizeof(struct vxfs_inode_info), 0, 0, NULL, NULL); - if (vxfs_inode_cachep) + if (vxfs_inode_cachep) { + kmem_set_pruner(vxfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return (register_filesystem(&vxfs_fs_type)); + } return -ENOMEM; } diff -uNr linux-2.5.27-rmap/fs/hfs/super.c linux-2.5.27-rmap-slablru/fs/hfs/super.c --- linux-2.5.27-rmap/fs/hfs/super.c Sat Jul 20 14:18:07 2002 +++ linux-2.5.27-rmap-slablru/fs/hfs/super.c Sat Jul 20 15:10:22 2002 @@ -75,6 +75,7 @@ init_once, NULL); if (hfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(hfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/hpfs/super.c linux-2.5.27-rmap-slablru/fs/hpfs/super.c --- linux-2.5.27-rmap/fs/hpfs/super.c Sat Jul 20 14:18:16 2002 +++ linux-2.5.27-rmap-slablru/fs/hpfs/super.c Sat Jul 20 15:10:22 2002 @@ -189,6 +189,7 @@ init_once, NULL); if (hpfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(hpfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/inode.c linux-2.5.27-rmap-slablru/fs/inode.c --- linux-2.5.27-rmap/fs/inode.c Sat Jul 20 14:19:42 2002 +++ linux-2.5.27-rmap-slablru/fs/inode.c Sat Jul 20 15:10:22 2002 @@ -377,10 +377,11 @@ count = 0; entry = inode_unused.prev; - while (entry != &inode_unused) - { + for(; goal; goal--) { struct list_head *tmp = entry; + if (entry == &inode_unused) + break; entry = entry->prev; inode = INODE(tmp); if (inode->i_state & (I_FREEING|I_CLEAR|I_LOCK)) @@ -394,8 +395,6 @@ list_add(tmp, freeable); inode->i_state |= I_FREEING; count++; - if (!--goal) - break; } inodes_stat.nr_unused -= count; spin_unlock(&inode_lock); @@ -405,19 +404,10 @@ /* * This is called from kswapd when we think we need some - * more memory, but aren't really sure how much. So we - * carefully try to free a _bit_ of our icache, but not - * too much. - * - * Priority: - * 1 - very urgent: shrink everything - * ... - * 6 - base-level: try to shrink a bit. + * more memory. */ -int shrink_icache_memory(int priority, int gfp_mask) +int age_icache_memory(kmem_cache_t *cachep, int entries, int gfp_mask) { - int count = 0; - /* * Nasty deadlock avoidance.. * @@ -428,11 +418,13 @@ if (!(gfp_mask & __GFP_FS)) return 0; - count = inodes_stat.nr_unused / priority; + if (entries > inodes_stat.nr_unused) + entries = inodes_stat.nr_unused; - prune_icache(count); - return kmem_cache_shrink(inode_cachep); + prune_icache(entries); + return entries; } +EXPORT_SYMBOL(age_icache_memory); /* * Called with the inode lock held. @@ -1098,4 +1090,6 @@ NULL); if (!inode_cachep) panic("cannot create inode slab cache"); + + kmem_set_pruner(inode_cachep, (kmem_pruner_t)age_icache_memory); } diff -uNr linux-2.5.27-rmap/fs/isofs/inode.c linux-2.5.27-rmap-slablru/fs/isofs/inode.c --- linux-2.5.27-rmap/fs/isofs/inode.c Sat Jul 20 14:18:11 2002 +++ linux-2.5.27-rmap-slablru/fs/isofs/inode.c Sat Jul 20 15:10:22 2002 @@ -114,6 +114,7 @@ init_once, NULL); if (isofs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(isofs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/jffs2/super.c linux-2.5.27-rmap-slablru/fs/jffs2/super.c --- linux-2.5.27-rmap/fs/jffs2/super.c Sat Jul 20 14:18:11 2002 +++ linux-2.5.27-rmap-slablru/fs/jffs2/super.c Sat Jul 20 15:10:22 2002 @@ -323,6 +323,7 @@ printk(KERN_ERR "JFFS2 error: Failed to initialise inode cache\n"); return -ENOMEM; } + kmem_set_pruner(jffs2_inode_cachep, (kmem_pruner_t)age_icache_memory); ret = jffs2_zlib_init(); if (ret) { printk(KERN_ERR "JFFS2 error: Failed to initialise zlib workspaces\n"); diff -uNr linux-2.5.27-rmap/fs/jfs/super.c linux-2.5.27-rmap-slablru/fs/jfs/super.c --- linux-2.5.27-rmap/fs/jfs/super.c Sat Jul 20 14:18:10 2002 +++ linux-2.5.27-rmap-slablru/fs/jfs/super.c Sat Jul 20 15:10:22 2002 @@ -398,6 +398,7 @@ 0, 0, init_once, NULL); if (jfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(jfs_inode_cachep, (kmem_pruner_t)age_icache_memory); /* * Metapage initialization diff -uNr linux-2.5.27-rmap/fs/minix/inode.c linux-2.5.27-rmap-slablru/fs/minix/inode.c --- linux-2.5.27-rmap/fs/minix/inode.c Sat Jul 20 14:18:22 2002 +++ linux-2.5.27-rmap-slablru/fs/minix/inode.c Sat Jul 20 15:10:22 2002 @@ -82,6 +82,7 @@ init_once, NULL); if (minix_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(minix_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/ncpfs/inode.c linux-2.5.27-rmap-slablru/fs/ncpfs/inode.c --- linux-2.5.27-rmap/fs/ncpfs/inode.c Sat Jul 20 14:18:16 2002 +++ linux-2.5.27-rmap-slablru/fs/ncpfs/inode.c Sat Jul 20 15:10:22 2002 @@ -71,6 +71,7 @@ init_once, NULL); if (ncp_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(ncp_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/nfs/inode.c linux-2.5.27-rmap-slablru/fs/nfs/inode.c --- linux-2.5.27-rmap/fs/nfs/inode.c Sat Jul 20 14:18:07 2002 +++ linux-2.5.27-rmap-slablru/fs/nfs/inode.c Sat Jul 20 15:10:22 2002 @@ -1287,6 +1287,7 @@ if (nfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(nfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/ntfs/super.c linux-2.5.27-rmap-slablru/fs/ntfs/super.c --- linux-2.5.27-rmap/fs/ntfs/super.c Sat Jul 20 14:18:11 2002 +++ linux-2.5.27-rmap-slablru/fs/ntfs/super.c Sat Jul 20 15:10:22 2002 @@ -1677,6 +1677,7 @@ ntfs_inode_cache_name); goto inode_err_out; } + kmem_set_pruner(ntfs_inode_cache, (kmem_pruner_t)age_icache_memory); ntfs_big_inode_cache = kmem_cache_create(ntfs_big_inode_cache_name, sizeof(big_ntfs_inode), 0, SLAB_HWCACHE_ALIGN, @@ -1686,6 +1687,7 @@ ntfs_big_inode_cache_name); goto big_inode_err_out; } + kmem_set_pruner(ntfs_big_inode_cache, (kmem_pruner_t)age_icache_memory); /* Register the ntfs sysctls. */ err = ntfs_sysctl(1); diff -uNr linux-2.5.27-rmap/fs/proc/inode.c linux-2.5.27-rmap-slablru/fs/proc/inode.c --- linux-2.5.27-rmap/fs/proc/inode.c Sat Jul 20 14:18:14 2002 +++ linux-2.5.27-rmap-slablru/fs/proc/inode.c Sat Jul 20 15:10:22 2002 @@ -114,6 +114,7 @@ init_once, NULL); if (proc_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(proc_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/qnx4/inode.c linux-2.5.27-rmap-slablru/fs/qnx4/inode.c --- linux-2.5.27-rmap/fs/qnx4/inode.c Sat Jul 20 14:18:23 2002 +++ linux-2.5.27-rmap-slablru/fs/qnx4/inode.c Sat Jul 20 15:10:22 2002 @@ -547,6 +547,7 @@ init_once, NULL); if (qnx4_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(qnx4_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/reiserfs/super.c linux-2.5.27-rmap-slablru/fs/reiserfs/super.c --- linux-2.5.27-rmap/fs/reiserfs/super.c Sat Jul 20 14:18:15 2002 +++ linux-2.5.27-rmap-slablru/fs/reiserfs/super.c Sat Jul 20 15:10:22 2002 @@ -437,6 +437,7 @@ init_once, NULL); if (reiserfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(reiserfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/romfs/inode.c linux-2.5.27-rmap-slablru/fs/romfs/inode.c --- linux-2.5.27-rmap/fs/romfs/inode.c Sat Jul 20 14:18:22 2002 +++ linux-2.5.27-rmap-slablru/fs/romfs/inode.c Sat Jul 20 15:10:22 2002 @@ -575,6 +575,7 @@ init_once, NULL); if (romfs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(romfs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/smbfs/inode.c linux-2.5.27-rmap-slablru/fs/smbfs/inode.c --- linux-2.5.27-rmap/fs/smbfs/inode.c Sat Jul 20 14:18:20 2002 +++ linux-2.5.27-rmap-slablru/fs/smbfs/inode.c Sat Jul 20 15:10:22 2002 @@ -81,6 +81,7 @@ init_once, NULL); if (smb_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(smb_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/sysv/inode.c linux-2.5.27-rmap-slablru/fs/sysv/inode.c --- linux-2.5.27-rmap/fs/sysv/inode.c Sat Jul 20 14:18:16 2002 +++ linux-2.5.27-rmap-slablru/fs/sysv/inode.c Sat Jul 20 15:10:22 2002 @@ -328,6 +328,7 @@ SLAB_HWCACHE_ALIGN, init_once, NULL); if (!sysv_inode_cachep) return -ENOMEM; + kmem_set_pruner(sysv_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/udf/super.c linux-2.5.27-rmap-slablru/fs/udf/super.c --- linux-2.5.27-rmap/fs/udf/super.c Sat Jul 20 14:18:15 2002 +++ linux-2.5.27-rmap-slablru/fs/udf/super.c Sat Jul 20 15:10:22 2002 @@ -144,6 +144,7 @@ init_once, NULL); if (udf_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(udf_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/fs/ufs/super.c linux-2.5.27-rmap-slablru/fs/ufs/super.c --- linux-2.5.27-rmap/fs/ufs/super.c Sat Jul 20 14:18:10 2002 +++ linux-2.5.27-rmap-slablru/fs/ufs/super.c Sat Jul 20 15:10:22 2002 @@ -1013,6 +1013,7 @@ init_once, NULL); if (ufs_inode_cachep == NULL) return -ENOMEM; + kmem_set_pruner(ufs_inode_cachep, (kmem_pruner_t)age_icache_memory); return 0; } diff -uNr linux-2.5.27-rmap/include/linux/dcache.h linux-2.5.27-rmap-slablru/include/linux/dcache.h --- linux-2.5.27-rmap/include/linux/dcache.h Sat Jul 20 14:15:52 2002 +++ linux-2.5.27-rmap-slablru/include/linux/dcache.h Sat Jul 20 15:10:22 2002 @@ -184,16 +184,11 @@ #define shrink_dcache() prune_dcache(0) struct zone_struct; /* dcache memory management */ -extern int shrink_dcache_memory(int, unsigned int); extern void prune_dcache(int); /* icache memory management (defined in linux/fs/inode.c) */ -extern int shrink_icache_memory(int, int); extern void prune_icache(int); -/* quota cache memory management (defined in linux/fs/dquot.c) */ -extern int shrink_dqcache_memory(int, unsigned int); - /* only used at mount-time */ extern struct dentry * d_alloc_root(struct inode *); diff -uNr linux-2.5.27-rmap/include/linux/slab.h linux-2.5.27-rmap-slablru/include/linux/slab.h --- linux-2.5.27-rmap/include/linux/slab.h Sat Jul 20 14:15:51 2002 +++ linux-2.5.27-rmap-slablru/include/linux/slab.h Sat Jul 20 15:10:22 2002 @@ -55,6 +55,26 @@ void (*)(void *, kmem_cache_t *, unsigned long)); extern int kmem_cache_destroy(kmem_cache_t *); extern int kmem_cache_shrink(kmem_cache_t *); + +typedef int (*kmem_pruner_t)(kmem_cache_t *, int, int); + +extern void kmem_set_pruner(kmem_cache_t *, kmem_pruner_t); +extern int kmem_do_prunes(int); +extern int kmem_count_page(struct page *, int); +#define kmem_touch_page(addr) SetPageReferenced(virt_to_page(addr)); + +/* shrink a slab */ +extern int kmem_shrink_slab(struct page *); + +/* dcache prune ( defined in linux/fs/dcache.c) */ +extern int age_dcache_memory(kmem_cache_t *, int, int); + +/* icache prune (defined in linux/fs/inode.c) */ +extern int age_icache_memory(kmem_cache_t *, int, int); + +/* quota cache prune (defined in linux/fs/dquot.c) */ +extern int age_dqcache_memory(kmem_cache_t *, int, int); + extern void *kmem_cache_alloc(kmem_cache_t *, int); extern void kmem_cache_free(kmem_cache_t *, void *); diff -uNr linux-2.5.27-rmap/kernel/ksyms.c linux-2.5.27-rmap-slablru/kernel/ksyms.c --- linux-2.5.27-rmap/kernel/ksyms.c Sat Jul 20 14:18:25 2002 +++ linux-2.5.27-rmap-slablru/kernel/ksyms.c Sat Jul 20 15:10:22 2002 @@ -100,6 +100,7 @@ EXPORT_SYMBOL(free_pages); EXPORT_SYMBOL(num_physpages); EXPORT_SYMBOL(kmem_find_general_cachep); +EXPORT_SYMBOL(kmem_set_pruner); EXPORT_SYMBOL(kmem_cache_create); EXPORT_SYMBOL(kmem_cache_destroy); EXPORT_SYMBOL(kmem_cache_shrink); diff -uNr linux-2.5.27-rmap/mm/slab.c linux-2.5.27-rmap-slablru/mm/slab.c --- linux-2.5.27-rmap/mm/slab.c Sat Jul 20 14:18:25 2002 +++ linux-2.5.27-rmap-slablru/mm/slab.c Sat Jul 20 15:10:23 2002 @@ -77,6 +77,7 @@ #include #include #include +#include #include /* @@ -215,6 +216,8 @@ kmem_cache_t *slabp_cache; unsigned int growing; unsigned int dflags; /* dynamic flags */ + kmem_pruner_t pruner; /* shrink callback */ + int count; /* count used to trigger shrink */ /* constructor func */ void (*ctor)(void *, kmem_cache_t *, unsigned long); @@ -253,10 +256,12 @@ /* c_dflags (dynamic flags). Need to hold the spinlock to access this member */ #define DFLGS_GROWN 0x000001UL /* don't reap a recently grown */ +#define DFLGS_NONLRU 0x000002UL /* there are reciently allocated + non lru pages in this cache */ #define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB) #define OPTIMIZE(x) ((x)->flags & CFLGS_OPTIMIZE) -#define GROWN(x) ((x)->dlags & DFLGS_GROWN) +#define GROWN(x) ((x)->dflags & DFLGS_GROWN) #if STATS #define STATS_INC_ACTIVE(x) ((x)->num_active++) @@ -412,6 +417,63 @@ static void enable_cpucache (kmem_cache_t *cachep); static void enable_all_cpucaches (void); #endif + +/* + * Note: For prunable caches object size must less than page size. + */ +void kmem_set_pruner(kmem_cache_t * cachep, kmem_pruner_t thepruner) +{ + BUG_ON(cachep->objsize > PAGE_SIZE); + cachep->pruner = thepruner; +} + +/* + * Used by refill_inactive_zone to determine caches that need pruning. + */ +int kmem_count_page(struct page *page, int cold) +{ + kmem_cache_t *cachep = GET_PAGE_CACHE(page); + slab_t *slabp = GET_PAGE_SLAB(page); + int ret = 0; + spin_lock(&cachep->spinlock); + if (cachep->pruner != NULL) { + cachep->count += slabp->inuse >> cachep->gfporder; + ret = !slabp->inuse; + } else + ret = cold && !slabp->inuse; + spin_unlock(&cachep->spinlock); + return ret; +} + + +/* Call the prune functions to age pruneable caches */ +int kmem_do_prunes(int gfp_mask) +{ + struct list_head *p; + int nr; + + if (gfp_mask & __GFP_WAIT) + down(&cache_chain_sem); + else + if (down_trylock(&cache_chain_sem)) + return 0; + + list_for_each(p,&cache_chain) { + kmem_cache_t *cachep = list_entry(p, kmem_cache_t, next); + if (cachep->pruner != NULL) { + spin_lock(&cachep->spinlock); + nr = cachep->count; + cachep->count = 0; + spin_unlock(&cachep->spinlock); + if (nr > 0) + (*cachep->pruner)(cachep, nr, gfp_mask); + + } + } + up(&cache_chain_sem); + return 1; +} + /* Cal the num objs, wastage, and bytes left over for a given slab size. */ static void kmem_cache_estimate (unsigned long gfporder, size_t size, @@ -451,8 +513,7 @@ kmem_cache_estimate(0, cache_cache.objsize, 0, &left_over, &cache_cache.num); - if (!cache_cache.num) - BUG(); + BUG_ON(!cache_cache.num); cache_cache.colour = left_over/cache_cache.colour_off; cache_cache.colour_next = 0; @@ -477,12 +538,10 @@ * eliminates "false sharing". * Note for systems short on memory removing the alignment will * allow tighter packing of the smaller caches. */ - if (!(sizes->cs_cachep = + BUG_ON(!(sizes->cs_cachep = kmem_cache_create(cache_names[sizes-cache_sizes].name, - sizes->cs_size, - 0, SLAB_HWCACHE_ALIGN, NULL, NULL))) { - BUG(); - } + sizes->cs_size, + 0, SLAB_HWCACHE_ALIGN, NULL, NULL))); /* Inc off-slab bufctl limit until the ceiling is hit. */ if (!(OFF_SLAB(sizes->cs_cachep))) { @@ -490,11 +549,10 @@ offslab_limit /= 2; } sizes->cs_dmacachep = kmem_cache_create( - cache_names[sizes-cache_sizes].name_dma, + cache_names[sizes-cache_sizes].name_dma, sizes->cs_size, 0, SLAB_CACHE_DMA|SLAB_HWCACHE_ALIGN, NULL, NULL); - if (!sizes->cs_dmacachep) - BUG(); + BUG_ON(!sizes->cs_dmacachep); sizes++; } while (sizes->cs_size); } @@ -510,7 +568,9 @@ __initcall(kmem_cpucache_init); -/* Interface to system's page allocator. No need to hold the cache-lock. +/* + * Interface to system's page allocator. No need to hold the cache-lock. + * Call with pagemap_lru_lock held */ static inline void * kmem_getpages (kmem_cache_t *cachep, unsigned long flags) { @@ -532,7 +592,8 @@ return addr; } -/* Interface to system's page release. */ +/* Interface to system's page release. + * Normally called with pagemap_lru_lock held */ static inline void kmem_freepages (kmem_cache_t *cachep, void *addr) { unsigned long i = (1<gfporder); @@ -545,6 +606,9 @@ */ while (i--) { ClearPageSlab(page); + if (PageActive(page)) + del_page_from_active_list(page); + ClearPageReferenced(page); page++; } free_pages((unsigned long)addr, cachep->gfporder); @@ -577,9 +641,11 @@ } #endif + /* Destroy all the objs in a slab, and release the mem back to the system. * Before calling the slab must have been unlinked from the cache. * The cache-lock is not held/needed. + * pagemap_lru_lock should be held for kmem_freepages */ static void kmem_slab_destroy (kmem_cache_t *cachep, slab_t *slabp) { @@ -593,11 +659,9 @@ void* objp = slabp->s_mem+cachep->objsize*i; #if DEBUG if (cachep->flags & SLAB_RED_ZONE) { - if (*((unsigned long*)(objp)) != RED_MAGIC1) - BUG(); - if (*((unsigned long*)(objp + cachep->objsize - -BYTES_PER_WORD)) != RED_MAGIC1) - BUG(); + BUG_ON(*((unsigned long*)(objp)) != RED_MAGIC1); + BUG_ON(*((unsigned long*)(objp + cachep->objsize + -BYTES_PER_WORD)) != RED_MAGIC1); objp += BYTES_PER_WORD; } #endif @@ -607,9 +671,8 @@ if (cachep->flags & SLAB_RED_ZONE) { objp -= BYTES_PER_WORD; } - if ((cachep->flags & SLAB_POISON) && - kmem_check_poison_obj(cachep, objp)) - BUG(); + BUG_ON((cachep->flags & SLAB_POISON) && + kmem_check_poison_obj(cachep, objp)); #endif } } @@ -664,13 +727,12 @@ /* * Sanity checks... these are all serious usage bugs. */ - if ((!name) || + BUG_ON((!name) || in_interrupt() || (size < BYTES_PER_WORD) || (size > (1< size)) - BUG(); + (offset < 0 || offset > size)); #if DEBUG if ((flags & SLAB_DEBUG_INITIAL) && !ctor) { @@ -700,8 +762,7 @@ * Always checks flags, a caller might be expecting debug * support which isn't available. */ - if (flags & ~CREATE_MASK) - BUG(); + BUG_ON(flags & ~CREATE_MASK); /* Get cache's description obj. */ cachep = (kmem_cache_t *) kmem_cache_alloc(&cache_cache, SLAB_KERNEL); @@ -816,6 +877,8 @@ flags |= CFLGS_OPTIMIZE; cachep->flags = flags; + cachep->pruner = NULL; + cachep->count = 0; cachep->gfpflags = 0; if (flags & SLAB_CACHE_DMA) cachep->gfpflags |= GFP_DMA; @@ -909,8 +972,7 @@ func(arg); local_irq_enable(); - if (smp_call_function(func, arg, 1, 1)) - BUG(); + BUG_ON(smp_call_function(func, arg, 1, 1)); } typedef struct ccupdate_struct_s { @@ -958,15 +1020,15 @@ #define drain_cpu_caches(cachep) do { } while (0) #endif -static int __kmem_cache_shrink(kmem_cache_t *cachep) + +/* + * Worker function for freeing slab caches; returns number of pages freed. + * Caller must hold pagemap_lru_lock + */ +static int __kmem_cache_shrink_locked(kmem_cache_t *cachep) { slab_t *slabp; - int ret; - - drain_cpu_caches(cachep); - - spin_lock_irq(&cachep->spinlock); - + int ret = 0; /* If the cache is growing, stop shrinking. */ while (!cachep->growing) { struct list_head *p; @@ -977,17 +1039,32 @@ slabp = list_entry(cachep->slabs_free.prev, slab_t, list); #if DEBUG - if (slabp->inuse) - BUG(); + BUG_ON(slabp->inuse); #endif list_del(&slabp->list); spin_unlock_irq(&cachep->spinlock); kmem_slab_destroy(cachep, slabp); + ret++; spin_lock_irq(&cachep->spinlock); } - ret = !list_empty(&cachep->slabs_full) || !list_empty(&cachep->slabs_partial); + return ret; +} + + +static int __kmem_cache_shrink(kmem_cache_t *cachep) +{ + int ret; + + drain_cpu_caches(cachep); + + spin_lock(&pagemap_lru_lock); + spin_lock_irq(&cachep->spinlock); + __kmem_cache_shrink_locked(cachep); + ret = !list_empty(&cachep->slabs_full) || + !list_empty(&cachep->slabs_partial); spin_unlock_irq(&cachep->spinlock); + spin_unlock(&pagemap_lru_lock); return ret; } @@ -1000,12 +1077,46 @@ */ int kmem_cache_shrink(kmem_cache_t *cachep) { - if (!cachep || in_interrupt() || !is_chained_kmem_cache(cachep)) - BUG(); - + BUG_ON(!cachep || in_interrupt() || !is_chained_kmem_cache(cachep)); return __kmem_cache_shrink(cachep); } + +/* + * Used by refill_inactive_zone to try to shrink a cache. The + * method we use to shrink depends on if we have added nonlru + * pages since the last time we shrunk this cache. + * - shrink works and we return the pages shrunk + * - shrink fails because the slab is in use, we return 0 + * called with pagemap_lru_lock held. + */ +int kmem_shrink_slab(struct page *page) +{ + kmem_cache_t *cachep = GET_PAGE_CACHE(page); + slab_t *slabp = GET_PAGE_SLAB(page); + + spin_lock_irq(&cachep->spinlock); + if (!slabp->inuse) { + if (!cachep->growing) { + if (cachep->dflags & DFLGS_NONLRU) { + int nr = __kmem_cache_shrink_locked(cachep); + cachep->dflags &= ~DFLGS_NONLRU; + spin_unlock_irq(&cachep->spinlock); + return nr<gfporder; + } else { + list_del(&slabp->list); + spin_unlock_irq(&cachep->spinlock); + kmem_slab_destroy(cachep, slabp); + return 1<gfporder; + } + BUG_ON(PageActive(page)); + } + } + spin_unlock_irq(&cachep->spinlock); + return 0; +} + + /** * kmem_cache_destroy - delete a cache * @cachep: the cache to destroy @@ -1023,8 +1134,7 @@ */ int kmem_cache_destroy (kmem_cache_t * cachep) { - if (!cachep || in_interrupt() || cachep->growing) - BUG(); + BUG_ON(!cachep || in_interrupt() || cachep->growing); /* Find the cache in the chain of caches. */ down(&cache_chain_sem); @@ -1112,11 +1222,9 @@ /* need to poison the objs */ kmem_poison_obj(cachep, objp); if (cachep->flags & SLAB_RED_ZONE) { - if (*((unsigned long*)(objp)) != RED_MAGIC1) - BUG(); - if (*((unsigned long*)(objp + cachep->objsize - - BYTES_PER_WORD)) != RED_MAGIC1) - BUG(); + BUG_ON(*((unsigned long*)(objp)) != RED_MAGIC1); + BUG_ON(*((unsigned long*)(objp + cachep->objsize - + BYTES_PER_WORD)) != RED_MAGIC1); } #endif slab_bufctl(slabp)[i] = i+1; @@ -1135,15 +1243,14 @@ struct page *page; void *objp; size_t offset; - unsigned int i, local_flags; + unsigned int i, local_flags, locked = 0; unsigned long ctor_flags; unsigned long save_flags; /* Be lazy and only check for valid flags here, * keeping it out of the critical path in kmem_cache_alloc(). */ - if (flags & ~(SLAB_DMA|SLAB_LEVEL_MASK|SLAB_NO_GROW)) - BUG(); + BUG_ON(flags & ~(SLAB_DMA|SLAB_LEVEL_MASK|SLAB_NO_GROW)); if (flags & SLAB_NO_GROW) return 0; @@ -1153,8 +1260,7 @@ * in kmem_cache_alloc(). If a caller is seriously mis-behaving they * will eventually be caught here (where it matters). */ - if (in_interrupt() && (flags & __GFP_WAIT)) - BUG(); + BUG_ON(in_interrupt() && (flags & __GFP_WAIT)); ctor_flags = SLAB_CTOR_CONSTRUCTOR; local_flags = (flags & SLAB_LEVEL_MASK); @@ -1192,6 +1298,21 @@ if (!(objp = kmem_getpages(cachep, flags))) goto failed; + /* + * We want the pagemap_lru_lock, in UP spin locks to not + * protect us in interrupt context... In SMP they do but, + * optimizating for speed, we process if we do not get it. + */ + if (!(cachep->flags & SLAB_NO_REAP)) { +#ifdef CONFIG_SMP + locked = spin_trylock(&pagemap_lru_lock); +#else + locked = !in_interrupt() && spin_trylock(&pagemap_lru_lock); +#endif + if (!locked && !in_interrupt()) + goto opps1; + } + /* Get slab management. */ if (!(slabp = kmem_cache_slabmgmt(cachep, objp, offset, local_flags))) goto opps1; @@ -1203,9 +1324,15 @@ SET_PAGE_CACHE(page, cachep); SET_PAGE_SLAB(page, slabp); SetPageSlab(page); + set_page_count(page, 1); + if (locked) + add_page_to_active_list(page); page++; } while (--i); + if (locked) + spin_unlock(&pagemap_lru_lock); + kmem_cache_init_objs(cachep, slabp, ctor_flags); spin_lock_irqsave(&cachep->spinlock, save_flags); @@ -1216,10 +1343,15 @@ STATS_INC_GROWN(cachep); cachep->failures = 0; + /* The pagemap_lru_lock was not quickly/safely available */ + if (!locked && !(cachep->flags & SLAB_NO_REAP)) + cachep->dflags |= DFLGS_NONLRU; + spin_unlock_irqrestore(&cachep->spinlock, save_flags); return 1; opps1: - kmem_freepages(cachep, objp); + /* do not use kmem_freepages - we are not in the lru yet... */ + free_pages((unsigned long)objp, cachep->gfporder); failed: spin_lock_irqsave(&cachep->spinlock, save_flags); cachep->growing--; @@ -1241,15 +1373,12 @@ int i; unsigned int objnr = (objp-slabp->s_mem)/cachep->objsize; - if (objnr >= cachep->num) - BUG(); - if (objp != slabp->s_mem + objnr*cachep->objsize) - BUG(); + BUG_ON(objnr >= cachep->num); + BUG_ON(objp != slabp->s_mem + objnr*cachep->objsize); /* Check slab's freelist to see if this obj is there. */ for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) { - if (i == objnr) - BUG(); + BUG_ON(i == objnr); } return 0; } @@ -1258,11 +1387,9 @@ static inline void kmem_cache_alloc_head(kmem_cache_t *cachep, int flags) { if (flags & SLAB_DMA) { - if (!(cachep->gfpflags & GFP_DMA)) - BUG(); + BUG_ON(!(cachep->gfpflags & GFP_DMA)); } else { - if (cachep->gfpflags & GFP_DMA) - BUG(); + BUG_ON(cachep->gfpflags & GFP_DMA); } } @@ -1284,18 +1411,16 @@ list_del(&slabp->list); list_add(&slabp->list, &cachep->slabs_full); } + kmem_touch_page(objp); #if DEBUG if (cachep->flags & SLAB_POISON) - if (kmem_check_poison_obj(cachep, objp)) - BUG(); + BUG_ON(kmem_check_poison_obj(cachep, objp)); if (cachep->flags & SLAB_RED_ZONE) { /* Set alloc red-zone, and check old one. */ - if (xchg((unsigned long *)objp, RED_MAGIC2) != - RED_MAGIC1) - BUG(); - if (xchg((unsigned long *)(objp+cachep->objsize - - BYTES_PER_WORD), RED_MAGIC2) != RED_MAGIC1) - BUG(); + BUG_ON(xchg((unsigned long *)objp, RED_MAGIC2) != + RED_MAGIC1); + BUG_ON(xchg((unsigned long *)(objp+cachep->objsize - + BYTES_PER_WORD), RED_MAGIC2) != RED_MAGIC1); objp += BYTES_PER_WORD; } #endif @@ -1463,13 +1588,11 @@ if (cachep->flags & SLAB_RED_ZONE) { objp -= BYTES_PER_WORD; - if (xchg((unsigned long *)objp, RED_MAGIC1) != RED_MAGIC2) - /* Either write before start, or a double free. */ - BUG(); - if (xchg((unsigned long *)(objp+cachep->objsize - - BYTES_PER_WORD), RED_MAGIC1) != RED_MAGIC2) - /* Either write past end, or a double free. */ - BUG(); + BUG_ON(xchg((unsigned long *)objp, RED_MAGIC1) != RED_MAGIC2); + /* Either write before start, or a double free. */ + BUG_ON(xchg((unsigned long *)(objp+cachep->objsize - + BYTES_PER_WORD), RED_MAGIC1) != RED_MAGIC2); + /* Either write past end, or a double free. */ } if (cachep->flags & SLAB_POISON) kmem_poison_obj(cachep, objp); @@ -1607,8 +1730,7 @@ unsigned long flags; #if DEBUG CHECK_PAGE(objp); - if (cachep != GET_PAGE_CACHE(virt_to_page(objp))) - BUG(); + BUG_ON(cachep != GET_PAGE_CACHE(virt_to_page(objp))); #endif local_irq_save(flags); @@ -1804,8 +1926,7 @@ while (p != &searchp->slabs_free) { slabp = list_entry(p, slab_t, list); #if DEBUG - if (slabp->inuse) - BUG(); + BUG_ON(slabp->inuse); #endif full_free++; p = p->next; @@ -1845,6 +1966,7 @@ spin_lock_irq(&best_cachep->spinlock); perfect: + spin_lock(&pagemap_lru_lock); /* free only 50% of the free slabs */ best_len = (best_len + 1)/2; for (scan = 0; scan < best_len; scan++) { @@ -1857,8 +1979,7 @@ break; slabp = list_entry(p,slab_t,list); #if DEBUG - if (slabp->inuse) - BUG(); + BUG_ON(slabp->inuse); #endif list_del(&slabp->list); STATS_INC_REAPED(best_cachep); @@ -1870,6 +1991,7 @@ kmem_slab_destroy(best_cachep, slabp); spin_lock_irq(&best_cachep->spinlock); } + spin_unlock(&pagemap_lru_lock); spin_unlock_irq(&best_cachep->spinlock); ret = scan * (1 << best_cachep->gfporder); out: @@ -1943,22 +2065,19 @@ num_slabs = 0; list_for_each(q,&cachep->slabs_full) { slabp = list_entry(q, slab_t, list); - if (slabp->inuse != cachep->num) - BUG(); + BUG_ON(slabp->inuse != cachep->num); active_objs += cachep->num; active_slabs++; } list_for_each(q,&cachep->slabs_partial) { slabp = list_entry(q, slab_t, list); - if (slabp->inuse == cachep->num || !slabp->inuse) - BUG(); + BUG_ON(slabp->inuse == cachep->num || !slabp->inuse); active_objs += slabp->inuse; active_slabs++; } list_for_each(q,&cachep->slabs_free) { slabp = list_entry(q, slab_t, list); - if (slabp->inuse) - BUG(); + BUG_ON(slabp->inuse); num_slabs++; } num_slabs+=active_slabs; diff -uNr linux-2.5.27-rmap/mm/vmscan.c linux-2.5.27-rmap-slablru/mm/vmscan.c --- linux-2.5.27-rmap/mm/vmscan.c Sat Jul 20 14:20:14 2002 +++ linux-2.5.27-rmap-slablru/mm/vmscan.c Sat Jul 20 15:10:23 2002 @@ -142,6 +142,9 @@ goto found_page; } + /* should not be on this list... */ + BUG_ON(PageSlab(page)); + /* We should never ever get here. */ printk(KERN_ERR "VM: reclaim_page, found unknown page\n"); list_del(page_lru); @@ -245,6 +248,9 @@ continue; } + /* Slab pages should never get here... */ + BUG_ON(PageSlab(page)); + /* * The page is in active use or really unfreeable. Move to * the active list and adjust the page age if needed. @@ -453,6 +459,7 @@ * This function will scan a portion of the active list of a zone to find * unused pages, those pages will then be moved to the inactive list. */ + int refill_inactive_zone(struct zone_struct * zone, int priority) { int maxscan = zone->active_pages >> priority; @@ -491,7 +498,7 @@ * both PG_locked and the pte_chain_lock are held. */ pte_chain_lock(page); - if (!page_mapping_inuse(page)) { + if (!page_mapping_inuse(page) && !PageSlab(page)) { pte_chain_unlock(page); unlock_page(page); drop_page(page); @@ -508,6 +515,30 @@ } /* + * For slab pages we count entries for caches with their + * own pruning/aging method. If we can count a page + * (i.e. its slab been been pruned) or it's cold we try + * to free it. Note: we skip inactive_clean -- freed pages + * go straight to the freelist after leaving the LRU. + */ + if (PageSlab(page)) { + pte_chain_unlock(page); + unlock_page(page); + if (kmem_count_page(page, !page->age)) { + int pages = kmem_shrink_slab(page); + if (pages) { + nr_deactivated += pages; + if (nr_deactivated > target) + goto done; + continue; + } + } + list_del(page_lru); + list_add(page_lru, &zone->active_list); + continue; + } + + /* * If the page age is 'hot' and the process using the * page doesn't exceed its RSS limit we keep the page. * Otherwise we move it to the inactive_dirty list. @@ -539,6 +570,7 @@ return nr_deactivated; } + /** * refill_inactive - checks all zones and refills the inactive list as needed * @@ -602,25 +634,14 @@ int ret = 0; /* - * Eat memory from filesystem page cache, - * dentry, inode and filesystem quota caches. + * Eat memory from filesystem page cache, buffer cache, and pruned + * slab caches. The rate we encounter slabs in refill_inactive() + * tells us how hard we should try to prune the slab caches in + * kmem_do_prunes(). Those slabs should be freed on the next pass. */ ret += page_launder(gfp_mask); - ret += shrink_dcache_memory(DEF_PRIORITY, gfp_mask); - ret += shrink_icache_memory(1, gfp_mask); -#ifdef CONFIG_QUOTA - ret += shrink_dqcache_memory(DEF_PRIORITY, gfp_mask); -#endif - /* - * Move pages from the active list to the inactive list. - */ refill_inactive(); - - /* - * Reclaim unused slab cache memory. - */ - ret += kmem_cache_reap(gfp_mask); - + kmem_do_prunes(gfp_mask); refill_freelist(); /* Start IO when needed. */ @@ -628,11 +649,13 @@ blk_run_queues(); /* - * Hmm.. Cache shrink failed - time to kill something? + * Last chance for cache reap! Then, hmm.. time to kill something? * Mhwahahhaha! This is the part I really like. Giggle. */ - if (!ret && free_min(ANY_ZONE) > 0) - out_of_memory(); + if (!ret && free_min(ANY_ZONE) > 0) { + if (!kmem_cache_reap(gfp_mask)) + out_of_memory(); + } return ret; } @@ -723,6 +746,7 @@ /* Do background page aging. */ background_aging(DEF_PRIORITY); + kmem_do_prunes(GFP_KSWAPD); } wakeup_memwaiters();