X-Git-Url: http://ftp.safe.ca/?p=safe%2Fjmp%2Flinux-2.6;a=blobdiff_plain;f=fs%2Finode.c;h=4d8e3be55976272732f6f42610ab7813f66b8133;hp=04536ebc5ac46c8012408168a87083fa442c3039;hb=9a1607071c293e48b08bd703733480b1d55c7b93;hpb=b227613841d4d211a10c5860acc73e133b613bc0 diff --git a/fs/inode.c b/fs/inode.c index 04536eb..4d8e3be 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -14,14 +14,19 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include #include +#include +#include /* * This is needed for the following functions: @@ -83,102 +88,174 @@ static struct hlist_head *inode_hashtable __read_mostly; DEFINE_SPINLOCK(inode_lock); /* - * iprune_mutex provides exclusion between the kswapd or try_to_free_pages + * iprune_sem provides exclusion between the kswapd or try_to_free_pages * icache shrinking path, and the umount path. Without this exclusion, * by the time prune_icache calls iput for the inode whose pages it has * been invalidating, or by the time it calls clear_inode & destroy_inode * from its final dispose_list, the struct super_block they refer to * (for inode->i_sb->s_op) may already have been freed and reused. + * + * We make this an rwsem because the fastpath is icache shrinking. In + * some cases a filesystem may be doing a significant amount of work in + * its inode reclaim code, so this should improve parallelism. */ -static DEFINE_MUTEX(iprune_mutex); +static DECLARE_RWSEM(iprune_sem); /* * Statistics gathering.. */ struct inodes_stat_t inodes_stat; -static struct kmem_cache * inode_cachep __read_mostly; +static struct kmem_cache *inode_cachep __read_mostly; -static struct inode *alloc_inode(struct super_block *sb) +static void wake_up_inode(struct inode *inode) +{ + /* + * Prevent speculative execution through spin_unlock(&inode_lock); + */ + smp_mb(); + wake_up_bit(&inode->i_state, __I_LOCK); +} + +/** + * inode_init_always - perform inode structure intialisation + * @sb: superblock inode belongs to + * @inode: inode to initialise + * + * These are initializations that need to be done on every inode + * allocation as the fields are not initialised by slab allocation. + */ +int inode_init_always(struct super_block *sb, struct inode *inode) { static const struct address_space_operations empty_aops; - static struct inode_operations empty_iops; + static const struct inode_operations empty_iops; static const struct file_operations empty_fops; + struct address_space *const mapping = &inode->i_data; + + inode->i_sb = sb; + inode->i_blkbits = sb->s_blocksize_bits; + inode->i_flags = 0; + atomic_set(&inode->i_count, 1); + inode->i_op = &empty_iops; + inode->i_fop = &empty_fops; + inode->i_nlink = 1; + inode->i_uid = 0; + inode->i_gid = 0; + atomic_set(&inode->i_writecount, 0); + inode->i_size = 0; + inode->i_blocks = 0; + inode->i_bytes = 0; + inode->i_generation = 0; +#ifdef CONFIG_QUOTA + memset(&inode->i_dquot, 0, sizeof(inode->i_dquot)); +#endif + inode->i_pipe = NULL; + inode->i_bdev = NULL; + inode->i_cdev = NULL; + inode->i_rdev = 0; + inode->dirtied_when = 0; + + if (security_inode_alloc(inode)) + goto out; + + /* allocate and initialize an i_integrity */ + if (ima_inode_alloc(inode)) + goto out_free_security; + + spin_lock_init(&inode->i_lock); + lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key); + + mutex_init(&inode->i_mutex); + lockdep_set_class(&inode->i_mutex, &sb->s_type->i_mutex_key); + + init_rwsem(&inode->i_alloc_sem); + lockdep_set_class(&inode->i_alloc_sem, &sb->s_type->i_alloc_sem_key); + + mapping->a_ops = &empty_aops; + mapping->host = inode; + mapping->flags = 0; + mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE); + mapping->assoc_mapping = NULL; + mapping->backing_dev_info = &default_backing_dev_info; + mapping->writeback_index = 0; + + /* + * If the block_device provides a backing_dev_info for client + * inodes then use that. Otherwise the inode share the bdev's + * backing_dev_info. + */ + if (sb->s_bdev) { + struct backing_dev_info *bdi; + + bdi = sb->s_bdev->bd_inode->i_mapping->backing_dev_info; + mapping->backing_dev_info = bdi; + } + inode->i_private = NULL; + inode->i_mapping = mapping; +#ifdef CONFIG_FS_POSIX_ACL + inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED; +#endif + +#ifdef CONFIG_FSNOTIFY + inode->i_fsnotify_mask = 0; +#endif + + return 0; + +out_free_security: + security_inode_free(inode); +out: + return -ENOMEM; +} +EXPORT_SYMBOL(inode_init_always); + +static struct inode *alloc_inode(struct super_block *sb) +{ struct inode *inode; if (sb->s_op->alloc_inode) inode = sb->s_op->alloc_inode(sb); else - inode = (struct inode *) kmem_cache_alloc(inode_cachep, GFP_KERNEL); - - if (inode) { - struct address_space * const mapping = &inode->i_data; - - inode->i_sb = sb; - inode->i_blkbits = sb->s_blocksize_bits; - inode->i_flags = 0; - atomic_set(&inode->i_count, 1); - inode->i_op = &empty_iops; - inode->i_fop = &empty_fops; - inode->i_nlink = 1; - atomic_set(&inode->i_writecount, 0); - inode->i_size = 0; - inode->i_blocks = 0; - inode->i_bytes = 0; - inode->i_generation = 0; -#ifdef CONFIG_QUOTA - memset(&inode->i_dquot, 0, sizeof(inode->i_dquot)); -#endif - inode->i_pipe = NULL; - inode->i_bdev = NULL; - inode->i_cdev = NULL; - inode->i_rdev = 0; - inode->dirtied_when = 0; - if (security_inode_alloc(inode)) { - if (inode->i_sb->s_op->destroy_inode) - inode->i_sb->s_op->destroy_inode(inode); - else - kmem_cache_free(inode_cachep, (inode)); - return NULL; - } + inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL); - mapping->a_ops = &empty_aops; - mapping->host = inode; - mapping->flags = 0; - mapping_set_gfp_mask(mapping, GFP_HIGHUSER); - mapping->assoc_mapping = NULL; - mapping->backing_dev_info = &default_backing_dev_info; + if (!inode) + return NULL; - /* - * If the block_device provides a backing_dev_info for client - * inodes then use that. Otherwise the inode share the bdev's - * backing_dev_info. - */ - if (sb->s_bdev) { - struct backing_dev_info *bdi; - - bdi = sb->s_bdev->bd_inode_backing_dev_info; - if (!bdi) - bdi = sb->s_bdev->bd_inode->i_mapping->backing_dev_info; - mapping->backing_dev_info = bdi; - } - inode->i_private = NULL; - inode->i_mapping = mapping; + if (unlikely(inode_init_always(sb, inode))) { + if (inode->i_sb->s_op->destroy_inode) + inode->i_sb->s_op->destroy_inode(inode); + else + kmem_cache_free(inode_cachep, inode); + return NULL; } + return inode; } -void destroy_inode(struct inode *inode) +void __destroy_inode(struct inode *inode) { BUG_ON(inode_has_buffers(inode)); + ima_inode_free(inode); security_inode_free(inode); + fsnotify_inode_delete(inode); +#ifdef CONFIG_FS_POSIX_ACL + if (inode->i_acl && inode->i_acl != ACL_NOT_CACHED) + posix_acl_release(inode->i_acl); + if (inode->i_default_acl && inode->i_default_acl != ACL_NOT_CACHED) + posix_acl_release(inode->i_default_acl); +#endif +} +EXPORT_SYMBOL(__destroy_inode); + +void destroy_inode(struct inode *inode) +{ + __destroy_inode(inode); if (inode->i_sb->s_op->destroy_inode) inode->i_sb->s_op->destroy_inode(inode); else kmem_cache_free(inode_cachep, (inode)); } - /* * These are initializations that only need to be done * once, because the fields are idempotent across use @@ -190,45 +267,42 @@ void inode_init_once(struct inode *inode) INIT_HLIST_NODE(&inode->i_hash); INIT_LIST_HEAD(&inode->i_dentry); INIT_LIST_HEAD(&inode->i_devices); - mutex_init(&inode->i_mutex); - init_rwsem(&inode->i_alloc_sem); INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC); - rwlock_init(&inode->i_data.tree_lock); + spin_lock_init(&inode->i_data.tree_lock); spin_lock_init(&inode->i_data.i_mmap_lock); INIT_LIST_HEAD(&inode->i_data.private_list); spin_lock_init(&inode->i_data.private_lock); INIT_RAW_PRIO_TREE_ROOT(&inode->i_data.i_mmap); INIT_LIST_HEAD(&inode->i_data.i_mmap_nonlinear); - spin_lock_init(&inode->i_lock); i_size_ordered_init(inode); #ifdef CONFIG_INOTIFY INIT_LIST_HEAD(&inode->inotify_watches); mutex_init(&inode->inotify_mutex); #endif +#ifdef CONFIG_FSNOTIFY + INIT_HLIST_HEAD(&inode->i_fsnotify_mark_entries); +#endif } - EXPORT_SYMBOL(inode_init_once); -static void init_once(void * foo, struct kmem_cache * cachep, unsigned long flags) +static void init_once(void *foo) { - struct inode * inode = (struct inode *) foo; + struct inode *inode = (struct inode *) foo; - if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == - SLAB_CTOR_CONSTRUCTOR) - inode_init_once(inode); + inode_init_once(inode); } /* * inode_lock must be held */ -void __iget(struct inode * inode) +void __iget(struct inode *inode) { if (atomic_read(&inode->i_count)) { atomic_inc(&inode->i_count); return; } atomic_inc(&inode->i_count); - if (!(inode->i_state & (I_DIRTY|I_LOCK))) + if (!(inode->i_state & (I_DIRTY|I_SYNC))) list_move(&inode->i_list, &inode_in_use); inodes_stat.nr_unused--; } @@ -245,13 +319,13 @@ void clear_inode(struct inode *inode) { might_sleep(); invalidate_inode_buffers(inode); - + BUG_ON(inode->i_data.nrpages); BUG_ON(!(inode->i_state & I_FREEING)); BUG_ON(inode->i_state & I_CLEAR); - wait_on_inode(inode); - DQUOT_DROP(inode); - if (inode->i_sb && inode->i_sb->s_op->clear_inode) + inode_sync_wait(inode); + vfs_dq_drop(inode); + if (inode->i_sb->s_op->clear_inode) inode->i_sb->s_op->clear_inode(inode); if (S_ISBLK(inode->i_mode) && inode->i_bdev) bd_forget(inode); @@ -259,7 +333,6 @@ void clear_inode(struct inode *inode) cd_forget(inode); inode->i_state = I_CLEAR; } - EXPORT_SYMBOL(clear_inode); /* @@ -276,7 +349,7 @@ static void dispose_list(struct list_head *head) while (!list_empty(head)) { struct inode *inode; - inode = list_entry(head->next, struct inode, i_list); + inode = list_first_entry(head, struct inode, i_list); list_del(&inode->i_list); if (inode->i_data.nrpages) @@ -307,13 +380,13 @@ static int invalidate_list(struct list_head *head, struct list_head *dispose) next = head->next; for (;;) { - struct list_head * tmp = next; - struct inode * inode; + struct list_head *tmp = next; + struct inode *inode; /* * We can reschedule here without worrying about the list's * consistency because the per-sb list of inodes must not - * change during umount anymore, and because iprune_mutex keeps + * change during umount anymore, and because iprune_sem keeps * shrink_icache_memory() away. */ cond_resched_lock(&inode_lock); @@ -322,9 +395,12 @@ static int invalidate_list(struct list_head *head, struct list_head *dispose) if (tmp == head) break; inode = list_entry(tmp, struct inode, i_sb_list); + if (inode->i_state & I_NEW) + continue; invalidate_inode_buffers(inode); if (!atomic_read(&inode->i_count)) { list_move(&inode->i_list, dispose); + WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; count++; continue; @@ -344,23 +420,23 @@ static int invalidate_list(struct list_head *head, struct list_head *dispose) * fails because there are busy inodes then a non zero value is returned. * If the discard is successful all the inodes have been discarded. */ -int invalidate_inodes(struct super_block * sb) +int invalidate_inodes(struct super_block *sb) { int busy; LIST_HEAD(throw_away); - mutex_lock(&iprune_mutex); + down_write(&iprune_sem); spin_lock(&inode_lock); inotify_unmount_inodes(&sb->s_inodes); + fsnotify_unmount_inodes(&sb->s_inodes); busy = invalidate_list(&sb->s_inodes, &throw_away); spin_unlock(&inode_lock); dispose_list(&throw_away); - mutex_unlock(&iprune_mutex); + up_write(&iprune_sem); return busy; } - EXPORT_SYMBOL(invalidate_inodes); static int can_unuse(struct inode *inode) @@ -396,7 +472,7 @@ static void prune_icache(int nr_to_scan) int nr_scanned; unsigned long reap = 0; - mutex_lock(&iprune_mutex); + down_read(&iprune_sem); spin_lock(&inode_lock); for (nr_scanned = 0; nr_scanned < nr_to_scan; nr_scanned++) { struct inode *inode; @@ -414,7 +490,8 @@ static void prune_icache(int nr_to_scan) __iget(inode); spin_unlock(&inode_lock); if (remove_inode_buffers(inode)) - reap += invalidate_inode_pages(&inode->i_data); + reap += invalidate_mapping_pages(&inode->i_data, + 0, -1); iput(inode); spin_lock(&inode_lock); @@ -425,6 +502,7 @@ static void prune_icache(int nr_to_scan) continue; } list_move(&inode->i_list, &freeable); + WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; nr_pruned++; } @@ -436,7 +514,7 @@ static void prune_icache(int nr_to_scan) spin_unlock(&inode_lock); dispose_list(&freeable); - mutex_unlock(&iprune_mutex); + up_read(&iprune_sem); } /* @@ -455,7 +533,7 @@ static int shrink_icache_memory(int nr, gfp_t gfp_mask) * Nasty deadlock avoidance. We may hold various FS locks, * and we don't want to recurse into the FS that called us * in clear_inode() and friends.. - */ + */ if (!(gfp_mask & __GFP_FS)) return -1; prune_icache(nr); @@ -463,6 +541,11 @@ static int shrink_icache_memory(int nr, gfp_t gfp_mask) return (inodes_stat.nr_unused / 100) * sysctl_vfs_cache_pressure; } +static struct shrinker icache_shrinker = { + .shrink = shrink_icache_memory, + .seeks = DEFAULT_SEEKS, +}; + static void __wait_on_freeing_inode(struct inode *inode); /* * Called with the inode lock held. @@ -470,14 +553,16 @@ static void __wait_on_freeing_inode(struct inode *inode); * by hand after calling find_inode now! This simplifies iunique and won't * add any additional branch in the common code. */ -static struct inode * find_inode(struct super_block * sb, struct hlist_head *head, int (*test)(struct inode *, void *), void *data) +static struct inode *find_inode(struct super_block *sb, + struct hlist_head *head, + int (*test)(struct inode *, void *), + void *data) { struct hlist_node *node; - struct inode * inode = NULL; + struct inode *inode = NULL; repeat: - hlist_for_each (node, head) { - inode = hlist_entry(node, struct inode, i_hash); + hlist_for_each_entry(inode, node, head, i_hash) { if (inode->i_sb != sb) continue; if (!test(inode, data)) @@ -495,14 +580,14 @@ repeat: * find_inode_fast is the fast path version of find_inode, see the comment at * iget_locked for details. */ -static struct inode * find_inode_fast(struct super_block * sb, struct hlist_head *head, unsigned long ino) +static struct inode *find_inode_fast(struct super_block *sb, + struct hlist_head *head, unsigned long ino) { struct hlist_node *node; - struct inode * inode = NULL; + struct inode *inode = NULL; repeat: - hlist_for_each (node, head) { - inode = hlist_entry(node, struct inode, i_hash); + hlist_for_each_entry(inode, node, head, i_hash) { if (inode->i_ino != ino) continue; if (inode->i_sb != sb) @@ -516,48 +601,118 @@ repeat: return node ? inode : NULL; } +static unsigned long hash(struct super_block *sb, unsigned long hashval) +{ + unsigned long tmp; + + tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) / + L1_CACHE_BYTES; + tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> I_HASHBITS); + return tmp & I_HASHMASK; +} + +static inline void +__inode_add_to_lists(struct super_block *sb, struct hlist_head *head, + struct inode *inode) +{ + inodes_stat.nr_inodes++; + list_add(&inode->i_list, &inode_in_use); + list_add(&inode->i_sb_list, &sb->s_inodes); + if (head) + hlist_add_head(&inode->i_hash, head); +} + +/** + * inode_add_to_lists - add a new inode to relevant lists + * @sb: superblock inode belongs to + * @inode: inode to mark in use + * + * When an inode is allocated it needs to be accounted for, added to the in use + * list, the owning superblock and the inode hash. This needs to be done under + * the inode_lock, so export a function to do this rather than the inode lock + * itself. We calculate the hash list to add to here so it is all internal + * which requires the caller to have already set up the inode number in the + * inode to add. + */ +void inode_add_to_lists(struct super_block *sb, struct inode *inode) +{ + struct hlist_head *head = inode_hashtable + hash(sb, inode->i_ino); + + spin_lock(&inode_lock); + __inode_add_to_lists(sb, head, inode); + spin_unlock(&inode_lock); +} +EXPORT_SYMBOL_GPL(inode_add_to_lists); + /** * new_inode - obtain an inode * @sb: superblock * - * Allocates a new inode for given superblock. + * Allocates a new inode for given superblock. The default gfp_mask + * for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE. + * If HIGHMEM pages are unsuitable or it is known that pages allocated + * for the page cache are not reclaimable or migratable, + * mapping_set_gfp_mask() must be called with suitable flags on the + * newly created inode's mapping + * */ struct inode *new_inode(struct super_block *sb) { - static unsigned long last_ino; - struct inode * inode; + /* + * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW + * error if st_ino won't fit in target struct field. Use 32bit counter + * here to attempt to avoid that. + */ + static unsigned int last_ino; + struct inode *inode; spin_lock_prefetch(&inode_lock); - + inode = alloc_inode(sb); if (inode) { spin_lock(&inode_lock); - inodes_stat.nr_inodes++; - list_add(&inode->i_list, &inode_in_use); - list_add(&inode->i_sb_list, &sb->s_inodes); + __inode_add_to_lists(sb, NULL, inode); inode->i_ino = ++last_ino; inode->i_state = 0; spin_unlock(&inode_lock); } return inode; } - EXPORT_SYMBOL(new_inode); void unlock_new_inode(struct inode *inode) { +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (inode->i_mode & S_IFDIR) { + struct file_system_type *type = inode->i_sb->s_type; + + /* Set new key only if filesystem hasn't already changed it */ + if (!lockdep_match_class(&inode->i_mutex, + &type->i_mutex_key)) { + /* + * ensure nobody is actually holding i_mutex + */ + mutex_destroy(&inode->i_mutex); + mutex_init(&inode->i_mutex); + lockdep_set_class(&inode->i_mutex, + &type->i_mutex_dir_key); + } + } +#endif /* - * This is special! We do not need the spinlock - * when clearing I_LOCK, because we're guaranteed - * that nobody else tries to do anything about the - * state of the inode when it is locked, as we - * just created it (so there can be no old holders - * that haven't tested I_LOCK). + * This is special! We do not need the spinlock when clearing I_LOCK, + * because we're guaranteed that nobody else tries to do anything about + * the state of the inode when it is locked, as we just created it (so + * there can be no old holders that haven't tested I_LOCK). + * However we must emit the memory barrier so that other CPUs reliably + * see the clearing of I_LOCK after the other inode initialisation has + * completed. */ + smp_mb(); + WARN_ON((inode->i_state & (I_LOCK|I_NEW)) != (I_LOCK|I_NEW)); inode->i_state &= ~(I_LOCK|I_NEW); wake_up_inode(inode); } - EXPORT_SYMBOL(unlock_new_inode); /* @@ -566,13 +721,17 @@ EXPORT_SYMBOL(unlock_new_inode); * We no longer cache the sb_flags in i_flags - see fs.h * -- rmk@arm.uk.linux.org */ -static struct inode * get_new_inode(struct super_block *sb, struct hlist_head *head, int (*test)(struct inode *, void *), int (*set)(struct inode *, void *), void *data) +static struct inode *get_new_inode(struct super_block *sb, + struct hlist_head *head, + int (*test)(struct inode *, void *), + int (*set)(struct inode *, void *), + void *data) { - struct inode * inode; + struct inode *inode; inode = alloc_inode(sb); if (inode) { - struct inode * old; + struct inode *old; spin_lock(&inode_lock); /* We released the lock, so.. */ @@ -581,10 +740,7 @@ static struct inode * get_new_inode(struct super_block *sb, struct hlist_head *h if (set(inode, data)) goto set_failed; - inodes_stat.nr_inodes++; - list_add(&inode->i_list, &inode_in_use); - list_add(&inode->i_sb_list, &sb->s_inodes); - hlist_add_head(&inode->i_hash, head); + __inode_add_to_lists(sb, head, inode); inode->i_state = I_LOCK|I_NEW; spin_unlock(&inode_lock); @@ -617,23 +773,21 @@ set_failed: * get_new_inode_fast is the fast path version of get_new_inode, see the * comment at iget_locked for details. */ -static struct inode * get_new_inode_fast(struct super_block *sb, struct hlist_head *head, unsigned long ino) +static struct inode *get_new_inode_fast(struct super_block *sb, + struct hlist_head *head, unsigned long ino) { - struct inode * inode; + struct inode *inode; inode = alloc_inode(sb); if (inode) { - struct inode * old; + struct inode *old; spin_lock(&inode_lock); /* We released the lock, so.. */ old = find_inode_fast(sb, head, ino); if (!old) { inode->i_ino = ino; - inodes_stat.nr_inodes++; - list_add(&inode->i_list, &inode_in_use); - list_add(&inode->i_sb_list, &sb->s_inodes); - hlist_add_head(&inode->i_hash, head); + __inode_add_to_lists(sb, head, inode); inode->i_state = I_LOCK|I_NEW; spin_unlock(&inode_lock); @@ -657,16 +811,6 @@ static struct inode * get_new_inode_fast(struct super_block *sb, struct hlist_he return inode; } -static unsigned long hash(struct super_block *sb, unsigned long hashval) -{ - unsigned long tmp; - - tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) / - L1_CACHE_BYTES; - tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> I_HASHBITS); - return tmp & I_HASHMASK; -} - /** * iunique - get a unique inode number * @sb: superblock @@ -683,33 +827,34 @@ static unsigned long hash(struct super_block *sb, unsigned long hashval) */ ino_t iunique(struct super_block *sb, ino_t max_reserved) { - static ino_t counter; + /* + * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW + * error if st_ino won't fit in target struct field. Use 32bit counter + * here to attempt to avoid that. + */ + static unsigned int counter; struct inode *inode; - struct hlist_head * head; + struct hlist_head *head; ino_t res; + spin_lock(&inode_lock); -retry: - if (counter > max_reserved) { - head = inode_hashtable + hash(sb,counter); + do { + if (counter <= max_reserved) + counter = max_reserved + 1; res = counter++; + head = inode_hashtable + hash(sb, res); inode = find_inode_fast(sb, head, res); - if (!inode) { - spin_unlock(&inode_lock); - return res; - } - } else { - counter = max_reserved + 1; - } - goto retry; - -} + } while (inode != NULL); + spin_unlock(&inode_lock); + return res; +} EXPORT_SYMBOL(iunique); struct inode *igrab(struct inode *inode) { spin_lock(&inode_lock); - if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) + if (!(inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE))) __iget(inode); else /* @@ -721,7 +866,6 @@ struct inode *igrab(struct inode *inode) spin_unlock(&inode_lock); return inode; } - EXPORT_SYMBOL(igrab); /** @@ -822,7 +966,6 @@ struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval, return ifind(sb, head, test, data, 0); } - EXPORT_SYMBOL(ilookup5_nowait); /** @@ -851,7 +994,6 @@ struct inode *ilookup5(struct super_block *sb, unsigned long hashval, return ifind(sb, head, test, data, 1); } - EXPORT_SYMBOL(ilookup5); /** @@ -874,7 +1016,6 @@ struct inode *ilookup(struct super_block *sb, unsigned long ino) return ifind_fast(sb, head, ino); } - EXPORT_SYMBOL(ilookup); /** @@ -885,8 +1026,6 @@ EXPORT_SYMBOL(ilookup); * @set: callback used to initialize a new struct inode * @data: opaque data pointer to pass to @test and @set * - * This is iget() without the read_inode() portion of get_new_inode(). - * * iget5_locked() uses ifind() to search for the inode specified by @hashval * and @data in the inode cache and if present it is returned with an increased * reference count. This is a generalized version of iget_locked() for file @@ -915,7 +1054,6 @@ struct inode *iget5_locked(struct super_block *sb, unsigned long hashval, */ return get_new_inode(sb, head, test, set, data); } - EXPORT_SYMBOL(iget5_locked); /** @@ -923,8 +1061,6 @@ EXPORT_SYMBOL(iget5_locked); * @sb: super block of file system * @ino: inode number to get * - * This is iget() without the read_inode() portion of get_new_inode_fast(). - * * iget_locked() uses ifind_fast() to search for the inode specified by @ino in * the inode cache and if present it is returned with an increased reference * count. This is for file systems where the inode number is sufficient for @@ -949,9 +1085,84 @@ struct inode *iget_locked(struct super_block *sb, unsigned long ino) */ return get_new_inode_fast(sb, head, ino); } - EXPORT_SYMBOL(iget_locked); +int insert_inode_locked(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + ino_t ino = inode->i_ino; + struct hlist_head *head = inode_hashtable + hash(sb, ino); + + inode->i_state |= I_LOCK|I_NEW; + while (1) { + struct hlist_node *node; + struct inode *old = NULL; + spin_lock(&inode_lock); + hlist_for_each_entry(old, node, head, i_hash) { + if (old->i_ino != ino) + continue; + if (old->i_sb != sb) + continue; + if (old->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE)) + continue; + break; + } + if (likely(!node)) { + hlist_add_head(&inode->i_hash, head); + spin_unlock(&inode_lock); + return 0; + } + __iget(old); + spin_unlock(&inode_lock); + wait_on_inode(old); + if (unlikely(!hlist_unhashed(&old->i_hash))) { + iput(old); + return -EBUSY; + } + iput(old); + } +} +EXPORT_SYMBOL(insert_inode_locked); + +int insert_inode_locked4(struct inode *inode, unsigned long hashval, + int (*test)(struct inode *, void *), void *data) +{ + struct super_block *sb = inode->i_sb; + struct hlist_head *head = inode_hashtable + hash(sb, hashval); + + inode->i_state |= I_LOCK|I_NEW; + + while (1) { + struct hlist_node *node; + struct inode *old = NULL; + + spin_lock(&inode_lock); + hlist_for_each_entry(old, node, head, i_hash) { + if (old->i_sb != sb) + continue; + if (!test(old, data)) + continue; + if (old->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE)) + continue; + break; + } + if (likely(!node)) { + hlist_add_head(&inode->i_hash, head); + spin_unlock(&inode_lock); + return 0; + } + __iget(old); + spin_unlock(&inode_lock); + wait_on_inode(old); + if (unlikely(!hlist_unhashed(&old->i_hash))) { + iput(old); + return -EBUSY; + } + iput(old); + } +} +EXPORT_SYMBOL(insert_inode_locked4); + /** * __insert_inode_hash - hash an inode * @inode: unhashed inode @@ -967,7 +1178,6 @@ void __insert_inode_hash(struct inode *inode, unsigned long hashval) hlist_add_head(&inode->i_hash, head); spin_unlock(&inode_lock); } - EXPORT_SYMBOL(__insert_inode_hash); /** @@ -982,7 +1192,6 @@ void remove_inode_hash(struct inode *inode) hlist_del_init(&inode->i_hash); spin_unlock(&inode_lock); } - EXPORT_SYMBOL(remove_inode_hash); /* @@ -999,10 +1208,11 @@ EXPORT_SYMBOL(remove_inode_hash); */ void generic_delete_inode(struct inode *inode) { - struct super_operations *op = inode->i_sb->s_op; + const struct super_operations *op = inode->i_sb->s_op; list_del_init(&inode->i_list); list_del_init(&inode->i_sb_list); + WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; inodes_stat.nr_inodes--; spin_unlock(&inode_lock); @@ -1012,7 +1222,7 @@ void generic_delete_inode(struct inode *inode) if (op->delete_inode) { void (*delete)(struct inode *) = op->delete_inode; if (!is_bad_inode(inode)) - DQUOT_INIT(inode); + vfs_dq_init(inode); /* Filesystems implementing their own * s_op->delete_inode are required to call * truncate_inode_pages and clear_inode() @@ -1029,34 +1239,53 @@ void generic_delete_inode(struct inode *inode) BUG_ON(inode->i_state != I_CLEAR); destroy_inode(inode); } - EXPORT_SYMBOL(generic_delete_inode); -static void generic_forget_inode(struct inode *inode) +/** + * generic_detach_inode - remove inode from inode lists + * @inode: inode to remove + * + * Remove inode from inode lists, write it if it's dirty. This is just an + * internal VFS helper exported for hugetlbfs. Do not use! + * + * Returns 1 if inode should be completely destroyed. + */ +int generic_detach_inode(struct inode *inode) { struct super_block *sb = inode->i_sb; if (!hlist_unhashed(&inode->i_hash)) { - if (!(inode->i_state & (I_DIRTY|I_LOCK))) + if (!(inode->i_state & (I_DIRTY|I_SYNC))) list_move(&inode->i_list, &inode_unused); inodes_stat.nr_unused++; - if (!sb || (sb->s_flags & MS_ACTIVE)) { + if (sb->s_flags & MS_ACTIVE) { spin_unlock(&inode_lock); - return; + return 0; } + WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_WILL_FREE; spin_unlock(&inode_lock); write_inode_now(inode, 1); spin_lock(&inode_lock); + WARN_ON(inode->i_state & I_NEW); inode->i_state &= ~I_WILL_FREE; inodes_stat.nr_unused--; hlist_del_init(&inode->i_hash); } list_del_init(&inode->i_list); list_del_init(&inode->i_sb_list); + WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; inodes_stat.nr_inodes--; spin_unlock(&inode_lock); + return 1; +} +EXPORT_SYMBOL_GPL(generic_detach_inode); + +static void generic_forget_inode(struct inode *inode) +{ + if (!generic_detach_inode(inode)) + return; if (inode->i_data.nrpages) truncate_inode_pages(&inode->i_data, 0); clear_inode(inode); @@ -1076,12 +1305,11 @@ void generic_drop_inode(struct inode *inode) else generic_forget_inode(inode); } - EXPORT_SYMBOL_GPL(generic_drop_inode); /* * Called when we're dropping the last reference - * to an inode. + * to an inode. * * Call the FS "drop()" function, defaulting to * the legacy UNIX filesystem behaviour.. @@ -1092,7 +1320,7 @@ EXPORT_SYMBOL_GPL(generic_drop_inode); */ static inline void iput_final(struct inode *inode) { - struct super_operations *op = inode->i_sb->s_op; + const struct super_operations *op = inode->i_sb->s_op; void (*drop)(struct inode *) = generic_drop_inode; if (op && op->drop_inode) @@ -1101,7 +1329,7 @@ static inline void iput_final(struct inode *inode) } /** - * iput - put an inode + * iput - put an inode * @inode: inode to put * * Puts an inode, dropping its usage count. If the inode use count hits @@ -1112,18 +1340,12 @@ static inline void iput_final(struct inode *inode) void iput(struct inode *inode) { if (inode) { - struct super_operations *op = inode->i_sb->s_op; - BUG_ON(inode->i_state == I_CLEAR); - if (op && op->put_inode) - op->put_inode(inode); - if (atomic_dec_and_lock(&inode->i_count, &inode_lock)) iput_final(inode); } } - EXPORT_SYMBOL(iput); /** @@ -1134,10 +1356,10 @@ EXPORT_SYMBOL(iput); * Returns the block number on the device holding the inode that * is the disk block number for the block of the file requested. * That is, asked for block 4 of inode 1 the function will return the - * disk block relative to the disk start that holds that block of the + * disk block relative to the disk start that holds that block of the * file. */ -sector_t bmap(struct inode * inode, sector_t block) +sector_t bmap(struct inode *inode, sector_t block) { sector_t res = 0; if (inode->i_mapping->a_ops->bmap) @@ -1146,6 +1368,40 @@ sector_t bmap(struct inode * inode, sector_t block) } EXPORT_SYMBOL(bmap); +/* + * With relative atime, only update atime if the previous atime is + * earlier than either the ctime or mtime or if at least a day has + * passed since the last atime update. + */ +static int relatime_need_update(struct vfsmount *mnt, struct inode *inode, + struct timespec now) +{ + + if (!(mnt->mnt_flags & MNT_RELATIME)) + return 1; + /* + * Is mtime younger than atime? If yes, update atime: + */ + if (timespec_compare(&inode->i_mtime, &inode->i_atime) >= 0) + return 1; + /* + * Is ctime younger than atime? If yes, update atime: + */ + if (timespec_compare(&inode->i_ctime, &inode->i_atime) >= 0) + return 1; + + /* + * Is the previous atime value older than a day? If yes, + * update atime: + */ + if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60) + return 1; + /* + * Good, we can skip the atime update: + */ + return 0; +} + /** * touch_atime - update the access time * @mnt: mount the inode is accessed on @@ -1160,30 +1416,32 @@ void touch_atime(struct vfsmount *mnt, struct dentry *dentry) struct inode *inode = dentry->d_inode; struct timespec now; - if (IS_RDONLY(inode)) - return; if (inode->i_flags & S_NOATIME) return; - if (inode->i_sb->s_flags & MS_NOATIME) + if (IS_NOATIME(inode)) return; if ((inode->i_sb->s_flags & MS_NODIRATIME) && S_ISDIR(inode->i_mode)) return; - /* - * We may have a NULL vfsmount when coming from NFSD - */ - if (mnt) { - if (mnt->mnt_flags & MNT_NOATIME) - return; - if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)) - return; - } + if (mnt->mnt_flags & MNT_NOATIME) + return; + if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)) + return; now = current_fs_time(inode->i_sb); - if (!timespec_equal(&inode->i_atime, &now)) { - inode->i_atime = now; - mark_inode_dirty_sync(inode); - } + + if (!relatime_need_update(mnt, inode, now)) + return; + + if (timespec_equal(&inode->i_atime, &now)) + return; + + if (mnt_want_write(mnt)) + return; + + inode->i_atime = now; + mark_inode_dirty_sync(inode); + mnt_drop_write(mnt); } EXPORT_SYMBOL(touch_atime); @@ -1195,7 +1453,7 @@ EXPORT_SYMBOL(touch_atime); * for writeback. Note that this function is meant exclusively for * usage in the file write path of filesystems, and filesystems may * choose to explicitly ignore update via this function with the - * S_NOCTIME inode flag, e.g. for network filesystem where these + * S_NOCMTIME inode flag, e.g. for network filesystem where these * timestamps are handled by the server. */ @@ -1203,28 +1461,39 @@ void file_update_time(struct file *file) { struct inode *inode = file->f_path.dentry->d_inode; struct timespec now; - int sync_it = 0; + enum { S_MTIME = 1, S_CTIME = 2, S_VERSION = 4 } sync_it = 0; + /* First try to exhaust all avenues to not sync */ if (IS_NOCMTIME(inode)) return; - if (IS_RDONLY(inode)) - return; now = current_fs_time(inode->i_sb); - if (!timespec_equal(&inode->i_mtime, &now)) { - inode->i_mtime = now; - sync_it = 1; - } + if (!timespec_equal(&inode->i_mtime, &now)) + sync_it = S_MTIME; - if (!timespec_equal(&inode->i_ctime, &now)) { - inode->i_ctime = now; - sync_it = 1; - } + if (!timespec_equal(&inode->i_ctime, &now)) + sync_it |= S_CTIME; - if (sync_it) - mark_inode_dirty_sync(inode); -} + if (IS_I_VERSION(inode)) + sync_it |= S_VERSION; + + if (!sync_it) + return; + + /* Finally allowed to write? Takes lock. */ + if (mnt_want_write_file(file)) + return; + /* Only change inode inside the lock region */ + if (sync_it & S_VERSION) + inode_inc_iversion(inode); + if (sync_it & S_CTIME) + inode->i_ctime = now; + if (sync_it & S_MTIME) + inode->i_mtime = now; + mark_inode_dirty_sync(inode); + mnt_drop_write(file->f_path.mnt); +} EXPORT_SYMBOL(file_update_time); int inode_needs_sync(struct inode *inode) @@ -1235,41 +1504,14 @@ int inode_needs_sync(struct inode *inode) return 1; return 0; } - EXPORT_SYMBOL(inode_needs_sync); -/* - * Quota functions that want to walk the inode lists.. - */ -#ifdef CONFIG_QUOTA - -void remove_dquot_ref(struct super_block *sb, int type, - struct list_head *tofree_head) -{ - struct inode *inode; - - if (!sb->dq_op) - return; /* nothing to do */ - spin_lock(&inode_lock); /* This lock is for inodes code */ - - /* - * We don't have to lock against quota code - test IS_QUOTAINIT is - * just for speedup... - */ - list_for_each_entry(inode, &sb->s_inodes, i_sb_list) - if (!IS_NOQUOTA(inode)) - remove_inode_dquot_ref(inode, type, tofree_head); - - spin_unlock(&inode_lock); -} - -#endif - int inode_wait(void *word) { schedule(); return 0; } +EXPORT_SYMBOL(inode_wait); /* * If we try to find an inode in the inode hash while it is being @@ -1295,51 +1537,6 @@ static void __wait_on_freeing_inode(struct inode *inode) spin_lock(&inode_lock); } -void wake_up_inode(struct inode *inode) -{ - /* - * Prevent speculative execution through spin_unlock(&inode_lock); - */ - smp_mb(); - wake_up_bit(&inode->i_state, __I_LOCK); -} - -/* - * We rarely want to lock two inodes that do not have a parent/child - * relationship (such as directory, child inode) simultaneously. The - * vast majority of file systems should be able to get along fine - * without this. Do not use these functions except as a last resort. - */ -void inode_double_lock(struct inode *inode1, struct inode *inode2) -{ - if (inode1 == NULL || inode2 == NULL || inode1 == inode2) { - if (inode1) - mutex_lock(&inode1->i_mutex); - else if (inode2) - mutex_lock(&inode2->i_mutex); - return; - } - - if (inode1 < inode2) { - mutex_lock_nested(&inode1->i_mutex, I_MUTEX_PARENT); - mutex_lock_nested(&inode2->i_mutex, I_MUTEX_CHILD); - } else { - mutex_lock_nested(&inode2->i_mutex, I_MUTEX_PARENT); - mutex_lock_nested(&inode1->i_mutex, I_MUTEX_CHILD); - } -} -EXPORT_SYMBOL(inode_double_lock); - -void inode_double_unlock(struct inode *inode1, struct inode *inode2) -{ - if (inode1) - mutex_unlock(&inode1->i_mutex); - - if (inode2 && inode2 != inode1) - mutex_unlock(&inode2->i_mutex); -} -EXPORT_SYMBOL(inode_double_unlock); - static __initdata unsigned long ihash_entries; static int __init set_ihash_entries(char *str) { @@ -1377,7 +1574,7 @@ void __init inode_init_early(void) INIT_HLIST_HEAD(&inode_hashtable[loop]); } -void __init inode_init(unsigned long mempages) +void __init inode_init(void) { int loop; @@ -1387,9 +1584,8 @@ void __init inode_init(unsigned long mempages) 0, (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| SLAB_MEM_SPREAD), - init_once, - NULL); - set_shrinker(DEFAULT_SEEKS, shrink_icache_memory); + init_once); + register_shrinker(&icache_shrinker); /* Hash may have been set up in inode_init_early */ if (!hashdist) @@ -1423,7 +1619,8 @@ void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev) else if (S_ISSOCK(mode)) inode->i_fop = &bad_sock_fops; else - printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o)\n", - mode); + printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for" + " inode %s:%lu\n", mode, inode->i_sb->s_id, + inode->i_ino); } EXPORT_SYMBOL(init_special_inode);