mntget(nd->path.mnt);
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
nd->flags &= ~LOOKUP_RCU;
return 0;
if (unlikely(!__d_rcu_to_refcount(dentry, nd->seq))) {
spin_unlock(&dentry->d_lock);
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
return -ECHILD;
}
BUG_ON(nd->inode != dentry->d_inode);
spin_unlock(&dentry->d_lock);
mntget(nd->path.mnt);
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
}
if (likely(!(nd->flags & LOOKUP_JUMPED)))
struct mount *parent;
struct dentry *mountpoint;
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
parent = mnt->mnt_parent;
if (&parent->mnt == path->mnt) {
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
return 0;
}
mntget(&parent->mnt);
mountpoint = dget(mnt->mnt_mountpoint);
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
dput(path->dentry);
path->dentry = mountpoint;
mntput(path->mnt);
if (!(nd->flags & LOOKUP_ROOT))
nd->root.mnt = NULL;
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
return -ECHILD;
}
* small and for now I'd prefer to have fast path as straight as possible.
* It _is_ time-critical.
*/
-static int do_lookup(struct nameidata *nd, struct qstr *name,
- struct path *path, struct inode **inode)
+static int lookup_fast(struct nameidata *nd, struct qstr *name,
+ struct path *path, struct inode **inode)
{
struct vfsmount *mnt = nd->path.mnt;
struct dentry *dentry, *parent = nd->path.dentry;
goto need_lookup;
}
}
-done:
+
path->mnt = mnt;
path->dentry = dentry;
err = follow_managed(path, nd->flags);
return 0;
need_lookup:
+ return 1;
+}
+
+/* Fast lookup failed, do it the slow way */
+static int lookup_slow(struct nameidata *nd, struct qstr *name,
+ struct path *path)
+{
+ struct dentry *dentry, *parent;
+ int err;
+
+ parent = nd->path.dentry;
BUG_ON(nd->inode != parent->d_inode);
mutex_lock(&parent->d_inode->i_mutex);
mutex_unlock(&parent->d_inode->i_mutex);
if (IS_ERR(dentry))
return PTR_ERR(dentry);
- goto done;
+ path->mnt = nd->path.mnt;
+ path->dentry = dentry;
+ err = follow_managed(path, nd->flags);
+ if (unlikely(err < 0)) {
+ path_put_conditional(path, nd);
+ return err;
+ }
+ if (err)
+ nd->flags |= LOOKUP_JUMPED;
+ return 0;
}
static inline int may_lookup(struct nameidata *nd)
if (!(nd->flags & LOOKUP_ROOT))
nd->root.mnt = NULL;
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
}
}
*/
if (unlikely(type != LAST_NORM))
return handle_dots(nd, type);
- err = do_lookup(nd, name, path, &inode);
+ err = lookup_fast(nd, name, path, &inode);
if (unlikely(err)) {
- terminate_walk(nd);
- return err;
- }
- if (!inode) {
- path_to_nameidata(path, nd);
- terminate_walk(nd);
- return -ENOENT;
+ if (err < 0)
+ goto out_err;
+
+ err = lookup_slow(nd, name, path);
+ if (err < 0)
+ goto out_err;
+
+ inode = path->dentry->d_inode;
}
+ err = -ENOENT;
+ if (!inode)
+ goto out_path_put;
+
if (should_follow_link(inode, follow)) {
if (nd->flags & LOOKUP_RCU) {
if (unlikely(unlazy_walk(nd, path->dentry))) {
- terminate_walk(nd);
- return -ECHILD;
+ err = -ECHILD;
+ goto out_err;
}
}
BUG_ON(inode != path->dentry->d_inode);
path_to_nameidata(path, nd);
nd->inode = inode;
return 0;
+
+out_path_put:
+ path_to_nameidata(path, nd);
+out_err:
+ terminate_walk(nd);
+ return err;
}
/*
*/
static inline unsigned long hash_name(const char *name, unsigned int *hashp)
{
- unsigned long a, mask, hash, len;
+ unsigned long a, b, adata, bdata, mask, hash, len;
+ const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
hash = a = 0;
len = -sizeof(unsigned long);
hash = (hash + a) * 9;
len += sizeof(unsigned long);
a = load_unaligned_zeropad(name+len);
- /* Do we have any NUL or '/' bytes in this word? */
- mask = has_zero(a) | has_zero(a ^ REPEAT_BYTE('/'));
- } while (!mask);
-
- /* The mask *below* the first high bit set */
- mask = (mask - 1) & ~mask;
- mask >>= 7;
- hash += a & mask;
+ b = a ^ REPEAT_BYTE('/');
+ } while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)));
+
+ adata = prep_zero_mask(a, adata, &constants);
+ bdata = prep_zero_mask(b, bdata, &constants);
+
+ mask = create_zero_mask(adata | bdata);
+
+ hash += a & zero_bytemask(mask);
*hashp = fold_hash(hash);
- return len + count_masked_bytes(mask);
+ return len + find_zero(mask);
}
#else
nd->path = nd->root;
nd->inode = inode;
if (flags & LOOKUP_RCU) {
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
} else {
if (*name=='/') {
if (flags & LOOKUP_RCU) {
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
set_root_rcu(nd);
} else {
struct fs_struct *fs = current->fs;
unsigned seq;
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
do {
if (fput_needed)
*fp = file;
nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
} else {
path_get(&file->f_path);
int want_write = 0;
int acc_mode = op->acc_mode;
struct file *filp;
+ struct inode *inode;
+ int symlink_ok = 0;
int error;
nd->flags &= ~LOOKUP_PARENT;
}
if (!(open_flag & O_CREAT)) {
- int symlink_ok = 0;
if (nd->last.name[nd->last.len])
nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
if (open_flag & O_PATH && !(nd->flags & LOOKUP_FOLLOW))
symlink_ok = 1;
/* we _can_ be in RCU mode here */
- error = walk_component(nd, path, &nd->last, LAST_NORM,
- !symlink_ok);
- if (error < 0)
- return ERR_PTR(error);
- if (error) /* symlink */
+ error = lookup_fast(nd, &nd->last, path, &inode);
+ if (unlikely(error)) {
+ if (error < 0)
+ goto exit;
+
+ error = lookup_slow(nd, &nd->last, path);
+ if (error < 0)
+ goto exit;
+
+ inode = path->dentry->d_inode;
+ }
+ error = -ENOENT;
+ if (!inode) {
+ path_to_nameidata(path, nd);
+ goto exit;
+ }
+
+ if (should_follow_link(inode, !symlink_ok)) {
+ if (nd->flags & LOOKUP_RCU) {
+ if (unlikely(unlazy_walk(nd, path->dentry))) {
+ error = -ECHILD;
+ goto exit;
+ }
+ }
+ BUG_ON(inode != path->dentry->d_inode);
return NULL;
+ }
+ path_to_nameidata(path, nd);
+ nd->inode = inode;
+
/* sayonara */
error = complete_walk(nd);
if (error)
if (error)
nd->flags |= LOOKUP_JUMPED;
+ BUG_ON(nd->flags & LOOKUP_RCU);
+ inode = path->dentry->d_inode;
error = -ENOENT;
- if (!path->dentry->d_inode)
- goto exit_dput;
+ if (!inode) {
+ path_to_nameidata(path, nd);
+ goto exit;
+ }
- if (path->dentry->d_inode->i_op->follow_link)
+ if (should_follow_link(inode, !symlink_ok)) {
+ if (nd->flags & LOOKUP_RCU) {
+ if (unlikely(unlazy_walk(nd, path->dentry))) {
+ error = -ECHILD;
+ goto exit;
+ }
+ }
+ BUG_ON(inode != path->dentry->d_inode);
return NULL;
+ }
path_to_nameidata(path, nd);
- nd->inode = path->dentry->d_inode;
+ nd->inode = inode;
/* Why this, you ask? _Now_ we might have grown LOOKUP_JUMPED... */
error = complete_walk(nd);
if (error)
return ERR_PTR(error);
error = -EISDIR;
- if (S_ISDIR(nd->inode->i_mode))
+ if ((open_flag & O_CREAT) && S_ISDIR(nd->inode->i_mode))
+ goto exit;
+ error = -ENOTDIR;
+ if ((nd->flags & LOOKUP_DIRECTORY) && !nd->inode->i_op->lookup)
goto exit;
+ audit_inode(pathname, nd->path.dentry);
ok:
if (!S_ISREG(nd->inode->i_mode))
will_truncate = 0;
out:
if (want_write)
mnt_drop_write(nd->path.mnt);
- path_put(&nd->path);
+ terminate_walk(nd);
return filp;
exit_mutex_unlock: