mntget(nd->path.mnt);
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
nd->flags &= ~LOOKUP_RCU;
return 0;
if (unlikely(!__d_rcu_to_refcount(dentry, nd->seq))) {
spin_unlock(&dentry->d_lock);
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
return -ECHILD;
}
BUG_ON(nd->inode != dentry->d_inode);
spin_unlock(&dentry->d_lock);
mntget(nd->path.mnt);
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
}
if (likely(!(nd->flags & LOOKUP_JUMPED)))
static inline void put_link(struct nameidata *nd, struct path *link, void *cookie)
{
struct inode *inode = link->dentry->d_inode;
- if (!IS_ERR(cookie) && inode->i_op->put_link)
+ if (inode->i_op->put_link)
inode->i_op->put_link(link->dentry, nd, cookie);
path_put(link);
}
static __always_inline int
follow_link(struct path *link, struct nameidata *nd, void **p)
{
- int error;
struct dentry *dentry = link->dentry;
+ int error;
+ char *s;
BUG_ON(nd->flags & LOOKUP_RCU);
if (link->mnt == nd->path.mnt)
mntget(link->mnt);
- if (unlikely(current->total_link_count >= 40)) {
- *p = ERR_PTR(-ELOOP); /* no ->put_link(), please */
- path_put(&nd->path);
- return -ELOOP;
- }
+ error = -ELOOP;
+ if (unlikely(current->total_link_count >= 40))
+ goto out_put_nd_path;
+
cond_resched();
current->total_link_count++;
nd_set_link(nd, NULL);
error = security_inode_follow_link(link->dentry, nd);
- if (error) {
- *p = ERR_PTR(error); /* no ->put_link(), please */
- path_put(&nd->path);
- return error;
- }
+ if (error)
+ goto out_put_nd_path;
nd->last_type = LAST_BIND;
*p = dentry->d_inode->i_op->follow_link(dentry, nd);
error = PTR_ERR(*p);
- if (!IS_ERR(*p)) {
- char *s = nd_get_link(nd);
- error = 0;
- if (s)
- error = __vfs_follow_link(nd, s);
- else if (nd->last_type == LAST_BIND) {
- nd->flags |= LOOKUP_JUMPED;
- nd->inode = nd->path.dentry->d_inode;
- if (nd->inode->i_op->follow_link) {
- /* stepped on a _really_ weird one */
- path_put(&nd->path);
- error = -ELOOP;
- }
+ if (IS_ERR(*p))
+ goto out_put_link;
+
+ error = 0;
+ s = nd_get_link(nd);
+ if (s) {
+ error = __vfs_follow_link(nd, s);
+ } else if (nd->last_type == LAST_BIND) {
+ nd->flags |= LOOKUP_JUMPED;
+ nd->inode = nd->path.dentry->d_inode;
+ if (nd->inode->i_op->follow_link) {
+ /* stepped on a _really_ weird one */
+ path_put(&nd->path);
+ error = -ELOOP;
}
}
+ if (unlikely(error))
+ put_link(nd, link, *p);
+
+ return error;
+
+out_put_nd_path:
+ path_put(&nd->path);
+out_put_link:
+ path_put(link);
return error;
}
struct mount *parent;
struct dentry *mountpoint;
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
parent = mnt->mnt_parent;
if (&parent->mnt == path->mnt) {
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
return 0;
}
mntget(&parent->mnt);
mountpoint = dget(mnt->mnt_mountpoint);
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
dput(path->dentry);
path->dentry = mountpoint;
mntput(path->mnt);
if (!(nd->flags & LOOKUP_ROOT))
nd->root.mnt = NULL;
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
return -ECHILD;
}
* small and for now I'd prefer to have fast path as straight as possible.
* It _is_ time-critical.
*/
-static int do_lookup(struct nameidata *nd, struct qstr *name,
- struct path *path, struct inode **inode)
+static int lookup_fast(struct nameidata *nd, struct qstr *name,
+ struct path *path, struct inode **inode)
{
struct vfsmount *mnt = nd->path.mnt;
struct dentry *dentry, *parent = nd->path.dentry;
goto need_lookup;
}
}
-done:
+
path->mnt = mnt;
path->dentry = dentry;
err = follow_managed(path, nd->flags);
return 0;
need_lookup:
+ return 1;
+}
+
+/* Fast lookup failed, do it the slow way */
+static int lookup_slow(struct nameidata *nd, struct qstr *name,
+ struct path *path)
+{
+ struct dentry *dentry, *parent;
+ int err;
+
+ parent = nd->path.dentry;
BUG_ON(nd->inode != parent->d_inode);
mutex_lock(&parent->d_inode->i_mutex);
mutex_unlock(&parent->d_inode->i_mutex);
if (IS_ERR(dentry))
return PTR_ERR(dentry);
- goto done;
+ path->mnt = nd->path.mnt;
+ path->dentry = dentry;
+ err = follow_managed(path, nd->flags);
+ if (unlikely(err < 0)) {
+ path_put_conditional(path, nd);
+ return err;
+ }
+ if (err)
+ nd->flags |= LOOKUP_JUMPED;
+ return 0;
}
static inline int may_lookup(struct nameidata *nd)
if (!(nd->flags & LOOKUP_ROOT))
nd->root.mnt = NULL;
rcu_read_unlock();
- br_read_unlock(vfsmount_lock);
+ br_read_unlock(&vfsmount_lock);
}
}
*/
if (unlikely(type != LAST_NORM))
return handle_dots(nd, type);
- err = do_lookup(nd, name, path, &inode);
+ err = lookup_fast(nd, name, path, &inode);
if (unlikely(err)) {
- terminate_walk(nd);
- return err;
- }
- if (!inode) {
- path_to_nameidata(path, nd);
- terminate_walk(nd);
- return -ENOENT;
+ if (err < 0)
+ goto out_err;
+
+ err = lookup_slow(nd, name, path);
+ if (err < 0)
+ goto out_err;
+
+ inode = path->dentry->d_inode;
}
+ err = -ENOENT;
+ if (!inode)
+ goto out_path_put;
+
if (should_follow_link(inode, follow)) {
if (nd->flags & LOOKUP_RCU) {
if (unlikely(unlazy_walk(nd, path->dentry))) {
- terminate_walk(nd);
- return -ECHILD;
+ err = -ECHILD;
+ goto out_err;
}
}
BUG_ON(inode != path->dentry->d_inode);
path_to_nameidata(path, nd);
nd->inode = inode;
return 0;
+
+out_path_put:
+ path_to_nameidata(path, nd);
+out_err:
+ terminate_walk(nd);
+ return err;
}
/*
void *cookie;
res = follow_link(&link, nd, &cookie);
- if (!res)
- res = walk_component(nd, path, &nd->last,
- nd->last_type, LOOKUP_FOLLOW);
+ if (res)
+ break;
+ res = walk_component(nd, path, &nd->last,
+ nd->last_type, LOOKUP_FOLLOW);
put_link(nd, &link, cookie);
} while (res > 0);
nd->path = nd->root;
nd->inode = inode;
if (flags & LOOKUP_RCU) {
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
} else {
if (*name=='/') {
if (flags & LOOKUP_RCU) {
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
set_root_rcu(nd);
} else {
struct fs_struct *fs = current->fs;
unsigned seq;
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
do {
if (fput_needed)
*fp = file;
nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
- br_read_lock(vfsmount_lock);
+ br_read_lock(&vfsmount_lock);
rcu_read_lock();
} else {
path_get(&file->f_path);
struct path link = path;
nd->flags |= LOOKUP_PARENT;
err = follow_link(&link, nd, &cookie);
- if (!err)
- err = lookup_last(nd, &path);
+ if (err)
+ break;
+ err = lookup_last(nd, &path);
put_link(nd, &link, cookie);
}
}
}
/*
+ * Lookup, maybe create and open the last component
+ *
+ * Must be called with i_mutex held on parent.
+ *
+ * Returns open file or NULL on success, error otherwise. NULL means no open
+ * was performed, only lookup.
+ */
+static struct file *lookup_open(struct nameidata *nd, struct path *path,
+ const struct open_flags *op,
+ int *want_write, bool *created)
+{
+ struct dentry *dir = nd->path.dentry;
+ struct dentry *dentry;
+ int error;
+
+ *created = false;
+ dentry = lookup_hash(nd);
+ if (IS_ERR(dentry))
+ return ERR_CAST(dentry);
+
+ /* Negative dentry, just create the file */
+ if (!dentry->d_inode && (op->open_flag & O_CREAT)) {
+ umode_t mode = op->mode;
+ if (!IS_POSIXACL(dir->d_inode))
+ mode &= ~current_umask();
+ /*
+ * This write is needed to ensure that a
+ * rw->ro transition does not occur between
+ * the time when the file is created and when
+ * a permanent write count is taken through
+ * the 'struct file' in nameidata_to_filp().
+ */
+ error = mnt_want_write(nd->path.mnt);
+ if (error)
+ goto out_dput;
+ *want_write = 1;
+ *created = true;
+ error = security_path_mknod(&nd->path, dentry, mode, 0);
+ if (error)
+ goto out_dput;
+ error = vfs_create(dir->d_inode, dentry, mode, nd);
+ if (error)
+ goto out_dput;
+ }
+ path->dentry = dentry;
+ path->mnt = nd->path.mnt;
+ return NULL;
+
+out_dput:
+ dput(dentry);
+ return ERR_PTR(error);
+}
+
+/*
* Handle the last step of open()
*/
static struct file *do_last(struct nameidata *nd, struct path *path,
const struct open_flags *op, const char *pathname)
{
struct dentry *dir = nd->path.dentry;
- struct dentry *dentry;
int open_flag = op->open_flag;
int will_truncate = open_flag & O_TRUNC;
int want_write = 0;
int acc_mode = op->acc_mode;
struct file *filp;
+ struct inode *inode;
+ bool created;
+ int symlink_ok = 0;
+ struct path save_parent = { .dentry = NULL, .mnt = NULL };
+ bool retried = false;
int error;
nd->flags &= ~LOOKUP_PARENT;
}
if (!(open_flag & O_CREAT)) {
- int symlink_ok = 0;
if (nd->last.name[nd->last.len])
nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
if (open_flag & O_PATH && !(nd->flags & LOOKUP_FOLLOW))
symlink_ok = 1;
/* we _can_ be in RCU mode here */
- error = walk_component(nd, path, &nd->last, LAST_NORM,
- !symlink_ok);
+ error = lookup_fast(nd, &nd->last, path, &inode);
+ if (likely(!error))
+ goto finish_lookup;
+
if (error < 0)
- return ERR_PTR(error);
- if (error) /* symlink */
- return NULL;
- /* sayonara */
+ goto exit;
+
+ BUG_ON(nd->inode != dir->d_inode);
+ } else {
+ /* create side of things */
+ /*
+ * This will *only* deal with leaving RCU mode - LOOKUP_JUMPED
+ * has been cleared when we got to the last component we are
+ * about to look up
+ */
error = complete_walk(nd);
if (error)
return ERR_PTR(error);
- error = -ENOTDIR;
- if (nd->flags & LOOKUP_DIRECTORY) {
- if (!nd->inode->i_op->lookup)
- goto exit;
- }
- audit_inode(pathname, nd->path.dentry);
- goto ok;
+ audit_inode(pathname, dir);
+ error = -EISDIR;
+ /* trailing slashes? */
+ if (nd->last.name[nd->last.len])
+ goto exit;
}
- /* create side of things */
- /*
- * This will *only* deal with leaving RCU mode - LOOKUP_JUMPED has been
- * cleared when we got to the last component we are about to look up
- */
- error = complete_walk(nd);
- if (error)
- return ERR_PTR(error);
-
- audit_inode(pathname, dir);
- error = -EISDIR;
- /* trailing slashes? */
- if (nd->last.name[nd->last.len])
- goto exit;
-
+retry_lookup:
mutex_lock(&dir->d_inode->i_mutex);
+ filp = lookup_open(nd, path, op, &want_write, &created);
+ mutex_unlock(&dir->d_inode->i_mutex);
- dentry = lookup_hash(nd);
- error = PTR_ERR(dentry);
- if (IS_ERR(dentry)) {
- mutex_unlock(&dir->d_inode->i_mutex);
- goto exit;
- }
-
- path->dentry = dentry;
- path->mnt = nd->path.mnt;
+ if (IS_ERR(filp))
+ goto out;
- /* Negative dentry, just create the file */
- if (!dentry->d_inode) {
- umode_t mode = op->mode;
- if (!IS_POSIXACL(dir->d_inode))
- mode &= ~current_umask();
- /*
- * This write is needed to ensure that a
- * rw->ro transition does not occur between
- * the time when the file is created and when
- * a permanent write count is taken through
- * the 'struct file' in nameidata_to_filp().
- */
- error = mnt_want_write(nd->path.mnt);
- if (error)
- goto exit_mutex_unlock;
- want_write = 1;
+ if (created) {
/* Don't check for write permission, don't truncate */
open_flag &= ~O_TRUNC;
will_truncate = 0;
acc_mode = MAY_OPEN;
- error = security_path_mknod(&nd->path, dentry, mode, 0);
- if (error)
- goto exit_mutex_unlock;
- error = vfs_create(dir->d_inode, dentry, mode, nd);
- if (error)
- goto exit_mutex_unlock;
- mutex_unlock(&dir->d_inode->i_mutex);
- dput(nd->path.dentry);
- nd->path.dentry = dentry;
+ path_to_nameidata(path, nd);
goto common;
}
/*
* It already exists.
*/
- mutex_unlock(&dir->d_inode->i_mutex);
audit_inode(pathname, path->dentry);
error = -EEXIST;
if (error)
nd->flags |= LOOKUP_JUMPED;
+ BUG_ON(nd->flags & LOOKUP_RCU);
+ inode = path->dentry->d_inode;
+finish_lookup:
+ /* we _can_ be in RCU mode here */
error = -ENOENT;
- if (!path->dentry->d_inode)
- goto exit_dput;
+ if (!inode) {
+ path_to_nameidata(path, nd);
+ goto exit;
+ }
- if (path->dentry->d_inode->i_op->follow_link)
+ if (should_follow_link(inode, !symlink_ok)) {
+ if (nd->flags & LOOKUP_RCU) {
+ if (unlikely(unlazy_walk(nd, path->dentry))) {
+ error = -ECHILD;
+ goto exit;
+ }
+ }
+ BUG_ON(inode != path->dentry->d_inode);
return NULL;
+ }
- path_to_nameidata(path, nd);
- nd->inode = path->dentry->d_inode;
+ if ((nd->flags & LOOKUP_RCU) || nd->path.mnt != path->mnt) {
+ path_to_nameidata(path, nd);
+ } else {
+ save_parent.dentry = nd->path.dentry;
+ save_parent.mnt = mntget(path->mnt);
+ nd->path.dentry = path->dentry;
+
+ }
+ nd->inode = inode;
/* Why this, you ask? _Now_ we might have grown LOOKUP_JUMPED... */
error = complete_walk(nd);
- if (error)
+ if (error) {
+ path_put(&save_parent);
return ERR_PTR(error);
+ }
error = -EISDIR;
- if (S_ISDIR(nd->inode->i_mode))
+ if ((open_flag & O_CREAT) && S_ISDIR(nd->inode->i_mode))
goto exit;
+ error = -ENOTDIR;
+ if ((nd->flags & LOOKUP_DIRECTORY) && !nd->inode->i_op->lookup)
+ goto exit;
+ audit_inode(pathname, nd->path.dentry);
ok:
if (!S_ISREG(nd->inode->i_mode))
will_truncate = 0;
if (error)
goto exit;
filp = nameidata_to_filp(nd);
+ if (filp == ERR_PTR(-EOPENSTALE) && save_parent.dentry && !retried) {
+ BUG_ON(save_parent.dentry != dir);
+ path_put(&nd->path);
+ nd->path = save_parent;
+ nd->inode = dir->d_inode;
+ save_parent.mnt = NULL;
+ save_parent.dentry = NULL;
+ if (want_write) {
+ mnt_drop_write(nd->path.mnt);
+ want_write = 0;
+ }
+ retried = true;
+ goto retry_lookup;
+ }
if (!IS_ERR(filp)) {
error = ima_file_check(filp, op->acc_mode);
if (error) {
out:
if (want_write)
mnt_drop_write(nd->path.mnt);
- path_put(&nd->path);
+ path_put(&save_parent);
+ terminate_walk(nd);
return filp;
-exit_mutex_unlock:
- mutex_unlock(&dir->d_inode->i_mutex);
exit_dput:
path_put_conditional(path, nd);
exit:
nd->flags &= ~(LOOKUP_OPEN|LOOKUP_CREATE|LOOKUP_EXCL);
error = follow_link(&link, nd, &cookie);
if (unlikely(error))
- filp = ERR_PTR(error);
- else
- filp = do_last(nd, &path, op, pathname);
+ goto out_filp;
+ filp = do_last(nd, &path, op, pathname);
put_link(nd, &link, cookie);
}
out:
if (base)
fput(base);
release_open_intent(nd);
+ if (filp == ERR_PTR(-EOPENSTALE)) {
+ if (flags & LOOKUP_RCU)
+ filp = ERR_PTR(-ECHILD);
+ else
+ filp = ERR_PTR(-ESTALE);
+ }
return filp;
out_filp: