static void drive_stat_acct(struct request *rq, int new_io)
{
+ struct hd_struct *part;
int rw = rq_data_dir(rq);
if (!blk_fs_request(rq) || !rq->rq_disk)
return;
- if (!new_io) {
- __all_stat_inc(rq->rq_disk, merges[rw], rq->sector);
- } else {
- struct hd_struct *part = get_part(rq->rq_disk, rq->sector);
+ part = get_part(rq->rq_disk, rq->sector);
+ if (!new_io)
+ __all_stat_inc(rq->rq_disk, part, merges[rw], rq->sector);
+ else {
disk_round_stats(rq->rq_disk);
rq->rq_disk->in_flight++;
if (part) {
**/
void generic_unplug_device(struct request_queue *q)
{
- spin_lock_irq(q->queue_lock);
- __generic_unplug_device(q);
- spin_unlock_irq(q->queue_lock);
+ if (blk_queue_plugged(q)) {
+ spin_lock_irq(q->queue_lock);
+ __generic_unplug_device(q);
+ spin_unlock_irq(q->queue_lock);
+ }
}
EXPORT_SYMBOL(generic_unplug_device);
}
if (blk_fs_request(req) && req->rq_disk) {
+ struct hd_struct *part = get_part(req->rq_disk, req->sector);
const int rw = rq_data_dir(req);
- all_stat_add(req->rq_disk, sectors[rw],
- nr_bytes >> 9, req->sector);
+ all_stat_add(req->rq_disk, part, sectors[rw],
+ nr_bytes >> 9, req->sector);
}
total_bytes = bio_nbytes = 0;
const int rw = rq_data_dir(req);
struct hd_struct *part = get_part(disk, req->sector);
- __all_stat_inc(disk, ios[rw], req->sector);
- __all_stat_add(disk, ticks[rw], duration, req->sector);
+ __all_stat_inc(disk, part, ios[rw], req->sector);
+ __all_stat_add(disk, part, ticks[rw], duration, req->sector);
disk_round_stats(disk);
disk->in_flight--;
if (part) {
rcu_read_lock();
if (ioc->aic && ioc->aic->dtor)
ioc->aic->dtor(ioc->aic);
- rcu_read_unlock();
cfq_dtor(ioc);
+ rcu_read_unlock();
kmem_cache_free(iocontext_cachep, ioc);
return 1;
static int blk_hw_contig_segment(struct request_queue *q, struct bio *bio,
struct bio *nxt)
{
- if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
+ if (!bio_flagged(bio, BIO_SEG_VALID))
blk_recount_segments(q, bio);
- if (unlikely(!bio_flagged(nxt, BIO_SEG_VALID)))
+ if (!bio_flagged(nxt, BIO_SEG_VALID))
blk_recount_segments(q, nxt);
if (!BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt)) ||
BIOVEC_VIRT_OVERSIZE(bio->bi_hw_back_size + nxt->bi_hw_front_size))
q->last_merge = NULL;
return 0;
}
- if (unlikely(!bio_flagged(req->biotail, BIO_SEG_VALID)))
+ if (!bio_flagged(req->biotail, BIO_SEG_VALID))
blk_recount_segments(q, req->biotail);
- if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
+ if (!bio_flagged(bio, BIO_SEG_VALID))
blk_recount_segments(q, bio);
len = req->biotail->bi_hw_back_size + bio->bi_hw_front_size;
if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(req->biotail), __BVEC_START(bio))
return 0;
}
len = bio->bi_hw_back_size + req->bio->bi_hw_front_size;
- if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
+ if (!bio_flagged(bio, BIO_SEG_VALID))
blk_recount_segments(q, bio);
- if (unlikely(!bio_flagged(req->bio, BIO_SEG_VALID)))
+ if (!bio_flagged(req->bio, BIO_SEG_VALID))
blk_recount_segments(q, req->bio);
if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(req->bio)) &&
!BIOVEC_VIRT_OVERSIZE(len)) {
unsigned long nm;
ssize_t ret = queue_var_store(&nm, page, count);
+ spin_lock_irq(q->queue_lock);
if (nm)
- set_bit(QUEUE_FLAG_NOMERGES, &q->queue_flags);
+ queue_flag_set(QUEUE_FLAG_NOMERGES, q);
else
- clear_bit(QUEUE_FLAG_NOMERGES, &q->queue_flags);
+ queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
+ spin_unlock_irq(q->queue_lock);
return ret;
}
__blk_free_tags(bqt);
q->queue_tags = NULL;
- queue_flag_clear(QUEUE_FLAG_QUEUED, q);
+ queue_flag_clear_unlocked(QUEUE_FLAG_QUEUED, q);
}
/**
**/
void blk_queue_free_tags(struct request_queue *q)
{
- queue_flag_clear(QUEUE_FLAG_QUEUED, q);
+ queue_flag_clear_unlocked(QUEUE_FLAG_QUEUED, q);
}
EXPORT_SYMBOL(blk_queue_free_tags);
* @q: the request queue for the device
* @depth: the maximum queue depth supported
* @tags: the tag to use
+ *
+ * Queue lock must be held here if the function is called to resize an
+ * existing map.
**/
int blk_queue_init_tags(struct request_queue *q, int depth,
struct blk_queue_tag *tags)
* assign it, all done
*/
q->queue_tags = tags;
- queue_flag_set(QUEUE_FLAG_QUEUED, q);
+ queue_flag_set_unlocked(QUEUE_FLAG_QUEUED, q);
INIT_LIST_HEAD(&q->tag_busy_list);
return 0;
fail:
kmem_cache_free(cfq_pool, cfqq);
}
+static void
+__call_for_each_cic(struct io_context *ioc,
+ void (*func)(struct io_context *, struct cfq_io_context *))
+{
+ struct cfq_io_context *cic;
+ struct hlist_node *n;
+
+ hlist_for_each_entry_rcu(cic, n, &ioc->cic_list, cic_list)
+ func(ioc, cic);
+}
+
/*
* Call func for each cic attached to this ioc.
*/
call_for_each_cic(struct io_context *ioc,
void (*func)(struct io_context *, struct cfq_io_context *))
{
- struct cfq_io_context *cic;
- struct hlist_node *n;
-
rcu_read_lock();
- hlist_for_each_entry_rcu(cic, n, &ioc->cic_list, cic_list)
- func(ioc, cic);
+ __call_for_each_cic(ioc, func);
rcu_read_unlock();
}
* should be ok to iterate over the known list, we will see all cic's
* since no new ones are added.
*/
- call_for_each_cic(ioc, cic_free_func);
+ __call_for_each_cic(ioc, cic_free_func);
}
static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class);
case IOPRIO_CLASS_NONE:
/*
- * no prio set, place us in the middle of the BE classes
+ * no prio set, inherit CPU scheduling settings
*/
cfqq->ioprio = task_nice_ioprio(tsk);
- cfqq->ioprio_class = IOPRIO_CLASS_BE;
+ cfqq->ioprio_class = task_nice_ioclass(tsk);
break;
case IOPRIO_CLASS_RT:
cfqq->ioprio = task_ioprio(ioc);
{
unsigned long n_sect = bio->bi_size >> 9;
const int rw = bio_data_dir(bio);
+ struct hd_struct *part;
- all_stat_inc(disk, ios[rw], sector);
- all_stat_add(disk, ticks[rw], duration, sector);
- all_stat_add(disk, sectors[rw], n_sect, sector);
- all_stat_add(disk, io_ticks, duration, sector);
+ part = get_part(disk, sector);
+ all_stat_inc(disk, part, ios[rw], sector);
+ all_stat_add(disk, part, ticks[rw], duration, sector);
+ all_stat_add(disk, part, sectors[rw], n_sect, sector);
+ all_stat_add(disk, part, io_ticks, duration, sector);
}
void
bio_init(bio);
if (likely(nr_iovecs)) {
- unsigned long idx = 0; /* shut up gcc */
+ unsigned long uninitialized_var(idx);
bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
if (unlikely(!bvl)) {
* @data: pointer to buffer to copy
* @len: length in bytes
* @gfp_mask: allocation flags for bio and page allocation
+ * @reading: data direction is READ
*
* copy the kernel address into a bio suitable for io to a block
* device. Returns an error pointer in case of error.
{
struct address_space *mapping = out->f_mapping;
struct inode *inode = mapping->host;
- int killsuid, killpriv;
+ struct splice_desc sd = {
+ .total_len = len,
+ .flags = flags,
+ .pos = *ppos,
+ .u.file = out,
+ };
ssize_t ret;
- int err = 0;
-
- killpriv = security_inode_need_killpriv(out->f_path.dentry);
- killsuid = should_remove_suid(out->f_path.dentry);
- if (unlikely(killsuid || killpriv)) {
- mutex_lock(&inode->i_mutex);
- if (killpriv)
- err = security_inode_killpriv(out->f_path.dentry);
- if (!err && killsuid)
- err = __remove_suid(out->f_path.dentry, killsuid);
- mutex_unlock(&inode->i_mutex);
- if (err)
- return err;
- }
- ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
+ inode_double_lock(inode, pipe->inode);
+ ret = remove_suid(out->f_path.dentry);
+ if (likely(!ret))
+ ret = __splice_from_pipe(pipe, &sd, pipe_to_file);
+ inode_double_unlock(inode, pipe->inode);
if (ret > 0) {
unsigned long nr_pages;
* sync it.
*/
if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
+ int err;
+
mutex_lock(&inode->i_mutex);
err = generic_osync_inode(inode, mapping,
OSYNC_METADATA|OSYNC_DATA);
ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
if (ret > 0)
- *ppos = sd.pos;
+ *ppos += ret;
return ret;
}
extern void clear_inode(struct inode *);
extern void destroy_inode(struct inode *);
extern struct inode *new_inode(struct super_block *);
-extern int __remove_suid(struct dentry *, int);
extern int should_remove_suid(struct dentry *);
extern int remove_suid(struct dentry *);
static inline void disk_stat_set_all(struct gendisk *gendiskp, int value) {
int i;
+
for_each_possible_cpu(i)
memset(per_cpu_ptr(gendiskp->dkstats, i), value,
- sizeof (struct disk_stats));
+ sizeof(struct disk_stats));
}
#define __part_stat_add(part, field, addnd) \
(per_cpu_ptr(part->dkstats, smp_processor_id())->field += addnd)
-#define __all_stat_add(gendiskp, field, addnd, sector) \
+#define __all_stat_add(gendiskp, part, field, addnd, sector) \
({ \
- struct hd_struct *part = get_part(gendiskp, sector); \
if (part) \
__part_stat_add(part, field, addnd); \
__disk_stat_add(gendiskp, field, addnd); \
res; \
})
-static inline void part_stat_set_all(struct hd_struct *part, int value) {
+static inline void part_stat_set_all(struct hd_struct *part, int value)
+{
int i;
+
for_each_possible_cpu(i)
memset(per_cpu_ptr(part->dkstats, i), value,
- sizeof(struct disk_stats));
+ sizeof(struct disk_stats));
}
#else /* !CONFIG_SMP */
#define __part_stat_add(part, field, addnd) \
(part->dkstats.field += addnd)
-#define __all_stat_add(gendiskp, field, addnd, sector) \
+#define __all_stat_add(gendiskp, part, field, addnd, sector) \
({ \
- struct hd_struct *part = get_part(gendiskp, sector); \
if (part) \
part->dkstats.field += addnd; \
__disk_stat_add(gendiskp, field, addnd); \
#define part_stat_sub(gendiskp, field, subnd) \
part_stat_add(gendiskp, field, -subnd)
-#define all_stat_add(gendiskp, field, addnd, sector) \
+#define all_stat_add(gendiskp, part, field, addnd, sector) \
do { \
preempt_disable(); \
- __all_stat_add(gendiskp, field, addnd, sector); \
+ __all_stat_add(gendiskp, part, field, addnd, sector); \
preempt_enable(); \
} while (0)
#define all_stat_dec(gendiskp, field, sector) \
all_stat_add(gendiskp, field, -1, sector)
-#define __all_stat_inc(gendiskp, field, sector) \
- __all_stat_add(gendiskp, field, 1, sector)
-#define all_stat_inc(gendiskp, field, sector) \
- all_stat_add(gendiskp, field, 1, sector)
+#define __all_stat_inc(gendiskp, part, field, sector) \
+ __all_stat_add(gendiskp, part, field, 1, sector)
+#define all_stat_inc(gendiskp, part, field, sector) \
+ all_stat_add(gendiskp, part, field, 1, sector)
-#define __all_stat_sub(gendiskp, field, subnd, sector) \
- __all_stat_add(gendiskp, field, -subnd, sector)
-#define all_stat_sub(gendiskp, field, subnd, sector) \
- all_stat_add(gendiskp, field, -subnd, sector)
+#define __all_stat_sub(gendiskp, part, field, subnd, sector) \
+ __all_stat_add(gendiskp, part, field, -subnd, sector)
+#define all_stat_sub(gendiskp, part, field, subnd, sector) \
+ all_stat_add(gendiskp, part, field, -subnd, sector)
/* Inlines to alloc and free disk stats in struct gendisk */
#ifdef CONFIG_SMP
}
/*
+ * This is for the case where the task hasn't asked for a specific IO class.
+ * Check for idle and rt task process, and return appropriate IO class.
+ */
+static inline int task_nice_ioclass(struct task_struct *task)
+{
+ if (task->policy == SCHED_IDLE)
+ return IOPRIO_CLASS_IDLE;
+ else if (task->policy == SCHED_FIFO || task->policy == SCHED_RR)
+ return IOPRIO_CLASS_RT;
+ else
+ return IOPRIO_CLASS_BE;
+}
+
+/*
* For inheritance, return the highest of the two given priorities
*/
extern int ioprio_best(unsigned short aprio, unsigned short bprio);
ret = 0;
spliced = 0;
- while (len && !spliced) {
+ while (len) {
ret = subbuf_splice_actor(in, ppos, pipe, len, flags, &nonpad_ret);
if (ret < 0)
break;
}
EXPORT_SYMBOL(should_remove_suid);
-int __remove_suid(struct dentry *dentry, int kill)
+static int __remove_suid(struct dentry *dentry, int kill)
{
struct iattr newattrs;