From 327108afb2279bb14ffb39a6da5195599b59e47f Mon Sep 17 00:00:00 2001 From: Arianna Avanzini Date: Wed, 5 Sep 2012 01:40:41 +0200 Subject: [PATCH] block: Switch from BFQ-v5r1 for 3.5.0 to BFQ-v6 for 3.5.0. Improvements: - Replacement of the cooperating-queue merging mechanism borrowed from CFQ with Early Queue Merge (EQM), a unified mechanism to get a sequential read pattern, and hence a high throughput with any set of processes performing interleaved I/O. The main idea is checking newly arrived requests against the next request of the active queue both in case of actual request insert and in case of request merge. EQM also preserves the typical low-latency properties of BFQ, by properly restoring the weight-raising state of a queue when it gets back to a non-merged state (see http://algo.ing.unimo.it/people/paolo/disk_sched/description.php for more details). The code for detecting whether two queues have to be merged is a slightly modified version of the CFQ code for detecting whether two queues belong to cooperating processes and whether the service of a queue should be preempted to boost the throughput. - Fix a bug that caused the peak rate of a disk to be computed as zero in case of multiple I/O errors. Subsequent estimations of the weight raising duration caused a division-by-zero error. Signed-off-by: Mauro Andreolini Signed-off-by: Arianna Avanzini Reviewed-by: Paolo Valente --- block/bfq-cgroup.c | 2 +- block/bfq-iosched.c | 600 ++++++++++++++++++++++++++++++++++----------------- block/bfq-sched.c | 28 --- block/bfq.h | 16 ++ 4 files changed, 422 insertions(+), 224 deletions(-) diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index e8efb03..49d74a0 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -694,7 +694,7 @@ static struct cgroup_subsys_state *bfqio_create(struct cgroup *cgroup) } /* - * We cannot support shared io contexts, as we have no mean to support + * We cannot support shared io contexts, as we have no means to support * two tasks with the same ioc in two different groups without major rework * of the main bic/bfqq data structures. By now we allow a task to change * its cgroup only if it's the only owner of its ioc; the drawback of this diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index aa9c55f..c9d57b0 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -444,6 +444,37 @@ static inline unsigned int bfq_wrais_duration(struct bfq_data *bfqd) return dur; } +static inline void +bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_io_cq *bic) +{ + if (bic->saved_idle_window) + bfq_mark_bfqq_idle_window(bfqq); + else + bfq_clear_bfqq_idle_window(bfqq); + if (bic->raising_time_left) { + /* + * Start a weight raising period with the duration given by + * the raising_time_left snapshot. + */ + bfqq->raising_coeff = bfqq->bfqd->bfq_raising_coeff; + bfqq->raising_cur_max_time = bic->raising_time_left; + bfqq->last_rais_start_finish = jiffies; + } +} + +/* + * Must be called with the queue_lock held. + */ +static int bfqq_process_refs(struct bfq_queue *bfqq) +{ + int process_refs, io_refs; + + io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE]; + process_refs = atomic_read(&bfqq->ref) - io_refs - bfqq->entity.on_st; + BUG_ON(process_refs < 0); + return process_refs; +} + static void bfq_add_rq_rb(struct request *rq) { struct bfq_queue *bfqq = RQ_BFQQ(rq); @@ -483,11 +514,20 @@ static void bfq_add_rq_rb(struct request *rq) if (! bfqd->low_latency) goto add_bfqq_busy; + if (bfq_bfqq_just_split(bfqq)) + goto set_ioprio_changed; + /* - * If the queue is not being boosted and has been idle - * for enough time, start a weight-raising period + * If the queue: + * - is not being boosted, + * - has been idle for enough time, + * - is not a sync queue or is linked to a bfq_io_cq (it is + * shared "for its nature" or it is not shared and its + * requests have not been redirected to a shared queue) + * start a weight-raising period. */ - if(old_raising_coeff == 1 && (idle_for_long_time || soft_rt)) { + if(old_raising_coeff == 1 && (idle_for_long_time || soft_rt) && + (!bfq_bfqq_sync(bfqq) || bfqq->bic != NULL)) { bfqq->raising_coeff = bfqd->bfq_raising_coeff; if (idle_for_long_time) bfqq->raising_cur_max_time = @@ -517,6 +557,7 @@ static void bfq_add_rq_rb(struct request *rq) raising_cur_max_time)); } } +set_ioprio_changed: if (old_raising_coeff != bfqq->raising_coeff) entity->ioprio_changed = 1; add_bfqq_busy: @@ -657,89 +698,35 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq, bfq_remove_request(next); } -static int bfq_allow_merge(struct request_queue *q, struct request *rq, - struct bio *bio) +static inline sector_t bfq_io_struct_pos(void *io_struct, bool request) { - struct bfq_data *bfqd = q->elevator->elevator_data; - struct bfq_io_cq *bic; - struct bfq_queue *bfqq; - - /* - * Disallow merge of a sync bio into an async request. - */ - if (bfq_bio_sync(bio) && !rq_is_sync(rq)) - return 0; - - /* - * Lookup the bfqq that this bio will be queued with. Allow - * merge only if rq is queued there. - * Queue lock is held here. - */ - bic = bfq_bic_lookup(bfqd, current->io_context); - if (bic == NULL) - return 0; - - bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); - return bfqq == RQ_BFQQ(rq); -} - -static void __bfq_set_active_queue(struct bfq_data *bfqd, - struct bfq_queue *bfqq) -{ - if (bfqq != NULL) { - bfq_mark_bfqq_must_alloc(bfqq); - bfq_mark_bfqq_budget_new(bfqq); - bfq_clear_bfqq_fifo_expire(bfqq); - - bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8; - - bfq_log_bfqq(bfqd, bfqq, "set_active_queue, cur-budget = %lu", - bfqq->entity.budget); - } - - bfqd->active_queue = bfqq; -} - -/* - * Get and set a new active queue for service. - */ -static struct bfq_queue *bfq_set_active_queue(struct bfq_data *bfqd, - struct bfq_queue *bfqq) -{ - if (!bfqq) - bfqq = bfq_get_next_queue(bfqd); + if (request) + return blk_rq_pos(io_struct); else - bfq_get_next_queue_forced(bfqd, bfqq); - - __bfq_set_active_queue(bfqd, bfqq); - return bfqq; + return ((struct bio *)io_struct)->bi_sector; } -static inline sector_t bfq_dist_from_last(struct bfq_data *bfqd, - struct request *rq) +static inline sector_t bfq_dist_from(sector_t pos1, + sector_t pos2) { - if (blk_rq_pos(rq) >= bfqd->last_position) - return blk_rq_pos(rq) - bfqd->last_position; + if (pos1 >= pos2) + return pos1 - pos2; else - return bfqd->last_position - blk_rq_pos(rq); + return pos2 - pos1; } -/* - * Return true if bfqq has no request pending and rq is close enough to - * bfqd->last_position, or if rq is closer to bfqd->last_position than - * bfqq->next_rq - */ -static inline int bfq_rq_close(struct bfq_data *bfqd, struct request *rq) +static inline int bfq_rq_close_to_sector(void *io_struct, bool request, + sector_t sector) { - return bfq_dist_from_last(bfqd, rq) <= BFQQ_SEEK_THR; + return bfq_dist_from(bfq_io_struct_pos(io_struct, request), sector) <= + BFQQ_SEEK_THR; } -static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) +static struct bfq_queue *bfqq_close(struct bfq_data *bfqd, sector_t sector) { struct rb_root *root = &bfqd->rq_pos_tree; struct rb_node *parent, *node; struct bfq_queue *__bfqq; - sector_t sector = bfqd->last_position; if (RB_EMPTY_ROOT(root)) return NULL; @@ -758,7 +745,7 @@ static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) * position). */ __bfqq = rb_entry(parent, struct bfq_queue, pos_node); - if (bfq_rq_close(bfqd, __bfqq->next_rq)) + if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) return __bfqq; if (blk_rq_pos(__bfqq->next_rq) < sector) @@ -769,7 +756,7 @@ static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) return NULL; __bfqq = rb_entry(node, struct bfq_queue, pos_node); - if (bfq_rq_close(bfqd, __bfqq->next_rq)) + if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) return __bfqq; return NULL; @@ -778,14 +765,12 @@ static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) /* * bfqd - obvious * cur_bfqq - passed in so that we don't decide that the current queue - * is closely cooperating with itself. - * - * We are assuming that cur_bfqq has dispatched at least one request, - * and that bfqd->last_position reflects a position on the disk associated - * with the I/O issued by cur_bfqq. + * is closely cooperating with itself + * sector - used as a reference point to search for a close queue */ static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd, - struct bfq_queue *cur_bfqq) + struct bfq_queue *cur_bfqq, + sector_t sector) { struct bfq_queue *bfqq; @@ -805,7 +790,7 @@ static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd, * working closely on the same area of the disk. In that case, * we can group them together and don't waste time idling. */ - bfqq = bfqq_close(bfqd); + bfqq = bfqq_close(bfqd, sector); if (bfqq == NULL || bfqq == cur_bfqq) return NULL; @@ -832,6 +817,267 @@ static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd, return bfqq; } +static struct bfq_queue * +bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) +{ + int process_refs, new_process_refs; + struct bfq_queue *__bfqq; + + /* + * If there are no process references on the new_bfqq, then it is + * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain + * may have dropped their last reference (not just their last process + * reference). + */ + if (!bfqq_process_refs(new_bfqq)) + return NULL; + + /* Avoid a circular list and skip interim queue merges. */ + while ((__bfqq = new_bfqq->new_bfqq)) { + if (__bfqq == bfqq) + return NULL; + new_bfqq = __bfqq; + } + + process_refs = bfqq_process_refs(bfqq); + new_process_refs = bfqq_process_refs(new_bfqq); + /* + * If the process for the bfqq has gone away, there is no + * sense in merging the queues. + */ + if (process_refs == 0 || new_process_refs == 0) + return NULL; + + bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", + new_bfqq->pid); + + /* + * Merging is just a redirection: the requests of the process owning + * one of the two queues are redirected to the other queue. The latter + * queue, in its turn, is set as shared if this is the first time that + * the requests of some process are redirected to it. + * + * We redirect bfqq to new_bfqq and not the opposite, because we + * are in the context of the process owning bfqq, hence we have the + * io_cq of this process. So we can immediately configure this io_cq + * to redirect the requests of the process to new_bfqq. + * + * NOTE, even if new_bfqq coincides with the active queue, the io_cq of + * new_bfqq is not available, because, if the active queue is shared, + * bfqd->active_bic may not point to the io_cq of the active queue. + * Redirecting the requests of the process owning bfqq to the currently + * active queue is in any case the best option, as we feed the active queue + * with new requests close to the last request served and, by doing so, + * hopefully increase the throughput. + */ + bfqq->new_bfqq = new_bfqq; + atomic_add(process_refs, &new_bfqq->ref); + return new_bfqq; +} + +/* + * Attempt to schedule a merge of bfqq with the currently active queue or + * with a close queue among the scheduled queues. + * Return NULL if no merge was scheduled, a pointer to the shared bfq_queue + * structure otherwise. + */ +static struct bfq_queue * +bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + void *io_struct, bool request) +{ + struct bfq_queue *active_bfqq, *new_bfqq; + + if (bfqq->new_bfqq) + return bfqq->new_bfqq; + + if (!io_struct) + return NULL; + + active_bfqq = bfqd->active_queue; + + if (active_bfqq == NULL || active_bfqq == bfqq || !bfqd->active_bic) + goto check_scheduled; + + if (bfq_class_idle(active_bfqq) || bfq_class_idle(bfqq)) + goto check_scheduled; + + if (bfq_class_rt(active_bfqq) != bfq_class_rt(bfqq)) + goto check_scheduled; + + if (active_bfqq->entity.parent != bfqq->entity.parent) + goto check_scheduled; + + if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && + bfq_bfqq_sync(active_bfqq) && bfq_bfqq_sync(bfqq)) + if ((new_bfqq = bfq_setup_merge(bfqq, active_bfqq))) + return new_bfqq; /* Merge with the active queue */ + + /* + * Check whether there is a cooperator among currently scheduled + * queues. The only thing we need is that the bio/request is not + * NULL, as we need it to establish whether a cooperator exists. + */ +check_scheduled: + new_bfqq = bfq_close_cooperator(bfqd, bfqq, + bfq_io_struct_pos(io_struct, request)); + if (new_bfqq) + return bfq_setup_merge(bfqq, new_bfqq); + + return NULL; +} + +static inline void +bfq_bfqq_save_state(struct bfq_queue *bfqq) +{ + /* + * If bfqq->bic == NULL, the queue is already shared or its requests + * have already been redirected to a shared queue; both idle window + * and weight raising state have already been saved. Do nothing. + */ + if (bfqq->bic == NULL) + return; + if (bfqq->raising_coeff > 1) { + unsigned long wrais_duration = + jiffies - bfqq->last_rais_start_finish; + /* + * It may happen that a queue's weight raising period lasts + * longer than its raising_cur_max_time, as weight raising is + * handled only when a request is enqueued or dispatched (it + * does not use any timer). If the weight raising period is + * about to end, don't save it. + */ + if (bfqq->raising_cur_max_time <= wrais_duration) + bfqq->bic->raising_time_left = 0; + else + bfqq->bic->raising_time_left = + bfqq->raising_cur_max_time - wrais_duration; + /* + * The bfq_queue is becoming shared or the requests of the + * process owning the queue are being redirected to a shared + * queue. Stop the weight raising period of the queue, as in + * both cases it should not be owned by an interactive or soft + * real-time application. + */ + bfqq->raising_coeff = 1; + bfqq->entity.ioprio_changed = 1; + } else + bfqq->bic->raising_time_left = 0; + bfqq->bic->saved_idle_window = bfq_bfqq_idle_window(bfqq); +} + +static inline void +bfq_get_bic_reference(struct bfq_queue *bfqq) +{ + /* + * If bfqq->bic has a non-NULL value, the bic to which it belongs + * is about to begin using a shared bfq_queue. + */ + if (bfqq->bic) + atomic_long_inc(&bfqq->bic->icq.ioc->refcount); +} + +static void +bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, + struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) +{ + bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", + (long unsigned)new_bfqq->pid); + /* Save weight raising and idle window of the merged queues */ + bfq_bfqq_save_state(bfqq); + bfq_bfqq_save_state(new_bfqq); + /* + * Grab a reference to the bic, to prevent it from being destroyed + * before being possibly touched by a bfq_split_bfqq(). + */ + bfq_get_bic_reference(bfqq); + bfq_get_bic_reference(new_bfqq); + /* Merge queues (that is, let bic redirect its requests to new_bfqq) */ + bic_set_bfqq(bic, new_bfqq, 1); + bfq_mark_bfqq_coop(new_bfqq); + /* + * new_bfqq now belongs to at least two bics (it is a shared queue): set + * new_bfqq->bic to NULL. bfqq either: + * - does not belong to any bic any more, and hence bfqq->bic must + * be set to NULL, or + * - is a queue whose owning bics have already been redirected to a + * different queue, hence the queue is destined to not belong to any + * bic soon and bfqq->bic is already NULL (therefore the next + * assignment causes no harm). + */ + new_bfqq->bic = NULL; + bfqq->bic = NULL; + bfq_put_queue(bfqq); +} + +static int bfq_allow_merge(struct request_queue *q, struct request *rq, + struct bio *bio) +{ + struct bfq_data *bfqd = q->elevator->elevator_data; + struct bfq_io_cq *bic; + struct bfq_queue *bfqq, *new_bfqq; + + /* + * Disallow merge of a sync bio into an async request. + */ + if (bfq_bio_sync(bio) && !rq_is_sync(rq)) + return 0; + + /* + * Lookup the bfqq that this bio will be queued with. Allow + * merge only if rq is queued there. + * Queue lock is held here. + */ + bic = bfq_bic_lookup(bfqd, current->io_context); + if (bic == NULL) + return 0; + + bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); + /* + * We take advantage of this function to perform an early merge + * of the queues of possible cooperating processes. + */ + if (bfqq != NULL && + (new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false))) { + bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq); + /* + * If we get here, the bio will be queued in the shared queue, + * i.e., new_bfqq, so use new_bfqq to decide whether bio and + * rq can be merged. + */ + bfqq = new_bfqq; + } + + return bfqq == RQ_BFQQ(rq); +} + +static void __bfq_set_active_queue(struct bfq_data *bfqd, + struct bfq_queue *bfqq) +{ + if (bfqq != NULL) { + bfq_mark_bfqq_must_alloc(bfqq); + bfq_mark_bfqq_budget_new(bfqq); + bfq_clear_bfqq_fifo_expire(bfqq); + + bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8; + + bfq_log_bfqq(bfqd, bfqq, "set_active_queue, cur-budget = %lu", + bfqq->entity.budget); + } + + bfqd->active_queue = bfqq; +} + +/* + * Get and set a new active queue for service. + */ +static struct bfq_queue *bfq_set_active_queue(struct bfq_data *bfqd) +{ + struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); + + __bfq_set_active_queue(bfqd, bfqq); + return bfqq; +} + /* * If enough samples have been computed, return the current max budget * stored in bfqd, which is dynamically updated according to the @@ -986,63 +1232,6 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq) return rq; } -/* - * Must be called with the queue_lock held. - */ -static int bfqq_process_refs(struct bfq_queue *bfqq) -{ - int process_refs, io_refs; - - io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE]; - process_refs = atomic_read(&bfqq->ref) - io_refs - bfqq->entity.on_st; - BUG_ON(process_refs < 0); - return process_refs; -} - -static void bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) -{ - int process_refs, new_process_refs; - struct bfq_queue *__bfqq; - - /* - * If there are no process references on the new_bfqq, then it is - * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain - * may have dropped their last reference (not just their last process - * reference). - */ - if (!bfqq_process_refs(new_bfqq)) - return; - - /* Avoid a circular list and skip interim queue merges. */ - while ((__bfqq = new_bfqq->new_bfqq)) { - if (__bfqq == bfqq) - return; - new_bfqq = __bfqq; - } - - process_refs = bfqq_process_refs(bfqq); - new_process_refs = bfqq_process_refs(new_bfqq); - /* - * If the process for the bfqq has gone away, there is no - * sense in merging the queues. - */ - if (process_refs == 0 || new_process_refs == 0) - return; - - /* - * Merge in the direction of the lesser amount of work. - */ - if (new_process_refs >= process_refs) { - bfqq->new_bfqq = new_bfqq; - atomic_add(process_refs, &new_bfqq->ref); - } else { - new_bfqq->new_bfqq = bfqq; - atomic_add(new_process_refs, &bfqq->ref); - } - bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", - new_bfqq->pid); -} - static inline unsigned long bfq_bfqq_budget_left(struct bfq_queue *bfqq) { struct bfq_entity *entity = &bfqq->entity; @@ -1281,6 +1470,8 @@ static int bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq, * new_rate = (7/8) * old_rate + (1/8) * bw */ do_div(bw, 8); + if (bw == 0) + return 0; bfqd->peak_rate *= 7; do_div(bfqd->peak_rate, 8); bfqd->peak_rate += bw; @@ -1456,7 +1647,7 @@ static inline int bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) */ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) { - struct bfq_queue *bfqq, *new_bfqq = NULL; + struct bfq_queue *bfqq; struct request *next_rq; enum bfqq_expiration reason = BFQ_BFQQ_BUDGET_TIMEOUT; @@ -1466,17 +1657,6 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) bfq_log_bfqq(bfqd, bfqq, "select_queue: already active queue"); - /* - * If another queue has a request waiting within our mean seek - * distance, let it run. The expire code will check for close - * cooperators and put the close queue at the front of the - * service tree. If possible, merge the expiring queue with the - * new bfqq. - */ - new_bfqq = bfq_close_cooperator(bfqd, bfqq); - if (new_bfqq != NULL && bfqq->new_bfqq == NULL) - bfq_setup_merge(bfqq, new_bfqq); - if (bfq_may_expire_for_budg_timeout(bfqq)) goto expire; @@ -1511,10 +1691,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) bfq_clear_bfqq_wait_request(bfqq); del_timer(&bfqd->idle_slice_timer); } - if (new_bfqq == NULL) - goto keep_queue; - else - goto expire; + goto keep_queue; } } @@ -1523,25 +1700,18 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * queue still has requests in flight or is idling for a new request, * then keep it. */ - if (new_bfqq == NULL && (timer_pending(&bfqd->idle_slice_timer) || - (bfqq->dispatched != 0 && bfq_bfqq_idle_window(bfqq) && - !bfq_queue_nonrot_noidle(bfqd, bfqq)))) { + if (timer_pending(&bfqd->idle_slice_timer) || + (bfqq->dispatched != 0 && bfq_bfqq_idle_window(bfqq) && + !bfq_queue_nonrot_noidle(bfqd, bfqq))) { bfqq = NULL; goto keep_queue; - } else if (new_bfqq != NULL && timer_pending(&bfqd->idle_slice_timer)) { - /* - * Expiring the queue because there is a close cooperator, - * cancel timer. - */ - bfq_clear_bfqq_wait_request(bfqq); - del_timer(&bfqd->idle_slice_timer); } reason = BFQ_BFQQ_NO_MORE_REQUESTS; expire: bfq_bfqq_expire(bfqd, bfqq, 0, reason); new_queue: - bfqq = bfq_set_active_queue(bfqd, new_bfqq); + bfqq = bfq_set_active_queue(bfqd); bfq_log(bfqd, "select_queue: new queue %d returned", bfqq != NULL ? bfqq->pid : 0); keep_queue: @@ -1550,9 +1720,8 @@ keep_queue: static void update_raising_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) { + struct bfq_entity *entity = &bfqq->entity; if (bfqq->raising_coeff > 1) { /* queue is being boosted */ - struct bfq_entity *entity = &bfqq->entity; - bfq_log_bfqq(bfqd, bfqq, "raising period dur %u/%u msec, " "old raising coeff %u, w %d(%d)", @@ -1590,12 +1759,14 @@ static void update_raising_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) raising_cur_max_time)); bfqq->raising_coeff = 1; entity->ioprio_changed = 1; - __bfq_entity_update_weight_prio( - bfq_entity_service_tree(entity), - entity); } } } + /* Update weight both if it must be raised and if it must be lowered */ + if ((entity->weight > entity->orig_weight) != (bfqq->raising_coeff > 1)) + __bfq_entity_update_weight_prio( + bfq_entity_service_tree(entity), + entity); } @@ -1836,6 +2007,7 @@ static void bfq_init_icq(struct io_cq *icq) struct bfq_io_cq *bic = icq_to_bic(icq); bic->ttime.last_end_request = jiffies; + bic->raising_time_left = 0; } static void bfq_exit_icq(struct io_cq *icq) @@ -1849,6 +2021,13 @@ static void bfq_exit_icq(struct io_cq *icq) } if (bic->bfqq[BLK_RW_SYNC]) { + /* + * If the bic is using a shared queue, put the reference + * taken on the io_context when the bic started using a + * shared bfq_queue. + */ + if (bfq_bfqq_coop(bic->bfqq[BLK_RW_SYNC])) + put_io_context(icq->ioc); bfq_exit_bfqq(bfqd, bic->bfqq[BLK_RW_SYNC]); bic->bfqq[BLK_RW_SYNC] = NULL; } @@ -2146,6 +2325,10 @@ static void bfq_update_idle_window(struct bfq_data *bfqd, if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq)) return; + /* Idle window just restored, statistics are meaningless. */ + if (bfq_bfqq_just_split(bfqq)) + return; + enable_idle = bfq_bfqq_idle_window(bfqq); if (atomic_read(&bic->icq.ioc->active_ref) == 0 || @@ -2186,6 +2369,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 || !BFQQ_SEEKY(bfqq)) bfq_update_idle_window(bfqd, bfqq, bic); + bfq_clear_bfqq_just_split(bfqq); bfq_log_bfqq(bfqd, bfqq, "rq_enqueued: idle_window=%d (seeky %d, mean %llu)", @@ -2237,9 +2421,33 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, static void bfq_insert_request(struct request_queue *q, struct request *rq) { struct bfq_data *bfqd = q->elevator->elevator_data; - struct bfq_queue *bfqq = RQ_BFQQ(rq); + struct bfq_queue *bfqq = RQ_BFQQ(rq), *new_bfqq; assert_spin_locked(bfqd->queue->queue_lock); + + /* + * An unplug may trigger a requeue of a request from the device + * driver: make sure we are in process context while trying to + * merge two bfq_queues. + */ + if (!in_interrupt() && + (new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true))) { + if (bic_to_bfqq(RQ_BIC(rq), 1) != bfqq) + new_bfqq = bic_to_bfqq(RQ_BIC(rq), 1); + /* + * Release the request's reference to the old bfqq + * and make sure one is taken to the shared queue. + */ + new_bfqq->allocated[rq_data_dir(rq)]++; + bfqq->allocated[rq_data_dir(rq)]--; + atomic_inc(&new_bfqq->ref); + bfq_put_queue(bfqq); + if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq) + bfq_merge_bfqqs(bfqd, RQ_BIC(rq), bfqq, new_bfqq); + rq->elv.priv[1] = new_bfqq; + bfqq = new_bfqq; + } + bfq_init_prio_data(bfqq, RQ_BIC(rq)); bfq_add_rq_rb(rq); @@ -2320,7 +2528,7 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq) (bfqd->rq_in_driver == 0 || bfqq->raising_coeff > 1) && RB_EMPTY_ROOT(&bfqq->sort_list) - && !bfq_close_cooperator(bfqd, bfqq) + && !bfq_close_cooperator(bfqd, bfqq, bfqd->last_position) && (!bfq_bfqq_coop(bfqq) || !bfq_bfqq_some_coop_idle(bfqq))) bfq_arm_slice_timer(bfqd); @@ -2389,18 +2597,6 @@ static void bfq_put_request(struct request *rq) } } -static struct bfq_queue * -bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, - struct bfq_queue *bfqq) -{ - bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", - (long unsigned)bfqq->new_bfqq->pid); - bic_set_bfqq(bic, bfqq->new_bfqq, 1); - bfq_mark_bfqq_coop(bfqq->new_bfqq); - bfq_put_queue(bfqq); - return bic_to_bfqq(bic, 1); -} - /* * Returns NULL if a new bfqq should be allocated, or the old bfqq if this * was the last process referring to said bfqq. @@ -2409,6 +2605,9 @@ static struct bfq_queue * bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) { bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); + + put_io_context(bic->icq.ioc); + if (bfqq_process_refs(bfqq) == 1) { bfqq->pid = current->pid; bfq_clear_bfqq_some_coop_idle(bfqq); @@ -2438,6 +2637,7 @@ static int bfq_set_request(struct request_queue *q, struct request *rq, struct bfq_queue *bfqq; struct bfq_group *bfqg; unsigned long flags; + bool split = false; might_sleep_if(gfp_mask & __GFP_WAIT); @@ -2456,24 +2656,14 @@ new_queue: bfqq = bfq_get_queue(bfqd, bfqg, is_sync, bic, gfp_mask); bic_set_bfqq(bic, bfqq, is_sync); } else { - /* - * If the queue was seeky for too long, break it apart. - */ + /* If the queue was seeky for too long, break it apart. */ if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); bfqq = bfq_split_bfqq(bic, bfqq); + split = true; if (!bfqq) goto new_queue; } - - /* - * Check to see if this queue is scheduled to merge with - * another closely cooperating queue. The merging of queues - * happens here as it must be done in process context. - * The reference on new_bfqq was taken in merge_bfqqs. - */ - if (bfqq->new_bfqq != NULL) - bfqq = bfq_merge_bfqqs(bfqd, bic, bfqq); } bfqq->allocated[rw]++; @@ -2484,6 +2674,26 @@ new_queue: rq->elv.priv[0] = bic; rq->elv.priv[1] = bfqq; + /* + * If a bfq_queue has only one process reference, it is owned + * by only one bfq_io_cq: we can set the bic field of the + * bfq_queue to the address of that structure. Also, if the + * queue has just been split, mark a flag so that the + * information is available to the other scheduler hooks. + */ + if (bfqq_process_refs(bfqq) == 1) { + bfqq->bic = bic; + if (split) { + bfq_mark_bfqq_just_split(bfqq); + /* + * If the queue has just been split from a shared queue, + * restore the idle window and the possible weight + * raising period. + */ + bfq_bfqq_resume_state(bfqq, bic); + } + } + spin_unlock_irqrestore(q->queue_lock, flags); return 0; diff --git a/block/bfq-sched.c b/block/bfq-sched.c index 03f8061..a0edaa2 100644 --- a/block/bfq-sched.c +++ b/block/bfq-sched.c @@ -978,34 +978,6 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) return bfqq; } -/* - * Forced extraction of the given queue. - */ -static void bfq_get_next_queue_forced(struct bfq_data *bfqd, - struct bfq_queue *bfqq) -{ - struct bfq_entity *entity; - struct bfq_sched_data *sd; - - BUG_ON(bfqd->active_queue != NULL); - - entity = &bfqq->entity; - /* - * Bubble up extraction/update from the leaf to the root. - */ - for_each_entity(entity) { - sd = entity->sched_data; - bfq_update_budget(entity); - bfq_update_vtime(bfq_entity_service_tree(entity)); - bfq_active_extract(bfq_entity_service_tree(entity), entity); - sd->active_entity = entity; - sd->next_active = NULL; - entity->service = 0; - } - - return; -} - static void __bfq_bfqd_reset_active(struct bfq_data *bfqd) { if (bfqd->active_bic != NULL) { diff --git a/block/bfq.h b/block/bfq.h index 3766529..bb3c003 100644 --- a/block/bfq.h +++ b/block/bfq.h @@ -188,6 +188,8 @@ struct bfq_group; * @pid: pid of the process owning the queue, used for logging purposes. * @last_rais_start_time: last (idle -> weight-raised) transition attempt * @raising_cur_max_time: current max raising time for this queue + * @bic: pointer to the bfq_io_cq owning the bfq_queue, set to %NULL if the + * queue is shared * * A bfq_queue is a leaf request queue; it can be associated to an io_context * or more (if it is an async one). @cgroup holds a reference to the @@ -231,6 +233,7 @@ struct bfq_queue { sector_t last_request_pos; pid_t pid; + struct bfq_io_cq *bic; /* weight-raising fields */ unsigned int raising_cur_max_time; @@ -257,12 +260,23 @@ struct bfq_ttime { * @icq: associated io_cq structure * @bfqq: array of two process queues, the sync and the async * @ttime: associated @bfq_ttime struct + * @raising_time_left: snapshot of the time left before weight raising ends + * for the sync queue associated to this process; this + * snapshot is taken to remember this value while the weight + * raising is suspended because the queue is merged with a + * shared queue, and is used to set @raising_cur_max_time + * when the queue is split from the shared queue and its + * weight is raised again + * @saved_idle_window: same purpose as the previous field for the idle window */ struct bfq_io_cq { struct io_cq icq; /* must be the first member */ struct bfq_queue *bfqq[2]; struct bfq_ttime ttime; int ioprio; + + unsigned int raising_time_left; + unsigned int saved_idle_window; }; /** @@ -403,6 +417,7 @@ enum bfqq_state_flags { BFQ_BFQQ_FLAG_coop, /* bfqq is shared */ BFQ_BFQQ_FLAG_split_coop, /* shared bfqq will be splitted */ BFQ_BFQQ_FLAG_some_coop_idle, /* some cooperator is inactive */ + BFQ_BFQQ_FLAG_just_split, /* queue has just been split */ }; #define BFQ_BFQQ_FNS(name) \ @@ -430,6 +445,7 @@ BFQ_BFQQ_FNS(budget_new); BFQ_BFQQ_FNS(coop); BFQ_BFQQ_FNS(split_coop); BFQ_BFQQ_FNS(some_coop_idle); +BFQ_BFQQ_FNS(just_split); #undef BFQ_BFQQ_FNS /* Logging facilities. */ -- 1.7.10.4