From dfc9ee24d8aabdf8f329fe1d5e5766a4df6d6675 Mon Sep 17 00:00:00 2001 From: Mauro Andreolini Date: Sat, 17 May 2014 22:55:21 +0200 Subject: [PATCH 4/4] block, bfq: add Early Queue Merge (EQM) to BFQ-v7r4 for 3.1.0 A set of processes may happen to perform interleaved reads, i.e., requests whose union would give rise to a sequential read pattern. There are two typical cases: in the first case, processes read fixed-size chunks of data at a fixed distance from each other, while in the second case processes may read variable-size chunks at variable distances. The latter case occurs for example with KVM, which splits the I/O generated by the guest into multiple chunks, and lets these chunks be served by a pool of cooperating processes, iteratively assigning the next chunk of I/O to the first available process. CFQ uses actual queue merging for the first type of rocesses, whereas it uses preemption to get a sequential read pattern out of the read requests performed by the second type of processes. In the end it uses two different mechanisms to achieve the same goal: boosting the throughput with interleaved I/O. This patch introduces Early Queue Merge (EQM), a unified mechanism to get a sequential read pattern with both types of processes. The main idea is checking newly arrived requests against the next request of the active queue both in case of actual request insert and in case of request merge. By doing so, both the types of processes can be handled by just merging their queues. EQM is then simpler and more compact than the pair of mechanisms used in CFQ. Finally, EQM also preserves the typical low-latency properties of BFQ, by properly restoring the weight-raising state of a queue when it gets back to a non-merged state. Signed-off-by: Mauro Andreolini Signed-off-by: Arianna Avanzini Reviewed-by: Paolo Valente --- block/bfq-cgroup.c | 17 +- block/bfq-ioc.c | 28 ++ block/bfq-iosched.c | 682 ++++++++++++++++++++++++++++++++-------------- block/bfq-sched.c | 28 -- block/bfq.h | 47 ++-- include/linux/iocontext.h | 3 + 6 files changed, 554 insertions(+), 251 deletions(-) diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 753d5ad..8eea9ee 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -313,11 +313,16 @@ static struct bfq_group *__bfq_cic_change_cgroup(struct bfq_data *bfqd, struct cfq_io_context *cic, struct cgroup *cgroup) { - struct bfq_queue *async_bfqq = cic_to_bfqq(cic, 0); - struct bfq_queue *sync_bfqq = cic_to_bfqq(cic, 1); + struct bfq_queue *async_bfqq; + struct bfq_queue *sync_bfqq; struct bfq_entity *entity; struct bfq_group *bfqg; + spin_lock(&bfqd->eqm_lock); + + async_bfqq = cic_to_bfqq(cic, 0); + sync_bfqq = cic_to_bfqq(cic, 1); + bfqg = bfq_find_alloc_group(bfqd, cgroup); if (async_bfqq != NULL) { entity = &async_bfqq->entity; @@ -337,6 +342,8 @@ static struct bfq_group *__bfq_cic_change_cgroup(struct bfq_data *bfqd, bfq_bfqq_move(bfqd, sync_bfqq, entity, bfqg); } + spin_unlock(&bfqd->eqm_lock); + return bfqg; } @@ -532,14 +539,14 @@ static void bfq_destroy_group(struct bfqio_cgroup *bgrp, struct bfq_group *bfqg) kfree(bfqg); } -static void bfq_end_raising_async(struct bfq_data *bfqd) +static void bfq_end_wr_async(struct bfq_data *bfqd) { struct hlist_node *pos, *n; struct bfq_group *bfqg; hlist_for_each_entry_safe(bfqg, pos, n, &bfqd->group_list, bfqd_node) - bfq_end_raising_async_queues(bfqd, bfqg); - bfq_end_raising_async_queues(bfqd, bfqd->root_group); + bfq_end_wr_async_queues(bfqd, bfqg); + bfq_end_wr_async_queues(bfqd, bfqd->root_group); } /** diff --git a/block/bfq-ioc.c b/block/bfq-ioc.c index c0671a6..2dd6699 100644 --- a/block/bfq-ioc.c +++ b/block/bfq-ioc.c @@ -122,10 +122,19 @@ static void __bfq_exit_single_io_context(struct bfq_data *bfqd, cic->cfqq[BLK_RW_ASYNC] = NULL; } + spin_lock(&bfqd->eqm_lock); if (cic->cfqq[BLK_RW_SYNC] != NULL) { + /* + * If the bic is using a shared queue, put the reference + * taken on the io_context when the bic started using a + * shared bfq_queue. + */ + if (bfq_bfqq_coop(cic->cfqq[BLK_RW_SYNC])) + put_io_context(ioc); bfq_exit_bfqq(bfqd, cic->cfqq[BLK_RW_SYNC]); cic->cfqq[BLK_RW_SYNC] = NULL; } + spin_unlock(&bfqd->eqm_lock); } /** @@ -172,6 +181,25 @@ static struct cfq_io_context *bfq_alloc_io_context(struct bfq_data *bfqd, bfqd->queue->node); if (cic != NULL) { cic->ttime.last_end_request = jiffies; + /* + * A newly created cic indicates that the process has just + * started doing I/O, and is probably mapping into memory its + * executable and libraries: it definitely needs weight raising. + * There is however the possibility that the process performs, + * for a while, I/O close to some other process. EQM intercepts + * this behavior and may merge the queue corresponding to the + * process with some other queue, BEFORE the weight of the queue + * is raised. Merged queues are not weight-raised (they are assumed + * to belong to processes that benefit only from high throughput). + * If the merge is basically the consequence of an accident, then + * the queue will be split soon and will get back its old weight. + * It is then important to write down somewhere that this queue + * does need weight raising, even if it did not make it to get its + * weight raised before being merged. To this purpose, we overload + * the field raising_time_left and assign 1 to it, to mark the queue + * as needing weight raising. + */ + cic->wr_time_left = 1; INIT_LIST_HEAD(&cic->queue_list); INIT_HLIST_NODE(&cic->cic_list); cic->dtor = bfq_free_io_context; diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 75958f7..1d3caef 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -574,6 +574,46 @@ static inline unsigned int bfq_wr_duration(struct bfq_data *bfqd) return dur; } +static inline void +bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct cfq_io_context *cic) +{ + if (cic->saved_idle_window) + bfq_mark_bfqq_idle_window(bfqq); + else + bfq_clear_bfqq_idle_window(bfqq); + if (cic->wr_time_left && bfqq->bfqd->low_latency) { + /* + * Start a weight raising period with the duration given by + * the raising_time_left snapshot. + */ + if (bfq_bfqq_busy(bfqq)) + bfqq->bfqd->raised_busy_queues++; + bfqq->wr_coeff = bfqq->bfqd->bfq_wr_coeff; + bfqq->wr_cur_max_time = cic->wr_time_left; + bfqq->last_wr_start_finish = jiffies; + bfqq->entity.ioprio_changed = 1; + } + /* + * Clear raising_time_left to prevent bfq_bfqq_save_state() from + * getting confused about the queue's need of a weight-raising + * period. + */ + cic->wr_time_left = 0; +} + +/* + * Must be called with the queue_lock held. + */ +static int bfqq_process_refs(struct bfq_queue *bfqq) +{ + int process_refs, io_refs; + + io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE]; + process_refs = atomic_read(&bfqq->ref) - io_refs - bfqq->entity.on_st; + BUG_ON(process_refs < 0); + return process_refs; +} + static void bfq_add_request(struct request *rq) { struct bfq_queue *bfqq = RQ_BFQQ(rq); @@ -589,6 +629,8 @@ static void bfq_add_request(struct request *rq) elv_rb_add(&bfqq->sort_list, rq); + spin_lock(&bfqd->eqm_lock); + /* * Check if this request is a better next-serve candidate. */ @@ -603,6 +645,8 @@ static void bfq_add_request(struct request *rq) if (prev != bfqq->next_rq) bfq_rq_pos_tree_add(bfqd, bfqq); + spin_unlock(&bfqd->eqm_lock); + if (!bfq_bfqq_busy(bfqq)) { int soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && time_is_before_jiffies(bfqq->soft_rt_next_start); @@ -615,14 +659,24 @@ static void bfq_add_request(struct request *rq) if (!bfqd->low_latency) goto add_bfqq_busy; + if (bfq_bfqq_just_split(bfqq)) + goto set_ioprio_changed; + /* - * If the queue is not being boosted and has been idle - * for enough time, start a weight-raising period + * If the queue: + * - is not being boosted, + * - has been idle for enough time, + * - is not a sync queue or is linked to a cfq_io_context (it is + * shared "for its nature" or it is not shared and its + * requests have not been redirected to a shared queue) + * start a weight-raising period. */ - if (old_wr_coeff == 1 && (idle_for_long_time || soft_rt)) { + if (old_wr_coeff == 1 && (idle_for_long_time || soft_rt) && + (!bfq_bfqq_sync(bfqq) || bfqq->cic != NULL)) { bfqq->wr_coeff = bfqd->bfq_wr_coeff; if (idle_for_long_time) - bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); + bfqq->wr_cur_max_time = + bfq_wr_duration(bfqd); else bfqq->wr_cur_max_time = bfqd->bfq_wr_rt_max_time; @@ -653,7 +707,7 @@ static void bfq_add_request(struct request *rq) /* * * The remaining weight-raising time is lower - * than bfqd->bfq_wr_rt_max_time, which + * than bfqd->bfq_raising_rt_max_time, which * means that the application is enjoying * weight raising either because deemed soft- * rt in the near past, or because deemed @@ -699,6 +753,7 @@ static void bfq_add_request(struct request *rq) bfqd->bfq_wr_rt_max_time; } } +set_ioprio_changed: if (old_wr_coeff != bfqq->wr_coeff) entity->ioprio_changed = 1; add_bfqq_busy: @@ -742,7 +797,9 @@ static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd, if (cic == NULL) return NULL; + spin_lock(&bfqd->eqm_lock); bfqq = cic_to_bfqq(cic, bfq_bio_sync(bio)); + spin_unlock(&bfqd->eqm_lock); if (bfqq != NULL) { sector_t sector = bio->bi_sector + bio_sectors(bio); @@ -776,6 +833,7 @@ static void bfq_remove_request(struct request *rq) struct bfq_data *bfqd = bfqq->bfqd; const int sync = rq_is_sync(rq); + spin_lock(&bfqq->bfqd->eqm_lock); if (bfqq->next_rq == rq) { bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); bfq_updated_next_req(bfqd, bfqq); @@ -798,6 +856,7 @@ static void bfq_remove_request(struct request *rq) bfqq->pos_root = NULL; } } + spin_unlock(&bfqq->bfqd->eqm_lock); if (rq->cmd_flags & REQ_META) { WARN_ON(bfqq->meta_pending == 0); @@ -866,8 +925,16 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq, rq_set_fifo_time(rq, rq_fifo_time(next)); } - if (bfqq->next_rq == next) + /* + * eqm_lock needed to avoid that other critical sections not holding + * the queue_lock read an inconsistent value from bfqq->next_rq while + * traversing the rq_pos_trees + */ + if (bfqq->next_rq == next) { + spin_lock(&bfqq->bfqd->eqm_lock); bfqq->next_rq = rq; + spin_unlock(&bfqq->bfqd->eqm_lock); + } bfq_remove_request(next); } @@ -912,87 +979,35 @@ static void bfq_end_wr(struct bfq_data *bfqd) spin_unlock_irq(bfqd->queue->queue_lock); } -static int bfq_allow_merge(struct request_queue *q, struct request *rq, - struct bio *bio) +static inline sector_t bfq_io_struct_pos(void *io_struct, bool request) { - struct bfq_data *bfqd = q->elevator->elevator_data; - struct cfq_io_context *cic; - struct bfq_queue *bfqq; - - /* Disallow merge of a sync bio into an async request. */ - if (bfq_bio_sync(bio) && !rq_is_sync(rq)) - return 0; - - /* - * Lookup the bfqq that this bio will be queued with. Allow - * merge only if rq is queued there. - */ - cic = bfq_cic_lookup(bfqd, current->io_context); - if (cic == NULL) - return 0; - - bfqq = cic_to_bfqq(cic, bfq_bio_sync(bio)); - return bfqq == RQ_BFQQ(rq); -} - -static void __bfq_set_in_service_queue(struct bfq_data *bfqd, - struct bfq_queue *bfqq) -{ - if (bfqq != NULL) { - bfq_mark_bfqq_must_alloc(bfqq); - bfq_mark_bfqq_budget_new(bfqq); - bfq_clear_bfqq_fifo_expire(bfqq); - - bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8; - - bfq_log_bfqq(bfqd, bfqq, - "set_in_service_queue, cur-budget = %lu", - bfqq->entity.budget); - } - - bfqd->in_service_queue = bfqq; -} - -/* - * Get and set a new active queue for service. - */ -static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd, - struct bfq_queue *bfqq) -{ - if (!bfqq) - bfqq = bfq_get_next_queue(bfqd); + if (request) + return blk_rq_pos(io_struct); else - bfq_get_next_queue_forced(bfqd, bfqq); - - __bfq_set_in_service_queue(bfqd, bfqq); - return bfqq; + return ((struct bio *)io_struct)->bi_sector; } -static inline sector_t bfq_dist_from_last(struct bfq_data *bfqd, - struct request *rq) +static inline sector_t bfq_dist_from(sector_t pos1, + sector_t pos2) { - if (blk_rq_pos(rq) >= bfqd->last_position) - return blk_rq_pos(rq) - bfqd->last_position; + if (pos1 >= pos2) + return pos1 - pos2; else - return bfqd->last_position - blk_rq_pos(rq); + return pos2 - pos1; } -/* - * Return true if bfqq has no request pending and rq is close enough to - * bfqd->last_position, or if rq is closer to bfqd->last_position than - * bfqq->next_rq - */ -static inline int bfq_rq_close(struct bfq_data *bfqd, struct request *rq) +static inline int bfq_rq_close_to_sector(void *io_struct, bool request, + sector_t sector) { - return bfq_dist_from_last(bfqd, rq) <= BFQQ_SEEK_THR; + return bfq_dist_from(bfq_io_struct_pos(io_struct, request), sector) <= + BFQQ_SEEK_THR; } -static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) +static struct bfq_queue *bfqq_close(struct bfq_data *bfqd, sector_t sector) { struct rb_root *root = &bfqd->rq_pos_tree; struct rb_node *parent, *node; struct bfq_queue *__bfqq; - sector_t sector = bfqd->last_position; if (RB_EMPTY_ROOT(root)) return NULL; @@ -1011,7 +1026,7 @@ static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) * position). */ __bfqq = rb_entry(parent, struct bfq_queue, pos_node); - if (bfq_rq_close(bfqd, __bfqq->next_rq)) + if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) return __bfqq; if (blk_rq_pos(__bfqq->next_rq) < sector) @@ -1022,7 +1037,7 @@ static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) return NULL; __bfqq = rb_entry(node, struct bfq_queue, pos_node); - if (bfq_rq_close(bfqd, __bfqq->next_rq)) + if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) return __bfqq; return NULL; @@ -1031,14 +1046,12 @@ static struct bfq_queue *bfqq_close(struct bfq_data *bfqd) /* * bfqd - obvious * cur_bfqq - passed in so that we don't decide that the current queue - * is closely cooperating with itself. - * - * We are assuming that cur_bfqq has dispatched at least one request, - * and that bfqd->last_position reflects a position on the disk associated - * with the I/O issued by cur_bfqq. + * is closely cooperating with itself + * sector - used as a reference point to search for a close queue */ static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd, - struct bfq_queue *cur_bfqq) + struct bfq_queue *cur_bfqq, + sector_t sector) { struct bfq_queue *bfqq; @@ -1058,7 +1071,7 @@ static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd, * working closely on the same area of the disk. In that case, * we can group them together and don't waste time idling. */ - bfqq = bfqq_close(bfqd); + bfqq = bfqq_close(bfqd, sector); if (bfqq == NULL || bfqq == cur_bfqq) return NULL; @@ -1085,6 +1098,285 @@ static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd, return bfqq; } +static struct bfq_queue * +bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) +{ + int process_refs, new_process_refs; + struct bfq_queue *__bfqq; + + /* + * If there are no process references on the new_bfqq, then it is + * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain + * may have dropped their last reference (not just their last process + * reference). + */ + if (!bfqq_process_refs(new_bfqq)) + return NULL; + + /* Avoid a circular list and skip interim queue merges. */ + while ((__bfqq = new_bfqq->new_bfqq)) { + if (__bfqq == bfqq) + return NULL; + new_bfqq = __bfqq; + } + + process_refs = bfqq_process_refs(bfqq); + new_process_refs = bfqq_process_refs(new_bfqq); + /* + * If the process for the bfqq has gone away, there is no + * sense in merging the queues. + */ + if (process_refs == 0 || new_process_refs == 0) + return NULL; + + bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", + new_bfqq->pid); + + /* + * Merging is just a redirection: the requests of the process owning + * one of the two queues are redirected to the other queue. The latter + * queue, in its turn, is set as shared if this is the first time that + * the requests of some process are redirected to it. + * + * We redirect bfqq to new_bfqq and not the opposite, because we + * are in the context of the process owning bfqq, hence we have the + * io_cq of this process. So we can immediately configure this io_cq + * to redirect the requests of the process to new_bfqq. + * + * NOTE, even if new_bfqq coincides with the in-service queue, the + * io_cq of new_bfqq is not available, because, if the in-service queue + * is shared, bfqd->in_service_cic may not point to the io_cq of the + * in-service queue. + * Redirecting the requests of the process owning bfqq to the currently + * in-service queue is in any case the best option, as we feed the + * in-service queue with new requests close to the last request served + * and, by doing so, hopefully increase the throughput. + */ + bfqq->new_bfqq = new_bfqq; + atomic_add(process_refs, &new_bfqq->ref); + return new_bfqq; +} + +/* + * Attempt to schedule a merge of bfqq with the currently in-service queue or + * with a close queue among the scheduled queues. + * Return NULL if no merge was scheduled, a pointer to the shared bfq_queue + * structure otherwise. + */ +static struct bfq_queue * +bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + void *io_struct, bool request) +{ + struct bfq_queue *in_service_bfqq, *new_bfqq; + + if (bfqq->new_bfqq) + return bfqq->new_bfqq; + + if (!io_struct) + return NULL; + + in_service_bfqq = bfqd->in_service_queue; + + if (in_service_bfqq == NULL || in_service_bfqq == bfqq || + !bfqd->in_service_cic) + goto check_scheduled; + + if (bfq_class_idle(in_service_bfqq) || bfq_class_idle(bfqq)) + goto check_scheduled; + + if (bfq_class_rt(in_service_bfqq) != bfq_class_rt(bfqq)) + goto check_scheduled; + + if (in_service_bfqq->entity.parent != bfqq->entity.parent) + goto check_scheduled; + + if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && + bfq_bfqq_sync(in_service_bfqq) && bfq_bfqq_sync(bfqq)) + if ((new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq))) + return new_bfqq; /* Merge with the in-service queue */ + + /* + * Check whether there is a cooperator among currently scheduled + * queues. The only thing we need is that the bio/request is not + * NULL, as we need it to establish whether a cooperator exists. + */ +check_scheduled: + new_bfqq = bfq_close_cooperator(bfqd, bfqq, + bfq_io_struct_pos(io_struct, request)); + if (new_bfqq) + return bfq_setup_merge(bfqq, new_bfqq); + + return NULL; +} + +static inline void +bfq_bfqq_save_state(struct bfq_queue *bfqq) +{ + /* + * If bfqq->cic == NULL, the queue is already shared or its requests + * have already been redirected to a shared queue; both idle window + * and weight raising state have already been saved. Do nothing. + */ + if (bfqq->cic == NULL) + return; + if (bfqq->cic->wr_time_left) + /* + * This is the queue of a just-started process, and would + * deserve weight raising: we set raising_time_left to the full + * weight-raising duration to trigger weight-raising when and + * if the queue is split and the first request of the queue + * is enqueued. + */ + bfqq->cic->wr_time_left = bfq_wr_duration(bfqq->bfqd); + else if (bfqq->wr_coeff > 1) { + unsigned long wr_duration = + jiffies - bfqq->last_wr_start_finish; + /* + * It may happen that a queue's weight raising period lasts + * longer than its raising_cur_max_time, as weight raising is + * handled only when a request is enqueued or dispatched (it + * does not use any timer). If the weight raising period is + * about to end, don't save it. + */ + if (bfqq->wr_cur_max_time <= wr_duration) + bfqq->cic->wr_time_left = 0; + else + bfqq->cic->wr_time_left = + bfqq->wr_cur_max_time - wr_duration; + /* + * The bfq_queue is becoming shared or the requests of the + * process owning the queue are being redirected to a shared + * queue. Stop the weight raising period of the queue, as in + * both cases it should not be owned by an interactive or soft + * real-time application. + */ + bfq_bfqq_end_wr(bfqq); + } else + bfqq->cic->wr_time_left = 0; + bfqq->cic->saved_idle_window = bfq_bfqq_idle_window(bfqq); +} + +static inline void +bfq_get_cic_reference(struct bfq_queue *bfqq) +{ + /* + * If bfqq->cic has a non-NULL value, the cic to which it belongs + * is about to begin using a shared bfq_queue. + */ + if (bfqq->cic) + atomic_long_inc(&bfqq->cic->ioc->refcount); +} + +static void +bfq_merge_bfqqs(struct bfq_data *bfqd, struct cfq_io_context *cic, + struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) +{ + bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", + (long unsigned)new_bfqq->pid); + /* Save weight raising and idle window of the merged queues */ + bfq_bfqq_save_state(bfqq); + bfq_bfqq_save_state(new_bfqq); + /* + * Grab a reference to the cic, to prevent it from being destroyed + * before being possibly touched by a bfq_split_bfqq(). + */ + bfq_get_cic_reference(bfqq); + bfq_get_cic_reference(new_bfqq); + /* Merge queues (that is, let cic redirect its requests to new_bfqq) */ + cic_set_bfqq(cic, new_bfqq, 1); + bfq_mark_bfqq_coop(new_bfqq); + /* + * new_bfqq now belongs to at least two cics (it is a shared queue): set + * new_bfqq->cic to NULL. bfqq either: + * - does not belong to any cic any more, and hence bfqq->cic must + * be set to NULL, or + * - is a queue whose owning cics have already been redirected to a + * different queue, hence the queue is destined to not belong to any + * cic soon and bfqq->cic is already NULL (therefore the next + * assignment causes no harm). + */ + new_bfqq->cic = NULL; + bfqq->cic = NULL; + bfq_put_queue(bfqq); +} + +static int bfq_allow_merge(struct request_queue *q, struct request *rq, + struct bio *bio) +{ + struct bfq_data *bfqd = q->elevator->elevator_data; + struct cfq_io_context *cic; + struct bfq_queue *bfqq, *new_bfqq; + unsigned long flags; + + /* Disallow merge of a sync bio into an async request. */ + if (bfq_bio_sync(bio) && !rq_is_sync(rq)) + return 0; + + /* + * Lookup the bfqq that this bio will be queued with. Allow + * merge only if rq is queued there. + */ + cic = bfq_cic_lookup(bfqd, current->io_context); + if (cic == NULL) + return 0; + + /* + * The allow_merge_fn scheduler hook may be called with or without + * the queue_lock being held. Access to the rq_pos_tree data + * structures and to cic->bfqq[] is protected by the eqm_lock. + */ + spin_lock_irqsave(&bfqd->eqm_lock, flags); + bfqq = cic_to_bfqq(cic, bfq_bio_sync(bio)); + /* + * We take advantage of this function to perform an early merge + * of the queues of possible cooperating processes. + */ + if (bfqq != NULL) { + new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); + if (new_bfqq != NULL) { + bfq_merge_bfqqs(bfqd, cic, bfqq, new_bfqq); + /* + * If we get here, the bio will be queued in the shared + * queue, i.e., new_bfqq, so use new_bfqq to decide + * whether bio and rq can be merged. + */ + bfqq = new_bfqq; + } + } + spin_unlock_irqrestore(&bfqd->eqm_lock, flags); + + return bfqq == RQ_BFQQ(rq); +} + +static void __bfq_set_in_service_queue(struct bfq_data *bfqd, + struct bfq_queue *bfqq) +{ + if (bfqq != NULL) { + bfq_mark_bfqq_must_alloc(bfqq); + bfq_mark_bfqq_budget_new(bfqq); + bfq_clear_bfqq_fifo_expire(bfqq); + + bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8; + + bfq_log_bfqq(bfqd, bfqq, + "set_in_service_queue, cur-budget = %lu", + bfqq->entity.budget); + } + + bfqd->in_service_queue = bfqq; +} + +/* + * Get and set a new queue for service. + */ +static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) +{ + struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); + + __bfq_set_in_service_queue(bfqd, bfqq); + return bfqq; +} + /* * If enough samples have been computed, return the current max budget * stored in bfqd, which is dynamically updated according to the @@ -1228,69 +1520,13 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq) return rq; } -/* - * Must be called with the queue_lock held. - */ -static int bfqq_process_refs(struct bfq_queue *bfqq) -{ - int process_refs, io_refs; - - io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE]; - process_refs = atomic_read(&bfqq->ref) - io_refs - bfqq->entity.on_st; - BUG_ON(process_refs < 0); - return process_refs; -} - -static void bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) -{ - int process_refs, new_process_refs; - struct bfq_queue *__bfqq; - - /* - * If there are no process references on the new_bfqq, then it is - * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain - * may have dropped their last reference (not just their last process - * reference). - */ - if (!bfqq_process_refs(new_bfqq)) - return; - - /* Avoid a circular list and skip interim queue merges. */ - while ((__bfqq = new_bfqq->new_bfqq)) { - if (__bfqq == bfqq) - return; - new_bfqq = __bfqq; - } - - process_refs = bfqq_process_refs(bfqq); - new_process_refs = bfqq_process_refs(new_bfqq); - /* - * If the process for the bfqq has gone away, there is no - * sense in merging the queues. - */ - if (process_refs == 0 || new_process_refs == 0) - return; - - /* - * Merge in the direction of the lesser amount of work. - */ - if (new_process_refs >= process_refs) { - bfqq->new_bfqq = new_bfqq; - atomic_add(process_refs, &new_bfqq->ref); - } else { - new_bfqq->new_bfqq = bfqq; - atomic_add(new_process_refs, &bfqq->ref); - } - bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", - new_bfqq->pid); -} - static inline unsigned long bfq_bfqq_budget_left(struct bfq_queue *bfqq) { struct bfq_entity *entity = &bfqq->entity; return entity->budget - entity->service; } +/* Must be called with eqm_lock held */ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq) { BUG_ON(bfqq != bfqd->in_service_queue); @@ -1778,7 +2014,9 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd, /* Increase, decrease or leave budget unchanged according to reason */ __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); + spin_lock(&bfqd->eqm_lock); __bfq_bfqq_expire(bfqd, bfqq); + spin_unlock(&bfqd->eqm_lock); } /* @@ -1986,7 +2224,7 @@ static inline bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) */ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) { - struct bfq_queue *bfqq, *new_bfqq = NULL; + struct bfq_queue *bfqq; struct request *next_rq; enum bfqq_expiration reason = BFQ_BFQQ_BUDGET_TIMEOUT; @@ -1996,17 +2234,6 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); - /* - * If another queue has a request waiting within our mean seek - * distance, let it run. The expire code will check for close - * cooperators and put the close queue at the front of the - * service tree. If possible, merge the expiring queue with the - * new bfqq. - */ - new_bfqq = bfq_close_cooperator(bfqd, bfqq); - if (new_bfqq != NULL && bfqq->new_bfqq == NULL) - bfq_setup_merge(bfqq, new_bfqq); - if (bfq_may_expire_for_budg_timeout(bfqq) && !timer_pending(&bfqd->idle_slice_timer) && !bfq_bfqq_must_idle(bfqq)) @@ -2043,10 +2270,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) bfq_clear_bfqq_wait_request(bfqq); del_timer(&bfqd->idle_slice_timer); } - if (new_bfqq == NULL) - goto keep_queue; - else - goto expire; + goto keep_queue; } } @@ -2055,24 +2279,17 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * flight (possibly waiting for a completion) or is idling for a new * request, then keep it. */ - if (new_bfqq == NULL && (timer_pending(&bfqd->idle_slice_timer) || - (bfqq->dispatched != 0 && bfq_bfqq_must_not_expire(bfqq)))) { + if (timer_pending(&bfqd->idle_slice_timer) || + (bfqq->dispatched != 0 && bfq_bfqq_must_not_expire(bfqq))) { bfqq = NULL; goto keep_queue; - } else if (new_bfqq != NULL && timer_pending(&bfqd->idle_slice_timer)) { - /* - * Expiring the queue because there is a close cooperator, - * cancel timer. - */ - bfq_clear_bfqq_wait_request(bfqq); - del_timer(&bfqd->idle_slice_timer); } reason = BFQ_BFQQ_NO_MORE_REQUESTS; expire: bfq_bfqq_expire(bfqd, bfqq, 0, reason); new_queue: - bfqq = bfq_set_in_service_queue(bfqd, new_bfqq); + bfqq = bfq_set_in_service_queue(bfqd); bfq_log(bfqd, "select_queue: new queue %d returned", bfqq != NULL ? bfqq->pid : 0); keep_queue: @@ -2082,9 +2299,8 @@ keep_queue: static void bfq_update_raising_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) { - if (bfqq->wr_coeff > 1) { /* queue is being boosted */ - struct bfq_entity *entity = &bfqq->entity; - + struct bfq_entity *entity = &bfqq->entity; + if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ bfq_log_bfqq(bfqd, bfqq, "raising period dur %u/%u msec, old raising coeff %u, w %d(%d)", jiffies_to_msecs(jiffies - @@ -2108,13 +2324,16 @@ static void bfq_update_raising_data(struct bfq_data *bfqd, bfq_log_bfqq(bfqd, bfqq, "wrais ending at %lu, rais_max_time %u", bfqq->last_wr_start_finish, - jiffies_to_msecs(bfqq->wr_cur_max_time)); + jiffies_to_msecs(bfqq-> + wr_cur_max_time)); bfq_bfqq_end_wr(bfqq); - __bfq_entity_update_weight_prio( - bfq_entity_service_tree(entity), - entity); } } + /* Update weight both if it must be raised and if it must be lowered */ + if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1)) + __bfq_entity_update_weight_prio( + bfq_entity_service_tree(entity), + entity); } /* @@ -2218,8 +2437,11 @@ static int bfq_forced_dispatch(struct bfq_data *bfqd) int dispatched = 0; bfqq = bfqd->in_service_queue; - if (bfqq != NULL) + if (bfqq != NULL) { + spin_lock(&bfqd->eqm_lock); __bfq_bfqq_expire(bfqd, bfqq); + spin_unlock(&bfqd->eqm_lock); + } /* * Loop through classes, and be careful to leave the scheduler @@ -2335,6 +2557,7 @@ static void bfq_put_cooperator(struct bfq_queue *bfqq) } } +/* Coop lock is taken in __bfq_exit_single_io_context() */ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) { if (bfqq == bfqd->in_service_queue) { @@ -2411,6 +2634,7 @@ static void bfq_changed_ioprio(struct io_context *ioc, if (unlikely(bfqd == NULL)) return; + spin_lock(&bfqd->eqm_lock); bfqq = cic->cfqq[BLK_RW_ASYNC]; if (bfqq != NULL) { bfqg = container_of(bfqq->entity.sched_data, struct bfq_group, @@ -2427,6 +2651,7 @@ static void bfq_changed_ioprio(struct io_context *ioc, } bfqq = cic->cfqq[BLK_RW_SYNC]; + spin_unlock(&bfqd->eqm_lock); if (bfqq != NULL) bfq_mark_bfqq_prio_changed(bfqq); @@ -2487,11 +2712,13 @@ retry: bfqq = new_bfqq; new_bfqq = NULL; } else if (gfp_mask & __GFP_WAIT) { + spin_unlock(&bfqd->eqm_lock); spin_unlock_irq(bfqd->queue->queue_lock); new_bfqq = kmem_cache_alloc_node(bfq_pool, gfp_mask | __GFP_ZERO, bfqd->queue->node); spin_lock_irq(bfqd->queue->queue_lock); + spin_lock(&bfqd->eqm_lock); if (new_bfqq != NULL) goto retry; } else { @@ -2627,6 +2854,10 @@ static void bfq_update_idle_window(struct bfq_data *bfqd, if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq)) return; + /* Idle window just restored, statistics are meaningless. */ + if (bfq_bfqq_just_split(bfqq)) + return; + enable_idle = bfq_bfqq_idle_window(bfqq); if (atomic_read(&cic->ioc->nr_tasks) == 0 || @@ -2674,6 +2905,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 || !BFQQ_SEEKY(bfqq)) bfq_update_idle_window(bfqd, bfqq, cic); + bfq_clear_bfqq_just_split(bfqq); bfq_log_bfqq(bfqd, bfqq, "rq_enqueued: idle_window=%d (seeky %d, mean %llu)", @@ -2734,13 +2966,50 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, static void bfq_insert_request(struct request_queue *q, struct request *rq) { struct bfq_data *bfqd = q->elevator->elevator_data; - struct bfq_queue *bfqq = RQ_BFQQ(rq); + struct bfq_queue *bfqq = RQ_BFQQ(rq), *new_bfqq; assert_spin_locked(bfqd->queue->queue_lock); + + /* + * An unplug may trigger a requeue of a request from the device + * driver: make sure we are in process context while trying to + * merge two bfq_queues. + */ + spin_lock(&bfqd->eqm_lock); + if (!in_interrupt()) { + new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); + if (new_bfqq != NULL) { + if (cic_to_bfqq(RQ_CIC(rq), 1) != bfqq) + new_bfqq = cic_to_bfqq(RQ_CIC(rq), 1); + /* + * Release the request's reference to the old bfqq + * and make sure one is taken to the shared queue. + */ + new_bfqq->allocated[rq_data_dir(rq)]++; + bfqq->allocated[rq_data_dir(rq)]--; + atomic_inc(&new_bfqq->ref); + bfq_put_queue(bfqq); + if (cic_to_bfqq(RQ_CIC(rq), 1) == bfqq) + bfq_merge_bfqqs(bfqd, RQ_CIC(rq), + bfqq, new_bfqq); + rq->elevator_private[1] = new_bfqq; + bfqq = new_bfqq; + } + } + spin_unlock(&bfqd->eqm_lock); + bfq_init_prio_data(bfqq, RQ_CIC(rq)->ioc); bfq_add_request(rq); + /* + * Here a newly-created bfq_queue has already started a weight-raising + * period: clear wr_time_left to prevent bfq_bfqq_save_state() + * from assigning it a full weight-raising period. See the detailed + * comments about this field in bfq_init_icq(). + */ + if (bfqq->cic != NULL) + bfqq->cic->wr_time_left = 0; rq_set_fifo_time(rq, jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)]); list_add_tail(&rq->queuelist, &bfqq->fifo); @@ -2873,7 +3142,9 @@ static int bfq_may_queue(struct request_queue *q, int rw) if (cic == NULL) return ELV_MQUEUE_MAY; + spin_lock(&bfqd->eqm_lock); bfqq = cic_to_bfqq(cic, rw_is_sync(rw)); + spin_unlock(&bfqd->eqm_lock); if (bfqq != NULL) { bfq_init_prio_data(bfqq, cic->ioc); @@ -2907,18 +3178,6 @@ static void bfq_put_request(struct request *rq) } } -static struct bfq_queue * -bfq_merge_bfqqs(struct bfq_data *bfqd, struct cfq_io_context *cic, - struct bfq_queue *bfqq) -{ - bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", - (long unsigned)bfqq->new_bfqq->pid); - cic_set_bfqq(cic, bfqq->new_bfqq, 1); - bfq_mark_bfqq_coop(bfqq->new_bfqq); - bfq_put_queue(bfqq); - return cic_to_bfqq(cic, 1); -} - /* * Returns NULL if a new bfqq should be allocated, or the old bfqq if this * was the last process referring to said bfqq. @@ -2927,6 +3186,9 @@ static struct bfq_queue * bfq_split_bfqq(struct cfq_io_context *cic, struct bfq_queue *bfqq) { bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); + + put_io_context(cic->ioc); + if (bfqq_process_refs(bfqq) == 1) { bfqq->pid = current->pid; bfq_clear_bfqq_coop(bfqq); @@ -2955,6 +3217,7 @@ static int bfq_set_request(struct request_queue *q, struct request *rq, struct bfq_queue *bfqq; struct bfq_group *bfqg; unsigned long flags; + bool split = false; might_sleep_if(gfp_mask & __GFP_WAIT); @@ -2967,30 +3230,22 @@ static int bfq_set_request(struct request_queue *q, struct request *rq, bfqg = bfq_cic_update_cgroup(cic); + spin_lock(&bfqd->eqm_lock); + new_queue: bfqq = cic_to_bfqq(cic, is_sync); if (bfqq == NULL || bfqq == &bfqd->oom_bfqq) { bfqq = bfq_get_queue(bfqd, bfqg, is_sync, cic->ioc, gfp_mask); cic_set_bfqq(cic, bfqq, is_sync); } else { - /* - * If the queue was seeky for too long, break it apart. - */ + /* If the queue was seeky for too long, break it apart. */ if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); bfqq = bfq_split_bfqq(cic, bfqq); + split = true; if (!bfqq) goto new_queue; } - - /* - * Check to see if this queue is scheduled to merge with - * another closely cooperating queue. The merging of queues - * happens here as it must be done in process context. - * The reference on new_bfqq was taken in merge_bfqqs. - */ - if (bfqq->new_bfqq != NULL) - bfqq = bfq_merge_bfqqs(bfqd, cic, bfqq); } bfqq->allocated[rw]++; @@ -2998,11 +3253,32 @@ new_queue: bfq_log_bfqq(bfqd, bfqq, "set_request: bfqq %p, %d", bfqq, atomic_read(&bfqq->ref)); - spin_unlock_irqrestore(q->queue_lock, flags); - rq->elevator_private[0] = cic; rq->elevator_private[1] = bfqq; + /* + * If a bfq_queue has only one process reference, it is owned + * by only one cfq_io_context: we can set the cic field of the + * bfq_queue to the address of that structure. Also, if the + * queue has just been split, mark a flag so that the + * information is available to the other scheduler hooks. + */ + if (bfqq_process_refs(bfqq) == 1) { + bfqq->cic = cic; + if (split) { + bfq_mark_bfqq_just_split(bfqq); + /* + * If the queue has just been split from a shared queue, + * restore the idle window and the possible weight + * raising period. + */ + bfq_bfqq_resume_state(bfqq, cic); + } + } + + spin_unlock(&bfqd->eqm_lock); + spin_unlock_irqrestore(q->queue_lock, flags); + return 0; queue_fail: @@ -3197,6 +3473,7 @@ static void *bfq_init_queue(struct request_queue *q) bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, 1, 0); atomic_inc(&bfqd->oom_bfqq.ref); + spin_lock_init(&bfqd->eqm_lock); INIT_LIST_HEAD(&bfqd->cic_list); bfqd->queue = q; @@ -3376,7 +3653,8 @@ SHOW_FUNCTION(bfq_timeout_async_show, bfqd->bfq_timeout[BLK_RW_ASYNC], 1); SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0); SHOW_FUNCTION(bfq_wr_coeff_show, bfqd->bfq_wr_coeff, 0); SHOW_FUNCTION(bfq_wr_rt_max_time_show, bfqd->bfq_wr_rt_max_time, 1); -SHOW_FUNCTION(bfq_wr_min_idle_time_show, bfqd->bfq_wr_min_idle_time, 1); +SHOW_FUNCTION(bfq_wr_min_idle_time_show, bfqd->bfq_wr_min_idle_time, + 1); SHOW_FUNCTION(bfq_wr_min_inter_arr_async_show, bfqd->bfq_wr_min_inter_arr_async, 1); SHOW_FUNCTION(bfq_wr_max_softrt_rate_show, bfqd->bfq_wr_max_softrt_rate, 0); @@ -3414,8 +3692,8 @@ STORE_FUNCTION(bfq_timeout_async_store, &bfqd->bfq_timeout[BLK_RW_ASYNC], 0, INT_MAX, 1); STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0); STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1); -STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX, - 1); +STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, + INT_MAX, 1); STORE_FUNCTION(bfq_wr_min_idle_time_store, &bfqd->bfq_wr_min_idle_time, 0, INT_MAX, 1); STORE_FUNCTION(bfq_wr_min_inter_arr_async_store, diff --git a/block/bfq-sched.c b/block/bfq-sched.c index b677c0f..c5be9ed 100644 --- a/block/bfq-sched.c +++ b/block/bfq-sched.c @@ -1079,34 +1079,6 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) return bfqq; } -/* - * Forced extraction of the given queue. - */ -static void bfq_get_next_queue_forced(struct bfq_data *bfqd, - struct bfq_queue *bfqq) -{ - struct bfq_entity *entity; - struct bfq_sched_data *sd; - - BUG_ON(bfqd->in_service_queue != NULL); - - entity = &bfqq->entity; - /* - * Bubble up extraction/update from the leaf to the root. - */ - for_each_entity(entity) { - sd = entity->sched_data; - bfq_update_budget(entity); - bfq_update_vtime(bfq_entity_service_tree(entity)); - bfq_active_extract(bfq_entity_service_tree(entity), entity); - sd->in_service_entity = entity; - sd->next_in_service = NULL; - entity->service = 0; - } - - return; -} - static void __bfq_bfqd_reset_in_service(struct bfq_data *bfqd) { if (bfqd->in_service_cic != NULL) { diff --git a/block/bfq.h b/block/bfq.h index faf40c6..663f790 100644 --- a/block/bfq.h +++ b/block/bfq.h @@ -212,6 +212,12 @@ struct bfq_group; * task associated with the queue it is deemed as soft * real-time (see the comments to the function * bfq_bfqq_softrt_next_start()) + * @last_idle_bklogged: time of the last transition of the @bfq_queue from + * idle to backlogged + * @service_from_backlogged: cumulative service received from the @bfq_queue + * since the last transition from idle to backlogged + * @cic: pointer to the cfq_io_context owning the bfq_queue, set to %NULL if the + * queue is shared * * A bfq_queue is a leaf request queue; it can be associated with an io_context * or more, if it is async or shared between cooperating processes. @cgroup @@ -263,6 +269,8 @@ struct bfq_queue { unsigned int wr_coeff; unsigned long last_idle_bklogged; unsigned long service_from_backlogged; + + struct cfq_io_context *cic; }; enum bfq_device_speed { @@ -274,11 +282,13 @@ enum bfq_device_speed { * struct bfq_data - per device data structure. * @queue: request queue for the managed device. * @root_group: root bfq_group for the device. - * @active_numerous_groups: number of bfq_groups containing more than one - * active @bfq_entity. * @rq_pos_tree: rbtree sorted by next_request position, * used when determining if two or more queues * have interleaving requests (see bfq_close_cooperator). + * @eqm_lock: spinlock used to protect all data structures pertaining + * the Early Queue Merge (EQM) mechanism. + * @active_numerous_groups: number of bfq_groups containing more than one + * active @bfq_entity. * @queue_weights_tree: rbtree of weight counters of @bfq_queues, sorted by * weight. Used to keep track of whether all @bfq_queues * have the same weight. The tree contains one counter @@ -360,20 +370,21 @@ enum bfq_device_speed { * they are charged for the whole allocated budget, to try * to preserve a behavior reasonably fair among them, but * without service-domain guarantees). - * @bfq_raising_coeff: Maximum factor by which the weight of a boosted - * queue is multiplied - * @bfq_raising_max_time: maximum duration of a weight-raising period (jiffies) - * @bfq_raising_rt_max_time: maximum duration for soft real-time processes - * @bfq_raising_min_idle_time: minimum idle period after which weight-raising - * may be reactivated for a queue (in jiffies) - * @bfq_raising_min_inter_arr_async: minimum period between request arrivals - * after which weight-raising may be - * reactivated for an already busy queue - * (in jiffies) - * @bfq_raising_max_softrt_rate: max service-rate for a soft real-time queue, - * sectors per seconds + * @bfq_wr_coeff: Maximum factor by which the weight of a boosted + * queue is multiplied + * @bfq_wr_max_time: maximum duration of a weight-raising period (jiffies) + * @bfq_wr_rt_max_time: maximum duration for soft real-time processes + * @bfq_wr_min_idle_time: minimum idle period after which weight-raising + * may be reactivated for a queue (in jiffies) + * @bfq_wr_min_inter_arr_async: minimum period between request arrivals + * after which weight-raising may be + * reactivated for an already busy queue + * (in jiffies) + * @bfq_wr_max_softrt_rate: max service-rate for a soft real-time queue, + * sectors per seconds * @RT_prod: cached value of the product R*T used for computing the maximum * duration of the weight raising automatically + * @device_speed: device speed class for the low-latency heuristic * @oom_bfqq: fallback dummy bfqq for extreme OOM conditions * * All the fields are protected by the @queue lock. @@ -387,6 +398,8 @@ struct bfq_data { #endif struct rb_root rq_pos_tree; + spinlock_t eqm_lock; + struct rb_root queue_weights_tree; struct rb_root group_weights_tree; @@ -464,8 +477,9 @@ enum bfqq_state_flags { * until budget timeout */ BFQ_BFQQ_FLAG_coop, /* bfqq is shared */ - BFQ_BFQQ_FLAG_split_coop, /* shared bfqq will be splitted */ - BFQ_BFQQ_FLAG_softrt_update, /* needs softrt-next-start update */ + BFQ_BFQQ_FLAG_split_coop, /* shared bfqq will be split */ + BFQ_BFQQ_FLAG_just_split, /* queue has just been split */ + BFQ_BFQQ_FLAG_softrt_update, /* may need softrt-next-start update */ }; #define BFQ_BFQQ_FNS(name) \ @@ -493,6 +507,7 @@ BFQ_BFQQ_FNS(budget_new); BFQ_BFQQ_FNS(constantly_seeky); BFQ_BFQQ_FNS(coop); BFQ_BFQQ_FNS(split_coop); +BFQ_BFQQ_FNS(just_split); BFQ_BFQQ_FNS(softrt_update); #undef BFQ_BFQQ_FNS diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 69fdd58..65b7a04 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -22,6 +22,9 @@ struct cfq_io_context { struct cfq_ttime ttime; + unsigned int wr_time_left; + unsigned int saved_idle_window; + struct list_head queue_list; struct hlist_node cic_list; -- 1.9.2