From: Chunguang Xu <[email protected]>
Some misc updates, put together mainly to facilitate patch management.
Chunguang Xu (8):
bfq: introduce bfq_entity_to_bfqg helper method
bfq: convert the type of bfq_group.bfqd to bfq_data*
bfq: limit the IO depth of CLASS_IDLE to 1
bfq: keep the minimun bandwidth for CLASS_BE
bfq: remove unnecessary initialization logic
bfq: optimize the calculation of bfq_weight_to_ioprio()
bfq: reset entity->prio_changed in bfq_init_entity()
bfq: remove unnecessary BFQ_DEFAULT_GRP_IOPRIO
block/bfq-cgroup.c | 13 +++-----
block/bfq-iosched.c | 17 +++++++++--
block/bfq-iosched.h | 15 ++++++----
block/bfq-wf2q.c | 86 +++++++++++++++++++++++++++++++++++++----------------
4 files changed, 88 insertions(+), 43 deletions(-)
--
1.8.3.1
From: Chunguang Xu <[email protected]>
Introduce bfq_entity_to_bfqg() to make it easier to obtain the
bfq_group corresponding to the entity.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-cgroup.c | 6 ++----
block/bfq-iosched.h | 1 +
block/bfq-wf2q.c | 16 ++++++++++++----
3 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index b791e20..a5f544a 100644
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -309,8 +309,7 @@ struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
{
struct bfq_entity *group_entity = bfqq->entity.parent;
- return group_entity ? container_of(group_entity, struct bfq_group,
- entity) :
+ return group_entity ? bfq_entity_to_bfqg(group_entity) :
bfqq->bfqd->root_group;
}
@@ -610,8 +609,7 @@ struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
*/
entity = &bfqg->entity;
for_each_entity(entity) {
- struct bfq_group *curr_bfqg = container_of(entity,
- struct bfq_group, entity);
+ struct bfq_group *curr_bfqg = bfq_entity_to_bfqg(entity);
if (curr_bfqg != bfqd->root_group) {
parent = bfqg_parent(curr_bfqg);
if (!parent)
diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index b8e793c..a6f98e9 100644
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -941,6 +941,7 @@ struct bfq_group {
#endif
struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity);
+struct bfq_group *bfq_entity_to_bfqg(struct bfq_entity *entity);
/* --------------- main algorithm interface ----------------- */
diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c
index 070e34a..5ff0028 100644
--- a/block/bfq-wf2q.c
+++ b/block/bfq-wf2q.c
@@ -149,7 +149,7 @@ struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq)
if (!group_entity)
group_entity = &bfqq->bfqd->root_group->entity;
- return container_of(group_entity, struct bfq_group, entity);
+ return bfq_entity_to_bfqg(group_entity);
}
/*
@@ -208,7 +208,7 @@ static bool bfq_no_longer_next_in_service(struct bfq_entity *entity)
if (bfq_entity_to_bfqq(entity))
return true;
- bfqg = container_of(entity, struct bfq_group, entity);
+ bfqg = bfq_entity_to_bfqg(entity);
/*
* The field active_entities does not always contain the
@@ -266,6 +266,15 @@ struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity)
return bfqq;
}
+struct bfq_group *bfq_entity_to_bfqg(struct bfq_entity *entity)
+{
+ struct bfq_group *bfqg = NULL;
+
+ if (entity->my_sched_data)
+ bfqg = container_of(entity, struct bfq_group, entity);
+
+ return bfqg;
+}
/**
* bfq_delta - map service into the virtual time domain.
@@ -1001,8 +1010,7 @@ static void __bfq_activate_entity(struct bfq_entity *entity,
#ifdef CONFIG_BFQ_GROUP_IOSCHED
if (!bfq_entity_to_bfqq(entity)) { /* bfq_group */
- struct bfq_group *bfqg =
- container_of(entity, struct bfq_group, entity);
+ struct bfq_group *bfqg = bfq_entity_to_bfqg(entity);
struct bfq_data *bfqd = bfqg->bfqd;
if (!entity->in_groups_with_pending_reqs) {
--
1.8.3.1
From: Chunguang Xu <[email protected]>
Setting bfq_group.bfqd to void* type does not seem to make much sense.
This will cause unnecessary type conversion. Perhaps it would be better
to change it to bfq_data* type.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-cgroup.c | 2 +-
block/bfq-iosched.h | 2 +-
block/bfq-wf2q.c | 6 +++---
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index a5f544a..50d06c7 100644
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -224,7 +224,7 @@ void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
{
blkg_rwstat_add(&bfqg->stats.queued, op, 1);
bfqg_stats_end_empty_time(&bfqg->stats);
- if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))
+ if (!(bfqq == bfqg->bfqd->in_service_queue))
bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
}
diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index a6f98e9..28d8590 100644
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -914,7 +914,7 @@ struct bfq_group {
struct bfq_entity entity;
struct bfq_sched_data sched_data;
- void *bfqd;
+ struct bfq_data *bfqd;
struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR];
struct bfq_queue *async_idle_bfqq;
diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c
index 5ff0028..276f225 100644
--- a/block/bfq-wf2q.c
+++ b/block/bfq-wf2q.c
@@ -498,7 +498,7 @@ static void bfq_active_insert(struct bfq_service_tree *st,
#ifdef CONFIG_BFQ_GROUP_IOSCHED
sd = entity->sched_data;
bfqg = container_of(sd, struct bfq_group, sched_data);
- bfqd = (struct bfq_data *)bfqg->bfqd;
+ bfqd = bfqg->bfqd;
#endif
if (bfqq)
list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list);
@@ -597,7 +597,7 @@ static void bfq_active_extract(struct bfq_service_tree *st,
#ifdef CONFIG_BFQ_GROUP_IOSCHED
sd = entity->sched_data;
bfqg = container_of(sd, struct bfq_group, sched_data);
- bfqd = (struct bfq_data *)bfqg->bfqd;
+ bfqd = bfqg->bfqd;
#endif
if (bfqq)
list_del(&bfqq->bfqq_list);
@@ -743,7 +743,7 @@ struct bfq_service_tree *
else {
sd = entity->my_sched_data;
bfqg = container_of(sd, struct bfq_group, sched_data);
- bfqd = (struct bfq_data *)bfqg->bfqd;
+ bfqd = bfqg->bfqd;
}
#endif
--
1.8.3.1
From: Chunguang Xu <[email protected]>
The IO depth of queues belong to CLASS_IDLE is limited to 1,
so that it can avoid introducing a larger tail latency under
a device with a larger IO depth. Although limiting the IO
depth may reduce the performance of idle_class, it is
generally not a big problem, because idle_class usually does
not have strict performance requirements.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-iosched.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index ec482e6..29940ec 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -4808,6 +4808,17 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
if (!bfqq)
goto exit;
+ /*
+ * Here, the IO depth of queues belong to CLASS_IDLE is limited
+ * to 1, so that it can avoid introducing a larger tail latency
+ * under a device with a larger IO depth. Although limiting the
+ * IO depth may reduce the performance of idle_class, it is
+ * generally not a big problem, because idle_class usually
+ * does not have strict performance requirements.
+ */
+ if (bfq_class_idle(bfqq) && bfqq->dispatched)
+ goto exit;
+
rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq);
if (rq) {
--
1.8.3.1
From: Chunguang Xu <[email protected]>
The value range of ioprio is [0, 7], but the result of
bfq_weight_to_ioprio() may exceed this range, so simple
optimization is required.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-wf2q.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c
index 619ed21..ea5b90d 100644
--- a/block/bfq-wf2q.c
+++ b/block/bfq-wf2q.c
@@ -527,8 +527,9 @@ unsigned short bfq_ioprio_to_weight(int ioprio)
*/
static unsigned short bfq_weight_to_ioprio(int weight)
{
- return max_t(int, 0,
- IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight);
+ int ioprio = IOPRIO_BE_NR - weight / BFQ_WEIGHT_CONVERSION_COEFF;
+
+ return ioprio < 0 ? 0 : min_t(int, ioprio, IOPRIO_BE_NR - 1);
}
static void bfq_get_entity(struct bfq_entity *entity)
--
1.8.3.1
From: Chunguang Xu <[email protected]>
Since we will initialize sched_data.service_tree[] in
bfq_init_root_group(), bfq_create_group_hierarchy() can
ignore this part of the initialization, which can avoid
repeated initialization.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-cgroup.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index 50d06c7..c8c68dc 100644
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -1429,15 +1429,11 @@ void bfqg_and_blkg_put(struct bfq_group *bfqg) {}
struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
{
struct bfq_group *bfqg;
- int i;
bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);
if (!bfqg)
return NULL;
- for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
- bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
-
return bfqg;
}
#endif /* CONFIG_BFQ_GROUP_IOSCHED */
--
1.8.3.1
From: Chunguang Xu <[email protected]>
CLASS_RT will preempt other classes, which may starve. At
present, CLASS_IDLE has alleviated the starvation problem
through the minimum bandwidth mechanism. Similarly, we
should do the same for CLASS_BE.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-iosched.c | 6 ++++--
block/bfq-iosched.h | 11 ++++++----
block/bfq-wf2q.c | 59 ++++++++++++++++++++++++++++++++++++++---------------
3 files changed, 53 insertions(+), 23 deletions(-)
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 29940ec..89d4646 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -6537,9 +6537,11 @@ static void bfq_init_root_group(struct bfq_group *root_group,
root_group->bfqd = bfqd;
#endif
root_group->rq_pos_tree = RB_ROOT;
- for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
+ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) {
root_group->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
- root_group->sched_data.bfq_class_idle_last_service = jiffies;
+ root_group->sched_data.bfq_class_last_service[i] = jiffies;
+ }
+ root_group->sched_data.class_timeout_last_check = jiffies;
}
static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index 28d8590..da636a8 100644
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -13,7 +13,7 @@
#include "blk-cgroup-rwstat.h"
#define BFQ_IOPRIO_CLASSES 3
-#define BFQ_CL_IDLE_TIMEOUT (HZ/5)
+#define BFQ_CLASS_TIMEOUT (HZ/5)
#define BFQ_MIN_WEIGHT 1
#define BFQ_MAX_WEIGHT 1000
@@ -97,9 +97,12 @@ struct bfq_sched_data {
struct bfq_entity *next_in_service;
/* array of service trees, one per ioprio_class */
struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES];
- /* last time CLASS_IDLE was served */
- unsigned long bfq_class_idle_last_service;
-
+ /* last time the class was served */
+ unsigned long bfq_class_last_service[BFQ_IOPRIO_CLASSES];
+ /* last time class timeout was checked */
+ unsigned long class_timeout_last_check;
+ /* next index to check class timeout */
+ unsigned int next_class_index;
};
/**
diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c
index 276f225..619ed21 100644
--- a/block/bfq-wf2q.c
+++ b/block/bfq-wf2q.c
@@ -1168,6 +1168,7 @@ bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree)
{
struct bfq_sched_data *sd = entity->sched_data;
struct bfq_service_tree *st;
+ int idx = bfq_class_idx(entity);
bool is_in_service;
if (!entity->on_st_or_in_serv) /*
@@ -1207,6 +1208,7 @@ bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree)
else
bfq_idle_insert(st, entity);
+ sd->bfq_class_last_service[idx] = jiffies;
return true;
}
@@ -1435,6 +1437,45 @@ static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st,
return entity;
}
+static int bfq_select_next_class(struct bfq_sched_data *sd)
+{
+ struct bfq_service_tree *st = sd->service_tree;
+ unsigned long last_check, last_serve;
+ int i, class_idx, next_class = 0;
+ bool found = false;
+
+ /*
+ * we needed to guarantee a minimum bandwidth for each class (if
+ * there is some active entity in this class). This should also
+ * mitigate priority-inversion problems in case a low priority
+ * task is holding file system resources.
+ */
+ last_check = sd->class_timeout_last_check;
+ if (time_is_after_jiffies(last_check + BFQ_CLASS_TIMEOUT))
+ return next_class;
+
+ sd->class_timeout_last_check = jiffies;
+ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) {
+ class_idx = (sd->next_class_index + i) % BFQ_IOPRIO_CLASSES;
+ last_serve = sd->bfq_class_last_service[class_idx];
+
+ if (time_is_after_jiffies(last_serve + BFQ_CLASS_TIMEOUT))
+ continue;
+
+ if (!RB_EMPTY_ROOT(&(st + class_idx)->active)) {
+ if (found)
+ continue;
+
+ next_class = class_idx++;
+ class_idx %= BFQ_IOPRIO_CLASSES;
+ sd->next_class_index = class_idx;
+ found = true;
+ }
+ sd->bfq_class_last_service[class_idx] = jiffies;
+ }
+ return next_class;
+}
+
/**
* bfq_lookup_next_entity - return the first eligible entity in @sd.
* @sd: the sched_data.
@@ -1448,24 +1489,8 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
bool expiration)
{
struct bfq_service_tree *st = sd->service_tree;
- struct bfq_service_tree *idle_class_st = st + (BFQ_IOPRIO_CLASSES - 1);
struct bfq_entity *entity = NULL;
- int class_idx = 0;
-
- /*
- * Choose from idle class, if needed to guarantee a minimum
- * bandwidth to this class (and if there is some active entity
- * in idle class). This should also mitigate
- * priority-inversion problems in case a low priority task is
- * holding file system resources.
- */
- if (time_is_before_jiffies(sd->bfq_class_idle_last_service +
- BFQ_CL_IDLE_TIMEOUT)) {
- if (!RB_EMPTY_ROOT(&idle_class_st->active))
- class_idx = BFQ_IOPRIO_CLASSES - 1;
- /* About to be served if backlogged, or not yet backlogged */
- sd->bfq_class_idle_last_service = jiffies;
- }
+ int class_idx = bfq_select_next_class(sd);
/*
* Find the next entity to serve for the highest-priority
--
1.8.3.1
From: Chunguang Xu <[email protected]>
Since weight, ioprio, ioprio_class will be updated in bfq_init_entity(),
st->wsum will be updated in __bfq_activate_entity(), so when it is first
active, it seems that __bfq_entity_update_weight_prio() has nothing to
do. By resetting entity->prio_change in bfq_init_entity(), we can avoid
unnecessary logic calls.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-cgroup.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index c8c68dc..79a5aaa 100644
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -426,6 +426,7 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
entity->weight = entity->new_weight;
entity->orig_weight = entity->new_weight;
+ entity->prio_changed = 0;
if (bfqq) {
bfqq->ioprio = bfqq->new_ioprio;
bfqq->ioprio_class = bfqq->new_ioprio_class;
--
1.8.3.1
From: Chunguang Xu <[email protected]>
BFQ_DEFAULT_GRP_IOPRIO seems to be unused, maybe we can remove it.
Signed-off-by: Chunguang Xu <[email protected]>
---
block/bfq-iosched.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index da636a8..91c8654 100644
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -22,7 +22,6 @@
#define BFQ_DEFAULT_QUEUE_IOPRIO 4
#define BFQ_WEIGHT_LEGACY_DFL 100
-#define BFQ_DEFAULT_GRP_IOPRIO 0
#define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE
#define MAX_PID_STR_LENGTH 12
--
1.8.3.1