Piteball
/
android_kernel_samsung_msm8976
mirror of https://github.com/team-infusion-developers/android_kernel_samsung_msm8976.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

block: introduce the BFQ-v7r8 I/O sched for 3.10.8+ 

Add the BFQ-v7r8 I/O scheduler to 3.10.8+.
The general structure is borrowed from CFQ, as much of the code for
handling I/O contexts Over time, several useful features have been
ported from CFQ as well (details in the changelog in README.BFQ). A
(bfq_)queue is associated to each task doing I/O on a device, and each
time a scheduling decision has to be made a queue is selected and served
until it expires.

    - Slices are given in the service domain: tasks are assigned
      budgets, measured in number of sectors. Once got the disk, a task
      must however consume its assigned budget within a configurable
      maximum time (by default, the maximum possible value of the
      budgets is automatically computed to comply with this timeout).
      This allows the desired latency vs "throughput boosting" tradeoff
      to be set.

    - Budgets are scheduled according to a variant of WF2Q+, implemented
      using an augmented rb-tree to take eligibility into account while
      preserving an O(log N) overall complexity.

    - A low-latency tunable is provided; if enabled, both interactive
      and soft real-time applications are guaranteed a very low latency.

    - Latency guarantees are preserved also in the presence of NCQ.

    - Also with flash-based devices, a high throughput is achieved
      while still preserving latency guarantees.

    - BFQ features Early Queue Merge (EQM), a sort of fusion of the
      cooperating-queue-merging and the preemption mechanisms present
      in CFQ. EQM is in fact a unified mechanism that tries to get a
      sequential read pattern, and hence a high throughput, with any
      set of processes performing interleaved I/O over a contiguous
      sequence of sectors.

    - BFQ supports full hierarchical scheduling, exporting a cgroups
      interface.  Since each node has a full scheduler, each group can
      be assigned its own weight.

    - If the cgroups interface is not used, only I/O priorities can be
      assigned to processes, with ioprio values mapped to weights
      with the relation weight = IOPRIO_BE_NR - ioprio.

    - ioprio classes are served in strict priority order, i.e., lower
      priority queues are not served as long as there are higher
      priority queues.  Among queues in the same class the bandwidth is
      distributed in proportion to the weight of each queue. A very
      thin extra bandwidth is however guaranteed to the Idle class, to
      prevent it from starving.

Change-Id: Iebf9be399041b89d79b54077da1a34a81d4e4238
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini.arianna@gmail.com>
This commit is contained in:
Paolo Valente 2013-05-09 19:10:02 +02:00 committed by LuK1337
parent 3c67a316fd
commit ac922cd71c
5 changed files with 6824 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

913
block/bfq-cgroup.c Normal file
View File

@ -0,0 +1,913 @@
/*
* BFQ: CGROUPS support.
*
* Based on ideas and code from CFQ:
* Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
*
* Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
* Paolo Valente <paolo.valente@unimore.it>
*
* Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
*
* Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ
* file.
*/
#ifdef CONFIG_CGROUP_BFQIO
static DEFINE_MUTEX(bfqio_mutex);
static bool bfqio_is_removed(struct cgroup *cgroup)
{
return test_bit(CGRP_REMOVED, &cgroup->flags);
}
static struct bfqio_cgroup bfqio_root_cgroup = {
.weight = BFQ_DEFAULT_GRP_WEIGHT,
.ioprio = BFQ_DEFAULT_GRP_IOPRIO,
.ioprio_class = BFQ_DEFAULT_GRP_CLASS,
};
static inline void bfq_init_entity(struct bfq_entity *entity,
struct bfq_group *bfqg)
{
entity->weight = entity->new_weight;
entity->orig_weight = entity->new_weight;
entity->ioprio = entity->new_ioprio;
entity->ioprio_class = entity->new_ioprio_class;
entity->parent = bfqg->my_entity;
entity->sched_data = &bfqg->sched_data;
}
static struct bfqio_cgroup *cgroup_to_bfqio(struct cgroup *cgroup)
{
return container_of(cgroup_subsys_state(cgroup, bfqio_subsys_id),
struct bfqio_cgroup, css);
}
/*
* Search the bfq_group for bfqd into the hash table (by now only a list)
* of bgrp. Must be called under rcu_read_lock().
*/
static struct bfq_group *bfqio_lookup_group(struct bfqio_cgroup *bgrp,
struct bfq_data *bfqd)
{
struct bfq_group *bfqg;
void *key;
hlist_for_each_entry_rcu(bfqg, &bgrp->group_data, group_node) {
key = rcu_dereference(bfqg->bfqd);
if (key == bfqd)
return bfqg;
}
return NULL;
}
static inline void bfq_group_init_entity(struct bfqio_cgroup *bgrp,
struct bfq_group *bfqg)
{
struct bfq_entity *entity = &bfqg->entity;
/*
* If the weight of the entity has never been set via the sysfs
* interface, then bgrp->weight == 0. In this case we initialize
* the weight from the current ioprio value. Otherwise, the group
* weight, if set, has priority over the ioprio value.
*/
if (bgrp->weight == 0) {
entity->new_weight = bfq_ioprio_to_weight(bgrp->ioprio);
entity->new_ioprio = bgrp->ioprio;
} else {
if (bgrp->weight < BFQ_MIN_WEIGHT ||
bgrp->weight > BFQ_MAX_WEIGHT) {
printk(KERN_CRIT "bfq_group_init_entity: "
"bgrp->weight %d\n", bgrp->weight);
BUG();
}
entity->new_weight = bgrp->weight;
entity->new_ioprio = bfq_weight_to_ioprio(bgrp->weight);
}
entity->orig_weight = entity->weight = entity->new_weight;
entity->ioprio = entity->new_ioprio;
entity->ioprio_class = entity->new_ioprio_class = bgrp->ioprio_class;
entity->my_sched_data = &bfqg->sched_data;
bfqg->active_entities = 0;
}
static inline void bfq_group_set_parent(struct bfq_group *bfqg,
struct bfq_group *parent)
{
struct bfq_entity *entity;
BUG_ON(parent == NULL);
BUG_ON(bfqg == NULL);
entity = &bfqg->entity;
entity->parent = parent->my_entity;
entity->sched_data = &parent->sched_data;
}
/**
* bfq_group_chain_alloc - allocate a chain of groups.
* @bfqd: queue descriptor.
* @cgroup: the leaf cgroup this chain starts from.
*
* Allocate a chain of groups starting from the one belonging to
* @cgroup up to the root cgroup. Stop if a cgroup on the chain
* to the root has already an allocated group on @bfqd.
*/
static struct bfq_group *bfq_group_chain_alloc(struct bfq_data *bfqd,
struct cgroup *cgroup)
{
struct bfqio_cgroup *bgrp;
struct bfq_group *bfqg, *prev = NULL, *leaf = NULL;
for (; cgroup != NULL; cgroup = cgroup->parent) {
bgrp = cgroup_to_bfqio(cgroup);
bfqg = bfqio_lookup_group(bgrp, bfqd);
if (bfqg != NULL) {
/*
* All the cgroups in the path from there to the
* root must have a bfq_group for bfqd, so we don't
* need any more allocations.
*/
break;
}
bfqg = kzalloc(sizeof(*bfqg), GFP_ATOMIC);
if (bfqg == NULL)
goto cleanup;
bfq_group_init_entity(bgrp, bfqg);
bfqg->my_entity = &bfqg->entity;
if (leaf == NULL) {
leaf = bfqg;
prev = leaf;
} else {
bfq_group_set_parent(prev, bfqg);
/*
* Build a list of allocated nodes using the bfqd
* filed, that is still unused and will be
* initialized only after the node will be
* connected.
*/
prev->bfqd = bfqg;
prev = bfqg;
}
}
return leaf;
cleanup:
while (leaf != NULL) {
prev = leaf;
leaf = leaf->bfqd;
kfree(prev);
}
return NULL;
}
/**
* bfq_group_chain_link - link an allocated group chain to a cgroup
* hierarchy.
* @bfqd: the queue descriptor.
* @cgroup: the leaf cgroup to start from.
* @leaf: the leaf group (to be associated to @cgroup).
*
* Try to link a chain of groups to a cgroup hierarchy, connecting the
* nodes bottom-up, so we can be sure that when we find a cgroup in the
* hierarchy that already as a group associated to @bfqd all the nodes
* in the path to the root cgroup have one too.
*
* On locking: the queue lock protects the hierarchy (there is a hierarchy
* per device) while the bfqio_cgroup lock protects the list of groups
* belonging to the same cgroup.
*/
static void bfq_group_chain_link(struct bfq_data *bfqd, struct cgroup *cgroup,
struct bfq_group *leaf)
{
struct bfqio_cgroup *bgrp;
struct bfq_group *bfqg, *next, *prev = NULL;
unsigned long flags;
assert_spin_locked(bfqd->queue->queue_lock);
for (; cgroup != NULL && leaf != NULL; cgroup = cgroup->parent) {
bgrp = cgroup_to_bfqio(cgroup);
next = leaf->bfqd;
bfqg = bfqio_lookup_group(bgrp, bfqd);
BUG_ON(bfqg != NULL);
spin_lock_irqsave(&bgrp->lock, flags);
rcu_assign_pointer(leaf->bfqd, bfqd);
hlist_add_head_rcu(&leaf->group_node, &bgrp->group_data);
hlist_add_head(&leaf->bfqd_node, &bfqd->group_list);
spin_unlock_irqrestore(&bgrp->lock, flags);
prev = leaf;
leaf = next;
}
BUG_ON(cgroup == NULL && leaf != NULL);
if (cgroup != NULL && prev != NULL) {
bgrp = cgroup_to_bfqio(cgroup);
bfqg = bfqio_lookup_group(bgrp, bfqd);
bfq_group_set_parent(prev, bfqg);
}
}
/**
* bfq_find_alloc_group - return the group associated to @bfqd in @cgroup.
* @bfqd: queue descriptor.
* @cgroup: cgroup being searched for.
*
* Return a group associated to @bfqd in @cgroup, allocating one if
* necessary. When a group is returned all the cgroups in the path
* to the root have a group associated to @bfqd.
*
* If the allocation fails, return the root group: this breaks guarantees
* but is a safe fallback. If this loss becomes a problem it can be
* mitigated using the equivalent weight (given by the product of the
* weights of the groups in the path from @group to the root) in the
* root scheduler.
*
* We allocate all the missing nodes in the path from the leaf cgroup
* to the root and we connect the nodes only after all the allocations
* have been successful.
*/
static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
struct cgroup *cgroup)
{
struct bfqio_cgroup *bgrp = cgroup_to_bfqio(cgroup);
struct bfq_group *bfqg;
bfqg = bfqio_lookup_group(bgrp, bfqd);
if (bfqg != NULL)
return bfqg;
bfqg = bfq_group_chain_alloc(bfqd, cgroup);
if (bfqg != NULL)
bfq_group_chain_link(bfqd, cgroup, bfqg);
else
bfqg = bfqd->root_group;
return bfqg;
}
/**
* bfq_bfqq_move - migrate @bfqq to @bfqg.
* @bfqd: queue descriptor.
* @bfqq: the queue to move.
* @entity: @bfqq's entity.
* @bfqg: the group to move to.
*
* Move @bfqq to @bfqg, deactivating it from its old group and reactivating
* it on the new one. Avoid putting the entity on the old group idle tree.
*
* Must be called under the queue lock; the cgroup owning @bfqg must
* not disappear (by now this just means that we are called under
* rcu_read_lock()).
*/
static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
struct bfq_entity *entity, struct bfq_group *bfqg)
{
int busy, resume;
busy = bfq_bfqq_busy(bfqq);
resume = !RB_EMPTY_ROOT(&bfqq->sort_list);
BUG_ON(resume && !entity->on_st);
BUG_ON(busy && !resume && entity->on_st &&
bfqq != bfqd->in_service_queue);
if (busy) {
BUG_ON(atomic_read(&bfqq->ref) < 2);
if (!resume)
bfq_del_bfqq_busy(bfqd, bfqq, 0);
else
bfq_deactivate_bfqq(bfqd, bfqq, 0);
} else if (entity->on_st)
bfq_put_idle_entity(bfq_entity_service_tree(entity), entity);
/*
* Here we use a reference to bfqg. We don't need a refcounter
* as the cgroup reference will not be dropped, so that its
* destroy() callback will not be invoked.
*/
entity->parent = bfqg->my_entity;
entity->sched_data = &bfqg->sched_data;
if (busy && resume)
bfq_activate_bfqq(bfqd, bfqq);
if (bfqd->in_service_queue == NULL && !bfqd->rq_in_driver)
bfq_schedule_dispatch(bfqd);
}
/**
* __bfq_bic_change_cgroup - move @bic to @cgroup.
* @bfqd: the queue descriptor.
* @bic: the bic to move.
* @cgroup: the cgroup to move to.
*
* Move bic to cgroup, assuming that bfqd->queue is locked; the caller
* has to make sure that the reference to cgroup is valid across the call.
*
* NOTE: an alternative approach might have been to store the current
* cgroup in bfqq and getting a reference to it, reducing the lookup
* time here, at the price of slightly more complex code.
*/
static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
struct bfq_io_cq *bic,
struct cgroup *cgroup)
{
struct bfq_queue *async_bfqq = bic_to_bfqq(bic, 0);
struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, 1);
struct bfq_entity *entity;
struct bfq_group *bfqg;
struct bfqio_cgroup *bgrp;
bgrp = cgroup_to_bfqio(cgroup);
bfqg = bfq_find_alloc_group(bfqd, cgroup);
if (async_bfqq != NULL) {
entity = &async_bfqq->entity;
if (entity->sched_data != &bfqg->sched_data) {
bic_set_bfqq(bic, NULL, 0);
bfq_log_bfqq(bfqd, async_bfqq,
"bic_change_group: %p %d",
async_bfqq, atomic_read(&async_bfqq->ref));
bfq_put_queue(async_bfqq);
}
}
if (sync_bfqq != NULL) {
entity = &sync_bfqq->entity;
if (entity->sched_data != &bfqg->sched_data)
bfq_bfqq_move(bfqd, sync_bfqq, entity, bfqg);
}
return bfqg;
}
/**
* bfq_bic_change_cgroup - move @bic to @cgroup.
* @bic: the bic being migrated.
* @cgroup: the destination cgroup.
*
* When the task owning @bic is moved to @cgroup, @bic is immediately
* moved into its new parent group.
*/
static void bfq_bic_change_cgroup(struct bfq_io_cq *bic,
struct cgroup *cgroup)
{
struct bfq_data *bfqd;
unsigned long uninitialized_var(flags);
bfqd = bfq_get_bfqd_locked(&(bic->icq.q->elevator->elevator_data),
&flags);
if (bfqd != NULL) {
__bfq_bic_change_cgroup(bfqd, bic, cgroup);
bfq_put_bfqd_unlock(bfqd, &flags);
}
}
/**
* bfq_bic_update_cgroup - update the cgroup of @bic.
* @bic: the @bic to update.
*
* Make sure that @bic is enqueued in the cgroup of the current task.
* We need this in addition to moving bics during the cgroup attach
* phase because the task owning @bic could be at its first disk
* access or we may end up in the root cgroup as the result of a
* memory allocation failure and here we try to move to the right
* group.
*
* Must be called under the queue lock. It is safe to use the returned
* value even after the rcu_read_unlock() as the migration/destruction
* paths act under the queue lock too. IOW it is impossible to race with
* group migration/destruction and end up with an invalid group as:
* a) here cgroup has not yet been destroyed, nor its destroy callback
* has started execution, as current holds a reference to it,
* b) if it is destroyed after rcu_read_unlock() [after current is
* migrated to a different cgroup] its attach() callback will have
* taken care of remove all the references to the old cgroup data.
*/
static struct bfq_group *bfq_bic_update_cgroup(struct bfq_io_cq *bic)
{
struct bfq_data *bfqd = bic_to_bfqd(bic);
struct bfq_group *bfqg;
struct cgroup *cgroup;
BUG_ON(bfqd == NULL);
rcu_read_lock();
cgroup = task_cgroup(current, bfqio_subsys_id);
bfqg = __bfq_bic_change_cgroup(bfqd, bic, cgroup);
rcu_read_unlock();
return bfqg;
}
/**
* bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
* @st: the service tree being flushed.
*/
static inline void bfq_flush_idle_tree(struct bfq_service_tree *st)
{
struct bfq_entity *entity = st->first_idle;
for (; entity != NULL; entity = st->first_idle)
__bfq_deactivate_entity(entity, 0);
}
/**
* bfq_reparent_leaf_entity - move leaf entity to the root_group.
* @bfqd: the device data structure with the root group.
* @entity: the entity to move.
*/
static inline void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
struct bfq_entity *entity)
{
struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
BUG_ON(bfqq == NULL);
bfq_bfqq_move(bfqd, bfqq, entity, bfqd->root_group);
return;
}
/**
* bfq_reparent_active_entities - move to the root group all active
* entities.
* @bfqd: the device data structure with the root group.
* @bfqg: the group to move from.
* @st: the service tree with the entities.
*
* Needs queue_lock to be taken and reference to be valid over the call.
*/
static inline void bfq_reparent_active_entities(struct bfq_data *bfqd,
struct bfq_group *bfqg,
struct bfq_service_tree *st)
{
struct rb_root *active = &st->active;
struct bfq_entity *entity = NULL;
if (!RB_EMPTY_ROOT(&st->active))
entity = bfq_entity_of(rb_first(active));
for (; entity != NULL; entity = bfq_entity_of(rb_first(active)))
bfq_reparent_leaf_entity(bfqd, entity);
if (bfqg->sched_data.in_service_entity != NULL)
bfq_reparent_leaf_entity(bfqd,
bfqg->sched_data.in_service_entity);
return;
}
/**
* bfq_destroy_group - destroy @bfqg.
* @bgrp: the bfqio_cgroup containing @bfqg.
* @bfqg: the group being destroyed.
*
* Destroy @bfqg, making sure that it is not referenced from its parent.
*/
static void bfq_destroy_group(struct bfqio_cgroup *bgrp, struct bfq_group *bfqg)
{
struct bfq_data *bfqd;
struct bfq_service_tree *st;
struct bfq_entity *entity = bfqg->my_entity;
unsigned long uninitialized_var(flags);
int i;
hlist_del(&bfqg->group_node);
/*
* Empty all service_trees belonging to this group before
* deactivating the group itself.
*/
for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) {
st = bfqg->sched_data.service_tree + i;
/*
* The idle tree may still contain bfq_queues belonging
* to exited task because they never migrated to a different
* cgroup from the one being destroyed now. No one else
* can access them so it's safe to act without any lock.
*/
bfq_flush_idle_tree(st);
/*
* It may happen that some queues are still active
* (busy) upon group destruction (if the corresponding
* processes have been forced to terminate). We move
* all the leaf entities corresponding to these queues
* to the root_group.
* Also, it may happen that the group has an entity
* in service, which is disconnected from the active
* tree: it must be moved, too.
* There is no need to put the sync queues, as the
* scheduler has taken no reference.
*/
bfqd = bfq_get_bfqd_locked(&bfqg->bfqd, &flags);
if (bfqd != NULL) {
bfq_reparent_active_entities(bfqd, bfqg, st);
bfq_put_bfqd_unlock(bfqd, &flags);
}
BUG_ON(!RB_EMPTY_ROOT(&st->active));
BUG_ON(!RB_EMPTY_ROOT(&st->idle));
}
BUG_ON(bfqg->sched_data.next_in_service != NULL);
BUG_ON(bfqg->sched_data.in_service_entity != NULL);
/*
* We may race with device destruction, take extra care when
* dereferencing bfqg->bfqd.
*/
bfqd = bfq_get_bfqd_locked(&bfqg->bfqd, &flags);
if (bfqd != NULL) {
hlist_del(&bfqg->bfqd_node);
__bfq_deactivate_entity(entity, 0);
bfq_put_async_queues(bfqd, bfqg);
bfq_put_bfqd_unlock(bfqd, &flags);
}
BUG_ON(entity->tree != NULL);
/*
* No need to defer the kfree() to the end of the RCU grace
* period: we are called from the destroy() callback of our
* cgroup, so we can be sure that no one is a) still using
* this cgroup or b) doing lookups in it.
*/
kfree(bfqg);
}
static void bfq_end_wr_async(struct bfq_data *bfqd)
{
struct hlist_node *tmp;
struct bfq_group *bfqg;
hlist_for_each_entry_safe(bfqg, tmp, &bfqd->group_list, bfqd_node)
bfq_end_wr_async_queues(bfqd, bfqg);
bfq_end_wr_async_queues(bfqd, bfqd->root_group);
}
/**
* bfq_disconnect_groups - disconnect @bfqd from all its groups.
* @bfqd: the device descriptor being exited.
*
* When the device exits we just make sure that no lookup can return
* the now unused group structures. They will be deallocated on cgroup
* destruction.
*/
static void bfq_disconnect_groups(struct bfq_data *bfqd)
{
struct hlist_node *tmp;
struct bfq_group *bfqg;
bfq_log(bfqd, "disconnect_groups beginning");
hlist_for_each_entry_safe(bfqg, tmp, &bfqd->group_list, bfqd_node) {
hlist_del(&bfqg->bfqd_node);
__bfq_deactivate_entity(bfqg->my_entity, 0);
/*
* Don't remove from the group hash, just set an
* invalid key. No lookups can race with the
* assignment as bfqd is being destroyed; this
* implies also that new elements cannot be added
* to the list.
*/
rcu_assign_pointer(bfqg->bfqd, NULL);
bfq_log(bfqd, "disconnect_groups: put async for group %p",
bfqg);
bfq_put_async_queues(bfqd, bfqg);
}
}
static inline void bfq_free_root_group(struct bfq_data *bfqd)
{
struct bfqio_cgroup *bgrp = &bfqio_root_cgroup;
struct bfq_group *bfqg = bfqd->root_group;
bfq_put_async_queues(bfqd, bfqg);
spin_lock_irq(&bgrp->lock);
hlist_del_rcu(&bfqg->group_node);
spin_unlock_irq(&bgrp->lock);
/*
* No need to synchronize_rcu() here: since the device is gone
* there cannot be any read-side access to its root_group.
*/
kfree(bfqg);
}
static struct bfq_group *bfq_alloc_root_group(struct bfq_data *bfqd, int node)
{
struct bfq_group *bfqg;
struct bfqio_cgroup *bgrp;
int i;
bfqg = kzalloc_node(sizeof(*bfqg), GFP_KERNEL, node);
if (bfqg == NULL)
return NULL;
bfqg->entity.parent = NULL;
for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
bgrp = &bfqio_root_cgroup;
spin_lock_irq(&bgrp->lock);
rcu_assign_pointer(bfqg->bfqd, bfqd);
hlist_add_head_rcu(&bfqg->group_node, &bgrp->group_data);
spin_unlock_irq(&bgrp->lock);
return bfqg;
}
#define SHOW_FUNCTION(__VAR) \
static u64 bfqio_cgroup_##__VAR##_read(struct cgroup *cgroup, \
struct cftype *cftype) \
{ \
struct bfqio_cgroup *bgrp; \
u64 ret = -ENODEV; \
\
mutex_lock(&bfqio_mutex); \
if (bfqio_is_removed(cgroup)) \
goto out_unlock; \
\
bgrp = cgroup_to_bfqio(cgroup); \
spin_lock_irq(&bgrp->lock); \
ret = bgrp->__VAR; \
spin_unlock_irq(&bgrp->lock); \
\
out_unlock: \
mutex_unlock(&bfqio_mutex); \
return ret; \
}
SHOW_FUNCTION(weight);
SHOW_FUNCTION(ioprio);
SHOW_FUNCTION(ioprio_class);
#undef SHOW_FUNCTION
#define STORE_FUNCTION(__VAR, __MIN, __MAX) \
static int bfqio_cgroup_##__VAR##_write(struct cgroup *cgroup, \
struct cftype *cftype, \
u64 val) \
{ \
struct bfqio_cgroup *bgrp; \
struct bfq_group *bfqg; \
int ret = -EINVAL; \
\
if (val < (__MIN) || val > (__MAX)) \
return ret; \
\
ret = -ENODEV; \
mutex_lock(&bfqio_mutex); \
if (bfqio_is_removed(cgroup)) \
goto out_unlock; \
ret = 0; \
\
bgrp = cgroup_to_bfqio(cgroup); \
\
spin_lock_irq(&bgrp->lock); \
bgrp->__VAR = (unsigned short)val; \
hlist_for_each_entry(bfqg, &bgrp->group_data, group_node) { \
/* \
* Setting the ioprio_changed flag of the entity \
* to 1 with new_##__VAR == ##__VAR would re-set \
* the value of the weight to its ioprio mapping. \
* Set the flag only if necessary. \
*/ \
if ((unsigned short)val != bfqg->entity.new_##__VAR) { \
bfqg->entity.new_##__VAR = (unsigned short)val; \
/* \
* Make sure that the above new value has been \
* stored in bfqg->entity.new_##__VAR before \
* setting the ioprio_changed flag. In fact, \
* this flag may be read asynchronously (in \
* critical sections protected by a different \
* lock than that held here), and finding this \
* flag set may cause the execution of the code \
* for updating parameters whose value may \
* depend also on bfqg->entity.new_##__VAR (in \
* __bfq_entity_update_weight_prio). \
* This barrier makes sure that the new value \
* of bfqg->entity.new_##__VAR is correctly \
* seen in that code. \
*/ \
smp_wmb(); \
bfqg->entity.ioprio_changed = 1; \
} \
} \
spin_unlock_irq(&bgrp->lock); \
\
out_unlock: \
mutex_unlock(&bfqio_mutex); \
return ret; \
}
STORE_FUNCTION(weight, BFQ_MIN_WEIGHT, BFQ_MAX_WEIGHT);
STORE_FUNCTION(ioprio, 0, IOPRIO_BE_NR - 1);
STORE_FUNCTION(ioprio_class, IOPRIO_CLASS_RT, IOPRIO_CLASS_IDLE);
#undef STORE_FUNCTION
static struct cftype bfqio_files[] = {
{
.name = "weight",
.read_u64 = bfqio_cgroup_weight_read,
.write_u64 = bfqio_cgroup_weight_write,
},
{
.name = "ioprio",
.read_u64 = bfqio_cgroup_ioprio_read,
.write_u64 = bfqio_cgroup_ioprio_write,
},
{
.name = "ioprio_class",
.read_u64 = bfqio_cgroup_ioprio_class_read,
.write_u64 = bfqio_cgroup_ioprio_class_write,
},
{ }, /* terminate */
};
static struct cgroup_subsys_state *bfqio_create(struct cgroup *cgroup)
{
struct bfqio_cgroup *bgrp;
if (cgroup->parent != NULL) {
bgrp = kzalloc(sizeof(*bgrp), GFP_KERNEL);
if (bgrp == NULL)
return ERR_PTR(-ENOMEM);
} else
bgrp = &bfqio_root_cgroup;
spin_lock_init(&bgrp->lock);
INIT_HLIST_HEAD(&bgrp->group_data);
bgrp->ioprio = BFQ_DEFAULT_GRP_IOPRIO;
bgrp->ioprio_class = BFQ_DEFAULT_GRP_CLASS;
return &bgrp->css;
}
/*
* We cannot support shared io contexts, as we have no means to support
* two tasks with the same ioc in two different groups without major rework
* of the main bic/bfqq data structures. By now we allow a task to change
* its cgroup only if it's the only owner of its ioc; the drawback of this
* behavior is that a group containing a task that forked using CLONE_IO
* will not be destroyed until the tasks sharing the ioc die.
*/
static int bfqio_can_attach(struct cgroup *cgroup, struct cgroup_taskset *tset)
{
struct task_struct *task;
struct io_context *ioc;
int ret = 0;
cgroup_taskset_for_each(task, cgroup, tset) {
/* task_lock() is needed to avoid races with exit_io_context() */
task_lock(task);
ioc = task->io_context;
if (ioc != NULL && atomic_read(&ioc->nr_tasks) > 1)
/*
* ioc == NULL means that the task is either too
* young or exiting: if it has still no ioc the
* ioc can't be shared, if the task is exiting the
* attach will fail anyway, no matter what we
* return here.
*/
ret = -EINVAL;
task_unlock(task);
if (ret)
break;
}
return ret;
}
static void bfqio_attach(struct cgroup *cgroup, struct cgroup_taskset *tset)
{
struct task_struct *task;
struct io_context *ioc;
struct io_cq *icq;
/*
* IMPORTANT NOTE: The move of more than one process at a time to a
* new group has not yet been tested.
*/
cgroup_taskset_for_each(task, cgroup, tset) {
ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
if (ioc) {
/*
* Handle cgroup change here.
*/
rcu_read_lock();
hlist_for_each_entry_rcu(icq, &ioc->icq_list, ioc_node)
if (!strncmp(
icq->q->elevator->type->elevator_name,
"bfq", ELV_NAME_MAX))
bfq_bic_change_cgroup(icq_to_bic(icq),
cgroup);
rcu_read_unlock();
put_io_context(ioc);
}
}
}
static void bfqio_destroy(struct cgroup *cgroup)
{
struct bfqio_cgroup *bgrp = cgroup_to_bfqio(cgroup);
struct hlist_node *tmp;
struct bfq_group *bfqg;
/*
* Since we are destroying the cgroup, there are no more tasks
* referencing it, and all the RCU grace periods that may have
* referenced it are ended (as the destruction of the parent
* cgroup is RCU-safe); bgrp->group_data will not be accessed by
* anything else and we don't need any synchronization.
*/
hlist_for_each_entry_safe(bfqg, tmp, &bgrp->group_data, group_node)
bfq_destroy_group(bgrp, bfqg);
BUG_ON(!hlist_empty(&bgrp->group_data));
kfree(bgrp);
}
struct cgroup_subsys bfqio_subsys = {
.name = "bfqio",
.css_alloc = bfqio_create,
.can_attach = bfqio_can_attach,
.attach = bfqio_attach,
.css_free = bfqio_destroy,
.subsys_id = bfqio_subsys_id,
.base_cftypes = bfqio_files,
};
#else
static inline void bfq_init_entity(struct bfq_entity *entity,
struct bfq_group *bfqg)
{
entity->weight = entity->new_weight;
entity->orig_weight = entity->new_weight;
entity->ioprio = entity->new_ioprio;
entity->ioprio_class = entity->new_ioprio_class;
entity->sched_data = &bfqg->sched_data;
}
static inline struct bfq_group *
bfq_bic_update_cgroup(struct bfq_io_cq *bic)
{
struct bfq_data *bfqd = bic_to_bfqd(bic);
return bfqd->root_group;
}
static inline void bfq_bfqq_move(struct bfq_data *bfqd,
struct bfq_queue *bfqq,
struct bfq_entity *entity,
struct bfq_group *bfqg)
{
}
static void bfq_end_wr_async(struct bfq_data *bfqd)
{
bfq_end_wr_async_queues(bfqd, bfqd->root_group);
}
static inline void bfq_disconnect_groups(struct bfq_data *bfqd)
{
bfq_put_async_queues(bfqd, bfqd->root_group);
}
static inline void bfq_free_root_group(struct bfq_data *bfqd)
{
kfree(bfqd->root_group);
}
static struct bfq_group *bfq_alloc_root_group(struct bfq_data *bfqd, int node)
{
struct bfq_group *bfqg;
int i;
bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);
if (bfqg == NULL)
return NULL;
for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
return bfqg;
}
#endif

36
block/bfq-ioc.c Normal file
View File

@ -0,0 +1,36 @@
/*
* BFQ: I/O context handling.
*
* Based on ideas and code from CFQ:
* Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
*
* Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
* Paolo Valente <paolo.valente@unimore.it>
*
* Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
*/
/**
* icq_to_bic - convert iocontext queue structure to bfq_io_cq.
* @icq: the iocontext queue.
*/
static inline struct bfq_io_cq *icq_to_bic(struct io_cq *icq)
{
/* bic->icq is the first member, %NULL will convert to %NULL */
return container_of(icq, struct bfq_io_cq, icq);
}
/**
* bfq_bic_lookup - search into @ioc a bic associated to @bfqd.
* @bfqd: the lookup key.
* @ioc: the io_context of the process doing I/O.
*
* Queue lock must be held.
*/
static inline struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd,
struct io_context *ioc)
{
if (ioc)
return icq_to_bic(ioc_lookup_icq(ioc, bfqd->queue));
return NULL;
}

3898
block/bfq-iosched.c Normal file
View File

File diff suppressed because it is too large Load Diff

1208
block/bfq-sched.c Normal file
View File

File diff suppressed because it is too large Load Diff

769
block/bfq.h Normal file
View File

@ -0,0 +1,769 @@
/*
* BFQ-v7r8 for 3.10.8+: data structures and common functions prototypes.
*
* Based on ideas and code from CFQ:
* Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
*
* Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
* Paolo Valente <paolo.valente@unimore.it>
*
* Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it>
*/
#ifndef _BFQ_H
#define _BFQ_H
#include <linux/blktrace_api.h>
#include <linux/hrtimer.h>
#include <linux/ioprio.h>
#include <linux/rbtree.h>
#define BFQ_IOPRIO_CLASSES 3
#define BFQ_CL_IDLE_TIMEOUT (HZ/5)
#define BFQ_MIN_WEIGHT 1
#define BFQ_MAX_WEIGHT 1000
#define BFQ_DEFAULT_QUEUE_IOPRIO 4
#define BFQ_DEFAULT_GRP_WEIGHT 10
#define BFQ_DEFAULT_GRP_IOPRIO 0
#define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE
struct bfq_entity;
/**
* struct bfq_service_tree - per ioprio_class service tree.
* @active: tree for active entities (i.e., those backlogged).
* @idle: tree for idle entities (i.e., those not backlogged, with V <= F_i).
* @first_idle: idle entity with minimum F_i.
* @last_idle: idle entity with maximum F_i.
* @vtime: scheduler virtual time.
* @wsum: scheduler weight sum; active and idle entities contribute to it.
*
* Each service tree represents a B-WF2Q+ scheduler on its own. Each
* ioprio_class has its own independent scheduler, and so its own
* bfq_service_tree. All the fields are protected by the queue lock
* of the containing bfqd.
*/
struct bfq_service_tree {
struct rb_root active;
struct rb_root idle;
struct bfq_entity *first_idle;
struct bfq_entity *last_idle;
u64 vtime;
unsigned long wsum;
};
/**
* struct bfq_sched_data - multi-class scheduler.
* @in_service_entity: entity in service.
* @next_in_service: head-of-the-line entity in the scheduler.
* @service_tree: array of service trees, one per ioprio_class.
*
* bfq_sched_data is the basic scheduler queue. It supports three
* ioprio_classes, and can be used either as a toplevel queue or as
* an intermediate queue on a hierarchical setup.
* @next_in_service points to the active entity of the sched_data
* service trees that will be scheduled next.
*
* The supported ioprio_classes are the same as in CFQ, in descending
* priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE.
* Requests from higher priority queues are served before all the
* requests from lower priority queues; among requests of the same
* queue requests are served according to B-WF2Q+.
* All the fields are protected by the queue lock of the containing bfqd.
*/
struct bfq_sched_data {
struct bfq_entity *in_service_entity;
struct bfq_entity *next_in_service;
struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES];
};
/**
* struct bfq_weight_counter - counter of the number of all active entities
* with a given weight.
* @weight: weight of the entities that this counter refers to.
* @num_active: number of active entities with this weight.
* @weights_node: weights tree member (see bfq_data's @queue_weights_tree
* and @group_weights_tree).
*/
struct bfq_weight_counter {
short int weight;
unsigned int num_active;
struct rb_node weights_node;
};
/**
* struct bfq_entity - schedulable entity.
* @rb_node: service_tree member.
* @weight_counter: pointer to the weight counter associated with this entity.
* @on_st: flag, true if the entity is on a tree (either the active or
* the idle one of its service_tree).
* @finish: B-WF2Q+ finish timestamp (aka F_i).
* @start: B-WF2Q+ start timestamp (aka S_i).
* @tree: tree the entity is enqueued into; %NULL if not on a tree.
* @min_start: minimum start time of the (active) subtree rooted at
* this entity; used for O(log N) lookups into active trees.
* @service: service received during the last round of service.
* @budget: budget used to calculate F_i; F_i = S_i + @budget / @weight.
* @weight: weight of the queue
* @parent: parent entity, for hierarchical scheduling.
* @my_sched_data: for non-leaf nodes in the cgroup hierarchy, the
* associated scheduler queue, %NULL on leaf nodes.
* @sched_data: the scheduler queue this entity belongs to.
* @ioprio: the ioprio in use.
* @new_weight: when a weight change is requested, the new weight value.
* @orig_weight: original weight, used to implement weight boosting
* @new_ioprio: when an ioprio change is requested, the new ioprio value.
* @ioprio_class: the ioprio_class in use.
* @new_ioprio_class: when an ioprio_class change is requested, the new
* ioprio_class value.
* @ioprio_changed: flag, true when the user requested a weight, ioprio or
* ioprio_class change.
*
* A bfq_entity is used to represent either a bfq_queue (leaf node in the
* cgroup hierarchy) or a bfq_group into the upper level scheduler. Each
* entity belongs to the sched_data of the parent group in the cgroup
* hierarchy. Non-leaf entities have also their own sched_data, stored
* in @my_sched_data.
*
* Each entity stores independently its priority values; this would
* allow different weights on different devices, but this
* functionality is not exported to userspace by now. Priorities and
* weights are updated lazily, first storing the new values into the
* new_* fields, then setting the @ioprio_changed flag. As soon as
* there is a transition in the entity state that allows the priority
* update to take place the effective and the requested priority
* values are synchronized.
*
* Unless cgroups are used, the weight value is calculated from the
* ioprio to export the same interface as CFQ. When dealing with
* ``well-behaved'' queues (i.e., queues that do not spend too much
* time to consume their budget and have true sequential behavior, and
* when there are no external factors breaking anticipation) the
* relative weights at each level of the cgroups hierarchy should be
* guaranteed. All the fields are protected by the queue lock of the
* containing bfqd.
*/
struct bfq_entity {
struct rb_node rb_node;
struct bfq_weight_counter *weight_counter;
int on_st;
u64 finish;
u64 start;
struct rb_root *tree;
u64 min_start;
unsigned long service, budget;
unsigned short weight, new_weight;
unsigned short orig_weight;
struct bfq_entity *parent;
struct bfq_sched_data *my_sched_data;
struct bfq_sched_data *sched_data;
unsigned short ioprio, new_ioprio;
unsigned short ioprio_class, new_ioprio_class;
int ioprio_changed;
};
struct bfq_group;
/**
* struct bfq_queue - leaf schedulable entity.
* @ref: reference counter.
* @bfqd: parent bfq_data.
* @new_bfqq: shared bfq_queue if queue is cooperating with
* one or more other queues.
* @pos_node: request-position tree member (see bfq_data's @rq_pos_tree).
* @pos_root: request-position tree root (see bfq_data's @rq_pos_tree).
* @sort_list: sorted list of pending requests.
* @next_rq: if fifo isn't expired, next request to serve.
* @queued: nr of requests queued in @sort_list.
* @allocated: currently allocated requests.
* @meta_pending: pending metadata requests.
* @fifo: fifo list of requests in sort_list.
* @entity: entity representing this queue in the scheduler.
* @max_budget: maximum budget allowed from the feedback mechanism.
* @budget_timeout: budget expiration (in jiffies).
* @dispatched: number of requests on the dispatch list or inside driver.
* @flags: status flags.
* @bfqq_list: node for active/idle bfqq list inside our bfqd.
* @burst_list_node: node for the device's burst list.
* @seek_samples: number of seeks sampled
* @seek_total: sum of the distances of the seeks sampled
* @seek_mean: mean seek distance
* @last_request_pos: position of the last request enqueued
* @requests_within_timer: number of consecutive pairs of request completion
* and arrival, such that the queue becomes idle
* after the completion, but the next request arrives
* within an idle time slice; used only if the queue's
* IO_bound has been cleared.
* @pid: pid of the process owning the queue, used for logging purposes.
* @last_wr_start_finish: start time of the current weight-raising period if
* the @bfq-queue is being weight-raised, otherwise
* finish time of the last weight-raising period
* @wr_cur_max_time: current max raising time for this queue
* @soft_rt_next_start: minimum time instant such that, only if a new
* request is enqueued after this time instant in an
* idle @bfq_queue with no outstanding requests, then
* the task associated with the queue it is deemed as
* soft real-time (see the comments to the function
* bfq_bfqq_softrt_next_start()).
* @last_idle_bklogged: time of the last transition of the @bfq_queue from
* idle to backlogged
* @service_from_backlogged: cumulative service received from the @bfq_queue
* since the last transition from idle to
* backlogged
*
* A bfq_queue is a leaf request queue; it can be associated with an io_context
* or more, if it is async or shared between cooperating processes. @cgroup
* holds a reference to the cgroup, to be sure that it does not disappear while
* a bfqq still references it (mostly to avoid races between request issuing and
* task migration followed by cgroup destruction).
* All the fields are protected by the queue lock of the containing bfqd.
*/
struct bfq_queue {
atomic_t ref;
struct bfq_data *bfqd;
/* fields for cooperating queues handling */
struct bfq_queue *new_bfqq;
struct rb_node pos_node;
struct rb_root *pos_root;
struct rb_root sort_list;
struct request *next_rq;
int queued[2];
int allocated[2];
int meta_pending;
struct list_head fifo;
struct bfq_entity entity;
unsigned long max_budget;
unsigned long budget_timeout;
int dispatched;
unsigned int flags;
struct list_head bfqq_list;
struct hlist_node burst_list_node;
unsigned int seek_samples;
u64 seek_total;
sector_t seek_mean;
sector_t last_request_pos;
unsigned int requests_within_timer;
pid_t pid;
/* weight-raising fields */
unsigned long wr_cur_max_time;
unsigned long soft_rt_next_start;
unsigned long last_wr_start_finish;
unsigned int wr_coeff;
unsigned long last_idle_bklogged;
unsigned long service_from_backlogged;
};
/**
* struct bfq_ttime - per process thinktime stats.
* @ttime_total: total process thinktime
* @ttime_samples: number of thinktime samples
* @ttime_mean: average process thinktime
*/
struct bfq_ttime {
unsigned long last_end_request;
unsigned long ttime_total;
unsigned long ttime_samples;
unsigned long ttime_mean;
};
/**
* struct bfq_io_cq - per (request_queue, io_context) structure.
* @icq: associated io_cq structure
* @bfqq: array of two process queues, the sync and the async
* @ttime: associated @bfq_ttime struct
*/
struct bfq_io_cq {
struct io_cq icq; /* must be the first member */
struct bfq_queue *bfqq[2];
struct bfq_ttime ttime;
int ioprio;
};
enum bfq_device_speed {
BFQ_BFQD_FAST,
BFQ_BFQD_SLOW,
};
/**
* struct bfq_data - per device data structure.
* @queue: request queue for the managed device.
* @root_group: root bfq_group for the device.
* @rq_pos_tree: rbtree sorted by next_request position, used when
* determining if two or more queues have interleaving
* requests (see bfq_close_cooperator()).
* @active_numerous_groups: number of bfq_groups containing more than one
* active @bfq_entity.
* @queue_weights_tree: rbtree of weight counters of @bfq_queues, sorted by
* weight. Used to keep track of whether all @bfq_queues
* have the same weight. The tree contains one counter
* for each distinct weight associated to some active
* and not weight-raised @bfq_queue (see the comments to
* the functions bfq_weights_tree_[add|remove] for
* further details).
* @group_weights_tree: rbtree of non-queue @bfq_entity weight counters, sorted
* by weight. Used to keep track of whether all
* @bfq_groups have the same weight. The tree contains
* one counter for each distinct weight associated to
* some active @bfq_group (see the comments to the
* functions bfq_weights_tree_[add|remove] for further
* details).
* @busy_queues: number of bfq_queues containing requests (including the
* queue in service, even if it is idling).
* @busy_in_flight_queues: number of @bfq_queues containing pending or
* in-flight requests, plus the @bfq_queue in
* service, even if idle but waiting for the
* possible arrival of its next sync request. This
* field is updated only if the device is rotational,
* but used only if the device is also NCQ-capable.
* The reason why the field is updated also for non-
* NCQ-capable rotational devices is related to the
* fact that the value of @hw_tag may be set also
* later than when busy_in_flight_queues may need to
* be incremented for the first time(s). Taking also
* this possibility into account, to avoid unbalanced
* increments/decrements, would imply more overhead
* than just updating busy_in_flight_queues
* regardless of the value of @hw_tag.
* @const_seeky_busy_in_flight_queues: number of constantly-seeky @bfq_queues
* (that is, seeky queues that expired
* for budget timeout at least once)
* containing pending or in-flight
* requests, including the in-service
* @bfq_queue if constantly seeky. This
* field is updated only if the device
* is rotational, but used only if the
* device is also NCQ-capable (see the
* comments to @busy_in_flight_queues).
* @wr_busy_queues: number of weight-raised busy @bfq_queues.
* @queued: number of queued requests.
* @rq_in_driver: number of requests dispatched and waiting for completion.
* @sync_flight: number of sync requests in the driver.
* @max_rq_in_driver: max number of reqs in driver in the last
* @hw_tag_samples completed requests.
* @hw_tag_samples: nr of samples used to calculate hw_tag.
* @hw_tag: flag set to one if the driver is showing a queueing behavior.
* @budgets_assigned: number of budgets assigned.
* @idle_slice_timer: timer set when idling for the next sequential request
* from the queue in service.
* @unplug_work: delayed work to restart dispatching on the request queue.
* @in_service_queue: bfq_queue in service.
* @in_service_bic: bfq_io_cq (bic) associated with the @in_service_queue.
* @last_position: on-disk position of the last served request.
* @last_budget_start: beginning of the last budget.
* @last_idling_start: beginning of the last idle slice.
* @peak_rate: peak transfer rate observed for a budget.
* @peak_rate_samples: number of samples used to calculate @peak_rate.
* @bfq_max_budget: maximum budget allotted to a bfq_queue before
* rescheduling.
* @group_list: list of all the bfq_groups active on the device.
* @active_list: list of all the bfq_queues active on the device.
* @idle_list: list of all the bfq_queues idle on the device.
* @bfq_fifo_expire: timeout for async/sync requests; when it expires
* requests are served in fifo order.
* @bfq_back_penalty: weight of backward seeks wrt forward ones.
* @bfq_back_max: maximum allowed backward seek.
* @bfq_slice_idle: maximum idling time.
* @bfq_user_max_budget: user-configured max budget value
* (0 for auto-tuning).
* @bfq_max_budget_async_rq: maximum budget (in nr of requests) allotted to
* async queues.
* @bfq_timeout: timeout for bfq_queues to consume their budget; used to
* to prevent seeky queues to impose long latencies to well
* behaved ones (this also implies that seeky queues cannot
* receive guarantees in the service domain; after a timeout
* they are charged for the whole allocated budget, to try
* to preserve a behavior reasonably fair among them, but
* without service-domain guarantees).
* @bfq_coop_thresh: number of queue merges after which a @bfq_queue is
* no more granted any weight-raising.
* @bfq_failed_cooperations: number of consecutive failed cooperation
* chances after which weight-raising is restored
* to a queue subject to more than bfq_coop_thresh
* queue merges.
* @bfq_requests_within_timer: number of consecutive requests that must be
* issued within the idle time slice to set
* again idling to a queue which was marked as
* non-I/O-bound (see the definition of the
* IO_bound flag for further details).
* @last_ins_in_burst: last time at which a queue entered the current
* burst of queues being activated shortly after
* each other; for more details about this and the
* following parameters related to a burst of
* activations, see the comments to the function
* @bfq_handle_burst.
* @bfq_burst_interval: reference time interval used to decide whether a
* queue has been activated shortly after
* @last_ins_in_burst.
* @burst_size: number of queues in the current burst of queue activations.
* @bfq_large_burst_thresh: maximum burst size above which the current
* queue-activation burst is deemed as 'large'.
* @large_burst: true if a large queue-activation burst is in progress.
* @burst_list: head of the burst list (as for the above fields, more details
* in the comments to the function bfq_handle_burst).
* @low_latency: if set to true, low-latency heuristics are enabled.
* @bfq_wr_coeff: maximum factor by which the weight of a weight-raised
* queue is multiplied.
* @bfq_wr_max_time: maximum duration of a weight-raising period (jiffies).
* @bfq_wr_rt_max_time: maximum duration for soft real-time processes.
* @bfq_wr_min_idle_time: minimum idle period after which weight-raising
* may be reactivated for a queue (in jiffies).
* @bfq_wr_min_inter_arr_async: minimum period between request arrivals
* after which weight-raising may be
* reactivated for an already busy queue
* (in jiffies).
* @bfq_wr_max_softrt_rate: max service-rate for a soft real-time queue,
* sectors per seconds.
* @RT_prod: cached value of the product R*T used for computing the maximum
* duration of the weight raising automatically.
* @device_speed: device-speed class for the low-latency heuristic.
* @oom_bfqq: fallback dummy bfqq for extreme OOM conditions.
*
* All the fields are protected by the @queue lock.
*/
struct bfq_data {
struct request_queue *queue;
struct bfq_group *root_group;
struct rb_root rq_pos_tree;
#ifdef CONFIG_CGROUP_BFQIO
int active_numerous_groups;
#endif
struct rb_root queue_weights_tree;
struct rb_root group_weights_tree;
int busy_queues;
int busy_in_flight_queues;
int const_seeky_busy_in_flight_queues;
int wr_busy_queues;
int queued;
int rq_in_driver;
int sync_flight;
int max_rq_in_driver;
int hw_tag_samples;
int hw_tag;
int budgets_assigned;
struct timer_list idle_slice_timer;
struct work_struct unplug_work;
struct bfq_queue *in_service_queue;
struct bfq_io_cq *in_service_bic;
sector_t last_position;
ktime_t last_budget_start;
ktime_t last_idling_start;
int peak_rate_samples;
u64 peak_rate;
unsigned long bfq_max_budget;
struct hlist_head group_list;
struct list_head active_list;
struct list_head idle_list;
unsigned int bfq_fifo_expire[2];
unsigned int bfq_back_penalty;
unsigned int bfq_back_max;
unsigned int bfq_slice_idle;
u64 bfq_class_idle_last_service;
unsigned int bfq_user_max_budget;
unsigned int bfq_max_budget_async_rq;
unsigned int bfq_timeout[2];
unsigned int bfq_coop_thresh;
unsigned int bfq_failed_cooperations;
unsigned int bfq_requests_within_timer;
unsigned long last_ins_in_burst;
unsigned long bfq_burst_interval;
int burst_size;
unsigned long bfq_large_burst_thresh;
bool large_burst;
struct hlist_head burst_list;
bool low_latency;
/* parameters of the low_latency heuristics */
unsigned int bfq_wr_coeff;
unsigned int bfq_wr_max_time;
unsigned int bfq_wr_rt_max_time;
unsigned int bfq_wr_min_idle_time;
unsigned long bfq_wr_min_inter_arr_async;
unsigned int bfq_wr_max_softrt_rate;
u64 RT_prod;
enum bfq_device_speed device_speed;
struct bfq_queue oom_bfqq;
};
enum bfqq_state_flags {
BFQ_BFQQ_FLAG_busy = 0, /* has requests or is in service */
BFQ_BFQQ_FLAG_wait_request, /* waiting for a request */
BFQ_BFQQ_FLAG_must_alloc, /* must be allowed rq alloc */
BFQ_BFQQ_FLAG_fifo_expire, /* FIFO checked in this slice */
BFQ_BFQQ_FLAG_idle_window, /* slice idling enabled */
BFQ_BFQQ_FLAG_sync, /* synchronous queue */
BFQ_BFQQ_FLAG_budget_new, /* no completion with this budget */
BFQ_BFQQ_FLAG_IO_bound, /*
* bfqq has timed-out at least once
* having consumed at most 2/10 of
* its budget
*/
BFQ_BFQQ_FLAG_in_large_burst, /*
* bfqq activated in a large burst,
* see comments to bfq_handle_burst.
*/
BFQ_BFQQ_FLAG_constantly_seeky, /*
* bfqq has proved to be slow and
* seeky until budget timeout
*/
BFQ_BFQQ_FLAG_softrt_update, /*
* may need softrt-next-start
* update
*/
BFQ_BFQQ_FLAG_coop, /* bfqq is shared */
BFQ_BFQQ_FLAG_split_coop, /* shared bfqq will be splitted */
};
#define BFQ_BFQQ_FNS(name) \
static inline void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \
{ \
(bfqq)->flags |= (1 << BFQ_BFQQ_FLAG_##name); \
} \
static inline void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \
{ \
(bfqq)->flags &= ~(1 << BFQ_BFQQ_FLAG_##name); \
} \
static inline int bfq_bfqq_##name(const struct bfq_queue *bfqq) \
{ \
return ((bfqq)->flags & (1 << BFQ_BFQQ_FLAG_##name)) != 0; \
}
BFQ_BFQQ_FNS(busy);
BFQ_BFQQ_FNS(wait_request);
BFQ_BFQQ_FNS(must_alloc);
BFQ_BFQQ_FNS(fifo_expire);
BFQ_BFQQ_FNS(idle_window);
BFQ_BFQQ_FNS(sync);
BFQ_BFQQ_FNS(budget_new);
BFQ_BFQQ_FNS(IO_bound);
BFQ_BFQQ_FNS(in_large_burst);
BFQ_BFQQ_FNS(constantly_seeky);
BFQ_BFQQ_FNS(coop);
BFQ_BFQQ_FNS(split_coop);
BFQ_BFQQ_FNS(softrt_update);
#undef BFQ_BFQQ_FNS
/* Logging facilities. */
#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \
blk_add_trace_msg((bfqd)->queue, "bfq%d " fmt, (bfqq)->pid, ##args)
#define bfq_log(bfqd, fmt, args...) \
blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args)
/* Expiration reasons. */
enum bfqq_expiration {
BFQ_BFQQ_TOO_IDLE = 0, /*
* queue has been idling for
* too long
*/
BFQ_BFQQ_BUDGET_TIMEOUT, /* budget took too long to be used */
BFQ_BFQQ_BUDGET_EXHAUSTED, /* budget consumed */
BFQ_BFQQ_NO_MORE_REQUESTS, /* the queue has no more requests */
};
#ifdef CONFIG_CGROUP_BFQIO
/**
* struct bfq_group - per (device, cgroup) data structure.
* @entity: schedulable entity to insert into the parent group sched_data.
* @sched_data: own sched_data, to contain child entities (they may be
* both bfq_queues and bfq_groups).
* @group_node: node to be inserted into the bfqio_cgroup->group_data
* list of the containing cgroup's bfqio_cgroup.
* @bfqd_node: node to be inserted into the @bfqd->group_list list
* of the groups active on the same device; used for cleanup.
* @bfqd: the bfq_data for the device this group acts upon.
* @async_bfqq: array of async queues for all the tasks belonging to
* the group, one queue per ioprio value per ioprio_class,
* except for the idle class that has only one queue.
* @async_idle_bfqq: async queue for the idle class (ioprio is ignored).
* @my_entity: pointer to @entity, %NULL for the toplevel group; used
* to avoid too many special cases during group creation/
* migration.
* @active_entities: number of active entities belonging to the group;
* unused for the root group. Used to know whether there
* are groups with more than one active @bfq_entity
* (see the comments to the function
* bfq_bfqq_must_not_expire()).
*
* Each (device, cgroup) pair has its own bfq_group, i.e., for each cgroup
* there is a set of bfq_groups, each one collecting the lower-level
* entities belonging to the group that are acting on the same device.
*
* Locking works as follows:
* o @group_node is protected by the bfqio_cgroup lock, and is accessed
* via RCU from its readers.
* o @bfqd is protected by the queue lock, RCU is used to access it
* from the readers.
* o All the other fields are protected by the @bfqd queue lock.
*/
struct bfq_group {
struct bfq_entity entity;
struct bfq_sched_data sched_data;
struct hlist_node group_node;
struct hlist_node bfqd_node;
void *bfqd;
struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR];
struct bfq_queue *async_idle_bfqq;
struct bfq_entity *my_entity;
int active_entities;
};
/**
* struct bfqio_cgroup - bfq cgroup data structure.
* @css: subsystem state for bfq in the containing cgroup.
* @weight: cgroup weight.
* @ioprio: cgroup ioprio.
* @ioprio_class: cgroup ioprio_class.
* @lock: spinlock that protects @ioprio, @ioprio_class and @group_data.
* @group_data: list containing the bfq_group belonging to this cgroup.
*
* @group_data is accessed using RCU, with @lock protecting the updates,
* @ioprio and @ioprio_class are protected by @lock.
*/
struct bfqio_cgroup {
struct cgroup_subsys_state css;
unsigned short weight, ioprio, ioprio_class;
spinlock_t lock;
struct hlist_head group_data;
};
#else
struct bfq_group {
struct bfq_sched_data sched_data;
struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR];
struct bfq_queue *async_idle_bfqq;
};
#endif
static inline struct bfq_service_tree *
bfq_entity_service_tree(struct bfq_entity *entity)
{
struct bfq_sched_data *sched_data = entity->sched_data;
unsigned int idx = entity->ioprio_class - 1;
BUG_ON(idx >= BFQ_IOPRIO_CLASSES);
BUG_ON(sched_data == NULL);
return sched_data->service_tree + idx;
}
static inline struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic,
bool is_sync)
{
return bic->bfqq[is_sync];
}
static inline void bic_set_bfqq(struct bfq_io_cq *bic,
struct bfq_queue *bfqq, bool is_sync)
{
bic->bfqq[is_sync] = bfqq;
}
static inline struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)
{
return bic->icq.q->elevator->elevator_data;
}
/**
* bfq_get_bfqd_locked - get a lock to a bfqd using a RCU protected pointer.
* @ptr: a pointer to a bfqd.
* @flags: storage for the flags to be saved.
*
* This function allows bfqg->bfqd to be protected by the
* queue lock of the bfqd they reference; the pointer is dereferenced
* under RCU, so the storage for bfqd is assured to be safe as long
* as the RCU read side critical section does not end. After the
* bfqd->queue->queue_lock is taken the pointer is rechecked, to be
* sure that no other writer accessed it. If we raced with a writer,
* the function returns NULL, with the queue unlocked, otherwise it
* returns the dereferenced pointer, with the queue locked.
*/
static inline struct bfq_data *bfq_get_bfqd_locked(void **ptr,
unsigned long *flags)
{
struct bfq_data *bfqd;
rcu_read_lock();
bfqd = rcu_dereference(*(struct bfq_data **)ptr);
if (bfqd != NULL) {
spin_lock_irqsave(bfqd->queue->queue_lock, *flags);
if (*ptr == bfqd)
goto out;
spin_unlock_irqrestore(bfqd->queue->queue_lock, *flags);
}
bfqd = NULL;
out:
rcu_read_unlock();
return bfqd;
}
static inline void bfq_put_bfqd_unlock(struct bfq_data *bfqd,
unsigned long *flags)
{
spin_unlock_irqrestore(bfqd->queue->queue_lock, *flags);
}
static void bfq_check_ioprio_change(struct bfq_io_cq *bic);
static void bfq_put_queue(struct bfq_queue *bfqq);
static void bfq_dispatch_insert(struct request_queue *q, struct request *rq);
static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
struct bfq_group *bfqg, int is_sync,
struct bfq_io_cq *bic, gfp_t gfp_mask);
static void bfq_end_wr_async_queues(struct bfq_data *bfqd,
struct bfq_group *bfqg);
static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);
static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq);
#endif /* _BFQ_H */