block/blk-mq.c

   1 /*
   2  * Block multiqueue core code
   3  *
   4  * Copyright (C) 2013-2014 Jens Axboe
   5  * Copyright (C) 2013-2014 Christoph Hellwig
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/backing-dev.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/mm.h>
  13 #include <linux/init.h>
  14 #include <linux/slab.h>
  15 #include <linux/workqueue.h>
  16 #include <linux/smp.h>
  17 #include <linux/llist.h>
  18 #include <linux/list_sort.h>
  19 #include <linux/cpu.h>
  20 #include <linux/cache.h>
  21 #include <linux/sched/sysctl.h>
  22 #include <linux/delay.h>
  23
  24 #include <trace/events/block.h>
  25
  26 #include <linux/blk-mq.h>
  27 #include "blk.h"
  28 #include "blk-mq.h"
  29 #include "blk-mq-tag.h"
  30
  31 static DEFINE_MUTEX(all_q_mutex);
  32 static LIST_HEAD(all_q_list);
  33
  34 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
  35
  36 /*
  37  * Check if any of the ctx's have pending work in this hardware queue
  38  */
  39 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
  40 {
  41         unsigned int i;
  42
  43         for (i = 0; i < hctx->ctx_map.map_size; i++)
  44                 if (hctx->ctx_map.map[i].word)
  45                         return true;
  46
  47         return false;
  48 }
  49
  50 static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
  51                                               struct blk_mq_ctx *ctx)
  52 {
  53         return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
  54 }
  55
  56 #define CTX_TO_BIT(hctx, ctx)   \
  57         ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
  58
  59 /*
  60  * Mark this ctx as having pending work in this hardware queue
  61  */
  62 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
  63                                      struct blk_mq_ctx *ctx)
  64 {
  65         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  66
  67         if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
  68                 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  69 }
  70
  71 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
  72                                       struct blk_mq_ctx *ctx)
  73 {
  74         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  75
  76         clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  77 }
  78
  79 static int blk_mq_queue_enter(struct request_queue *q)
  80 {
  81         int ret;
  82
  83         __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
  84         smp_wmb();
  85
  86         /* we have problems freezing the queue if it's initializing */
  87         if (!blk_queue_dying(q) &&
  88             (!blk_queue_bypass(q) || !blk_queue_init_done(q)))
  89                 return 0;
  90
  91         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
  92
  93         spin_lock_irq(q->queue_lock);
  94         ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
  95                 !blk_queue_bypass(q) || blk_queue_dying(q),
  96                 *q->queue_lock);
  97         /* inc usage with lock hold to avoid freeze_queue runs here */
  98         if (!ret && !blk_queue_dying(q))
  99                 __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
 100         else if (blk_queue_dying(q))
 101                 ret = -ENODEV;
 102         spin_unlock_irq(q->queue_lock);
 103
 104         return ret;
 105 }
 106
 107 static void blk_mq_queue_exit(struct request_queue *q)
 108 {
 109         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 110 }
 111
 112 static void __blk_mq_drain_queue(struct request_queue *q)
 113 {
 114         while (true) {
 115                 s64 count;
 116
 117                 spin_lock_irq(q->queue_lock);
 118                 count = percpu_counter_sum(&q->mq_usage_counter);
 119                 spin_unlock_irq(q->queue_lock);
 120
 121                 if (count == 0)
 122                         break;
 123                 blk_mq_run_queues(q, false);
 124                 msleep(10);
 125         }
 126 }
 127
 128 /*
 129  * Guarantee no request is in use, so we can change any data structure of
 130  * the queue afterward.
 131  */
 132 static void blk_mq_freeze_queue(struct request_queue *q)
 133 {
 134         bool drain;
 135
 136         spin_lock_irq(q->queue_lock);
 137         drain = !q->bypass_depth++;
 138         queue_flag_set(QUEUE_FLAG_BYPASS, q);
 139         spin_unlock_irq(q->queue_lock);
 140
 141         if (drain)
 142                 __blk_mq_drain_queue(q);
 143 }
 144
 145 void blk_mq_drain_queue(struct request_queue *q)
 146 {
 147         __blk_mq_drain_queue(q);
 148 }
 149
 150 static void blk_mq_unfreeze_queue(struct request_queue *q)
 151 {
 152         bool wake = false;
 153
 154         spin_lock_irq(q->queue_lock);
 155         if (!--q->bypass_depth) {
 156                 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
 157                 wake = true;
 158         }
 159         WARN_ON_ONCE(q->bypass_depth < 0);
 160         spin_unlock_irq(q->queue_lock);
 161         if (wake)
 162                 wake_up_all(&q->mq_freeze_wq);
 163 }
 164
 165 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 166 {
 167         return blk_mq_has_free_tags(hctx->tags);
 168 }
 169 EXPORT_SYMBOL(blk_mq_can_queue);
 170
 171 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 172                                struct request *rq, unsigned int rw_flags)
 173 {
 174         if (blk_queue_io_stat(q))
 175                 rw_flags |= REQ_IO_STAT;
 176
 177         INIT_LIST_HEAD(&rq->queuelist);
 178         /* csd/requeue_work/fifo_time is initialized before use */
 179         rq->q = q;
 180         rq->mq_ctx = ctx;
 181         rq->cmd_flags |= rw_flags;
 182         /* do not touch atomic flags, it needs atomic ops against the timer */
 183         rq->cpu = -1;
 184         INIT_HLIST_NODE(&rq->hash);
 185         RB_CLEAR_NODE(&rq->rb_node);
 186         rq->rq_disk = NULL;
 187         rq->part = NULL;
 188 #ifdef CONFIG_BLK_CGROUP
 189         rq->rl = NULL;
 190         set_start_time_ns(rq);
 191         rq->io_start_time_ns = 0;
 192 #endif
 193         rq->nr_phys_segments = 0;
 194 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 195         rq->nr_integrity_segments = 0;
 196 #endif
 197         rq->special = NULL;
 198         /* tag was already set */
 199         rq->errors = 0;
 200
 201         rq->extra_len = 0;
 202         rq->sense_len = 0;
 203         rq->resid_len = 0;
 204         rq->sense = NULL;
 205
 206         INIT_LIST_HEAD(&rq->timeout_list);
 207         rq->timeout = 0;
 208
 209         rq->end_io = NULL;
 210         rq->end_io_data = NULL;
 211         rq->next_rq = NULL;
 212
 213         ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
 214 }
 215
 216 static struct request *
 217 __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
 218 {
 219         struct request *rq;
 220         unsigned int tag;
 221
 222         tag = blk_mq_get_tag(data);
 223         if (tag != BLK_MQ_TAG_FAIL) {
 224                 rq = data->hctx->tags->rqs[tag];
 225
 226                 rq->cmd_flags = 0;
 227                 if (blk_mq_tag_busy(data->hctx)) {
 228                         rq->cmd_flags = REQ_MQ_INFLIGHT;
 229                         atomic_inc(&data->hctx->nr_active);
 230                 }
 231
 232                 rq->tag = tag;
 233                 blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw);
 234                 return rq;
 235         }
 236
 237         return NULL;
 238 }
 239
 240 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
 241                 bool reserved)
 242 {
 243         struct blk_mq_ctx *ctx;
 244         struct blk_mq_hw_ctx *hctx;
 245         struct request *rq;
 246         struct blk_mq_alloc_data alloc_data;
 247
 248         if (blk_mq_queue_enter(q))
 249                 return NULL;
 250
 251         ctx = blk_mq_get_ctx(q);
 252         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 253         blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_WAIT,
 254                         reserved, ctx, hctx);
 255
 256         rq = __blk_mq_alloc_request(&alloc_data, rw);
 257         if (!rq && (gfp & __GFP_WAIT)) {
 258                 __blk_mq_run_hw_queue(hctx);
 259                 blk_mq_put_ctx(ctx);
 260
 261                 ctx = blk_mq_get_ctx(q);
 262                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
 263                 blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
 264                                 hctx);
 265                 rq =  __blk_mq_alloc_request(&alloc_data, rw);
 266                 ctx = alloc_data.ctx;
 267         }
 268         blk_mq_put_ctx(ctx);
 269         return rq;
 270 }
 271 EXPORT_SYMBOL(blk_mq_alloc_request);
 272
 273 static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 274                                   struct blk_mq_ctx *ctx, struct request *rq)
 275 {
 276         const int tag = rq->tag;
 277         struct request_queue *q = rq->q;
 278
 279         if (rq->cmd_flags & REQ_MQ_INFLIGHT)
 280                 atomic_dec(&hctx->nr_active);
 281
 282         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 283         blk_mq_put_tag(hctx, tag, &ctx->last_tag);
 284         blk_mq_queue_exit(q);
 285 }
 286
 287 void blk_mq_free_request(struct request *rq)
 288 {
 289         struct blk_mq_ctx *ctx = rq->mq_ctx;
 290         struct blk_mq_hw_ctx *hctx;
 291         struct request_queue *q = rq->q;
 292
 293         ctx->rq_completed[rq_is_sync(rq)]++;
 294
 295         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 296         __blk_mq_free_request(hctx, ctx, rq);
 297 }
 298
 299 /*
 300  * Clone all relevant state from a request that has been put on hold in
 301  * the flush state machine into the preallocated flush request that hangs
 302  * off the request queue.
 303  *
 304  * For a driver the flush request should be invisible, that's why we are
 305  * impersonating the original request here.
 306  */
 307 void blk_mq_clone_flush_request(struct request *flush_rq,
 308                 struct request *orig_rq)
 309 {
 310         struct blk_mq_hw_ctx *hctx =
 311                 orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu);
 312
 313         flush_rq->mq_ctx = orig_rq->mq_ctx;
 314         flush_rq->tag = orig_rq->tag;
 315         memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq),
 316                 hctx->cmd_size);
 317 }
 318
 319 inline void __blk_mq_end_io(struct request *rq, int error)
 320 {
 321         blk_account_io_done(rq);
 322
 323         if (rq->end_io) {
 324                 rq->end_io(rq, error);
 325         } else {
 326                 if (unlikely(blk_bidi_rq(rq)))
 327                         blk_mq_free_request(rq->next_rq);
 328                 blk_mq_free_request(rq);
 329         }
 330 }
 331 EXPORT_SYMBOL(__blk_mq_end_io);
 332
 333 void blk_mq_end_io(struct request *rq, int error)
 334 {
 335         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 336                 BUG();
 337         __blk_mq_end_io(rq, error);
 338 }
 339 EXPORT_SYMBOL(blk_mq_end_io);
 340
 341 static void __blk_mq_complete_request_remote(void *data)
 342 {
 343         struct request *rq = data;
 344
 345         rq->q->softirq_done_fn(rq);
 346 }
 347
 348 static void blk_mq_ipi_complete_request(struct request *rq)
 349 {
 350         struct blk_mq_ctx *ctx = rq->mq_ctx;
 351         bool shared = false;
 352         int cpu;
 353
 354         if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 355                 rq->q->softirq_done_fn(rq);
 356                 return;
 357         }
 358
 359         cpu = get_cpu();
 360         if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
 361                 shared = cpus_share_cache(cpu, ctx->cpu);
 362
 363         if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 364                 rq->csd.func = __blk_mq_complete_request_remote;
 365                 rq->csd.info = rq;
 366                 rq->csd.flags = 0;
 367                 smp_call_function_single_async(ctx->cpu, &rq->csd);
 368         } else {
 369                 rq->q->softirq_done_fn(rq);
 370         }
 371         put_cpu();
 372 }
 373
 374 void __blk_mq_complete_request(struct request *rq)
 375 {
 376         struct request_queue *q = rq->q;
 377
 378         if (!q->softirq_done_fn)
 379                 blk_mq_end_io(rq, rq->errors);
 380         else
 381                 blk_mq_ipi_complete_request(rq);
 382 }
 383
 384 /**
 385  * blk_mq_complete_request - end I/O on a request
 386  * @rq:         the request being processed
 387  *
 388  * Description:
 389  *      Ends all I/O on a request. It does not handle partial completions.
 390  *      The actual completion happens out-of-order, through a IPI handler.
 391  **/
 392 void blk_mq_complete_request(struct request *rq)
 393 {
 394         struct request_queue *q = rq->q;
 395
 396         if (unlikely(blk_should_fake_timeout(q)))
 397                 return;
 398         if (!blk_mark_rq_complete(rq))
 399                 __blk_mq_complete_request(rq);
 400 }
 401 EXPORT_SYMBOL(blk_mq_complete_request);
 402
 403 static void blk_mq_start_request(struct request *rq, bool last)
 404 {
 405         struct request_queue *q = rq->q;
 406
 407         trace_block_rq_issue(q, rq);
 408
 409         rq->resid_len = blk_rq_bytes(rq);
 410         if (unlikely(blk_bidi_rq(rq)))
 411                 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 412
 413         /*
 414          * Just mark start time and set the started bit. Due to memory
 415          * ordering, we know we'll see the correct deadline as long as
 416          * REQ_ATOMIC_STARTED is seen. Use the default queue timeout,
 417          * unless one has been set in the request.
 418          */
 419         if (!rq->timeout)
 420                 rq->deadline = jiffies + q->rq_timeout;
 421         else
 422                 rq->deadline = jiffies + rq->timeout;
 423
 424         /*
 425          * Mark us as started and clear complete. Complete might have been
 426          * set if requeue raced with timeout, which then marked it as
 427          * complete. So be sure to clear complete again when we start
 428          * the request, otherwise we'll ignore the completion event.
 429          */
 430         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 431                 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 432         if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
 433                 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
 434
 435         if (q->dma_drain_size && blk_rq_bytes(rq)) {
 436                 /*
 437                  * Make sure space for the drain appears.  We know we can do
 438                  * this because max_hw_segments has been adjusted to be one
 439                  * fewer than the device can handle.
 440                  */
 441                 rq->nr_phys_segments++;
 442         }
 443
 444         /*
 445          * Flag the last request in the series so that drivers know when IO
 446          * should be kicked off, if they don't do it on a per-request basis.
 447          *
 448          * Note: the flag isn't the only condition drivers should do kick off.
 449          * If drive is busy, the last request might not have the bit set.
 450          */
 451         if (last)
 452                 rq->cmd_flags |= REQ_END;
 453 }
 454
 455 static void __blk_mq_requeue_request(struct request *rq)
 456 {
 457         struct request_queue *q = rq->q;
 458
 459         trace_block_rq_requeue(q, rq);
 460         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 461
 462         rq->cmd_flags &= ~REQ_END;
 463
 464         if (q->dma_drain_size && blk_rq_bytes(rq))
 465                 rq->nr_phys_segments--;
 466 }
 467
 468 void blk_mq_requeue_request(struct request *rq)
 469 {
 470         __blk_mq_requeue_request(rq);
 471         blk_clear_rq_complete(rq);
 472
 473         BUG_ON(blk_queued_rq(rq));
 474         blk_mq_add_to_requeue_list(rq, true);
 475 }
 476 EXPORT_SYMBOL(blk_mq_requeue_request);
 477
 478 static void blk_mq_requeue_work(struct work_struct *work)
 479 {
 480         struct request_queue *q =
 481                 container_of(work, struct request_queue, requeue_work);
 482         LIST_HEAD(rq_list);
 483         struct request *rq, *next;
 484         unsigned long flags;
 485
 486         spin_lock_irqsave(&q->requeue_lock, flags);
 487         list_splice_init(&q->requeue_list, &rq_list);
 488         spin_unlock_irqrestore(&q->requeue_lock, flags);
 489
 490         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
 491                 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
 492                         continue;
 493
 494                 rq->cmd_flags &= ~REQ_SOFTBARRIER;
 495                 list_del_init(&rq->queuelist);
 496                 blk_mq_insert_request(rq, true, false, false);
 497         }
 498
 499         while (!list_empty(&rq_list)) {
 500                 rq = list_entry(rq_list.next, struct request, queuelist);
 501                 list_del_init(&rq->queuelist);
 502                 blk_mq_insert_request(rq, false, false, false);
 503         }
 504
 505         blk_mq_run_queues(q, false);
 506 }
 507
 508 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 509 {
 510         struct request_queue *q = rq->q;
 511         unsigned long flags;
 512
 513         /*
 514          * We abuse this flag that is otherwise used by the I/O scheduler to
 515          * request head insertation from the workqueue.
 516          */
 517         BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
 518
 519         spin_lock_irqsave(&q->requeue_lock, flags);
 520         if (at_head) {
 521                 rq->cmd_flags |= REQ_SOFTBARRIER;
 522                 list_add(&rq->queuelist, &q->requeue_list);
 523         } else {
 524                 list_add_tail(&rq->queuelist, &q->requeue_list);
 525         }
 526         spin_unlock_irqrestore(&q->requeue_lock, flags);
 527 }
 528 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 529
 530 void blk_mq_kick_requeue_list(struct request_queue *q)
 531 {
 532         kblockd_schedule_work(&q->requeue_work);
 533 }
 534 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 535
 536 static inline bool is_flush_request(struct request *rq, unsigned int tag)
 537 {
 538         return ((rq->cmd_flags & REQ_FLUSH_SEQ) &&
 539                         rq->q->flush_rq->tag == tag);
 540 }
 541
 542 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
 543 {
 544         struct request *rq = tags->rqs[tag];
 545
 546         if (!is_flush_request(rq, tag))
 547                 return rq;
 548
 549         return rq->q->flush_rq;
 550 }
 551 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 552
 553 struct blk_mq_timeout_data {
 554         struct blk_mq_hw_ctx *hctx;
 555         unsigned long *next;
 556         unsigned int *next_set;
 557 };
 558
 559 static void blk_mq_timeout_check(void *__data, unsigned long *free_tags)
 560 {
 561         struct blk_mq_timeout_data *data = __data;
 562         struct blk_mq_hw_ctx *hctx = data->hctx;
 563         unsigned int tag;
 564
 565          /* It may not be in flight yet (this is where
 566          * the REQ_ATOMIC_STARTED flag comes in). The requests are
 567          * statically allocated, so we know it's always safe to access the
 568          * memory associated with a bit offset into ->rqs[].
 569          */
 570         tag = 0;
 571         do {
 572                 struct request *rq;
 573
 574                 tag = find_next_zero_bit(free_tags, hctx->tags->nr_tags, tag);
 575                 if (tag >= hctx->tags->nr_tags)
 576                         break;
 577
 578                 rq = blk_mq_tag_to_rq(hctx->tags, tag++);
 579                 if (rq->q != hctx->queue)
 580                         continue;
 581                 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 582                         continue;
 583
 584                 blk_rq_check_expired(rq, data->next, data->next_set);
 585         } while (1);
 586 }
 587
 588 static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx,
 589                                         unsigned long *next,
 590                                         unsigned int *next_set)
 591 {
 592         struct blk_mq_timeout_data data = {
 593                 .hctx           = hctx,
 594                 .next           = next,
 595                 .next_set       = next_set,
 596         };
 597
 598         /*
 599          * Ask the tagging code to iterate busy requests, so we can
 600          * check them for timeout.
 601          */
 602         blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data);
 603 }
 604
 605 static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq)
 606 {
 607         struct request_queue *q = rq->q;
 608
 609         /*
 610          * We know that complete is set at this point. If STARTED isn't set
 611          * anymore, then the request isn't active and the "timeout" should
 612          * just be ignored. This can happen due to the bitflag ordering.
 613          * Timeout first checks if STARTED is set, and if it is, assumes
 614          * the request is active. But if we race with completion, then
 615          * we both flags will get cleared. So check here again, and ignore
 616          * a timeout event with a request that isn't active.
 617          */
 618         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 619                 return BLK_EH_NOT_HANDLED;
 620
 621         if (!q->mq_ops->timeout)
 622                 return BLK_EH_RESET_TIMER;
 623
 624         return q->mq_ops->timeout(rq);
 625 }
 626
 627 static void blk_mq_rq_timer(unsigned long data)
 628 {
 629         struct request_queue *q = (struct request_queue *) data;
 630         struct blk_mq_hw_ctx *hctx;
 631         unsigned long next = 0;
 632         int i, next_set = 0;
 633
 634         queue_for_each_hw_ctx(q, hctx, i) {
 635                 /*
 636                  * If not software queues are currently mapped to this
 637                  * hardware queue, there's nothing to check
 638                  */
 639                 if (!hctx->nr_ctx || !hctx->tags)
 640                         continue;
 641
 642                 blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set);
 643         }
 644
 645         if (next_set) {
 646                 next = blk_rq_timeout(round_jiffies_up(next));
 647                 mod_timer(&q->timeout, next);
 648         } else {
 649                 queue_for_each_hw_ctx(q, hctx, i)
 650                         blk_mq_tag_idle(hctx);
 651         }
 652 }
 653
 654 /*
 655  * Reverse check our software queue for entries that we could potentially
 656  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 657  * too much time checking for merges.
 658  */
 659 static bool blk_mq_attempt_merge(struct request_queue *q,
 660                                  struct blk_mq_ctx *ctx, struct bio *bio)
 661 {
 662         struct request *rq;
 663         int checked = 8;
 664
 665         list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
 666                 int el_ret;
 667
 668                 if (!checked--)
 669                         break;
 670
 671                 if (!blk_rq_merge_ok(rq, bio))
 672                         continue;
 673
 674                 el_ret = blk_try_merge(rq, bio);
 675                 if (el_ret == ELEVATOR_BACK_MERGE) {
 676                         if (bio_attempt_back_merge(q, rq, bio)) {
 677                                 ctx->rq_merged++;
 678                                 return true;
 679                         }
 680                         break;
 681                 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
 682                         if (bio_attempt_front_merge(q, rq, bio)) {
 683                                 ctx->rq_merged++;
 684                                 return true;
 685                         }
 686                         break;
 687                 }
 688         }
 689
 690         return false;
 691 }
 692
 693 /*
 694  * Process software queues that have been marked busy, splicing them
 695  * to the for-dispatch
 696  */
 697 static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 698 {
 699         struct blk_mq_ctx *ctx;
 700         int i;
 701
 702         for (i = 0; i < hctx->ctx_map.map_size; i++) {
 703                 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
 704                 unsigned int off, bit;
 705
 706                 if (!bm->word)
 707                         continue;
 708
 709                 bit = 0;
 710                 off = i * hctx->ctx_map.bits_per_word;
 711                 do {
 712                         bit = find_next_bit(&bm->word, bm->depth, bit);
 713                         if (bit >= bm->depth)
 714                                 break;
 715
 716                         ctx = hctx->ctxs[bit + off];
 717                         clear_bit(bit, &bm->word);
 718                         spin_lock(&ctx->lock);
 719                         list_splice_tail_init(&ctx->rq_list, list);
 720                         spin_unlock(&ctx->lock);
 721
 722                         bit++;
 723                 } while (1);
 724         }
 725 }
 726
 727 /*
 728  * Run this hardware queue, pulling any software queues mapped to it in.
 729  * Note that this function currently has various problems around ordering
 730  * of IO. In particular, we'd like FIFO behaviour on handling existing
 731  * items on the hctx->dispatch list. Ignore that for now.
 732  */
 733 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 734 {
 735         struct request_queue *q = hctx->queue;
 736         struct request *rq;
 737         LIST_HEAD(rq_list);
 738         int queued;
 739
 740         WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 741
 742         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 743                 return;
 744
 745         hctx->run++;
 746
 747         /*
 748          * Touch any software queue that has pending entries.
 749          */
 750         flush_busy_ctxs(hctx, &rq_list);
 751
 752         /*
 753          * If we have previous entries on our dispatch list, grab them
 754          * and stuff them at the front for more fair dispatch.
 755          */
 756         if (!list_empty_careful(&hctx->dispatch)) {
 757                 spin_lock(&hctx->lock);
 758                 if (!list_empty(&hctx->dispatch))
 759                         list_splice_init(&hctx->dispatch, &rq_list);
 760                 spin_unlock(&hctx->lock);
 761         }
 762
 763         /*
 764          * Now process all the entries, sending them to the driver.
 765          */
 766         queued = 0;
 767         while (!list_empty(&rq_list)) {
 768                 int ret;
 769
 770                 rq = list_first_entry(&rq_list, struct request, queuelist);
 771                 list_del_init(&rq->queuelist);
 772
 773                 blk_mq_start_request(rq, list_empty(&rq_list));
 774
 775                 ret = q->mq_ops->queue_rq(hctx, rq);
 776                 switch (ret) {
 777                 case BLK_MQ_RQ_QUEUE_OK:
 778                         queued++;
 779                         continue;
 780                 case BLK_MQ_RQ_QUEUE_BUSY:
 781                         list_add(&rq->queuelist, &rq_list);
 782                         __blk_mq_requeue_request(rq);
 783                         break;
 784                 default:
 785                         pr_err("blk-mq: bad return on queue: %d\n", ret);
 786                 case BLK_MQ_RQ_QUEUE_ERROR:
 787                         rq->errors = -EIO;
 788                         blk_mq_end_io(rq, rq->errors);
 789                         break;
 790                 }
 791
 792                 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
 793                         break;
 794         }
 795
 796         if (!queued)
 797                 hctx->dispatched[0]++;
 798         else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
 799                 hctx->dispatched[ilog2(queued) + 1]++;
 800
 801         /*
 802          * Any items that need requeuing? Stuff them into hctx->dispatch,
 803          * that is where we will continue on next queue run.
 804          */
 805         if (!list_empty(&rq_list)) {
 806                 spin_lock(&hctx->lock);
 807                 list_splice(&rq_list, &hctx->dispatch);
 808                 spin_unlock(&hctx->lock);
 809         }
 810 }
 811
 812 /*
 813  * It'd be great if the workqueue API had a way to pass
 814  * in a mask and had some smarts for more clever placement.
 815  * For now we just round-robin here, switching for every
 816  * BLK_MQ_CPU_WORK_BATCH queued items.
 817  */
 818 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 819 {
 820         int cpu = hctx->next_cpu;
 821
 822         if (--hctx->next_cpu_batch <= 0) {
 823                 int next_cpu;
 824
 825                 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 826                 if (next_cpu >= nr_cpu_ids)
 827                         next_cpu = cpumask_first(hctx->cpumask);
 828
 829                 hctx->next_cpu = next_cpu;
 830                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 831         }
 832
 833         return cpu;
 834 }
 835
 836 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 837 {
 838         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 839                 return;
 840
 841         if (!async && cpumask_test_cpu(smp_processor_id(), hctx->cpumask))
 842                 __blk_mq_run_hw_queue(hctx);
 843         else if (hctx->queue->nr_hw_queues == 1)
 844                 kblockd_schedule_delayed_work(&hctx->run_work, 0);
 845         else {
 846                 unsigned int cpu;
 847
 848                 cpu = blk_mq_hctx_next_cpu(hctx);
 849                 kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
 850         }
 851 }
 852
 853 void blk_mq_run_queues(struct request_queue *q, bool async)
 854 {
 855         struct blk_mq_hw_ctx *hctx;
 856         int i;
 857
 858         queue_for_each_hw_ctx(q, hctx, i) {
 859                 if ((!blk_mq_hctx_has_pending(hctx) &&
 860                     list_empty_careful(&hctx->dispatch)) ||
 861                     test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 862                         continue;
 863
 864                 preempt_disable();
 865                 blk_mq_run_hw_queue(hctx, async);
 866                 preempt_enable();
 867         }
 868 }
 869 EXPORT_SYMBOL(blk_mq_run_queues);
 870
 871 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 872 {
 873         cancel_delayed_work(&hctx->run_work);
 874         cancel_delayed_work(&hctx->delay_work);
 875         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 876 }
 877 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 878
 879 void blk_mq_stop_hw_queues(struct request_queue *q)
 880 {
 881         struct blk_mq_hw_ctx *hctx;
 882         int i;
 883
 884         queue_for_each_hw_ctx(q, hctx, i)
 885                 blk_mq_stop_hw_queue(hctx);
 886 }
 887 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 888
 889 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 890 {
 891         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 892
 893         preempt_disable();
 894         __blk_mq_run_hw_queue(hctx);
 895         preempt_enable();
 896 }
 897 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 898
 899 void blk_mq_start_hw_queues(struct request_queue *q)
 900 {
 901         struct blk_mq_hw_ctx *hctx;
 902         int i;
 903
 904         queue_for_each_hw_ctx(q, hctx, i)
 905                 blk_mq_start_hw_queue(hctx);
 906 }
 907 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 908
 909
 910 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 911 {
 912         struct blk_mq_hw_ctx *hctx;
 913         int i;
 914
 915         queue_for_each_hw_ctx(q, hctx, i) {
 916                 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 917                         continue;
 918
 919                 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 920                 preempt_disable();
 921                 blk_mq_run_hw_queue(hctx, async);
 922                 preempt_enable();
 923         }
 924 }
 925 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 926
 927 static void blk_mq_run_work_fn(struct work_struct *work)
 928 {
 929         struct blk_mq_hw_ctx *hctx;
 930
 931         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 932
 933         __blk_mq_run_hw_queue(hctx);
 934 }
 935
 936 static void blk_mq_delay_work_fn(struct work_struct *work)
 937 {
 938         struct blk_mq_hw_ctx *hctx;
 939
 940         hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
 941
 942         if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
 943                 __blk_mq_run_hw_queue(hctx);
 944 }
 945
 946 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 947 {
 948         unsigned long tmo = msecs_to_jiffies(msecs);
 949
 950         if (hctx->queue->nr_hw_queues == 1)
 951                 kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
 952         else {
 953                 unsigned int cpu;
 954
 955                 cpu = blk_mq_hctx_next_cpu(hctx);
 956                 kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
 957         }
 958 }
 959 EXPORT_SYMBOL(blk_mq_delay_queue);
 960
 961 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
 962                                     struct request *rq, bool at_head)
 963 {
 964         struct blk_mq_ctx *ctx = rq->mq_ctx;
 965
 966         trace_block_rq_insert(hctx->queue, rq);
 967
 968         if (at_head)
 969                 list_add(&rq->queuelist, &ctx->rq_list);
 970         else
 971                 list_add_tail(&rq->queuelist, &ctx->rq_list);
 972
 973         blk_mq_hctx_mark_pending(hctx, ctx);
 974
 975         /*
 976          * We do this early, to ensure we are on the right CPU.
 977          */
 978         blk_add_timer(rq);
 979 }
 980
 981 void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
 982                 bool async)
 983 {
 984         struct request_queue *q = rq->q;
 985         struct blk_mq_hw_ctx *hctx;
 986         struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
 987
 988         current_ctx = blk_mq_get_ctx(q);
 989         if (!cpu_online(ctx->cpu))
 990                 rq->mq_ctx = ctx = current_ctx;
 991
 992         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 993
 994         if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA) &&
 995             !(rq->cmd_flags & (REQ_FLUSH_SEQ))) {
 996                 blk_insert_flush(rq);
 997         } else {
 998                 spin_lock(&ctx->lock);
 999                 __blk_mq_insert_request(hctx, rq, at_head);
1000                 spin_unlock(&ctx->lock);
1001         }
1002
1003         if (run_queue)
1004                 blk_mq_run_hw_queue(hctx, async);
1005
1006         blk_mq_put_ctx(current_ctx);
1007 }
1008
1009 static void blk_mq_insert_requests(struct request_queue *q,
1010                                      struct blk_mq_ctx *ctx,
1011                                      struct list_head *list,
1012                                      int depth,
1013                                      bool from_schedule)
1014
1015 {
1016         struct blk_mq_hw_ctx *hctx;
1017         struct blk_mq_ctx *current_ctx;
1018
1019         trace_block_unplug(q, depth, !from_schedule);
1020
1021         current_ctx = blk_mq_get_ctx(q);
1022
1023         if (!cpu_online(ctx->cpu))
1024                 ctx = current_ctx;
1025         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1026
1027         /*
1028          * preemption doesn't flush plug list, so it's possible ctx->cpu is
1029          * offline now
1030          */
1031         spin_lock(&ctx->lock);
1032         while (!list_empty(list)) {
1033                 struct request *rq;
1034
1035                 rq = list_first_entry(list, struct request, queuelist);
1036                 list_del_init(&rq->queuelist);
1037                 rq->mq_ctx = ctx;
1038                 __blk_mq_insert_request(hctx, rq, false);
1039         }
1040         spin_unlock(&ctx->lock);
1041
1042         blk_mq_run_hw_queue(hctx, from_schedule);
1043         blk_mq_put_ctx(current_ctx);
1044 }
1045
1046 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1047 {
1048         struct request *rqa = container_of(a, struct request, queuelist);
1049         struct request *rqb = container_of(b, struct request, queuelist);
1050
1051         return !(rqa->mq_ctx < rqb->mq_ctx ||
1052                  (rqa->mq_ctx == rqb->mq_ctx &&
1053                   blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1054 }
1055
1056 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1057 {
1058         struct blk_mq_ctx *this_ctx;
1059         struct request_queue *this_q;
1060         struct request *rq;
1061         LIST_HEAD(list);
1062         LIST_HEAD(ctx_list);
1063         unsigned int depth;
1064
1065         list_splice_init(&plug->mq_list, &list);
1066
1067         list_sort(NULL, &list, plug_ctx_cmp);
1068
1069         this_q = NULL;
1070         this_ctx = NULL;
1071         depth = 0;
1072
1073         while (!list_empty(&list)) {
1074                 rq = list_entry_rq(list.next);
1075                 list_del_init(&rq->queuelist);
1076                 BUG_ON(!rq->q);
1077                 if (rq->mq_ctx != this_ctx) {
1078                         if (this_ctx) {
1079                                 blk_mq_insert_requests(this_q, this_ctx,
1080                                                         &ctx_list, depth,
1081                                                         from_schedule);
1082                         }
1083
1084                         this_ctx = rq->mq_ctx;
1085                         this_q = rq->q;
1086                         depth = 0;
1087                 }
1088
1089                 depth++;
1090                 list_add_tail(&rq->queuelist, &ctx_list);
1091         }
1092
1093         /*
1094          * If 'this_ctx' is set, we know we have entries to complete
1095          * on 'ctx_list'. Do those.
1096          */
1097         if (this_ctx) {
1098                 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1099                                        from_schedule);
1100         }
1101 }
1102
1103 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1104 {
1105         init_request_from_bio(rq, bio);
1106
1107         if (blk_do_io_stat(rq)) {
1108                 rq->start_time = jiffies;
1109                 blk_account_io_start(rq, 1);
1110         }
1111 }
1112
1113 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1114                                          struct blk_mq_ctx *ctx,
1115                                          struct request *rq, struct bio *bio)
1116 {
1117         struct request_queue *q = hctx->queue;
1118
1119         if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
1120                 blk_mq_bio_to_request(rq, bio);
1121                 spin_lock(&ctx->lock);
1122 insert_rq:
1123                 __blk_mq_insert_request(hctx, rq, false);
1124                 spin_unlock(&ctx->lock);
1125                 return false;
1126         } else {
1127                 spin_lock(&ctx->lock);
1128                 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1129                         blk_mq_bio_to_request(rq, bio);
1130                         goto insert_rq;
1131                 }
1132
1133                 spin_unlock(&ctx->lock);
1134                 __blk_mq_free_request(hctx, ctx, rq);
1135                 return true;
1136         }
1137 }
1138
1139 struct blk_map_ctx {
1140         struct blk_mq_hw_ctx *hctx;
1141         struct blk_mq_ctx *ctx;
1142 };
1143
1144 static struct request *blk_mq_map_request(struct request_queue *q,
1145                                           struct bio *bio,
1146                                           struct blk_map_ctx *data)
1147 {
1148         struct blk_mq_hw_ctx *hctx;
1149         struct blk_mq_ctx *ctx;
1150         struct request *rq;
1151         int rw = bio_data_dir(bio);
1152         struct blk_mq_alloc_data alloc_data;
1153
1154         if (unlikely(blk_mq_queue_enter(q))) {
1155                 bio_endio(bio, -EIO);
1156                 return NULL;
1157         }
1158
1159         ctx = blk_mq_get_ctx(q);
1160         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1161
1162         if (rw_is_sync(bio->bi_rw))
1163                 rw |= REQ_SYNC;
1164
1165         trace_block_getrq(q, bio, rw);
1166         blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
1167                         hctx);
1168         rq = __blk_mq_alloc_request(&alloc_data, rw);
1169         if (unlikely(!rq)) {
1170                 __blk_mq_run_hw_queue(hctx);
1171                 blk_mq_put_ctx(ctx);
1172                 trace_block_sleeprq(q, bio, rw);
1173
1174                 ctx = blk_mq_get_ctx(q);
1175                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1176                 blk_mq_set_alloc_data(&alloc_data, q,
1177                                 __GFP_WAIT|GFP_ATOMIC, false, ctx, hctx);
1178                 rq = __blk_mq_alloc_request(&alloc_data, rw);
1179                 ctx = alloc_data.ctx;
1180                 hctx = alloc_data.hctx;
1181         }
1182
1183         hctx->queued++;
1184         data->hctx = hctx;
1185         data->ctx = ctx;
1186         return rq;
1187 }
1188
1189 /*
1190  * Multiple hardware queue variant. This will not use per-process plugs,
1191  * but will attempt to bypass the hctx queueing if we can go straight to
1192  * hardware for SYNC IO.
1193  */
1194 static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1195 {
1196         const int is_sync = rw_is_sync(bio->bi_rw);
1197         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1198         struct blk_map_ctx data;
1199         struct request *rq;
1200
1201         blk_queue_bounce(q, &bio);
1202
1203         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1204                 bio_endio(bio, -EIO);
1205                 return;
1206         }
1207
1208         rq = blk_mq_map_request(q, bio, &data);
1209         if (unlikely(!rq))
1210                 return;
1211
1212         if (unlikely(is_flush_fua)) {
1213                 blk_mq_bio_to_request(rq, bio);
1214                 blk_insert_flush(rq);
1215                 goto run_queue;
1216         }
1217
1218         if (is_sync) {
1219                 int ret;
1220
1221                 blk_mq_bio_to_request(rq, bio);
1222                 blk_mq_start_request(rq, true);
1223                 blk_add_timer(rq);
1224
1225                 /*
1226                  * For OK queue, we are done. For error, kill it. Any other
1227                  * error (busy), just add it to our list as we previously
1228                  * would have done
1229                  */
1230                 ret = q->mq_ops->queue_rq(data.hctx, rq);
1231                 if (ret == BLK_MQ_RQ_QUEUE_OK)
1232                         goto done;
1233                 else {
1234                         __blk_mq_requeue_request(rq);
1235
1236                         if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1237                                 rq->errors = -EIO;
1238                                 blk_mq_end_io(rq, rq->errors);
1239                                 goto done;
1240                         }
1241                 }
1242         }
1243
1244         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1245                 /*
1246                  * For a SYNC request, send it to the hardware immediately. For
1247                  * an ASYNC request, just ensure that we run it later on. The
1248                  * latter allows for merging opportunities and more efficient
1249                  * dispatching.
1250                  */
1251 run_queue:
1252                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1253         }
1254 done:
1255         blk_mq_put_ctx(data.ctx);
1256 }
1257
1258 /*
1259  * Single hardware queue variant. This will attempt to use any per-process
1260  * plug for merging and IO deferral.
1261  */
1262 static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1263 {
1264         const int is_sync = rw_is_sync(bio->bi_rw);
1265         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1266         unsigned int use_plug, request_count = 0;
1267         struct blk_map_ctx data;
1268         struct request *rq;
1269
1270         /*
1271          * If we have multiple hardware queues, just go directly to
1272          * one of those for sync IO.
1273          */
1274         use_plug = !is_flush_fua && !is_sync;
1275
1276         blk_queue_bounce(q, &bio);
1277
1278         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1279                 bio_endio(bio, -EIO);
1280                 return;
1281         }
1282
1283         if (use_plug && !blk_queue_nomerges(q) &&
1284             blk_attempt_plug_merge(q, bio, &request_count))
1285                 return;
1286
1287         rq = blk_mq_map_request(q, bio, &data);
1288         if (unlikely(!rq))
1289                 return;
1290
1291         if (unlikely(is_flush_fua)) {
1292                 blk_mq_bio_to_request(rq, bio);
1293                 blk_insert_flush(rq);
1294                 goto run_queue;
1295         }
1296
1297         /*
1298          * A task plug currently exists. Since this is completely lockless,
1299          * utilize that to temporarily store requests until the task is
1300          * either done or scheduled away.
1301          */
1302         if (use_plug) {
1303                 struct blk_plug *plug = current->plug;
1304
1305                 if (plug) {
1306                         blk_mq_bio_to_request(rq, bio);
1307                         if (list_empty(&plug->mq_list))
1308                                 trace_block_plug(q);
1309                         else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1310                                 blk_flush_plug_list(plug, false);
1311                                 trace_block_plug(q);
1312                         }
1313                         list_add_tail(&rq->queuelist, &plug->mq_list);
1314                         blk_mq_put_ctx(data.ctx);
1315                         return;
1316                 }
1317         }
1318
1319         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1320                 /*
1321                  * For a SYNC request, send it to the hardware immediately. For
1322                  * an ASYNC request, just ensure that we run it later on. The
1323                  * latter allows for merging opportunities and more efficient
1324                  * dispatching.
1325                  */
1326 run_queue:
1327                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1328         }
1329
1330         blk_mq_put_ctx(data.ctx);
1331 }
1332
1333 /*
1334  * Default mapping to a software queue, since we use one per CPU.
1335  */
1336 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1337 {
1338         return q->queue_hw_ctx[q->mq_map[cpu]];
1339 }
1340 EXPORT_SYMBOL(blk_mq_map_queue);
1341
1342 static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1343                 struct blk_mq_tags *tags, unsigned int hctx_idx)
1344 {
1345         struct page *page;
1346
1347         if (tags->rqs && set->ops->exit_request) {
1348                 int i;
1349
1350                 for (i = 0; i < tags->nr_tags; i++) {
1351                         if (!tags->rqs[i])
1352                                 continue;
1353                         set->ops->exit_request(set->driver_data, tags->rqs[i],
1354                                                 hctx_idx, i);
1355                 }
1356         }
1357
1358         while (!list_empty(&tags->page_list)) {
1359                 page = list_first_entry(&tags->page_list, struct page, lru);
1360                 list_del_init(&page->lru);
1361                 __free_pages(page, page->private);
1362         }
1363
1364         kfree(tags->rqs);
1365
1366         blk_mq_free_tags(tags);
1367 }
1368
1369 static size_t order_to_size(unsigned int order)
1370 {
1371         return (size_t)PAGE_SIZE << order;
1372 }
1373
1374 static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1375                 unsigned int hctx_idx)
1376 {
1377         struct blk_mq_tags *tags;
1378         unsigned int i, j, entries_per_page, max_order = 4;
1379         size_t rq_size, left;
1380
1381         tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1382                                 set->numa_node);
1383         if (!tags)
1384                 return NULL;
1385
1386         INIT_LIST_HEAD(&tags->page_list);
1387
1388         tags->rqs = kmalloc_node(set->queue_depth * sizeof(struct request *),
1389                                         GFP_KERNEL, set->numa_node);
1390         if (!tags->rqs) {
1391                 blk_mq_free_tags(tags);
1392                 return NULL;
1393         }
1394
1395         /*
1396          * rq_size is the size of the request plus driver payload, rounded
1397          * to the cacheline size
1398          */
1399         rq_size = round_up(sizeof(struct request) + set->cmd_size,
1400                                 cache_line_size());
1401         left = rq_size * set->queue_depth;
1402
1403         for (i = 0; i < set->queue_depth; ) {
1404                 int this_order = max_order;
1405                 struct page *page;
1406                 int to_do;
1407                 void *p;
1408
1409                 while (left < order_to_size(this_order - 1) && this_order)
1410                         this_order--;
1411
1412                 do {
1413                         page = alloc_pages_node(set->numa_node, GFP_KERNEL,
1414                                                 this_order);
1415                         if (page)
1416                                 break;
1417                         if (!this_order--)
1418                                 break;
1419                         if (order_to_size(this_order) < rq_size)
1420                                 break;
1421                 } while (1);
1422
1423                 if (!page)
1424                         goto fail;
1425
1426                 page->private = this_order;
1427                 list_add_tail(&page->lru, &tags->page_list);
1428
1429                 p = page_address(page);
1430                 entries_per_page = order_to_size(this_order) / rq_size;
1431                 to_do = min(entries_per_page, set->queue_depth - i);
1432                 left -= to_do * rq_size;
1433                 for (j = 0; j < to_do; j++) {
1434                         tags->rqs[i] = p;
1435                         if (set->ops->init_request) {
1436                                 if (set->ops->init_request(set->driver_data,
1437                                                 tags->rqs[i], hctx_idx, i,
1438                                                 set->numa_node))
1439                                         goto fail;
1440                         }
1441
1442                         p += rq_size;
1443                         i++;
1444                 }
1445         }
1446
1447         return tags;
1448
1449 fail:
1450         pr_warn("%s: failed to allocate requests\n", __func__);
1451         blk_mq_free_rq_map(set, tags, hctx_idx);
1452         return NULL;
1453 }
1454
1455 static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1456 {
1457         kfree(bitmap->map);
1458 }
1459
1460 static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1461 {
1462         unsigned int bpw = 8, total, num_maps, i;
1463
1464         bitmap->bits_per_word = bpw;
1465
1466         num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1467         bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1468                                         GFP_KERNEL, node);
1469         if (!bitmap->map)
1470                 return -ENOMEM;
1471
1472         bitmap->map_size = num_maps;
1473
1474         total = nr_cpu_ids;
1475         for (i = 0; i < num_maps; i++) {
1476                 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1477                 total -= bitmap->map[i].depth;
1478         }
1479
1480         return 0;
1481 }
1482
1483 static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1484 {
1485         struct request_queue *q = hctx->queue;
1486         struct blk_mq_ctx *ctx;
1487         LIST_HEAD(tmp);
1488
1489         /*
1490          * Move ctx entries to new CPU, if this one is going away.
1491          */
1492         ctx = __blk_mq_get_ctx(q, cpu);
1493
1494         spin_lock(&ctx->lock);
1495         if (!list_empty(&ctx->rq_list)) {
1496                 list_splice_init(&ctx->rq_list, &tmp);
1497                 blk_mq_hctx_clear_pending(hctx, ctx);
1498         }
1499         spin_unlock(&ctx->lock);
1500
1501         if (list_empty(&tmp))
1502                 return NOTIFY_OK;
1503
1504         ctx = blk_mq_get_ctx(q);
1505         spin_lock(&ctx->lock);
1506
1507         while (!list_empty(&tmp)) {
1508                 struct request *rq;
1509
1510                 rq = list_first_entry(&tmp, struct request, queuelist);
1511                 rq->mq_ctx = ctx;
1512                 list_move_tail(&rq->queuelist, &ctx->rq_list);
1513         }
1514
1515         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1516         blk_mq_hctx_mark_pending(hctx, ctx);
1517
1518         spin_unlock(&ctx->lock);
1519
1520         blk_mq_run_hw_queue(hctx, true);
1521         blk_mq_put_ctx(ctx);
1522         return NOTIFY_OK;
1523 }
1524
1525 static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1526 {
1527         struct request_queue *q = hctx->queue;
1528         struct blk_mq_tag_set *set = q->tag_set;
1529
1530         if (set->tags[hctx->queue_num])
1531                 return NOTIFY_OK;
1532
1533         set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1534         if (!set->tags[hctx->queue_num])
1535                 return NOTIFY_STOP;
1536
1537         hctx->tags = set->tags[hctx->queue_num];
1538         return NOTIFY_OK;
1539 }
1540
1541 static int blk_mq_hctx_notify(void *data, unsigned long action,
1542                               unsigned int cpu)
1543 {
1544         struct blk_mq_hw_ctx *hctx = data;
1545
1546         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1547                 return blk_mq_hctx_cpu_offline(hctx, cpu);
1548         else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1549                 return blk_mq_hctx_cpu_online(hctx, cpu);
1550
1551         return NOTIFY_OK;
1552 }
1553
1554 static void blk_mq_exit_hw_queues(struct request_queue *q,
1555                 struct blk_mq_tag_set *set, int nr_queue)
1556 {
1557         struct blk_mq_hw_ctx *hctx;
1558         unsigned int i;
1559
1560         queue_for_each_hw_ctx(q, hctx, i) {
1561                 if (i == nr_queue)
1562                         break;
1563
1564                 blk_mq_tag_idle(hctx);
1565
1566                 if (set->ops->exit_hctx)
1567                         set->ops->exit_hctx(hctx, i);
1568
1569                 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1570                 kfree(hctx->ctxs);
1571                 blk_mq_free_bitmap(&hctx->ctx_map);
1572         }
1573
1574 }
1575
1576 static void blk_mq_free_hw_queues(struct request_queue *q,
1577                 struct blk_mq_tag_set *set)
1578 {
1579         struct blk_mq_hw_ctx *hctx;
1580         unsigned int i;
1581
1582         queue_for_each_hw_ctx(q, hctx, i) {
1583                 free_cpumask_var(hctx->cpumask);
1584                 kfree(hctx);
1585         }
1586 }
1587
1588 static int blk_mq_init_hw_queues(struct request_queue *q,
1589                 struct blk_mq_tag_set *set)
1590 {
1591         struct blk_mq_hw_ctx *hctx;
1592         unsigned int i;
1593
1594         /*
1595          * Initialize hardware queues
1596          */
1597         queue_for_each_hw_ctx(q, hctx, i) {
1598                 int node;
1599
1600                 node = hctx->numa_node;
1601                 if (node == NUMA_NO_NODE)
1602                         node = hctx->numa_node = set->numa_node;
1603
1604                 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1605                 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1606                 spin_lock_init(&hctx->lock);
1607                 INIT_LIST_HEAD(&hctx->dispatch);
1608                 hctx->queue = q;
1609                 hctx->queue_num = i;
1610                 hctx->flags = set->flags;
1611                 hctx->cmd_size = set->cmd_size;
1612
1613                 blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1614                                                 blk_mq_hctx_notify, hctx);
1615                 blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1616
1617                 hctx->tags = set->tags[i];
1618
1619                 /*
1620                  * Allocate space for all possible cpus to avoid allocation in
1621                  * runtime
1622                  */
1623                 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1624                                                 GFP_KERNEL, node);
1625                 if (!hctx->ctxs)
1626                         break;
1627
1628                 if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
1629                         break;
1630
1631                 hctx->nr_ctx = 0;
1632
1633                 if (set->ops->init_hctx &&
1634                     set->ops->init_hctx(hctx, set->driver_data, i))
1635                         break;
1636         }
1637
1638         if (i == q->nr_hw_queues)
1639                 return 0;
1640
1641         /*
1642          * Init failed
1643          */
1644         blk_mq_exit_hw_queues(q, set, i);
1645
1646         return 1;
1647 }
1648
1649 static void blk_mq_init_cpu_queues(struct request_queue *q,
1650                                    unsigned int nr_hw_queues)
1651 {
1652         unsigned int i;
1653
1654         for_each_possible_cpu(i) {
1655                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1656                 struct blk_mq_hw_ctx *hctx;
1657
1658                 memset(__ctx, 0, sizeof(*__ctx));
1659                 __ctx->cpu = i;
1660                 spin_lock_init(&__ctx->lock);
1661                 INIT_LIST_HEAD(&__ctx->rq_list);
1662                 __ctx->queue = q;
1663
1664                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1665                 if (!cpu_online(i))
1666                         continue;
1667
1668                 hctx = q->mq_ops->map_queue(q, i);
1669                 cpumask_set_cpu(i, hctx->cpumask);
1670                 hctx->nr_ctx++;
1671
1672                 /*
1673                  * Set local node, IFF we have more than one hw queue. If
1674                  * not, we remain on the home node of the device
1675                  */
1676                 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1677                         hctx->numa_node = cpu_to_node(i);
1678         }
1679 }
1680
1681 static void blk_mq_map_swqueue(struct request_queue *q)
1682 {
1683         unsigned int i;
1684         struct blk_mq_hw_ctx *hctx;
1685         struct blk_mq_ctx *ctx;
1686
1687         queue_for_each_hw_ctx(q, hctx, i) {
1688                 cpumask_clear(hctx->cpumask);
1689                 hctx->nr_ctx = 0;
1690         }
1691
1692         /*
1693          * Map software to hardware queues
1694          */
1695         queue_for_each_ctx(q, ctx, i) {
1696                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1697                 if (!cpu_online(i))
1698                         continue;
1699
1700                 hctx = q->mq_ops->map_queue(q, i);
1701                 cpumask_set_cpu(i, hctx->cpumask);
1702                 ctx->index_hw = hctx->nr_ctx;
1703                 hctx->ctxs[hctx->nr_ctx++] = ctx;
1704         }
1705
1706         queue_for_each_hw_ctx(q, hctx, i) {
1707                 /*
1708                  * If not software queues are mapped to this hardware queue,
1709                  * disable it and free the request entries
1710                  */
1711                 if (!hctx->nr_ctx) {
1712                         struct blk_mq_tag_set *set = q->tag_set;
1713
1714                         if (set->tags[i]) {
1715                                 blk_mq_free_rq_map(set, set->tags[i], i);
1716                                 set->tags[i] = NULL;
1717                                 hctx->tags = NULL;
1718                         }
1719                         continue;
1720                 }
1721
1722                 /*
1723                  * Initialize batch roundrobin counts
1724                  */
1725                 hctx->next_cpu = cpumask_first(hctx->cpumask);
1726                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1727         }
1728 }
1729
1730 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1731 {
1732         struct blk_mq_hw_ctx *hctx;
1733         struct request_queue *q;
1734         bool shared;
1735         int i;
1736
1737         if (set->tag_list.next == set->tag_list.prev)
1738                 shared = false;
1739         else
1740                 shared = true;
1741
1742         list_for_each_entry(q, &set->tag_list, tag_set_list) {
1743                 blk_mq_freeze_queue(q);
1744
1745                 queue_for_each_hw_ctx(q, hctx, i) {
1746                         if (shared)
1747                                 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1748                         else
1749                                 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1750                 }
1751                 blk_mq_unfreeze_queue(q);
1752         }
1753 }
1754
1755 static void blk_mq_del_queue_tag_set(struct request_queue *q)
1756 {
1757         struct blk_mq_tag_set *set = q->tag_set;
1758
1759         blk_mq_freeze_queue(q);
1760
1761         mutex_lock(&set->tag_list_lock);
1762         list_del_init(&q->tag_set_list);
1763         blk_mq_update_tag_set_depth(set);
1764         mutex_unlock(&set->tag_list_lock);
1765
1766         blk_mq_unfreeze_queue(q);
1767 }
1768
1769 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1770                                      struct request_queue *q)
1771 {
1772         q->tag_set = set;
1773
1774         mutex_lock(&set->tag_list_lock);
1775         list_add_tail(&q->tag_set_list, &set->tag_list);
1776         blk_mq_update_tag_set_depth(set);
1777         mutex_unlock(&set->tag_list_lock);
1778 }
1779
1780 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1781 {
1782         struct blk_mq_hw_ctx **hctxs;
1783         struct blk_mq_ctx __percpu *ctx;
1784         struct request_queue *q;
1785         unsigned int *map;
1786         int i;
1787
1788         ctx = alloc_percpu(struct blk_mq_ctx);
1789         if (!ctx)
1790                 return ERR_PTR(-ENOMEM);
1791
1792         hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1793                         set->numa_node);
1794
1795         if (!hctxs)
1796                 goto err_percpu;
1797
1798         map = blk_mq_make_queue_map(set);
1799         if (!map)
1800                 goto err_map;
1801
1802         for (i = 0; i < set->nr_hw_queues; i++) {
1803                 int node = blk_mq_hw_queue_to_node(map, i);
1804
1805                 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
1806                                         GFP_KERNEL, node);
1807                 if (!hctxs[i])
1808                         goto err_hctxs;
1809
1810                 if (!zalloc_cpumask_var(&hctxs[i]->cpumask, GFP_KERNEL))
1811                         goto err_hctxs;
1812
1813                 atomic_set(&hctxs[i]->nr_active, 0);
1814                 hctxs[i]->numa_node = node;
1815                 hctxs[i]->queue_num = i;
1816         }
1817
1818         q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1819         if (!q)
1820                 goto err_hctxs;
1821
1822         if (percpu_counter_init(&q->mq_usage_counter, 0))
1823                 goto err_map;
1824
1825         setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1826         blk_queue_rq_timeout(q, 30000);
1827
1828         q->nr_queues = nr_cpu_ids;
1829         q->nr_hw_queues = set->nr_hw_queues;
1830         q->mq_map = map;
1831
1832         q->queue_ctx = ctx;
1833         q->queue_hw_ctx = hctxs;
1834
1835         q->mq_ops = set->ops;
1836         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1837
1838         if (!(set->flags & BLK_MQ_F_SG_MERGE))
1839                 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
1840
1841         q->sg_reserved_size = INT_MAX;
1842
1843         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1844         INIT_LIST_HEAD(&q->requeue_list);
1845         spin_lock_init(&q->requeue_lock);
1846
1847         if (q->nr_hw_queues > 1)
1848                 blk_queue_make_request(q, blk_mq_make_request);
1849         else
1850                 blk_queue_make_request(q, blk_sq_make_request);
1851
1852         blk_queue_rq_timed_out(q, blk_mq_rq_timed_out);
1853         if (set->timeout)
1854                 blk_queue_rq_timeout(q, set->timeout);
1855
1856         /*
1857          * Do this after blk_queue_make_request() overrides it...
1858          */
1859         q->nr_requests = set->queue_depth;
1860
1861         if (set->ops->complete)
1862                 blk_queue_softirq_done(q, set->ops->complete);
1863
1864         blk_mq_init_flush(q);
1865         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1866
1867         q->flush_rq = kzalloc(round_up(sizeof(struct request) +
1868                                 set->cmd_size, cache_line_size()),
1869                                 GFP_KERNEL);
1870         if (!q->flush_rq)
1871                 goto err_hw;
1872
1873         if (blk_mq_init_hw_queues(q, set))
1874                 goto err_flush_rq;
1875
1876         mutex_lock(&all_q_mutex);
1877         list_add_tail(&q->all_q_node, &all_q_list);
1878         mutex_unlock(&all_q_mutex);
1879
1880         blk_mq_add_queue_tag_set(set, q);
1881
1882         blk_mq_map_swqueue(q);
1883
1884         return q;
1885
1886 err_flush_rq:
1887         kfree(q->flush_rq);
1888 err_hw:
1889         blk_cleanup_queue(q);
1890 err_hctxs:
1891         kfree(map);
1892         for (i = 0; i < set->nr_hw_queues; i++) {
1893                 if (!hctxs[i])
1894                         break;
1895                 free_cpumask_var(hctxs[i]->cpumask);
1896                 kfree(hctxs[i]);
1897         }
1898 err_map:
1899         kfree(hctxs);
1900 err_percpu:
1901         free_percpu(ctx);
1902         return ERR_PTR(-ENOMEM);
1903 }
1904 EXPORT_SYMBOL(blk_mq_init_queue);
1905
1906 void blk_mq_free_queue(struct request_queue *q)
1907 {
1908         struct blk_mq_tag_set   *set = q->tag_set;
1909
1910         blk_mq_del_queue_tag_set(q);
1911
1912         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1913         blk_mq_free_hw_queues(q, set);
1914
1915         percpu_counter_destroy(&q->mq_usage_counter);
1916
1917         free_percpu(q->queue_ctx);
1918         kfree(q->queue_hw_ctx);
1919         kfree(q->mq_map);
1920
1921         q->queue_ctx = NULL;
1922         q->queue_hw_ctx = NULL;
1923         q->mq_map = NULL;
1924
1925         mutex_lock(&all_q_mutex);
1926         list_del_init(&q->all_q_node);
1927         mutex_unlock(&all_q_mutex);
1928 }
1929
1930 /* Basically redo blk_mq_init_queue with queue frozen */
1931 static void blk_mq_queue_reinit(struct request_queue *q)
1932 {
1933         blk_mq_freeze_queue(q);
1934
1935         blk_mq_sysfs_unregister(q);
1936
1937         blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1938
1939         /*
1940          * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1941          * we should change hctx numa_node according to new topology (this
1942          * involves free and re-allocate memory, worthy doing?)
1943          */
1944
1945         blk_mq_map_swqueue(q);
1946
1947         blk_mq_sysfs_register(q);
1948
1949         blk_mq_unfreeze_queue(q);
1950 }
1951
1952 static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1953                                       unsigned long action, void *hcpu)
1954 {
1955         struct request_queue *q;
1956
1957         /*
1958          * Before new mappings are established, hotadded cpu might already
1959          * start handling requests. This doesn't break anything as we map
1960          * offline CPUs to first hardware queue. We will re-init the queue
1961          * below to get optimal settings.
1962          */
1963         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1964             action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1965                 return NOTIFY_OK;
1966
1967         mutex_lock(&all_q_mutex);
1968         list_for_each_entry(q, &all_q_list, all_q_node)
1969                 blk_mq_queue_reinit(q);
1970         mutex_unlock(&all_q_mutex);
1971         return NOTIFY_OK;
1972 }
1973
1974 /*
1975  * Alloc a tag set to be associated with one or more request queues.
1976  * May fail with EINVAL for various error conditions. May adjust the
1977  * requested depth down, if if it too large. In that case, the set
1978  * value will be stored in set->queue_depth.
1979  */
1980 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
1981 {
1982         int i;
1983
1984         if (!set->nr_hw_queues)
1985                 return -EINVAL;
1986         if (!set->queue_depth)
1987                 return -EINVAL;
1988         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
1989                 return -EINVAL;
1990
1991         if (!set->nr_hw_queues || !set->ops->queue_rq || !set->ops->map_queue)
1992                 return -EINVAL;
1993
1994         if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
1995                 pr_info("blk-mq: reduced tag depth to %u\n",
1996                         BLK_MQ_MAX_DEPTH);
1997                 set->queue_depth = BLK_MQ_MAX_DEPTH;
1998         }
1999
2000         set->tags = kmalloc_node(set->nr_hw_queues *
2001                                  sizeof(struct blk_mq_tags *),
2002                                  GFP_KERNEL, set->numa_node);
2003         if (!set->tags)
2004                 goto out;
2005
2006         for (i = 0; i < set->nr_hw_queues; i++) {
2007                 set->tags[i] = blk_mq_init_rq_map(set, i);
2008                 if (!set->tags[i])
2009                         goto out_unwind;
2010         }
2011
2012         mutex_init(&set->tag_list_lock);
2013         INIT_LIST_HEAD(&set->tag_list);
2014
2015         return 0;
2016
2017 out_unwind:
2018         while (--i >= 0)
2019                 blk_mq_free_rq_map(set, set->tags[i], i);
2020 out:
2021         return -ENOMEM;
2022 }
2023 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2024
2025 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2026 {
2027         int i;
2028
2029         for (i = 0; i < set->nr_hw_queues; i++) {
2030                 if (set->tags[i])
2031                         blk_mq_free_rq_map(set, set->tags[i], i);
2032         }
2033
2034         kfree(set->tags);
2035 }
2036 EXPORT_SYMBOL(blk_mq_free_tag_set);
2037
2038 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2039 {
2040         struct blk_mq_tag_set *set = q->tag_set;
2041         struct blk_mq_hw_ctx *hctx;
2042         int i, ret;
2043
2044         if (!set || nr > set->queue_depth)
2045                 return -EINVAL;
2046
2047         ret = 0;
2048         queue_for_each_hw_ctx(q, hctx, i) {
2049                 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2050                 if (ret)
2051                         break;
2052         }
2053
2054         if (!ret)
2055                 q->nr_requests = nr;
2056
2057         return ret;
2058 }
2059
2060 void blk_mq_disable_hotplug(void)
2061 {
2062         mutex_lock(&all_q_mutex);
2063 }
2064
2065 void blk_mq_enable_hotplug(void)
2066 {
2067         mutex_unlock(&all_q_mutex);
2068 }
2069
2070 static int __init blk_mq_init(void)
2071 {
2072         blk_mq_cpu_init();
2073
2074         /* Must be called after percpu_counter_hotcpu_callback() */
2075         hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
2076
2077         return 0;
2078 }
2079 subsys_initcall(blk_mq_init);