block/blk-mq.c

   1 /*
   2  * Block multiqueue core code
   3  *
   4  * Copyright (C) 2013-2014 Jens Axboe
   5  * Copyright (C) 2013-2014 Christoph Hellwig
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/backing-dev.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/mm.h>
  13 #include <linux/init.h>
  14 #include <linux/slab.h>
  15 #include <linux/workqueue.h>
  16 #include <linux/smp.h>
  17 #include <linux/llist.h>
  18 #include <linux/list_sort.h>
  19 #include <linux/cpu.h>
  20 #include <linux/cache.h>
  21 #include <linux/sched/sysctl.h>
  22 #include <linux/delay.h>
  23 #include <linux/crash_dump.h>
  24
  25 #include <trace/events/block.h>
  26
  27 #include <linux/blk-mq.h>
  28 #include "blk.h"
  29 #include "blk-mq.h"
  30 #include "blk-mq-tag.h"
  31
  32 static DEFINE_MUTEX(all_q_mutex);
  33 static LIST_HEAD(all_q_list);
  34
  35 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
  36
  37 /*
  38  * Check if any of the ctx's have pending work in this hardware queue
  39  */
  40 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
  41 {
  42         unsigned int i;
  43
  44         for (i = 0; i < hctx->ctx_map.map_size; i++)
  45                 if (hctx->ctx_map.map[i].word)
  46                         return true;
  47
  48         return false;
  49 }
  50
  51 static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
  52                                               struct blk_mq_ctx *ctx)
  53 {
  54         return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
  55 }
  56
  57 #define CTX_TO_BIT(hctx, ctx)   \
  58         ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
  59
  60 /*
  61  * Mark this ctx as having pending work in this hardware queue
  62  */
  63 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
  64                                      struct blk_mq_ctx *ctx)
  65 {
  66         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  67
  68         if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
  69                 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  70 }
  71
  72 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
  73                                       struct blk_mq_ctx *ctx)
  74 {
  75         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  76
  77         clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  78 }
  79
  80 static int blk_mq_queue_enter(struct request_queue *q)
  81 {
  82         while (true) {
  83                 int ret;
  84
  85                 if (percpu_ref_tryget_live(&q->mq_usage_counter))
  86                         return 0;
  87
  88                 ret = wait_event_interruptible(q->mq_freeze_wq,
  89                                 !q->mq_freeze_depth || blk_queue_dying(q));
  90                 if (blk_queue_dying(q))
  91                         return -ENODEV;
  92                 if (ret)
  93                         return ret;
  94         }
  95 }
  96
  97 static void blk_mq_queue_exit(struct request_queue *q)
  98 {
  99         percpu_ref_put(&q->mq_usage_counter);
 100 }
 101
 102 static void blk_mq_usage_counter_release(struct percpu_ref *ref)
 103 {
 104         struct request_queue *q =
 105                 container_of(ref, struct request_queue, mq_usage_counter);
 106
 107         wake_up_all(&q->mq_freeze_wq);
 108 }
 109
 110 /*
 111  * Guarantee no request is in use, so we can change any data structure of
 112  * the queue afterward.
 113  */
 114 void blk_mq_freeze_queue(struct request_queue *q)
 115 {
 116         bool freeze;
 117
 118         spin_lock_irq(q->queue_lock);
 119         freeze = !q->mq_freeze_depth++;
 120         spin_unlock_irq(q->queue_lock);
 121
 122         if (freeze) {
 123                 percpu_ref_kill(&q->mq_usage_counter);
 124                 blk_mq_run_queues(q, false);
 125         }
 126         wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->mq_usage_counter));
 127 }
 128
 129 static void blk_mq_unfreeze_queue(struct request_queue *q)
 130 {
 131         bool wake;
 132
 133         spin_lock_irq(q->queue_lock);
 134         wake = !--q->mq_freeze_depth;
 135         WARN_ON_ONCE(q->mq_freeze_depth < 0);
 136         spin_unlock_irq(q->queue_lock);
 137         if (wake) {
 138                 percpu_ref_reinit(&q->mq_usage_counter);
 139                 wake_up_all(&q->mq_freeze_wq);
 140         }
 141 }
 142
 143 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 144 {
 145         return blk_mq_has_free_tags(hctx->tags);
 146 }
 147 EXPORT_SYMBOL(blk_mq_can_queue);
 148
 149 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 150                                struct request *rq, unsigned int rw_flags)
 151 {
 152         if (blk_queue_io_stat(q))
 153                 rw_flags |= REQ_IO_STAT;
 154
 155         INIT_LIST_HEAD(&rq->queuelist);
 156         /* csd/requeue_work/fifo_time is initialized before use */
 157         rq->q = q;
 158         rq->mq_ctx = ctx;
 159         rq->cmd_flags |= rw_flags;
 160         /* do not touch atomic flags, it needs atomic ops against the timer */
 161         rq->cpu = -1;
 162         INIT_HLIST_NODE(&rq->hash);
 163         RB_CLEAR_NODE(&rq->rb_node);
 164         rq->rq_disk = NULL;
 165         rq->part = NULL;
 166         rq->start_time = jiffies;
 167 #ifdef CONFIG_BLK_CGROUP
 168         rq->rl = NULL;
 169         set_start_time_ns(rq);
 170         rq->io_start_time_ns = 0;
 171 #endif
 172         rq->nr_phys_segments = 0;
 173 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 174         rq->nr_integrity_segments = 0;
 175 #endif
 176         rq->special = NULL;
 177         /* tag was already set */
 178         rq->errors = 0;
 179
 180         rq->cmd = rq->__cmd;
 181
 182         rq->extra_len = 0;
 183         rq->sense_len = 0;
 184         rq->resid_len = 0;
 185         rq->sense = NULL;
 186
 187         INIT_LIST_HEAD(&rq->timeout_list);
 188         rq->timeout = 0;
 189
 190         rq->end_io = NULL;
 191         rq->end_io_data = NULL;
 192         rq->next_rq = NULL;
 193
 194         ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
 195 }
 196
 197 static struct request *
 198 __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
 199 {
 200         struct request *rq;
 201         unsigned int tag;
 202
 203         tag = blk_mq_get_tag(data);
 204         if (tag != BLK_MQ_TAG_FAIL) {
 205                 rq = data->hctx->tags->rqs[tag];
 206
 207                 if (blk_mq_tag_busy(data->hctx)) {
 208                         rq->cmd_flags = REQ_MQ_INFLIGHT;
 209                         atomic_inc(&data->hctx->nr_active);
 210                 }
 211
 212                 rq->tag = tag;
 213                 blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw);
 214                 return rq;
 215         }
 216
 217         return NULL;
 218 }
 219
 220 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
 221                 bool reserved)
 222 {
 223         struct blk_mq_ctx *ctx;
 224         struct blk_mq_hw_ctx *hctx;
 225         struct request *rq;
 226         struct blk_mq_alloc_data alloc_data;
 227         int ret;
 228
 229         ret = blk_mq_queue_enter(q);
 230         if (ret)
 231                 return ERR_PTR(ret);
 232
 233         ctx = blk_mq_get_ctx(q);
 234         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 235         blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_WAIT,
 236                         reserved, ctx, hctx);
 237
 238         rq = __blk_mq_alloc_request(&alloc_data, rw);
 239         if (!rq && (gfp & __GFP_WAIT)) {
 240                 __blk_mq_run_hw_queue(hctx);
 241                 blk_mq_put_ctx(ctx);
 242
 243                 ctx = blk_mq_get_ctx(q);
 244                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
 245                 blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
 246                                 hctx);
 247                 rq =  __blk_mq_alloc_request(&alloc_data, rw);
 248                 ctx = alloc_data.ctx;
 249         }
 250         blk_mq_put_ctx(ctx);
 251         if (!rq)
 252                 return ERR_PTR(-EWOULDBLOCK);
 253         return rq;
 254 }
 255 EXPORT_SYMBOL(blk_mq_alloc_request);
 256
 257 static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 258                                   struct blk_mq_ctx *ctx, struct request *rq)
 259 {
 260         const int tag = rq->tag;
 261         struct request_queue *q = rq->q;
 262
 263         if (rq->cmd_flags & REQ_MQ_INFLIGHT)
 264                 atomic_dec(&hctx->nr_active);
 265         rq->cmd_flags = 0;
 266
 267         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 268         blk_mq_put_tag(hctx, tag, &ctx->last_tag);
 269         blk_mq_queue_exit(q);
 270 }
 271
 272 void blk_mq_free_request(struct request *rq)
 273 {
 274         struct blk_mq_ctx *ctx = rq->mq_ctx;
 275         struct blk_mq_hw_ctx *hctx;
 276         struct request_queue *q = rq->q;
 277
 278         ctx->rq_completed[rq_is_sync(rq)]++;
 279
 280         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 281         __blk_mq_free_request(hctx, ctx, rq);
 282 }
 283 EXPORT_SYMBOL_GPL(blk_mq_free_request);
 284
 285 inline void __blk_mq_end_request(struct request *rq, int error)
 286 {
 287         blk_account_io_done(rq);
 288
 289         if (rq->end_io) {
 290                 rq->end_io(rq, error);
 291         } else {
 292                 if (unlikely(blk_bidi_rq(rq)))
 293                         blk_mq_free_request(rq->next_rq);
 294                 blk_mq_free_request(rq);
 295         }
 296 }
 297 EXPORT_SYMBOL(__blk_mq_end_request);
 298
 299 void blk_mq_end_request(struct request *rq, int error)
 300 {
 301         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 302                 BUG();
 303         __blk_mq_end_request(rq, error);
 304 }
 305 EXPORT_SYMBOL(blk_mq_end_request);
 306
 307 static void __blk_mq_complete_request_remote(void *data)
 308 {
 309         struct request *rq = data;
 310
 311         rq->q->softirq_done_fn(rq);
 312 }
 313
 314 static void blk_mq_ipi_complete_request(struct request *rq)
 315 {
 316         struct blk_mq_ctx *ctx = rq->mq_ctx;
 317         bool shared = false;
 318         int cpu;
 319
 320         if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 321                 rq->q->softirq_done_fn(rq);
 322                 return;
 323         }
 324
 325         cpu = get_cpu();
 326         if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
 327                 shared = cpus_share_cache(cpu, ctx->cpu);
 328
 329         if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 330                 rq->csd.func = __blk_mq_complete_request_remote;
 331                 rq->csd.info = rq;
 332                 rq->csd.flags = 0;
 333                 smp_call_function_single_async(ctx->cpu, &rq->csd);
 334         } else {
 335                 rq->q->softirq_done_fn(rq);
 336         }
 337         put_cpu();
 338 }
 339
 340 void __blk_mq_complete_request(struct request *rq)
 341 {
 342         struct request_queue *q = rq->q;
 343
 344         if (!q->softirq_done_fn)
 345                 blk_mq_end_request(rq, rq->errors);
 346         else
 347                 blk_mq_ipi_complete_request(rq);
 348 }
 349
 350 /**
 351  * blk_mq_complete_request - end I/O on a request
 352  * @rq:         the request being processed
 353  *
 354  * Description:
 355  *      Ends all I/O on a request. It does not handle partial completions.
 356  *      The actual completion happens out-of-order, through a IPI handler.
 357  **/
 358 void blk_mq_complete_request(struct request *rq)
 359 {
 360         struct request_queue *q = rq->q;
 361
 362         if (unlikely(blk_should_fake_timeout(q)))
 363                 return;
 364         if (!blk_mark_rq_complete(rq))
 365                 __blk_mq_complete_request(rq);
 366 }
 367 EXPORT_SYMBOL(blk_mq_complete_request);
 368
 369 void blk_mq_start_request(struct request *rq)
 370 {
 371         struct request_queue *q = rq->q;
 372
 373         trace_block_rq_issue(q, rq);
 374
 375         rq->resid_len = blk_rq_bytes(rq);
 376         if (unlikely(blk_bidi_rq(rq)))
 377                 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 378
 379         blk_add_timer(rq);
 380
 381         /*
 382          * Ensure that ->deadline is visible before set the started
 383          * flag and clear the completed flag.
 384          */
 385         smp_mb__before_atomic();
 386
 387         /*
 388          * Mark us as started and clear complete. Complete might have been
 389          * set if requeue raced with timeout, which then marked it as
 390          * complete. So be sure to clear complete again when we start
 391          * the request, otherwise we'll ignore the completion event.
 392          */
 393         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 394                 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 395         if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
 396                 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
 397
 398         if (q->dma_drain_size && blk_rq_bytes(rq)) {
 399                 /*
 400                  * Make sure space for the drain appears.  We know we can do
 401                  * this because max_hw_segments has been adjusted to be one
 402                  * fewer than the device can handle.
 403                  */
 404                 rq->nr_phys_segments++;
 405         }
 406 }
 407 EXPORT_SYMBOL(blk_mq_start_request);
 408
 409 static void __blk_mq_requeue_request(struct request *rq)
 410 {
 411         struct request_queue *q = rq->q;
 412
 413         trace_block_rq_requeue(q, rq);
 414
 415         if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
 416                 if (q->dma_drain_size && blk_rq_bytes(rq))
 417                         rq->nr_phys_segments--;
 418         }
 419 }
 420
 421 void blk_mq_requeue_request(struct request *rq)
 422 {
 423         __blk_mq_requeue_request(rq);
 424
 425         BUG_ON(blk_queued_rq(rq));
 426         blk_mq_add_to_requeue_list(rq, true);
 427 }
 428 EXPORT_SYMBOL(blk_mq_requeue_request);
 429
 430 static void blk_mq_requeue_work(struct work_struct *work)
 431 {
 432         struct request_queue *q =
 433                 container_of(work, struct request_queue, requeue_work);
 434         LIST_HEAD(rq_list);
 435         struct request *rq, *next;
 436         unsigned long flags;
 437
 438         spin_lock_irqsave(&q->requeue_lock, flags);
 439         list_splice_init(&q->requeue_list, &rq_list);
 440         spin_unlock_irqrestore(&q->requeue_lock, flags);
 441
 442         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
 443                 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
 444                         continue;
 445
 446                 rq->cmd_flags &= ~REQ_SOFTBARRIER;
 447                 list_del_init(&rq->queuelist);
 448                 blk_mq_insert_request(rq, true, false, false);
 449         }
 450
 451         while (!list_empty(&rq_list)) {
 452                 rq = list_entry(rq_list.next, struct request, queuelist);
 453                 list_del_init(&rq->queuelist);
 454                 blk_mq_insert_request(rq, false, false, false);
 455         }
 456
 457         /*
 458          * Use the start variant of queue running here, so that running
 459          * the requeue work will kick stopped queues.
 460          */
 461         blk_mq_start_hw_queues(q);
 462 }
 463
 464 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 465 {
 466         struct request_queue *q = rq->q;
 467         unsigned long flags;
 468
 469         /*
 470          * We abuse this flag that is otherwise used by the I/O scheduler to
 471          * request head insertation from the workqueue.
 472          */
 473         BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
 474
 475         spin_lock_irqsave(&q->requeue_lock, flags);
 476         if (at_head) {
 477                 rq->cmd_flags |= REQ_SOFTBARRIER;
 478                 list_add(&rq->queuelist, &q->requeue_list);
 479         } else {
 480                 list_add_tail(&rq->queuelist, &q->requeue_list);
 481         }
 482         spin_unlock_irqrestore(&q->requeue_lock, flags);
 483 }
 484 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 485
 486 void blk_mq_kick_requeue_list(struct request_queue *q)
 487 {
 488         kblockd_schedule_work(&q->requeue_work);
 489 }
 490 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 491
 492 static inline bool is_flush_request(struct request *rq,
 493                 struct blk_flush_queue *fq, unsigned int tag)
 494 {
 495         return ((rq->cmd_flags & REQ_FLUSH_SEQ) &&
 496                         fq->flush_rq->tag == tag);
 497 }
 498
 499 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
 500 {
 501         struct request *rq = tags->rqs[tag];
 502         /* mq_ctx of flush rq is always cloned from the corresponding req */
 503         struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx);
 504
 505         if (!is_flush_request(rq, fq, tag))
 506                 return rq;
 507
 508         return fq->flush_rq;
 509 }
 510 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 511
 512 struct blk_mq_timeout_data {
 513         unsigned long next;
 514         unsigned int next_set;
 515 };
 516
 517 void blk_mq_rq_timed_out(struct request *req, bool reserved)
 518 {
 519         struct blk_mq_ops *ops = req->q->mq_ops;
 520         enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
 521
 522         /*
 523          * We know that complete is set at this point. If STARTED isn't set
 524          * anymore, then the request isn't active and the "timeout" should
 525          * just be ignored. This can happen due to the bitflag ordering.
 526          * Timeout first checks if STARTED is set, and if it is, assumes
 527          * the request is active. But if we race with completion, then
 528          * we both flags will get cleared. So check here again, and ignore
 529          * a timeout event with a request that isn't active.
 530          */
 531         if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
 532                 return;
 533
 534         if (ops->timeout)
 535                 ret = ops->timeout(req, reserved);
 536
 537         switch (ret) {
 538         case BLK_EH_HANDLED:
 539                 __blk_mq_complete_request(req);
 540                 break;
 541         case BLK_EH_RESET_TIMER:
 542                 blk_add_timer(req);
 543                 blk_clear_rq_complete(req);
 544                 break;
 545         case BLK_EH_NOT_HANDLED:
 546                 break;
 547         default:
 548                 printk(KERN_ERR "block: bad eh return: %d\n", ret);
 549                 break;
 550         }
 551 }
 552
 553 static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
 554                 struct request *rq, void *priv, bool reserved)
 555 {
 556         struct blk_mq_timeout_data *data = priv;
 557
 558         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 559                 return;
 560
 561         if (time_after_eq(jiffies, rq->deadline)) {
 562                 if (!blk_mark_rq_complete(rq))
 563                         blk_mq_rq_timed_out(rq, reserved);
 564         } else if (!data->next_set || time_after(data->next, rq->deadline)) {
 565                 data->next = rq->deadline;
 566                 data->next_set = 1;
 567         }
 568 }
 569
 570 static void blk_mq_rq_timer(unsigned long priv)
 571 {
 572         struct request_queue *q = (struct request_queue *)priv;
 573         struct blk_mq_timeout_data data = {
 574                 .next           = 0,
 575                 .next_set       = 0,
 576         };
 577         struct blk_mq_hw_ctx *hctx;
 578         int i;
 579
 580         queue_for_each_hw_ctx(q, hctx, i) {
 581                 /*
 582                  * If not software queues are currently mapped to this
 583                  * hardware queue, there's nothing to check
 584                  */
 585                 if (!hctx->nr_ctx || !hctx->tags)
 586                         continue;
 587
 588                 blk_mq_tag_busy_iter(hctx, blk_mq_check_expired, &data);
 589         }
 590
 591         if (data.next_set) {
 592                 data.next = blk_rq_timeout(round_jiffies_up(data.next));
 593                 mod_timer(&q->timeout, data.next);
 594         } else {
 595                 queue_for_each_hw_ctx(q, hctx, i)
 596                         blk_mq_tag_idle(hctx);
 597         }
 598 }
 599
 600 /*
 601  * Reverse check our software queue for entries that we could potentially
 602  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 603  * too much time checking for merges.
 604  */
 605 static bool blk_mq_attempt_merge(struct request_queue *q,
 606                                  struct blk_mq_ctx *ctx, struct bio *bio)
 607 {
 608         struct request *rq;
 609         int checked = 8;
 610
 611         list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
 612                 int el_ret;
 613
 614                 if (!checked--)
 615                         break;
 616
 617                 if (!blk_rq_merge_ok(rq, bio))
 618                         continue;
 619
 620                 el_ret = blk_try_merge(rq, bio);
 621                 if (el_ret == ELEVATOR_BACK_MERGE) {
 622                         if (bio_attempt_back_merge(q, rq, bio)) {
 623                                 ctx->rq_merged++;
 624                                 return true;
 625                         }
 626                         break;
 627                 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
 628                         if (bio_attempt_front_merge(q, rq, bio)) {
 629                                 ctx->rq_merged++;
 630                                 return true;
 631                         }
 632                         break;
 633                 }
 634         }
 635
 636         return false;
 637 }
 638
 639 /*
 640  * Process software queues that have been marked busy, splicing them
 641  * to the for-dispatch
 642  */
 643 static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 644 {
 645         struct blk_mq_ctx *ctx;
 646         int i;
 647
 648         for (i = 0; i < hctx->ctx_map.map_size; i++) {
 649                 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
 650                 unsigned int off, bit;
 651
 652                 if (!bm->word)
 653                         continue;
 654
 655                 bit = 0;
 656                 off = i * hctx->ctx_map.bits_per_word;
 657                 do {
 658                         bit = find_next_bit(&bm->word, bm->depth, bit);
 659                         if (bit >= bm->depth)
 660                                 break;
 661
 662                         ctx = hctx->ctxs[bit + off];
 663                         clear_bit(bit, &bm->word);
 664                         spin_lock(&ctx->lock);
 665                         list_splice_tail_init(&ctx->rq_list, list);
 666                         spin_unlock(&ctx->lock);
 667
 668                         bit++;
 669                 } while (1);
 670         }
 671 }
 672
 673 /*
 674  * Run this hardware queue, pulling any software queues mapped to it in.
 675  * Note that this function currently has various problems around ordering
 676  * of IO. In particular, we'd like FIFO behaviour on handling existing
 677  * items on the hctx->dispatch list. Ignore that for now.
 678  */
 679 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 680 {
 681         struct request_queue *q = hctx->queue;
 682         struct request *rq;
 683         LIST_HEAD(rq_list);
 684         LIST_HEAD(driver_list);
 685         struct list_head *dptr;
 686         int queued;
 687
 688         WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 689
 690         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 691                 return;
 692
 693         hctx->run++;
 694
 695         /*
 696          * Touch any software queue that has pending entries.
 697          */
 698         flush_busy_ctxs(hctx, &rq_list);
 699
 700         /*
 701          * If we have previous entries on our dispatch list, grab them
 702          * and stuff them at the front for more fair dispatch.
 703          */
 704         if (!list_empty_careful(&hctx->dispatch)) {
 705                 spin_lock(&hctx->lock);
 706                 if (!list_empty(&hctx->dispatch))
 707                         list_splice_init(&hctx->dispatch, &rq_list);
 708                 spin_unlock(&hctx->lock);
 709         }
 710
 711         /*
 712          * Start off with dptr being NULL, so we start the first request
 713          * immediately, even if we have more pending.
 714          */
 715         dptr = NULL;
 716
 717         /*
 718          * Now process all the entries, sending them to the driver.
 719          */
 720         queued = 0;
 721         while (!list_empty(&rq_list)) {
 722                 struct blk_mq_queue_data bd;
 723                 int ret;
 724
 725                 rq = list_first_entry(&rq_list, struct request, queuelist);
 726                 list_del_init(&rq->queuelist);
 727
 728                 bd.rq = rq;
 729                 bd.list = dptr;
 730                 bd.last = list_empty(&rq_list);
 731
 732                 ret = q->mq_ops->queue_rq(hctx, &bd);
 733                 switch (ret) {
 734                 case BLK_MQ_RQ_QUEUE_OK:
 735                         queued++;
 736                         continue;
 737                 case BLK_MQ_RQ_QUEUE_BUSY:
 738                         list_add(&rq->queuelist, &rq_list);
 739                         __blk_mq_requeue_request(rq);
 740                         break;
 741                 default:
 742                         pr_err("blk-mq: bad return on queue: %d\n", ret);
 743                 case BLK_MQ_RQ_QUEUE_ERROR:
 744                         rq->errors = -EIO;
 745                         blk_mq_end_request(rq, rq->errors);
 746                         break;
 747                 }
 748
 749                 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
 750                         break;
 751
 752                 /*
 753                  * We've done the first request. If we have more than 1
 754                  * left in the list, set dptr to defer issue.
 755                  */
 756                 if (!dptr && rq_list.next != rq_list.prev)
 757                         dptr = &driver_list;
 758         }
 759
 760         if (!queued)
 761                 hctx->dispatched[0]++;
 762         else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
 763                 hctx->dispatched[ilog2(queued) + 1]++;
 764
 765         /*
 766          * Any items that need requeuing? Stuff them into hctx->dispatch,
 767          * that is where we will continue on next queue run.
 768          */
 769         if (!list_empty(&rq_list)) {
 770                 spin_lock(&hctx->lock);
 771                 list_splice(&rq_list, &hctx->dispatch);
 772                 spin_unlock(&hctx->lock);
 773         }
 774 }
 775
 776 /*
 777  * It'd be great if the workqueue API had a way to pass
 778  * in a mask and had some smarts for more clever placement.
 779  * For now we just round-robin here, switching for every
 780  * BLK_MQ_CPU_WORK_BATCH queued items.
 781  */
 782 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 783 {
 784         int cpu = hctx->next_cpu;
 785
 786         if (--hctx->next_cpu_batch <= 0) {
 787                 int next_cpu;
 788
 789                 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 790                 if (next_cpu >= nr_cpu_ids)
 791                         next_cpu = cpumask_first(hctx->cpumask);
 792
 793                 hctx->next_cpu = next_cpu;
 794                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 795         }
 796
 797         return cpu;
 798 }
 799
 800 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 801 {
 802         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 803                 return;
 804
 805         if (!async) {
 806                 int cpu = get_cpu();
 807                 if (cpumask_test_cpu(cpu, hctx->cpumask)) {
 808                         __blk_mq_run_hw_queue(hctx);
 809                         put_cpu();
 810                         return;
 811                 }
 812
 813                 put_cpu();
 814         }
 815
 816         if (hctx->queue->nr_hw_queues == 1)
 817                 kblockd_schedule_delayed_work(&hctx->run_work, 0);
 818         else {
 819                 unsigned int cpu;
 820
 821                 cpu = blk_mq_hctx_next_cpu(hctx);
 822                 kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
 823         }
 824 }
 825
 826 void blk_mq_run_queues(struct request_queue *q, bool async)
 827 {
 828         struct blk_mq_hw_ctx *hctx;
 829         int i;
 830
 831         queue_for_each_hw_ctx(q, hctx, i) {
 832                 if ((!blk_mq_hctx_has_pending(hctx) &&
 833                     list_empty_careful(&hctx->dispatch)) ||
 834                     test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 835                         continue;
 836
 837                 blk_mq_run_hw_queue(hctx, async);
 838         }
 839 }
 840 EXPORT_SYMBOL(blk_mq_run_queues);
 841
 842 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 843 {
 844         cancel_delayed_work(&hctx->run_work);
 845         cancel_delayed_work(&hctx->delay_work);
 846         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 847 }
 848 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 849
 850 void blk_mq_stop_hw_queues(struct request_queue *q)
 851 {
 852         struct blk_mq_hw_ctx *hctx;
 853         int i;
 854
 855         queue_for_each_hw_ctx(q, hctx, i)
 856                 blk_mq_stop_hw_queue(hctx);
 857 }
 858 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 859
 860 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 861 {
 862         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 863
 864         blk_mq_run_hw_queue(hctx, false);
 865 }
 866 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 867
 868 void blk_mq_start_hw_queues(struct request_queue *q)
 869 {
 870         struct blk_mq_hw_ctx *hctx;
 871         int i;
 872
 873         queue_for_each_hw_ctx(q, hctx, i)
 874                 blk_mq_start_hw_queue(hctx);
 875 }
 876 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 877
 878
 879 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 880 {
 881         struct blk_mq_hw_ctx *hctx;
 882         int i;
 883
 884         queue_for_each_hw_ctx(q, hctx, i) {
 885                 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 886                         continue;
 887
 888                 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 889                 blk_mq_run_hw_queue(hctx, async);
 890         }
 891 }
 892 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 893
 894 static void blk_mq_run_work_fn(struct work_struct *work)
 895 {
 896         struct blk_mq_hw_ctx *hctx;
 897
 898         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 899
 900         __blk_mq_run_hw_queue(hctx);
 901 }
 902
 903 static void blk_mq_delay_work_fn(struct work_struct *work)
 904 {
 905         struct blk_mq_hw_ctx *hctx;
 906
 907         hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
 908
 909         if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
 910                 __blk_mq_run_hw_queue(hctx);
 911 }
 912
 913 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 914 {
 915         unsigned long tmo = msecs_to_jiffies(msecs);
 916
 917         if (hctx->queue->nr_hw_queues == 1)
 918                 kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
 919         else {
 920                 unsigned int cpu;
 921
 922                 cpu = blk_mq_hctx_next_cpu(hctx);
 923                 kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
 924         }
 925 }
 926 EXPORT_SYMBOL(blk_mq_delay_queue);
 927
 928 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
 929                                     struct request *rq, bool at_head)
 930 {
 931         struct blk_mq_ctx *ctx = rq->mq_ctx;
 932
 933         trace_block_rq_insert(hctx->queue, rq);
 934
 935         if (at_head)
 936                 list_add(&rq->queuelist, &ctx->rq_list);
 937         else
 938                 list_add_tail(&rq->queuelist, &ctx->rq_list);
 939
 940         blk_mq_hctx_mark_pending(hctx, ctx);
 941 }
 942
 943 void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
 944                 bool async)
 945 {
 946         struct request_queue *q = rq->q;
 947         struct blk_mq_hw_ctx *hctx;
 948         struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
 949
 950         current_ctx = blk_mq_get_ctx(q);
 951         if (!cpu_online(ctx->cpu))
 952                 rq->mq_ctx = ctx = current_ctx;
 953
 954         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 955
 956         spin_lock(&ctx->lock);
 957         __blk_mq_insert_request(hctx, rq, at_head);
 958         spin_unlock(&ctx->lock);
 959
 960         if (run_queue)
 961                 blk_mq_run_hw_queue(hctx, async);
 962
 963         blk_mq_put_ctx(current_ctx);
 964 }
 965
 966 static void blk_mq_insert_requests(struct request_queue *q,
 967                                      struct blk_mq_ctx *ctx,
 968                                      struct list_head *list,
 969                                      int depth,
 970                                      bool from_schedule)
 971
 972 {
 973         struct blk_mq_hw_ctx *hctx;
 974         struct blk_mq_ctx *current_ctx;
 975
 976         trace_block_unplug(q, depth, !from_schedule);
 977
 978         current_ctx = blk_mq_get_ctx(q);
 979
 980         if (!cpu_online(ctx->cpu))
 981                 ctx = current_ctx;
 982         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 983
 984         /*
 985          * preemption doesn't flush plug list, so it's possible ctx->cpu is
 986          * offline now
 987          */
 988         spin_lock(&ctx->lock);
 989         while (!list_empty(list)) {
 990                 struct request *rq;
 991
 992                 rq = list_first_entry(list, struct request, queuelist);
 993                 list_del_init(&rq->queuelist);
 994                 rq->mq_ctx = ctx;
 995                 __blk_mq_insert_request(hctx, rq, false);
 996         }
 997         spin_unlock(&ctx->lock);
 998
 999         blk_mq_run_hw_queue(hctx, from_schedule);
1000         blk_mq_put_ctx(current_ctx);
1001 }
1002
1003 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1004 {
1005         struct request *rqa = container_of(a, struct request, queuelist);
1006         struct request *rqb = container_of(b, struct request, queuelist);
1007
1008         return !(rqa->mq_ctx < rqb->mq_ctx ||
1009                  (rqa->mq_ctx == rqb->mq_ctx &&
1010                   blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1011 }
1012
1013 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1014 {
1015         struct blk_mq_ctx *this_ctx;
1016         struct request_queue *this_q;
1017         struct request *rq;
1018         LIST_HEAD(list);
1019         LIST_HEAD(ctx_list);
1020         unsigned int depth;
1021
1022         list_splice_init(&plug->mq_list, &list);
1023
1024         list_sort(NULL, &list, plug_ctx_cmp);
1025
1026         this_q = NULL;
1027         this_ctx = NULL;
1028         depth = 0;
1029
1030         while (!list_empty(&list)) {
1031                 rq = list_entry_rq(list.next);
1032                 list_del_init(&rq->queuelist);
1033                 BUG_ON(!rq->q);
1034                 if (rq->mq_ctx != this_ctx) {
1035                         if (this_ctx) {
1036                                 blk_mq_insert_requests(this_q, this_ctx,
1037                                                         &ctx_list, depth,
1038                                                         from_schedule);
1039                         }
1040
1041                         this_ctx = rq->mq_ctx;
1042                         this_q = rq->q;
1043                         depth = 0;
1044                 }
1045
1046                 depth++;
1047                 list_add_tail(&rq->queuelist, &ctx_list);
1048         }
1049
1050         /*
1051          * If 'this_ctx' is set, we know we have entries to complete
1052          * on 'ctx_list'. Do those.
1053          */
1054         if (this_ctx) {
1055                 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1056                                        from_schedule);
1057         }
1058 }
1059
1060 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1061 {
1062         init_request_from_bio(rq, bio);
1063
1064         if (blk_do_io_stat(rq))
1065                 blk_account_io_start(rq, 1);
1066 }
1067
1068 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
1069 {
1070         return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
1071                 !blk_queue_nomerges(hctx->queue);
1072 }
1073
1074 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1075                                          struct blk_mq_ctx *ctx,
1076                                          struct request *rq, struct bio *bio)
1077 {
1078         if (!hctx_allow_merges(hctx)) {
1079                 blk_mq_bio_to_request(rq, bio);
1080                 spin_lock(&ctx->lock);
1081 insert_rq:
1082                 __blk_mq_insert_request(hctx, rq, false);
1083                 spin_unlock(&ctx->lock);
1084                 return false;
1085         } else {
1086                 struct request_queue *q = hctx->queue;
1087
1088                 spin_lock(&ctx->lock);
1089                 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1090                         blk_mq_bio_to_request(rq, bio);
1091                         goto insert_rq;
1092                 }
1093
1094                 spin_unlock(&ctx->lock);
1095                 __blk_mq_free_request(hctx, ctx, rq);
1096                 return true;
1097         }
1098 }
1099
1100 struct blk_map_ctx {
1101         struct blk_mq_hw_ctx *hctx;
1102         struct blk_mq_ctx *ctx;
1103 };
1104
1105 static struct request *blk_mq_map_request(struct request_queue *q,
1106                                           struct bio *bio,
1107                                           struct blk_map_ctx *data)
1108 {
1109         struct blk_mq_hw_ctx *hctx;
1110         struct blk_mq_ctx *ctx;
1111         struct request *rq;
1112         int rw = bio_data_dir(bio);
1113         struct blk_mq_alloc_data alloc_data;
1114
1115         if (unlikely(blk_mq_queue_enter(q))) {
1116                 bio_endio(bio, -EIO);
1117                 return NULL;
1118         }
1119
1120         ctx = blk_mq_get_ctx(q);
1121         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1122
1123         if (rw_is_sync(bio->bi_rw))
1124                 rw |= REQ_SYNC;
1125
1126         trace_block_getrq(q, bio, rw);
1127         blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
1128                         hctx);
1129         rq = __blk_mq_alloc_request(&alloc_data, rw);
1130         if (unlikely(!rq)) {
1131                 __blk_mq_run_hw_queue(hctx);
1132                 blk_mq_put_ctx(ctx);
1133                 trace_block_sleeprq(q, bio, rw);
1134
1135                 ctx = blk_mq_get_ctx(q);
1136                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1137                 blk_mq_set_alloc_data(&alloc_data, q,
1138                                 __GFP_WAIT|GFP_ATOMIC, false, ctx, hctx);
1139                 rq = __blk_mq_alloc_request(&alloc_data, rw);
1140                 ctx = alloc_data.ctx;
1141                 hctx = alloc_data.hctx;
1142         }
1143
1144         hctx->queued++;
1145         data->hctx = hctx;
1146         data->ctx = ctx;
1147         return rq;
1148 }
1149
1150 /*
1151  * Multiple hardware queue variant. This will not use per-process plugs,
1152  * but will attempt to bypass the hctx queueing if we can go straight to
1153  * hardware for SYNC IO.
1154  */
1155 static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1156 {
1157         const int is_sync = rw_is_sync(bio->bi_rw);
1158         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1159         struct blk_map_ctx data;
1160         struct request *rq;
1161
1162         blk_queue_bounce(q, &bio);
1163
1164         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1165                 bio_endio(bio, -EIO);
1166                 return;
1167         }
1168
1169         rq = blk_mq_map_request(q, bio, &data);
1170         if (unlikely(!rq))
1171                 return;
1172
1173         if (unlikely(is_flush_fua)) {
1174                 blk_mq_bio_to_request(rq, bio);
1175                 blk_insert_flush(rq);
1176                 goto run_queue;
1177         }
1178
1179         /*
1180          * If the driver supports defer issued based on 'last', then
1181          * queue it up like normal since we can potentially save some
1182          * CPU this way.
1183          */
1184         if (is_sync && !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
1185                 struct blk_mq_queue_data bd = {
1186                         .rq = rq,
1187                         .list = NULL,
1188                         .last = 1
1189                 };
1190                 int ret;
1191
1192                 blk_mq_bio_to_request(rq, bio);
1193
1194                 /*
1195                  * For OK queue, we are done. For error, kill it. Any other
1196                  * error (busy), just add it to our list as we previously
1197                  * would have done
1198                  */
1199                 ret = q->mq_ops->queue_rq(data.hctx, &bd);
1200                 if (ret == BLK_MQ_RQ_QUEUE_OK)
1201                         goto done;
1202                 else {
1203                         __blk_mq_requeue_request(rq);
1204
1205                         if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1206                                 rq->errors = -EIO;
1207                                 blk_mq_end_request(rq, rq->errors);
1208                                 goto done;
1209                         }
1210                 }
1211         }
1212
1213         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1214                 /*
1215                  * For a SYNC request, send it to the hardware immediately. For
1216                  * an ASYNC request, just ensure that we run it later on. The
1217                  * latter allows for merging opportunities and more efficient
1218                  * dispatching.
1219                  */
1220 run_queue:
1221                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1222         }
1223 done:
1224         blk_mq_put_ctx(data.ctx);
1225 }
1226
1227 /*
1228  * Single hardware queue variant. This will attempt to use any per-process
1229  * plug for merging and IO deferral.
1230  */
1231 static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1232 {
1233         const int is_sync = rw_is_sync(bio->bi_rw);
1234         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1235         unsigned int use_plug, request_count = 0;
1236         struct blk_map_ctx data;
1237         struct request *rq;
1238
1239         /*
1240          * If we have multiple hardware queues, just go directly to
1241          * one of those for sync IO.
1242          */
1243         use_plug = !is_flush_fua && !is_sync;
1244
1245         blk_queue_bounce(q, &bio);
1246
1247         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1248                 bio_endio(bio, -EIO);
1249                 return;
1250         }
1251
1252         if (use_plug && !blk_queue_nomerges(q) &&
1253             blk_attempt_plug_merge(q, bio, &request_count))
1254                 return;
1255
1256         rq = blk_mq_map_request(q, bio, &data);
1257         if (unlikely(!rq))
1258                 return;
1259
1260         if (unlikely(is_flush_fua)) {
1261                 blk_mq_bio_to_request(rq, bio);
1262                 blk_insert_flush(rq);
1263                 goto run_queue;
1264         }
1265
1266         /*
1267          * A task plug currently exists. Since this is completely lockless,
1268          * utilize that to temporarily store requests until the task is
1269          * either done or scheduled away.
1270          */
1271         if (use_plug) {
1272                 struct blk_plug *plug = current->plug;
1273
1274                 if (plug) {
1275                         blk_mq_bio_to_request(rq, bio);
1276                         if (list_empty(&plug->mq_list))
1277                                 trace_block_plug(q);
1278                         else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1279                                 blk_flush_plug_list(plug, false);
1280                                 trace_block_plug(q);
1281                         }
1282                         list_add_tail(&rq->queuelist, &plug->mq_list);
1283                         blk_mq_put_ctx(data.ctx);
1284                         return;
1285                 }
1286         }
1287
1288         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1289                 /*
1290                  * For a SYNC request, send it to the hardware immediately. For
1291                  * an ASYNC request, just ensure that we run it later on. The
1292                  * latter allows for merging opportunities and more efficient
1293                  * dispatching.
1294                  */
1295 run_queue:
1296                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1297         }
1298
1299         blk_mq_put_ctx(data.ctx);
1300 }
1301
1302 /*
1303  * Default mapping to a software queue, since we use one per CPU.
1304  */
1305 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1306 {
1307         return q->queue_hw_ctx[q->mq_map[cpu]];
1308 }
1309 EXPORT_SYMBOL(blk_mq_map_queue);
1310
1311 static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1312                 struct blk_mq_tags *tags, unsigned int hctx_idx)
1313 {
1314         struct page *page;
1315
1316         if (tags->rqs && set->ops->exit_request) {
1317                 int i;
1318
1319                 for (i = 0; i < tags->nr_tags; i++) {
1320                         if (!tags->rqs[i])
1321                                 continue;
1322                         set->ops->exit_request(set->driver_data, tags->rqs[i],
1323                                                 hctx_idx, i);
1324                         tags->rqs[i] = NULL;
1325                 }
1326         }
1327
1328         while (!list_empty(&tags->page_list)) {
1329                 page = list_first_entry(&tags->page_list, struct page, lru);
1330                 list_del_init(&page->lru);
1331                 __free_pages(page, page->private);
1332         }
1333
1334         kfree(tags->rqs);
1335
1336         blk_mq_free_tags(tags);
1337 }
1338
1339 static size_t order_to_size(unsigned int order)
1340 {
1341         return (size_t)PAGE_SIZE << order;
1342 }
1343
1344 static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1345                 unsigned int hctx_idx)
1346 {
1347         struct blk_mq_tags *tags;
1348         unsigned int i, j, entries_per_page, max_order = 4;
1349         size_t rq_size, left;
1350
1351         tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1352                                 set->numa_node);
1353         if (!tags)
1354                 return NULL;
1355
1356         INIT_LIST_HEAD(&tags->page_list);
1357
1358         tags->rqs = kzalloc_node(set->queue_depth * sizeof(struct request *),
1359                                  GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
1360                                  set->numa_node);
1361         if (!tags->rqs) {
1362                 blk_mq_free_tags(tags);
1363                 return NULL;
1364         }
1365
1366         /*
1367          * rq_size is the size of the request plus driver payload, rounded
1368          * to the cacheline size
1369          */
1370         rq_size = round_up(sizeof(struct request) + set->cmd_size,
1371                                 cache_line_size());
1372         left = rq_size * set->queue_depth;
1373
1374         for (i = 0; i < set->queue_depth; ) {
1375                 int this_order = max_order;
1376                 struct page *page;
1377                 int to_do;
1378                 void *p;
1379
1380                 while (left < order_to_size(this_order - 1) && this_order)
1381                         this_order--;
1382
1383                 do {
1384                         page = alloc_pages_node(set->numa_node,
1385                                 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
1386                                 this_order);
1387                         if (page)
1388                                 break;
1389                         if (!this_order--)
1390                                 break;
1391                         if (order_to_size(this_order) < rq_size)
1392                                 break;
1393                 } while (1);
1394
1395                 if (!page)
1396                         goto fail;
1397
1398                 page->private = this_order;
1399                 list_add_tail(&page->lru, &tags->page_list);
1400
1401                 p = page_address(page);
1402                 entries_per_page = order_to_size(this_order) / rq_size;
1403                 to_do = min(entries_per_page, set->queue_depth - i);
1404                 left -= to_do * rq_size;
1405                 for (j = 0; j < to_do; j++) {
1406                         tags->rqs[i] = p;
1407                         tags->rqs[i]->atomic_flags = 0;
1408                         tags->rqs[i]->cmd_flags = 0;
1409                         if (set->ops->init_request) {
1410                                 if (set->ops->init_request(set->driver_data,
1411                                                 tags->rqs[i], hctx_idx, i,
1412                                                 set->numa_node)) {
1413                                         tags->rqs[i] = NULL;
1414                                         goto fail;
1415                                 }
1416                         }
1417
1418                         p += rq_size;
1419                         i++;
1420                 }
1421         }
1422
1423         return tags;
1424
1425 fail:
1426         blk_mq_free_rq_map(set, tags, hctx_idx);
1427         return NULL;
1428 }
1429
1430 static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1431 {
1432         kfree(bitmap->map);
1433 }
1434
1435 static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1436 {
1437         unsigned int bpw = 8, total, num_maps, i;
1438
1439         bitmap->bits_per_word = bpw;
1440
1441         num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1442         bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1443                                         GFP_KERNEL, node);
1444         if (!bitmap->map)
1445                 return -ENOMEM;
1446
1447         bitmap->map_size = num_maps;
1448
1449         total = nr_cpu_ids;
1450         for (i = 0; i < num_maps; i++) {
1451                 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1452                 total -= bitmap->map[i].depth;
1453         }
1454
1455         return 0;
1456 }
1457
1458 static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1459 {
1460         struct request_queue *q = hctx->queue;
1461         struct blk_mq_ctx *ctx;
1462         LIST_HEAD(tmp);
1463
1464         /*
1465          * Move ctx entries to new CPU, if this one is going away.
1466          */
1467         ctx = __blk_mq_get_ctx(q, cpu);
1468
1469         spin_lock(&ctx->lock);
1470         if (!list_empty(&ctx->rq_list)) {
1471                 list_splice_init(&ctx->rq_list, &tmp);
1472                 blk_mq_hctx_clear_pending(hctx, ctx);
1473         }
1474         spin_unlock(&ctx->lock);
1475
1476         if (list_empty(&tmp))
1477                 return NOTIFY_OK;
1478
1479         ctx = blk_mq_get_ctx(q);
1480         spin_lock(&ctx->lock);
1481
1482         while (!list_empty(&tmp)) {
1483                 struct request *rq;
1484
1485                 rq = list_first_entry(&tmp, struct request, queuelist);
1486                 rq->mq_ctx = ctx;
1487                 list_move_tail(&rq->queuelist, &ctx->rq_list);
1488         }
1489
1490         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1491         blk_mq_hctx_mark_pending(hctx, ctx);
1492
1493         spin_unlock(&ctx->lock);
1494
1495         blk_mq_run_hw_queue(hctx, true);
1496         blk_mq_put_ctx(ctx);
1497         return NOTIFY_OK;
1498 }
1499
1500 static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1501 {
1502         struct request_queue *q = hctx->queue;
1503         struct blk_mq_tag_set *set = q->tag_set;
1504
1505         if (set->tags[hctx->queue_num])
1506                 return NOTIFY_OK;
1507
1508         set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1509         if (!set->tags[hctx->queue_num])
1510                 return NOTIFY_STOP;
1511
1512         hctx->tags = set->tags[hctx->queue_num];
1513         return NOTIFY_OK;
1514 }
1515
1516 static int blk_mq_hctx_notify(void *data, unsigned long action,
1517                               unsigned int cpu)
1518 {
1519         struct blk_mq_hw_ctx *hctx = data;
1520
1521         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1522                 return blk_mq_hctx_cpu_offline(hctx, cpu);
1523         else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1524                 return blk_mq_hctx_cpu_online(hctx, cpu);
1525
1526         return NOTIFY_OK;
1527 }
1528
1529 static void blk_mq_exit_hctx(struct request_queue *q,
1530                 struct blk_mq_tag_set *set,
1531                 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1532 {
1533         unsigned flush_start_tag = set->queue_depth;
1534
1535         blk_mq_tag_idle(hctx);
1536
1537         if (set->ops->exit_request)
1538                 set->ops->exit_request(set->driver_data,
1539                                        hctx->fq->flush_rq, hctx_idx,
1540                                        flush_start_tag + hctx_idx);
1541
1542         if (set->ops->exit_hctx)
1543                 set->ops->exit_hctx(hctx, hctx_idx);
1544
1545         blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1546         blk_free_flush_queue(hctx->fq);
1547         kfree(hctx->ctxs);
1548         blk_mq_free_bitmap(&hctx->ctx_map);
1549 }
1550
1551 static void blk_mq_exit_hw_queues(struct request_queue *q,
1552                 struct blk_mq_tag_set *set, int nr_queue)
1553 {
1554         struct blk_mq_hw_ctx *hctx;
1555         unsigned int i;
1556
1557         queue_for_each_hw_ctx(q, hctx, i) {
1558                 if (i == nr_queue)
1559                         break;
1560                 blk_mq_exit_hctx(q, set, hctx, i);
1561         }
1562 }
1563
1564 static void blk_mq_free_hw_queues(struct request_queue *q,
1565                 struct blk_mq_tag_set *set)
1566 {
1567         struct blk_mq_hw_ctx *hctx;
1568         unsigned int i;
1569
1570         queue_for_each_hw_ctx(q, hctx, i) {
1571                 free_cpumask_var(hctx->cpumask);
1572                 kfree(hctx);
1573         }
1574 }
1575
1576 static int blk_mq_init_hctx(struct request_queue *q,
1577                 struct blk_mq_tag_set *set,
1578                 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1579 {
1580         int node;
1581         unsigned flush_start_tag = set->queue_depth;
1582
1583         node = hctx->numa_node;
1584         if (node == NUMA_NO_NODE)
1585                 node = hctx->numa_node = set->numa_node;
1586
1587         INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1588         INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1589         spin_lock_init(&hctx->lock);
1590         INIT_LIST_HEAD(&hctx->dispatch);
1591         hctx->queue = q;
1592         hctx->queue_num = hctx_idx;
1593         hctx->flags = set->flags;
1594         hctx->cmd_size = set->cmd_size;
1595
1596         blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1597                                         blk_mq_hctx_notify, hctx);
1598         blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1599
1600         hctx->tags = set->tags[hctx_idx];
1601
1602         /*
1603          * Allocate space for all possible cpus to avoid allocation at
1604          * runtime
1605          */
1606         hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1607                                         GFP_KERNEL, node);
1608         if (!hctx->ctxs)
1609                 goto unregister_cpu_notifier;
1610
1611         if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
1612                 goto free_ctxs;
1613
1614         hctx->nr_ctx = 0;
1615
1616         if (set->ops->init_hctx &&
1617             set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
1618                 goto free_bitmap;
1619
1620         hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
1621         if (!hctx->fq)
1622                 goto exit_hctx;
1623
1624         if (set->ops->init_request &&
1625             set->ops->init_request(set->driver_data,
1626                                    hctx->fq->flush_rq, hctx_idx,
1627                                    flush_start_tag + hctx_idx, node))
1628                 goto free_fq;
1629
1630         return 0;
1631
1632  free_fq:
1633         kfree(hctx->fq);
1634  exit_hctx:
1635         if (set->ops->exit_hctx)
1636                 set->ops->exit_hctx(hctx, hctx_idx);
1637  free_bitmap:
1638         blk_mq_free_bitmap(&hctx->ctx_map);
1639  free_ctxs:
1640         kfree(hctx->ctxs);
1641  unregister_cpu_notifier:
1642         blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1643
1644         return -1;
1645 }
1646
1647 static int blk_mq_init_hw_queues(struct request_queue *q,
1648                 struct blk_mq_tag_set *set)
1649 {
1650         struct blk_mq_hw_ctx *hctx;
1651         unsigned int i;
1652
1653         /*
1654          * Initialize hardware queues
1655          */
1656         queue_for_each_hw_ctx(q, hctx, i) {
1657                 if (blk_mq_init_hctx(q, set, hctx, i))
1658                         break;
1659         }
1660
1661         if (i == q->nr_hw_queues)
1662                 return 0;
1663
1664         /*
1665          * Init failed
1666          */
1667         blk_mq_exit_hw_queues(q, set, i);
1668
1669         return 1;
1670 }
1671
1672 static void blk_mq_init_cpu_queues(struct request_queue *q,
1673                                    unsigned int nr_hw_queues)
1674 {
1675         unsigned int i;
1676
1677         for_each_possible_cpu(i) {
1678                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1679                 struct blk_mq_hw_ctx *hctx;
1680
1681                 memset(__ctx, 0, sizeof(*__ctx));
1682                 __ctx->cpu = i;
1683                 spin_lock_init(&__ctx->lock);
1684                 INIT_LIST_HEAD(&__ctx->rq_list);
1685                 __ctx->queue = q;
1686
1687                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1688                 if (!cpu_online(i))
1689                         continue;
1690
1691                 hctx = q->mq_ops->map_queue(q, i);
1692                 cpumask_set_cpu(i, hctx->cpumask);
1693                 hctx->nr_ctx++;
1694
1695                 /*
1696                  * Set local node, IFF we have more than one hw queue. If
1697                  * not, we remain on the home node of the device
1698                  */
1699                 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1700                         hctx->numa_node = cpu_to_node(i);
1701         }
1702 }
1703
1704 static void blk_mq_map_swqueue(struct request_queue *q)
1705 {
1706         unsigned int i;
1707         struct blk_mq_hw_ctx *hctx;
1708         struct blk_mq_ctx *ctx;
1709
1710         queue_for_each_hw_ctx(q, hctx, i) {
1711                 cpumask_clear(hctx->cpumask);
1712                 hctx->nr_ctx = 0;
1713         }
1714
1715         /*
1716          * Map software to hardware queues
1717          */
1718         queue_for_each_ctx(q, ctx, i) {
1719                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1720                 if (!cpu_online(i))
1721                         continue;
1722
1723                 hctx = q->mq_ops->map_queue(q, i);
1724                 cpumask_set_cpu(i, hctx->cpumask);
1725                 ctx->index_hw = hctx->nr_ctx;
1726                 hctx->ctxs[hctx->nr_ctx++] = ctx;
1727         }
1728
1729         queue_for_each_hw_ctx(q, hctx, i) {
1730                 /*
1731                  * If no software queues are mapped to this hardware queue,
1732                  * disable it and free the request entries.
1733                  */
1734                 if (!hctx->nr_ctx) {
1735                         struct blk_mq_tag_set *set = q->tag_set;
1736
1737                         if (set->tags[i]) {
1738                                 blk_mq_free_rq_map(set, set->tags[i], i);
1739                                 set->tags[i] = NULL;
1740                                 hctx->tags = NULL;
1741                         }
1742                         continue;
1743                 }
1744
1745                 /*
1746                  * Initialize batch roundrobin counts
1747                  */
1748                 hctx->next_cpu = cpumask_first(hctx->cpumask);
1749                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1750         }
1751 }
1752
1753 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1754 {
1755         struct blk_mq_hw_ctx *hctx;
1756         struct request_queue *q;
1757         bool shared;
1758         int i;
1759
1760         if (set->tag_list.next == set->tag_list.prev)
1761                 shared = false;
1762         else
1763                 shared = true;
1764
1765         list_for_each_entry(q, &set->tag_list, tag_set_list) {
1766                 blk_mq_freeze_queue(q);
1767
1768                 queue_for_each_hw_ctx(q, hctx, i) {
1769                         if (shared)
1770                                 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1771                         else
1772                                 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1773                 }
1774                 blk_mq_unfreeze_queue(q);
1775         }
1776 }
1777
1778 static void blk_mq_del_queue_tag_set(struct request_queue *q)
1779 {
1780         struct blk_mq_tag_set *set = q->tag_set;
1781
1782         mutex_lock(&set->tag_list_lock);
1783         list_del_init(&q->tag_set_list);
1784         blk_mq_update_tag_set_depth(set);
1785         mutex_unlock(&set->tag_list_lock);
1786 }
1787
1788 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1789                                      struct request_queue *q)
1790 {
1791         q->tag_set = set;
1792
1793         mutex_lock(&set->tag_list_lock);
1794         list_add_tail(&q->tag_set_list, &set->tag_list);
1795         blk_mq_update_tag_set_depth(set);
1796         mutex_unlock(&set->tag_list_lock);
1797 }
1798
1799 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1800 {
1801         struct blk_mq_hw_ctx **hctxs;
1802         struct blk_mq_ctx __percpu *ctx;
1803         struct request_queue *q;
1804         unsigned int *map;
1805         int i;
1806
1807         ctx = alloc_percpu(struct blk_mq_ctx);
1808         if (!ctx)
1809                 return ERR_PTR(-ENOMEM);
1810
1811         /*
1812          * If a crashdump is active, then we are potentially in a very
1813          * memory constrained environment. Limit us to 1 queue and
1814          * 64 tags to prevent using too much memory.
1815          */
1816         if (is_kdump_kernel()) {
1817                 set->nr_hw_queues = 1;
1818                 set->queue_depth = min(64U, set->queue_depth);
1819         }
1820
1821         hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1822                         set->numa_node);
1823
1824         if (!hctxs)
1825                 goto err_percpu;
1826
1827         map = blk_mq_make_queue_map(set);
1828         if (!map)
1829                 goto err_map;
1830
1831         for (i = 0; i < set->nr_hw_queues; i++) {
1832                 int node = blk_mq_hw_queue_to_node(map, i);
1833
1834                 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
1835                                         GFP_KERNEL, node);
1836                 if (!hctxs[i])
1837                         goto err_hctxs;
1838
1839                 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
1840                                                 node))
1841                         goto err_hctxs;
1842
1843                 atomic_set(&hctxs[i]->nr_active, 0);
1844                 hctxs[i]->numa_node = node;
1845                 hctxs[i]->queue_num = i;
1846         }
1847
1848         q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1849         if (!q)
1850                 goto err_hctxs;
1851
1852         /*
1853          * Init percpu_ref in atomic mode so that it's faster to shutdown.
1854          * See blk_register_queue() for details.
1855          */
1856         if (percpu_ref_init(&q->mq_usage_counter, blk_mq_usage_counter_release,
1857                             PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1858                 goto err_map;
1859
1860         setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1861         blk_queue_rq_timeout(q, 30000);
1862
1863         q->nr_queues = nr_cpu_ids;
1864         q->nr_hw_queues = set->nr_hw_queues;
1865         q->mq_map = map;
1866
1867         q->queue_ctx = ctx;
1868         q->queue_hw_ctx = hctxs;
1869
1870         q->mq_ops = set->ops;
1871         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1872
1873         if (!(set->flags & BLK_MQ_F_SG_MERGE))
1874                 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
1875
1876         q->sg_reserved_size = INT_MAX;
1877
1878         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1879         INIT_LIST_HEAD(&q->requeue_list);
1880         spin_lock_init(&q->requeue_lock);
1881
1882         if (q->nr_hw_queues > 1)
1883                 blk_queue_make_request(q, blk_mq_make_request);
1884         else
1885                 blk_queue_make_request(q, blk_sq_make_request);
1886
1887         if (set->timeout)
1888                 blk_queue_rq_timeout(q, set->timeout);
1889
1890         /*
1891          * Do this after blk_queue_make_request() overrides it...
1892          */
1893         q->nr_requests = set->queue_depth;
1894
1895         if (set->ops->complete)
1896                 blk_queue_softirq_done(q, set->ops->complete);
1897
1898         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1899
1900         if (blk_mq_init_hw_queues(q, set))
1901                 goto err_hw;
1902
1903         mutex_lock(&all_q_mutex);
1904         list_add_tail(&q->all_q_node, &all_q_list);
1905         mutex_unlock(&all_q_mutex);
1906
1907         blk_mq_add_queue_tag_set(set, q);
1908
1909         blk_mq_map_swqueue(q);
1910
1911         return q;
1912
1913 err_hw:
1914         blk_cleanup_queue(q);
1915 err_hctxs:
1916         kfree(map);
1917         for (i = 0; i < set->nr_hw_queues; i++) {
1918                 if (!hctxs[i])
1919                         break;
1920                 free_cpumask_var(hctxs[i]->cpumask);
1921                 kfree(hctxs[i]);
1922         }
1923 err_map:
1924         kfree(hctxs);
1925 err_percpu:
1926         free_percpu(ctx);
1927         return ERR_PTR(-ENOMEM);
1928 }
1929 EXPORT_SYMBOL(blk_mq_init_queue);
1930
1931 void blk_mq_free_queue(struct request_queue *q)
1932 {
1933         struct blk_mq_tag_set   *set = q->tag_set;
1934
1935         blk_mq_del_queue_tag_set(q);
1936
1937         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1938         blk_mq_free_hw_queues(q, set);
1939
1940         percpu_ref_exit(&q->mq_usage_counter);
1941
1942         free_percpu(q->queue_ctx);
1943         kfree(q->queue_hw_ctx);
1944         kfree(q->mq_map);
1945
1946         q->queue_ctx = NULL;
1947         q->queue_hw_ctx = NULL;
1948         q->mq_map = NULL;
1949
1950         mutex_lock(&all_q_mutex);
1951         list_del_init(&q->all_q_node);
1952         mutex_unlock(&all_q_mutex);
1953 }
1954
1955 /* Basically redo blk_mq_init_queue with queue frozen */
1956 static void blk_mq_queue_reinit(struct request_queue *q)
1957 {
1958         blk_mq_freeze_queue(q);
1959
1960         blk_mq_sysfs_unregister(q);
1961
1962         blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1963
1964         /*
1965          * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1966          * we should change hctx numa_node according to new topology (this
1967          * involves free and re-allocate memory, worthy doing?)
1968          */
1969
1970         blk_mq_map_swqueue(q);
1971
1972         blk_mq_sysfs_register(q);
1973
1974         blk_mq_unfreeze_queue(q);
1975 }
1976
1977 static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1978                                       unsigned long action, void *hcpu)
1979 {
1980         struct request_queue *q;
1981
1982         /*
1983          * Before new mappings are established, hotadded cpu might already
1984          * start handling requests. This doesn't break anything as we map
1985          * offline CPUs to first hardware queue. We will re-init the queue
1986          * below to get optimal settings.
1987          */
1988         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1989             action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1990                 return NOTIFY_OK;
1991
1992         mutex_lock(&all_q_mutex);
1993         list_for_each_entry(q, &all_q_list, all_q_node)
1994                 blk_mq_queue_reinit(q);
1995         mutex_unlock(&all_q_mutex);
1996         return NOTIFY_OK;
1997 }
1998
1999 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2000 {
2001         int i;
2002
2003         for (i = 0; i < set->nr_hw_queues; i++) {
2004                 set->tags[i] = blk_mq_init_rq_map(set, i);
2005                 if (!set->tags[i])
2006                         goto out_unwind;
2007         }
2008
2009         return 0;
2010
2011 out_unwind:
2012         while (--i >= 0)
2013                 blk_mq_free_rq_map(set, set->tags[i], i);
2014
2015         return -ENOMEM;
2016 }
2017
2018 /*
2019  * Allocate the request maps associated with this tag_set. Note that this
2020  * may reduce the depth asked for, if memory is tight. set->queue_depth
2021  * will be updated to reflect the allocated depth.
2022  */
2023 static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2024 {
2025         unsigned int depth;
2026         int err;
2027
2028         depth = set->queue_depth;
2029         do {
2030                 err = __blk_mq_alloc_rq_maps(set);
2031                 if (!err)
2032                         break;
2033
2034                 set->queue_depth >>= 1;
2035                 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
2036                         err = -ENOMEM;
2037                         break;
2038                 }
2039         } while (set->queue_depth);
2040
2041         if (!set->queue_depth || err) {
2042                 pr_err("blk-mq: failed to allocate request map\n");
2043                 return -ENOMEM;
2044         }
2045
2046         if (depth != set->queue_depth)
2047                 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
2048                                                 depth, set->queue_depth);
2049
2050         return 0;
2051 }
2052
2053 /*
2054  * Alloc a tag set to be associated with one or more request queues.
2055  * May fail with EINVAL for various error conditions. May adjust the
2056  * requested depth down, if if it too large. In that case, the set
2057  * value will be stored in set->queue_depth.
2058  */
2059 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
2060 {
2061         if (!set->nr_hw_queues)
2062                 return -EINVAL;
2063         if (!set->queue_depth)
2064                 return -EINVAL;
2065         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
2066                 return -EINVAL;
2067
2068         if (!set->nr_hw_queues || !set->ops->queue_rq || !set->ops->map_queue)
2069                 return -EINVAL;
2070
2071         if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
2072                 pr_info("blk-mq: reduced tag depth to %u\n",
2073                         BLK_MQ_MAX_DEPTH);
2074                 set->queue_depth = BLK_MQ_MAX_DEPTH;
2075         }
2076
2077         set->tags = kmalloc_node(set->nr_hw_queues *
2078                                  sizeof(struct blk_mq_tags *),
2079                                  GFP_KERNEL, set->numa_node);
2080         if (!set->tags)
2081                 return -ENOMEM;
2082
2083         if (blk_mq_alloc_rq_maps(set))
2084                 goto enomem;
2085
2086         mutex_init(&set->tag_list_lock);
2087         INIT_LIST_HEAD(&set->tag_list);
2088
2089         return 0;
2090 enomem:
2091         kfree(set->tags);
2092         set->tags = NULL;
2093         return -ENOMEM;
2094 }
2095 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2096
2097 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2098 {
2099         int i;
2100
2101         for (i = 0; i < set->nr_hw_queues; i++) {
2102                 if (set->tags[i])
2103                         blk_mq_free_rq_map(set, set->tags[i], i);
2104         }
2105
2106         kfree(set->tags);
2107         set->tags = NULL;
2108 }
2109 EXPORT_SYMBOL(blk_mq_free_tag_set);
2110
2111 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2112 {
2113         struct blk_mq_tag_set *set = q->tag_set;
2114         struct blk_mq_hw_ctx *hctx;
2115         int i, ret;
2116
2117         if (!set || nr > set->queue_depth)
2118                 return -EINVAL;
2119
2120         ret = 0;
2121         queue_for_each_hw_ctx(q, hctx, i) {
2122                 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2123                 if (ret)
2124                         break;
2125         }
2126
2127         if (!ret)
2128                 q->nr_requests = nr;
2129
2130         return ret;
2131 }
2132
2133 void blk_mq_disable_hotplug(void)
2134 {
2135         mutex_lock(&all_q_mutex);
2136 }
2137
2138 void blk_mq_enable_hotplug(void)
2139 {
2140         mutex_unlock(&all_q_mutex);
2141 }
2142
2143 static int __init blk_mq_init(void)
2144 {
2145         blk_mq_cpu_init();
2146
2147         hotcpu_notifier(blk_mq_queue_reinit_notify, 0);
2148
2149         return 0;
2150 }
2151 subsys_initcall(blk_mq_init);