drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/timer.h>
  37 #include <linux/iova.h>
  38 #include <linux/iommu.h>
  39 #include <linux/intel-iommu.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/tboot.h>
  42 #include <linux/dmi.h>
  43 #include <linux/pci-ats.h>
  44 #include <linux/memblock.h>
  45 #include <linux/dma-contiguous.h>
  46 #include <linux/crash_dump.h>
  47 #include <asm/irq_remapping.h>
  48 #include <asm/cacheflush.h>
  49 #include <asm/iommu.h>
  50
  51 #include "irq_remapping.h"
  52
  53 #define ROOT_SIZE               VTD_PAGE_SIZE
  54 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  55
  56 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  57 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  58 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  59 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  60
  61 #define IOAPIC_RANGE_START      (0xfee00000)
  62 #define IOAPIC_RANGE_END        (0xfeefffff)
  63 #define IOVA_START_ADDR         (0x1000)
  64
  65 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  66
  67 #define MAX_AGAW_WIDTH 64
  68 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  69
  70 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  71 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  72
  73 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  74    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  75 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  76                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  77 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  78
  79 /* IO virtual address start page frame number */
  80 #define IOVA_START_PFN          (1)
  81
  82 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  83 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  84 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  85
  86 /* page table handling */
  87 #define LEVEL_STRIDE            (9)
  88 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  89
  90 /*
  91  * This bitmap is used to advertise the page sizes our hardware support
  92  * to the IOMMU core, which will then use this information to split
  93  * physically contiguous memory regions it is mapping into page sizes
  94  * that we support.
  95  *
  96  * Traditionally the IOMMU core just handed us the mappings directly,
  97  * after making sure the size is an order of a 4KiB page and that the
  98  * mapping has natural alignment.
  99  *
 100  * To retain this behavior, we currently advertise that we support
 101  * all page sizes that are an order of 4KiB.
 102  *
 103  * If at some point we'd like to utilize the IOMMU core's new behavior,
 104  * we could change this to advertise the real page sizes we support.
 105  */
 106 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 107
 108 static inline int agaw_to_level(int agaw)
 109 {
 110         return agaw + 2;
 111 }
 112
 113 static inline int agaw_to_width(int agaw)
 114 {
 115         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 116 }
 117
 118 static inline int width_to_agaw(int width)
 119 {
 120         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 121 }
 122
 123 static inline unsigned int level_to_offset_bits(int level)
 124 {
 125         return (level - 1) * LEVEL_STRIDE;
 126 }
 127
 128 static inline int pfn_level_offset(unsigned long pfn, int level)
 129 {
 130         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 131 }
 132
 133 static inline unsigned long level_mask(int level)
 134 {
 135         return -1UL << level_to_offset_bits(level);
 136 }
 137
 138 static inline unsigned long level_size(int level)
 139 {
 140         return 1UL << level_to_offset_bits(level);
 141 }
 142
 143 static inline unsigned long align_to_level(unsigned long pfn, int level)
 144 {
 145         return (pfn + level_size(level) - 1) & level_mask(level);
 146 }
 147
 148 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 149 {
 150         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 151 }
 152
 153 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 154    are never going to work. */
 155 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 156 {
 157         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 158 }
 159
 160 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 161 {
 162         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 163 }
 164 static inline unsigned long page_to_dma_pfn(struct page *pg)
 165 {
 166         return mm_to_dma_pfn(page_to_pfn(pg));
 167 }
 168 static inline unsigned long virt_to_dma_pfn(void *p)
 169 {
 170         return page_to_dma_pfn(virt_to_page(p));
 171 }
 172
 173 /* global iommu list, set NULL for ignored DMAR units */
 174 static struct intel_iommu **g_iommus;
 175
 176 static void __init check_tylersburg_isoch(void);
 177 static int rwbf_quirk;
 178
 179 /*
 180  * set to 1 to panic kernel if can't successfully enable VT-d
 181  * (used when kernel is launched w/ TXT)
 182  */
 183 static int force_on = 0;
 184
 185 /*
 186  * 0: Present
 187  * 1-11: Reserved
 188  * 12-63: Context Ptr (12 - (haw-1))
 189  * 64-127: Reserved
 190  */
 191 struct root_entry {
 192         u64     lo;
 193         u64     hi;
 194 };
 195 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 196
 197 /*
 198  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 199  * if marked present.
 200  */
 201 static phys_addr_t root_entry_lctp(struct root_entry *re)
 202 {
 203         if (!(re->lo & 1))
 204                 return 0;
 205
 206         return re->lo & VTD_PAGE_MASK;
 207 }
 208
 209 /*
 210  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 211  * if marked present.
 212  */
 213 static phys_addr_t root_entry_uctp(struct root_entry *re)
 214 {
 215         if (!(re->hi & 1))
 216                 return 0;
 217
 218         return re->hi & VTD_PAGE_MASK;
 219 }
 220 /*
 221  * low 64 bits:
 222  * 0: present
 223  * 1: fault processing disable
 224  * 2-3: translation type
 225  * 12-63: address space root
 226  * high 64 bits:
 227  * 0-2: address width
 228  * 3-6: aval
 229  * 8-23: domain id
 230  */
 231 struct context_entry {
 232         u64 lo;
 233         u64 hi;
 234 };
 235
 236 static inline void context_clear_pasid_enable(struct context_entry *context)
 237 {
 238         context->lo &= ~(1ULL << 11);
 239 }
 240
 241 static inline bool context_pasid_enabled(struct context_entry *context)
 242 {
 243         return !!(context->lo & (1ULL << 11));
 244 }
 245
 246 static inline void context_set_copied(struct context_entry *context)
 247 {
 248         context->hi |= (1ull << 3);
 249 }
 250
 251 static inline bool context_copied(struct context_entry *context)
 252 {
 253         return !!(context->hi & (1ULL << 3));
 254 }
 255
 256 static inline bool __context_present(struct context_entry *context)
 257 {
 258         return (context->lo & 1);
 259 }
 260
 261 static inline bool context_present(struct context_entry *context)
 262 {
 263         return context_pasid_enabled(context) ?
 264              __context_present(context) :
 265              __context_present(context) && !context_copied(context);
 266 }
 267
 268 static inline void context_set_present(struct context_entry *context)
 269 {
 270         context->lo |= 1;
 271 }
 272
 273 static inline void context_set_fault_enable(struct context_entry *context)
 274 {
 275         context->lo &= (((u64)-1) << 2) | 1;
 276 }
 277
 278 static inline void context_set_translation_type(struct context_entry *context,
 279                                                 unsigned long value)
 280 {
 281         context->lo &= (((u64)-1) << 4) | 3;
 282         context->lo |= (value & 3) << 2;
 283 }
 284
 285 static inline void context_set_address_root(struct context_entry *context,
 286                                             unsigned long value)
 287 {
 288         context->lo &= ~VTD_PAGE_MASK;
 289         context->lo |= value & VTD_PAGE_MASK;
 290 }
 291
 292 static inline void context_set_address_width(struct context_entry *context,
 293                                              unsigned long value)
 294 {
 295         context->hi |= value & 7;
 296 }
 297
 298 static inline void context_set_domain_id(struct context_entry *context,
 299                                          unsigned long value)
 300 {
 301         context->hi |= (value & ((1 << 16) - 1)) << 8;
 302 }
 303
 304 static inline int context_domain_id(struct context_entry *c)
 305 {
 306         return((c->hi >> 8) & 0xffff);
 307 }
 308
 309 static inline void context_clear_entry(struct context_entry *context)
 310 {
 311         context->lo = 0;
 312         context->hi = 0;
 313 }
 314
 315 /*
 316  * 0: readable
 317  * 1: writable
 318  * 2-6: reserved
 319  * 7: super page
 320  * 8-10: available
 321  * 11: snoop behavior
 322  * 12-63: Host physcial address
 323  */
 324 struct dma_pte {
 325         u64 val;
 326 };
 327
 328 static inline void dma_clear_pte(struct dma_pte *pte)
 329 {
 330         pte->val = 0;
 331 }
 332
 333 static inline u64 dma_pte_addr(struct dma_pte *pte)
 334 {
 335 #ifdef CONFIG_64BIT
 336         return pte->val & VTD_PAGE_MASK;
 337 #else
 338         /* Must have a full atomic 64-bit read */
 339         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 340 #endif
 341 }
 342
 343 static inline bool dma_pte_present(struct dma_pte *pte)
 344 {
 345         return (pte->val & 3) != 0;
 346 }
 347
 348 static inline bool dma_pte_superpage(struct dma_pte *pte)
 349 {
 350         return (pte->val & DMA_PTE_LARGE_PAGE);
 351 }
 352
 353 static inline int first_pte_in_page(struct dma_pte *pte)
 354 {
 355         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 356 }
 357
 358 /*
 359  * This domain is a statically identity mapping domain.
 360  *      1. This domain creats a static 1:1 mapping to all usable memory.
 361  *      2. It maps to each iommu if successful.
 362  *      3. Each iommu mapps to this domain if successful.
 363  */
 364 static struct dmar_domain *si_domain;
 365 static int hw_pass_through = 1;
 366
 367 /*
 368  * Domain represents a virtual machine, more than one devices
 369  * across iommus may be owned in one domain, e.g. kvm guest.
 370  */
 371 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 372
 373 /* si_domain contains mulitple devices */
 374 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 375
 376 #define for_each_domain_iommu(idx, domain)                      \
 377         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 378                 if (domain->iommu_refcnt[idx])
 379
 380 struct dmar_domain {
 381         int     nid;                    /* node id */
 382
 383         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 384                                         /* Refcount of devices per iommu */
 385
 386
 387         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 388                                         /* Domain ids per IOMMU. Use u16 since
 389                                          * domain ids are 16 bit wide according
 390                                          * to VT-d spec, section 9.3 */
 391
 392         struct list_head devices;       /* all devices' list */
 393         struct iova_domain iovad;       /* iova's that belong to this domain */
 394
 395         struct dma_pte  *pgd;           /* virtual address */
 396         int             gaw;            /* max guest address width */
 397
 398         /* adjusted guest address width, 0 is level 2 30-bit */
 399         int             agaw;
 400
 401         int             flags;          /* flags to find out type of domain */
 402
 403         int             iommu_coherency;/* indicate coherency of iommu access */
 404         int             iommu_snooping; /* indicate snooping control feature*/
 405         int             iommu_count;    /* reference count of iommu */
 406         int             iommu_superpage;/* Level of superpages supported:
 407                                            0 == 4KiB (no superpages), 1 == 2MiB,
 408                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 409         u64             max_addr;       /* maximum mapped address */
 410
 411         struct iommu_domain domain;     /* generic domain data structure for
 412                                            iommu core */
 413 };
 414
 415 /* PCI domain-device relationship */
 416 struct device_domain_info {
 417         struct list_head link;  /* link to domain siblings */
 418         struct list_head global; /* link to global list */
 419         u8 bus;                 /* PCI bus number */
 420         u8 devfn;               /* PCI devfn number */
 421         struct {
 422                 u8 enabled:1;
 423                 u8 qdep;
 424         } ats;                  /* ATS state */
 425         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 426         struct intel_iommu *iommu; /* IOMMU used by this device */
 427         struct dmar_domain *domain; /* pointer to domain */
 428 };
 429
 430 struct dmar_rmrr_unit {
 431         struct list_head list;          /* list of rmrr units   */
 432         struct acpi_dmar_header *hdr;   /* ACPI header          */
 433         u64     base_address;           /* reserved base address*/
 434         u64     end_address;            /* reserved end address */
 435         struct dmar_dev_scope *devices; /* target devices */
 436         int     devices_cnt;            /* target device count */
 437 };
 438
 439 struct dmar_atsr_unit {
 440         struct list_head list;          /* list of ATSR units */
 441         struct acpi_dmar_header *hdr;   /* ACPI header */
 442         struct dmar_dev_scope *devices; /* target devices */
 443         int devices_cnt;                /* target device count */
 444         u8 include_all:1;               /* include all ports */
 445 };
 446
 447 static LIST_HEAD(dmar_atsr_units);
 448 static LIST_HEAD(dmar_rmrr_units);
 449
 450 #define for_each_rmrr_units(rmrr) \
 451         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 452
 453 static void flush_unmaps_timeout(unsigned long data);
 454
 455 static DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 456
 457 #define HIGH_WATER_MARK 250
 458 struct deferred_flush_tables {
 459         int next;
 460         struct iova *iova[HIGH_WATER_MARK];
 461         struct dmar_domain *domain[HIGH_WATER_MARK];
 462         struct page *freelist[HIGH_WATER_MARK];
 463 };
 464
 465 static struct deferred_flush_tables *deferred_flush;
 466
 467 /* bitmap for indexing intel_iommus */
 468 static int g_num_of_iommus;
 469
 470 static DEFINE_SPINLOCK(async_umap_flush_lock);
 471 static LIST_HEAD(unmaps_to_do);
 472
 473 static int timer_on;
 474 static long list_size;
 475
 476 static void domain_exit(struct dmar_domain *domain);
 477 static void domain_remove_dev_info(struct dmar_domain *domain);
 478 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 479                                      struct device *dev);
 480 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 481 static void domain_context_clear(struct intel_iommu *iommu,
 482                                  struct device *dev);
 483 static int domain_detach_iommu(struct dmar_domain *domain,
 484                                struct intel_iommu *iommu);
 485
 486 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 487 int dmar_disabled = 0;
 488 #else
 489 int dmar_disabled = 1;
 490 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 491
 492 int intel_iommu_enabled = 0;
 493 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 494
 495 static int dmar_map_gfx = 1;
 496 static int dmar_forcedac;
 497 static int intel_iommu_strict;
 498 static int intel_iommu_superpage = 1;
 499 static int intel_iommu_ecs = 1;
 500
 501 /* We only actually use ECS when PASID support (on the new bit 40)
 502  * is also advertised. Some early implementations — the ones with
 503  * PASID support on bit 28 — have issues even when we *only* use
 504  * extended root/context tables. */
 505 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 506                             ecap_pasid(iommu->ecap))
 507
 508 int intel_iommu_gfx_mapped;
 509 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 510
 511 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 512 static DEFINE_SPINLOCK(device_domain_lock);
 513 static LIST_HEAD(device_domain_list);
 514
 515 static const struct iommu_ops intel_iommu_ops;
 516
 517 static bool translation_pre_enabled(struct intel_iommu *iommu)
 518 {
 519         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 520 }
 521
 522 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 523 {
 524         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 525 }
 526
 527 static void init_translation_status(struct intel_iommu *iommu)
 528 {
 529         u32 gsts;
 530
 531         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 532         if (gsts & DMA_GSTS_TES)
 533                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 534 }
 535
 536 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 537 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 538 {
 539         return container_of(dom, struct dmar_domain, domain);
 540 }
 541
 542 static int __init intel_iommu_setup(char *str)
 543 {
 544         if (!str)
 545                 return -EINVAL;
 546         while (*str) {
 547                 if (!strncmp(str, "on", 2)) {
 548                         dmar_disabled = 0;
 549                         pr_info("IOMMU enabled\n");
 550                 } else if (!strncmp(str, "off", 3)) {
 551                         dmar_disabled = 1;
 552                         pr_info("IOMMU disabled\n");
 553                 } else if (!strncmp(str, "igfx_off", 8)) {
 554                         dmar_map_gfx = 0;
 555                         pr_info("Disable GFX device mapping\n");
 556                 } else if (!strncmp(str, "forcedac", 8)) {
 557                         pr_info("Forcing DAC for PCI devices\n");
 558                         dmar_forcedac = 1;
 559                 } else if (!strncmp(str, "strict", 6)) {
 560                         pr_info("Disable batched IOTLB flush\n");
 561                         intel_iommu_strict = 1;
 562                 } else if (!strncmp(str, "sp_off", 6)) {
 563                         pr_info("Disable supported super page\n");
 564                         intel_iommu_superpage = 0;
 565                 } else if (!strncmp(str, "ecs_off", 7)) {
 566                         printk(KERN_INFO
 567                                 "Intel-IOMMU: disable extended context table support\n");
 568                         intel_iommu_ecs = 0;
 569                 }
 570
 571                 str += strcspn(str, ",");
 572                 while (*str == ',')
 573                         str++;
 574         }
 575         return 0;
 576 }
 577 __setup("intel_iommu=", intel_iommu_setup);
 578
 579 static struct kmem_cache *iommu_domain_cache;
 580 static struct kmem_cache *iommu_devinfo_cache;
 581
 582 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 583 {
 584         struct dmar_domain **domains;
 585         int idx = did >> 8;
 586
 587         domains = iommu->domains[idx];
 588         if (!domains)
 589                 return NULL;
 590
 591         return domains[did & 0xff];
 592 }
 593
 594 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 595                              struct dmar_domain *domain)
 596 {
 597         struct dmar_domain **domains;
 598         int idx = did >> 8;
 599
 600         if (!iommu->domains[idx]) {
 601                 size_t size = 256 * sizeof(struct dmar_domain *);
 602                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 603         }
 604
 605         domains = iommu->domains[idx];
 606         if (WARN_ON(!domains))
 607                 return;
 608         else
 609                 domains[did & 0xff] = domain;
 610 }
 611
 612 static inline void *alloc_pgtable_page(int node)
 613 {
 614         struct page *page;
 615         void *vaddr = NULL;
 616
 617         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 618         if (page)
 619                 vaddr = page_address(page);
 620         return vaddr;
 621 }
 622
 623 static inline void free_pgtable_page(void *vaddr)
 624 {
 625         free_page((unsigned long)vaddr);
 626 }
 627
 628 static inline void *alloc_domain_mem(void)
 629 {
 630         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 631 }
 632
 633 static void free_domain_mem(void *vaddr)
 634 {
 635         kmem_cache_free(iommu_domain_cache, vaddr);
 636 }
 637
 638 static inline void * alloc_devinfo_mem(void)
 639 {
 640         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 641 }
 642
 643 static inline void free_devinfo_mem(void *vaddr)
 644 {
 645         kmem_cache_free(iommu_devinfo_cache, vaddr);
 646 }
 647
 648 static inline int domain_type_is_vm(struct dmar_domain *domain)
 649 {
 650         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 651 }
 652
 653 static inline int domain_type_is_si(struct dmar_domain *domain)
 654 {
 655         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 656 }
 657
 658 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 659 {
 660         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 661                                 DOMAIN_FLAG_STATIC_IDENTITY);
 662 }
 663
 664 static inline int domain_pfn_supported(struct dmar_domain *domain,
 665                                        unsigned long pfn)
 666 {
 667         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 668
 669         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 670 }
 671
 672 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 673 {
 674         unsigned long sagaw;
 675         int agaw = -1;
 676
 677         sagaw = cap_sagaw(iommu->cap);
 678         for (agaw = width_to_agaw(max_gaw);
 679              agaw >= 0; agaw--) {
 680                 if (test_bit(agaw, &sagaw))
 681                         break;
 682         }
 683
 684         return agaw;
 685 }
 686
 687 /*
 688  * Calculate max SAGAW for each iommu.
 689  */
 690 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 691 {
 692         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 693 }
 694
 695 /*
 696  * calculate agaw for each iommu.
 697  * "SAGAW" may be different across iommus, use a default agaw, and
 698  * get a supported less agaw for iommus that don't support the default agaw.
 699  */
 700 int iommu_calculate_agaw(struct intel_iommu *iommu)
 701 {
 702         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 703 }
 704
 705 /* This functionin only returns single iommu in a domain */
 706 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 707 {
 708         int iommu_id;
 709
 710         /* si_domain and vm domain should not get here. */
 711         BUG_ON(domain_type_is_vm_or_si(domain));
 712         for_each_domain_iommu(iommu_id, domain)
 713                 break;
 714
 715         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 716                 return NULL;
 717
 718         return g_iommus[iommu_id];
 719 }
 720
 721 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 722 {
 723         struct dmar_drhd_unit *drhd;
 724         struct intel_iommu *iommu;
 725         bool found = false;
 726         int i;
 727
 728         domain->iommu_coherency = 1;
 729
 730         for_each_domain_iommu(i, domain) {
 731                 found = true;
 732                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 733                         domain->iommu_coherency = 0;
 734                         break;
 735                 }
 736         }
 737         if (found)
 738                 return;
 739
 740         /* No hardware attached; use lowest common denominator */
 741         rcu_read_lock();
 742         for_each_active_iommu(iommu, drhd) {
 743                 if (!ecap_coherent(iommu->ecap)) {
 744                         domain->iommu_coherency = 0;
 745                         break;
 746                 }
 747         }
 748         rcu_read_unlock();
 749 }
 750
 751 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 752 {
 753         struct dmar_drhd_unit *drhd;
 754         struct intel_iommu *iommu;
 755         int ret = 1;
 756
 757         rcu_read_lock();
 758         for_each_active_iommu(iommu, drhd) {
 759                 if (iommu != skip) {
 760                         if (!ecap_sc_support(iommu->ecap)) {
 761                                 ret = 0;
 762                                 break;
 763                         }
 764                 }
 765         }
 766         rcu_read_unlock();
 767
 768         return ret;
 769 }
 770
 771 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 772 {
 773         struct dmar_drhd_unit *drhd;
 774         struct intel_iommu *iommu;
 775         int mask = 0xf;
 776
 777         if (!intel_iommu_superpage) {
 778                 return 0;
 779         }
 780
 781         /* set iommu_superpage to the smallest common denominator */
 782         rcu_read_lock();
 783         for_each_active_iommu(iommu, drhd) {
 784                 if (iommu != skip) {
 785                         mask &= cap_super_page_val(iommu->cap);
 786                         if (!mask)
 787                                 break;
 788                 }
 789         }
 790         rcu_read_unlock();
 791
 792         return fls(mask);
 793 }
 794
 795 /* Some capabilities may be different across iommus */
 796 static void domain_update_iommu_cap(struct dmar_domain *domain)
 797 {
 798         domain_update_iommu_coherency(domain);
 799         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 800         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 801 }
 802
 803 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 804                                                        u8 bus, u8 devfn, int alloc)
 805 {
 806         struct root_entry *root = &iommu->root_entry[bus];
 807         struct context_entry *context;
 808         u64 *entry;
 809
 810         entry = &root->lo;
 811         if (ecs_enabled(iommu)) {
 812                 if (devfn >= 0x80) {
 813                         devfn -= 0x80;
 814                         entry = &root->hi;
 815                 }
 816                 devfn *= 2;
 817         }
 818         if (*entry & 1)
 819                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 820         else {
 821                 unsigned long phy_addr;
 822                 if (!alloc)
 823                         return NULL;
 824
 825                 context = alloc_pgtable_page(iommu->node);
 826                 if (!context)
 827                         return NULL;
 828
 829                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 830                 phy_addr = virt_to_phys((void *)context);
 831                 *entry = phy_addr | 1;
 832                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 833         }
 834         return &context[devfn];
 835 }
 836
 837 static int iommu_dummy(struct device *dev)
 838 {
 839         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 840 }
 841
 842 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 843 {
 844         struct dmar_drhd_unit *drhd = NULL;
 845         struct intel_iommu *iommu;
 846         struct device *tmp;
 847         struct pci_dev *ptmp, *pdev = NULL;
 848         u16 segment = 0;
 849         int i;
 850
 851         if (iommu_dummy(dev))
 852                 return NULL;
 853
 854         if (dev_is_pci(dev)) {
 855                 pdev = to_pci_dev(dev);
 856                 segment = pci_domain_nr(pdev->bus);
 857         } else if (has_acpi_companion(dev))
 858                 dev = &ACPI_COMPANION(dev)->dev;
 859
 860         rcu_read_lock();
 861         for_each_active_iommu(iommu, drhd) {
 862                 if (pdev && segment != drhd->segment)
 863                         continue;
 864
 865                 for_each_active_dev_scope(drhd->devices,
 866                                           drhd->devices_cnt, i, tmp) {
 867                         if (tmp == dev) {
 868                                 *bus = drhd->devices[i].bus;
 869                                 *devfn = drhd->devices[i].devfn;
 870                                 goto out;
 871                         }
 872
 873                         if (!pdev || !dev_is_pci(tmp))
 874                                 continue;
 875
 876                         ptmp = to_pci_dev(tmp);
 877                         if (ptmp->subordinate &&
 878                             ptmp->subordinate->number <= pdev->bus->number &&
 879                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 880                                 goto got_pdev;
 881                 }
 882
 883                 if (pdev && drhd->include_all) {
 884                 got_pdev:
 885                         *bus = pdev->bus->number;
 886                         *devfn = pdev->devfn;
 887                         goto out;
 888                 }
 889         }
 890         iommu = NULL;
 891  out:
 892         rcu_read_unlock();
 893
 894         return iommu;
 895 }
 896
 897 static void domain_flush_cache(struct dmar_domain *domain,
 898                                void *addr, int size)
 899 {
 900         if (!domain->iommu_coherency)
 901                 clflush_cache_range(addr, size);
 902 }
 903
 904 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 905 {
 906         struct context_entry *context;
 907         int ret = 0;
 908         unsigned long flags;
 909
 910         spin_lock_irqsave(&iommu->lock, flags);
 911         context = iommu_context_addr(iommu, bus, devfn, 0);
 912         if (context)
 913                 ret = context_present(context);
 914         spin_unlock_irqrestore(&iommu->lock, flags);
 915         return ret;
 916 }
 917
 918 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 919 {
 920         struct context_entry *context;
 921         unsigned long flags;
 922
 923         spin_lock_irqsave(&iommu->lock, flags);
 924         context = iommu_context_addr(iommu, bus, devfn, 0);
 925         if (context) {
 926                 context_clear_entry(context);
 927                 __iommu_flush_cache(iommu, context, sizeof(*context));
 928         }
 929         spin_unlock_irqrestore(&iommu->lock, flags);
 930 }
 931
 932 static void free_context_table(struct intel_iommu *iommu)
 933 {
 934         int i;
 935         unsigned long flags;
 936         struct context_entry *context;
 937
 938         spin_lock_irqsave(&iommu->lock, flags);
 939         if (!iommu->root_entry) {
 940                 goto out;
 941         }
 942         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 943                 context = iommu_context_addr(iommu, i, 0, 0);
 944                 if (context)
 945                         free_pgtable_page(context);
 946
 947                 if (!ecs_enabled(iommu))
 948                         continue;
 949
 950                 context = iommu_context_addr(iommu, i, 0x80, 0);
 951                 if (context)
 952                         free_pgtable_page(context);
 953
 954         }
 955         free_pgtable_page(iommu->root_entry);
 956         iommu->root_entry = NULL;
 957 out:
 958         spin_unlock_irqrestore(&iommu->lock, flags);
 959 }
 960
 961 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 962                                       unsigned long pfn, int *target_level)
 963 {
 964         struct dma_pte *parent, *pte = NULL;
 965         int level = agaw_to_level(domain->agaw);
 966         int offset;
 967
 968         BUG_ON(!domain->pgd);
 969
 970         if (!domain_pfn_supported(domain, pfn))
 971                 /* Address beyond IOMMU's addressing capabilities. */
 972                 return NULL;
 973
 974         parent = domain->pgd;
 975
 976         while (1) {
 977                 void *tmp_page;
 978
 979                 offset = pfn_level_offset(pfn, level);
 980                 pte = &parent[offset];
 981                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 982                         break;
 983                 if (level == *target_level)
 984                         break;
 985
 986                 if (!dma_pte_present(pte)) {
 987                         uint64_t pteval;
 988
 989                         tmp_page = alloc_pgtable_page(domain->nid);
 990
 991                         if (!tmp_page)
 992                                 return NULL;
 993
 994                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 995                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 996                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 997                                 /* Someone else set it while we were thinking; use theirs. */
 998                                 free_pgtable_page(tmp_page);
 999                         else
1000                                 domain_flush_cache(domain, pte, sizeof(*pte));
1001                 }
1002                 if (level == 1)
1003                         break;
1004
1005                 parent = phys_to_virt(dma_pte_addr(pte));
1006                 level--;
1007         }
1008
1009         if (!*target_level)
1010                 *target_level = level;
1011
1012         return pte;
1013 }
1014
1015
1016 /* return address's pte at specific level */
1017 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1018                                          unsigned long pfn,
1019                                          int level, int *large_page)
1020 {
1021         struct dma_pte *parent, *pte = NULL;
1022         int total = agaw_to_level(domain->agaw);
1023         int offset;
1024
1025         parent = domain->pgd;
1026         while (level <= total) {
1027                 offset = pfn_level_offset(pfn, total);
1028                 pte = &parent[offset];
1029                 if (level == total)
1030                         return pte;
1031
1032                 if (!dma_pte_present(pte)) {
1033                         *large_page = total;
1034                         break;
1035                 }
1036
1037                 if (dma_pte_superpage(pte)) {
1038                         *large_page = total;
1039                         return pte;
1040                 }
1041
1042                 parent = phys_to_virt(dma_pte_addr(pte));
1043                 total--;
1044         }
1045         return NULL;
1046 }
1047
1048 /* clear last level pte, a tlb flush should be followed */
1049 static void dma_pte_clear_range(struct dmar_domain *domain,
1050                                 unsigned long start_pfn,
1051                                 unsigned long last_pfn)
1052 {
1053         unsigned int large_page = 1;
1054         struct dma_pte *first_pte, *pte;
1055
1056         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1057         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1058         BUG_ON(start_pfn > last_pfn);
1059
1060         /* we don't need lock here; nobody else touches the iova range */
1061         do {
1062                 large_page = 1;
1063                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1064                 if (!pte) {
1065                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1066                         continue;
1067                 }
1068                 do {
1069                         dma_clear_pte(pte);
1070                         start_pfn += lvl_to_nr_pages(large_page);
1071                         pte++;
1072                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1073
1074                 domain_flush_cache(domain, first_pte,
1075                                    (void *)pte - (void *)first_pte);
1076
1077         } while (start_pfn && start_pfn <= last_pfn);
1078 }
1079
1080 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1081                                struct dma_pte *pte, unsigned long pfn,
1082                                unsigned long start_pfn, unsigned long last_pfn)
1083 {
1084         pfn = max(start_pfn, pfn);
1085         pte = &pte[pfn_level_offset(pfn, level)];
1086
1087         do {
1088                 unsigned long level_pfn;
1089                 struct dma_pte *level_pte;
1090
1091                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1092                         goto next;
1093
1094                 level_pfn = pfn & level_mask(level - 1);
1095                 level_pte = phys_to_virt(dma_pte_addr(pte));
1096
1097                 if (level > 2)
1098                         dma_pte_free_level(domain, level - 1, level_pte,
1099                                            level_pfn, start_pfn, last_pfn);
1100
1101                 /* If range covers entire pagetable, free it */
1102                 if (!(start_pfn > level_pfn ||
1103                       last_pfn < level_pfn + level_size(level) - 1)) {
1104                         dma_clear_pte(pte);
1105                         domain_flush_cache(domain, pte, sizeof(*pte));
1106                         free_pgtable_page(level_pte);
1107                 }
1108 next:
1109                 pfn += level_size(level);
1110         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1111 }
1112
1113 /* free page table pages. last level pte should already be cleared */
1114 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1115                                    unsigned long start_pfn,
1116                                    unsigned long last_pfn)
1117 {
1118         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1119         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1120         BUG_ON(start_pfn > last_pfn);
1121
1122         dma_pte_clear_range(domain, start_pfn, last_pfn);
1123
1124         /* We don't need lock here; nobody else touches the iova range */
1125         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1126                            domain->pgd, 0, start_pfn, last_pfn);
1127
1128         /* free pgd */
1129         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1130                 free_pgtable_page(domain->pgd);
1131                 domain->pgd = NULL;
1132         }
1133 }
1134
1135 /* When a page at a given level is being unlinked from its parent, we don't
1136    need to *modify* it at all. All we need to do is make a list of all the
1137    pages which can be freed just as soon as we've flushed the IOTLB and we
1138    know the hardware page-walk will no longer touch them.
1139    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1140    be freed. */
1141 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1142                                             int level, struct dma_pte *pte,
1143                                             struct page *freelist)
1144 {
1145         struct page *pg;
1146
1147         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1148         pg->freelist = freelist;
1149         freelist = pg;
1150
1151         if (level == 1)
1152                 return freelist;
1153
1154         pte = page_address(pg);
1155         do {
1156                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1157                         freelist = dma_pte_list_pagetables(domain, level - 1,
1158                                                            pte, freelist);
1159                 pte++;
1160         } while (!first_pte_in_page(pte));
1161
1162         return freelist;
1163 }
1164
1165 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1166                                         struct dma_pte *pte, unsigned long pfn,
1167                                         unsigned long start_pfn,
1168                                         unsigned long last_pfn,
1169                                         struct page *freelist)
1170 {
1171         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1172
1173         pfn = max(start_pfn, pfn);
1174         pte = &pte[pfn_level_offset(pfn, level)];
1175
1176         do {
1177                 unsigned long level_pfn;
1178
1179                 if (!dma_pte_present(pte))
1180                         goto next;
1181
1182                 level_pfn = pfn & level_mask(level);
1183
1184                 /* If range covers entire pagetable, free it */
1185                 if (start_pfn <= level_pfn &&
1186                     last_pfn >= level_pfn + level_size(level) - 1) {
1187                         /* These suborbinate page tables are going away entirely. Don't
1188                            bother to clear them; we're just going to *free* them. */
1189                         if (level > 1 && !dma_pte_superpage(pte))
1190                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1191
1192                         dma_clear_pte(pte);
1193                         if (!first_pte)
1194                                 first_pte = pte;
1195                         last_pte = pte;
1196                 } else if (level > 1) {
1197                         /* Recurse down into a level that isn't *entirely* obsolete */
1198                         freelist = dma_pte_clear_level(domain, level - 1,
1199                                                        phys_to_virt(dma_pte_addr(pte)),
1200                                                        level_pfn, start_pfn, last_pfn,
1201                                                        freelist);
1202                 }
1203 next:
1204                 pfn += level_size(level);
1205         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1206
1207         if (first_pte)
1208                 domain_flush_cache(domain, first_pte,
1209                                    (void *)++last_pte - (void *)first_pte);
1210
1211         return freelist;
1212 }
1213
1214 /* We can't just free the pages because the IOMMU may still be walking
1215    the page tables, and may have cached the intermediate levels. The
1216    pages can only be freed after the IOTLB flush has been done. */
1217 static struct page *domain_unmap(struct dmar_domain *domain,
1218                                  unsigned long start_pfn,
1219                                  unsigned long last_pfn)
1220 {
1221         struct page *freelist = NULL;
1222
1223         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1224         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1225         BUG_ON(start_pfn > last_pfn);
1226
1227         /* we don't need lock here; nobody else touches the iova range */
1228         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1229                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1230
1231         /* free pgd */
1232         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1233                 struct page *pgd_page = virt_to_page(domain->pgd);
1234                 pgd_page->freelist = freelist;
1235                 freelist = pgd_page;
1236
1237                 domain->pgd = NULL;
1238         }
1239
1240         return freelist;
1241 }
1242
1243 static void dma_free_pagelist(struct page *freelist)
1244 {
1245         struct page *pg;
1246
1247         while ((pg = freelist)) {
1248                 freelist = pg->freelist;
1249                 free_pgtable_page(page_address(pg));
1250         }
1251 }
1252
1253 /* iommu handling */
1254 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1255 {
1256         struct root_entry *root;
1257         unsigned long flags;
1258
1259         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1260         if (!root) {
1261                 pr_err("Allocating root entry for %s failed\n",
1262                         iommu->name);
1263                 return -ENOMEM;
1264         }
1265
1266         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1267
1268         spin_lock_irqsave(&iommu->lock, flags);
1269         iommu->root_entry = root;
1270         spin_unlock_irqrestore(&iommu->lock, flags);
1271
1272         return 0;
1273 }
1274
1275 static void iommu_set_root_entry(struct intel_iommu *iommu)
1276 {
1277         u64 addr;
1278         u32 sts;
1279         unsigned long flag;
1280
1281         addr = virt_to_phys(iommu->root_entry);
1282         if (ecs_enabled(iommu))
1283                 addr |= DMA_RTADDR_RTT;
1284
1285         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1286         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1287
1288         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1289
1290         /* Make sure hardware complete it */
1291         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1292                       readl, (sts & DMA_GSTS_RTPS), sts);
1293
1294         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1295 }
1296
1297 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1298 {
1299         u32 val;
1300         unsigned long flag;
1301
1302         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1303                 return;
1304
1305         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1306         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1307
1308         /* Make sure hardware complete it */
1309         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1310                       readl, (!(val & DMA_GSTS_WBFS)), val);
1311
1312         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1313 }
1314
1315 /* return value determine if we need a write buffer flush */
1316 static void __iommu_flush_context(struct intel_iommu *iommu,
1317                                   u16 did, u16 source_id, u8 function_mask,
1318                                   u64 type)
1319 {
1320         u64 val = 0;
1321         unsigned long flag;
1322
1323         switch (type) {
1324         case DMA_CCMD_GLOBAL_INVL:
1325                 val = DMA_CCMD_GLOBAL_INVL;
1326                 break;
1327         case DMA_CCMD_DOMAIN_INVL:
1328                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1329                 break;
1330         case DMA_CCMD_DEVICE_INVL:
1331                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1332                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1333                 break;
1334         default:
1335                 BUG();
1336         }
1337         val |= DMA_CCMD_ICC;
1338
1339         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1340         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1341
1342         /* Make sure hardware complete it */
1343         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1344                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1345
1346         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1347 }
1348
1349 /* return value determine if we need a write buffer flush */
1350 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1351                                 u64 addr, unsigned int size_order, u64 type)
1352 {
1353         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1354         u64 val = 0, val_iva = 0;
1355         unsigned long flag;
1356
1357         switch (type) {
1358         case DMA_TLB_GLOBAL_FLUSH:
1359                 /* global flush doesn't need set IVA_REG */
1360                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1361                 break;
1362         case DMA_TLB_DSI_FLUSH:
1363                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1364                 break;
1365         case DMA_TLB_PSI_FLUSH:
1366                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1367                 /* IH bit is passed in as part of address */
1368                 val_iva = size_order | addr;
1369                 break;
1370         default:
1371                 BUG();
1372         }
1373         /* Note: set drain read/write */
1374 #if 0
1375         /*
1376          * This is probably to be super secure.. Looks like we can
1377          * ignore it without any impact.
1378          */
1379         if (cap_read_drain(iommu->cap))
1380                 val |= DMA_TLB_READ_DRAIN;
1381 #endif
1382         if (cap_write_drain(iommu->cap))
1383                 val |= DMA_TLB_WRITE_DRAIN;
1384
1385         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1386         /* Note: Only uses first TLB reg currently */
1387         if (val_iva)
1388                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1389         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1390
1391         /* Make sure hardware complete it */
1392         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1393                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1394
1395         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1396
1397         /* check IOTLB invalidation granularity */
1398         if (DMA_TLB_IAIG(val) == 0)
1399                 pr_err("Flush IOTLB failed\n");
1400         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1401                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1402                         (unsigned long long)DMA_TLB_IIRG(type),
1403                         (unsigned long long)DMA_TLB_IAIG(val));
1404 }
1405
1406 static struct device_domain_info *
1407 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1408                          u8 bus, u8 devfn)
1409 {
1410         bool found = false;
1411         struct device_domain_info *info;
1412         struct pci_dev *pdev;
1413
1414         assert_spin_locked(&device_domain_lock);
1415
1416         if (!ecap_dev_iotlb_support(iommu->ecap))
1417                 return NULL;
1418
1419         if (!iommu->qi)
1420                 return NULL;
1421
1422         list_for_each_entry(info, &domain->devices, link)
1423                 if (info->iommu == iommu && info->bus == bus &&
1424                     info->devfn == devfn) {
1425                         found = true;
1426                         break;
1427                 }
1428
1429         if (!found || !info->dev || !dev_is_pci(info->dev))
1430                 return NULL;
1431
1432         pdev = to_pci_dev(info->dev);
1433
1434         if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1435                 return NULL;
1436
1437         if (!dmar_find_matched_atsr_unit(pdev))
1438                 return NULL;
1439
1440         return info;
1441 }
1442
1443 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1444 {
1445         struct pci_dev *pdev;
1446
1447         if (!info || !dev_is_pci(info->dev))
1448                 return;
1449
1450         pdev = to_pci_dev(info->dev);
1451         if (pci_enable_ats(pdev, VTD_PAGE_SHIFT))
1452                 return;
1453
1454         info->ats.enabled = 1;
1455         info->ats.qdep = pci_ats_queue_depth(pdev);
1456 }
1457
1458 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1459 {
1460         if (!info->ats.enabled)
1461                 return;
1462
1463         pci_disable_ats(to_pci_dev(info->dev));
1464         info->ats.enabled = 0;
1465 }
1466
1467 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1468                                   u64 addr, unsigned mask)
1469 {
1470         u16 sid, qdep;
1471         unsigned long flags;
1472         struct device_domain_info *info;
1473
1474         spin_lock_irqsave(&device_domain_lock, flags);
1475         list_for_each_entry(info, &domain->devices, link) {
1476                 if (!info->ats.enabled)
1477                         continue;
1478
1479                 sid = info->bus << 8 | info->devfn;
1480                 qdep = info->ats.qdep;
1481                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1482         }
1483         spin_unlock_irqrestore(&device_domain_lock, flags);
1484 }
1485
1486 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1487                                   struct dmar_domain *domain,
1488                                   unsigned long pfn, unsigned int pages,
1489                                   int ih, int map)
1490 {
1491         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1492         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1493         u16 did = domain->iommu_did[iommu->seq_id];
1494
1495         BUG_ON(pages == 0);
1496
1497         if (ih)
1498                 ih = 1 << 6;
1499         /*
1500          * Fallback to domain selective flush if no PSI support or the size is
1501          * too big.
1502          * PSI requires page size to be 2 ^ x, and the base address is naturally
1503          * aligned to the size
1504          */
1505         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1506                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1507                                                 DMA_TLB_DSI_FLUSH);
1508         else
1509                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1510                                                 DMA_TLB_PSI_FLUSH);
1511
1512         /*
1513          * In caching mode, changes of pages from non-present to present require
1514          * flush. However, device IOTLB doesn't need to be flushed in this case.
1515          */
1516         if (!cap_caching_mode(iommu->cap) || !map)
1517                 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1518                                       addr, mask);
1519 }
1520
1521 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1522 {
1523         u32 pmen;
1524         unsigned long flags;
1525
1526         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1527         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1528         pmen &= ~DMA_PMEN_EPM;
1529         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1530
1531         /* wait for the protected region status bit to clear */
1532         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1533                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1534
1535         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1536 }
1537
1538 static void iommu_enable_translation(struct intel_iommu *iommu)
1539 {
1540         u32 sts;
1541         unsigned long flags;
1542
1543         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1544         iommu->gcmd |= DMA_GCMD_TE;
1545         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1546
1547         /* Make sure hardware complete it */
1548         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1549                       readl, (sts & DMA_GSTS_TES), sts);
1550
1551         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1552 }
1553
1554 static void iommu_disable_translation(struct intel_iommu *iommu)
1555 {
1556         u32 sts;
1557         unsigned long flag;
1558
1559         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1560         iommu->gcmd &= ~DMA_GCMD_TE;
1561         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1562
1563         /* Make sure hardware complete it */
1564         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1565                       readl, (!(sts & DMA_GSTS_TES)), sts);
1566
1567         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1568 }
1569
1570
1571 static int iommu_init_domains(struct intel_iommu *iommu)
1572 {
1573         u32 ndomains, nlongs;
1574         size_t size;
1575
1576         ndomains = cap_ndoms(iommu->cap);
1577         pr_debug("%s: Number of Domains supported <%d>\n",
1578                  iommu->name, ndomains);
1579         nlongs = BITS_TO_LONGS(ndomains);
1580
1581         spin_lock_init(&iommu->lock);
1582
1583         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1584         if (!iommu->domain_ids) {
1585                 pr_err("%s: Allocating domain id array failed\n",
1586                        iommu->name);
1587                 return -ENOMEM;
1588         }
1589
1590         size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1591         iommu->domains = kzalloc(size, GFP_KERNEL);
1592
1593         if (iommu->domains) {
1594                 size = 256 * sizeof(struct dmar_domain *);
1595                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1596         }
1597
1598         if (!iommu->domains || !iommu->domains[0]) {
1599                 pr_err("%s: Allocating domain array failed\n",
1600                        iommu->name);
1601                 kfree(iommu->domain_ids);
1602                 kfree(iommu->domains);
1603                 iommu->domain_ids = NULL;
1604                 iommu->domains    = NULL;
1605                 return -ENOMEM;
1606         }
1607
1608
1609
1610         /*
1611          * If Caching mode is set, then invalid translations are tagged
1612          * with domain-id 0, hence we need to pre-allocate it. We also
1613          * use domain-id 0 as a marker for non-allocated domain-id, so
1614          * make sure it is not used for a real domain.
1615          */
1616         set_bit(0, iommu->domain_ids);
1617
1618         return 0;
1619 }
1620
1621 static void disable_dmar_iommu(struct intel_iommu *iommu)
1622 {
1623         struct device_domain_info *info, *tmp;
1624         unsigned long flags;
1625
1626         if (!iommu->domains || !iommu->domain_ids)
1627                 return;
1628
1629         spin_lock_irqsave(&device_domain_lock, flags);
1630         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1631                 struct dmar_domain *domain;
1632
1633                 if (info->iommu != iommu)
1634                         continue;
1635
1636                 if (!info->dev || !info->domain)
1637                         continue;
1638
1639                 domain = info->domain;
1640
1641                 dmar_remove_one_dev_info(domain, info->dev);
1642
1643                 if (!domain_type_is_vm_or_si(domain))
1644                         domain_exit(domain);
1645         }
1646         spin_unlock_irqrestore(&device_domain_lock, flags);
1647
1648         if (iommu->gcmd & DMA_GCMD_TE)
1649                 iommu_disable_translation(iommu);
1650 }
1651
1652 static void free_dmar_iommu(struct intel_iommu *iommu)
1653 {
1654         if ((iommu->domains) && (iommu->domain_ids)) {
1655                 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1656                 int i;
1657
1658                 for (i = 0; i < elems; i++)
1659                         kfree(iommu->domains[i]);
1660                 kfree(iommu->domains);
1661                 kfree(iommu->domain_ids);
1662                 iommu->domains = NULL;
1663                 iommu->domain_ids = NULL;
1664         }
1665
1666         g_iommus[iommu->seq_id] = NULL;
1667
1668         /* free context mapping */
1669         free_context_table(iommu);
1670 }
1671
1672 static struct dmar_domain *alloc_domain(int flags)
1673 {
1674         struct dmar_domain *domain;
1675
1676         domain = alloc_domain_mem();
1677         if (!domain)
1678                 return NULL;
1679
1680         memset(domain, 0, sizeof(*domain));
1681         domain->nid = -1;
1682         domain->flags = flags;
1683         INIT_LIST_HEAD(&domain->devices);
1684
1685         return domain;
1686 }
1687
1688 /* Must be called with iommu->lock */
1689 static int domain_attach_iommu(struct dmar_domain *domain,
1690                                struct intel_iommu *iommu)
1691 {
1692         unsigned long ndomains;
1693         int num;
1694
1695         assert_spin_locked(&device_domain_lock);
1696         assert_spin_locked(&iommu->lock);
1697
1698         domain->iommu_refcnt[iommu->seq_id] += 1;
1699         domain->iommu_count += 1;
1700         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1701                 ndomains = cap_ndoms(iommu->cap);
1702                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1703
1704                 if (num >= ndomains) {
1705                         pr_err("%s: No free domain ids\n", iommu->name);
1706                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1707                         domain->iommu_count -= 1;
1708                         return -ENOSPC;
1709                 }
1710
1711                 set_bit(num, iommu->domain_ids);
1712                 set_iommu_domain(iommu, num, domain);
1713
1714                 domain->iommu_did[iommu->seq_id] = num;
1715                 domain->nid                      = iommu->node;
1716
1717                 domain_update_iommu_cap(domain);
1718         }
1719
1720         return 0;
1721 }
1722
1723 static int domain_detach_iommu(struct dmar_domain *domain,
1724                                struct intel_iommu *iommu)
1725 {
1726         int num, count = INT_MAX;
1727
1728         assert_spin_locked(&device_domain_lock);
1729         assert_spin_locked(&iommu->lock);
1730
1731         domain->iommu_refcnt[iommu->seq_id] -= 1;
1732         count = --domain->iommu_count;
1733         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1734                 num = domain->iommu_did[iommu->seq_id];
1735                 clear_bit(num, iommu->domain_ids);
1736                 set_iommu_domain(iommu, num, NULL);
1737
1738                 domain_update_iommu_cap(domain);
1739                 domain->iommu_did[iommu->seq_id] = 0;
1740         }
1741
1742         return count;
1743 }
1744
1745 static struct iova_domain reserved_iova_list;
1746 static struct lock_class_key reserved_rbtree_key;
1747
1748 static int dmar_init_reserved_ranges(void)
1749 {
1750         struct pci_dev *pdev = NULL;
1751         struct iova *iova;
1752         int i;
1753
1754         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1755                         DMA_32BIT_PFN);
1756
1757         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1758                 &reserved_rbtree_key);
1759
1760         /* IOAPIC ranges shouldn't be accessed by DMA */
1761         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1762                 IOVA_PFN(IOAPIC_RANGE_END));
1763         if (!iova) {
1764                 pr_err("Reserve IOAPIC range failed\n");
1765                 return -ENODEV;
1766         }
1767
1768         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1769         for_each_pci_dev(pdev) {
1770                 struct resource *r;
1771
1772                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1773                         r = &pdev->resource[i];
1774                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1775                                 continue;
1776                         iova = reserve_iova(&reserved_iova_list,
1777                                             IOVA_PFN(r->start),
1778                                             IOVA_PFN(r->end));
1779                         if (!iova) {
1780                                 pr_err("Reserve iova failed\n");
1781                                 return -ENODEV;
1782                         }
1783                 }
1784         }
1785         return 0;
1786 }
1787
1788 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1789 {
1790         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1791 }
1792
1793 static inline int guestwidth_to_adjustwidth(int gaw)
1794 {
1795         int agaw;
1796         int r = (gaw - 12) % 9;
1797
1798         if (r == 0)
1799                 agaw = gaw;
1800         else
1801                 agaw = gaw + 9 - r;
1802         if (agaw > 64)
1803                 agaw = 64;
1804         return agaw;
1805 }
1806
1807 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1808                        int guest_width)
1809 {
1810         int adjust_width, agaw;
1811         unsigned long sagaw;
1812
1813         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1814                         DMA_32BIT_PFN);
1815         domain_reserve_special_ranges(domain);
1816
1817         /* calculate AGAW */
1818         if (guest_width > cap_mgaw(iommu->cap))
1819                 guest_width = cap_mgaw(iommu->cap);
1820         domain->gaw = guest_width;
1821         adjust_width = guestwidth_to_adjustwidth(guest_width);
1822         agaw = width_to_agaw(adjust_width);
1823         sagaw = cap_sagaw(iommu->cap);
1824         if (!test_bit(agaw, &sagaw)) {
1825                 /* hardware doesn't support it, choose a bigger one */
1826                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1827                 agaw = find_next_bit(&sagaw, 5, agaw);
1828                 if (agaw >= 5)
1829                         return -ENODEV;
1830         }
1831         domain->agaw = agaw;
1832
1833         if (ecap_coherent(iommu->ecap))
1834                 domain->iommu_coherency = 1;
1835         else
1836                 domain->iommu_coherency = 0;
1837
1838         if (ecap_sc_support(iommu->ecap))
1839                 domain->iommu_snooping = 1;
1840         else
1841                 domain->iommu_snooping = 0;
1842
1843         if (intel_iommu_superpage)
1844                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1845         else
1846                 domain->iommu_superpage = 0;
1847
1848         domain->nid = iommu->node;
1849
1850         /* always allocate the top pgd */
1851         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1852         if (!domain->pgd)
1853                 return -ENOMEM;
1854         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1855         return 0;
1856 }
1857
1858 static void domain_exit(struct dmar_domain *domain)
1859 {
1860         struct page *freelist = NULL;
1861
1862         /* Domain 0 is reserved, so dont process it */
1863         if (!domain)
1864                 return;
1865
1866         /* Flush any lazy unmaps that may reference this domain */
1867         if (!intel_iommu_strict)
1868                 flush_unmaps_timeout(0);
1869
1870         /* Remove associated devices and clear attached or cached domains */
1871         rcu_read_lock();
1872         domain_remove_dev_info(domain);
1873         rcu_read_unlock();
1874
1875         /* destroy iovas */
1876         put_iova_domain(&domain->iovad);
1877
1878         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1879
1880         dma_free_pagelist(freelist);
1881
1882         free_domain_mem(domain);
1883 }
1884
1885 static int domain_context_mapping_one(struct dmar_domain *domain,
1886                                       struct intel_iommu *iommu,
1887                                       u8 bus, u8 devfn)
1888 {
1889         u16 did = domain->iommu_did[iommu->seq_id];
1890         int translation = CONTEXT_TT_MULTI_LEVEL;
1891         struct device_domain_info *info = NULL;
1892         struct context_entry *context;
1893         unsigned long flags;
1894         struct dma_pte *pgd;
1895         int ret, agaw;
1896
1897         WARN_ON(did == 0);
1898
1899         if (hw_pass_through && domain_type_is_si(domain))
1900                 translation = CONTEXT_TT_PASS_THROUGH;
1901
1902         pr_debug("Set context mapping for %02x:%02x.%d\n",
1903                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1904
1905         BUG_ON(!domain->pgd);
1906
1907         spin_lock_irqsave(&device_domain_lock, flags);
1908         spin_lock(&iommu->lock);
1909
1910         ret = -ENOMEM;
1911         context = iommu_context_addr(iommu, bus, devfn, 1);
1912         if (!context)
1913                 goto out_unlock;
1914
1915         ret = 0;
1916         if (context_present(context))
1917                 goto out_unlock;
1918
1919         pgd = domain->pgd;
1920
1921         context_clear_entry(context);
1922         context_set_domain_id(context, did);
1923
1924         /*
1925          * Skip top levels of page tables for iommu which has less agaw
1926          * than default.  Unnecessary for PT mode.
1927          */
1928         if (translation != CONTEXT_TT_PASS_THROUGH) {
1929                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1930                         ret = -ENOMEM;
1931                         pgd = phys_to_virt(dma_pte_addr(pgd));
1932                         if (!dma_pte_present(pgd))
1933                                 goto out_unlock;
1934                 }
1935
1936                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1937                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1938                                      CONTEXT_TT_MULTI_LEVEL;
1939
1940                 context_set_address_root(context, virt_to_phys(pgd));
1941                 context_set_address_width(context, iommu->agaw);
1942         } else {
1943                 /*
1944                  * In pass through mode, AW must be programmed to
1945                  * indicate the largest AGAW value supported by
1946                  * hardware. And ASR is ignored by hardware.
1947                  */
1948                 context_set_address_width(context, iommu->msagaw);
1949         }
1950
1951         context_set_translation_type(context, translation);
1952         context_set_fault_enable(context);
1953         context_set_present(context);
1954         domain_flush_cache(domain, context, sizeof(*context));
1955
1956         /*
1957          * It's a non-present to present mapping. If hardware doesn't cache
1958          * non-present entry we only need to flush the write-buffer. If the
1959          * _does_ cache non-present entries, then it does so in the special
1960          * domain #0, which we have to flush:
1961          */
1962         if (cap_caching_mode(iommu->cap)) {
1963                 iommu->flush.flush_context(iommu, 0,
1964                                            (((u16)bus) << 8) | devfn,
1965                                            DMA_CCMD_MASK_NOBIT,
1966                                            DMA_CCMD_DEVICE_INVL);
1967                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1968         } else {
1969                 iommu_flush_write_buffer(iommu);
1970         }
1971         iommu_enable_dev_iotlb(info);
1972
1973         ret = 0;
1974
1975 out_unlock:
1976         spin_unlock(&iommu->lock);
1977         spin_unlock_irqrestore(&device_domain_lock, flags);
1978
1979         return 0;
1980 }
1981
1982 struct domain_context_mapping_data {
1983         struct dmar_domain *domain;
1984         struct intel_iommu *iommu;
1985 };
1986
1987 static int domain_context_mapping_cb(struct pci_dev *pdev,
1988                                      u16 alias, void *opaque)
1989 {
1990         struct domain_context_mapping_data *data = opaque;
1991
1992         return domain_context_mapping_one(data->domain, data->iommu,
1993                                           PCI_BUS_NUM(alias), alias & 0xff);
1994 }
1995
1996 static int
1997 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
1998 {
1999         struct intel_iommu *iommu;
2000         u8 bus, devfn;
2001         struct domain_context_mapping_data data;
2002
2003         iommu = device_to_iommu(dev, &bus, &devfn);
2004         if (!iommu)
2005                 return -ENODEV;
2006
2007         if (!dev_is_pci(dev))
2008                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2009
2010         data.domain = domain;
2011         data.iommu = iommu;
2012
2013         return pci_for_each_dma_alias(to_pci_dev(dev),
2014                                       &domain_context_mapping_cb, &data);
2015 }
2016
2017 static int domain_context_mapped_cb(struct pci_dev *pdev,
2018                                     u16 alias, void *opaque)
2019 {
2020         struct intel_iommu *iommu = opaque;
2021
2022         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2023 }
2024
2025 static int domain_context_mapped(struct device *dev)
2026 {
2027         struct intel_iommu *iommu;
2028         u8 bus, devfn;
2029
2030         iommu = device_to_iommu(dev, &bus, &devfn);
2031         if (!iommu)
2032                 return -ENODEV;
2033
2034         if (!dev_is_pci(dev))
2035                 return device_context_mapped(iommu, bus, devfn);
2036
2037         return !pci_for_each_dma_alias(to_pci_dev(dev),
2038                                        domain_context_mapped_cb, iommu);
2039 }
2040
2041 /* Returns a number of VTD pages, but aligned to MM page size */
2042 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2043                                             size_t size)
2044 {
2045         host_addr &= ~PAGE_MASK;
2046         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2047 }
2048
2049 /* Return largest possible superpage level for a given mapping */
2050 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2051                                           unsigned long iov_pfn,
2052                                           unsigned long phy_pfn,
2053                                           unsigned long pages)
2054 {
2055         int support, level = 1;
2056         unsigned long pfnmerge;
2057
2058         support = domain->iommu_superpage;
2059
2060         /* To use a large page, the virtual *and* physical addresses
2061            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2062            of them will mean we have to use smaller pages. So just
2063            merge them and check both at once. */
2064         pfnmerge = iov_pfn | phy_pfn;
2065
2066         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2067                 pages >>= VTD_STRIDE_SHIFT;
2068                 if (!pages)
2069                         break;
2070                 pfnmerge >>= VTD_STRIDE_SHIFT;
2071                 level++;
2072                 support--;
2073         }
2074         return level;
2075 }
2076
2077 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2078                             struct scatterlist *sg, unsigned long phys_pfn,
2079                             unsigned long nr_pages, int prot)
2080 {
2081         struct dma_pte *first_pte = NULL, *pte = NULL;
2082         phys_addr_t uninitialized_var(pteval);
2083         unsigned long sg_res = 0;
2084         unsigned int largepage_lvl = 0;
2085         unsigned long lvl_pages = 0;
2086
2087         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2088
2089         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2090                 return -EINVAL;
2091
2092         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2093
2094         if (!sg) {
2095                 sg_res = nr_pages;
2096                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2097         }
2098
2099         while (nr_pages > 0) {
2100                 uint64_t tmp;
2101
2102                 if (!sg_res) {
2103                         sg_res = aligned_nrpages(sg->offset, sg->length);
2104                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2105                         sg->dma_length = sg->length;
2106                         pteval = (sg_phys(sg) & PAGE_MASK) | prot;
2107                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2108                 }
2109
2110                 if (!pte) {
2111                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2112
2113                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2114                         if (!pte)
2115                                 return -ENOMEM;
2116                         /* It is large page*/
2117                         if (largepage_lvl > 1) {
2118                                 pteval |= DMA_PTE_LARGE_PAGE;
2119                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2120                                 /*
2121                                  * Ensure that old small page tables are
2122                                  * removed to make room for superpage,
2123                                  * if they exist.
2124                                  */
2125                                 dma_pte_free_pagetable(domain, iov_pfn,
2126                                                        iov_pfn + lvl_pages - 1);
2127                         } else {
2128                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2129                         }
2130
2131                 }
2132                 /* We don't need lock here, nobody else
2133                  * touches the iova range
2134                  */
2135                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2136                 if (tmp) {
2137                         static int dumps = 5;
2138                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2139                                 iov_pfn, tmp, (unsigned long long)pteval);
2140                         if (dumps) {
2141                                 dumps--;
2142                                 debug_dma_dump_mappings(NULL);
2143                         }
2144                         WARN_ON(1);
2145                 }
2146
2147                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2148
2149                 BUG_ON(nr_pages < lvl_pages);
2150                 BUG_ON(sg_res < lvl_pages);
2151
2152                 nr_pages -= lvl_pages;
2153                 iov_pfn += lvl_pages;
2154                 phys_pfn += lvl_pages;
2155                 pteval += lvl_pages * VTD_PAGE_SIZE;
2156                 sg_res -= lvl_pages;
2157
2158                 /* If the next PTE would be the first in a new page, then we
2159                    need to flush the cache on the entries we've just written.
2160                    And then we'll need to recalculate 'pte', so clear it and
2161                    let it get set again in the if (!pte) block above.
2162
2163                    If we're done (!nr_pages) we need to flush the cache too.
2164
2165                    Also if we've been setting superpages, we may need to
2166                    recalculate 'pte' and switch back to smaller pages for the
2167                    end of the mapping, if the trailing size is not enough to
2168                    use another superpage (i.e. sg_res < lvl_pages). */
2169                 pte++;
2170                 if (!nr_pages || first_pte_in_page(pte) ||
2171                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2172                         domain_flush_cache(domain, first_pte,
2173                                            (void *)pte - (void *)first_pte);
2174                         pte = NULL;
2175                 }
2176
2177                 if (!sg_res && nr_pages)
2178                         sg = sg_next(sg);
2179         }
2180         return 0;
2181 }
2182
2183 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2184                                     struct scatterlist *sg, unsigned long nr_pages,
2185                                     int prot)
2186 {
2187         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2188 }
2189
2190 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2191                                      unsigned long phys_pfn, unsigned long nr_pages,
2192                                      int prot)
2193 {
2194         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2195 }
2196
2197 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2198 {
2199         if (!iommu)
2200                 return;
2201
2202         clear_context_table(iommu, bus, devfn);
2203         iommu->flush.flush_context(iommu, 0, 0, 0,
2204                                            DMA_CCMD_GLOBAL_INVL);
2205         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2206 }
2207
2208 static inline void unlink_domain_info(struct device_domain_info *info)
2209 {
2210         assert_spin_locked(&device_domain_lock);
2211         list_del(&info->link);
2212         list_del(&info->global);
2213         if (info->dev)
2214                 info->dev->archdata.iommu = NULL;
2215 }
2216
2217 static void domain_remove_dev_info(struct dmar_domain *domain)
2218 {
2219         struct device_domain_info *info, *tmp;
2220         unsigned long flags;
2221
2222         spin_lock_irqsave(&device_domain_lock, flags);
2223         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2224                 __dmar_remove_one_dev_info(info);
2225         spin_unlock_irqrestore(&device_domain_lock, flags);
2226 }
2227
2228 /*
2229  * find_domain
2230  * Note: we use struct device->archdata.iommu stores the info
2231  */
2232 static struct dmar_domain *find_domain(struct device *dev)
2233 {
2234         struct device_domain_info *info;
2235
2236         /* No lock here, assumes no domain exit in normal case */
2237         info = dev->archdata.iommu;
2238         if (info)
2239                 return info->domain;
2240         return NULL;
2241 }
2242
2243 static inline struct device_domain_info *
2244 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2245 {
2246         struct device_domain_info *info;
2247
2248         list_for_each_entry(info, &device_domain_list, global)
2249                 if (info->iommu->segment == segment && info->bus == bus &&
2250                     info->devfn == devfn)
2251                         return info;
2252
2253         return NULL;
2254 }
2255
2256 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2257                                                     int bus, int devfn,
2258                                                     struct device *dev,
2259                                                     struct dmar_domain *domain)
2260 {
2261         struct dmar_domain *found = NULL;
2262         struct device_domain_info *info;
2263         unsigned long flags;
2264         int ret;
2265
2266         info = alloc_devinfo_mem();
2267         if (!info)
2268                 return NULL;
2269
2270         info->bus = bus;
2271         info->devfn = devfn;
2272         info->ats.enabled = 0;
2273         info->ats.qdep = 0;
2274         info->dev = dev;
2275         info->domain = domain;
2276         info->iommu = iommu;
2277
2278         spin_lock_irqsave(&device_domain_lock, flags);
2279         if (dev)
2280                 found = find_domain(dev);
2281
2282         if (!found) {
2283                 struct device_domain_info *info2;
2284                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2285                 if (info2) {
2286                         found      = info2->domain;
2287                         info2->dev = dev;
2288                 }
2289         }
2290
2291         if (found) {
2292                 spin_unlock_irqrestore(&device_domain_lock, flags);
2293                 free_devinfo_mem(info);
2294                 /* Caller must free the original domain */
2295                 return found;
2296         }
2297
2298         spin_lock(&iommu->lock);
2299         ret = domain_attach_iommu(domain, iommu);
2300         spin_unlock(&iommu->lock);
2301
2302         if (ret) {
2303                 spin_unlock_irqrestore(&device_domain_lock, flags);
2304                 return NULL;
2305         }
2306
2307         list_add(&info->link, &domain->devices);
2308         list_add(&info->global, &device_domain_list);
2309         if (dev)
2310                 dev->archdata.iommu = info;
2311         spin_unlock_irqrestore(&device_domain_lock, flags);
2312
2313         if (dev && domain_context_mapping(domain, dev)) {
2314                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2315                 dmar_remove_one_dev_info(domain, dev);
2316                 return NULL;
2317         }
2318
2319         return domain;
2320 }
2321
2322 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2323 {
2324         *(u16 *)opaque = alias;
2325         return 0;
2326 }
2327
2328 /* domain is initialized */
2329 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2330 {
2331         struct device_domain_info *info = NULL;
2332         struct dmar_domain *domain, *tmp;
2333         struct intel_iommu *iommu;
2334         u16 req_id, dma_alias;
2335         unsigned long flags;
2336         u8 bus, devfn;
2337
2338         domain = find_domain(dev);
2339         if (domain)
2340                 return domain;
2341
2342         iommu = device_to_iommu(dev, &bus, &devfn);
2343         if (!iommu)
2344                 return NULL;
2345
2346         req_id = ((u16)bus << 8) | devfn;
2347
2348         if (dev_is_pci(dev)) {
2349                 struct pci_dev *pdev = to_pci_dev(dev);
2350
2351                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2352
2353                 spin_lock_irqsave(&device_domain_lock, flags);
2354                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2355                                                       PCI_BUS_NUM(dma_alias),
2356                                                       dma_alias & 0xff);
2357                 if (info) {
2358                         iommu = info->iommu;
2359                         domain = info->domain;
2360                 }
2361                 spin_unlock_irqrestore(&device_domain_lock, flags);
2362
2363                 /* DMA alias already has a domain, uses it */
2364                 if (info)
2365                         goto found_domain;
2366         }
2367
2368         /* Allocate and initialize new domain for the device */
2369         domain = alloc_domain(0);
2370         if (!domain)
2371                 return NULL;
2372         if (domain_init(domain, iommu, gaw)) {
2373                 domain_exit(domain);
2374                 return NULL;
2375         }
2376
2377         /* register PCI DMA alias device */
2378         if (req_id != dma_alias && dev_is_pci(dev)) {
2379                 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2380                                                dma_alias & 0xff, NULL, domain);
2381
2382                 if (!tmp || tmp != domain) {
2383                         domain_exit(domain);
2384                         domain = tmp;
2385                 }
2386
2387                 if (!domain)
2388                         return NULL;
2389         }
2390
2391 found_domain:
2392         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2393
2394         if (!tmp || tmp != domain) {
2395                 domain_exit(domain);
2396                 domain = tmp;
2397         }
2398
2399         return domain;
2400 }
2401
2402 static int iommu_identity_mapping;
2403 #define IDENTMAP_ALL            1
2404 #define IDENTMAP_GFX            2
2405 #define IDENTMAP_AZALIA         4
2406
2407 static int iommu_domain_identity_map(struct dmar_domain *domain,
2408                                      unsigned long long start,
2409                                      unsigned long long end)
2410 {
2411         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2412         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2413
2414         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2415                           dma_to_mm_pfn(last_vpfn))) {
2416                 pr_err("Reserving iova failed\n");
2417                 return -ENOMEM;
2418         }
2419
2420         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2421         /*
2422          * RMRR range might have overlap with physical memory range,
2423          * clear it first
2424          */
2425         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2426
2427         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2428                                   last_vpfn - first_vpfn + 1,
2429                                   DMA_PTE_READ|DMA_PTE_WRITE);
2430 }
2431
2432 static int iommu_prepare_identity_map(struct device *dev,
2433                                       unsigned long long start,
2434                                       unsigned long long end)
2435 {
2436         struct dmar_domain *domain;
2437         int ret;
2438
2439         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2440         if (!domain)
2441                 return -ENOMEM;
2442
2443         /* For _hardware_ passthrough, don't bother. But for software
2444            passthrough, we do it anyway -- it may indicate a memory
2445            range which is reserved in E820, so which didn't get set
2446            up to start with in si_domain */
2447         if (domain == si_domain && hw_pass_through) {
2448                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2449                         dev_name(dev), start, end);
2450                 return 0;
2451         }
2452
2453         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2454                 dev_name(dev), start, end);
2455
2456         if (end < start) {
2457                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2458                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2459                         dmi_get_system_info(DMI_BIOS_VENDOR),
2460                         dmi_get_system_info(DMI_BIOS_VERSION),
2461                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2462                 ret = -EIO;
2463                 goto error;
2464         }
2465
2466         if (end >> agaw_to_width(domain->agaw)) {
2467                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2468                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2469                      agaw_to_width(domain->agaw),
2470                      dmi_get_system_info(DMI_BIOS_VENDOR),
2471                      dmi_get_system_info(DMI_BIOS_VERSION),
2472                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2473                 ret = -EIO;
2474                 goto error;
2475         }
2476
2477         ret = iommu_domain_identity_map(domain, start, end);
2478         if (ret)
2479                 goto error;
2480
2481         return 0;
2482
2483  error:
2484         domain_exit(domain);
2485         return ret;
2486 }
2487
2488 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2489                                          struct device *dev)
2490 {
2491         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2492                 return 0;
2493         return iommu_prepare_identity_map(dev, rmrr->base_address,
2494                                           rmrr->end_address);
2495 }
2496
2497 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2498 static inline void iommu_prepare_isa(void)
2499 {
2500         struct pci_dev *pdev;
2501         int ret;
2502
2503         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2504         if (!pdev)
2505                 return;
2506
2507         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2508         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2509
2510         if (ret)
2511                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2512
2513         pci_dev_put(pdev);
2514 }
2515 #else
2516 static inline void iommu_prepare_isa(void)
2517 {
2518         return;
2519 }
2520 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2521
2522 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2523
2524 static int __init si_domain_init(int hw)
2525 {
2526         int nid, ret = 0;
2527
2528         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2529         if (!si_domain)
2530                 return -EFAULT;
2531
2532         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2533                 domain_exit(si_domain);
2534                 return -EFAULT;
2535         }
2536
2537         pr_debug("Identity mapping domain allocated\n");
2538
2539         if (hw)
2540                 return 0;
2541
2542         for_each_online_node(nid) {
2543                 unsigned long start_pfn, end_pfn;
2544                 int i;
2545
2546                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2547                         ret = iommu_domain_identity_map(si_domain,
2548                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2549                         if (ret)
2550                                 return ret;
2551                 }
2552         }
2553
2554         return 0;
2555 }
2556
2557 static int identity_mapping(struct device *dev)
2558 {
2559         struct device_domain_info *info;
2560
2561         if (likely(!iommu_identity_mapping))
2562                 return 0;
2563
2564         info = dev->archdata.iommu;
2565         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2566                 return (info->domain == si_domain);
2567
2568         return 0;
2569 }
2570
2571 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2572 {
2573         struct dmar_domain *ndomain;
2574         struct intel_iommu *iommu;
2575         u8 bus, devfn;
2576
2577         iommu = device_to_iommu(dev, &bus, &devfn);
2578         if (!iommu)
2579                 return -ENODEV;
2580
2581         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2582         if (ndomain != domain)
2583                 return -EBUSY;
2584
2585         return 0;
2586 }
2587
2588 static bool device_has_rmrr(struct device *dev)
2589 {
2590         struct dmar_rmrr_unit *rmrr;
2591         struct device *tmp;
2592         int i;
2593
2594         rcu_read_lock();
2595         for_each_rmrr_units(rmrr) {
2596                 /*
2597                  * Return TRUE if this RMRR contains the device that
2598                  * is passed in.
2599                  */
2600                 for_each_active_dev_scope(rmrr->devices,
2601                                           rmrr->devices_cnt, i, tmp)
2602                         if (tmp == dev) {
2603                                 rcu_read_unlock();
2604                                 return true;
2605                         }
2606         }
2607         rcu_read_unlock();
2608         return false;
2609 }
2610
2611 /*
2612  * There are a couple cases where we need to restrict the functionality of
2613  * devices associated with RMRRs.  The first is when evaluating a device for
2614  * identity mapping because problems exist when devices are moved in and out
2615  * of domains and their respective RMRR information is lost.  This means that
2616  * a device with associated RMRRs will never be in a "passthrough" domain.
2617  * The second is use of the device through the IOMMU API.  This interface
2618  * expects to have full control of the IOVA space for the device.  We cannot
2619  * satisfy both the requirement that RMRR access is maintained and have an
2620  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2621  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2622  * We therefore prevent devices associated with an RMRR from participating in
2623  * the IOMMU API, which eliminates them from device assignment.
2624  *
2625  * In both cases we assume that PCI USB devices with RMRRs have them largely
2626  * for historical reasons and that the RMRR space is not actively used post
2627  * boot.  This exclusion may change if vendors begin to abuse it.
2628  *
2629  * The same exception is made for graphics devices, with the requirement that
2630  * any use of the RMRR regions will be torn down before assigning the device
2631  * to a guest.
2632  */
2633 static bool device_is_rmrr_locked(struct device *dev)
2634 {
2635         if (!device_has_rmrr(dev))
2636                 return false;
2637
2638         if (dev_is_pci(dev)) {
2639                 struct pci_dev *pdev = to_pci_dev(dev);
2640
2641                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2642                         return false;
2643         }
2644
2645         return true;
2646 }
2647
2648 static int iommu_should_identity_map(struct device *dev, int startup)
2649 {
2650
2651         if (dev_is_pci(dev)) {
2652                 struct pci_dev *pdev = to_pci_dev(dev);
2653
2654                 if (device_is_rmrr_locked(dev))
2655                         return 0;
2656
2657                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2658                         return 1;
2659
2660                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2661                         return 1;
2662
2663                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2664                         return 0;
2665
2666                 /*
2667                  * We want to start off with all devices in the 1:1 domain, and
2668                  * take them out later if we find they can't access all of memory.
2669                  *
2670                  * However, we can't do this for PCI devices behind bridges,
2671                  * because all PCI devices behind the same bridge will end up
2672                  * with the same source-id on their transactions.
2673                  *
2674                  * Practically speaking, we can't change things around for these
2675                  * devices at run-time, because we can't be sure there'll be no
2676                  * DMA transactions in flight for any of their siblings.
2677                  *
2678                  * So PCI devices (unless they're on the root bus) as well as
2679                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2680                  * the 1:1 domain, just in _case_ one of their siblings turns out
2681                  * not to be able to map all of memory.
2682                  */
2683                 if (!pci_is_pcie(pdev)) {
2684                         if (!pci_is_root_bus(pdev->bus))
2685                                 return 0;
2686                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2687                                 return 0;
2688                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2689                         return 0;
2690         } else {
2691                 if (device_has_rmrr(dev))
2692                         return 0;
2693         }
2694
2695         /*
2696          * At boot time, we don't yet know if devices will be 64-bit capable.
2697          * Assume that they will — if they turn out not to be, then we can
2698          * take them out of the 1:1 domain later.
2699          */
2700         if (!startup) {
2701                 /*
2702                  * If the device's dma_mask is less than the system's memory
2703                  * size then this is not a candidate for identity mapping.
2704                  */
2705                 u64 dma_mask = *dev->dma_mask;
2706
2707                 if (dev->coherent_dma_mask &&
2708                     dev->coherent_dma_mask < dma_mask)
2709                         dma_mask = dev->coherent_dma_mask;
2710
2711                 return dma_mask >= dma_get_required_mask(dev);
2712         }
2713
2714         return 1;
2715 }
2716
2717 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2718 {
2719         int ret;
2720
2721         if (!iommu_should_identity_map(dev, 1))
2722                 return 0;
2723
2724         ret = domain_add_dev_info(si_domain, dev);
2725         if (!ret)
2726                 pr_info("%s identity mapping for device %s\n",
2727                         hw ? "Hardware" : "Software", dev_name(dev));
2728         else if (ret == -ENODEV)
2729                 /* device not associated with an iommu */
2730                 ret = 0;
2731
2732         return ret;
2733 }
2734
2735
2736 static int __init iommu_prepare_static_identity_mapping(int hw)
2737 {
2738         struct pci_dev *pdev = NULL;
2739         struct dmar_drhd_unit *drhd;
2740         struct intel_iommu *iommu;
2741         struct device *dev;
2742         int i;
2743         int ret = 0;
2744
2745         for_each_pci_dev(pdev) {
2746                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2747                 if (ret)
2748                         return ret;
2749         }
2750
2751         for_each_active_iommu(iommu, drhd)
2752                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2753                         struct acpi_device_physical_node *pn;
2754                         struct acpi_device *adev;
2755
2756                         if (dev->bus != &acpi_bus_type)
2757                                 continue;
2758
2759                         adev= to_acpi_device(dev);
2760                         mutex_lock(&adev->physical_node_lock);
2761                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2762                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2763                                 if (ret)
2764                                         break;
2765                         }
2766                         mutex_unlock(&adev->physical_node_lock);
2767                         if (ret)
2768                                 return ret;
2769                 }
2770
2771         return 0;
2772 }
2773
2774 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2775 {
2776         /*
2777          * Start from the sane iommu hardware state.
2778          * If the queued invalidation is already initialized by us
2779          * (for example, while enabling interrupt-remapping) then
2780          * we got the things already rolling from a sane state.
2781          */
2782         if (!iommu->qi) {
2783                 /*
2784                  * Clear any previous faults.
2785                  */
2786                 dmar_fault(-1, iommu);
2787                 /*
2788                  * Disable queued invalidation if supported and already enabled
2789                  * before OS handover.
2790                  */
2791                 dmar_disable_qi(iommu);
2792         }
2793
2794         if (dmar_enable_qi(iommu)) {
2795                 /*
2796                  * Queued Invalidate not enabled, use Register Based Invalidate
2797                  */
2798                 iommu->flush.flush_context = __iommu_flush_context;
2799                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2800                 pr_info("%s: Using Register based invalidation\n",
2801                         iommu->name);
2802         } else {
2803                 iommu->flush.flush_context = qi_flush_context;
2804                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2805                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2806         }
2807 }
2808
2809 static int copy_context_table(struct intel_iommu *iommu,
2810                               struct root_entry __iomem *old_re,
2811                               struct context_entry **tbl,
2812                               int bus, bool ext)
2813 {
2814         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2815         struct context_entry __iomem *old_ce = NULL;
2816         struct context_entry *new_ce = NULL, ce;
2817         struct root_entry re;
2818         phys_addr_t old_ce_phys;
2819
2820         tbl_idx = ext ? bus * 2 : bus;
2821         memcpy_fromio(&re, old_re, sizeof(re));
2822
2823         for (devfn = 0; devfn < 256; devfn++) {
2824                 /* First calculate the correct index */
2825                 idx = (ext ? devfn * 2 : devfn) % 256;
2826
2827                 if (idx == 0) {
2828                         /* First save what we may have and clean up */
2829                         if (new_ce) {
2830                                 tbl[tbl_idx] = new_ce;
2831                                 __iommu_flush_cache(iommu, new_ce,
2832                                                     VTD_PAGE_SIZE);
2833                                 pos = 1;
2834                         }
2835
2836                         if (old_ce)
2837                                 iounmap(old_ce);
2838
2839                         ret = 0;
2840                         if (devfn < 0x80)
2841                                 old_ce_phys = root_entry_lctp(&re);
2842                         else
2843                                 old_ce_phys = root_entry_uctp(&re);
2844
2845                         if (!old_ce_phys) {
2846                                 if (ext && devfn == 0) {
2847                                         /* No LCTP, try UCTP */
2848                                         devfn = 0x7f;
2849                                         continue;
2850                                 } else {
2851                                         goto out;
2852                                 }
2853                         }
2854
2855                         ret = -ENOMEM;
2856                         old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2857                         if (!old_ce)
2858                                 goto out;
2859
2860                         new_ce = alloc_pgtable_page(iommu->node);
2861                         if (!new_ce)
2862                                 goto out_unmap;
2863
2864                         ret = 0;
2865                 }
2866
2867                 /* Now copy the context entry */
2868                 memcpy_fromio(&ce, old_ce + idx, sizeof(ce));
2869
2870                 if (!__context_present(&ce))
2871                         continue;
2872
2873                 did = context_domain_id(&ce);
2874                 if (did >= 0 && did < cap_ndoms(iommu->cap))
2875                         set_bit(did, iommu->domain_ids);
2876
2877                 /*
2878                  * We need a marker for copied context entries. This
2879                  * marker needs to work for the old format as well as
2880                  * for extended context entries.
2881                  *
2882                  * Bit 67 of the context entry is used. In the old
2883                  * format this bit is available to software, in the
2884                  * extended format it is the PGE bit, but PGE is ignored
2885                  * by HW if PASIDs are disabled (and thus still
2886                  * available).
2887                  *
2888                  * So disable PASIDs first and then mark the entry
2889                  * copied. This means that we don't copy PASID
2890                  * translations from the old kernel, but this is fine as
2891                  * faults there are not fatal.
2892                  */
2893                 context_clear_pasid_enable(&ce);
2894                 context_set_copied(&ce);
2895
2896                 new_ce[idx] = ce;
2897         }
2898
2899         tbl[tbl_idx + pos] = new_ce;
2900
2901         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2902
2903 out_unmap:
2904         iounmap(old_ce);
2905
2906 out:
2907         return ret;
2908 }
2909
2910 static int copy_translation_tables(struct intel_iommu *iommu)
2911 {
2912         struct root_entry __iomem *old_rt;
2913         struct context_entry **ctxt_tbls;
2914         phys_addr_t old_rt_phys;
2915         int ctxt_table_entries;
2916         unsigned long flags;
2917         u64 rtaddr_reg;
2918         int bus, ret;
2919         bool new_ext, ext;
2920
2921         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2922         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
2923         new_ext    = !!ecap_ecs(iommu->ecap);
2924
2925         /*
2926          * The RTT bit can only be changed when translation is disabled,
2927          * but disabling translation means to open a window for data
2928          * corruption. So bail out and don't copy anything if we would
2929          * have to change the bit.
2930          */
2931         if (new_ext != ext)
2932                 return -EINVAL;
2933
2934         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2935         if (!old_rt_phys)
2936                 return -EINVAL;
2937
2938         old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2939         if (!old_rt)
2940                 return -ENOMEM;
2941
2942         /* This is too big for the stack - allocate it from slab */
2943         ctxt_table_entries = ext ? 512 : 256;
2944         ret = -ENOMEM;
2945         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2946         if (!ctxt_tbls)
2947                 goto out_unmap;
2948
2949         for (bus = 0; bus < 256; bus++) {
2950                 ret = copy_context_table(iommu, &old_rt[bus],
2951                                          ctxt_tbls, bus, ext);
2952                 if (ret) {
2953                         pr_err("%s: Failed to copy context table for bus %d\n",
2954                                 iommu->name, bus);
2955                         continue;
2956                 }
2957         }
2958
2959         spin_lock_irqsave(&iommu->lock, flags);
2960
2961         /* Context tables are copied, now write them to the root_entry table */
2962         for (bus = 0; bus < 256; bus++) {
2963                 int idx = ext ? bus * 2 : bus;
2964                 u64 val;
2965
2966                 if (ctxt_tbls[idx]) {
2967                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
2968                         iommu->root_entry[bus].lo = val;
2969                 }
2970
2971                 if (!ext || !ctxt_tbls[idx + 1])
2972                         continue;
2973
2974                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2975                 iommu->root_entry[bus].hi = val;
2976         }
2977
2978         spin_unlock_irqrestore(&iommu->lock, flags);
2979
2980         kfree(ctxt_tbls);
2981
2982         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2983
2984         ret = 0;
2985
2986 out_unmap:
2987         iounmap(old_rt);
2988
2989         return ret;
2990 }
2991
2992 static int __init init_dmars(void)
2993 {
2994         struct dmar_drhd_unit *drhd;
2995         struct dmar_rmrr_unit *rmrr;
2996         bool copied_tables = false;
2997         struct device *dev;
2998         struct intel_iommu *iommu;
2999         int i, ret;
3000
3001         /*
3002          * for each drhd
3003          *    allocate root
3004          *    initialize and program root entry to not present
3005          * endfor
3006          */
3007         for_each_drhd_unit(drhd) {
3008                 /*
3009                  * lock not needed as this is only incremented in the single
3010                  * threaded kernel __init code path all other access are read
3011                  * only
3012                  */
3013                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3014                         g_num_of_iommus++;
3015                         continue;
3016                 }
3017                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3018         }
3019
3020         /* Preallocate enough resources for IOMMU hot-addition */
3021         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3022                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3023
3024         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3025                         GFP_KERNEL);
3026         if (!g_iommus) {
3027                 pr_err("Allocating global iommu array failed\n");
3028                 ret = -ENOMEM;
3029                 goto error;
3030         }
3031
3032         deferred_flush = kzalloc(g_num_of_iommus *
3033                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3034         if (!deferred_flush) {
3035                 ret = -ENOMEM;
3036                 goto free_g_iommus;
3037         }
3038
3039         for_each_active_iommu(iommu, drhd) {
3040                 g_iommus[iommu->seq_id] = iommu;
3041
3042                 intel_iommu_init_qi(iommu);
3043
3044                 ret = iommu_init_domains(iommu);
3045                 if (ret)
3046                         goto free_iommu;
3047
3048                 init_translation_status(iommu);
3049
3050                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3051                         iommu_disable_translation(iommu);
3052                         clear_translation_pre_enabled(iommu);
3053                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3054                                 iommu->name);
3055                 }
3056
3057                 /*
3058                  * TBD:
3059                  * we could share the same root & context tables
3060                  * among all IOMMU's. Need to Split it later.
3061                  */
3062                 ret = iommu_alloc_root_entry(iommu);
3063                 if (ret)
3064                         goto free_iommu;
3065
3066                 if (translation_pre_enabled(iommu)) {
3067                         pr_info("Translation already enabled - trying to copy translation structures\n");
3068
3069                         ret = copy_translation_tables(iommu);
3070                         if (ret) {
3071                                 /*
3072                                  * We found the IOMMU with translation
3073                                  * enabled - but failed to copy over the
3074                                  * old root-entry table. Try to proceed
3075                                  * by disabling translation now and
3076                                  * allocating a clean root-entry table.
3077                                  * This might cause DMAR faults, but
3078                                  * probably the dump will still succeed.
3079                                  */
3080                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3081                                        iommu->name);
3082                                 iommu_disable_translation(iommu);
3083                                 clear_translation_pre_enabled(iommu);
3084                         } else {
3085                                 pr_info("Copied translation tables from previous kernel for %s\n",
3086                                         iommu->name);
3087                                 copied_tables = true;
3088                         }
3089                 }
3090
3091                 iommu_flush_write_buffer(iommu);
3092                 iommu_set_root_entry(iommu);
3093                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3094                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3095
3096                 if (!ecap_pass_through(iommu->ecap))
3097                         hw_pass_through = 0;
3098         }
3099
3100         if (iommu_pass_through)
3101                 iommu_identity_mapping |= IDENTMAP_ALL;
3102
3103 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3104         iommu_identity_mapping |= IDENTMAP_GFX;
3105 #endif
3106
3107         if (iommu_identity_mapping) {
3108                 ret = si_domain_init(hw_pass_through);
3109                 if (ret)
3110                         goto free_iommu;
3111         }
3112
3113         check_tylersburg_isoch();
3114
3115         /*
3116          * If we copied translations from a previous kernel in the kdump
3117          * case, we can not assign the devices to domains now, as that
3118          * would eliminate the old mappings. So skip this part and defer
3119          * the assignment to device driver initialization time.
3120          */
3121         if (copied_tables)
3122                 goto domains_done;
3123
3124         /*
3125          * If pass through is not set or not enabled, setup context entries for
3126          * identity mappings for rmrr, gfx, and isa and may fall back to static
3127          * identity mapping if iommu_identity_mapping is set.
3128          */
3129         if (iommu_identity_mapping) {
3130                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3131                 if (ret) {
3132                         pr_crit("Failed to setup IOMMU pass-through\n");
3133                         goto free_iommu;
3134                 }
3135         }
3136         /*
3137          * For each rmrr
3138          *   for each dev attached to rmrr
3139          *   do
3140          *     locate drhd for dev, alloc domain for dev
3141          *     allocate free domain
3142          *     allocate page table entries for rmrr
3143          *     if context not allocated for bus
3144          *           allocate and init context
3145          *           set present in root table for this bus
3146          *     init context with domain, translation etc
3147          *    endfor
3148          * endfor
3149          */
3150         pr_info("Setting RMRR:\n");
3151         for_each_rmrr_units(rmrr) {
3152                 /* some BIOS lists non-exist devices in DMAR table. */
3153                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3154                                           i, dev) {
3155                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3156                         if (ret)
3157                                 pr_err("Mapping reserved region failed\n");
3158                 }
3159         }
3160
3161         iommu_prepare_isa();
3162
3163 domains_done:
3164
3165         /*
3166          * for each drhd
3167          *   enable fault log
3168          *   global invalidate context cache
3169          *   global invalidate iotlb
3170          *   enable translation
3171          */
3172         for_each_iommu(iommu, drhd) {
3173                 if (drhd->ignored) {
3174                         /*
3175                          * we always have to disable PMRs or DMA may fail on
3176                          * this device
3177                          */
3178                         if (force_on)
3179                                 iommu_disable_protect_mem_regions(iommu);
3180                         continue;
3181                 }
3182
3183                 iommu_flush_write_buffer(iommu);
3184
3185                 ret = dmar_set_interrupt(iommu);
3186                 if (ret)
3187                         goto free_iommu;
3188
3189                 if (!translation_pre_enabled(iommu))
3190                         iommu_enable_translation(iommu);
3191
3192                 iommu_disable_protect_mem_regions(iommu);
3193         }
3194
3195         return 0;
3196
3197 free_iommu:
3198         for_each_active_iommu(iommu, drhd) {
3199                 disable_dmar_iommu(iommu);
3200                 free_dmar_iommu(iommu);
3201         }
3202         kfree(deferred_flush);
3203 free_g_iommus:
3204         kfree(g_iommus);
3205 error:
3206         return ret;
3207 }
3208
3209 /* This takes a number of _MM_ pages, not VTD pages */
3210 static struct iova *intel_alloc_iova(struct device *dev,
3211                                      struct dmar_domain *domain,
3212                                      unsigned long nrpages, uint64_t dma_mask)
3213 {
3214         struct iova *iova = NULL;
3215
3216         /* Restrict dma_mask to the width that the iommu can handle */
3217         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3218         /* Ensure we reserve the whole size-aligned region */
3219         nrpages = __roundup_pow_of_two(nrpages);
3220
3221         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3222                 /*
3223                  * First try to allocate an io virtual address in
3224                  * DMA_BIT_MASK(32) and if that fails then try allocating
3225                  * from higher range
3226                  */
3227                 iova = alloc_iova(&domain->iovad, nrpages,
3228                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
3229                 if (iova)
3230                         return iova;
3231         }
3232         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3233         if (unlikely(!iova)) {
3234                 pr_err("Allocating %ld-page iova for %s failed",
3235                        nrpages, dev_name(dev));
3236                 return NULL;
3237         }
3238
3239         return iova;
3240 }
3241
3242 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3243 {
3244         struct dmar_domain *domain;
3245
3246         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3247         if (!domain) {
3248                 pr_err("Allocating domain for %s failed\n",
3249                        dev_name(dev));
3250                 return NULL;
3251         }
3252
3253         return domain;
3254 }
3255
3256 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3257 {
3258         struct device_domain_info *info;
3259
3260         /* No lock here, assumes no domain exit in normal case */
3261         info = dev->archdata.iommu;
3262         if (likely(info))
3263                 return info->domain;
3264
3265         return __get_valid_domain_for_dev(dev);
3266 }
3267
3268 /* Check if the dev needs to go through non-identity map and unmap process.*/
3269 static int iommu_no_mapping(struct device *dev)
3270 {
3271         int found;
3272
3273         if (iommu_dummy(dev))
3274                 return 1;
3275
3276         if (!iommu_identity_mapping)
3277                 return 0;
3278
3279         found = identity_mapping(dev);
3280         if (found) {
3281                 if (iommu_should_identity_map(dev, 0))
3282                         return 1;
3283                 else {
3284                         /*
3285                          * 32 bit DMA is removed from si_domain and fall back
3286                          * to non-identity mapping.
3287                          */
3288                         dmar_remove_one_dev_info(si_domain, dev);
3289                         pr_info("32bit %s uses non-identity mapping\n",
3290                                 dev_name(dev));
3291                         return 0;
3292                 }
3293         } else {
3294                 /*
3295                  * In case of a detached 64 bit DMA device from vm, the device
3296                  * is put into si_domain for identity mapping.
3297                  */
3298                 if (iommu_should_identity_map(dev, 0)) {
3299                         int ret;
3300                         ret = domain_add_dev_info(si_domain, dev);
3301                         if (!ret) {
3302                                 pr_info("64bit %s uses identity mapping\n",
3303                                         dev_name(dev));
3304                                 return 1;
3305                         }
3306                 }
3307         }
3308
3309         return 0;
3310 }
3311
3312 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3313                                      size_t size, int dir, u64 dma_mask)
3314 {
3315         struct dmar_domain *domain;
3316         phys_addr_t start_paddr;
3317         struct iova *iova;
3318         int prot = 0;
3319         int ret;
3320         struct intel_iommu *iommu;
3321         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3322
3323         BUG_ON(dir == DMA_NONE);
3324
3325         if (iommu_no_mapping(dev))
3326                 return paddr;
3327
3328         domain = get_valid_domain_for_dev(dev);
3329         if (!domain)
3330                 return 0;
3331
3332         iommu = domain_get_iommu(domain);
3333         size = aligned_nrpages(paddr, size);
3334
3335         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3336         if (!iova)
3337                 goto error;
3338
3339         /*
3340          * Check if DMAR supports zero-length reads on write only
3341          * mappings..
3342          */
3343         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3344                         !cap_zlr(iommu->cap))
3345                 prot |= DMA_PTE_READ;
3346         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3347                 prot |= DMA_PTE_WRITE;
3348         /*
3349          * paddr - (paddr + size) might be partial page, we should map the whole
3350          * page.  Note: if two part of one page are separately mapped, we
3351          * might have two guest_addr mapping to the same host paddr, but this
3352          * is not a big problem
3353          */
3354         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3355                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3356         if (ret)
3357                 goto error;
3358
3359         /* it's a non-present to present mapping. Only flush if caching mode */
3360         if (cap_caching_mode(iommu->cap))
3361                 iommu_flush_iotlb_psi(iommu, domain,
3362                                       mm_to_dma_pfn(iova->pfn_lo),
3363                                       size, 0, 1);
3364         else
3365                 iommu_flush_write_buffer(iommu);
3366
3367         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3368         start_paddr += paddr & ~PAGE_MASK;
3369         return start_paddr;
3370
3371 error:
3372         if (iova)
3373                 __free_iova(&domain->iovad, iova);
3374         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3375                 dev_name(dev), size, (unsigned long long)paddr, dir);
3376         return 0;
3377 }
3378
3379 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3380                                  unsigned long offset, size_t size,
3381                                  enum dma_data_direction dir,
3382                                  struct dma_attrs *attrs)
3383 {
3384         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3385                                   dir, *dev->dma_mask);
3386 }
3387
3388 static void flush_unmaps(void)
3389 {
3390         int i, j;
3391
3392         timer_on = 0;
3393
3394         /* just flush them all */
3395         for (i = 0; i < g_num_of_iommus; i++) {
3396                 struct intel_iommu *iommu = g_iommus[i];
3397                 if (!iommu)
3398                         continue;
3399
3400                 if (!deferred_flush[i].next)
3401                         continue;
3402
3403                 /* In caching mode, global flushes turn emulation expensive */
3404                 if (!cap_caching_mode(iommu->cap))
3405                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3406                                          DMA_TLB_GLOBAL_FLUSH);
3407                 for (j = 0; j < deferred_flush[i].next; j++) {
3408                         unsigned long mask;
3409                         struct iova *iova = deferred_flush[i].iova[j];
3410                         struct dmar_domain *domain = deferred_flush[i].domain[j];
3411
3412                         /* On real hardware multiple invalidations are expensive */
3413                         if (cap_caching_mode(iommu->cap))
3414                                 iommu_flush_iotlb_psi(iommu, domain,
3415                                         iova->pfn_lo, iova_size(iova),
3416                                         !deferred_flush[i].freelist[j], 0);
3417                         else {
3418                                 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3419                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3420                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3421                         }
3422                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3423                         if (deferred_flush[i].freelist[j])
3424                                 dma_free_pagelist(deferred_flush[i].freelist[j]);
3425                 }
3426                 deferred_flush[i].next = 0;
3427         }
3428
3429         list_size = 0;
3430 }
3431
3432 static void flush_unmaps_timeout(unsigned long data)
3433 {
3434         unsigned long flags;
3435
3436         spin_lock_irqsave(&async_umap_flush_lock, flags);
3437         flush_unmaps();
3438         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3439 }
3440
3441 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3442 {
3443         unsigned long flags;
3444         int next, iommu_id;
3445         struct intel_iommu *iommu;
3446
3447         spin_lock_irqsave(&async_umap_flush_lock, flags);
3448         if (list_size == HIGH_WATER_MARK)
3449                 flush_unmaps();
3450
3451         iommu = domain_get_iommu(dom);
3452         iommu_id = iommu->seq_id;
3453
3454         next = deferred_flush[iommu_id].next;
3455         deferred_flush[iommu_id].domain[next] = dom;
3456         deferred_flush[iommu_id].iova[next] = iova;
3457         deferred_flush[iommu_id].freelist[next] = freelist;
3458         deferred_flush[iommu_id].next++;
3459
3460         if (!timer_on) {
3461                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3462                 timer_on = 1;
3463         }
3464         list_size++;
3465         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3466 }
3467
3468 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3469 {
3470         struct dmar_domain *domain;
3471         unsigned long start_pfn, last_pfn;
3472         struct iova *iova;
3473         struct intel_iommu *iommu;
3474         struct page *freelist;
3475
3476         if (iommu_no_mapping(dev))
3477                 return;
3478
3479         domain = find_domain(dev);
3480         BUG_ON(!domain);
3481
3482         iommu = domain_get_iommu(domain);
3483
3484         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3485         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3486                       (unsigned long long)dev_addr))
3487                 return;
3488
3489         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3490         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3491
3492         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3493                  dev_name(dev), start_pfn, last_pfn);
3494
3495         freelist = domain_unmap(domain, start_pfn, last_pfn);
3496
3497         if (intel_iommu_strict) {
3498                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3499                                       last_pfn - start_pfn + 1, !freelist, 0);
3500                 /* free iova */
3501                 __free_iova(&domain->iovad, iova);
3502                 dma_free_pagelist(freelist);
3503         } else {
3504                 add_unmap(domain, iova, freelist);
3505                 /*
3506                  * queue up the release of the unmap to save the 1/6th of the
3507                  * cpu used up by the iotlb flush operation...
3508                  */
3509         }
3510 }
3511
3512 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3513                              size_t size, enum dma_data_direction dir,
3514                              struct dma_attrs *attrs)
3515 {
3516         intel_unmap(dev, dev_addr);
3517 }
3518
3519 static void *intel_alloc_coherent(struct device *dev, size_t size,
3520                                   dma_addr_t *dma_handle, gfp_t flags,
3521                                   struct dma_attrs *attrs)
3522 {
3523         struct page *page = NULL;
3524         int order;
3525
3526         size = PAGE_ALIGN(size);
3527         order = get_order(size);
3528
3529         if (!iommu_no_mapping(dev))
3530                 flags &= ~(GFP_DMA | GFP_DMA32);
3531         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3532                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3533                         flags |= GFP_DMA;
3534                 else
3535                         flags |= GFP_DMA32;
3536         }
3537
3538         if (flags & __GFP_WAIT) {
3539                 unsigned int count = size >> PAGE_SHIFT;
3540
3541                 page = dma_alloc_from_contiguous(dev, count, order);
3542                 if (page && iommu_no_mapping(dev) &&
3543                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3544                         dma_release_from_contiguous(dev, page, count);
3545                         page = NULL;
3546                 }
3547         }
3548
3549         if (!page)
3550                 page = alloc_pages(flags, order);
3551         if (!page)
3552                 return NULL;
3553         memset(page_address(page), 0, size);
3554
3555         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3556                                          DMA_BIDIRECTIONAL,
3557                                          dev->coherent_dma_mask);
3558         if (*dma_handle)
3559                 return page_address(page);
3560         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3561                 __free_pages(page, order);
3562
3563         return NULL;
3564 }
3565
3566 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3567                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
3568 {
3569         int order;
3570         struct page *page = virt_to_page(vaddr);
3571
3572         size = PAGE_ALIGN(size);
3573         order = get_order(size);
3574
3575         intel_unmap(dev, dma_handle);
3576         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3577                 __free_pages(page, order);
3578 }
3579
3580 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3581                            int nelems, enum dma_data_direction dir,
3582                            struct dma_attrs *attrs)
3583 {
3584         intel_unmap(dev, sglist[0].dma_address);
3585 }
3586
3587 static int intel_nontranslate_map_sg(struct device *hddev,
3588         struct scatterlist *sglist, int nelems, int dir)
3589 {
3590         int i;
3591         struct scatterlist *sg;
3592
3593         for_each_sg(sglist, sg, nelems, i) {
3594                 BUG_ON(!sg_page(sg));
3595                 sg->dma_address = sg_phys(sg);
3596                 sg->dma_length = sg->length;
3597         }
3598         return nelems;
3599 }
3600
3601 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3602                         enum dma_data_direction dir, struct dma_attrs *attrs)
3603 {
3604         int i;
3605         struct dmar_domain *domain;
3606         size_t size = 0;
3607         int prot = 0;
3608         struct iova *iova = NULL;
3609         int ret;
3610         struct scatterlist *sg;
3611         unsigned long start_vpfn;
3612         struct intel_iommu *iommu;
3613
3614         BUG_ON(dir == DMA_NONE);
3615         if (iommu_no_mapping(dev))
3616                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3617
3618         domain = get_valid_domain_for_dev(dev);
3619         if (!domain)
3620                 return 0;
3621
3622         iommu = domain_get_iommu(domain);
3623
3624         for_each_sg(sglist, sg, nelems, i)
3625                 size += aligned_nrpages(sg->offset, sg->length);
3626
3627         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3628                                 *dev->dma_mask);
3629         if (!iova) {
3630                 sglist->dma_length = 0;
3631                 return 0;
3632         }
3633
3634         /*
3635          * Check if DMAR supports zero-length reads on write only
3636          * mappings..
3637          */
3638         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3639                         !cap_zlr(iommu->cap))
3640                 prot |= DMA_PTE_READ;
3641         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3642                 prot |= DMA_PTE_WRITE;
3643
3644         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3645
3646         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3647         if (unlikely(ret)) {
3648                 dma_pte_free_pagetable(domain, start_vpfn,
3649                                        start_vpfn + size - 1);
3650                 __free_iova(&domain->iovad, iova);
3651                 return 0;
3652         }
3653
3654         /* it's a non-present to present mapping. Only flush if caching mode */
3655         if (cap_caching_mode(iommu->cap))
3656                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3657         else
3658                 iommu_flush_write_buffer(iommu);
3659
3660         return nelems;
3661 }
3662
3663 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3664 {
3665         return !dma_addr;
3666 }
3667
3668 struct dma_map_ops intel_dma_ops = {
3669         .alloc = intel_alloc_coherent,
3670         .free = intel_free_coherent,
3671         .map_sg = intel_map_sg,
3672         .unmap_sg = intel_unmap_sg,
3673         .map_page = intel_map_page,
3674         .unmap_page = intel_unmap_page,
3675         .mapping_error = intel_mapping_error,
3676 };
3677
3678 static inline int iommu_domain_cache_init(void)
3679 {
3680         int ret = 0;
3681
3682         iommu_domain_cache = kmem_cache_create("iommu_domain",
3683                                          sizeof(struct dmar_domain),
3684                                          0,
3685                                          SLAB_HWCACHE_ALIGN,
3686
3687                                          NULL);
3688         if (!iommu_domain_cache) {
3689                 pr_err("Couldn't create iommu_domain cache\n");
3690                 ret = -ENOMEM;
3691         }
3692
3693         return ret;
3694 }
3695
3696 static inline int iommu_devinfo_cache_init(void)
3697 {
3698         int ret = 0;
3699
3700         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3701                                          sizeof(struct device_domain_info),
3702                                          0,
3703                                          SLAB_HWCACHE_ALIGN,
3704                                          NULL);
3705         if (!iommu_devinfo_cache) {
3706                 pr_err("Couldn't create devinfo cache\n");
3707                 ret = -ENOMEM;
3708         }
3709
3710         return ret;
3711 }
3712
3713 static int __init iommu_init_mempool(void)
3714 {
3715         int ret;
3716         ret = iova_cache_get();
3717         if (ret)
3718                 return ret;
3719
3720         ret = iommu_domain_cache_init();
3721         if (ret)
3722                 goto domain_error;
3723
3724         ret = iommu_devinfo_cache_init();
3725         if (!ret)
3726                 return ret;
3727
3728         kmem_cache_destroy(iommu_domain_cache);
3729 domain_error:
3730         iova_cache_put();
3731
3732         return -ENOMEM;
3733 }
3734
3735 static void __init iommu_exit_mempool(void)
3736 {
3737         kmem_cache_destroy(iommu_devinfo_cache);
3738         kmem_cache_destroy(iommu_domain_cache);
3739         iova_cache_put();
3740 }
3741
3742 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3743 {
3744         struct dmar_drhd_unit *drhd;
3745         u32 vtbar;
3746         int rc;
3747
3748         /* We know that this device on this chipset has its own IOMMU.
3749          * If we find it under a different IOMMU, then the BIOS is lying
3750          * to us. Hope that the IOMMU for this device is actually
3751          * disabled, and it needs no translation...
3752          */
3753         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3754         if (rc) {
3755                 /* "can't" happen */
3756                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3757                 return;
3758         }
3759         vtbar &= 0xffff0000;
3760
3761         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3762         drhd = dmar_find_matched_drhd_unit(pdev);
3763         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3764                             TAINT_FIRMWARE_WORKAROUND,
3765                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3766                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3767 }
3768 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3769
3770 static void __init init_no_remapping_devices(void)
3771 {
3772         struct dmar_drhd_unit *drhd;
3773         struct device *dev;
3774         int i;
3775
3776         for_each_drhd_unit(drhd) {
3777                 if (!drhd->include_all) {
3778                         for_each_active_dev_scope(drhd->devices,
3779                                                   drhd->devices_cnt, i, dev)
3780                                 break;
3781                         /* ignore DMAR unit if no devices exist */
3782                         if (i == drhd->devices_cnt)
3783                                 drhd->ignored = 1;
3784                 }
3785         }
3786
3787         for_each_active_drhd_unit(drhd) {
3788                 if (drhd->include_all)
3789                         continue;
3790
3791                 for_each_active_dev_scope(drhd->devices,
3792                                           drhd->devices_cnt, i, dev)
3793                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3794                                 break;
3795                 if (i < drhd->devices_cnt)
3796                         continue;
3797
3798                 /* This IOMMU has *only* gfx devices. Either bypass it or
3799                    set the gfx_mapped flag, as appropriate */
3800                 if (dmar_map_gfx) {
3801                         intel_iommu_gfx_mapped = 1;
3802                 } else {
3803                         drhd->ignored = 1;
3804                         for_each_active_dev_scope(drhd->devices,
3805                                                   drhd->devices_cnt, i, dev)
3806                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3807                 }
3808         }
3809 }
3810
3811 #ifdef CONFIG_SUSPEND
3812 static int init_iommu_hw(void)
3813 {
3814         struct dmar_drhd_unit *drhd;
3815         struct intel_iommu *iommu = NULL;
3816
3817         for_each_active_iommu(iommu, drhd)
3818                 if (iommu->qi)
3819                         dmar_reenable_qi(iommu);
3820
3821         for_each_iommu(iommu, drhd) {
3822                 if (drhd->ignored) {
3823                         /*
3824                          * we always have to disable PMRs or DMA may fail on
3825                          * this device
3826                          */
3827                         if (force_on)
3828                                 iommu_disable_protect_mem_regions(iommu);
3829                         continue;
3830                 }
3831
3832                 iommu_flush_write_buffer(iommu);
3833
3834                 iommu_set_root_entry(iommu);
3835
3836                 iommu->flush.flush_context(iommu, 0, 0, 0,
3837                                            DMA_CCMD_GLOBAL_INVL);
3838                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3839                 iommu_enable_translation(iommu);
3840                 iommu_disable_protect_mem_regions(iommu);
3841         }
3842
3843         return 0;
3844 }
3845
3846 static void iommu_flush_all(void)
3847 {
3848         struct dmar_drhd_unit *drhd;
3849         struct intel_iommu *iommu;
3850
3851         for_each_active_iommu(iommu, drhd) {
3852                 iommu->flush.flush_context(iommu, 0, 0, 0,
3853                                            DMA_CCMD_GLOBAL_INVL);
3854                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3855                                          DMA_TLB_GLOBAL_FLUSH);
3856         }
3857 }
3858
3859 static int iommu_suspend(void)
3860 {
3861         struct dmar_drhd_unit *drhd;
3862         struct intel_iommu *iommu = NULL;
3863         unsigned long flag;
3864
3865         for_each_active_iommu(iommu, drhd) {
3866                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3867                                                  GFP_ATOMIC);
3868                 if (!iommu->iommu_state)
3869                         goto nomem;
3870         }
3871
3872         iommu_flush_all();
3873
3874         for_each_active_iommu(iommu, drhd) {
3875                 iommu_disable_translation(iommu);
3876
3877                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3878
3879                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3880                         readl(iommu->reg + DMAR_FECTL_REG);
3881                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3882                         readl(iommu->reg + DMAR_FEDATA_REG);
3883                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3884                         readl(iommu->reg + DMAR_FEADDR_REG);
3885                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3886                         readl(iommu->reg + DMAR_FEUADDR_REG);
3887
3888                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3889         }
3890         return 0;
3891
3892 nomem:
3893         for_each_active_iommu(iommu, drhd)
3894                 kfree(iommu->iommu_state);
3895
3896         return -ENOMEM;
3897 }
3898
3899 static void iommu_resume(void)
3900 {
3901         struct dmar_drhd_unit *drhd;
3902         struct intel_iommu *iommu = NULL;
3903         unsigned long flag;
3904
3905         if (init_iommu_hw()) {
3906                 if (force_on)
3907                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3908                 else
3909                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3910                 return;
3911         }
3912
3913         for_each_active_iommu(iommu, drhd) {
3914
3915                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3916
3917                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3918                         iommu->reg + DMAR_FECTL_REG);
3919                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3920                         iommu->reg + DMAR_FEDATA_REG);
3921                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3922                         iommu->reg + DMAR_FEADDR_REG);
3923                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3924                         iommu->reg + DMAR_FEUADDR_REG);
3925
3926                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3927         }
3928
3929         for_each_active_iommu(iommu, drhd)
3930                 kfree(iommu->iommu_state);
3931 }
3932
3933 static struct syscore_ops iommu_syscore_ops = {
3934         .resume         = iommu_resume,
3935         .suspend        = iommu_suspend,
3936 };
3937
3938 static void __init init_iommu_pm_ops(void)
3939 {
3940         register_syscore_ops(&iommu_syscore_ops);
3941 }
3942
3943 #else
3944 static inline void init_iommu_pm_ops(void) {}
3945 #endif  /* CONFIG_PM */
3946
3947
3948 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3949 {
3950         struct acpi_dmar_reserved_memory *rmrr;
3951         struct dmar_rmrr_unit *rmrru;
3952
3953         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3954         if (!rmrru)
3955                 return -ENOMEM;
3956
3957         rmrru->hdr = header;
3958         rmrr = (struct acpi_dmar_reserved_memory *)header;
3959         rmrru->base_address = rmrr->base_address;
3960         rmrru->end_address = rmrr->end_address;
3961         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3962                                 ((void *)rmrr) + rmrr->header.length,
3963                                 &rmrru->devices_cnt);
3964         if (rmrru->devices_cnt && rmrru->devices == NULL) {
3965                 kfree(rmrru);
3966                 return -ENOMEM;
3967         }
3968
3969         list_add(&rmrru->list, &dmar_rmrr_units);
3970
3971         return 0;
3972 }
3973
3974 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3975 {
3976         struct dmar_atsr_unit *atsru;
3977         struct acpi_dmar_atsr *tmp;
3978
3979         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3980                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3981                 if (atsr->segment != tmp->segment)
3982                         continue;
3983                 if (atsr->header.length != tmp->header.length)
3984                         continue;
3985                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3986                         return atsru;
3987         }
3988
3989         return NULL;
3990 }
3991
3992 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3993 {
3994         struct acpi_dmar_atsr *atsr;
3995         struct dmar_atsr_unit *atsru;
3996
3997         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3998                 return 0;
3999
4000         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4001         atsru = dmar_find_atsr(atsr);
4002         if (atsru)
4003                 return 0;
4004
4005         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4006         if (!atsru)
4007                 return -ENOMEM;
4008
4009         /*
4010          * If memory is allocated from slab by ACPI _DSM method, we need to
4011          * copy the memory content because the memory buffer will be freed
4012          * on return.
4013          */
4014         atsru->hdr = (void *)(atsru + 1);
4015         memcpy(atsru->hdr, hdr, hdr->length);
4016         atsru->include_all = atsr->flags & 0x1;
4017         if (!atsru->include_all) {
4018                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4019                                 (void *)atsr + atsr->header.length,
4020                                 &atsru->devices_cnt);
4021                 if (atsru->devices_cnt && atsru->devices == NULL) {
4022                         kfree(atsru);
4023                         return -ENOMEM;
4024                 }
4025         }
4026
4027         list_add_rcu(&atsru->list, &dmar_atsr_units);
4028
4029         return 0;
4030 }
4031
4032 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4033 {
4034         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4035         kfree(atsru);
4036 }
4037
4038 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4039 {
4040         struct acpi_dmar_atsr *atsr;
4041         struct dmar_atsr_unit *atsru;
4042
4043         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4044         atsru = dmar_find_atsr(atsr);
4045         if (atsru) {
4046                 list_del_rcu(&atsru->list);
4047                 synchronize_rcu();
4048                 intel_iommu_free_atsr(atsru);
4049         }
4050
4051         return 0;
4052 }
4053
4054 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4055 {
4056         int i;
4057         struct device *dev;
4058         struct acpi_dmar_atsr *atsr;
4059         struct dmar_atsr_unit *atsru;
4060
4061         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4062         atsru = dmar_find_atsr(atsr);
4063         if (!atsru)
4064                 return 0;
4065
4066         if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4067                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4068                                           i, dev)
4069                         return -EBUSY;
4070
4071         return 0;
4072 }
4073
4074 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4075 {
4076         int sp, ret = 0;
4077         struct intel_iommu *iommu = dmaru->iommu;
4078
4079         if (g_iommus[iommu->seq_id])
4080                 return 0;
4081
4082         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4083                 pr_warn("%s: Doesn't support hardware pass through.\n",
4084                         iommu->name);
4085                 return -ENXIO;
4086         }
4087         if (!ecap_sc_support(iommu->ecap) &&
4088             domain_update_iommu_snooping(iommu)) {
4089                 pr_warn("%s: Doesn't support snooping.\n",
4090                         iommu->name);
4091                 return -ENXIO;
4092         }
4093         sp = domain_update_iommu_superpage(iommu) - 1;
4094         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4095                 pr_warn("%s: Doesn't support large page.\n",
4096                         iommu->name);
4097                 return -ENXIO;
4098         }
4099
4100         /*
4101          * Disable translation if already enabled prior to OS handover.
4102          */
4103         if (iommu->gcmd & DMA_GCMD_TE)
4104                 iommu_disable_translation(iommu);
4105
4106         g_iommus[iommu->seq_id] = iommu;
4107         ret = iommu_init_domains(iommu);
4108         if (ret == 0)
4109                 ret = iommu_alloc_root_entry(iommu);
4110         if (ret)
4111                 goto out;
4112
4113         if (dmaru->ignored) {
4114                 /*
4115                  * we always have to disable PMRs or DMA may fail on this device
4116                  */
4117                 if (force_on)
4118                         iommu_disable_protect_mem_regions(iommu);
4119                 return 0;
4120         }
4121
4122         intel_iommu_init_qi(iommu);
4123         iommu_flush_write_buffer(iommu);
4124         ret = dmar_set_interrupt(iommu);
4125         if (ret)
4126                 goto disable_iommu;
4127
4128         iommu_set_root_entry(iommu);
4129         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4130         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4131         iommu_enable_translation(iommu);
4132
4133         iommu_disable_protect_mem_regions(iommu);
4134         return 0;
4135
4136 disable_iommu:
4137         disable_dmar_iommu(iommu);
4138 out:
4139         free_dmar_iommu(iommu);
4140         return ret;
4141 }
4142
4143 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4144 {
4145         int ret = 0;
4146         struct intel_iommu *iommu = dmaru->iommu;
4147
4148         if (!intel_iommu_enabled)
4149                 return 0;
4150         if (iommu == NULL)
4151                 return -EINVAL;
4152
4153         if (insert) {
4154                 ret = intel_iommu_add(dmaru);
4155         } else {
4156                 disable_dmar_iommu(iommu);
4157                 free_dmar_iommu(iommu);
4158         }
4159
4160         return ret;
4161 }
4162
4163 static void intel_iommu_free_dmars(void)
4164 {
4165         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4166         struct dmar_atsr_unit *atsru, *atsr_n;
4167
4168         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4169                 list_del(&rmrru->list);
4170                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4171                 kfree(rmrru);
4172         }
4173
4174         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4175                 list_del(&atsru->list);
4176                 intel_iommu_free_atsr(atsru);
4177         }
4178 }
4179
4180 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4181 {
4182         int i, ret = 1;
4183         struct pci_bus *bus;
4184         struct pci_dev *bridge = NULL;
4185         struct device *tmp;
4186         struct acpi_dmar_atsr *atsr;
4187         struct dmar_atsr_unit *atsru;
4188
4189         dev = pci_physfn(dev);
4190         for (bus = dev->bus; bus; bus = bus->parent) {
4191                 bridge = bus->self;
4192                 if (!bridge || !pci_is_pcie(bridge) ||
4193                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4194                         return 0;
4195                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4196                         break;
4197         }
4198         if (!bridge)
4199                 return 0;
4200
4201         rcu_read_lock();
4202         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4203                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4204                 if (atsr->segment != pci_domain_nr(dev->bus))
4205                         continue;
4206
4207                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4208                         if (tmp == &bridge->dev)
4209                                 goto out;
4210
4211                 if (atsru->include_all)
4212                         goto out;
4213         }
4214         ret = 0;
4215 out:
4216         rcu_read_unlock();
4217
4218         return ret;
4219 }
4220
4221 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4222 {
4223         int ret = 0;
4224         struct dmar_rmrr_unit *rmrru;
4225         struct dmar_atsr_unit *atsru;
4226         struct acpi_dmar_atsr *atsr;
4227         struct acpi_dmar_reserved_memory *rmrr;
4228
4229         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4230                 return 0;
4231
4232         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4233                 rmrr = container_of(rmrru->hdr,
4234                                     struct acpi_dmar_reserved_memory, header);
4235                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4236                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4237                                 ((void *)rmrr) + rmrr->header.length,
4238                                 rmrr->segment, rmrru->devices,
4239                                 rmrru->devices_cnt);
4240                         if(ret < 0)
4241                                 return ret;
4242                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4243                         dmar_remove_dev_scope(info, rmrr->segment,
4244                                 rmrru->devices, rmrru->devices_cnt);
4245                 }
4246         }
4247
4248         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4249                 if (atsru->include_all)
4250                         continue;
4251
4252                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4253                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4254                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4255                                         (void *)atsr + atsr->header.length,
4256                                         atsr->segment, atsru->devices,
4257                                         atsru->devices_cnt);
4258                         if (ret > 0)
4259                                 break;
4260                         else if(ret < 0)
4261                                 return ret;
4262                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4263                         if (dmar_remove_dev_scope(info, atsr->segment,
4264                                         atsru->devices, atsru->devices_cnt))
4265                                 break;
4266                 }
4267         }
4268
4269         return 0;
4270 }
4271
4272 /*
4273  * Here we only respond to action of unbound device from driver.
4274  *
4275  * Added device is not attached to its DMAR domain here yet. That will happen
4276  * when mapping the device to iova.
4277  */
4278 static int device_notifier(struct notifier_block *nb,
4279                                   unsigned long action, void *data)
4280 {
4281         struct device *dev = data;
4282         struct dmar_domain *domain;
4283
4284         if (iommu_dummy(dev))
4285                 return 0;
4286
4287         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4288                 return 0;
4289
4290         domain = find_domain(dev);
4291         if (!domain)
4292                 return 0;
4293
4294         dmar_remove_one_dev_info(domain, dev);
4295         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4296                 domain_exit(domain);
4297
4298         return 0;
4299 }
4300
4301 static struct notifier_block device_nb = {
4302         .notifier_call = device_notifier,
4303 };
4304
4305 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4306                                        unsigned long val, void *v)
4307 {
4308         struct memory_notify *mhp = v;
4309         unsigned long long start, end;
4310         unsigned long start_vpfn, last_vpfn;
4311
4312         switch (val) {
4313         case MEM_GOING_ONLINE:
4314                 start = mhp->start_pfn << PAGE_SHIFT;
4315                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4316                 if (iommu_domain_identity_map(si_domain, start, end)) {
4317                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4318                                 start, end);
4319                         return NOTIFY_BAD;
4320                 }
4321                 break;
4322
4323         case MEM_OFFLINE:
4324         case MEM_CANCEL_ONLINE:
4325                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4326                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4327                 while (start_vpfn <= last_vpfn) {
4328                         struct iova *iova;
4329                         struct dmar_drhd_unit *drhd;
4330                         struct intel_iommu *iommu;
4331                         struct page *freelist;
4332
4333                         iova = find_iova(&si_domain->iovad, start_vpfn);
4334                         if (iova == NULL) {
4335                                 pr_debug("Failed get IOVA for PFN %lx\n",
4336                                          start_vpfn);
4337                                 break;
4338                         }
4339
4340                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4341                                                      start_vpfn, last_vpfn);
4342                         if (iova == NULL) {
4343                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4344                                         start_vpfn, last_vpfn);
4345                                 return NOTIFY_BAD;
4346                         }
4347
4348                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4349                                                iova->pfn_hi);
4350
4351                         rcu_read_lock();
4352                         for_each_active_iommu(iommu, drhd)
4353                                 iommu_flush_iotlb_psi(iommu, si_domain,
4354                                         iova->pfn_lo, iova_size(iova),
4355                                         !freelist, 0);
4356                         rcu_read_unlock();
4357                         dma_free_pagelist(freelist);
4358
4359                         start_vpfn = iova->pfn_hi + 1;
4360                         free_iova_mem(iova);
4361                 }
4362                 break;
4363         }
4364
4365         return NOTIFY_OK;
4366 }
4367
4368 static struct notifier_block intel_iommu_memory_nb = {
4369         .notifier_call = intel_iommu_memory_notifier,
4370         .priority = 0
4371 };
4372
4373
4374 static ssize_t intel_iommu_show_version(struct device *dev,
4375                                         struct device_attribute *attr,
4376                                         char *buf)
4377 {
4378         struct intel_iommu *iommu = dev_get_drvdata(dev);
4379         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4380         return sprintf(buf, "%d:%d\n",
4381                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4382 }
4383 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4384
4385 static ssize_t intel_iommu_show_address(struct device *dev,
4386                                         struct device_attribute *attr,
4387                                         char *buf)
4388 {
4389         struct intel_iommu *iommu = dev_get_drvdata(dev);
4390         return sprintf(buf, "%llx\n", iommu->reg_phys);
4391 }
4392 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4393
4394 static ssize_t intel_iommu_show_cap(struct device *dev,
4395                                     struct device_attribute *attr,
4396                                     char *buf)
4397 {
4398         struct intel_iommu *iommu = dev_get_drvdata(dev);
4399         return sprintf(buf, "%llx\n", iommu->cap);
4400 }
4401 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4402
4403 static ssize_t intel_iommu_show_ecap(struct device *dev,
4404                                     struct device_attribute *attr,
4405                                     char *buf)
4406 {
4407         struct intel_iommu *iommu = dev_get_drvdata(dev);
4408         return sprintf(buf, "%llx\n", iommu->ecap);
4409 }
4410 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4411
4412 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4413                                       struct device_attribute *attr,
4414                                       char *buf)
4415 {
4416         struct intel_iommu *iommu = dev_get_drvdata(dev);
4417         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4418 }
4419 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4420
4421 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4422                                            struct device_attribute *attr,
4423                                            char *buf)
4424 {
4425         struct intel_iommu *iommu = dev_get_drvdata(dev);
4426         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4427                                                   cap_ndoms(iommu->cap)));
4428 }
4429 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4430
4431 static struct attribute *intel_iommu_attrs[] = {
4432         &dev_attr_version.attr,
4433         &dev_attr_address.attr,
4434         &dev_attr_cap.attr,
4435         &dev_attr_ecap.attr,
4436         &dev_attr_domains_supported.attr,
4437         &dev_attr_domains_used.attr,
4438         NULL,
4439 };
4440
4441 static struct attribute_group intel_iommu_group = {
4442         .name = "intel-iommu",
4443         .attrs = intel_iommu_attrs,
4444 };
4445
4446 const struct attribute_group *intel_iommu_groups[] = {
4447         &intel_iommu_group,
4448         NULL,
4449 };
4450
4451 int __init intel_iommu_init(void)
4452 {
4453         int ret = -ENODEV;
4454         struct dmar_drhd_unit *drhd;
4455         struct intel_iommu *iommu;
4456
4457         /* VT-d is required for a TXT/tboot launch, so enforce that */
4458         force_on = tboot_force_iommu();
4459
4460         if (iommu_init_mempool()) {
4461                 if (force_on)
4462                         panic("tboot: Failed to initialize iommu memory\n");
4463                 return -ENOMEM;
4464         }
4465
4466         down_write(&dmar_global_lock);
4467         if (dmar_table_init()) {
4468                 if (force_on)
4469                         panic("tboot: Failed to initialize DMAR table\n");
4470                 goto out_free_dmar;
4471         }
4472
4473         if (dmar_dev_scope_init() < 0) {
4474                 if (force_on)
4475                         panic("tboot: Failed to initialize DMAR device scope\n");
4476                 goto out_free_dmar;
4477         }
4478
4479         if (no_iommu || dmar_disabled)
4480                 goto out_free_dmar;
4481
4482         if (list_empty(&dmar_rmrr_units))
4483                 pr_info("No RMRR found\n");
4484
4485         if (list_empty(&dmar_atsr_units))
4486                 pr_info("No ATSR found\n");
4487
4488         if (dmar_init_reserved_ranges()) {
4489                 if (force_on)
4490                         panic("tboot: Failed to reserve iommu ranges\n");
4491                 goto out_free_reserved_range;
4492         }
4493
4494         init_no_remapping_devices();
4495
4496         ret = init_dmars();
4497         if (ret) {
4498                 if (force_on)
4499                         panic("tboot: Failed to initialize DMARs\n");
4500                 pr_err("Initialization failed\n");
4501                 goto out_free_reserved_range;
4502         }
4503         up_write(&dmar_global_lock);
4504         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4505
4506         init_timer(&unmap_timer);
4507 #ifdef CONFIG_SWIOTLB
4508         swiotlb = 0;
4509 #endif
4510         dma_ops = &intel_dma_ops;
4511
4512         init_iommu_pm_ops();
4513
4514         for_each_active_iommu(iommu, drhd)
4515                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4516                                                        intel_iommu_groups,
4517                                                        "%s", iommu->name);
4518
4519         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4520         bus_register_notifier(&pci_bus_type, &device_nb);
4521         if (si_domain && !hw_pass_through)
4522                 register_memory_notifier(&intel_iommu_memory_nb);
4523
4524         intel_iommu_enabled = 1;
4525
4526         return 0;
4527
4528 out_free_reserved_range:
4529         put_iova_domain(&reserved_iova_list);
4530 out_free_dmar:
4531         intel_iommu_free_dmars();
4532         up_write(&dmar_global_lock);
4533         iommu_exit_mempool();
4534         return ret;
4535 }
4536
4537 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4538 {
4539         struct intel_iommu *iommu = opaque;
4540
4541         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4542         return 0;
4543 }
4544
4545 /*
4546  * NB - intel-iommu lacks any sort of reference counting for the users of
4547  * dependent devices.  If multiple endpoints have intersecting dependent
4548  * devices, unbinding the driver from any one of them will possibly leave
4549  * the others unable to operate.
4550  */
4551 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4552 {
4553         if (!iommu || !dev || !dev_is_pci(dev))
4554                 return;
4555
4556         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4557 }
4558
4559 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4560 {
4561         struct intel_iommu *iommu;
4562         unsigned long flags;
4563
4564         assert_spin_locked(&device_domain_lock);
4565
4566         if (WARN_ON(!info))
4567                 return;
4568
4569         iommu = info->iommu;
4570
4571         if (info->dev) {
4572                 iommu_disable_dev_iotlb(info);
4573                 domain_context_clear(iommu, info->dev);
4574         }
4575
4576         unlink_domain_info(info);
4577
4578         spin_lock_irqsave(&iommu->lock, flags);
4579         domain_detach_iommu(info->domain, iommu);
4580         spin_unlock_irqrestore(&iommu->lock, flags);
4581
4582         free_devinfo_mem(info);
4583 }
4584
4585 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4586                                      struct device *dev)
4587 {
4588         struct device_domain_info *info;
4589         unsigned long flags;
4590
4591         spin_lock_irqsave(&device_domain_lock, flags);
4592         info = dev->archdata.iommu;
4593         __dmar_remove_one_dev_info(info);
4594         spin_unlock_irqrestore(&device_domain_lock, flags);
4595 }
4596
4597 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4598 {
4599         int adjust_width;
4600
4601         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4602                         DMA_32BIT_PFN);
4603         domain_reserve_special_ranges(domain);
4604
4605         /* calculate AGAW */
4606         domain->gaw = guest_width;
4607         adjust_width = guestwidth_to_adjustwidth(guest_width);
4608         domain->agaw = width_to_agaw(adjust_width);
4609
4610         domain->iommu_coherency = 0;
4611         domain->iommu_snooping = 0;
4612         domain->iommu_superpage = 0;
4613         domain->max_addr = 0;
4614
4615         /* always allocate the top pgd */
4616         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4617         if (!domain->pgd)
4618                 return -ENOMEM;
4619         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4620         return 0;
4621 }
4622
4623 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4624 {
4625         struct dmar_domain *dmar_domain;
4626         struct iommu_domain *domain;
4627
4628         if (type != IOMMU_DOMAIN_UNMANAGED)
4629                 return NULL;
4630
4631         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4632         if (!dmar_domain) {
4633                 pr_err("Can't allocate dmar_domain\n");
4634                 return NULL;
4635         }
4636         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4637                 pr_err("Domain initialization failed\n");
4638                 domain_exit(dmar_domain);
4639                 return NULL;
4640         }
4641         domain_update_iommu_cap(dmar_domain);
4642
4643         domain = &dmar_domain->domain;
4644         domain->geometry.aperture_start = 0;
4645         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4646         domain->geometry.force_aperture = true;
4647
4648         return domain;
4649 }
4650
4651 static void intel_iommu_domain_free(struct iommu_domain *domain)
4652 {
4653         domain_exit(to_dmar_domain(domain));
4654 }
4655
4656 static int intel_iommu_attach_device(struct iommu_domain *domain,
4657                                      struct device *dev)
4658 {
4659         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4660         struct intel_iommu *iommu;
4661         int addr_width;
4662         u8 bus, devfn;
4663
4664         if (device_is_rmrr_locked(dev)) {
4665                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
4666                 return -EPERM;
4667         }
4668
4669         /* normally dev is not mapped */
4670         if (unlikely(domain_context_mapped(dev))) {
4671                 struct dmar_domain *old_domain;
4672
4673                 old_domain = find_domain(dev);
4674                 if (old_domain) {
4675                         rcu_read_lock();
4676                         dmar_remove_one_dev_info(old_domain, dev);
4677                         rcu_read_unlock();
4678
4679                         if (!domain_type_is_vm_or_si(old_domain) &&
4680                              list_empty(&old_domain->devices))
4681                                 domain_exit(old_domain);
4682                 }
4683         }
4684
4685         iommu = device_to_iommu(dev, &bus, &devfn);
4686         if (!iommu)
4687                 return -ENODEV;
4688
4689         /* check if this iommu agaw is sufficient for max mapped address */
4690         addr_width = agaw_to_width(iommu->agaw);
4691         if (addr_width > cap_mgaw(iommu->cap))
4692                 addr_width = cap_mgaw(iommu->cap);
4693
4694         if (dmar_domain->max_addr > (1LL << addr_width)) {
4695                 pr_err("%s: iommu width (%d) is not "
4696                        "sufficient for the mapped address (%llx)\n",
4697                        __func__, addr_width, dmar_domain->max_addr);
4698                 return -EFAULT;
4699         }
4700         dmar_domain->gaw = addr_width;
4701
4702         /*
4703          * Knock out extra levels of page tables if necessary
4704          */
4705         while (iommu->agaw < dmar_domain->agaw) {
4706                 struct dma_pte *pte;
4707
4708                 pte = dmar_domain->pgd;
4709                 if (dma_pte_present(pte)) {
4710                         dmar_domain->pgd = (struct dma_pte *)
4711                                 phys_to_virt(dma_pte_addr(pte));
4712                         free_pgtable_page(pte);
4713                 }
4714                 dmar_domain->agaw--;
4715         }
4716
4717         return domain_add_dev_info(dmar_domain, dev);
4718 }
4719
4720 static void intel_iommu_detach_device(struct iommu_domain *domain,
4721                                       struct device *dev)
4722 {
4723         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
4724 }
4725
4726 static int intel_iommu_map(struct iommu_domain *domain,
4727                            unsigned long iova, phys_addr_t hpa,
4728                            size_t size, int iommu_prot)
4729 {
4730         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4731         u64 max_addr;
4732         int prot = 0;
4733         int ret;
4734
4735         if (iommu_prot & IOMMU_READ)
4736                 prot |= DMA_PTE_READ;
4737         if (iommu_prot & IOMMU_WRITE)
4738                 prot |= DMA_PTE_WRITE;
4739         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4740                 prot |= DMA_PTE_SNP;
4741
4742         max_addr = iova + size;
4743         if (dmar_domain->max_addr < max_addr) {
4744                 u64 end;
4745
4746                 /* check if minimum agaw is sufficient for mapped address */
4747                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4748                 if (end < max_addr) {
4749                         pr_err("%s: iommu width (%d) is not "
4750                                "sufficient for the mapped address (%llx)\n",
4751                                __func__, dmar_domain->gaw, max_addr);
4752                         return -EFAULT;
4753                 }
4754                 dmar_domain->max_addr = max_addr;
4755         }
4756         /* Round up size to next multiple of PAGE_SIZE, if it and
4757            the low bits of hpa would take us onto the next page */
4758         size = aligned_nrpages(hpa, size);
4759         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4760                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4761         return ret;
4762 }
4763
4764 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4765                                 unsigned long iova, size_t size)
4766 {
4767         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4768         struct page *freelist = NULL;
4769         struct intel_iommu *iommu;
4770         unsigned long start_pfn, last_pfn;
4771         unsigned int npages;
4772         int iommu_id, level = 0;
4773
4774         /* Cope with horrid API which requires us to unmap more than the
4775            size argument if it happens to be a large-page mapping. */
4776         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
4777
4778         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4779                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4780
4781         start_pfn = iova >> VTD_PAGE_SHIFT;
4782         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4783
4784         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4785
4786         npages = last_pfn - start_pfn + 1;
4787
4788         for_each_domain_iommu(iommu_id, dmar_domain) {
4789                 iommu = g_iommus[iommu_id];
4790
4791                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
4792                                       start_pfn, npages, !freelist, 0);
4793         }
4794
4795         dma_free_pagelist(freelist);
4796
4797         if (dmar_domain->max_addr == iova + size)
4798                 dmar_domain->max_addr = iova;
4799
4800         return size;
4801 }
4802
4803 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4804                                             dma_addr_t iova)
4805 {
4806         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4807         struct dma_pte *pte;
4808         int level = 0;
4809         u64 phys = 0;
4810
4811         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4812         if (pte)
4813                 phys = dma_pte_addr(pte);
4814
4815         return phys;
4816 }
4817
4818 static bool intel_iommu_capable(enum iommu_cap cap)
4819 {
4820         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4821                 return domain_update_iommu_snooping(NULL) == 1;
4822         if (cap == IOMMU_CAP_INTR_REMAP)
4823                 return irq_remapping_enabled == 1;
4824
4825         return false;
4826 }
4827
4828 static int intel_iommu_add_device(struct device *dev)
4829 {
4830         struct intel_iommu *iommu;
4831         struct iommu_group *group;
4832         u8 bus, devfn;
4833
4834         iommu = device_to_iommu(dev, &bus, &devfn);
4835         if (!iommu)
4836                 return -ENODEV;
4837
4838         iommu_device_link(iommu->iommu_dev, dev);
4839
4840         group = iommu_group_get_for_dev(dev);
4841
4842         if (IS_ERR(group))
4843                 return PTR_ERR(group);
4844
4845         iommu_group_put(group);
4846         return 0;
4847 }
4848
4849 static void intel_iommu_remove_device(struct device *dev)
4850 {
4851         struct intel_iommu *iommu;
4852         u8 bus, devfn;
4853
4854         iommu = device_to_iommu(dev, &bus, &devfn);
4855         if (!iommu)
4856                 return;
4857
4858         iommu_group_remove_device(dev);
4859
4860         iommu_device_unlink(iommu->iommu_dev, dev);
4861 }
4862
4863 static const struct iommu_ops intel_iommu_ops = {
4864         .capable        = intel_iommu_capable,
4865         .domain_alloc   = intel_iommu_domain_alloc,
4866         .domain_free    = intel_iommu_domain_free,
4867         .attach_dev     = intel_iommu_attach_device,
4868         .detach_dev     = intel_iommu_detach_device,
4869         .map            = intel_iommu_map,
4870         .unmap          = intel_iommu_unmap,
4871         .map_sg         = default_iommu_map_sg,
4872         .iova_to_phys   = intel_iommu_iova_to_phys,
4873         .add_device     = intel_iommu_add_device,
4874         .remove_device  = intel_iommu_remove_device,
4875         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
4876 };
4877
4878 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4879 {
4880         /* G4x/GM45 integrated gfx dmar support is totally busted. */
4881         pr_info("Disabling IOMMU for graphics on this chipset\n");
4882         dmar_map_gfx = 0;
4883 }
4884
4885 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4886 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4887 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4888 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4889 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4890 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4891 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4892
4893 static void quirk_iommu_rwbf(struct pci_dev *dev)
4894 {
4895         /*
4896          * Mobile 4 Series Chipset neglects to set RWBF capability,
4897          * but needs it. Same seems to hold for the desktop versions.
4898          */
4899         pr_info("Forcing write-buffer flush capability\n");
4900         rwbf_quirk = 1;
4901 }
4902
4903 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4904 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4905 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4906 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4907 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4908 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4909 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4910
4911 #define GGC 0x52
4912 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4913 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4914 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4915 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4916 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4917 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4918 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4919 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4920
4921 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4922 {
4923         unsigned short ggc;
4924
4925         if (pci_read_config_word(dev, GGC, &ggc))
4926                 return;
4927
4928         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4929                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4930                 dmar_map_gfx = 0;
4931         } else if (dmar_map_gfx) {
4932                 /* we have to ensure the gfx device is idle before we flush */
4933                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
4934                 intel_iommu_strict = 1;
4935        }
4936 }
4937 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4938 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4939 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4940 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4941
4942 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4943    ISOCH DMAR unit for the Azalia sound device, but not give it any
4944    TLB entries, which causes it to deadlock. Check for that.  We do
4945    this in a function called from init_dmars(), instead of in a PCI
4946    quirk, because we don't want to print the obnoxious "BIOS broken"
4947    message if VT-d is actually disabled.
4948 */
4949 static void __init check_tylersburg_isoch(void)
4950 {
4951         struct pci_dev *pdev;
4952         uint32_t vtisochctrl;
4953
4954         /* If there's no Azalia in the system anyway, forget it. */
4955         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4956         if (!pdev)
4957                 return;
4958         pci_dev_put(pdev);
4959
4960         /* System Management Registers. Might be hidden, in which case
4961            we can't do the sanity check. But that's OK, because the
4962            known-broken BIOSes _don't_ actually hide it, so far. */
4963         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4964         if (!pdev)
4965                 return;
4966
4967         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4968                 pci_dev_put(pdev);
4969                 return;
4970         }
4971
4972         pci_dev_put(pdev);
4973
4974         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4975         if (vtisochctrl & 1)
4976                 return;
4977
4978         /* Drop all bits other than the number of TLB entries */
4979         vtisochctrl &= 0x1c;
4980
4981         /* If we have the recommended number of TLB entries (16), fine. */
4982         if (vtisochctrl == 0x10)
4983                 return;
4984
4985         /* Zero TLB entries? You get to ride the short bus to school. */
4986         if (!vtisochctrl) {
4987                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4988                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4989                      dmi_get_system_info(DMI_BIOS_VENDOR),
4990                      dmi_get_system_info(DMI_BIOS_VERSION),
4991                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4992                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4993                 return;
4994         }
4995
4996         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4997                vtisochctrl);
4998 }