arch/mips/kvm/kvm_tlb.c

   1 /*
   2 * This file is subject to the terms and conditions of the GNU General Public
   3 * License.  See the file "COPYING" in the main directory of this archive
   4 * for more details.
   5 *
   6 * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
   7 * TLB handlers run from KSEG0
   8 *
   9 * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
  10 * Authors: Sanjay Lal <sanjayl@kymasys.com>
  11 */
  12
  13 #include <linux/init.h>
  14 #include <linux/sched.h>
  15 #include <linux/smp.h>
  16 #include <linux/mm.h>
  17 #include <linux/delay.h>
  18 #include <linux/module.h>
  19 #include <linux/kvm_host.h>
  20 #include <linux/srcu.h>
  21
  22
  23 #include <asm/cpu.h>
  24 #include <asm/bootinfo.h>
  25 #include <asm/mmu_context.h>
  26 #include <asm/pgtable.h>
  27 #include <asm/cacheflush.h>
  28 #include <asm/tlb.h>
  29
  30 #undef CONFIG_MIPS_MT
  31 #include <asm/r4kcache.h>
  32 #define CONFIG_MIPS_MT
  33
  34 #define KVM_GUEST_PC_TLB    0
  35 #define KVM_GUEST_SP_TLB    1
  36
  37 #define PRIx64 "llx"
  38
  39 atomic_t kvm_mips_instance;
  40 EXPORT_SYMBOL(kvm_mips_instance);
  41
  42 /* These function pointers are initialized once the KVM module is loaded */
  43 pfn_t(*kvm_mips_gfn_to_pfn) (struct kvm *kvm, gfn_t gfn);
  44 EXPORT_SYMBOL(kvm_mips_gfn_to_pfn);
  45
  46 void (*kvm_mips_release_pfn_clean) (pfn_t pfn);
  47 EXPORT_SYMBOL(kvm_mips_release_pfn_clean);
  48
  49 bool(*kvm_mips_is_error_pfn) (pfn_t pfn);
  50 EXPORT_SYMBOL(kvm_mips_is_error_pfn);
  51
  52 uint32_t kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
  53 {
  54         return vcpu->arch.guest_kernel_asid[smp_processor_id()] & ASID_MASK;
  55 }
  56
  57
  58 uint32_t kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
  59 {
  60         return vcpu->arch.guest_user_asid[smp_processor_id()] & ASID_MASK;
  61 }
  62
  63 inline uint32_t kvm_mips_get_commpage_asid (struct kvm_vcpu *vcpu)
  64 {
  65         return vcpu->kvm->arch.commpage_tlb;
  66 }
  67
  68
  69 /*
  70  * Structure defining an tlb entry data set.
  71  */
  72
  73 void kvm_mips_dump_host_tlbs(void)
  74 {
  75         unsigned long old_entryhi;
  76         unsigned long old_pagemask;
  77         struct kvm_mips_tlb tlb;
  78         unsigned long flags;
  79         int i;
  80
  81         local_irq_save(flags);
  82
  83         old_entryhi = read_c0_entryhi();
  84         old_pagemask = read_c0_pagemask();
  85
  86         printk("HOST TLBs:\n");
  87         printk("ASID: %#lx\n", read_c0_entryhi() & ASID_MASK);
  88
  89         for (i = 0; i < current_cpu_data.tlbsize; i++) {
  90                 write_c0_index(i);
  91                 mtc0_tlbw_hazard();
  92
  93                 tlb_read();
  94                 tlbw_use_hazard();
  95
  96                 tlb.tlb_hi = read_c0_entryhi();
  97                 tlb.tlb_lo0 = read_c0_entrylo0();
  98                 tlb.tlb_lo1 = read_c0_entrylo1();
  99                 tlb.tlb_mask = read_c0_pagemask();
 100
 101                 printk("TLB%c%3d Hi 0x%08lx ",
 102                        (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
 103                        i, tlb.tlb_hi);
 104                 printk("Lo0=0x%09" PRIx64 " %c%c attr %lx ",
 105                        (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
 106                        (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
 107                        (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
 108                        (tlb.tlb_lo0 >> 3) & 7);
 109                 printk("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n",
 110                        (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
 111                        (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
 112                        (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
 113                        (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
 114         }
 115         write_c0_entryhi(old_entryhi);
 116         write_c0_pagemask(old_pagemask);
 117         mtc0_tlbw_hazard();
 118         local_irq_restore(flags);
 119 }
 120
 121 void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
 122 {
 123         struct mips_coproc *cop0 = vcpu->arch.cop0;
 124         struct kvm_mips_tlb tlb;
 125         int i;
 126
 127         printk("Guest TLBs:\n");
 128         printk("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
 129
 130         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
 131                 tlb = vcpu->arch.guest_tlb[i];
 132                 printk("TLB%c%3d Hi 0x%08lx ",
 133                        (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
 134                        i, tlb.tlb_hi);
 135                 printk("Lo0=0x%09" PRIx64 " %c%c attr %lx ",
 136                        (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
 137                        (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
 138                        (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
 139                        (tlb.tlb_lo0 >> 3) & 7);
 140                 printk("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n",
 141                        (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
 142                        (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
 143                        (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
 144                        (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
 145         }
 146 }
 147
 148 static int kvm_mips_map_page(struct kvm *kvm, gfn_t gfn)
 149 {
 150         int srcu_idx, err = 0;
 151         pfn_t pfn;
 152
 153         if (kvm->arch.guest_pmap[gfn] != KVM_INVALID_PAGE)
 154                 return 0;
 155
 156         srcu_idx = srcu_read_lock(&kvm->srcu);
 157         pfn = kvm_mips_gfn_to_pfn(kvm, gfn);
 158
 159         if (kvm_mips_is_error_pfn(pfn)) {
 160                 kvm_err("Couldn't get pfn for gfn %#" PRIx64 "!\n", gfn);
 161                 err = -EFAULT;
 162                 goto out;
 163         }
 164
 165         kvm->arch.guest_pmap[gfn] = pfn;
 166 out:
 167         srcu_read_unlock(&kvm->srcu, srcu_idx);
 168         return err;
 169 }
 170
 171 /* Translate guest KSEG0 addresses to Host PA */
 172 unsigned long kvm_mips_translate_guest_kseg0_to_hpa(struct kvm_vcpu *vcpu,
 173         unsigned long gva)
 174 {
 175         gfn_t gfn;
 176         uint32_t offset = gva & ~PAGE_MASK;
 177         struct kvm *kvm = vcpu->kvm;
 178
 179         if (KVM_GUEST_KSEGX(gva) != KVM_GUEST_KSEG0) {
 180                 kvm_err("%s/%p: Invalid gva: %#lx\n", __func__,
 181                         __builtin_return_address(0), gva);
 182                 return KVM_INVALID_PAGE;
 183         }
 184
 185         gfn = (KVM_GUEST_CPHYSADDR(gva) >> PAGE_SHIFT);
 186
 187         if (gfn >= kvm->arch.guest_pmap_npages) {
 188                 kvm_err("%s: Invalid gfn: %#llx, GVA: %#lx\n", __func__, gfn,
 189                         gva);
 190                 return KVM_INVALID_PAGE;
 191         }
 192
 193         if (kvm_mips_map_page(vcpu->kvm, gfn) < 0)
 194                 return KVM_INVALID_ADDR;
 195
 196         return (kvm->arch.guest_pmap[gfn] << PAGE_SHIFT) + offset;
 197 }
 198
 199 /* XXXKYMA: Must be called with interrupts disabled */
 200 /* set flush_dcache_mask == 0 if no dcache flush required */
 201 int
 202 kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi,
 203         unsigned long entrylo0, unsigned long entrylo1, int flush_dcache_mask)
 204 {
 205         unsigned long flags;
 206         unsigned long old_entryhi;
 207         volatile int idx;
 208
 209         local_irq_save(flags);
 210
 211
 212         old_entryhi = read_c0_entryhi();
 213         write_c0_entryhi(entryhi);
 214         mtc0_tlbw_hazard();
 215
 216         tlb_probe();
 217         tlb_probe_hazard();
 218         idx = read_c0_index();
 219
 220         if (idx > current_cpu_data.tlbsize) {
 221                 kvm_err("%s: Invalid Index: %d\n", __func__, idx);
 222                 kvm_mips_dump_host_tlbs();
 223                 return -1;
 224         }
 225
 226         if (idx < 0) {
 227                 idx = read_c0_random() % current_cpu_data.tlbsize;
 228                 write_c0_index(idx);
 229                 mtc0_tlbw_hazard();
 230         }
 231         write_c0_entrylo0(entrylo0);
 232         write_c0_entrylo1(entrylo1);
 233         mtc0_tlbw_hazard();
 234
 235         tlb_write_indexed();
 236         tlbw_use_hazard();
 237
 238 #ifdef DEBUG
 239         if (debug) {
 240                 kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] "
 241                           "entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n",
 242                           vcpu->arch.pc, idx, read_c0_entryhi(),
 243                           read_c0_entrylo0(), read_c0_entrylo1());
 244         }
 245 #endif
 246
 247         /* Flush D-cache */
 248         if (flush_dcache_mask) {
 249                 if (entrylo0 & MIPS3_PG_V) {
 250                         ++vcpu->stat.flush_dcache_exits;
 251                         flush_data_cache_page((entryhi & VPN2_MASK) & ~flush_dcache_mask);
 252                 }
 253                 if (entrylo1 & MIPS3_PG_V) {
 254                         ++vcpu->stat.flush_dcache_exits;
 255                         flush_data_cache_page(((entryhi & VPN2_MASK) & ~flush_dcache_mask) |
 256                                 (0x1 << PAGE_SHIFT));
 257                 }
 258         }
 259
 260         /* Restore old ASID */
 261         write_c0_entryhi(old_entryhi);
 262         mtc0_tlbw_hazard();
 263         tlbw_use_hazard();
 264         local_irq_restore(flags);
 265         return 0;
 266 }
 267
 268
 269 /* XXXKYMA: Must be called with interrupts disabled */
 270 int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr,
 271         struct kvm_vcpu *vcpu)
 272 {
 273         gfn_t gfn;
 274         pfn_t pfn0, pfn1;
 275         unsigned long vaddr = 0;
 276         unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0;
 277         int even;
 278         struct kvm *kvm = vcpu->kvm;
 279         const int flush_dcache_mask = 0;
 280
 281
 282         if (KVM_GUEST_KSEGX(badvaddr) != KVM_GUEST_KSEG0) {
 283                 kvm_err("%s: Invalid BadVaddr: %#lx\n", __func__, badvaddr);
 284                 kvm_mips_dump_host_tlbs();
 285                 return -1;
 286         }
 287
 288         gfn = (KVM_GUEST_CPHYSADDR(badvaddr) >> PAGE_SHIFT);
 289         if (gfn >= kvm->arch.guest_pmap_npages) {
 290                 kvm_err("%s: Invalid gfn: %#llx, BadVaddr: %#lx\n", __func__,
 291                         gfn, badvaddr);
 292                 kvm_mips_dump_host_tlbs();
 293                 return -1;
 294         }
 295         even = !(gfn & 0x1);
 296         vaddr = badvaddr & (PAGE_MASK << 1);
 297
 298         if (kvm_mips_map_page(vcpu->kvm, gfn) < 0)
 299                 return -1;
 300
 301         if (kvm_mips_map_page(vcpu->kvm, gfn ^ 0x1) < 0)
 302                 return -1;
 303
 304         if (even) {
 305                 pfn0 = kvm->arch.guest_pmap[gfn];
 306                 pfn1 = kvm->arch.guest_pmap[gfn ^ 0x1];
 307         } else {
 308                 pfn0 = kvm->arch.guest_pmap[gfn ^ 0x1];
 309                 pfn1 = kvm->arch.guest_pmap[gfn];
 310         }
 311
 312         entryhi = (vaddr | kvm_mips_get_kernel_asid(vcpu));
 313         entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | (1 << 2) |
 314                         (0x1 << 1);
 315         entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) | (1 << 2) |
 316                         (0x1 << 1);
 317
 318         return kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1,
 319                                        flush_dcache_mask);
 320 }
 321
 322 int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
 323         struct kvm_vcpu *vcpu)
 324 {
 325         pfn_t pfn0, pfn1;
 326         unsigned long flags, old_entryhi = 0, vaddr = 0;
 327         unsigned long entrylo0 = 0, entrylo1 = 0;
 328
 329
 330         pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT;
 331         pfn1 = 0;
 332         entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | (1 << 2) |
 333                         (0x1 << 1);
 334         entrylo1 = 0;
 335
 336         local_irq_save(flags);
 337
 338         old_entryhi = read_c0_entryhi();
 339         vaddr = badvaddr & (PAGE_MASK << 1);
 340         write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu));
 341         mtc0_tlbw_hazard();
 342         write_c0_entrylo0(entrylo0);
 343         mtc0_tlbw_hazard();
 344         write_c0_entrylo1(entrylo1);
 345         mtc0_tlbw_hazard();
 346         write_c0_index(kvm_mips_get_commpage_asid(vcpu));
 347         mtc0_tlbw_hazard();
 348         tlb_write_indexed();
 349         mtc0_tlbw_hazard();
 350         tlbw_use_hazard();
 351
 352 #ifdef DEBUG
 353         kvm_debug ("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n",
 354              vcpu->arch.pc, read_c0_index(), read_c0_entryhi(),
 355              read_c0_entrylo0(), read_c0_entrylo1());
 356 #endif
 357
 358         /* Restore old ASID */
 359         write_c0_entryhi(old_entryhi);
 360         mtc0_tlbw_hazard();
 361         tlbw_use_hazard();
 362         local_irq_restore(flags);
 363
 364         return 0;
 365 }
 366
 367 int
 368 kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu,
 369         struct kvm_mips_tlb *tlb, unsigned long *hpa0, unsigned long *hpa1)
 370 {
 371         unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0;
 372         struct kvm *kvm = vcpu->kvm;
 373         pfn_t pfn0, pfn1;
 374
 375
 376         if ((tlb->tlb_hi & VPN2_MASK) == 0) {
 377                 pfn0 = 0;
 378                 pfn1 = 0;
 379         } else {
 380                 if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo0) >> PAGE_SHIFT) < 0)
 381                         return -1;
 382
 383                 if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo1) >> PAGE_SHIFT) < 0)
 384                         return -1;
 385
 386                 pfn0 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo0) >> PAGE_SHIFT];
 387                 pfn1 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo1) >> PAGE_SHIFT];
 388         }
 389
 390         if (hpa0)
 391                 *hpa0 = pfn0 << PAGE_SHIFT;
 392
 393         if (hpa1)
 394                 *hpa1 = pfn1 << PAGE_SHIFT;
 395
 396         /* Get attributes from the Guest TLB */
 397         entryhi = (tlb->tlb_hi & VPN2_MASK) | (KVM_GUEST_KERNEL_MODE(vcpu) ?
 398                         kvm_mips_get_kernel_asid(vcpu) : kvm_mips_get_user_asid(vcpu));
 399         entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) |
 400                         (tlb->tlb_lo0 & MIPS3_PG_D) | (tlb->tlb_lo0 & MIPS3_PG_V);
 401         entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) |
 402                         (tlb->tlb_lo1 & MIPS3_PG_D) | (tlb->tlb_lo1 & MIPS3_PG_V);
 403
 404 #ifdef DEBUG
 405         kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc,
 406                   tlb->tlb_lo0, tlb->tlb_lo1);
 407 #endif
 408
 409         return kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1,
 410                                        tlb->tlb_mask);
 411 }
 412
 413 int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
 414 {
 415         int i;
 416         int index = -1;
 417         struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
 418
 419
 420         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
 421                 if (((TLB_VPN2(tlb[i]) & ~tlb[i].tlb_mask) == ((entryhi & VPN2_MASK) & ~tlb[i].tlb_mask)) &&
 422                         (TLB_IS_GLOBAL(tlb[i]) || (TLB_ASID(tlb[i]) == (entryhi & ASID_MASK)))) {
 423                         index = i;
 424                         break;
 425                 }
 426         }
 427
 428 #ifdef DEBUG
 429         kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
 430                   __func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1);
 431 #endif
 432
 433         return index;
 434 }
 435
 436 int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr)
 437 {
 438         unsigned long old_entryhi, flags;
 439         volatile int idx;
 440
 441
 442         local_irq_save(flags);
 443
 444         old_entryhi = read_c0_entryhi();
 445
 446         if (KVM_GUEST_KERNEL_MODE(vcpu))
 447                 write_c0_entryhi((vaddr & VPN2_MASK) | kvm_mips_get_kernel_asid(vcpu));
 448         else {
 449                 write_c0_entryhi((vaddr & VPN2_MASK) | kvm_mips_get_user_asid(vcpu));
 450         }
 451
 452         mtc0_tlbw_hazard();
 453
 454         tlb_probe();
 455         tlb_probe_hazard();
 456         idx = read_c0_index();
 457
 458         /* Restore old ASID */
 459         write_c0_entryhi(old_entryhi);
 460         mtc0_tlbw_hazard();
 461         tlbw_use_hazard();
 462
 463         local_irq_restore(flags);
 464
 465 #ifdef DEBUG
 466         kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx);
 467 #endif
 468
 469         return idx;
 470 }
 471
 472 int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
 473 {
 474         int idx;
 475         unsigned long flags, old_entryhi;
 476
 477         local_irq_save(flags);
 478
 479
 480         old_entryhi = read_c0_entryhi();
 481
 482         write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu));
 483         mtc0_tlbw_hazard();
 484
 485         tlb_probe();
 486         tlb_probe_hazard();
 487         idx = read_c0_index();
 488
 489         if (idx >= current_cpu_data.tlbsize)
 490                 BUG();
 491
 492         if (idx > 0) {
 493                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 494                 mtc0_tlbw_hazard();
 495
 496                 write_c0_entrylo0(0);
 497                 mtc0_tlbw_hazard();
 498
 499                 write_c0_entrylo1(0);
 500                 mtc0_tlbw_hazard();
 501
 502                 tlb_write_indexed();
 503                 mtc0_tlbw_hazard();
 504         }
 505
 506         write_c0_entryhi(old_entryhi);
 507         mtc0_tlbw_hazard();
 508         tlbw_use_hazard();
 509
 510         local_irq_restore(flags);
 511
 512 #ifdef DEBUG
 513         if (idx > 0) {
 514                 kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__,
 515                           (va & VPN2_MASK) | (vcpu->arch.asid_map[va & ASID_MASK] & ASID_MASK), idx);
 516         }
 517 #endif
 518
 519         return 0;
 520 }
 521
 522 /* XXXKYMA: Fix Guest USER/KERNEL no longer share the same ASID*/
 523 int kvm_mips_host_tlb_inv_index(struct kvm_vcpu *vcpu, int index)
 524 {
 525         unsigned long flags, old_entryhi;
 526
 527         if (index >= current_cpu_data.tlbsize)
 528                 BUG();
 529
 530         local_irq_save(flags);
 531
 532
 533         old_entryhi = read_c0_entryhi();
 534
 535         write_c0_entryhi(UNIQUE_ENTRYHI(index));
 536         mtc0_tlbw_hazard();
 537
 538         write_c0_index(index);
 539         mtc0_tlbw_hazard();
 540
 541         write_c0_entrylo0(0);
 542         mtc0_tlbw_hazard();
 543
 544         write_c0_entrylo1(0);
 545         mtc0_tlbw_hazard();
 546
 547         tlb_write_indexed();
 548         mtc0_tlbw_hazard();
 549         tlbw_use_hazard();
 550
 551         write_c0_entryhi(old_entryhi);
 552         mtc0_tlbw_hazard();
 553         tlbw_use_hazard();
 554
 555         local_irq_restore(flags);
 556
 557         return 0;
 558 }
 559
 560 void kvm_mips_flush_host_tlb(int skip_kseg0)
 561 {
 562         unsigned long flags;
 563         unsigned long old_entryhi, entryhi;
 564         unsigned long old_pagemask;
 565         int entry = 0;
 566         int maxentry = current_cpu_data.tlbsize;
 567
 568
 569         local_irq_save(flags);
 570
 571         old_entryhi = read_c0_entryhi();
 572         old_pagemask = read_c0_pagemask();
 573
 574         /* Blast 'em all away. */
 575         for (entry = 0; entry < maxentry; entry++) {
 576
 577                 write_c0_index(entry);
 578                 mtc0_tlbw_hazard();
 579
 580                 if (skip_kseg0) {
 581                         tlb_read();
 582                         tlbw_use_hazard();
 583
 584                         entryhi = read_c0_entryhi();
 585
 586                         /* Don't blow away guest kernel entries */
 587                         if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0) {
 588                                 continue;
 589                         }
 590                 }
 591
 592                 /* Make sure all entries differ. */
 593                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 594                 mtc0_tlbw_hazard();
 595                 write_c0_entrylo0(0);
 596                 mtc0_tlbw_hazard();
 597                 write_c0_entrylo1(0);
 598                 mtc0_tlbw_hazard();
 599
 600                 tlb_write_indexed();
 601                 mtc0_tlbw_hazard();
 602         }
 603
 604         tlbw_use_hazard();
 605
 606         write_c0_entryhi(old_entryhi);
 607         write_c0_pagemask(old_pagemask);
 608         mtc0_tlbw_hazard();
 609         tlbw_use_hazard();
 610
 611         local_irq_restore(flags);
 612 }
 613
 614 void
 615 kvm_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu,
 616                         struct kvm_vcpu *vcpu)
 617 {
 618         unsigned long asid = asid_cache(cpu);
 619
 620         if (!((asid += ASID_INC) & ASID_MASK)) {
 621                 if (cpu_has_vtag_icache) {
 622                         flush_icache_all();
 623                 }
 624
 625                 kvm_local_flush_tlb_all();      /* start new asid cycle */
 626
 627                 if (!asid)      /* fix version if needed */
 628                         asid = ASID_FIRST_VERSION;
 629         }
 630
 631         cpu_context(cpu, mm) = asid_cache(cpu) = asid;
 632 }
 633
 634 void kvm_local_flush_tlb_all(void)
 635 {
 636         unsigned long flags;
 637         unsigned long old_ctx;
 638         int entry = 0;
 639
 640         local_irq_save(flags);
 641         /* Save old context and create impossible VPN2 value */
 642         old_ctx = read_c0_entryhi();
 643         write_c0_entrylo0(0);
 644         write_c0_entrylo1(0);
 645
 646         /* Blast 'em all away. */
 647         while (entry < current_cpu_data.tlbsize) {
 648                 /* Make sure all entries differ. */
 649                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 650                 write_c0_index(entry);
 651                 mtc0_tlbw_hazard();
 652                 tlb_write_indexed();
 653                 entry++;
 654         }
 655         tlbw_use_hazard();
 656         write_c0_entryhi(old_ctx);
 657         mtc0_tlbw_hazard();
 658
 659         local_irq_restore(flags);
 660 }
 661
 662 /* Restore ASID once we are scheduled back after preemption */
 663 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 664 {
 665         unsigned long flags;
 666         int newasid = 0;
 667
 668 #ifdef DEBUG
 669         kvm_debug("%s: vcpu %p, cpu: %d\n", __func__, vcpu, cpu);
 670 #endif
 671
 672         /* Alocate new kernel and user ASIDs if needed */
 673
 674         local_irq_save(flags);
 675
 676         if (((vcpu->arch.
 677               guest_kernel_asid[cpu] ^ asid_cache(cpu)) & ASID_VERSION_MASK)) {
 678                 kvm_get_new_mmu_context(&vcpu->arch.guest_kernel_mm, cpu, vcpu);
 679                 vcpu->arch.guest_kernel_asid[cpu] =
 680                     vcpu->arch.guest_kernel_mm.context.asid[cpu];
 681                 kvm_get_new_mmu_context(&vcpu->arch.guest_user_mm, cpu, vcpu);
 682                 vcpu->arch.guest_user_asid[cpu] =
 683                     vcpu->arch.guest_user_mm.context.asid[cpu];
 684                 newasid++;
 685
 686                 kvm_info("[%d]: cpu_context: %#lx\n", cpu,
 687                          cpu_context(cpu, current->mm));
 688                 kvm_info("[%d]: Allocated new ASID for Guest Kernel: %#x\n",
 689                          cpu, vcpu->arch.guest_kernel_asid[cpu]);
 690                 kvm_info("[%d]: Allocated new ASID for Guest User: %#x\n", cpu,
 691                          vcpu->arch.guest_user_asid[cpu]);
 692         }
 693
 694         if (vcpu->arch.last_sched_cpu != cpu) {
 695                 kvm_info("[%d->%d]KVM VCPU[%d] switch\n",
 696                          vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id);
 697         }
 698
 699         if (!newasid) {
 700                 /* If we preempted while the guest was executing, then reload the pre-empted ASID */
 701                 if (current->flags & PF_VCPU) {
 702                         write_c0_entryhi(vcpu->arch.
 703                                          preempt_entryhi & ASID_MASK);
 704                         ehb();
 705                 }
 706         } else {
 707                 /* New ASIDs were allocated for the VM */
 708
 709                 /* Were we in guest context? If so then the pre-empted ASID is no longer
 710                  * valid, we need to set it to what it should be based on the mode of
 711                  * the Guest (Kernel/User)
 712                  */
 713                 if (current->flags & PF_VCPU) {
 714                         if (KVM_GUEST_KERNEL_MODE(vcpu))
 715                                 write_c0_entryhi(vcpu->arch.
 716                                                  guest_kernel_asid[cpu] &
 717                                                  ASID_MASK);
 718                         else
 719                                 write_c0_entryhi(vcpu->arch.
 720                                                  guest_user_asid[cpu] &
 721                                                  ASID_MASK);
 722                         ehb();
 723                 }
 724         }
 725
 726         local_irq_restore(flags);
 727
 728 }
 729
 730 /* ASID can change if another task is scheduled during preemption */
 731 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 732 {
 733         unsigned long flags;
 734         uint32_t cpu;
 735
 736         local_irq_save(flags);
 737
 738         cpu = smp_processor_id();
 739
 740
 741         vcpu->arch.preempt_entryhi = read_c0_entryhi();
 742         vcpu->arch.last_sched_cpu = cpu;
 743
 744         if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
 745              ASID_VERSION_MASK)) {
 746                 kvm_debug("%s: Dropping MMU Context:  %#lx\n", __func__,
 747                           cpu_context(cpu, current->mm));
 748                 drop_mmu_context(current->mm, cpu);
 749         }
 750         write_c0_entryhi(cpu_asid(cpu, current->mm));
 751         ehb();
 752
 753         local_irq_restore(flags);
 754 }
 755
 756 uint32_t kvm_get_inst(uint32_t *opc, struct kvm_vcpu *vcpu)
 757 {
 758         struct mips_coproc *cop0 = vcpu->arch.cop0;
 759         unsigned long paddr, flags;
 760         uint32_t inst;
 761         int index;
 762
 763         if (KVM_GUEST_KSEGX((unsigned long) opc) < KVM_GUEST_KSEG0 ||
 764             KVM_GUEST_KSEGX((unsigned long) opc) == KVM_GUEST_KSEG23) {
 765                 local_irq_save(flags);
 766                 index = kvm_mips_host_tlb_lookup(vcpu, (unsigned long) opc);
 767                 if (index >= 0) {
 768                         inst = *(opc);
 769                 } else {
 770                         index =
 771                             kvm_mips_guest_tlb_lookup(vcpu,
 772                                                       ((unsigned long) opc & VPN2_MASK)
 773                                                       |
 774                                                       (kvm_read_c0_guest_entryhi
 775                                                        (cop0) & ASID_MASK));
 776                         if (index < 0) {
 777                                 kvm_err
 778                                     ("%s: get_user_failed for %p, vcpu: %p, ASID: %#lx\n",
 779                                      __func__, opc, vcpu, read_c0_entryhi());
 780                                 kvm_mips_dump_host_tlbs();
 781                                 local_irq_restore(flags);
 782                                 return KVM_INVALID_INST;
 783                         }
 784                         kvm_mips_handle_mapped_seg_tlb_fault(vcpu,
 785                                                              &vcpu->arch.
 786                                                              guest_tlb[index],
 787                                                              NULL, NULL);
 788                         inst = *(opc);
 789                 }
 790                 local_irq_restore(flags);
 791         } else if (KVM_GUEST_KSEGX(opc) == KVM_GUEST_KSEG0) {
 792                 paddr =
 793                     kvm_mips_translate_guest_kseg0_to_hpa(vcpu,
 794                                                          (unsigned long) opc);
 795                 inst = *(uint32_t *) CKSEG0ADDR(paddr);
 796         } else {
 797                 kvm_err("%s: illegal address: %p\n", __func__, opc);
 798                 return KVM_INVALID_INST;
 799         }
 800
 801         return inst;
 802 }
 803
 804 EXPORT_SYMBOL(kvm_local_flush_tlb_all);
 805 EXPORT_SYMBOL(kvm_mips_handle_mapped_seg_tlb_fault);
 806 EXPORT_SYMBOL(kvm_mips_handle_commpage_tlb_fault);
 807 EXPORT_SYMBOL(kvm_mips_dump_host_tlbs);
 808 EXPORT_SYMBOL(kvm_mips_handle_kseg0_tlb_fault);
 809 EXPORT_SYMBOL(kvm_mips_host_tlb_lookup);
 810 EXPORT_SYMBOL(kvm_mips_flush_host_tlb);
 811 EXPORT_SYMBOL(kvm_mips_guest_tlb_lookup);
 812 EXPORT_SYMBOL(kvm_mips_host_tlb_inv);
 813 EXPORT_SYMBOL(kvm_mips_translate_guest_kseg0_to_hpa);
 814 EXPORT_SYMBOL(kvm_mips_dump_guest_tlbs);
 815 EXPORT_SYMBOL(kvm_get_inst);
 816 EXPORT_SYMBOL(kvm_arch_vcpu_load);
 817 EXPORT_SYMBOL(kvm_arch_vcpu_put);