arch/mips/cavium-octeon/executive/cvmx-l2c.c

   1 /***********************license start***************
   2  * Author: Cavium Networks
   3  *
   4  * Contact: support@caviumnetworks.com
   5  * This file is part of the OCTEON SDK
   6  *
   7  * Copyright (c) 2003-2010 Cavium Networks
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more
  17  * details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this file; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  22  * or visit http://www.gnu.org/licenses/.
  23  *
  24  * This file may also be available under a different license from Cavium.
  25  * Contact Cavium Networks for more information
  26  ***********************license end**************************************/
  27
  28 /*
  29  * Implementation of the Level 2 Cache (L2C) control,
  30  * measurement, and debugging facilities.
  31  */
  32
  33 #include <linux/compiler.h>
  34 #include <linux/irqflags.h>
  35 #include <asm/octeon/cvmx.h>
  36 #include <asm/octeon/cvmx-l2c.h>
  37 #include <asm/octeon/cvmx-spinlock.h>
  38
  39 /*
  40  * This spinlock is used internally to ensure that only one core is
  41  * performing certain L2 operations at a time.
  42  *
  43  * NOTE: This only protects calls from within a single application -
  44  * if multiple applications or operating systems are running, then it
  45  * is up to the user program to coordinate between them.
  46  */
  47 cvmx_spinlock_t cvmx_l2c_spinlock;
  48
  49 int cvmx_l2c_get_core_way_partition(uint32_t core)
  50 {
  51         uint32_t field;
  52
  53         /* Validate the core number */
  54         if (core >= cvmx_octeon_num_cores())
  55                 return -1;
  56
  57         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
  58                 return cvmx_read_csr(CVMX_L2C_WPAR_PPX(core)) & 0xffff;
  59
  60         /*
  61          * Use the lower two bits of the coreNumber to determine the
  62          * bit offset of the UMSK[] field in the L2C_SPAR register.
  63          */
  64         field = (core & 0x3) * 8;
  65
  66         /*
  67          * Return the UMSK[] field from the appropriate L2C_SPAR
  68          * register based on the coreNumber.
  69          */
  70
  71         switch (core & 0xC) {
  72         case 0x0:
  73                 return (cvmx_read_csr(CVMX_L2C_SPAR0) & (0xFF << field)) >> field;
  74         case 0x4:
  75                 return (cvmx_read_csr(CVMX_L2C_SPAR1) & (0xFF << field)) >> field;
  76         case 0x8:
  77                 return (cvmx_read_csr(CVMX_L2C_SPAR2) & (0xFF << field)) >> field;
  78         case 0xC:
  79                 return (cvmx_read_csr(CVMX_L2C_SPAR3) & (0xFF << field)) >> field;
  80         }
  81         return 0;
  82 }
  83
  84 int cvmx_l2c_set_core_way_partition(uint32_t core, uint32_t mask)
  85 {
  86         uint32_t field;
  87         uint32_t valid_mask;
  88
  89         valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
  90
  91         mask &= valid_mask;
  92
  93         /* A UMSK setting which blocks all L2C Ways is an error on some chips */
  94         if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX))
  95                 return -1;
  96
  97         /* Validate the core number */
  98         if (core >= cvmx_octeon_num_cores())
  99                 return -1;
 100
 101         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 102                 cvmx_write_csr(CVMX_L2C_WPAR_PPX(core), mask);
 103                 return 0;
 104         }
 105
 106         /*
 107          * Use the lower two bits of core to determine the bit offset of the
 108          * UMSK[] field in the L2C_SPAR register.
 109          */
 110         field = (core & 0x3) * 8;
 111
 112         /*
 113          * Assign the new mask setting to the UMSK[] field in the appropriate
 114          * L2C_SPAR register based on the core_num.
 115          *
 116          */
 117         switch (core & 0xC) {
 118         case 0x0:
 119                 cvmx_write_csr(CVMX_L2C_SPAR0,
 120                                (cvmx_read_csr(CVMX_L2C_SPAR0) & ~(0xFF << field)) |
 121                                mask << field);
 122                 break;
 123         case 0x4:
 124                 cvmx_write_csr(CVMX_L2C_SPAR1,
 125                                (cvmx_read_csr(CVMX_L2C_SPAR1) & ~(0xFF << field)) |
 126                                mask << field);
 127                 break;
 128         case 0x8:
 129                 cvmx_write_csr(CVMX_L2C_SPAR2,
 130                                (cvmx_read_csr(CVMX_L2C_SPAR2) & ~(0xFF << field)) |
 131                                mask << field);
 132                 break;
 133         case 0xC:
 134                 cvmx_write_csr(CVMX_L2C_SPAR3,
 135                                (cvmx_read_csr(CVMX_L2C_SPAR3) & ~(0xFF << field)) |
 136                                mask << field);
 137                 break;
 138         }
 139         return 0;
 140 }
 141
 142 int cvmx_l2c_set_hw_way_partition(uint32_t mask)
 143 {
 144         uint32_t valid_mask;
 145
 146         valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
 147         mask &= valid_mask;
 148
 149         /* A UMSK setting which blocks all L2C Ways is an error on some chips */
 150         if (mask == valid_mask  && !OCTEON_IS_MODEL(OCTEON_CN63XX))
 151                 return -1;
 152
 153         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 154                 cvmx_write_csr(CVMX_L2C_WPAR_IOBX(0), mask);
 155         else
 156                 cvmx_write_csr(CVMX_L2C_SPAR4,
 157                                (cvmx_read_csr(CVMX_L2C_SPAR4) & ~0xFF) | mask);
 158         return 0;
 159 }
 160
 161 int cvmx_l2c_get_hw_way_partition(void)
 162 {
 163         if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 164                 return cvmx_read_csr(CVMX_L2C_WPAR_IOBX(0)) & 0xffff;
 165         else
 166                 return cvmx_read_csr(CVMX_L2C_SPAR4) & (0xFF);
 167 }
 168
 169 void cvmx_l2c_config_perf(uint32_t counter, enum cvmx_l2c_event event,
 170                           uint32_t clear_on_read)
 171 {
 172         if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 173                 union cvmx_l2c_pfctl pfctl;
 174
 175                 pfctl.u64 = cvmx_read_csr(CVMX_L2C_PFCTL);
 176
 177                 switch (counter) {
 178                 case 0:
 179                         pfctl.s.cnt0sel = event;
 180                         pfctl.s.cnt0ena = 1;
 181                         pfctl.s.cnt0rdclr = clear_on_read;
 182                         break;
 183                 case 1:
 184                         pfctl.s.cnt1sel = event;
 185                         pfctl.s.cnt1ena = 1;
 186                         pfctl.s.cnt1rdclr = clear_on_read;
 187                         break;
 188                 case 2:
 189                         pfctl.s.cnt2sel = event;
 190                         pfctl.s.cnt2ena = 1;
 191                         pfctl.s.cnt2rdclr = clear_on_read;
 192                         break;
 193                 case 3:
 194                 default:
 195                         pfctl.s.cnt3sel = event;
 196                         pfctl.s.cnt3ena = 1;
 197                         pfctl.s.cnt3rdclr = clear_on_read;
 198                         break;
 199                 }
 200
 201                 cvmx_write_csr(CVMX_L2C_PFCTL, pfctl.u64);
 202         } else {
 203                 union cvmx_l2c_tadx_prf l2c_tadx_prf;
 204                 int tad;
 205
 206                 cvmx_dprintf("L2C performance counter events are different for this chip, mapping 'event' to cvmx_l2c_tad_event_t\n");
 207                 if (clear_on_read)
 208                         cvmx_dprintf("L2C counters don't support clear on read for this chip\n");
 209
 210                 l2c_tadx_prf.u64 = cvmx_read_csr(CVMX_L2C_TADX_PRF(0));
 211
 212                 switch (counter) {
 213                 case 0:
 214                         l2c_tadx_prf.s.cnt0sel = event;
 215                         break;
 216                 case 1:
 217                         l2c_tadx_prf.s.cnt1sel = event;
 218                         break;
 219                 case 2:
 220                         l2c_tadx_prf.s.cnt2sel = event;
 221                         break;
 222                 default:
 223                 case 3:
 224                         l2c_tadx_prf.s.cnt3sel = event;
 225                         break;
 226                 }
 227                 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 228                         cvmx_write_csr(CVMX_L2C_TADX_PRF(tad),
 229                                        l2c_tadx_prf.u64);
 230         }
 231 }
 232
 233 uint64_t cvmx_l2c_read_perf(uint32_t counter)
 234 {
 235         switch (counter) {
 236         case 0:
 237                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 238                         return cvmx_read_csr(CVMX_L2C_PFC0);
 239                 else {
 240                         uint64_t counter = 0;
 241                         int tad;
 242                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 243                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC0(tad));
 244                         return counter;
 245                 }
 246         case 1:
 247                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 248                         return cvmx_read_csr(CVMX_L2C_PFC1);
 249                 else {
 250                         uint64_t counter = 0;
 251                         int tad;
 252                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 253                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC1(tad));
 254                         return counter;
 255                 }
 256         case 2:
 257                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 258                         return cvmx_read_csr(CVMX_L2C_PFC2);
 259                 else {
 260                         uint64_t counter = 0;
 261                         int tad;
 262                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 263                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC2(tad));
 264                         return counter;
 265                 }
 266         case 3:
 267         default:
 268                 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
 269                         return cvmx_read_csr(CVMX_L2C_PFC3);
 270                 else {
 271                         uint64_t counter = 0;
 272                         int tad;
 273                         for (tad = 0; tad < CVMX_L2C_TADS; tad++)
 274                                 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC3(tad));
 275                         return counter;
 276                 }
 277         }
 278 }
 279
 280 /**
 281  * @INTERNAL
 282  * Helper function use to fault in cache lines for L2 cache locking
 283  *
 284  * @addr:   Address of base of memory region to read into L2 cache
 285  * @len:    Length (in bytes) of region to fault in
 286  */
 287 static void fault_in(uint64_t addr, int len)
 288 {
 289         char *ptr;
 290
 291         /*
 292          * Adjust addr and length so we get all cache lines even for
 293          * small ranges spanning two cache lines.
 294          */
 295         len += addr & CVMX_CACHE_LINE_MASK;
 296         addr &= ~CVMX_CACHE_LINE_MASK;
 297         ptr = cvmx_phys_to_ptr(addr);
 298         /*
 299          * Invalidate L1 cache to make sure all loads result in data
 300          * being in L2.
 301          */
 302         CVMX_DCACHE_INVALIDATE;
 303         while (len > 0) {
 304                 ACCESS_ONCE(*ptr);
 305                 len -= CVMX_CACHE_LINE_SIZE;
 306                 ptr += CVMX_CACHE_LINE_SIZE;
 307         }
 308 }
 309
 310 int cvmx_l2c_lock_line(uint64_t addr)
 311 {
 312         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 313                 int shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
 314                 uint64_t assoc = cvmx_l2c_get_num_assoc();
 315                 uint64_t tag = addr >> shift;
 316                 uint64_t index = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, cvmx_l2c_address_to_index(addr) << CVMX_L2C_IDX_ADDR_SHIFT);
 317                 uint64_t way;
 318                 union cvmx_l2c_tadx_tag l2c_tadx_tag;
 319
 320                 CVMX_CACHE_LCKL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, addr), 0);
 321
 322                 /* Make sure we were able to lock the line */
 323                 for (way = 0; way < assoc; way++) {
 324                         CVMX_CACHE_LTGL2I(index | (way << shift), 0);
 325                         /* make sure CVMX_L2C_TADX_TAG is updated */
 326                         CVMX_SYNC;
 327                         l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
 328                         if (l2c_tadx_tag.s.valid && l2c_tadx_tag.s.tag == tag)
 329                                 break;
 330                 }
 331
 332                 /* Check if a valid line is found */
 333                 if (way >= assoc) {
 334                         /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: line not found for locking at 0x%llx address\n", (unsigned long long)addr); */
 335                         return -1;
 336                 }
 337
 338                 /* Check if lock bit is not set */
 339                 if (!l2c_tadx_tag.s.lock) {
 340                         /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: Not able to lock at 0x%llx address\n", (unsigned long long)addr); */
 341                         return -1;
 342                 }
 343                 return way;
 344         } else {
 345                 int retval = 0;
 346                 union cvmx_l2c_dbg l2cdbg;
 347                 union cvmx_l2c_lckbase lckbase;
 348                 union cvmx_l2c_lckoff lckoff;
 349                 union cvmx_l2t_err l2t_err;
 350
 351                 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
 352
 353                 l2cdbg.u64 = 0;
 354                 lckbase.u64 = 0;
 355                 lckoff.u64 = 0;
 356
 357                 /* Clear l2t error bits if set */
 358                 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
 359                 l2t_err.s.lckerr = 1;
 360                 l2t_err.s.lckerr2 = 1;
 361                 cvmx_write_csr(CVMX_L2T_ERR, l2t_err.u64);
 362
 363                 addr &= ~CVMX_CACHE_LINE_MASK;
 364
 365                 /* Set this core as debug core */
 366                 l2cdbg.s.ppnum = cvmx_get_core_num();
 367                 CVMX_SYNC;
 368                 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
 369                 cvmx_read_csr(CVMX_L2C_DBG);
 370
 371                 lckoff.s.lck_offset = 0; /* Only lock 1 line at a time */
 372                 cvmx_write_csr(CVMX_L2C_LCKOFF, lckoff.u64);
 373                 cvmx_read_csr(CVMX_L2C_LCKOFF);
 374
 375                 if (((union cvmx_l2c_cfg)(cvmx_read_csr(CVMX_L2C_CFG))).s.idxalias) {
 376                         int alias_shift = CVMX_L2C_IDX_ADDR_SHIFT + 2 * CVMX_L2_SET_BITS - 1;
 377                         uint64_t addr_tmp = addr ^ (addr & ((1 << alias_shift) - 1)) >> CVMX_L2_SET_BITS;
 378                         lckbase.s.lck_base = addr_tmp >> 7;
 379                 } else {
 380                         lckbase.s.lck_base = addr >> 7;
 381                 }
 382
 383                 lckbase.s.lck_ena = 1;
 384                 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
 385                 /* Make sure it gets there */
 386                 cvmx_read_csr(CVMX_L2C_LCKBASE);
 387
 388                 fault_in(addr, CVMX_CACHE_LINE_SIZE);
 389
 390                 lckbase.s.lck_ena = 0;
 391                 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
 392                 /* Make sure it gets there */
 393                 cvmx_read_csr(CVMX_L2C_LCKBASE);
 394
 395                 /* Stop being debug core */
 396                 cvmx_write_csr(CVMX_L2C_DBG, 0);
 397                 cvmx_read_csr(CVMX_L2C_DBG);
 398
 399                 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
 400                 if (l2t_err.s.lckerr || l2t_err.s.lckerr2)
 401                         retval = 1;  /* We were unable to lock the line */
 402
 403                 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
 404                 return retval;
 405         }
 406 }
 407
 408 int cvmx_l2c_lock_mem_region(uint64_t start, uint64_t len)
 409 {
 410         int retval = 0;
 411
 412         /* Round start/end to cache line boundaries */
 413         len += start & CVMX_CACHE_LINE_MASK;
 414         start &= ~CVMX_CACHE_LINE_MASK;
 415         len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
 416
 417         while (len) {
 418                 retval += cvmx_l2c_lock_line(start);
 419                 start += CVMX_CACHE_LINE_SIZE;
 420                 len -= CVMX_CACHE_LINE_SIZE;
 421         }
 422         return retval;
 423 }
 424
 425 void cvmx_l2c_flush(void)
 426 {
 427         uint64_t assoc, set;
 428         uint64_t n_assoc, n_set;
 429
 430         n_set = cvmx_l2c_get_num_sets();
 431         n_assoc = cvmx_l2c_get_num_assoc();
 432
 433         if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 434                 uint64_t address;
 435                 /* These may look like constants, but they aren't... */
 436                 int assoc_shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
 437                 int set_shift = CVMX_L2C_IDX_ADDR_SHIFT;
 438                 for (set = 0; set < n_set; set++) {
 439                         for (assoc = 0; assoc < n_assoc; assoc++) {
 440                                 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 441                                                        (assoc << assoc_shift) | (set << set_shift));
 442                                 CVMX_CACHE_WBIL2I(address, 0);
 443                         }
 444                 }
 445         } else {
 446                 for (set = 0; set < n_set; set++)
 447                         for (assoc = 0; assoc < n_assoc; assoc++)
 448                                 cvmx_l2c_flush_line(assoc, set);
 449         }
 450 }
 451
 452
 453 int cvmx_l2c_unlock_line(uint64_t address)
 454 {
 455
 456         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 457                 int assoc;
 458                 union cvmx_l2c_tag tag;
 459                 uint32_t tag_addr;
 460                 uint32_t index = cvmx_l2c_address_to_index(address);
 461
 462                 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
 463
 464                 /*
 465                  * For 63XX, we can flush a line by using the physical
 466                  * address directly, so finding the cache line used by
 467                  * the address is only required to provide the proper
 468                  * return value for the function.
 469                  */
 470                 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
 471                         tag = cvmx_l2c_get_tag(assoc, index);
 472
 473                         if (tag.s.V && (tag.s.addr == tag_addr)) {
 474                                 CVMX_CACHE_WBIL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, address), 0);
 475                                 return tag.s.L;
 476                         }
 477                 }
 478         } else {
 479                 int assoc;
 480                 union cvmx_l2c_tag tag;
 481                 uint32_t tag_addr;
 482
 483                 uint32_t index = cvmx_l2c_address_to_index(address);
 484
 485                 /* Compute portion of address that is stored in tag */
 486                 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
 487                 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
 488                         tag = cvmx_l2c_get_tag(assoc, index);
 489
 490                         if (tag.s.V && (tag.s.addr == tag_addr)) {
 491                                 cvmx_l2c_flush_line(assoc, index);
 492                                 return tag.s.L;
 493                         }
 494                 }
 495         }
 496         return 0;
 497 }
 498
 499 int cvmx_l2c_unlock_mem_region(uint64_t start, uint64_t len)
 500 {
 501         int num_unlocked = 0;
 502         /* Round start/end to cache line boundaries */
 503         len += start & CVMX_CACHE_LINE_MASK;
 504         start &= ~CVMX_CACHE_LINE_MASK;
 505         len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
 506         while (len > 0) {
 507                 num_unlocked += cvmx_l2c_unlock_line(start);
 508                 start += CVMX_CACHE_LINE_SIZE;
 509                 len -= CVMX_CACHE_LINE_SIZE;
 510         }
 511
 512         return num_unlocked;
 513 }
 514
 515 /*
 516  * Internal l2c tag types.  These are converted to a generic structure
 517  * that can be used on all chips.
 518  */
 519 union __cvmx_l2c_tag {
 520         uint64_t u64;
 521         struct cvmx_l2c_tag_cn50xx {
 522                 uint64_t reserved:40;
 523                 uint64_t V:1;           /* Line valid */
 524                 uint64_t D:1;           /* Line dirty */
 525                 uint64_t L:1;           /* Line locked */
 526                 uint64_t U:1;           /* Use, LRU eviction */
 527                 uint64_t addr:20;       /* Phys mem addr (33..14) */
 528         } cn50xx;
 529         struct cvmx_l2c_tag_cn30xx {
 530                 uint64_t reserved:41;
 531                 uint64_t V:1;           /* Line valid */
 532                 uint64_t D:1;           /* Line dirty */
 533                 uint64_t L:1;           /* Line locked */
 534                 uint64_t U:1;           /* Use, LRU eviction */
 535                 uint64_t addr:19;       /* Phys mem addr (33..15) */
 536         } cn30xx;
 537         struct cvmx_l2c_tag_cn31xx {
 538                 uint64_t reserved:42;
 539                 uint64_t V:1;           /* Line valid */
 540                 uint64_t D:1;           /* Line dirty */
 541                 uint64_t L:1;           /* Line locked */
 542                 uint64_t U:1;           /* Use, LRU eviction */
 543                 uint64_t addr:18;       /* Phys mem addr (33..16) */
 544         } cn31xx;
 545         struct cvmx_l2c_tag_cn38xx {
 546                 uint64_t reserved:43;
 547                 uint64_t V:1;           /* Line valid */
 548                 uint64_t D:1;           /* Line dirty */
 549                 uint64_t L:1;           /* Line locked */
 550                 uint64_t U:1;           /* Use, LRU eviction */
 551                 uint64_t addr:17;       /* Phys mem addr (33..17) */
 552         } cn38xx;
 553         struct cvmx_l2c_tag_cn58xx {
 554                 uint64_t reserved:44;
 555                 uint64_t V:1;           /* Line valid */
 556                 uint64_t D:1;           /* Line dirty */
 557                 uint64_t L:1;           /* Line locked */
 558                 uint64_t U:1;           /* Use, LRU eviction */
 559                 uint64_t addr:16;       /* Phys mem addr (33..18) */
 560         } cn58xx;
 561         struct cvmx_l2c_tag_cn58xx cn56xx;      /* 2048 sets */
 562         struct cvmx_l2c_tag_cn31xx cn52xx;      /* 512 sets */
 563 };
 564
 565
 566 /**
 567  * @INTERNAL
 568  * Function to read a L2C tag.  This code make the current core
 569  * the 'debug core' for the L2.  This code must only be executed by
 570  * 1 core at a time.
 571  *
 572  * @assoc:  Association (way) of the tag to dump
 573  * @index:  Index of the cacheline
 574  *
 575  * Returns The Octeon model specific tag structure.  This is
 576  *         translated by a wrapper function to a generic form that is
 577  *         easier for applications to use.
 578  */
 579 static union __cvmx_l2c_tag __read_l2_tag(uint64_t assoc, uint64_t index)
 580 {
 581
 582         uint64_t debug_tag_addr = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, (index << 7) + 96);
 583         uint64_t core = cvmx_get_core_num();
 584         union __cvmx_l2c_tag tag_val;
 585         uint64_t dbg_addr = CVMX_L2C_DBG;
 586         unsigned long flags;
 587
 588         union cvmx_l2c_dbg debug_val;
 589         debug_val.u64 = 0;
 590         /*
 591          * For low core count parts, the core number is always small
 592          * enough to stay in the correct field and not set any
 593          * reserved bits.
 594          */
 595         debug_val.s.ppnum = core;
 596         debug_val.s.l2t = 1;
 597         debug_val.s.set = assoc;
 598
 599         local_irq_save(flags);
 600         /*
 601          * Make sure core is quiet (no prefetches, etc.) before
 602          * entering debug mode.
 603          */
 604         CVMX_SYNC;
 605         /* Flush L1 to make sure debug load misses L1 */
 606         CVMX_DCACHE_INVALIDATE;
 607
 608         /*
 609          * The following must be done in assembly as when in debug
 610          * mode all data loads from L2 return special debug data, not
 611          * normal memory contents.  Also, interrupts must be disabled,
 612          * since if an interrupt occurs while in debug mode the ISR
 613          * will get debug data from all its memory * reads instead of
 614          * the contents of memory.
 615          */
 616
 617         asm volatile (
 618                 ".set push\n\t"
 619                 ".set mips64\n\t"
 620                 ".set noreorder\n\t"
 621                 "sd    %[dbg_val], 0(%[dbg_addr])\n\t"   /* Enter debug mode, wait for store */
 622                 "ld    $0, 0(%[dbg_addr])\n\t"
 623                 "ld    %[tag_val], 0(%[tag_addr])\n\t"   /* Read L2C tag data */
 624                 "sd    $0, 0(%[dbg_addr])\n\t"          /* Exit debug mode, wait for store */
 625                 "ld    $0, 0(%[dbg_addr])\n\t"
 626                 "cache 9, 0($0)\n\t"             /* Invalidate dcache to discard debug data */
 627                 ".set pop"
 628                 : [tag_val] "=r" (tag_val)
 629                 : [dbg_addr] "r" (dbg_addr), [dbg_val] "r" (debug_val), [tag_addr] "r" (debug_tag_addr)
 630                 : "memory");
 631
 632         local_irq_restore(flags);
 633
 634         return tag_val;
 635 }
 636
 637
 638 union cvmx_l2c_tag cvmx_l2c_get_tag(uint32_t association, uint32_t index)
 639 {
 640         union cvmx_l2c_tag tag;
 641         tag.u64 = 0;
 642
 643         if ((int)association >= cvmx_l2c_get_num_assoc()) {
 644                 cvmx_dprintf("ERROR: cvmx_l2c_get_tag association out of range\n");
 645                 return tag;
 646         }
 647         if ((int)index >= cvmx_l2c_get_num_sets()) {
 648                 cvmx_dprintf("ERROR: cvmx_l2c_get_tag index out of range (arg: %d, max: %d)\n",
 649                              (int)index, cvmx_l2c_get_num_sets());
 650                 return tag;
 651         }
 652         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 653                 union cvmx_l2c_tadx_tag l2c_tadx_tag;
 654                 uint64_t address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 655                                                 (association << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
 656                                                 (index << CVMX_L2C_IDX_ADDR_SHIFT));
 657                 /*
 658                  * Use L2 cache Index load tag cache instruction, as
 659                  * hardware loads the virtual tag for the L2 cache
 660                  * block with the contents of L2C_TAD0_TAG
 661                  * register.
 662                  */
 663                 CVMX_CACHE_LTGL2I(address, 0);
 664                 CVMX_SYNC;   /* make sure CVMX_L2C_TADX_TAG is updated */
 665                 l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
 666
 667                 tag.s.V     = l2c_tadx_tag.s.valid;
 668                 tag.s.D     = l2c_tadx_tag.s.dirty;
 669                 tag.s.L     = l2c_tadx_tag.s.lock;
 670                 tag.s.U     = l2c_tadx_tag.s.use;
 671                 tag.s.addr  = l2c_tadx_tag.s.tag;
 672         } else {
 673                 union __cvmx_l2c_tag tmp_tag;
 674                 /* __read_l2_tag is intended for internal use only */
 675                 tmp_tag = __read_l2_tag(association, index);
 676
 677                 /*
 678                  * Convert all tag structure types to generic version,
 679                  * as it can represent all models.
 680                  */
 681                 if (OCTEON_IS_MODEL(OCTEON_CN58XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
 682                         tag.s.V    = tmp_tag.cn58xx.V;
 683                         tag.s.D    = tmp_tag.cn58xx.D;
 684                         tag.s.L    = tmp_tag.cn58xx.L;
 685                         tag.s.U    = tmp_tag.cn58xx.U;
 686                         tag.s.addr = tmp_tag.cn58xx.addr;
 687                 } else if (OCTEON_IS_MODEL(OCTEON_CN38XX)) {
 688                         tag.s.V    = tmp_tag.cn38xx.V;
 689                         tag.s.D    = tmp_tag.cn38xx.D;
 690                         tag.s.L    = tmp_tag.cn38xx.L;
 691                         tag.s.U    = tmp_tag.cn38xx.U;
 692                         tag.s.addr = tmp_tag.cn38xx.addr;
 693                 } else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) {
 694                         tag.s.V    = tmp_tag.cn31xx.V;
 695                         tag.s.D    = tmp_tag.cn31xx.D;
 696                         tag.s.L    = tmp_tag.cn31xx.L;
 697                         tag.s.U    = tmp_tag.cn31xx.U;
 698                         tag.s.addr = tmp_tag.cn31xx.addr;
 699                 } else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) {
 700                         tag.s.V    = tmp_tag.cn30xx.V;
 701                         tag.s.D    = tmp_tag.cn30xx.D;
 702                         tag.s.L    = tmp_tag.cn30xx.L;
 703                         tag.s.U    = tmp_tag.cn30xx.U;
 704                         tag.s.addr = tmp_tag.cn30xx.addr;
 705                 } else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) {
 706                         tag.s.V    = tmp_tag.cn50xx.V;
 707                         tag.s.D    = tmp_tag.cn50xx.D;
 708                         tag.s.L    = tmp_tag.cn50xx.L;
 709                         tag.s.U    = tmp_tag.cn50xx.U;
 710                         tag.s.addr = tmp_tag.cn50xx.addr;
 711                 } else {
 712                         cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 713                 }
 714         }
 715         return tag;
 716 }
 717
 718 uint32_t cvmx_l2c_address_to_index(uint64_t addr)
 719 {
 720         uint64_t idx = addr >> CVMX_L2C_IDX_ADDR_SHIFT;
 721         int indxalias = 0;
 722
 723         if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 724                 union cvmx_l2c_ctl l2c_ctl;
 725                 l2c_ctl.u64 = cvmx_read_csr(CVMX_L2C_CTL);
 726                 indxalias = !l2c_ctl.s.disidxalias;
 727         } else {
 728                 union cvmx_l2c_cfg l2c_cfg;
 729                 l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG);
 730                 indxalias = l2c_cfg.s.idxalias;
 731         }
 732
 733         if (indxalias) {
 734                 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 735                         uint32_t a_14_12 = (idx / (CVMX_L2C_MEMBANK_SELECT_SIZE/(1<<CVMX_L2C_IDX_ADDR_SHIFT))) & 0x7;
 736                         idx ^= idx / cvmx_l2c_get_num_sets();
 737                         idx ^= a_14_12;
 738                 } else {
 739                         idx ^= ((addr & CVMX_L2C_ALIAS_MASK) >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT);
 740                 }
 741         }
 742         idx &= CVMX_L2C_IDX_MASK;
 743         return idx;
 744 }
 745
 746 int cvmx_l2c_get_cache_size_bytes(void)
 747 {
 748         return cvmx_l2c_get_num_sets() * cvmx_l2c_get_num_assoc() *
 749                 CVMX_CACHE_LINE_SIZE;
 750 }
 751
 752 /**
 753  * Return log base 2 of the number of sets in the L2 cache
 754  * Returns
 755  */
 756 int cvmx_l2c_get_set_bits(void)
 757 {
 758         int l2_set_bits;
 759         if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN58XX))
 760                 l2_set_bits = 11;       /* 2048 sets */
 761         else if (OCTEON_IS_MODEL(OCTEON_CN38XX) || OCTEON_IS_MODEL(OCTEON_CN63XX))
 762                 l2_set_bits = 10;       /* 1024 sets */
 763         else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX))
 764                 l2_set_bits = 9;        /* 512 sets */
 765         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 766                 l2_set_bits = 8;        /* 256 sets */
 767         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 768                 l2_set_bits = 7;        /* 128 sets */
 769         else {
 770                 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 771                 l2_set_bits = 11;       /* 2048 sets */
 772         }
 773         return l2_set_bits;
 774 }
 775
 776 /* Return the number of sets in the L2 Cache */
 777 int cvmx_l2c_get_num_sets(void)
 778 {
 779         return 1 << cvmx_l2c_get_set_bits();
 780 }
 781
 782 /* Return the number of associations in the L2 Cache */
 783 int cvmx_l2c_get_num_assoc(void)
 784 {
 785         int l2_assoc;
 786         if (OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 787             OCTEON_IS_MODEL(OCTEON_CN52XX) ||
 788             OCTEON_IS_MODEL(OCTEON_CN58XX) ||
 789             OCTEON_IS_MODEL(OCTEON_CN50XX) ||
 790             OCTEON_IS_MODEL(OCTEON_CN38XX))
 791                 l2_assoc = 8;
 792         else if (OCTEON_IS_MODEL(OCTEON_CN63XX))
 793                 l2_assoc = 16;
 794         else if (OCTEON_IS_MODEL(OCTEON_CN31XX) ||
 795                  OCTEON_IS_MODEL(OCTEON_CN30XX))
 796                 l2_assoc = 4;
 797         else {
 798                 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
 799                 l2_assoc = 8;
 800         }
 801
 802         /* Check to see if part of the cache is disabled */
 803         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 804                 union cvmx_mio_fus_dat3 mio_fus_dat3;
 805
 806                 mio_fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3);
 807                 /*
 808                  * cvmx_mio_fus_dat3.s.l2c_crip fuses map as follows
 809                  * <2> will be not used for 63xx
 810                  * <1> disables 1/2 ways
 811                  * <0> disables 1/4 ways
 812                  * They are cumulative, so for 63xx:
 813                  * <1> <0>
 814                  * 0 0 16-way 2MB cache
 815                  * 0 1 12-way 1.5MB cache
 816                  * 1 0 8-way 1MB cache
 817                  * 1 1 4-way 512KB cache
 818                  */
 819
 820                 if (mio_fus_dat3.s.l2c_crip == 3)
 821                         l2_assoc = 4;
 822                 else if (mio_fus_dat3.s.l2c_crip == 2)
 823                         l2_assoc = 8;
 824                 else if (mio_fus_dat3.s.l2c_crip == 1)
 825                         l2_assoc = 12;
 826         } else {
 827                 union cvmx_l2d_fus3 val;
 828                 val.u64 = cvmx_read_csr(CVMX_L2D_FUS3);
 829                 /*
 830                  * Using shifts here, as bit position names are
 831                  * different for each model but they all mean the
 832                  * same.
 833                  */
 834                 if ((val.u64 >> 35) & 0x1)
 835                         l2_assoc = l2_assoc >> 2;
 836                 else if ((val.u64 >> 34) & 0x1)
 837                         l2_assoc = l2_assoc >> 1;
 838         }
 839         return l2_assoc;
 840 }
 841
 842 /**
 843  * Flush a line from the L2 cache
 844  * This should only be called from one core at a time, as this routine
 845  * sets the core to the 'debug' core in order to flush the line.
 846  *
 847  * @assoc:  Association (or way) to flush
 848  * @index:  Index to flush
 849  */
 850 void cvmx_l2c_flush_line(uint32_t assoc, uint32_t index)
 851 {
 852         /* Check the range of the index. */
 853         if (index > (uint32_t)cvmx_l2c_get_num_sets()) {
 854                 cvmx_dprintf("ERROR: cvmx_l2c_flush_line index out of range.\n");
 855                 return;
 856         }
 857
 858         /* Check the range of association. */
 859         if (assoc > (uint32_t)cvmx_l2c_get_num_assoc()) {
 860                 cvmx_dprintf("ERROR: cvmx_l2c_flush_line association out of range.\n");
 861                 return;
 862         }
 863
 864         if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
 865                 uint64_t address;
 866                 /* Create the address based on index and association.
 867                  * Bits<20:17> select the way of the cache block involved in
 868                  *             the operation
 869                  * Bits<16:7> of the effect address select the index
 870                  */
 871                 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 872                                 (assoc << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
 873                                 (index << CVMX_L2C_IDX_ADDR_SHIFT));
 874                 CVMX_CACHE_WBIL2I(address, 0);
 875         } else {
 876                 union cvmx_l2c_dbg l2cdbg;
 877
 878                 l2cdbg.u64 = 0;
 879                 if (!OCTEON_IS_MODEL(OCTEON_CN30XX))
 880                         l2cdbg.s.ppnum = cvmx_get_core_num();
 881                 l2cdbg.s.finv = 1;
 882
 883                 l2cdbg.s.set = assoc;
 884                 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
 885                 /*
 886                  * Enter debug mode, and make sure all other writes
 887                  * complete before we enter debug mode
 888                  */
 889                 CVMX_SYNC;
 890                 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
 891                 cvmx_read_csr(CVMX_L2C_DBG);
 892
 893                 CVMX_PREPARE_FOR_STORE(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 894                                                     index * CVMX_CACHE_LINE_SIZE),
 895                                        0);
 896                 /* Exit debug mode */
 897                 CVMX_SYNC;
 898                 cvmx_write_csr(CVMX_L2C_DBG, 0);
 899                 cvmx_read_csr(CVMX_L2C_DBG);
 900                 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
 901         }
 902 }