drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/oom.h>
  35 #include <linux/shmem_fs.h>
  36 #include <linux/slab.h>
  37 #include <linux/swap.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-buf.h>
  40
  41 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  42 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
  43                                                    bool force);
  44 static __must_check int
  45 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  46                                bool readonly);
  47 static void
  48 i915_gem_object_retire(struct drm_i915_gem_object *obj);
  49
  50 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  51                                  struct drm_i915_gem_object *obj);
  52 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  53                                          struct drm_i915_fence_reg *fence,
  54                                          bool enable);
  55
  56 static unsigned long i915_gem_shrinker_count(struct shrinker *shrinker,
  57                                              struct shrink_control *sc);
  58 static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker,
  59                                             struct shrink_control *sc);
  60 static int i915_gem_shrinker_oom(struct notifier_block *nb,
  61                                  unsigned long event,
  62                                  void *ptr);
  63 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  64
  65 static bool cpu_cache_is_coherent(struct drm_device *dev,
  66                                   enum i915_cache_level level)
  67 {
  68         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  69 }
  70
  71 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  72 {
  73         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  74                 return true;
  75
  76         return obj->pin_display;
  77 }
  78
  79 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  80 {
  81         if (obj->tiling_mode)
  82                 i915_gem_release_mmap(obj);
  83
  84         /* As we do not have an associated fence register, we will force
  85          * a tiling change if we ever need to acquire one.
  86          */
  87         obj->fence_dirty = false;
  88         obj->fence_reg = I915_FENCE_REG_NONE;
  89 }
  90
  91 /* some bookkeeping */
  92 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  93                                   size_t size)
  94 {
  95         spin_lock(&dev_priv->mm.object_stat_lock);
  96         dev_priv->mm.object_count++;
  97         dev_priv->mm.object_memory += size;
  98         spin_unlock(&dev_priv->mm.object_stat_lock);
  99 }
 100
 101 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 102                                      size_t size)
 103 {
 104         spin_lock(&dev_priv->mm.object_stat_lock);
 105         dev_priv->mm.object_count--;
 106         dev_priv->mm.object_memory -= size;
 107         spin_unlock(&dev_priv->mm.object_stat_lock);
 108 }
 109
 110 static int
 111 i915_gem_wait_for_error(struct i915_gpu_error *error)
 112 {
 113         int ret;
 114
 115 #define EXIT_COND (!i915_reset_in_progress(error) || \
 116                    i915_terminally_wedged(error))
 117         if (EXIT_COND)
 118                 return 0;
 119
 120         /*
 121          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 122          * userspace. If it takes that long something really bad is going on and
 123          * we should simply try to bail out and fail as gracefully as possible.
 124          */
 125         ret = wait_event_interruptible_timeout(error->reset_queue,
 126                                                EXIT_COND,
 127                                                10*HZ);
 128         if (ret == 0) {
 129                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 130                 return -EIO;
 131         } else if (ret < 0) {
 132                 return ret;
 133         }
 134 #undef EXIT_COND
 135
 136         return 0;
 137 }
 138
 139 int i915_mutex_lock_interruptible(struct drm_device *dev)
 140 {
 141         struct drm_i915_private *dev_priv = dev->dev_private;
 142         int ret;
 143
 144         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 145         if (ret)
 146                 return ret;
 147
 148         ret = mutex_lock_interruptible(&dev->struct_mutex);
 149         if (ret)
 150                 return ret;
 151
 152         WARN_ON(i915_verify_lists(dev));
 153         return 0;
 154 }
 155
 156 static inline bool
 157 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 158 {
 159         return i915_gem_obj_bound_any(obj) && !obj->active;
 160 }
 161
 162 int
 163 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 164                             struct drm_file *file)
 165 {
 166         struct drm_i915_private *dev_priv = dev->dev_private;
 167         struct drm_i915_gem_get_aperture *args = data;
 168         struct drm_i915_gem_object *obj;
 169         size_t pinned;
 170
 171         pinned = 0;
 172         mutex_lock(&dev->struct_mutex);
 173         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
 174                 if (i915_gem_obj_is_pinned(obj))
 175                         pinned += i915_gem_obj_ggtt_size(obj);
 176         mutex_unlock(&dev->struct_mutex);
 177
 178         args->aper_size = dev_priv->gtt.base.total;
 179         args->aper_available_size = args->aper_size - pinned;
 180
 181         return 0;
 182 }
 183
 184 static int
 185 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 186 {
 187         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 188         char *vaddr = obj->phys_handle->vaddr;
 189         struct sg_table *st;
 190         struct scatterlist *sg;
 191         int i;
 192
 193         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 194                 return -EINVAL;
 195
 196         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 197                 struct page *page;
 198                 char *src;
 199
 200                 page = shmem_read_mapping_page(mapping, i);
 201                 if (IS_ERR(page))
 202                         return PTR_ERR(page);
 203
 204                 src = kmap_atomic(page);
 205                 memcpy(vaddr, src, PAGE_SIZE);
 206                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 207                 kunmap_atomic(src);
 208
 209                 page_cache_release(page);
 210                 vaddr += PAGE_SIZE;
 211         }
 212
 213         i915_gem_chipset_flush(obj->base.dev);
 214
 215         st = kmalloc(sizeof(*st), GFP_KERNEL);
 216         if (st == NULL)
 217                 return -ENOMEM;
 218
 219         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 220                 kfree(st);
 221                 return -ENOMEM;
 222         }
 223
 224         sg = st->sgl;
 225         sg->offset = 0;
 226         sg->length = obj->base.size;
 227
 228         sg_dma_address(sg) = obj->phys_handle->busaddr;
 229         sg_dma_len(sg) = obj->base.size;
 230
 231         obj->pages = st;
 232         obj->has_dma_mapping = true;
 233         return 0;
 234 }
 235
 236 static void
 237 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 238 {
 239         int ret;
 240
 241         BUG_ON(obj->madv == __I915_MADV_PURGED);
 242
 243         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 244         if (ret) {
 245                 /* In the event of a disaster, abandon all caches and
 246                  * hope for the best.
 247                  */
 248                 WARN_ON(ret != -EIO);
 249                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 250         }
 251
 252         if (obj->madv == I915_MADV_DONTNEED)
 253                 obj->dirty = 0;
 254
 255         if (obj->dirty) {
 256                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 257                 char *vaddr = obj->phys_handle->vaddr;
 258                 int i;
 259
 260                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 261                         struct page *page;
 262                         char *dst;
 263
 264                         page = shmem_read_mapping_page(mapping, i);
 265                         if (IS_ERR(page))
 266                                 continue;
 267
 268                         dst = kmap_atomic(page);
 269                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 270                         memcpy(dst, vaddr, PAGE_SIZE);
 271                         kunmap_atomic(dst);
 272
 273                         set_page_dirty(page);
 274                         if (obj->madv == I915_MADV_WILLNEED)
 275                                 mark_page_accessed(page);
 276                         page_cache_release(page);
 277                         vaddr += PAGE_SIZE;
 278                 }
 279                 obj->dirty = 0;
 280         }
 281
 282         sg_free_table(obj->pages);
 283         kfree(obj->pages);
 284
 285         obj->has_dma_mapping = false;
 286 }
 287
 288 static void
 289 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 290 {
 291         drm_pci_free(obj->base.dev, obj->phys_handle);
 292 }
 293
 294 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 295         .get_pages = i915_gem_object_get_pages_phys,
 296         .put_pages = i915_gem_object_put_pages_phys,
 297         .release = i915_gem_object_release_phys,
 298 };
 299
 300 static int
 301 drop_pages(struct drm_i915_gem_object *obj)
 302 {
 303         struct i915_vma *vma, *next;
 304         int ret;
 305
 306         drm_gem_object_reference(&obj->base);
 307         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
 308                 if (i915_vma_unbind(vma))
 309                         break;
 310
 311         ret = i915_gem_object_put_pages(obj);
 312         drm_gem_object_unreference(&obj->base);
 313
 314         return ret;
 315 }
 316
 317 int
 318 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 319                             int align)
 320 {
 321         drm_dma_handle_t *phys;
 322         int ret;
 323
 324         if (obj->phys_handle) {
 325                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 326                         return -EBUSY;
 327
 328                 return 0;
 329         }
 330
 331         if (obj->madv != I915_MADV_WILLNEED)
 332                 return -EFAULT;
 333
 334         if (obj->base.filp == NULL)
 335                 return -EINVAL;
 336
 337         ret = drop_pages(obj);
 338         if (ret)
 339                 return ret;
 340
 341         /* create a new object */
 342         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 343         if (!phys)
 344                 return -ENOMEM;
 345
 346         obj->phys_handle = phys;
 347         obj->ops = &i915_gem_phys_ops;
 348
 349         return i915_gem_object_get_pages(obj);
 350 }
 351
 352 static int
 353 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 354                      struct drm_i915_gem_pwrite *args,
 355                      struct drm_file *file_priv)
 356 {
 357         struct drm_device *dev = obj->base.dev;
 358         void *vaddr = obj->phys_handle->vaddr + args->offset;
 359         char __user *user_data = to_user_ptr(args->data_ptr);
 360         int ret;
 361
 362         /* We manually control the domain here and pretend that it
 363          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 364          */
 365         ret = i915_gem_object_wait_rendering(obj, false);
 366         if (ret)
 367                 return ret;
 368
 369         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 370                 unsigned long unwritten;
 371
 372                 /* The physical object once assigned is fixed for the lifetime
 373                  * of the obj, so we can safely drop the lock and continue
 374                  * to access vaddr.
 375                  */
 376                 mutex_unlock(&dev->struct_mutex);
 377                 unwritten = copy_from_user(vaddr, user_data, args->size);
 378                 mutex_lock(&dev->struct_mutex);
 379                 if (unwritten)
 380                         return -EFAULT;
 381         }
 382
 383         drm_clflush_virt_range(vaddr, args->size);
 384         i915_gem_chipset_flush(dev);
 385         return 0;
 386 }
 387
 388 void *i915_gem_object_alloc(struct drm_device *dev)
 389 {
 390         struct drm_i915_private *dev_priv = dev->dev_private;
 391         return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
 392 }
 393
 394 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 395 {
 396         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 397         kmem_cache_free(dev_priv->slab, obj);
 398 }
 399
 400 static int
 401 i915_gem_create(struct drm_file *file,
 402                 struct drm_device *dev,
 403                 uint64_t size,
 404                 uint32_t *handle_p)
 405 {
 406         struct drm_i915_gem_object *obj;
 407         int ret;
 408         u32 handle;
 409
 410         size = roundup(size, PAGE_SIZE);
 411         if (size == 0)
 412                 return -EINVAL;
 413
 414         /* Allocate the new object */
 415         obj = i915_gem_alloc_object(dev, size);
 416         if (obj == NULL)
 417                 return -ENOMEM;
 418
 419         ret = drm_gem_handle_create(file, &obj->base, &handle);
 420         /* drop reference from allocate - handle holds it now */
 421         drm_gem_object_unreference_unlocked(&obj->base);
 422         if (ret)
 423                 return ret;
 424
 425         *handle_p = handle;
 426         return 0;
 427 }
 428
 429 int
 430 i915_gem_dumb_create(struct drm_file *file,
 431                      struct drm_device *dev,
 432                      struct drm_mode_create_dumb *args)
 433 {
 434         /* have to work out size/pitch and return them */
 435         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 436         args->size = args->pitch * args->height;
 437         return i915_gem_create(file, dev,
 438                                args->size, &args->handle);
 439 }
 440
 441 /**
 442  * Creates a new mm object and returns a handle to it.
 443  */
 444 int
 445 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 446                       struct drm_file *file)
 447 {
 448         struct drm_i915_gem_create *args = data;
 449
 450         return i915_gem_create(file, dev,
 451                                args->size, &args->handle);
 452 }
 453
 454 static inline int
 455 __copy_to_user_swizzled(char __user *cpu_vaddr,
 456                         const char *gpu_vaddr, int gpu_offset,
 457                         int length)
 458 {
 459         int ret, cpu_offset = 0;
 460
 461         while (length > 0) {
 462                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 463                 int this_length = min(cacheline_end - gpu_offset, length);
 464                 int swizzled_gpu_offset = gpu_offset ^ 64;
 465
 466                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 467                                      gpu_vaddr + swizzled_gpu_offset,
 468                                      this_length);
 469                 if (ret)
 470                         return ret + length;
 471
 472                 cpu_offset += this_length;
 473                 gpu_offset += this_length;
 474                 length -= this_length;
 475         }
 476
 477         return 0;
 478 }
 479
 480 static inline int
 481 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 482                           const char __user *cpu_vaddr,
 483                           int length)
 484 {
 485         int ret, cpu_offset = 0;
 486
 487         while (length > 0) {
 488                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 489                 int this_length = min(cacheline_end - gpu_offset, length);
 490                 int swizzled_gpu_offset = gpu_offset ^ 64;
 491
 492                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 493                                        cpu_vaddr + cpu_offset,
 494                                        this_length);
 495                 if (ret)
 496                         return ret + length;
 497
 498                 cpu_offset += this_length;
 499                 gpu_offset += this_length;
 500                 length -= this_length;
 501         }
 502
 503         return 0;
 504 }
 505
 506 /*
 507  * Pins the specified object's pages and synchronizes the object with
 508  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 509  * flush the object from the CPU cache.
 510  */
 511 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 512                                     int *needs_clflush)
 513 {
 514         int ret;
 515
 516         *needs_clflush = 0;
 517
 518         if (!obj->base.filp)
 519                 return -EINVAL;
 520
 521         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 522                 /* If we're not in the cpu read domain, set ourself into the gtt
 523                  * read domain and manually flush cachelines (if required). This
 524                  * optimizes for the case when the gpu will dirty the data
 525                  * anyway again before the next pread happens. */
 526                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 527                                                         obj->cache_level);
 528                 ret = i915_gem_object_wait_rendering(obj, true);
 529                 if (ret)
 530                         return ret;
 531
 532                 i915_gem_object_retire(obj);
 533         }
 534
 535         ret = i915_gem_object_get_pages(obj);
 536         if (ret)
 537                 return ret;
 538
 539         i915_gem_object_pin_pages(obj);
 540
 541         return ret;
 542 }
 543
 544 /* Per-page copy function for the shmem pread fastpath.
 545  * Flushes invalid cachelines before reading the target if
 546  * needs_clflush is set. */
 547 static int
 548 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 549                  char __user *user_data,
 550                  bool page_do_bit17_swizzling, bool needs_clflush)
 551 {
 552         char *vaddr;
 553         int ret;
 554
 555         if (unlikely(page_do_bit17_swizzling))
 556                 return -EINVAL;
 557
 558         vaddr = kmap_atomic(page);
 559         if (needs_clflush)
 560                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 561                                        page_length);
 562         ret = __copy_to_user_inatomic(user_data,
 563                                       vaddr + shmem_page_offset,
 564                                       page_length);
 565         kunmap_atomic(vaddr);
 566
 567         return ret ? -EFAULT : 0;
 568 }
 569
 570 static void
 571 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 572                              bool swizzled)
 573 {
 574         if (unlikely(swizzled)) {
 575                 unsigned long start = (unsigned long) addr;
 576                 unsigned long end = (unsigned long) addr + length;
 577
 578                 /* For swizzling simply ensure that we always flush both
 579                  * channels. Lame, but simple and it works. Swizzled
 580                  * pwrite/pread is far from a hotpath - current userspace
 581                  * doesn't use it at all. */
 582                 start = round_down(start, 128);
 583                 end = round_up(end, 128);
 584
 585                 drm_clflush_virt_range((void *)start, end - start);
 586         } else {
 587                 drm_clflush_virt_range(addr, length);
 588         }
 589
 590 }
 591
 592 /* Only difference to the fast-path function is that this can handle bit17
 593  * and uses non-atomic copy and kmap functions. */
 594 static int
 595 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 596                  char __user *user_data,
 597                  bool page_do_bit17_swizzling, bool needs_clflush)
 598 {
 599         char *vaddr;
 600         int ret;
 601
 602         vaddr = kmap(page);
 603         if (needs_clflush)
 604                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 605                                              page_length,
 606                                              page_do_bit17_swizzling);
 607
 608         if (page_do_bit17_swizzling)
 609                 ret = __copy_to_user_swizzled(user_data,
 610                                               vaddr, shmem_page_offset,
 611                                               page_length);
 612         else
 613                 ret = __copy_to_user(user_data,
 614                                      vaddr + shmem_page_offset,
 615                                      page_length);
 616         kunmap(page);
 617
 618         return ret ? - EFAULT : 0;
 619 }
 620
 621 static int
 622 i915_gem_shmem_pread(struct drm_device *dev,
 623                      struct drm_i915_gem_object *obj,
 624                      struct drm_i915_gem_pread *args,
 625                      struct drm_file *file)
 626 {
 627         char __user *user_data;
 628         ssize_t remain;
 629         loff_t offset;
 630         int shmem_page_offset, page_length, ret = 0;
 631         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 632         int prefaulted = 0;
 633         int needs_clflush = 0;
 634         struct sg_page_iter sg_iter;
 635
 636         user_data = to_user_ptr(args->data_ptr);
 637         remain = args->size;
 638
 639         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 640
 641         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 642         if (ret)
 643                 return ret;
 644
 645         offset = args->offset;
 646
 647         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 648                          offset >> PAGE_SHIFT) {
 649                 struct page *page = sg_page_iter_page(&sg_iter);
 650
 651                 if (remain <= 0)
 652                         break;
 653
 654                 /* Operation in this page
 655                  *
 656                  * shmem_page_offset = offset within page in shmem file
 657                  * page_length = bytes to copy for this page
 658                  */
 659                 shmem_page_offset = offset_in_page(offset);
 660                 page_length = remain;
 661                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 662                         page_length = PAGE_SIZE - shmem_page_offset;
 663
 664                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 665                         (page_to_phys(page) & (1 << 17)) != 0;
 666
 667                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 668                                        user_data, page_do_bit17_swizzling,
 669                                        needs_clflush);
 670                 if (ret == 0)
 671                         goto next_page;
 672
 673                 mutex_unlock(&dev->struct_mutex);
 674
 675                 if (likely(!i915.prefault_disable) && !prefaulted) {
 676                         ret = fault_in_multipages_writeable(user_data, remain);
 677                         /* Userspace is tricking us, but we've already clobbered
 678                          * its pages with the prefault and promised to write the
 679                          * data up to the first fault. Hence ignore any errors
 680                          * and just continue. */
 681                         (void)ret;
 682                         prefaulted = 1;
 683                 }
 684
 685                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 686                                        user_data, page_do_bit17_swizzling,
 687                                        needs_clflush);
 688
 689                 mutex_lock(&dev->struct_mutex);
 690
 691                 if (ret)
 692                         goto out;
 693
 694 next_page:
 695                 remain -= page_length;
 696                 user_data += page_length;
 697                 offset += page_length;
 698         }
 699
 700 out:
 701         i915_gem_object_unpin_pages(obj);
 702
 703         return ret;
 704 }
 705
 706 /**
 707  * Reads data from the object referenced by handle.
 708  *
 709  * On error, the contents of *data are undefined.
 710  */
 711 int
 712 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 713                      struct drm_file *file)
 714 {
 715         struct drm_i915_gem_pread *args = data;
 716         struct drm_i915_gem_object *obj;
 717         int ret = 0;
 718
 719         if (args->size == 0)
 720                 return 0;
 721
 722         if (!access_ok(VERIFY_WRITE,
 723                        to_user_ptr(args->data_ptr),
 724                        args->size))
 725                 return -EFAULT;
 726
 727         ret = i915_mutex_lock_interruptible(dev);
 728         if (ret)
 729                 return ret;
 730
 731         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 732         if (&obj->base == NULL) {
 733                 ret = -ENOENT;
 734                 goto unlock;
 735         }
 736
 737         /* Bounds check source.  */
 738         if (args->offset > obj->base.size ||
 739             args->size > obj->base.size - args->offset) {
 740                 ret = -EINVAL;
 741                 goto out;
 742         }
 743
 744         /* prime objects have no backing filp to GEM pread/pwrite
 745          * pages from.
 746          */
 747         if (!obj->base.filp) {
 748                 ret = -EINVAL;
 749                 goto out;
 750         }
 751
 752         trace_i915_gem_object_pread(obj, args->offset, args->size);
 753
 754         ret = i915_gem_shmem_pread(dev, obj, args, file);
 755
 756 out:
 757         drm_gem_object_unreference(&obj->base);
 758 unlock:
 759         mutex_unlock(&dev->struct_mutex);
 760         return ret;
 761 }
 762
 763 /* This is the fast write path which cannot handle
 764  * page faults in the source data
 765  */
 766
 767 static inline int
 768 fast_user_write(struct io_mapping *mapping,
 769                 loff_t page_base, int page_offset,
 770                 char __user *user_data,
 771                 int length)
 772 {
 773         void __iomem *vaddr_atomic;
 774         void *vaddr;
 775         unsigned long unwritten;
 776
 777         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 778         /* We can use the cpu mem copy function because this is X86. */
 779         vaddr = (void __force*)vaddr_atomic + page_offset;
 780         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 781                                                       user_data, length);
 782         io_mapping_unmap_atomic(vaddr_atomic);
 783         return unwritten;
 784 }
 785
 786 /**
 787  * This is the fast pwrite path, where we copy the data directly from the
 788  * user into the GTT, uncached.
 789  */
 790 static int
 791 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 792                          struct drm_i915_gem_object *obj,
 793                          struct drm_i915_gem_pwrite *args,
 794                          struct drm_file *file)
 795 {
 796         struct drm_i915_private *dev_priv = dev->dev_private;
 797         ssize_t remain;
 798         loff_t offset, page_base;
 799         char __user *user_data;
 800         int page_offset, page_length, ret;
 801
 802         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 803         if (ret)
 804                 goto out;
 805
 806         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 807         if (ret)
 808                 goto out_unpin;
 809
 810         ret = i915_gem_object_put_fence(obj);
 811         if (ret)
 812                 goto out_unpin;
 813
 814         user_data = to_user_ptr(args->data_ptr);
 815         remain = args->size;
 816
 817         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
 818
 819         while (remain > 0) {
 820                 /* Operation in this page
 821                  *
 822                  * page_base = page offset within aperture
 823                  * page_offset = offset within page
 824                  * page_length = bytes to copy for this page
 825                  */
 826                 page_base = offset & PAGE_MASK;
 827                 page_offset = offset_in_page(offset);
 828                 page_length = remain;
 829                 if ((page_offset + remain) > PAGE_SIZE)
 830                         page_length = PAGE_SIZE - page_offset;
 831
 832                 /* If we get a fault while copying data, then (presumably) our
 833                  * source page isn't available.  Return the error and we'll
 834                  * retry in the slow path.
 835                  */
 836                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
 837                                     page_offset, user_data, page_length)) {
 838                         ret = -EFAULT;
 839                         goto out_unpin;
 840                 }
 841
 842                 remain -= page_length;
 843                 user_data += page_length;
 844                 offset += page_length;
 845         }
 846
 847 out_unpin:
 848         i915_gem_object_ggtt_unpin(obj);
 849 out:
 850         return ret;
 851 }
 852
 853 /* Per-page copy function for the shmem pwrite fastpath.
 854  * Flushes invalid cachelines before writing to the target if
 855  * needs_clflush_before is set and flushes out any written cachelines after
 856  * writing if needs_clflush is set. */
 857 static int
 858 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 859                   char __user *user_data,
 860                   bool page_do_bit17_swizzling,
 861                   bool needs_clflush_before,
 862                   bool needs_clflush_after)
 863 {
 864         char *vaddr;
 865         int ret;
 866
 867         if (unlikely(page_do_bit17_swizzling))
 868                 return -EINVAL;
 869
 870         vaddr = kmap_atomic(page);
 871         if (needs_clflush_before)
 872                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 873                                        page_length);
 874         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
 875                                         user_data, page_length);
 876         if (needs_clflush_after)
 877                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 878                                        page_length);
 879         kunmap_atomic(vaddr);
 880
 881         return ret ? -EFAULT : 0;
 882 }
 883
 884 /* Only difference to the fast-path function is that this can handle bit17
 885  * and uses non-atomic copy and kmap functions. */
 886 static int
 887 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 888                   char __user *user_data,
 889                   bool page_do_bit17_swizzling,
 890                   bool needs_clflush_before,
 891                   bool needs_clflush_after)
 892 {
 893         char *vaddr;
 894         int ret;
 895
 896         vaddr = kmap(page);
 897         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 898                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 899                                              page_length,
 900                                              page_do_bit17_swizzling);
 901         if (page_do_bit17_swizzling)
 902                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 903                                                 user_data,
 904                                                 page_length);
 905         else
 906                 ret = __copy_from_user(vaddr + shmem_page_offset,
 907                                        user_data,
 908                                        page_length);
 909         if (needs_clflush_after)
 910                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 911                                              page_length,
 912                                              page_do_bit17_swizzling);
 913         kunmap(page);
 914
 915         return ret ? -EFAULT : 0;
 916 }
 917
 918 static int
 919 i915_gem_shmem_pwrite(struct drm_device *dev,
 920                       struct drm_i915_gem_object *obj,
 921                       struct drm_i915_gem_pwrite *args,
 922                       struct drm_file *file)
 923 {
 924         ssize_t remain;
 925         loff_t offset;
 926         char __user *user_data;
 927         int shmem_page_offset, page_length, ret = 0;
 928         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 929         int hit_slowpath = 0;
 930         int needs_clflush_after = 0;
 931         int needs_clflush_before = 0;
 932         struct sg_page_iter sg_iter;
 933
 934         user_data = to_user_ptr(args->data_ptr);
 935         remain = args->size;
 936
 937         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 938
 939         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 940                 /* If we're not in the cpu write domain, set ourself into the gtt
 941                  * write domain and manually flush cachelines (if required). This
 942                  * optimizes for the case when the gpu will use the data
 943                  * right away and we therefore have to clflush anyway. */
 944                 needs_clflush_after = cpu_write_needs_clflush(obj);
 945                 ret = i915_gem_object_wait_rendering(obj, false);
 946                 if (ret)
 947                         return ret;
 948
 949                 i915_gem_object_retire(obj);
 950         }
 951         /* Same trick applies to invalidate partially written cachelines read
 952          * before writing. */
 953         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
 954                 needs_clflush_before =
 955                         !cpu_cache_is_coherent(dev, obj->cache_level);
 956
 957         ret = i915_gem_object_get_pages(obj);
 958         if (ret)
 959                 return ret;
 960
 961         i915_gem_object_pin_pages(obj);
 962
 963         offset = args->offset;
 964         obj->dirty = 1;
 965
 966         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 967                          offset >> PAGE_SHIFT) {
 968                 struct page *page = sg_page_iter_page(&sg_iter);
 969                 int partial_cacheline_write;
 970
 971                 if (remain <= 0)
 972                         break;
 973
 974                 /* Operation in this page
 975                  *
 976                  * shmem_page_offset = offset within page in shmem file
 977                  * page_length = bytes to copy for this page
 978                  */
 979                 shmem_page_offset = offset_in_page(offset);
 980
 981                 page_length = remain;
 982                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 983                         page_length = PAGE_SIZE - shmem_page_offset;
 984
 985                 /* If we don't overwrite a cacheline completely we need to be
 986                  * careful to have up-to-date data by first clflushing. Don't
 987                  * overcomplicate things and flush the entire patch. */
 988                 partial_cacheline_write = needs_clflush_before &&
 989                         ((shmem_page_offset | page_length)
 990                                 & (boot_cpu_data.x86_clflush_size - 1));
 991
 992                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 993                         (page_to_phys(page) & (1 << 17)) != 0;
 994
 995                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 996                                         user_data, page_do_bit17_swizzling,
 997                                         partial_cacheline_write,
 998                                         needs_clflush_after);
 999                 if (ret == 0)
1000                         goto next_page;
1001
1002                 hit_slowpath = 1;
1003                 mutex_unlock(&dev->struct_mutex);
1004                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1005                                         user_data, page_do_bit17_swizzling,
1006                                         partial_cacheline_write,
1007                                         needs_clflush_after);
1008
1009                 mutex_lock(&dev->struct_mutex);
1010
1011                 if (ret)
1012                         goto out;
1013
1014 next_page:
1015                 remain -= page_length;
1016                 user_data += page_length;
1017                 offset += page_length;
1018         }
1019
1020 out:
1021         i915_gem_object_unpin_pages(obj);
1022
1023         if (hit_slowpath) {
1024                 /*
1025                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1026                  * cachelines in-line while writing and the object moved
1027                  * out of the cpu write domain while we've dropped the lock.
1028                  */
1029                 if (!needs_clflush_after &&
1030                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1031                         if (i915_gem_clflush_object(obj, obj->pin_display))
1032                                 i915_gem_chipset_flush(dev);
1033                 }
1034         }
1035
1036         if (needs_clflush_after)
1037                 i915_gem_chipset_flush(dev);
1038
1039         return ret;
1040 }
1041
1042 /**
1043  * Writes data to the object referenced by handle.
1044  *
1045  * On error, the contents of the buffer that were to be modified are undefined.
1046  */
1047 int
1048 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1049                       struct drm_file *file)
1050 {
1051         struct drm_i915_private *dev_priv = dev->dev_private;
1052         struct drm_i915_gem_pwrite *args = data;
1053         struct drm_i915_gem_object *obj;
1054         int ret;
1055
1056         if (args->size == 0)
1057                 return 0;
1058
1059         if (!access_ok(VERIFY_READ,
1060                        to_user_ptr(args->data_ptr),
1061                        args->size))
1062                 return -EFAULT;
1063
1064         if (likely(!i915.prefault_disable)) {
1065                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1066                                                    args->size);
1067                 if (ret)
1068                         return -EFAULT;
1069         }
1070
1071         intel_runtime_pm_get(dev_priv);
1072
1073         ret = i915_mutex_lock_interruptible(dev);
1074         if (ret)
1075                 goto put_rpm;
1076
1077         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1078         if (&obj->base == NULL) {
1079                 ret = -ENOENT;
1080                 goto unlock;
1081         }
1082
1083         /* Bounds check destination. */
1084         if (args->offset > obj->base.size ||
1085             args->size > obj->base.size - args->offset) {
1086                 ret = -EINVAL;
1087                 goto out;
1088         }
1089
1090         /* prime objects have no backing filp to GEM pread/pwrite
1091          * pages from.
1092          */
1093         if (!obj->base.filp) {
1094                 ret = -EINVAL;
1095                 goto out;
1096         }
1097
1098         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1099
1100         ret = -EFAULT;
1101         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1102          * it would end up going through the fenced access, and we'll get
1103          * different detiling behavior between reading and writing.
1104          * pread/pwrite currently are reading and writing from the CPU
1105          * perspective, requiring manual detiling by the client.
1106          */
1107         if (obj->tiling_mode == I915_TILING_NONE &&
1108             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1109             cpu_write_needs_clflush(obj)) {
1110                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1111                 /* Note that the gtt paths might fail with non-page-backed user
1112                  * pointers (e.g. gtt mappings when moving data between
1113                  * textures). Fallback to the shmem path in that case. */
1114         }
1115
1116         if (ret == -EFAULT || ret == -ENOSPC) {
1117                 if (obj->phys_handle)
1118                         ret = i915_gem_phys_pwrite(obj, args, file);
1119                 else
1120                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1121         }
1122
1123 out:
1124         drm_gem_object_unreference(&obj->base);
1125 unlock:
1126         mutex_unlock(&dev->struct_mutex);
1127 put_rpm:
1128         intel_runtime_pm_put(dev_priv);
1129
1130         return ret;
1131 }
1132
1133 int
1134 i915_gem_check_wedge(struct i915_gpu_error *error,
1135                      bool interruptible)
1136 {
1137         if (i915_reset_in_progress(error)) {
1138                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1139                  * -EIO unconditionally for these. */
1140                 if (!interruptible)
1141                         return -EIO;
1142
1143                 /* Recovery complete, but the reset failed ... */
1144                 if (i915_terminally_wedged(error))
1145                         return -EIO;
1146
1147                 /*
1148                  * Check if GPU Reset is in progress - we need intel_ring_begin
1149                  * to work properly to reinit the hw state while the gpu is
1150                  * still marked as reset-in-progress. Handle this with a flag.
1151                  */
1152                 if (!error->reload_in_reset)
1153                         return -EAGAIN;
1154         }
1155
1156         return 0;
1157 }
1158
1159 /*
1160  * Compare seqno against outstanding lazy request. Emit a request if they are
1161  * equal.
1162  */
1163 int
1164 i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno)
1165 {
1166         int ret;
1167
1168         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
1169
1170         ret = 0;
1171         if (seqno == ring->outstanding_lazy_seqno)
1172                 ret = i915_add_request(ring, NULL);
1173
1174         return ret;
1175 }
1176
1177 static void fake_irq(unsigned long data)
1178 {
1179         wake_up_process((struct task_struct *)data);
1180 }
1181
1182 static bool missed_irq(struct drm_i915_private *dev_priv,
1183                        struct intel_engine_cs *ring)
1184 {
1185         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1186 }
1187
1188 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
1189 {
1190         if (file_priv == NULL)
1191                 return true;
1192
1193         return !atomic_xchg(&file_priv->rps_wait_boost, true);
1194 }
1195
1196 /**
1197  * __i915_wait_seqno - wait until execution of seqno has finished
1198  * @ring: the ring expected to report seqno
1199  * @seqno: duh!
1200  * @reset_counter: reset sequence associated with the given seqno
1201  * @interruptible: do an interruptible wait (normally yes)
1202  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1203  *
1204  * Note: It is of utmost importance that the passed in seqno and reset_counter
1205  * values have been read by the caller in an smp safe manner. Where read-side
1206  * locks are involved, it is sufficient to read the reset_counter before
1207  * unlocking the lock that protects the seqno. For lockless tricks, the
1208  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1209  * inserted.
1210  *
1211  * Returns 0 if the seqno was found within the alloted time. Else returns the
1212  * errno with remaining time filled in timeout argument.
1213  */
1214 int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno,
1215                         unsigned reset_counter,
1216                         bool interruptible,
1217                         s64 *timeout,
1218                         struct drm_i915_file_private *file_priv)
1219 {
1220         struct drm_device *dev = ring->dev;
1221         struct drm_i915_private *dev_priv = dev->dev_private;
1222         const bool irq_test_in_progress =
1223                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1224         DEFINE_WAIT(wait);
1225         unsigned long timeout_expire;
1226         s64 before, now;
1227         int ret;
1228
1229         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1230
1231         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
1232                 return 0;
1233
1234         timeout_expire = timeout ?
1235                 jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
1236
1237         if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
1238                 gen6_rps_boost(dev_priv);
1239                 if (file_priv)
1240                         mod_delayed_work(dev_priv->wq,
1241                                          &file_priv->mm.idle_work,
1242                                          msecs_to_jiffies(100));
1243         }
1244
1245         if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
1246                 return -ENODEV;
1247
1248         /* Record current time in case interrupted by signal, or wedged */
1249         trace_i915_gem_request_wait_begin(ring, seqno);
1250         before = ktime_get_raw_ns();
1251         for (;;) {
1252                 struct timer_list timer;
1253
1254                 prepare_to_wait(&ring->irq_queue, &wait,
1255                                 interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1256
1257                 /* We need to check whether any gpu reset happened in between
1258                  * the caller grabbing the seqno and now ... */
1259                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1260                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1261                          * is truely gone. */
1262                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1263                         if (ret == 0)
1264                                 ret = -EAGAIN;
1265                         break;
1266                 }
1267
1268                 if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
1269                         ret = 0;
1270                         break;
1271                 }
1272
1273                 if (interruptible && signal_pending(current)) {
1274                         ret = -ERESTARTSYS;
1275                         break;
1276                 }
1277
1278                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1279                         ret = -ETIME;
1280                         break;
1281                 }
1282
1283                 timer.function = NULL;
1284                 if (timeout || missed_irq(dev_priv, ring)) {
1285                         unsigned long expire;
1286
1287                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1288                         expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1289                         mod_timer(&timer, expire);
1290                 }
1291
1292                 io_schedule();
1293
1294                 if (timer.function) {
1295                         del_singleshot_timer_sync(&timer);
1296                         destroy_timer_on_stack(&timer);
1297                 }
1298         }
1299         now = ktime_get_raw_ns();
1300         trace_i915_gem_request_wait_end(ring, seqno);
1301
1302         if (!irq_test_in_progress)
1303                 ring->irq_put(ring);
1304
1305         finish_wait(&ring->irq_queue, &wait);
1306
1307         if (timeout) {
1308                 s64 tres = *timeout - (now - before);
1309
1310                 *timeout = tres < 0 ? 0 : tres;
1311
1312                 /*
1313                  * Apparently ktime isn't accurate enough and occasionally has a
1314                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1315                  * things up to make the test happy. We allow up to 1 jiffy.
1316                  *
1317                  * This is a regrssion from the timespec->ktime conversion.
1318                  */
1319                 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1320                         *timeout = 0;
1321         }
1322
1323         return ret;
1324 }
1325
1326 /**
1327  * Waits for a sequence number to be signaled, and cleans up the
1328  * request and object lists appropriately for that event.
1329  */
1330 int
1331 i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
1332 {
1333         struct drm_device *dev = ring->dev;
1334         struct drm_i915_private *dev_priv = dev->dev_private;
1335         bool interruptible = dev_priv->mm.interruptible;
1336         unsigned reset_counter;
1337         int ret;
1338
1339         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1340         BUG_ON(seqno == 0);
1341
1342         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1343         if (ret)
1344                 return ret;
1345
1346         ret = i915_gem_check_olr(ring, seqno);
1347         if (ret)
1348                 return ret;
1349
1350         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1351         return __i915_wait_seqno(ring, seqno, reset_counter, interruptible,
1352                                  NULL, NULL);
1353 }
1354
1355 static int
1356 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
1357 {
1358         if (!obj->active)
1359                 return 0;
1360
1361         /* Manually manage the write flush as we may have not yet
1362          * retired the buffer.
1363          *
1364          * Note that the last_write_seqno is always the earlier of
1365          * the two (read/write) seqno, so if we haved successfully waited,
1366          * we know we have passed the last write.
1367          */
1368         obj->last_write_seqno = 0;
1369
1370         return 0;
1371 }
1372
1373 /**
1374  * Ensures that all rendering to the object has completed and the object is
1375  * safe to unbind from the GTT or access from the CPU.
1376  */
1377 static __must_check int
1378 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1379                                bool readonly)
1380 {
1381         struct intel_engine_cs *ring = obj->ring;
1382         u32 seqno;
1383         int ret;
1384
1385         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1386         if (seqno == 0)
1387                 return 0;
1388
1389         ret = i915_wait_seqno(ring, seqno);
1390         if (ret)
1391                 return ret;
1392
1393         return i915_gem_object_wait_rendering__tail(obj);
1394 }
1395
1396 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1397  * as the object state may change during this call.
1398  */
1399 static __must_check int
1400 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1401                                             struct drm_i915_file_private *file_priv,
1402                                             bool readonly)
1403 {
1404         struct drm_device *dev = obj->base.dev;
1405         struct drm_i915_private *dev_priv = dev->dev_private;
1406         struct intel_engine_cs *ring = obj->ring;
1407         unsigned reset_counter;
1408         u32 seqno;
1409         int ret;
1410
1411         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1412         BUG_ON(!dev_priv->mm.interruptible);
1413
1414         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1415         if (seqno == 0)
1416                 return 0;
1417
1418         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1419         if (ret)
1420                 return ret;
1421
1422         ret = i915_gem_check_olr(ring, seqno);
1423         if (ret)
1424                 return ret;
1425
1426         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1427         mutex_unlock(&dev->struct_mutex);
1428         ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL,
1429                                 file_priv);
1430         mutex_lock(&dev->struct_mutex);
1431         if (ret)
1432                 return ret;
1433
1434         return i915_gem_object_wait_rendering__tail(obj);
1435 }
1436
1437 /**
1438  * Called when user space prepares to use an object with the CPU, either
1439  * through the mmap ioctl's mapping or a GTT mapping.
1440  */
1441 int
1442 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1443                           struct drm_file *file)
1444 {
1445         struct drm_i915_gem_set_domain *args = data;
1446         struct drm_i915_gem_object *obj;
1447         uint32_t read_domains = args->read_domains;
1448         uint32_t write_domain = args->write_domain;
1449         int ret;
1450
1451         /* Only handle setting domains to types used by the CPU. */
1452         if (write_domain & I915_GEM_GPU_DOMAINS)
1453                 return -EINVAL;
1454
1455         if (read_domains & I915_GEM_GPU_DOMAINS)
1456                 return -EINVAL;
1457
1458         /* Having something in the write domain implies it's in the read
1459          * domain, and only that read domain.  Enforce that in the request.
1460          */
1461         if (write_domain != 0 && read_domains != write_domain)
1462                 return -EINVAL;
1463
1464         ret = i915_mutex_lock_interruptible(dev);
1465         if (ret)
1466                 return ret;
1467
1468         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1469         if (&obj->base == NULL) {
1470                 ret = -ENOENT;
1471                 goto unlock;
1472         }
1473
1474         /* Try to flush the object off the GPU without holding the lock.
1475          * We will repeat the flush holding the lock in the normal manner
1476          * to catch cases where we are gazumped.
1477          */
1478         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1479                                                           file->driver_priv,
1480                                                           !write_domain);
1481         if (ret)
1482                 goto unref;
1483
1484         if (read_domains & I915_GEM_DOMAIN_GTT) {
1485                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1486
1487                 /* Silently promote "you're not bound, there was nothing to do"
1488                  * to success, since the client was just asking us to
1489                  * make sure everything was done.
1490                  */
1491                 if (ret == -EINVAL)
1492                         ret = 0;
1493         } else {
1494                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1495         }
1496
1497 unref:
1498         drm_gem_object_unreference(&obj->base);
1499 unlock:
1500         mutex_unlock(&dev->struct_mutex);
1501         return ret;
1502 }
1503
1504 /**
1505  * Called when user space has done writes to this buffer
1506  */
1507 int
1508 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1509                          struct drm_file *file)
1510 {
1511         struct drm_i915_gem_sw_finish *args = data;
1512         struct drm_i915_gem_object *obj;
1513         int ret = 0;
1514
1515         ret = i915_mutex_lock_interruptible(dev);
1516         if (ret)
1517                 return ret;
1518
1519         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1520         if (&obj->base == NULL) {
1521                 ret = -ENOENT;
1522                 goto unlock;
1523         }
1524
1525         /* Pinned buffers may be scanout, so flush the cache */
1526         if (obj->pin_display)
1527                 i915_gem_object_flush_cpu_write_domain(obj, true);
1528
1529         drm_gem_object_unreference(&obj->base);
1530 unlock:
1531         mutex_unlock(&dev->struct_mutex);
1532         return ret;
1533 }
1534
1535 /**
1536  * Maps the contents of an object, returning the address it is mapped
1537  * into.
1538  *
1539  * While the mapping holds a reference on the contents of the object, it doesn't
1540  * imply a ref on the object itself.
1541  *
1542  * IMPORTANT:
1543  *
1544  * DRM driver writers who look a this function as an example for how to do GEM
1545  * mmap support, please don't implement mmap support like here. The modern way
1546  * to implement DRM mmap support is with an mmap offset ioctl (like
1547  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1548  * That way debug tooling like valgrind will understand what's going on, hiding
1549  * the mmap call in a driver private ioctl will break that. The i915 driver only
1550  * does cpu mmaps this way because we didn't know better.
1551  */
1552 int
1553 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1554                     struct drm_file *file)
1555 {
1556         struct drm_i915_gem_mmap *args = data;
1557         struct drm_gem_object *obj;
1558         unsigned long addr;
1559
1560         obj = drm_gem_object_lookup(dev, file, args->handle);
1561         if (obj == NULL)
1562                 return -ENOENT;
1563
1564         /* prime objects have no backing filp to GEM mmap
1565          * pages from.
1566          */
1567         if (!obj->filp) {
1568                 drm_gem_object_unreference_unlocked(obj);
1569                 return -EINVAL;
1570         }
1571
1572         addr = vm_mmap(obj->filp, 0, args->size,
1573                        PROT_READ | PROT_WRITE, MAP_SHARED,
1574                        args->offset);
1575         drm_gem_object_unreference_unlocked(obj);
1576         if (IS_ERR((void *)addr))
1577                 return addr;
1578
1579         args->addr_ptr = (uint64_t) addr;
1580
1581         return 0;
1582 }
1583
1584 /**
1585  * i915_gem_fault - fault a page into the GTT
1586  * vma: VMA in question
1587  * vmf: fault info
1588  *
1589  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1590  * from userspace.  The fault handler takes care of binding the object to
1591  * the GTT (if needed), allocating and programming a fence register (again,
1592  * only if needed based on whether the old reg is still valid or the object
1593  * is tiled) and inserting a new PTE into the faulting process.
1594  *
1595  * Note that the faulting process may involve evicting existing objects
1596  * from the GTT and/or fence registers to make room.  So performance may
1597  * suffer if the GTT working set is large or there are few fence registers
1598  * left.
1599  */
1600 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1601 {
1602         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1603         struct drm_device *dev = obj->base.dev;
1604         struct drm_i915_private *dev_priv = dev->dev_private;
1605         pgoff_t page_offset;
1606         unsigned long pfn;
1607         int ret = 0;
1608         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1609
1610         intel_runtime_pm_get(dev_priv);
1611
1612         /* We don't use vmf->pgoff since that has the fake offset */
1613         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1614                 PAGE_SHIFT;
1615
1616         ret = i915_mutex_lock_interruptible(dev);
1617         if (ret)
1618                 goto out;
1619
1620         trace_i915_gem_object_fault(obj, page_offset, true, write);
1621
1622         /* Try to flush the object off the GPU first without holding the lock.
1623          * Upon reacquiring the lock, we will perform our sanity checks and then
1624          * repeat the flush holding the lock in the normal manner to catch cases
1625          * where we are gazumped.
1626          */
1627         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1628         if (ret)
1629                 goto unlock;
1630
1631         /* Access to snoopable pages through the GTT is incoherent. */
1632         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1633                 ret = -EFAULT;
1634                 goto unlock;
1635         }
1636
1637         /* Now bind it into the GTT if needed */
1638         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
1639         if (ret)
1640                 goto unlock;
1641
1642         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1643         if (ret)
1644                 goto unpin;
1645
1646         ret = i915_gem_object_get_fence(obj);
1647         if (ret)
1648                 goto unpin;
1649
1650         /* Finally, remap it using the new GTT offset */
1651         pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1652         pfn >>= PAGE_SHIFT;
1653
1654         if (!obj->fault_mappable) {
1655                 unsigned long size = min_t(unsigned long,
1656                                            vma->vm_end - vma->vm_start,
1657                                            obj->base.size);
1658                 int i;
1659
1660                 for (i = 0; i < size >> PAGE_SHIFT; i++) {
1661                         ret = vm_insert_pfn(vma,
1662                                             (unsigned long)vma->vm_start + i * PAGE_SIZE,
1663                                             pfn + i);
1664                         if (ret)
1665                                 break;
1666                 }
1667
1668                 obj->fault_mappable = true;
1669         } else
1670                 ret = vm_insert_pfn(vma,
1671                                     (unsigned long)vmf->virtual_address,
1672                                     pfn + page_offset);
1673 unpin:
1674         i915_gem_object_ggtt_unpin(obj);
1675 unlock:
1676         mutex_unlock(&dev->struct_mutex);
1677 out:
1678         switch (ret) {
1679         case -EIO:
1680                 /*
1681                  * We eat errors when the gpu is terminally wedged to avoid
1682                  * userspace unduly crashing (gl has no provisions for mmaps to
1683                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1684                  * and so needs to be reported.
1685                  */
1686                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1687                         ret = VM_FAULT_SIGBUS;
1688                         break;
1689                 }
1690         case -EAGAIN:
1691                 /*
1692                  * EAGAIN means the gpu is hung and we'll wait for the error
1693                  * handler to reset everything when re-faulting in
1694                  * i915_mutex_lock_interruptible.
1695                  */
1696         case 0:
1697         case -ERESTARTSYS:
1698         case -EINTR:
1699         case -EBUSY:
1700                 /*
1701                  * EBUSY is ok: this just means that another thread
1702                  * already did the job.
1703                  */
1704                 ret = VM_FAULT_NOPAGE;
1705                 break;
1706         case -ENOMEM:
1707                 ret = VM_FAULT_OOM;
1708                 break;
1709         case -ENOSPC:
1710         case -EFAULT:
1711                 ret = VM_FAULT_SIGBUS;
1712                 break;
1713         default:
1714                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1715                 ret = VM_FAULT_SIGBUS;
1716                 break;
1717         }
1718
1719         intel_runtime_pm_put(dev_priv);
1720         return ret;
1721 }
1722
1723 /**
1724  * i915_gem_release_mmap - remove physical page mappings
1725  * @obj: obj in question
1726  *
1727  * Preserve the reservation of the mmapping with the DRM core code, but
1728  * relinquish ownership of the pages back to the system.
1729  *
1730  * It is vital that we remove the page mapping if we have mapped a tiled
1731  * object through the GTT and then lose the fence register due to
1732  * resource pressure. Similarly if the object has been moved out of the
1733  * aperture, than pages mapped into userspace must be revoked. Removing the
1734  * mapping will then trigger a page fault on the next user access, allowing
1735  * fixup by i915_gem_fault().
1736  */
1737 void
1738 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1739 {
1740         if (!obj->fault_mappable)
1741                 return;
1742
1743         drm_vma_node_unmap(&obj->base.vma_node,
1744                            obj->base.dev->anon_inode->i_mapping);
1745         obj->fault_mappable = false;
1746 }
1747
1748 void
1749 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1750 {
1751         struct drm_i915_gem_object *obj;
1752
1753         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1754                 i915_gem_release_mmap(obj);
1755 }
1756
1757 uint32_t
1758 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1759 {
1760         uint32_t gtt_size;
1761
1762         if (INTEL_INFO(dev)->gen >= 4 ||
1763             tiling_mode == I915_TILING_NONE)
1764                 return size;
1765
1766         /* Previous chips need a power-of-two fence region when tiling */
1767         if (INTEL_INFO(dev)->gen == 3)
1768                 gtt_size = 1024*1024;
1769         else
1770                 gtt_size = 512*1024;
1771
1772         while (gtt_size < size)
1773                 gtt_size <<= 1;
1774
1775         return gtt_size;
1776 }
1777
1778 /**
1779  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1780  * @obj: object to check
1781  *
1782  * Return the required GTT alignment for an object, taking into account
1783  * potential fence register mapping.
1784  */
1785 uint32_t
1786 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1787                            int tiling_mode, bool fenced)
1788 {
1789         /*
1790          * Minimum alignment is 4k (GTT page size), but might be greater
1791          * if a fence register is needed for the object.
1792          */
1793         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1794             tiling_mode == I915_TILING_NONE)
1795                 return 4096;
1796
1797         /*
1798          * Previous chips need to be aligned to the size of the smallest
1799          * fence register that can contain the object.
1800          */
1801         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1802 }
1803
1804 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1805 {
1806         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1807         int ret;
1808
1809         if (drm_vma_node_has_offset(&obj->base.vma_node))
1810                 return 0;
1811
1812         dev_priv->mm.shrinker_no_lock_stealing = true;
1813
1814         ret = drm_gem_create_mmap_offset(&obj->base);
1815         if (ret != -ENOSPC)
1816                 goto out;
1817
1818         /* Badly fragmented mmap space? The only way we can recover
1819          * space is by destroying unwanted objects. We can't randomly release
1820          * mmap_offsets as userspace expects them to be persistent for the
1821          * lifetime of the objects. The closest we can is to release the
1822          * offsets on purgeable objects by truncating it and marking it purged,
1823          * which prevents userspace from ever using that object again.
1824          */
1825         i915_gem_shrink(dev_priv,
1826                         obj->base.size >> PAGE_SHIFT,
1827                         I915_SHRINK_BOUND |
1828                         I915_SHRINK_UNBOUND |
1829                         I915_SHRINK_PURGEABLE);
1830         ret = drm_gem_create_mmap_offset(&obj->base);
1831         if (ret != -ENOSPC)
1832                 goto out;
1833
1834         i915_gem_shrink_all(dev_priv);
1835         ret = drm_gem_create_mmap_offset(&obj->base);
1836 out:
1837         dev_priv->mm.shrinker_no_lock_stealing = false;
1838
1839         return ret;
1840 }
1841
1842 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1843 {
1844         drm_gem_free_mmap_offset(&obj->base);
1845 }
1846
1847 int
1848 i915_gem_mmap_gtt(struct drm_file *file,
1849                   struct drm_device *dev,
1850                   uint32_t handle,
1851                   uint64_t *offset)
1852 {
1853         struct drm_i915_private *dev_priv = dev->dev_private;
1854         struct drm_i915_gem_object *obj;
1855         int ret;
1856
1857         ret = i915_mutex_lock_interruptible(dev);
1858         if (ret)
1859                 return ret;
1860
1861         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1862         if (&obj->base == NULL) {
1863                 ret = -ENOENT;
1864                 goto unlock;
1865         }
1866
1867         if (obj->base.size > dev_priv->gtt.mappable_end) {
1868                 ret = -E2BIG;
1869                 goto out;
1870         }
1871
1872         if (obj->madv != I915_MADV_WILLNEED) {
1873                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1874                 ret = -EFAULT;
1875                 goto out;
1876         }
1877
1878         ret = i915_gem_object_create_mmap_offset(obj);
1879         if (ret)
1880                 goto out;
1881
1882         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1883
1884 out:
1885         drm_gem_object_unreference(&obj->base);
1886 unlock:
1887         mutex_unlock(&dev->struct_mutex);
1888         return ret;
1889 }
1890
1891 /**
1892  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1893  * @dev: DRM device
1894  * @data: GTT mapping ioctl data
1895  * @file: GEM object info
1896  *
1897  * Simply returns the fake offset to userspace so it can mmap it.
1898  * The mmap call will end up in drm_gem_mmap(), which will set things
1899  * up so we can get faults in the handler above.
1900  *
1901  * The fault handler will take care of binding the object into the GTT
1902  * (since it may have been evicted to make room for something), allocating
1903  * a fence register, and mapping the appropriate aperture address into
1904  * userspace.
1905  */
1906 int
1907 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1908                         struct drm_file *file)
1909 {
1910         struct drm_i915_gem_mmap_gtt *args = data;
1911
1912         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1913 }
1914
1915 static inline int
1916 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1917 {
1918         return obj->madv == I915_MADV_DONTNEED;
1919 }
1920
1921 /* Immediately discard the backing storage */
1922 static void
1923 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1924 {
1925         i915_gem_object_free_mmap_offset(obj);
1926
1927         if (obj->base.filp == NULL)
1928                 return;
1929
1930         /* Our goal here is to return as much of the memory as
1931          * is possible back to the system as we are called from OOM.
1932          * To do this we must instruct the shmfs to drop all of its
1933          * backing pages, *now*.
1934          */
1935         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
1936         obj->madv = __I915_MADV_PURGED;
1937 }
1938
1939 /* Try to discard unwanted pages */
1940 static void
1941 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
1942 {
1943         struct address_space *mapping;
1944
1945         switch (obj->madv) {
1946         case I915_MADV_DONTNEED:
1947                 i915_gem_object_truncate(obj);
1948         case __I915_MADV_PURGED:
1949                 return;
1950         }
1951
1952         if (obj->base.filp == NULL)
1953                 return;
1954
1955         mapping = file_inode(obj->base.filp)->i_mapping,
1956         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
1957 }
1958
1959 static void
1960 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1961 {
1962         struct sg_page_iter sg_iter;
1963         int ret;
1964
1965         BUG_ON(obj->madv == __I915_MADV_PURGED);
1966
1967         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1968         if (ret) {
1969                 /* In the event of a disaster, abandon all caches and
1970                  * hope for the best.
1971                  */
1972                 WARN_ON(ret != -EIO);
1973                 i915_gem_clflush_object(obj, true);
1974                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1975         }
1976
1977         if (i915_gem_object_needs_bit17_swizzle(obj))
1978                 i915_gem_object_save_bit_17_swizzle(obj);
1979
1980         if (obj->madv == I915_MADV_DONTNEED)
1981                 obj->dirty = 0;
1982
1983         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1984                 struct page *page = sg_page_iter_page(&sg_iter);
1985
1986                 if (obj->dirty)
1987                         set_page_dirty(page);
1988
1989                 if (obj->madv == I915_MADV_WILLNEED)
1990                         mark_page_accessed(page);
1991
1992                 page_cache_release(page);
1993         }
1994         obj->dirty = 0;
1995
1996         sg_free_table(obj->pages);
1997         kfree(obj->pages);
1998 }
1999
2000 int
2001 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2002 {
2003         const struct drm_i915_gem_object_ops *ops = obj->ops;
2004
2005         if (obj->pages == NULL)
2006                 return 0;
2007
2008         if (obj->pages_pin_count)
2009                 return -EBUSY;
2010
2011         BUG_ON(i915_gem_obj_bound_any(obj));
2012
2013         /* ->put_pages might need to allocate memory for the bit17 swizzle
2014          * array, hence protect them from being reaped by removing them from gtt
2015          * lists early. */
2016         list_del(&obj->global_list);
2017
2018         ops->put_pages(obj);
2019         obj->pages = NULL;
2020
2021         i915_gem_object_invalidate(obj);
2022
2023         return 0;
2024 }
2025
2026 unsigned long
2027 i915_gem_shrink(struct drm_i915_private *dev_priv,
2028                 long target, unsigned flags)
2029 {
2030         const struct {
2031                 struct list_head *list;
2032                 unsigned int bit;
2033         } phases[] = {
2034                 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
2035                 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
2036                 { NULL, 0 },
2037         }, *phase;
2038         unsigned long count = 0;
2039
2040         /*
2041          * As we may completely rewrite the (un)bound list whilst unbinding
2042          * (due to retiring requests) we have to strictly process only
2043          * one element of the list at the time, and recheck the list
2044          * on every iteration.
2045          *
2046          * In particular, we must hold a reference whilst removing the
2047          * object as we may end up waiting for and/or retiring the objects.
2048          * This might release the final reference (held by the active list)
2049          * and result in the object being freed from under us. This is
2050          * similar to the precautions the eviction code must take whilst
2051          * removing objects.
2052          *
2053          * Also note that although these lists do not hold a reference to
2054          * the object we can safely grab one here: The final object
2055          * unreferencing and the bound_list are both protected by the
2056          * dev->struct_mutex and so we won't ever be able to observe an
2057          * object on the bound_list with a reference count equals 0.
2058          */
2059         for (phase = phases; phase->list; phase++) {
2060                 struct list_head still_in_list;
2061
2062                 if ((flags & phase->bit) == 0)
2063                         continue;
2064
2065                 INIT_LIST_HEAD(&still_in_list);
2066                 while (count < target && !list_empty(phase->list)) {
2067                         struct drm_i915_gem_object *obj;
2068                         struct i915_vma *vma, *v;
2069
2070                         obj = list_first_entry(phase->list,
2071                                                typeof(*obj), global_list);
2072                         list_move_tail(&obj->global_list, &still_in_list);
2073
2074                         if (flags & I915_SHRINK_PURGEABLE &&
2075                             !i915_gem_object_is_purgeable(obj))
2076                                 continue;
2077
2078                         drm_gem_object_reference(&obj->base);
2079
2080                         /* For the unbound phase, this should be a no-op! */
2081                         list_for_each_entry_safe(vma, v,
2082                                                  &obj->vma_list, vma_link)
2083                                 if (i915_vma_unbind(vma))
2084                                         break;
2085
2086                         if (i915_gem_object_put_pages(obj) == 0)
2087                                 count += obj->base.size >> PAGE_SHIFT;
2088
2089                         drm_gem_object_unreference(&obj->base);
2090                 }
2091                 list_splice(&still_in_list, phase->list);
2092         }
2093
2094         return count;
2095 }
2096
2097 static unsigned long
2098 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
2099 {
2100         i915_gem_evict_everything(dev_priv->dev);
2101         return i915_gem_shrink(dev_priv, LONG_MAX,
2102                                I915_SHRINK_BOUND | I915_SHRINK_UNBOUND);
2103 }
2104
2105 static int
2106 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2107 {
2108         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2109         int page_count, i;
2110         struct address_space *mapping;
2111         struct sg_table *st;
2112         struct scatterlist *sg;
2113         struct sg_page_iter sg_iter;
2114         struct page *page;
2115         unsigned long last_pfn = 0;     /* suppress gcc warning */
2116         gfp_t gfp;
2117
2118         /* Assert that the object is not currently in any GPU domain. As it
2119          * wasn't in the GTT, there shouldn't be any way it could have been in
2120          * a GPU cache
2121          */
2122         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2123         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2124
2125         st = kmalloc(sizeof(*st), GFP_KERNEL);
2126         if (st == NULL)
2127                 return -ENOMEM;
2128
2129         page_count = obj->base.size / PAGE_SIZE;
2130         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2131                 kfree(st);
2132                 return -ENOMEM;
2133         }
2134
2135         /* Get the list of pages out of our struct file.  They'll be pinned
2136          * at this point until we release them.
2137          *
2138          * Fail silently without starting the shrinker
2139          */
2140         mapping = file_inode(obj->base.filp)->i_mapping;
2141         gfp = mapping_gfp_mask(mapping);
2142         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
2143         gfp &= ~(__GFP_IO | __GFP_WAIT);
2144         sg = st->sgl;
2145         st->nents = 0;
2146         for (i = 0; i < page_count; i++) {
2147                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2148                 if (IS_ERR(page)) {
2149                         i915_gem_shrink(dev_priv,
2150                                         page_count,
2151                                         I915_SHRINK_BOUND |
2152                                         I915_SHRINK_UNBOUND |
2153                                         I915_SHRINK_PURGEABLE);
2154                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2155                 }
2156                 if (IS_ERR(page)) {
2157                         /* We've tried hard to allocate the memory by reaping
2158                          * our own buffer, now let the real VM do its job and
2159                          * go down in flames if truly OOM.
2160                          */
2161                         i915_gem_shrink_all(dev_priv);
2162                         page = shmem_read_mapping_page(mapping, i);
2163                         if (IS_ERR(page))
2164                                 goto err_pages;
2165                 }
2166 #ifdef CONFIG_SWIOTLB
2167                 if (swiotlb_nr_tbl()) {
2168                         st->nents++;
2169                         sg_set_page(sg, page, PAGE_SIZE, 0);
2170                         sg = sg_next(sg);
2171                         continue;
2172                 }
2173 #endif
2174                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2175                         if (i)
2176                                 sg = sg_next(sg);
2177                         st->nents++;
2178                         sg_set_page(sg, page, PAGE_SIZE, 0);
2179                 } else {
2180                         sg->length += PAGE_SIZE;
2181                 }
2182                 last_pfn = page_to_pfn(page);
2183
2184                 /* Check that the i965g/gm workaround works. */
2185                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2186         }
2187 #ifdef CONFIG_SWIOTLB
2188         if (!swiotlb_nr_tbl())
2189 #endif
2190                 sg_mark_end(sg);
2191         obj->pages = st;
2192
2193         if (i915_gem_object_needs_bit17_swizzle(obj))
2194                 i915_gem_object_do_bit_17_swizzle(obj);
2195
2196         if (obj->tiling_mode != I915_TILING_NONE &&
2197             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2198                 i915_gem_object_pin_pages(obj);
2199
2200         return 0;
2201
2202 err_pages:
2203         sg_mark_end(sg);
2204         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2205                 page_cache_release(sg_page_iter_page(&sg_iter));
2206         sg_free_table(st);
2207         kfree(st);
2208
2209         /* shmemfs first checks if there is enough memory to allocate the page
2210          * and reports ENOSPC should there be insufficient, along with the usual
2211          * ENOMEM for a genuine allocation failure.
2212          *
2213          * We use ENOSPC in our driver to mean that we have run out of aperture
2214          * space and so want to translate the error from shmemfs back to our
2215          * usual understanding of ENOMEM.
2216          */
2217         if (PTR_ERR(page) == -ENOSPC)
2218                 return -ENOMEM;
2219         else
2220                 return PTR_ERR(page);
2221 }
2222
2223 /* Ensure that the associated pages are gathered from the backing storage
2224  * and pinned into our object. i915_gem_object_get_pages() may be called
2225  * multiple times before they are released by a single call to
2226  * i915_gem_object_put_pages() - once the pages are no longer referenced
2227  * either as a result of memory pressure (reaping pages under the shrinker)
2228  * or as the object is itself released.
2229  */
2230 int
2231 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2232 {
2233         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2234         const struct drm_i915_gem_object_ops *ops = obj->ops;
2235         int ret;
2236
2237         if (obj->pages)
2238                 return 0;
2239
2240         if (obj->madv != I915_MADV_WILLNEED) {
2241                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2242                 return -EFAULT;
2243         }
2244
2245         BUG_ON(obj->pages_pin_count);
2246
2247         ret = ops->get_pages(obj);
2248         if (ret)
2249                 return ret;
2250
2251         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2252         return 0;
2253 }
2254
2255 static void
2256 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2257                                struct intel_engine_cs *ring)
2258 {
2259         u32 seqno = intel_ring_get_seqno(ring);
2260
2261         BUG_ON(ring == NULL);
2262         if (obj->ring != ring && obj->last_write_seqno) {
2263                 /* Keep the seqno relative to the current ring */
2264                 obj->last_write_seqno = seqno;
2265         }
2266         obj->ring = ring;
2267
2268         /* Add a reference if we're newly entering the active list. */
2269         if (!obj->active) {
2270                 drm_gem_object_reference(&obj->base);
2271                 obj->active = 1;
2272         }
2273
2274         list_move_tail(&obj->ring_list, &ring->active_list);
2275
2276         obj->last_read_seqno = seqno;
2277 }
2278
2279 void i915_vma_move_to_active(struct i915_vma *vma,
2280                              struct intel_engine_cs *ring)
2281 {
2282         list_move_tail(&vma->mm_list, &vma->vm->active_list);
2283         return i915_gem_object_move_to_active(vma->obj, ring);
2284 }
2285
2286 static void
2287 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2288 {
2289         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2290         struct i915_address_space *vm;
2291         struct i915_vma *vma;
2292
2293         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2294         BUG_ON(!obj->active);
2295
2296         list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
2297                 vma = i915_gem_obj_to_vma(obj, vm);
2298                 if (vma && !list_empty(&vma->mm_list))
2299                         list_move_tail(&vma->mm_list, &vm->inactive_list);
2300         }
2301
2302         intel_fb_obj_flush(obj, true);
2303
2304         list_del_init(&obj->ring_list);
2305         obj->ring = NULL;
2306
2307         obj->last_read_seqno = 0;
2308         obj->last_write_seqno = 0;
2309         obj->base.write_domain = 0;
2310
2311         obj->last_fenced_seqno = 0;
2312
2313         obj->active = 0;
2314         drm_gem_object_unreference(&obj->base);
2315
2316         WARN_ON(i915_verify_lists(dev));
2317 }
2318
2319 static void
2320 i915_gem_object_retire(struct drm_i915_gem_object *obj)
2321 {
2322         struct intel_engine_cs *ring = obj->ring;
2323
2324         if (ring == NULL)
2325                 return;
2326
2327         if (i915_seqno_passed(ring->get_seqno(ring, true),
2328                               obj->last_read_seqno))
2329                 i915_gem_object_move_to_inactive(obj);
2330 }
2331
2332 static int
2333 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2334 {
2335         struct drm_i915_private *dev_priv = dev->dev_private;
2336         struct intel_engine_cs *ring;
2337         int ret, i, j;
2338
2339         /* Carefully retire all requests without writing to the rings */
2340         for_each_ring(ring, dev_priv, i) {
2341                 ret = intel_ring_idle(ring);
2342                 if (ret)
2343                         return ret;
2344         }
2345         i915_gem_retire_requests(dev);
2346
2347         /* Finally reset hw state */
2348         for_each_ring(ring, dev_priv, i) {
2349                 intel_ring_init_seqno(ring, seqno);
2350
2351                 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
2352                         ring->semaphore.sync_seqno[j] = 0;
2353         }
2354
2355         return 0;
2356 }
2357
2358 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2359 {
2360         struct drm_i915_private *dev_priv = dev->dev_private;
2361         int ret;
2362
2363         if (seqno == 0)
2364                 return -EINVAL;
2365
2366         /* HWS page needs to be set less than what we
2367          * will inject to ring
2368          */
2369         ret = i915_gem_init_seqno(dev, seqno - 1);
2370         if (ret)
2371                 return ret;
2372
2373         /* Carefully set the last_seqno value so that wrap
2374          * detection still works
2375          */
2376         dev_priv->next_seqno = seqno;
2377         dev_priv->last_seqno = seqno - 1;
2378         if (dev_priv->last_seqno == 0)
2379                 dev_priv->last_seqno--;
2380
2381         return 0;
2382 }
2383
2384 int
2385 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2386 {
2387         struct drm_i915_private *dev_priv = dev->dev_private;
2388
2389         /* reserve 0 for non-seqno */
2390         if (dev_priv->next_seqno == 0) {
2391                 int ret = i915_gem_init_seqno(dev, 0);
2392                 if (ret)
2393                         return ret;
2394
2395                 dev_priv->next_seqno = 1;
2396         }
2397
2398         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2399         return 0;
2400 }
2401
2402 int __i915_add_request(struct intel_engine_cs *ring,
2403                        struct drm_file *file,
2404                        struct drm_i915_gem_object *obj,
2405                        u32 *out_seqno)
2406 {
2407         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2408         struct drm_i915_gem_request *request;
2409         struct intel_ringbuffer *ringbuf;
2410         u32 request_ring_position, request_start;
2411         int ret;
2412
2413         request = ring->preallocated_lazy_request;
2414         if (WARN_ON(request == NULL))
2415                 return -ENOMEM;
2416
2417         if (i915.enable_execlists) {
2418                 struct intel_context *ctx = request->ctx;
2419                 ringbuf = ctx->engine[ring->id].ringbuf;
2420         } else
2421                 ringbuf = ring->buffer;
2422
2423         request_start = intel_ring_get_tail(ringbuf);
2424         /*
2425          * Emit any outstanding flushes - execbuf can fail to emit the flush
2426          * after having emitted the batchbuffer command. Hence we need to fix
2427          * things up similar to emitting the lazy request. The difference here
2428          * is that the flush _must_ happen before the next request, no matter
2429          * what.
2430          */
2431         if (i915.enable_execlists) {
2432                 ret = logical_ring_flush_all_caches(ringbuf);
2433                 if (ret)
2434                         return ret;
2435         } else {
2436                 ret = intel_ring_flush_all_caches(ring);
2437                 if (ret)
2438                         return ret;
2439         }
2440
2441         /* Record the position of the start of the request so that
2442          * should we detect the updated seqno part-way through the
2443          * GPU processing the request, we never over-estimate the
2444          * position of the head.
2445          */
2446         request_ring_position = intel_ring_get_tail(ringbuf);
2447
2448         if (i915.enable_execlists) {
2449                 ret = ring->emit_request(ringbuf);
2450                 if (ret)
2451                         return ret;
2452         } else {
2453                 ret = ring->add_request(ring);
2454                 if (ret)
2455                         return ret;
2456         }
2457
2458         request->seqno = intel_ring_get_seqno(ring);
2459         request->ring = ring;
2460         request->head = request_start;
2461         request->tail = request_ring_position;
2462
2463         /* Whilst this request exists, batch_obj will be on the
2464          * active_list, and so will hold the active reference. Only when this
2465          * request is retired will the the batch_obj be moved onto the
2466          * inactive_list and lose its active reference. Hence we do not need
2467          * to explicitly hold another reference here.
2468          */
2469         request->batch_obj = obj;
2470
2471         if (!i915.enable_execlists) {
2472                 /* Hold a reference to the current context so that we can inspect
2473                  * it later in case a hangcheck error event fires.
2474                  */
2475                 request->ctx = ring->last_context;
2476                 if (request->ctx)
2477                         i915_gem_context_reference(request->ctx);
2478         }
2479
2480         request->emitted_jiffies = jiffies;
2481         list_add_tail(&request->list, &ring->request_list);
2482         request->file_priv = NULL;
2483
2484         if (file) {
2485                 struct drm_i915_file_private *file_priv = file->driver_priv;
2486
2487                 spin_lock(&file_priv->mm.lock);
2488                 request->file_priv = file_priv;
2489                 list_add_tail(&request->client_list,
2490                               &file_priv->mm.request_list);
2491                 spin_unlock(&file_priv->mm.lock);
2492         }
2493
2494         trace_i915_gem_request_add(ring, request->seqno);
2495         ring->outstanding_lazy_seqno = 0;
2496         ring->preallocated_lazy_request = NULL;
2497
2498         i915_queue_hangcheck(ring->dev);
2499
2500         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
2501         queue_delayed_work(dev_priv->wq,
2502                            &dev_priv->mm.retire_work,
2503                            round_jiffies_up_relative(HZ));
2504         intel_mark_busy(dev_priv->dev);
2505
2506         if (out_seqno)
2507                 *out_seqno = request->seqno;
2508         return 0;
2509 }
2510
2511 static inline void
2512 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2513 {
2514         struct drm_i915_file_private *file_priv = request->file_priv;
2515
2516         if (!file_priv)
2517                 return;
2518
2519         spin_lock(&file_priv->mm.lock);
2520         list_del(&request->client_list);
2521         request->file_priv = NULL;
2522         spin_unlock(&file_priv->mm.lock);
2523 }
2524
2525 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2526                                    const struct intel_context *ctx)
2527 {
2528         unsigned long elapsed;
2529
2530         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2531
2532         if (ctx->hang_stats.banned)
2533                 return true;
2534
2535         if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
2536                 if (!i915_gem_context_is_default(ctx)) {
2537                         DRM_DEBUG("context hanging too fast, banning!\n");
2538                         return true;
2539                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2540                         if (i915_stop_ring_allow_warn(dev_priv))
2541                                 DRM_ERROR("gpu hanging too fast, banning!\n");
2542                         return true;
2543                 }
2544         }
2545
2546         return false;
2547 }
2548
2549 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2550                                   struct intel_context *ctx,
2551                                   const bool guilty)
2552 {
2553         struct i915_ctx_hang_stats *hs;
2554
2555         if (WARN_ON(!ctx))
2556                 return;
2557
2558         hs = &ctx->hang_stats;
2559
2560         if (guilty) {
2561                 hs->banned = i915_context_is_banned(dev_priv, ctx);
2562                 hs->batch_active++;
2563                 hs->guilty_ts = get_seconds();
2564         } else {
2565                 hs->batch_pending++;
2566         }
2567 }
2568
2569 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2570 {
2571         struct intel_context *ctx = request->ctx;
2572
2573         list_del(&request->list);
2574         i915_gem_request_remove_from_client(request);
2575
2576         if (ctx) {
2577                 if (i915.enable_execlists) {
2578                         struct intel_engine_cs *ring = request->ring;
2579
2580                         if (ctx != ring->default_context)
2581                                 intel_lr_context_unpin(ring, ctx);
2582                 }
2583                 i915_gem_context_unreference(ctx);
2584         }
2585         kfree(request);
2586 }
2587
2588 struct drm_i915_gem_request *
2589 i915_gem_find_active_request(struct intel_engine_cs *ring)
2590 {
2591         struct drm_i915_gem_request *request;
2592         u32 completed_seqno;
2593
2594         completed_seqno = ring->get_seqno(ring, false);
2595
2596         list_for_each_entry(request, &ring->request_list, list) {
2597                 if (i915_seqno_passed(completed_seqno, request->seqno))
2598                         continue;
2599
2600                 return request;
2601         }
2602
2603         return NULL;
2604 }
2605
2606 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2607                                        struct intel_engine_cs *ring)
2608 {
2609         struct drm_i915_gem_request *request;
2610         bool ring_hung;
2611
2612         request = i915_gem_find_active_request(ring);
2613
2614         if (request == NULL)
2615                 return;
2616
2617         ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2618
2619         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2620
2621         list_for_each_entry_continue(request, &ring->request_list, list)
2622                 i915_set_reset_status(dev_priv, request->ctx, false);
2623 }
2624
2625 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2626                                         struct intel_engine_cs *ring)
2627 {
2628         while (!list_empty(&ring->active_list)) {
2629                 struct drm_i915_gem_object *obj;
2630
2631                 obj = list_first_entry(&ring->active_list,
2632                                        struct drm_i915_gem_object,
2633                                        ring_list);
2634
2635                 i915_gem_object_move_to_inactive(obj);
2636         }
2637
2638         /*
2639          * Clear the execlists queue up before freeing the requests, as those
2640          * are the ones that keep the context and ringbuffer backing objects
2641          * pinned in place.
2642          */
2643         while (!list_empty(&ring->execlist_queue)) {
2644                 struct intel_ctx_submit_request *submit_req;
2645
2646                 submit_req = list_first_entry(&ring->execlist_queue,
2647                                 struct intel_ctx_submit_request,
2648                                 execlist_link);
2649                 list_del(&submit_req->execlist_link);
2650                 intel_runtime_pm_put(dev_priv);
2651                 i915_gem_context_unreference(submit_req->ctx);
2652                 kfree(submit_req);
2653         }
2654
2655         /*
2656          * We must free the requests after all the corresponding objects have
2657          * been moved off active lists. Which is the same order as the normal
2658          * retire_requests function does. This is important if object hold
2659          * implicit references on things like e.g. ppgtt address spaces through
2660          * the request.
2661          */
2662         while (!list_empty(&ring->request_list)) {
2663                 struct drm_i915_gem_request *request;
2664
2665                 request = list_first_entry(&ring->request_list,
2666                                            struct drm_i915_gem_request,
2667                                            list);
2668
2669                 i915_gem_free_request(request);
2670         }
2671
2672         /* These may not have been flush before the reset, do so now */
2673         kfree(ring->preallocated_lazy_request);
2674         ring->preallocated_lazy_request = NULL;
2675         ring->outstanding_lazy_seqno = 0;
2676 }
2677
2678 void i915_gem_restore_fences(struct drm_device *dev)
2679 {
2680         struct drm_i915_private *dev_priv = dev->dev_private;
2681         int i;
2682
2683         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2684                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2685
2686                 /*
2687                  * Commit delayed tiling changes if we have an object still
2688                  * attached to the fence, otherwise just clear the fence.
2689                  */
2690                 if (reg->obj) {
2691                         i915_gem_object_update_fence(reg->obj, reg,
2692                                                      reg->obj->tiling_mode);
2693                 } else {
2694                         i915_gem_write_fence(dev, i, NULL);
2695                 }
2696         }
2697 }
2698
2699 void i915_gem_reset(struct drm_device *dev)
2700 {
2701         struct drm_i915_private *dev_priv = dev->dev_private;
2702         struct intel_engine_cs *ring;
2703         int i;
2704
2705         /*
2706          * Before we free the objects from the requests, we need to inspect
2707          * them for finding the guilty party. As the requests only borrow
2708          * their reference to the objects, the inspection must be done first.
2709          */
2710         for_each_ring(ring, dev_priv, i)
2711                 i915_gem_reset_ring_status(dev_priv, ring);
2712
2713         for_each_ring(ring, dev_priv, i)
2714                 i915_gem_reset_ring_cleanup(dev_priv, ring);
2715
2716         i915_gem_context_reset(dev);
2717
2718         i915_gem_restore_fences(dev);
2719 }
2720
2721 /**
2722  * This function clears the request list as sequence numbers are passed.
2723  */
2724 void
2725 i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2726 {
2727         uint32_t seqno;
2728
2729         if (list_empty(&ring->request_list))
2730                 return;
2731
2732         WARN_ON(i915_verify_lists(ring->dev));
2733
2734         seqno = ring->get_seqno(ring, true);
2735
2736         /* Move any buffers on the active list that are no longer referenced
2737          * by the ringbuffer to the flushing/inactive lists as appropriate,
2738          * before we free the context associated with the requests.
2739          */
2740         while (!list_empty(&ring->active_list)) {
2741                 struct drm_i915_gem_object *obj;
2742
2743                 obj = list_first_entry(&ring->active_list,
2744                                       struct drm_i915_gem_object,
2745                                       ring_list);
2746
2747                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2748                         break;
2749
2750                 i915_gem_object_move_to_inactive(obj);
2751         }
2752
2753
2754         while (!list_empty(&ring->request_list)) {
2755                 struct drm_i915_gem_request *request;
2756                 struct intel_ringbuffer *ringbuf;
2757
2758                 request = list_first_entry(&ring->request_list,
2759                                            struct drm_i915_gem_request,
2760                                            list);
2761
2762                 if (!i915_seqno_passed(seqno, request->seqno))
2763                         break;
2764
2765                 trace_i915_gem_request_retire(ring, request->seqno);
2766
2767                 /* This is one of the few common intersection points
2768                  * between legacy ringbuffer submission and execlists:
2769                  * we need to tell them apart in order to find the correct
2770                  * ringbuffer to which the request belongs to.
2771                  */
2772                 if (i915.enable_execlists) {
2773                         struct intel_context *ctx = request->ctx;
2774                         ringbuf = ctx->engine[ring->id].ringbuf;
2775                 } else
2776                         ringbuf = ring->buffer;
2777
2778                 /* We know the GPU must have read the request to have
2779                  * sent us the seqno + interrupt, so use the position
2780                  * of tail of the request to update the last known position
2781                  * of the GPU head.
2782                  */
2783                 ringbuf->last_retired_head = request->tail;
2784
2785                 i915_gem_free_request(request);
2786         }
2787
2788         if (unlikely(ring->trace_irq_seqno &&
2789                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2790                 ring->irq_put(ring);
2791                 ring->trace_irq_seqno = 0;
2792         }
2793
2794         WARN_ON(i915_verify_lists(ring->dev));
2795 }
2796
2797 bool
2798 i915_gem_retire_requests(struct drm_device *dev)
2799 {
2800         struct drm_i915_private *dev_priv = dev->dev_private;
2801         struct intel_engine_cs *ring;
2802         bool idle = true;
2803         int i;
2804
2805         for_each_ring(ring, dev_priv, i) {
2806                 i915_gem_retire_requests_ring(ring);
2807                 idle &= list_empty(&ring->request_list);
2808                 if (i915.enable_execlists) {
2809                         unsigned long flags;
2810
2811                         spin_lock_irqsave(&ring->execlist_lock, flags);
2812                         idle &= list_empty(&ring->execlist_queue);
2813                         spin_unlock_irqrestore(&ring->execlist_lock, flags);
2814
2815                         intel_execlists_retire_requests(ring);
2816                 }
2817         }
2818
2819         if (idle)
2820                 mod_delayed_work(dev_priv->wq,
2821                                    &dev_priv->mm.idle_work,
2822                                    msecs_to_jiffies(100));
2823
2824         return idle;
2825 }
2826
2827 static void
2828 i915_gem_retire_work_handler(struct work_struct *work)
2829 {
2830         struct drm_i915_private *dev_priv =
2831                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2832         struct drm_device *dev = dev_priv->dev;
2833         bool idle;
2834
2835         /* Come back later if the device is busy... */
2836         idle = false;
2837         if (mutex_trylock(&dev->struct_mutex)) {
2838                 idle = i915_gem_retire_requests(dev);
2839                 mutex_unlock(&dev->struct_mutex);
2840         }
2841         if (!idle)
2842                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2843                                    round_jiffies_up_relative(HZ));
2844 }
2845
2846 static void
2847 i915_gem_idle_work_handler(struct work_struct *work)
2848 {
2849         struct drm_i915_private *dev_priv =
2850                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
2851
2852         intel_mark_idle(dev_priv->dev);
2853 }
2854
2855 /**
2856  * Ensures that an object will eventually get non-busy by flushing any required
2857  * write domains, emitting any outstanding lazy request and retiring and
2858  * completed requests.
2859  */
2860 static int
2861 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2862 {
2863         int ret;
2864
2865         if (obj->active) {
2866                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2867                 if (ret)
2868                         return ret;
2869
2870                 i915_gem_retire_requests_ring(obj->ring);
2871         }
2872
2873         return 0;
2874 }
2875
2876 /**
2877  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2878  * @DRM_IOCTL_ARGS: standard ioctl arguments
2879  *
2880  * Returns 0 if successful, else an error is returned with the remaining time in
2881  * the timeout parameter.
2882  *  -ETIME: object is still busy after timeout
2883  *  -ERESTARTSYS: signal interrupted the wait
2884  *  -ENONENT: object doesn't exist
2885  * Also possible, but rare:
2886  *  -EAGAIN: GPU wedged
2887  *  -ENOMEM: damn
2888  *  -ENODEV: Internal IRQ fail
2889  *  -E?: The add request failed
2890  *
2891  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2892  * non-zero timeout parameter the wait ioctl will wait for the given number of
2893  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2894  * without holding struct_mutex the object may become re-busied before this
2895  * function completes. A similar but shorter * race condition exists in the busy
2896  * ioctl
2897  */
2898 int
2899 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2900 {
2901         struct drm_i915_private *dev_priv = dev->dev_private;
2902         struct drm_i915_gem_wait *args = data;
2903         struct drm_i915_gem_object *obj;
2904         struct intel_engine_cs *ring = NULL;
2905         unsigned reset_counter;
2906         u32 seqno = 0;
2907         int ret = 0;
2908
2909         if (args->flags != 0)
2910                 return -EINVAL;
2911
2912         ret = i915_mutex_lock_interruptible(dev);
2913         if (ret)
2914                 return ret;
2915
2916         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2917         if (&obj->base == NULL) {
2918                 mutex_unlock(&dev->struct_mutex);
2919                 return -ENOENT;
2920         }
2921
2922         /* Need to make sure the object gets inactive eventually. */
2923         ret = i915_gem_object_flush_active(obj);
2924         if (ret)
2925                 goto out;
2926
2927         if (obj->active) {
2928                 seqno = obj->last_read_seqno;
2929                 ring = obj->ring;
2930         }
2931
2932         if (seqno == 0)
2933                  goto out;
2934
2935         /* Do this after OLR check to make sure we make forward progress polling
2936          * on this IOCTL with a timeout <=0 (like busy ioctl)
2937          */
2938         if (args->timeout_ns <= 0) {
2939                 ret = -ETIME;
2940                 goto out;
2941         }
2942
2943         drm_gem_object_unreference(&obj->base);
2944         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2945         mutex_unlock(&dev->struct_mutex);
2946
2947         return __i915_wait_seqno(ring, seqno, reset_counter, true,
2948                                  &args->timeout_ns, file->driver_priv);
2949
2950 out:
2951         drm_gem_object_unreference(&obj->base);
2952         mutex_unlock(&dev->struct_mutex);
2953         return ret;
2954 }
2955
2956 /**
2957  * i915_gem_object_sync - sync an object to a ring.
2958  *
2959  * @obj: object which may be in use on another ring.
2960  * @to: ring we wish to use the object on. May be NULL.
2961  *
2962  * This code is meant to abstract object synchronization with the GPU.
2963  * Calling with NULL implies synchronizing the object with the CPU
2964  * rather than a particular GPU ring.
2965  *
2966  * Returns 0 if successful, else propagates up the lower layer error.
2967  */
2968 int
2969 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2970                      struct intel_engine_cs *to)
2971 {
2972         struct intel_engine_cs *from = obj->ring;
2973         u32 seqno;
2974         int ret, idx;
2975
2976         if (from == NULL || to == from)
2977                 return 0;
2978
2979         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2980                 return i915_gem_object_wait_rendering(obj, false);
2981
2982         idx = intel_ring_sync_index(from, to);
2983
2984         seqno = obj->last_read_seqno;
2985         /* Optimization: Avoid semaphore sync when we are sure we already
2986          * waited for an object with higher seqno */
2987         if (seqno <= from->semaphore.sync_seqno[idx])
2988                 return 0;
2989
2990         ret = i915_gem_check_olr(obj->ring, seqno);
2991         if (ret)
2992                 return ret;
2993
2994         trace_i915_gem_ring_sync_to(from, to, seqno);
2995         ret = to->semaphore.sync_to(to, from, seqno);
2996         if (!ret)
2997                 /* We use last_read_seqno because sync_to()
2998                  * might have just caused seqno wrap under
2999                  * the radar.
3000                  */
3001                 from->semaphore.sync_seqno[idx] = obj->last_read_seqno;
3002
3003         return ret;
3004 }
3005
3006 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3007 {
3008         u32 old_write_domain, old_read_domains;
3009
3010         /* Force a pagefault for domain tracking on next user access */
3011         i915_gem_release_mmap(obj);
3012
3013         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3014                 return;
3015
3016         /* Wait for any direct GTT access to complete */
3017         mb();
3018
3019         old_read_domains = obj->base.read_domains;
3020         old_write_domain = obj->base.write_domain;
3021
3022         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3023         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3024
3025         trace_i915_gem_object_change_domain(obj,
3026                                             old_read_domains,
3027                                             old_write_domain);
3028 }
3029
3030 int i915_vma_unbind(struct i915_vma *vma)
3031 {
3032         struct drm_i915_gem_object *obj = vma->obj;
3033         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3034         int ret;
3035
3036         if (list_empty(&vma->vma_link))
3037                 return 0;
3038
3039         if (!drm_mm_node_allocated(&vma->node)) {
3040                 i915_gem_vma_destroy(vma);
3041                 return 0;
3042         }
3043
3044         if (vma->pin_count)
3045                 return -EBUSY;
3046
3047         BUG_ON(obj->pages == NULL);
3048
3049         ret = i915_gem_object_finish_gpu(obj);
3050         if (ret)
3051                 return ret;
3052         /* Continue on if we fail due to EIO, the GPU is hung so we
3053          * should be safe and we need to cleanup or else we might
3054          * cause memory corruption through use-after-free.
3055          */
3056
3057         /* Throw away the active reference before moving to the unbound list */
3058         i915_gem_object_retire(obj);
3059
3060         if (i915_is_ggtt(vma->vm)) {
3061                 i915_gem_object_finish_gtt(obj);
3062
3063                 /* release the fence reg _after_ flushing */
3064                 ret = i915_gem_object_put_fence(obj);
3065                 if (ret)
3066                         return ret;
3067         }
3068
3069         trace_i915_vma_unbind(vma);
3070
3071         vma->unbind_vma(vma);
3072
3073         list_del_init(&vma->mm_list);
3074         if (i915_is_ggtt(vma->vm))
3075                 obj->map_and_fenceable = false;
3076
3077         drm_mm_remove_node(&vma->node);
3078         i915_gem_vma_destroy(vma);
3079
3080         /* Since the unbound list is global, only move to that list if
3081          * no more VMAs exist. */
3082         if (list_empty(&obj->vma_list)) {
3083                 i915_gem_gtt_finish_object(obj);
3084                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3085         }
3086
3087         /* And finally now the object is completely decoupled from this vma,
3088          * we can drop its hold on the backing storage and allow it to be
3089          * reaped by the shrinker.
3090          */
3091         i915_gem_object_unpin_pages(obj);
3092
3093         return 0;
3094 }
3095
3096 int i915_gpu_idle(struct drm_device *dev)
3097 {
3098         struct drm_i915_private *dev_priv = dev->dev_private;
3099         struct intel_engine_cs *ring;
3100         int ret, i;
3101
3102         /* Flush everything onto the inactive list. */
3103         for_each_ring(ring, dev_priv, i) {
3104                 if (!i915.enable_execlists) {
3105                         ret = i915_switch_context(ring, ring->default_context);
3106                         if (ret)
3107                                 return ret;
3108                 }
3109
3110                 ret = intel_ring_idle(ring);
3111                 if (ret)
3112                         return ret;
3113         }
3114
3115         return 0;
3116 }
3117
3118 static void i965_write_fence_reg(struct drm_device *dev, int reg,
3119                                  struct drm_i915_gem_object *obj)
3120 {
3121         struct drm_i915_private *dev_priv = dev->dev_private;
3122         int fence_reg;
3123         int fence_pitch_shift;
3124
3125         if (INTEL_INFO(dev)->gen >= 6) {
3126                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
3127                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
3128         } else {
3129                 fence_reg = FENCE_REG_965_0;
3130                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
3131         }
3132
3133         fence_reg += reg * 8;
3134
3135         /* To w/a incoherency with non-atomic 64-bit register updates,
3136          * we split the 64-bit update into two 32-bit writes. In order
3137          * for a partial fence not to be evaluated between writes, we
3138          * precede the update with write to turn off the fence register,
3139          * and only enable the fence as the last step.
3140          *
3141          * For extra levels of paranoia, we make sure each step lands
3142          * before applying the next step.
3143          */
3144         I915_WRITE(fence_reg, 0);
3145         POSTING_READ(fence_reg);
3146
3147         if (obj) {
3148                 u32 size = i915_gem_obj_ggtt_size(obj);
3149                 uint64_t val;
3150
3151                 /* Adjust fence size to match tiled area */
3152                 if (obj->tiling_mode != I915_TILING_NONE) {
3153                         uint32_t row_size = obj->stride *
3154                                 (obj->tiling_mode == I915_TILING_Y ? 32 : 8);
3155                         size = (size / row_size) * row_size;
3156                 }
3157
3158                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3159                                  0xfffff000) << 32;
3160                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3161                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3162                 if (obj->tiling_mode == I915_TILING_Y)
3163                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3164                 val |= I965_FENCE_REG_VALID;
3165
3166                 I915_WRITE(fence_reg + 4, val >> 32);
3167                 POSTING_READ(fence_reg + 4);
3168
3169                 I915_WRITE(fence_reg + 0, val);
3170                 POSTING_READ(fence_reg);
3171         } else {
3172                 I915_WRITE(fence_reg + 4, 0);
3173                 POSTING_READ(fence_reg + 4);
3174         }
3175 }
3176
3177 static void i915_write_fence_reg(struct drm_device *dev, int reg,
3178                                  struct drm_i915_gem_object *obj)
3179 {
3180         struct drm_i915_private *dev_priv = dev->dev_private;
3181         u32 val;
3182
3183         if (obj) {
3184                 u32 size = i915_gem_obj_ggtt_size(obj);
3185                 int pitch_val;
3186                 int tile_width;
3187
3188                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
3189                      (size & -size) != size ||
3190                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3191                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
3192                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
3193
3194                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
3195                         tile_width = 128;
3196                 else
3197                         tile_width = 512;
3198
3199                 /* Note: pitch better be a power of two tile widths */
3200                 pitch_val = obj->stride / tile_width;
3201                 pitch_val = ffs(pitch_val) - 1;
3202
3203                 val = i915_gem_obj_ggtt_offset(obj);
3204                 if (obj->tiling_mode == I915_TILING_Y)
3205                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3206                 val |= I915_FENCE_SIZE_BITS(size);
3207                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3208                 val |= I830_FENCE_REG_VALID;
3209         } else
3210                 val = 0;
3211
3212         if (reg < 8)
3213                 reg = FENCE_REG_830_0 + reg * 4;
3214         else
3215                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
3216
3217         I915_WRITE(reg, val);
3218         POSTING_READ(reg);
3219 }
3220
3221 static void i830_write_fence_reg(struct drm_device *dev, int reg,
3222                                 struct drm_i915_gem_object *obj)
3223 {
3224         struct drm_i915_private *dev_priv = dev->dev_private;
3225         uint32_t val;
3226
3227         if (obj) {
3228                 u32 size = i915_gem_obj_ggtt_size(obj);
3229                 uint32_t pitch_val;
3230
3231                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
3232                      (size & -size) != size ||
3233                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3234                      "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
3235                      i915_gem_obj_ggtt_offset(obj), size);
3236
3237                 pitch_val = obj->stride / 128;
3238                 pitch_val = ffs(pitch_val) - 1;
3239
3240                 val = i915_gem_obj_ggtt_offset(obj);
3241                 if (obj->tiling_mode == I915_TILING_Y)
3242                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3243                 val |= I830_FENCE_SIZE_BITS(size);
3244                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3245                 val |= I830_FENCE_REG_VALID;
3246         } else
3247                 val = 0;
3248
3249         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
3250         POSTING_READ(FENCE_REG_830_0 + reg * 4);
3251 }
3252
3253 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
3254 {
3255         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
3256 }
3257
3258 static void i915_gem_write_fence(struct drm_device *dev, int reg,
3259                                  struct drm_i915_gem_object *obj)
3260 {
3261         struct drm_i915_private *dev_priv = dev->dev_private;
3262
3263         /* Ensure that all CPU reads are completed before installing a fence
3264          * and all writes before removing the fence.
3265          */
3266         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
3267                 mb();
3268
3269         WARN(obj && (!obj->stride || !obj->tiling_mode),
3270              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
3271              obj->stride, obj->tiling_mode);
3272
3273         switch (INTEL_INFO(dev)->gen) {
3274         case 9:
3275         case 8:
3276         case 7:
3277         case 6:
3278         case 5:
3279         case 4: i965_write_fence_reg(dev, reg, obj); break;
3280         case 3: i915_write_fence_reg(dev, reg, obj); break;
3281         case 2: i830_write_fence_reg(dev, reg, obj); break;
3282         default: BUG();
3283         }
3284
3285         /* And similarly be paranoid that no direct access to this region
3286          * is reordered to before the fence is installed.
3287          */
3288         if (i915_gem_object_needs_mb(obj))
3289                 mb();
3290 }
3291
3292 static inline int fence_number(struct drm_i915_private *dev_priv,
3293                                struct drm_i915_fence_reg *fence)
3294 {
3295         return fence - dev_priv->fence_regs;
3296 }
3297
3298 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
3299                                          struct drm_i915_fence_reg *fence,
3300                                          bool enable)
3301 {
3302         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3303         int reg = fence_number(dev_priv, fence);
3304
3305         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3306
3307         if (enable) {
3308                 obj->fence_reg = reg;
3309                 fence->obj = obj;
3310                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
3311         } else {
3312                 obj->fence_reg = I915_FENCE_REG_NONE;
3313                 fence->obj = NULL;
3314                 list_del_init(&fence->lru_list);
3315         }
3316         obj->fence_dirty = false;
3317 }
3318
3319 static int
3320 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3321 {
3322         if (obj->last_fenced_seqno) {
3323                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
3324                 if (ret)
3325                         return ret;
3326
3327                 obj->last_fenced_seqno = 0;
3328         }
3329
3330         return 0;
3331 }
3332
3333 int
3334 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3335 {
3336         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3337         struct drm_i915_fence_reg *fence;
3338         int ret;
3339
3340         ret = i915_gem_object_wait_fence(obj);
3341         if (ret)
3342                 return ret;
3343
3344         if (obj->fence_reg == I915_FENCE_REG_NONE)
3345                 return 0;
3346
3347         fence = &dev_priv->fence_regs[obj->fence_reg];
3348
3349         if (WARN_ON(fence->pin_count))
3350                 return -EBUSY;
3351
3352         i915_gem_object_fence_lost(obj);
3353         i915_gem_object_update_fence(obj, fence, false);
3354
3355         return 0;
3356 }
3357
3358 static struct drm_i915_fence_reg *
3359 i915_find_fence_reg(struct drm_device *dev)
3360 {
3361         struct drm_i915_private *dev_priv = dev->dev_private;
3362         struct drm_i915_fence_reg *reg, *avail;
3363         int i;
3364
3365         /* First try to find a free reg */
3366         avail = NULL;
3367         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3368                 reg = &dev_priv->fence_regs[i];
3369                 if (!reg->obj)
3370                         return reg;
3371
3372                 if (!reg->pin_count)
3373                         avail = reg;
3374         }
3375
3376         if (avail == NULL)
3377                 goto deadlock;
3378
3379         /* None available, try to steal one or wait for a user to finish */
3380         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3381                 if (reg->pin_count)
3382                         continue;
3383
3384                 return reg;
3385         }
3386
3387 deadlock:
3388         /* Wait for completion of pending flips which consume fences */
3389         if (intel_has_pending_fb_unpin(dev))
3390                 return ERR_PTR(-EAGAIN);
3391
3392         return ERR_PTR(-EDEADLK);
3393 }
3394
3395 /**
3396  * i915_gem_object_get_fence - set up fencing for an object
3397  * @obj: object to map through a fence reg
3398  *
3399  * When mapping objects through the GTT, userspace wants to be able to write
3400  * to them without having to worry about swizzling if the object is tiled.
3401  * This function walks the fence regs looking for a free one for @obj,
3402  * stealing one if it can't find any.
3403  *
3404  * It then sets up the reg based on the object's properties: address, pitch
3405  * and tiling format.
3406  *
3407  * For an untiled surface, this removes any existing fence.
3408  */
3409 int
3410 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3411 {
3412         struct drm_device *dev = obj->base.dev;
3413         struct drm_i915_private *dev_priv = dev->dev_private;
3414         bool enable = obj->tiling_mode != I915_TILING_NONE;
3415         struct drm_i915_fence_reg *reg;
3416         int ret;
3417
3418         /* Have we updated the tiling parameters upon the object and so
3419          * will need to serialise the write to the associated fence register?
3420          */
3421         if (obj->fence_dirty) {
3422                 ret = i915_gem_object_wait_fence(obj);
3423                 if (ret)
3424                         return ret;
3425         }
3426
3427         /* Just update our place in the LRU if our fence is getting reused. */
3428         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3429                 reg = &dev_priv->fence_regs[obj->fence_reg];
3430                 if (!obj->fence_dirty) {
3431                         list_move_tail(&reg->lru_list,
3432                                        &dev_priv->mm.fence_list);
3433                         return 0;
3434                 }
3435         } else if (enable) {
3436                 if (WARN_ON(!obj->map_and_fenceable))
3437                         return -EINVAL;
3438
3439                 reg = i915_find_fence_reg(dev);
3440                 if (IS_ERR(reg))
3441                         return PTR_ERR(reg);
3442
3443                 if (reg->obj) {
3444                         struct drm_i915_gem_object *old = reg->obj;
3445
3446                         ret = i915_gem_object_wait_fence(old);
3447                         if (ret)
3448                                 return ret;
3449
3450                         i915_gem_object_fence_lost(old);
3451                 }
3452         } else
3453                 return 0;
3454
3455         i915_gem_object_update_fence(obj, reg, enable);
3456
3457         return 0;
3458 }
3459
3460 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3461                                      unsigned long cache_level)
3462 {
3463         struct drm_mm_node *gtt_space = &vma->node;
3464         struct drm_mm_node *other;
3465
3466         /*
3467          * On some machines we have to be careful when putting differing types
3468          * of snoopable memory together to avoid the prefetcher crossing memory
3469          * domains and dying. During vm initialisation, we decide whether or not
3470          * these constraints apply and set the drm_mm.color_adjust
3471          * appropriately.
3472          */
3473         if (vma->vm->mm.color_adjust == NULL)
3474                 return true;
3475
3476         if (!drm_mm_node_allocated(gtt_space))
3477                 return true;
3478
3479         if (list_empty(&gtt_space->node_list))
3480                 return true;
3481
3482         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3483         if (other->allocated && !other->hole_follows && other->color != cache_level)
3484                 return false;
3485
3486         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3487         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3488                 return false;
3489
3490         return true;
3491 }
3492
3493 /**
3494  * Finds free space in the GTT aperture and binds the object there.
3495  */
3496 static struct i915_vma *
3497 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3498                            struct i915_address_space *vm,
3499                            unsigned alignment,
3500                            uint64_t flags)
3501 {
3502         struct drm_device *dev = obj->base.dev;
3503         struct drm_i915_private *dev_priv = dev->dev_private;
3504         u32 size, fence_size, fence_alignment, unfenced_alignment;
3505         unsigned long start =
3506                 flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3507         unsigned long end =
3508                 flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
3509         struct i915_vma *vma;
3510         int ret;
3511
3512         fence_size = i915_gem_get_gtt_size(dev,
3513                                            obj->base.size,
3514                                            obj->tiling_mode);
3515         fence_alignment = i915_gem_get_gtt_alignment(dev,
3516                                                      obj->base.size,
3517                                                      obj->tiling_mode, true);
3518         unfenced_alignment =
3519                 i915_gem_get_gtt_alignment(dev,
3520                                            obj->base.size,
3521                                            obj->tiling_mode, false);
3522
3523         if (alignment == 0)
3524                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3525                                                 unfenced_alignment;
3526         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3527                 DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
3528                 return ERR_PTR(-EINVAL);
3529         }
3530
3531         size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3532
3533         /* If the object is bigger than the entire aperture, reject it early
3534          * before evicting everything in a vain attempt to find space.
3535          */
3536         if (obj->base.size > end) {
3537                 DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n",
3538                           obj->base.size,
3539                           flags & PIN_MAPPABLE ? "mappable" : "total",
3540                           end);
3541                 return ERR_PTR(-E2BIG);
3542         }
3543
3544         ret = i915_gem_object_get_pages(obj);
3545         if (ret)
3546                 return ERR_PTR(ret);
3547
3548         i915_gem_object_pin_pages(obj);
3549
3550         vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
3551         if (IS_ERR(vma))
3552                 goto err_unpin;
3553
3554 search_free:
3555         ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3556                                                   size, alignment,
3557                                                   obj->cache_level,
3558                                                   start, end,
3559                                                   DRM_MM_SEARCH_DEFAULT,
3560                                                   DRM_MM_CREATE_DEFAULT);
3561         if (ret) {
3562                 ret = i915_gem_evict_something(dev, vm, size, alignment,
3563                                                obj->cache_level,
3564                                                start, end,
3565                                                flags);
3566                 if (ret == 0)
3567                         goto search_free;
3568
3569                 goto err_free_vma;
3570         }
3571         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3572                 ret = -EINVAL;
3573                 goto err_remove_node;
3574         }
3575
3576         ret = i915_gem_gtt_prepare_object(obj);
3577         if (ret)
3578                 goto err_remove_node;
3579
3580         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3581         list_add_tail(&vma->mm_list, &vm->inactive_list);
3582
3583         trace_i915_vma_bind(vma, flags);
3584         vma->bind_vma(vma, obj->cache_level,
3585                       flags & PIN_GLOBAL ? GLOBAL_BIND : 0);
3586
3587         return vma;
3588
3589 err_remove_node:
3590         drm_mm_remove_node(&vma->node);
3591 err_free_vma:
3592         i915_gem_vma_destroy(vma);
3593         vma = ERR_PTR(ret);
3594 err_unpin:
3595         i915_gem_object_unpin_pages(obj);
3596         return vma;
3597 }
3598
3599 bool
3600 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3601                         bool force)
3602 {
3603         /* If we don't have a page list set up, then we're not pinned
3604          * to GPU, and we can ignore the cache flush because it'll happen
3605          * again at bind time.
3606          */
3607         if (obj->pages == NULL)
3608                 return false;
3609
3610         /*
3611          * Stolen memory is always coherent with the GPU as it is explicitly
3612          * marked as wc by the system, or the system is cache-coherent.
3613          */
3614         if (obj->stolen || obj->phys_handle)
3615                 return false;
3616
3617         /* If the GPU is snooping the contents of the CPU cache,
3618          * we do not need to manually clear the CPU cache lines.  However,
3619          * the caches are only snooped when the render cache is
3620          * flushed/invalidated.  As we always have to emit invalidations
3621          * and flushes when moving into and out of the RENDER domain, correct
3622          * snooping behaviour occurs naturally as the result of our domain
3623          * tracking.
3624          */
3625         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3626                 return false;
3627
3628         trace_i915_gem_object_clflush(obj);
3629         drm_clflush_sg(obj->pages);
3630
3631         return true;
3632 }
3633
3634 /** Flushes the GTT write domain for the object if it's dirty. */
3635 static void
3636 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3637 {
3638         uint32_t old_write_domain;
3639
3640         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3641                 return;
3642
3643         /* No actual flushing is required for the GTT write domain.  Writes
3644          * to it immediately go to main memory as far as we know, so there's
3645          * no chipset flush.  It also doesn't land in render cache.
3646          *
3647          * However, we do have to enforce the order so that all writes through
3648          * the GTT land before any writes to the device, such as updates to
3649          * the GATT itself.
3650          */
3651         wmb();
3652
3653         old_write_domain = obj->base.write_domain;
3654         obj->base.write_domain = 0;
3655
3656         intel_fb_obj_flush(obj, false);
3657
3658         trace_i915_gem_object_change_domain(obj,
3659                                             obj->base.read_domains,
3660                                             old_write_domain);
3661 }
3662
3663 /** Flushes the CPU write domain for the object if it's dirty. */
3664 static void
3665 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
3666                                        bool force)
3667 {
3668         uint32_t old_write_domain;
3669
3670         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3671                 return;
3672
3673         if (i915_gem_clflush_object(obj, force))
3674                 i915_gem_chipset_flush(obj->base.dev);
3675
3676         old_write_domain = obj->base.write_domain;
3677         obj->base.write_domain = 0;
3678
3679         intel_fb_obj_flush(obj, false);
3680
3681         trace_i915_gem_object_change_domain(obj,
3682                                             obj->base.read_domains,
3683                                             old_write_domain);
3684 }
3685
3686 /**
3687  * Moves a single object to the GTT read, and possibly write domain.
3688  *
3689  * This function returns when the move is complete, including waiting on
3690  * flushes to occur.
3691  */
3692 int
3693 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3694 {
3695         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3696         struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
3697         uint32_t old_write_domain, old_read_domains;
3698         int ret;
3699
3700         /* Not valid to be called on unbound objects. */
3701         if (vma == NULL)
3702                 return -EINVAL;
3703
3704         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3705                 return 0;
3706
3707         ret = i915_gem_object_wait_rendering(obj, !write);
3708         if (ret)
3709                 return ret;
3710
3711         i915_gem_object_retire(obj);
3712         i915_gem_object_flush_cpu_write_domain(obj, false);
3713
3714         /* Serialise direct access to this object with the barriers for
3715          * coherent writes from the GPU, by effectively invalidating the
3716          * GTT domain upon first access.
3717          */
3718         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3719                 mb();
3720
3721         old_write_domain = obj->base.write_domain;
3722         old_read_domains = obj->base.read_domains;
3723
3724         /* It should now be out of any other write domains, and we can update
3725          * the domain values for our changes.
3726          */
3727         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3728         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3729         if (write) {
3730                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3731                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3732                 obj->dirty = 1;
3733         }
3734
3735         if (write)
3736                 intel_fb_obj_invalidate(obj, NULL);
3737
3738         trace_i915_gem_object_change_domain(obj,
3739                                             old_read_domains,
3740                                             old_write_domain);
3741
3742         /* And bump the LRU for this access */
3743         if (i915_gem_object_is_inactive(obj))
3744                 list_move_tail(&vma->mm_list,
3745                                &dev_priv->gtt.base.inactive_list);
3746
3747         return 0;
3748 }
3749
3750 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3751                                     enum i915_cache_level cache_level)
3752 {
3753         struct drm_device *dev = obj->base.dev;
3754         struct i915_vma *vma, *next;
3755         int ret;
3756
3757         if (obj->cache_level == cache_level)
3758                 return 0;
3759
3760         if (i915_gem_obj_is_pinned(obj)) {
3761                 DRM_DEBUG("can not change the cache level of pinned objects\n");
3762                 return -EBUSY;
3763         }
3764
3765         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
3766                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3767                         ret = i915_vma_unbind(vma);
3768                         if (ret)
3769                                 return ret;
3770                 }
3771         }
3772
3773         if (i915_gem_obj_bound_any(obj)) {
3774                 ret = i915_gem_object_finish_gpu(obj);
3775                 if (ret)
3776                         return ret;
3777
3778                 i915_gem_object_finish_gtt(obj);
3779
3780                 /* Before SandyBridge, you could not use tiling or fence
3781                  * registers with snooped memory, so relinquish any fences
3782                  * currently pointing to our region in the aperture.
3783                  */
3784                 if (INTEL_INFO(dev)->gen < 6) {
3785                         ret = i915_gem_object_put_fence(obj);
3786                         if (ret)
3787                                 return ret;
3788                 }
3789
3790                 list_for_each_entry(vma, &obj->vma_list, vma_link)
3791                         if (drm_mm_node_allocated(&vma->node))
3792                                 vma->bind_vma(vma, cache_level,
3793                                                 vma->bound & GLOBAL_BIND);
3794         }
3795
3796         list_for_each_entry(vma, &obj->vma_list, vma_link)
3797                 vma->node.color = cache_level;
3798         obj->cache_level = cache_level;
3799
3800         if (cpu_write_needs_clflush(obj)) {
3801                 u32 old_read_domains, old_write_domain;
3802
3803                 /* If we're coming from LLC cached, then we haven't
3804                  * actually been tracking whether the data is in the
3805                  * CPU cache or not, since we only allow one bit set
3806                  * in obj->write_domain and have been skipping the clflushes.
3807                  * Just set it to the CPU cache for now.
3808                  */
3809                 i915_gem_object_retire(obj);
3810                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3811
3812                 old_read_domains = obj->base.read_domains;
3813                 old_write_domain = obj->base.write_domain;
3814
3815                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3816                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3817
3818                 trace_i915_gem_object_change_domain(obj,
3819                                                     old_read_domains,
3820                                                     old_write_domain);
3821         }
3822
3823         return 0;
3824 }
3825
3826 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3827                                struct drm_file *file)
3828 {
3829         struct drm_i915_gem_caching *args = data;
3830         struct drm_i915_gem_object *obj;
3831         int ret;
3832
3833         ret = i915_mutex_lock_interruptible(dev);
3834         if (ret)
3835                 return ret;
3836
3837         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3838         if (&obj->base == NULL) {
3839                 ret = -ENOENT;
3840                 goto unlock;
3841         }
3842
3843         switch (obj->cache_level) {
3844         case I915_CACHE_LLC:
3845         case I915_CACHE_L3_LLC:
3846                 args->caching = I915_CACHING_CACHED;
3847                 break;
3848
3849         case I915_CACHE_WT:
3850                 args->caching = I915_CACHING_DISPLAY;
3851                 break;
3852
3853         default:
3854                 args->caching = I915_CACHING_NONE;
3855                 break;
3856         }
3857
3858         drm_gem_object_unreference(&obj->base);
3859 unlock:
3860         mutex_unlock(&dev->struct_mutex);
3861         return ret;
3862 }
3863
3864 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3865                                struct drm_file *file)
3866 {
3867         struct drm_i915_gem_caching *args = data;
3868         struct drm_i915_gem_object *obj;
3869         enum i915_cache_level level;
3870         int ret;
3871
3872         switch (args->caching) {
3873         case I915_CACHING_NONE:
3874                 level = I915_CACHE_NONE;
3875                 break;
3876         case I915_CACHING_CACHED:
3877                 level = I915_CACHE_LLC;
3878                 break;
3879         case I915_CACHING_DISPLAY:
3880                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3881                 break;
3882         default:
3883                 return -EINVAL;
3884         }
3885
3886         ret = i915_mutex_lock_interruptible(dev);
3887         if (ret)
3888                 return ret;
3889
3890         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3891         if (&obj->base == NULL) {
3892                 ret = -ENOENT;
3893                 goto unlock;
3894         }
3895
3896         ret = i915_gem_object_set_cache_level(obj, level);
3897
3898         drm_gem_object_unreference(&obj->base);
3899 unlock:
3900         mutex_unlock(&dev->struct_mutex);
3901         return ret;
3902 }
3903
3904 static bool is_pin_display(struct drm_i915_gem_object *obj)
3905 {
3906         struct i915_vma *vma;
3907
3908         vma = i915_gem_obj_to_ggtt(obj);
3909         if (!vma)
3910                 return false;
3911
3912         /* There are 3 sources that pin objects:
3913          *   1. The display engine (scanouts, sprites, cursors);
3914          *   2. Reservations for execbuffer;
3915          *   3. The user.
3916          *
3917          * We can ignore reservations as we hold the struct_mutex and
3918          * are only called outside of the reservation path.  The user
3919          * can only increment pin_count once, and so if after
3920          * subtracting the potential reference by the user, any pin_count
3921          * remains, it must be due to another use by the display engine.
3922          */
3923         return vma->pin_count - !!obj->user_pin_count;
3924 }
3925
3926 /*
3927  * Prepare buffer for display plane (scanout, cursors, etc).
3928  * Can be called from an uninterruptible phase (modesetting) and allows
3929  * any flushes to be pipelined (for pageflips).
3930  */
3931 int
3932 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3933                                      u32 alignment,
3934                                      struct intel_engine_cs *pipelined)
3935 {
3936         u32 old_read_domains, old_write_domain;
3937         bool was_pin_display;
3938         int ret;
3939
3940         if (pipelined != obj->ring) {
3941                 ret = i915_gem_object_sync(obj, pipelined);
3942                 if (ret)
3943                         return ret;
3944         }
3945
3946         /* Mark the pin_display early so that we account for the
3947          * display coherency whilst setting up the cache domains.
3948          */
3949         was_pin_display = obj->pin_display;
3950         obj->pin_display = true;
3951
3952         /* The display engine is not coherent with the LLC cache on gen6.  As
3953          * a result, we make sure that the pinning that is about to occur is
3954          * done with uncached PTEs. This is lowest common denominator for all
3955          * chipsets.
3956          *
3957          * However for gen6+, we could do better by using the GFDT bit instead
3958          * of uncaching, which would allow us to flush all the LLC-cached data
3959          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3960          */
3961         ret = i915_gem_object_set_cache_level(obj,
3962                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3963         if (ret)
3964                 goto err_unpin_display;
3965
3966         /* As the user may map the buffer once pinned in the display plane
3967          * (e.g. libkms for the bootup splash), we have to ensure that we
3968          * always use map_and_fenceable for all scanout buffers.
3969          */
3970         ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE);
3971         if (ret)
3972                 goto err_unpin_display;
3973
3974         i915_gem_object_flush_cpu_write_domain(obj, true);
3975
3976         old_write_domain = obj->base.write_domain;
3977         old_read_domains = obj->base.read_domains;
3978
3979         /* It should now be out of any other write domains, and we can update
3980          * the domain values for our changes.
3981          */
3982         obj->base.write_domain = 0;
3983         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3984
3985         trace_i915_gem_object_change_domain(obj,
3986                                             old_read_domains,
3987                                             old_write_domain);
3988
3989         return 0;
3990
3991 err_unpin_display:
3992         WARN_ON(was_pin_display != is_pin_display(obj));
3993         obj->pin_display = was_pin_display;
3994         return ret;
3995 }
3996
3997 void
3998 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
3999 {
4000         i915_gem_object_ggtt_unpin(obj);
4001         obj->pin_display = is_pin_display(obj);
4002 }
4003
4004 int
4005 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
4006 {
4007         int ret;
4008
4009         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
4010                 return 0;
4011
4012         ret = i915_gem_object_wait_rendering(obj, false);
4013         if (ret)
4014                 return ret;
4015
4016         /* Ensure that we invalidate the GPU's caches and TLBs. */
4017         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
4018         return 0;
4019 }
4020
4021 /**
4022  * Moves a single object to the CPU read, and possibly write domain.
4023  *
4024  * This function returns when the move is complete, including waiting on
4025  * flushes to occur.
4026  */
4027 int
4028 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4029 {
4030         uint32_t old_write_domain, old_read_domains;
4031         int ret;
4032
4033         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4034                 return 0;
4035
4036         ret = i915_gem_object_wait_rendering(obj, !write);
4037         if (ret)
4038                 return ret;
4039
4040         i915_gem_object_retire(obj);
4041         i915_gem_object_flush_gtt_write_domain(obj);
4042
4043         old_write_domain = obj->base.write_domain;
4044         old_read_domains = obj->base.read_domains;
4045
4046         /* Flush the CPU cache if it's still invalid. */
4047         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4048                 i915_gem_clflush_object(obj, false);
4049
4050                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4051         }
4052
4053         /* It should now be out of any other write domains, and we can update
4054          * the domain values for our changes.
4055          */
4056         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4057
4058         /* If we're writing through the CPU, then the GPU read domains will
4059          * need to be invalidated at next use.
4060          */
4061         if (write) {
4062                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4063                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4064         }
4065
4066         if (write)
4067                 intel_fb_obj_invalidate(obj, NULL);
4068
4069         trace_i915_gem_object_change_domain(obj,
4070                                             old_read_domains,
4071                                             old_write_domain);
4072
4073         return 0;
4074 }
4075
4076 /* Throttle our rendering by waiting until the ring has completed our requests
4077  * emitted over 20 msec ago.
4078  *
4079  * Note that if we were to use the current jiffies each time around the loop,
4080  * we wouldn't escape the function with any frames outstanding if the time to
4081  * render a frame was over 20ms.
4082  *
4083  * This should get us reasonable parallelism between CPU and GPU but also
4084  * relatively low latency when blocking on a particular request to finish.
4085  */
4086 static int
4087 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4088 {
4089         struct drm_i915_private *dev_priv = dev->dev_private;
4090         struct drm_i915_file_private *file_priv = file->driver_priv;
4091         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
4092         struct drm_i915_gem_request *request;
4093         struct intel_engine_cs *ring = NULL;
4094         unsigned reset_counter;
4095         u32 seqno = 0;
4096         int ret;
4097
4098         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4099         if (ret)
4100                 return ret;
4101
4102         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
4103         if (ret)
4104                 return ret;
4105
4106         spin_lock(&file_priv->mm.lock);
4107         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4108                 if (time_after_eq(request->emitted_jiffies, recent_enough))
4109                         break;
4110
4111                 ring = request->ring;
4112                 seqno = request->seqno;
4113         }
4114         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
4115         spin_unlock(&file_priv->mm.lock);
4116
4117         if (seqno == 0)
4118                 return 0;
4119
4120         ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
4121         if (ret == 0)
4122                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4123
4124         return ret;
4125 }
4126
4127 static bool
4128 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4129 {
4130         struct drm_i915_gem_object *obj = vma->obj;
4131
4132         if (alignment &&
4133             vma->node.start & (alignment - 1))
4134                 return true;
4135
4136         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4137                 return true;
4138
4139         if (flags & PIN_OFFSET_BIAS &&
4140             vma->node.start < (flags & PIN_OFFSET_MASK))
4141                 return true;
4142
4143         return false;
4144 }
4145
4146 int
4147 i915_gem_object_pin(struct drm_i915_gem_object *obj,
4148                     struct i915_address_space *vm,
4149                     uint32_t alignment,
4150                     uint64_t flags)
4151 {
4152         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4153         struct i915_vma *vma;
4154         unsigned bound;
4155         int ret;
4156
4157         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4158                 return -ENODEV;
4159
4160         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4161                 return -EINVAL;
4162
4163         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4164                 return -EINVAL;
4165
4166         vma = i915_gem_obj_to_vma(obj, vm);
4167         if (vma) {
4168                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4169                         return -EBUSY;
4170
4171                 if (i915_vma_misplaced(vma, alignment, flags)) {
4172                         WARN(vma->pin_count,
4173                              "bo is already pinned with incorrect alignment:"
4174                              " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
4175                              " obj->map_and_fenceable=%d\n",
4176                              i915_gem_obj_offset(obj, vm), alignment,
4177                              !!(flags & PIN_MAPPABLE),
4178                              obj->map_and_fenceable);
4179                         ret = i915_vma_unbind(vma);
4180                         if (ret)
4181                                 return ret;
4182
4183                         vma = NULL;
4184                 }
4185         }
4186
4187         bound = vma ? vma->bound : 0;
4188         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4189                 vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
4190                 if (IS_ERR(vma))
4191                         return PTR_ERR(vma);
4192         }
4193
4194         if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND))
4195                 vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
4196
4197         if ((bound ^ vma->bound) & GLOBAL_BIND) {
4198                 bool mappable, fenceable;
4199                 u32 fence_size, fence_alignment;
4200
4201                 fence_size = i915_gem_get_gtt_size(obj->base.dev,
4202                                                    obj->base.size,
4203                                                    obj->tiling_mode);
4204                 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4205                                                              obj->base.size,
4206                                                              obj->tiling_mode,
4207                                                              true);
4208
4209                 fenceable = (vma->node.size == fence_size &&
4210                              (vma->node.start & (fence_alignment - 1)) == 0);
4211
4212                 mappable = (vma->node.start + obj->base.size <=
4213                             dev_priv->gtt.mappable_end);
4214
4215                 obj->map_and_fenceable = mappable && fenceable;
4216         }
4217
4218         WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4219
4220         vma->pin_count++;
4221         if (flags & PIN_MAPPABLE)
4222                 obj->pin_mappable |= true;
4223
4224         return 0;
4225 }
4226
4227 void
4228 i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
4229 {
4230         struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
4231
4232         BUG_ON(!vma);
4233         BUG_ON(vma->pin_count == 0);
4234         BUG_ON(!i915_gem_obj_ggtt_bound(obj));
4235
4236         if (--vma->pin_count == 0)
4237                 obj->pin_mappable = false;
4238 }
4239
4240 bool
4241 i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
4242 {
4243         if (obj->fence_reg != I915_FENCE_REG_NONE) {
4244                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4245                 struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj);
4246
4247                 WARN_ON(!ggtt_vma ||
4248                         dev_priv->fence_regs[obj->fence_reg].pin_count >
4249                         ggtt_vma->pin_count);
4250                 dev_priv->fence_regs[obj->fence_reg].pin_count++;
4251                 return true;
4252         } else
4253                 return false;
4254 }
4255
4256 void
4257 i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
4258 {
4259         if (obj->fence_reg != I915_FENCE_REG_NONE) {
4260                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4261                 WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
4262                 dev_priv->fence_regs[obj->fence_reg].pin_count--;
4263         }
4264 }
4265
4266 int
4267 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4268                    struct drm_file *file)
4269 {
4270         struct drm_i915_gem_pin *args = data;
4271         struct drm_i915_gem_object *obj;
4272         int ret;
4273
4274         if (drm_core_check_feature(dev, DRIVER_MODESET))
4275                 return -ENODEV;
4276
4277         ret = i915_mutex_lock_interruptible(dev);
4278         if (ret)
4279                 return ret;
4280
4281         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4282         if (&obj->base == NULL) {
4283                 ret = -ENOENT;
4284                 goto unlock;
4285         }
4286
4287         if (obj->madv != I915_MADV_WILLNEED) {
4288                 DRM_DEBUG("Attempting to pin a purgeable buffer\n");
4289                 ret = -EFAULT;
4290                 goto out;
4291         }
4292
4293         if (obj->pin_filp != NULL && obj->pin_filp != file) {
4294                 DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n",
4295                           args->handle);
4296                 ret = -EINVAL;
4297                 goto out;
4298         }
4299
4300         if (obj->user_pin_count == ULONG_MAX) {
4301                 ret = -EBUSY;
4302                 goto out;
4303         }
4304
4305         if (obj->user_pin_count == 0) {
4306                 ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE);
4307                 if (ret)
4308                         goto out;
4309         }
4310
4311         obj->user_pin_count++;
4312         obj->pin_filp = file;
4313
4314         args->offset = i915_gem_obj_ggtt_offset(obj);
4315 out:
4316         drm_gem_object_unreference(&obj->base);
4317 unlock:
4318         mutex_unlock(&dev->struct_mutex);
4319         return ret;
4320 }
4321
4322 int
4323 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4324                      struct drm_file *file)
4325 {
4326         struct drm_i915_gem_pin *args = data;
4327         struct drm_i915_gem_object *obj;
4328         int ret;
4329
4330         if (drm_core_check_feature(dev, DRIVER_MODESET))
4331                 return -ENODEV;
4332
4333         ret = i915_mutex_lock_interruptible(dev);
4334         if (ret)
4335                 return ret;
4336
4337         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4338         if (&obj->base == NULL) {
4339                 ret = -ENOENT;
4340                 goto unlock;
4341         }
4342
4343         if (obj->pin_filp != file) {
4344                 DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4345                           args->handle);
4346                 ret = -EINVAL;
4347                 goto out;
4348         }
4349         obj->user_pin_count--;
4350         if (obj->user_pin_count == 0) {
4351                 obj->pin_filp = NULL;
4352                 i915_gem_object_ggtt_unpin(obj);
4353         }
4354
4355 out:
4356         drm_gem_object_unreference(&obj->base);
4357 unlock:
4358         mutex_unlock(&dev->struct_mutex);
4359         return ret;
4360 }
4361
4362 int
4363 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4364                     struct drm_file *file)
4365 {
4366         struct drm_i915_gem_busy *args = data;
4367         struct drm_i915_gem_object *obj;
4368         int ret;
4369
4370         ret = i915_mutex_lock_interruptible(dev);
4371         if (ret)
4372                 return ret;
4373
4374         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4375         if (&obj->base == NULL) {
4376                 ret = -ENOENT;
4377                 goto unlock;
4378         }
4379
4380         /* Count all active objects as busy, even if they are currently not used
4381          * by the gpu. Users of this interface expect objects to eventually
4382          * become non-busy without any further actions, therefore emit any
4383          * necessary flushes here.
4384          */
4385         ret = i915_gem_object_flush_active(obj);
4386
4387         args->busy = obj->active;
4388         if (obj->ring) {
4389                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
4390                 args->busy |= intel_ring_flag(obj->ring) << 16;
4391         }
4392
4393         drm_gem_object_unreference(&obj->base);
4394 unlock:
4395         mutex_unlock(&dev->struct_mutex);
4396         return ret;
4397 }
4398
4399 int
4400 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4401                         struct drm_file *file_priv)
4402 {
4403         return i915_gem_ring_throttle(dev, file_priv);
4404 }
4405
4406 int
4407 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4408                        struct drm_file *file_priv)
4409 {
4410         struct drm_i915_private *dev_priv = dev->dev_private;
4411         struct drm_i915_gem_madvise *args = data;
4412         struct drm_i915_gem_object *obj;
4413         int ret;
4414
4415         switch (args->madv) {
4416         case I915_MADV_DONTNEED:
4417         case I915_MADV_WILLNEED:
4418             break;
4419         default:
4420             return -EINVAL;
4421         }
4422
4423         ret = i915_mutex_lock_interruptible(dev);
4424         if (ret)
4425                 return ret;
4426
4427         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4428         if (&obj->base == NULL) {
4429                 ret = -ENOENT;
4430                 goto unlock;
4431         }
4432
4433         if (i915_gem_obj_is_pinned(obj)) {
4434                 ret = -EINVAL;
4435                 goto out;
4436         }
4437
4438         if (obj->pages &&
4439             obj->tiling_mode != I915_TILING_NONE &&
4440             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4441                 if (obj->madv == I915_MADV_WILLNEED)
4442                         i915_gem_object_unpin_pages(obj);
4443                 if (args->madv == I915_MADV_WILLNEED)
4444                         i915_gem_object_pin_pages(obj);
4445         }
4446
4447         if (obj->madv != __I915_MADV_PURGED)
4448                 obj->madv = args->madv;
4449
4450         /* if the object is no longer attached, discard its backing storage */
4451         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
4452                 i915_gem_object_truncate(obj);
4453
4454         args->retained = obj->madv != __I915_MADV_PURGED;
4455
4456 out:
4457         drm_gem_object_unreference(&obj->base);
4458 unlock:
4459         mutex_unlock(&dev->struct_mutex);
4460         return ret;
4461 }
4462
4463 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4464                           const struct drm_i915_gem_object_ops *ops)
4465 {
4466         INIT_LIST_HEAD(&obj->global_list);
4467         INIT_LIST_HEAD(&obj->ring_list);
4468         INIT_LIST_HEAD(&obj->obj_exec_link);
4469         INIT_LIST_HEAD(&obj->vma_list);
4470
4471         obj->ops = ops;
4472
4473         obj->fence_reg = I915_FENCE_REG_NONE;
4474         obj->madv = I915_MADV_WILLNEED;
4475
4476         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4477 }
4478
4479 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4480         .get_pages = i915_gem_object_get_pages_gtt,
4481         .put_pages = i915_gem_object_put_pages_gtt,
4482 };
4483
4484 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4485                                                   size_t size)
4486 {
4487         struct drm_i915_gem_object *obj;
4488         struct address_space *mapping;
4489         gfp_t mask;
4490
4491         obj = i915_gem_object_alloc(dev);
4492         if (obj == NULL)
4493                 return NULL;
4494
4495         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4496                 i915_gem_object_free(obj);
4497                 return NULL;
4498         }
4499
4500         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4501         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4502                 /* 965gm cannot relocate objects above 4GiB. */
4503                 mask &= ~__GFP_HIGHMEM;
4504                 mask |= __GFP_DMA32;
4505         }
4506
4507         mapping = file_inode(obj->base.filp)->i_mapping;
4508         mapping_set_gfp_mask(mapping, mask);
4509
4510         i915_gem_object_init(obj, &i915_gem_object_ops);
4511
4512         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4513         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4514
4515         if (HAS_LLC(dev)) {
4516                 /* On some devices, we can have the GPU use the LLC (the CPU
4517                  * cache) for about a 10% performance improvement
4518                  * compared to uncached.  Graphics requests other than
4519                  * display scanout are coherent with the CPU in
4520                  * accessing this cache.  This means in this mode we
4521                  * don't need to clflush on the CPU side, and on the
4522                  * GPU side we only need to flush internal caches to
4523                  * get data visible to the CPU.
4524                  *
4525                  * However, we maintain the display planes as UC, and so
4526                  * need to rebind when first used as such.
4527                  */
4528                 obj->cache_level = I915_CACHE_LLC;
4529         } else
4530                 obj->cache_level = I915_CACHE_NONE;
4531
4532         trace_i915_gem_object_create(obj);
4533
4534         return obj;
4535 }
4536
4537 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4538 {
4539         /* If we are the last user of the backing storage (be it shmemfs
4540          * pages or stolen etc), we know that the pages are going to be
4541          * immediately released. In this case, we can then skip copying
4542          * back the contents from the GPU.
4543          */
4544
4545         if (obj->madv != I915_MADV_WILLNEED)
4546                 return false;
4547
4548         if (obj->base.filp == NULL)
4549                 return true;
4550
4551         /* At first glance, this looks racy, but then again so would be
4552          * userspace racing mmap against close. However, the first external
4553          * reference to the filp can only be obtained through the
4554          * i915_gem_mmap_ioctl() which safeguards us against the user
4555          * acquiring such a reference whilst we are in the middle of
4556          * freeing the object.
4557          */
4558         return atomic_long_read(&obj->base.filp->f_count) == 1;
4559 }
4560
4561 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4562 {
4563         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4564         struct drm_device *dev = obj->base.dev;
4565         struct drm_i915_private *dev_priv = dev->dev_private;
4566         struct i915_vma *vma, *next;
4567
4568         intel_runtime_pm_get(dev_priv);
4569
4570         trace_i915_gem_object_destroy(obj);
4571
4572         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4573                 int ret;
4574
4575                 vma->pin_count = 0;
4576                 ret = i915_vma_unbind(vma);
4577                 if (WARN_ON(ret == -ERESTARTSYS)) {
4578                         bool was_interruptible;
4579
4580                         was_interruptible = dev_priv->mm.interruptible;
4581                         dev_priv->mm.interruptible = false;
4582
4583                         WARN_ON(i915_vma_unbind(vma));
4584
4585                         dev_priv->mm.interruptible = was_interruptible;
4586                 }
4587         }
4588
4589         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4590          * before progressing. */
4591         if (obj->stolen)
4592                 i915_gem_object_unpin_pages(obj);
4593
4594         WARN_ON(obj->frontbuffer_bits);
4595
4596         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4597             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4598             obj->tiling_mode != I915_TILING_NONE)
4599                 i915_gem_object_unpin_pages(obj);
4600
4601         if (WARN_ON(obj->pages_pin_count))
4602                 obj->pages_pin_count = 0;
4603         if (discard_backing_storage(obj))
4604                 obj->madv = I915_MADV_DONTNEED;
4605         i915_gem_object_put_pages(obj);
4606         i915_gem_object_free_mmap_offset(obj);
4607
4608         BUG_ON(obj->pages);
4609
4610         if (obj->base.import_attach)
4611                 drm_prime_gem_destroy(&obj->base, NULL);
4612
4613         if (obj->ops->release)
4614                 obj->ops->release(obj);
4615
4616         drm_gem_object_release(&obj->base);
4617         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4618
4619         kfree(obj->bit_17);
4620         i915_gem_object_free(obj);
4621
4622         intel_runtime_pm_put(dev_priv);
4623 }
4624
4625 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4626                                      struct i915_address_space *vm)
4627 {
4628         struct i915_vma *vma;
4629         list_for_each_entry(vma, &obj->vma_list, vma_link)
4630                 if (vma->vm == vm)
4631                         return vma;
4632
4633         return NULL;
4634 }
4635
4636 void i915_gem_vma_destroy(struct i915_vma *vma)
4637 {
4638         struct i915_address_space *vm = NULL;
4639         WARN_ON(vma->node.allocated);
4640
4641         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4642         if (!list_empty(&vma->exec_list))
4643                 return;
4644
4645         vm = vma->vm;
4646
4647         if (!i915_is_ggtt(vm))
4648                 i915_ppgtt_put(i915_vm_to_ppgtt(vm));
4649
4650         list_del(&vma->vma_link);
4651
4652         kfree(vma);
4653 }
4654
4655 static void
4656 i915_gem_stop_ringbuffers(struct drm_device *dev)
4657 {
4658         struct drm_i915_private *dev_priv = dev->dev_private;
4659         struct intel_engine_cs *ring;
4660         int i;
4661
4662         for_each_ring(ring, dev_priv, i)
4663                 dev_priv->gt.stop_ring(ring);
4664 }
4665
4666 int
4667 i915_gem_suspend(struct drm_device *dev)
4668 {
4669         struct drm_i915_private *dev_priv = dev->dev_private;
4670         int ret = 0;
4671
4672         mutex_lock(&dev->struct_mutex);
4673         ret = i915_gpu_idle(dev);
4674         if (ret)
4675                 goto err;
4676
4677         i915_gem_retire_requests(dev);
4678
4679         /* Under UMS, be paranoid and evict. */
4680         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4681                 i915_gem_evict_everything(dev);
4682
4683         i915_gem_stop_ringbuffers(dev);
4684         mutex_unlock(&dev->struct_mutex);
4685
4686         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4687         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4688         flush_delayed_work(&dev_priv->mm.idle_work);
4689
4690         return 0;
4691
4692 err:
4693         mutex_unlock(&dev->struct_mutex);
4694         return ret;
4695 }
4696
4697 int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice)
4698 {
4699         struct drm_device *dev = ring->dev;
4700         struct drm_i915_private *dev_priv = dev->dev_private;
4701         u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
4702         u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4703         int i, ret;
4704
4705         if (!HAS_L3_DPF(dev) || !remap_info)
4706                 return 0;
4707
4708         ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
4709         if (ret)
4710                 return ret;
4711
4712         /*
4713          * Note: We do not worry about the concurrent register cacheline hang
4714          * here because no other code should access these registers other than
4715          * at initialization time.
4716          */
4717         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4718                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4719                 intel_ring_emit(ring, reg_base + i);
4720                 intel_ring_emit(ring, remap_info[i/4]);
4721         }
4722
4723         intel_ring_advance(ring);
4724
4725         return ret;
4726 }
4727
4728 void i915_gem_init_swizzling(struct drm_device *dev)
4729 {
4730         struct drm_i915_private *dev_priv = dev->dev_private;
4731
4732         if (INTEL_INFO(dev)->gen < 5 ||
4733             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4734                 return;
4735
4736         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4737                                  DISP_TILE_SURFACE_SWIZZLING);
4738
4739         if (IS_GEN5(dev))
4740                 return;
4741
4742         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4743         if (IS_GEN6(dev))
4744                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4745         else if (IS_GEN7(dev))
4746                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4747         else if (IS_GEN8(dev))
4748                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4749         else
4750                 BUG();
4751 }
4752
4753 static bool
4754 intel_enable_blt(struct drm_device *dev)
4755 {
4756         if (!HAS_BLT(dev))
4757                 return false;
4758
4759         /* The blitter was dysfunctional on early prototypes */
4760         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
4761                 DRM_INFO("BLT not supported on this pre-production hardware;"
4762                          " graphics performance will be degraded.\n");
4763                 return false;
4764         }
4765
4766         return true;
4767 }
4768
4769 static void init_unused_ring(struct drm_device *dev, u32 base)
4770 {
4771         struct drm_i915_private *dev_priv = dev->dev_private;
4772
4773         I915_WRITE(RING_CTL(base), 0);
4774         I915_WRITE(RING_HEAD(base), 0);
4775         I915_WRITE(RING_TAIL(base), 0);
4776         I915_WRITE(RING_START(base), 0);
4777 }
4778
4779 static void init_unused_rings(struct drm_device *dev)
4780 {
4781         if (IS_I830(dev)) {
4782                 init_unused_ring(dev, PRB1_BASE);
4783                 init_unused_ring(dev, SRB0_BASE);
4784                 init_unused_ring(dev, SRB1_BASE);
4785                 init_unused_ring(dev, SRB2_BASE);
4786                 init_unused_ring(dev, SRB3_BASE);
4787         } else if (IS_GEN2(dev)) {
4788                 init_unused_ring(dev, SRB0_BASE);
4789                 init_unused_ring(dev, SRB1_BASE);
4790         } else if (IS_GEN3(dev)) {
4791                 init_unused_ring(dev, PRB1_BASE);
4792                 init_unused_ring(dev, PRB2_BASE);
4793         }
4794 }
4795
4796 int i915_gem_init_rings(struct drm_device *dev)
4797 {
4798         struct drm_i915_private *dev_priv = dev->dev_private;
4799         int ret;
4800
4801         /*
4802          * At least 830 can leave some of the unused rings
4803          * "active" (ie. head != tail) after resume which
4804          * will prevent c3 entry. Makes sure all unused rings
4805          * are totally idle.
4806          */
4807         init_unused_rings(dev);
4808
4809         ret = intel_init_render_ring_buffer(dev);
4810         if (ret)
4811                 return ret;
4812
4813         if (HAS_BSD(dev)) {
4814                 ret = intel_init_bsd_ring_buffer(dev);
4815                 if (ret)
4816                         goto cleanup_render_ring;
4817         }
4818
4819         if (intel_enable_blt(dev)) {
4820                 ret = intel_init_blt_ring_buffer(dev);
4821                 if (ret)
4822                         goto cleanup_bsd_ring;
4823         }
4824
4825         if (HAS_VEBOX(dev)) {
4826                 ret = intel_init_vebox_ring_buffer(dev);
4827                 if (ret)
4828                         goto cleanup_blt_ring;
4829         }
4830
4831         if (HAS_BSD2(dev)) {
4832                 ret = intel_init_bsd2_ring_buffer(dev);
4833                 if (ret)
4834                         goto cleanup_vebox_ring;
4835         }
4836
4837         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4838         if (ret)
4839                 goto cleanup_bsd2_ring;
4840
4841         return 0;
4842
4843 cleanup_bsd2_ring:
4844         intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]);
4845 cleanup_vebox_ring:
4846         intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4847 cleanup_blt_ring:
4848         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4849 cleanup_bsd_ring:
4850         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4851 cleanup_render_ring:
4852         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4853
4854         return ret;
4855 }
4856
4857 int
4858 i915_gem_init_hw(struct drm_device *dev)
4859 {
4860         struct drm_i915_private *dev_priv = dev->dev_private;
4861         int ret, i;
4862
4863         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4864                 return -EIO;
4865
4866         if (dev_priv->ellc_size)
4867                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4868
4869         if (IS_HASWELL(dev))
4870                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4871                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4872
4873         if (HAS_PCH_NOP(dev)) {
4874                 if (IS_IVYBRIDGE(dev)) {
4875                         u32 temp = I915_READ(GEN7_MSG_CTL);
4876                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4877                         I915_WRITE(GEN7_MSG_CTL, temp);
4878                 } else if (INTEL_INFO(dev)->gen >= 7) {
4879                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4880                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4881                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4882                 }
4883         }
4884
4885         i915_gem_init_swizzling(dev);
4886
4887         ret = dev_priv->gt.init_rings(dev);
4888         if (ret)
4889                 return ret;
4890
4891         for (i = 0; i < NUM_L3_SLICES(dev); i++)
4892                 i915_gem_l3_remap(&dev_priv->ring[RCS], i);
4893
4894         ret = i915_ppgtt_init_hw(dev);
4895         if (ret && ret != -EIO) {
4896                 DRM_ERROR("PPGTT enable failed %d\n", ret);
4897                 i915_gem_cleanup_ringbuffer(dev);
4898         }
4899
4900         ret = i915_gem_context_enable(dev_priv);
4901         if (ret && ret != -EIO) {
4902                 DRM_ERROR("Context enable failed %d\n", ret);
4903                 i915_gem_cleanup_ringbuffer(dev);
4904
4905                 return ret;
4906         }
4907
4908         return ret;
4909 }
4910
4911 int i915_gem_init(struct drm_device *dev)
4912 {
4913         struct drm_i915_private *dev_priv = dev->dev_private;
4914         int ret;
4915
4916         i915.enable_execlists = intel_sanitize_enable_execlists(dev,
4917                         i915.enable_execlists);
4918
4919         mutex_lock(&dev->struct_mutex);
4920
4921         if (IS_VALLEYVIEW(dev)) {
4922                 /* VLVA0 (potential hack), BIOS isn't actually waking us */
4923                 I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
4924                 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
4925                               VLV_GTLC_ALLOWWAKEACK), 10))
4926                         DRM_DEBUG_DRIVER("allow wake ack timed out\n");
4927         }
4928
4929         if (!i915.enable_execlists) {
4930                 dev_priv->gt.do_execbuf = i915_gem_ringbuffer_submission;
4931                 dev_priv->gt.init_rings = i915_gem_init_rings;
4932                 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
4933                 dev_priv->gt.stop_ring = intel_stop_ring_buffer;
4934         } else {
4935                 dev_priv->gt.do_execbuf = intel_execlists_submission;
4936                 dev_priv->gt.init_rings = intel_logical_rings_init;
4937                 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
4938                 dev_priv->gt.stop_ring = intel_logical_ring_stop;
4939         }
4940
4941         ret = i915_gem_init_userptr(dev);
4942         if (ret) {
4943                 mutex_unlock(&dev->struct_mutex);
4944                 return ret;
4945         }
4946
4947         i915_gem_init_global_gtt(dev);
4948
4949         ret = i915_gem_context_init(dev);
4950         if (ret) {
4951                 mutex_unlock(&dev->struct_mutex);
4952                 return ret;
4953         }
4954
4955         ret = i915_gem_init_hw(dev);
4956         if (ret == -EIO) {
4957                 /* Allow ring initialisation to fail by marking the GPU as
4958                  * wedged. But we only want to do this where the GPU is angry,
4959                  * for all other failure, such as an allocation failure, bail.
4960                  */
4961                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4962                 atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4963                 ret = 0;
4964         }
4965         mutex_unlock(&dev->struct_mutex);
4966
4967         return ret;
4968 }
4969
4970 void
4971 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4972 {
4973         struct drm_i915_private *dev_priv = dev->dev_private;
4974         struct intel_engine_cs *ring;
4975         int i;
4976
4977         for_each_ring(ring, dev_priv, i)
4978                 dev_priv->gt.cleanup_ring(ring);
4979 }
4980
4981 static void
4982 init_ring_lists(struct intel_engine_cs *ring)
4983 {
4984         INIT_LIST_HEAD(&ring->active_list);
4985         INIT_LIST_HEAD(&ring->request_list);
4986 }
4987
4988 void i915_init_vm(struct drm_i915_private *dev_priv,
4989                   struct i915_address_space *vm)
4990 {
4991         if (!i915_is_ggtt(vm))
4992                 drm_mm_init(&vm->mm, vm->start, vm->total);
4993         vm->dev = dev_priv->dev;
4994         INIT_LIST_HEAD(&vm->active_list);
4995         INIT_LIST_HEAD(&vm->inactive_list);
4996         INIT_LIST_HEAD(&vm->global_link);
4997         list_add_tail(&vm->global_link, &dev_priv->vm_list);
4998 }
4999
5000 void
5001 i915_gem_load(struct drm_device *dev)
5002 {
5003         struct drm_i915_private *dev_priv = dev->dev_private;
5004         int i;
5005
5006         dev_priv->slab =
5007                 kmem_cache_create("i915_gem_object",
5008                                   sizeof(struct drm_i915_gem_object), 0,
5009                                   SLAB_HWCACHE_ALIGN,
5010                                   NULL);
5011
5012         INIT_LIST_HEAD(&dev_priv->vm_list);
5013         i915_init_vm(dev_priv, &dev_priv->gtt.base);
5014
5015         INIT_LIST_HEAD(&dev_priv->context_list);
5016         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5017         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5018         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5019         for (i = 0; i < I915_NUM_RINGS; i++)
5020                 init_ring_lists(&dev_priv->ring[i]);
5021         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5022                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5023         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5024                           i915_gem_retire_work_handler);
5025         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5026                           i915_gem_idle_work_handler);
5027         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5028
5029         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
5030         if (!drm_core_check_feature(dev, DRIVER_MODESET) && IS_GEN3(dev)) {
5031                 I915_WRITE(MI_ARB_STATE,
5032                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5033         }
5034
5035         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5036
5037         /* Old X drivers will take 0-2 for front, back, depth buffers */
5038         if (!drm_core_check_feature(dev, DRIVER_MODESET))
5039                 dev_priv->fence_reg_start = 3;
5040
5041         if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
5042                 dev_priv->num_fence_regs = 32;
5043         else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5044                 dev_priv->num_fence_regs = 16;
5045         else
5046                 dev_priv->num_fence_regs = 8;
5047
5048         /* Initialize fence registers to zero */
5049         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5050         i915_gem_restore_fences(dev);
5051
5052         i915_gem_detect_bit_6_swizzle(dev);
5053         init_waitqueue_head(&dev_priv->pending_flip_queue);
5054
5055         dev_priv->mm.interruptible = true;
5056
5057         dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
5058         dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
5059         dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
5060         register_shrinker(&dev_priv->mm.shrinker);
5061
5062         dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
5063         register_oom_notifier(&dev_priv->mm.oom_notifier);
5064
5065         mutex_init(&dev_priv->fb_tracking.lock);
5066 }
5067
5068 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5069 {
5070         struct drm_i915_file_private *file_priv = file->driver_priv;
5071
5072         cancel_delayed_work_sync(&file_priv->mm.idle_work);
5073
5074         /* Clean up our request list when the client is going away, so that
5075          * later retire_requests won't dereference our soon-to-be-gone
5076          * file_priv.
5077          */
5078         spin_lock(&file_priv->mm.lock);
5079         while (!list_empty(&file_priv->mm.request_list)) {
5080                 struct drm_i915_gem_request *request;
5081
5082                 request = list_first_entry(&file_priv->mm.request_list,
5083                                            struct drm_i915_gem_request,
5084                                            client_list);
5085                 list_del(&request->client_list);
5086                 request->file_priv = NULL;
5087         }
5088         spin_unlock(&file_priv->mm.lock);
5089 }
5090
5091 static void
5092 i915_gem_file_idle_work_handler(struct work_struct *work)
5093 {
5094         struct drm_i915_file_private *file_priv =
5095                 container_of(work, typeof(*file_priv), mm.idle_work.work);
5096
5097         atomic_set(&file_priv->rps_wait_boost, false);
5098 }
5099
5100 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5101 {
5102         struct drm_i915_file_private *file_priv;
5103         int ret;
5104
5105         DRM_DEBUG_DRIVER("\n");
5106
5107         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5108         if (!file_priv)
5109                 return -ENOMEM;
5110
5111         file->driver_priv = file_priv;
5112         file_priv->dev_priv = dev->dev_private;
5113         file_priv->file = file;
5114
5115         spin_lock_init(&file_priv->mm.lock);
5116         INIT_LIST_HEAD(&file_priv->mm.request_list);
5117         INIT_DELAYED_WORK(&file_priv->mm.idle_work,
5118                           i915_gem_file_idle_work_handler);
5119
5120         ret = i915_gem_context_open(dev, file);
5121         if (ret)
5122                 kfree(file_priv);
5123
5124         return ret;
5125 }
5126
5127 /**
5128  * i915_gem_track_fb - update frontbuffer tracking
5129  * old: current GEM buffer for the frontbuffer slots
5130  * new: new GEM buffer for the frontbuffer slots
5131  * frontbuffer_bits: bitmask of frontbuffer slots
5132  *
5133  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5134  * from @old and setting them in @new. Both @old and @new can be NULL.
5135  */
5136 void i915_gem_track_fb(struct drm_i915_gem_object *old,
5137                        struct drm_i915_gem_object *new,
5138                        unsigned frontbuffer_bits)
5139 {
5140         if (old) {
5141                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5142                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5143                 old->frontbuffer_bits &= ~frontbuffer_bits;
5144         }
5145
5146         if (new) {
5147                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5148                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5149                 new->frontbuffer_bits |= frontbuffer_bits;
5150         }
5151 }
5152
5153 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
5154 {
5155         if (!mutex_is_locked(mutex))
5156                 return false;
5157
5158 #if defined(CONFIG_SMP) && !defined(CONFIG_DEBUG_MUTEXES)
5159         return mutex->owner == task;
5160 #else
5161         /* Since UP may be pre-empted, we cannot assume that we own the lock */
5162         return false;
5163 #endif
5164 }
5165
5166 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
5167 {
5168         if (!mutex_trylock(&dev->struct_mutex)) {
5169                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
5170                         return false;
5171
5172                 if (to_i915(dev)->mm.shrinker_no_lock_stealing)
5173                         return false;
5174
5175                 *unlock = false;
5176         } else
5177                 *unlock = true;
5178
5179         return true;
5180 }
5181
5182 static int num_vma_bound(struct drm_i915_gem_object *obj)
5183 {
5184         struct i915_vma *vma;
5185         int count = 0;
5186
5187         list_for_each_entry(vma, &obj->vma_list, vma_link)
5188                 if (drm_mm_node_allocated(&vma->node))
5189                         count++;
5190
5191         return count;
5192 }
5193
5194 static unsigned long
5195 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
5196 {
5197         struct drm_i915_private *dev_priv =
5198                 container_of(shrinker, struct drm_i915_private, mm.shrinker);
5199         struct drm_device *dev = dev_priv->dev;
5200         struct drm_i915_gem_object *obj;
5201         unsigned long count;
5202         bool unlock;
5203
5204         if (!i915_gem_shrinker_lock(dev, &unlock))
5205                 return 0;
5206
5207         count = 0;
5208         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
5209                 if (obj->pages_pin_count == 0)
5210                         count += obj->base.size >> PAGE_SHIFT;
5211
5212         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
5213                 if (!i915_gem_obj_is_pinned(obj) &&
5214                     obj->pages_pin_count == num_vma_bound(obj))
5215                         count += obj->base.size >> PAGE_SHIFT;
5216         }
5217
5218         if (unlock)
5219                 mutex_unlock(&dev->struct_mutex);
5220
5221         return count;
5222 }
5223
5224 /* All the new VM stuff */
5225 unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
5226                                   struct i915_address_space *vm)
5227 {
5228         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5229         struct i915_vma *vma;
5230
5231         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5232
5233         list_for_each_entry(vma, &o->vma_list, vma_link) {
5234                 if (vma->vm == vm)
5235                         return vma->node.start;
5236
5237         }
5238         WARN(1, "%s vma for this object not found.\n",
5239              i915_is_ggtt(vm) ? "global" : "ppgtt");
5240         return -1;
5241 }
5242
5243 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5244                         struct i915_address_space *vm)
5245 {
5246         struct i915_vma *vma;
5247
5248         list_for_each_entry(vma, &o->vma_list, vma_link)
5249                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5250                         return true;
5251
5252         return false;
5253 }
5254
5255 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5256 {
5257         struct i915_vma *vma;
5258
5259         list_for_each_entry(vma, &o->vma_list, vma_link)
5260                 if (drm_mm_node_allocated(&vma->node))
5261                         return true;
5262
5263         return false;
5264 }
5265
5266 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5267                                 struct i915_address_space *vm)
5268 {
5269         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5270         struct i915_vma *vma;
5271
5272         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5273
5274         BUG_ON(list_empty(&o->vma_list));
5275
5276         list_for_each_entry(vma, &o->vma_list, vma_link)
5277                 if (vma->vm == vm)
5278                         return vma->node.size;
5279
5280         return 0;
5281 }
5282
5283 static unsigned long
5284 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
5285 {
5286         struct drm_i915_private *dev_priv =
5287                 container_of(shrinker, struct drm_i915_private, mm.shrinker);
5288         struct drm_device *dev = dev_priv->dev;
5289         unsigned long freed;
5290         bool unlock;
5291
5292         if (!i915_gem_shrinker_lock(dev, &unlock))
5293                 return SHRINK_STOP;
5294
5295         freed = i915_gem_shrink(dev_priv,
5296                                 sc->nr_to_scan,
5297                                 I915_SHRINK_BOUND |
5298                                 I915_SHRINK_UNBOUND |
5299                                 I915_SHRINK_PURGEABLE);
5300         if (freed < sc->nr_to_scan)
5301                 freed += i915_gem_shrink(dev_priv,
5302                                          sc->nr_to_scan - freed,
5303                                          I915_SHRINK_BOUND |
5304                                          I915_SHRINK_UNBOUND);
5305         if (unlock)
5306                 mutex_unlock(&dev->struct_mutex);
5307
5308         return freed;
5309 }
5310
5311 static int
5312 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
5313 {
5314         struct drm_i915_private *dev_priv =
5315                 container_of(nb, struct drm_i915_private, mm.oom_notifier);
5316         struct drm_device *dev = dev_priv->dev;
5317         struct drm_i915_gem_object *obj;
5318         unsigned long timeout = msecs_to_jiffies(5000) + 1;
5319         unsigned long pinned, bound, unbound, freed_pages;
5320         bool was_interruptible;
5321         bool unlock;
5322
5323         while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) {
5324                 schedule_timeout_killable(1);
5325                 if (fatal_signal_pending(current))
5326                         return NOTIFY_DONE;
5327         }
5328         if (timeout == 0) {
5329                 pr_err("Unable to purge GPU memory due lock contention.\n");
5330                 return NOTIFY_DONE;
5331         }
5332
5333         was_interruptible = dev_priv->mm.interruptible;
5334         dev_priv->mm.interruptible = false;
5335
5336         freed_pages = i915_gem_shrink_all(dev_priv);
5337
5338         dev_priv->mm.interruptible = was_interruptible;
5339
5340         /* Because we may be allocating inside our own driver, we cannot
5341          * assert that there are no objects with pinned pages that are not
5342          * being pointed to by hardware.
5343          */
5344         unbound = bound = pinned = 0;
5345         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
5346                 if (!obj->base.filp) /* not backed by a freeable object */
5347                         continue;
5348
5349                 if (obj->pages_pin_count)
5350                         pinned += obj->base.size;
5351                 else
5352                         unbound += obj->base.size;
5353         }
5354         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
5355                 if (!obj->base.filp)
5356                         continue;
5357
5358                 if (obj->pages_pin_count)
5359                         pinned += obj->base.size;
5360                 else
5361                         bound += obj->base.size;
5362         }
5363
5364         if (unlock)
5365                 mutex_unlock(&dev->struct_mutex);
5366
5367         if (freed_pages || unbound || bound)
5368                 pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n",
5369                         freed_pages << PAGE_SHIFT, pinned);
5370         if (unbound || bound)
5371                 pr_err("%lu and %lu bytes still available in the "
5372                        "bound and unbound GPU page lists.\n",
5373                        bound, unbound);
5374
5375         *(unsigned long *)ptr += freed_pages;
5376         return NOTIFY_DONE;
5377 }
5378
5379 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
5380 {
5381         struct i915_vma *vma;
5382
5383         vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
5384         if (vma->vm != i915_obj_to_ggtt(obj))
5385                 return NULL;
5386
5387         return vma;
5388 }