drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_vgpu.h"
  33 #include "i915_trace.h"
  34 #include "intel_drv.h"
  35 #include <linux/shmem_fs.h>
  36 #include <linux/slab.h>
  37 #include <linux/swap.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-buf.h>
  40
  41 #define RQ_BUG_ON(expr)
  42
  43 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  44 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  45 static void
  46 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
  47 static void
  48 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
  49 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  50                                  struct drm_i915_gem_object *obj);
  51 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  52                                          struct drm_i915_fence_reg *fence,
  53                                          bool enable);
  54
  55 static bool cpu_cache_is_coherent(struct drm_device *dev,
  56                                   enum i915_cache_level level)
  57 {
  58         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  59 }
  60
  61 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  62 {
  63         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  64                 return true;
  65
  66         return obj->pin_display;
  67 }
  68
  69 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  70 {
  71         if (obj->tiling_mode)
  72                 i915_gem_release_mmap(obj);
  73
  74         /* As we do not have an associated fence register, we will force
  75          * a tiling change if we ever need to acquire one.
  76          */
  77         obj->fence_dirty = false;
  78         obj->fence_reg = I915_FENCE_REG_NONE;
  79 }
  80
  81 /* some bookkeeping */
  82 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  83                                   size_t size)
  84 {
  85         spin_lock(&dev_priv->mm.object_stat_lock);
  86         dev_priv->mm.object_count++;
  87         dev_priv->mm.object_memory += size;
  88         spin_unlock(&dev_priv->mm.object_stat_lock);
  89 }
  90
  91 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  92                                      size_t size)
  93 {
  94         spin_lock(&dev_priv->mm.object_stat_lock);
  95         dev_priv->mm.object_count--;
  96         dev_priv->mm.object_memory -= size;
  97         spin_unlock(&dev_priv->mm.object_stat_lock);
  98 }
  99
 100 static int
 101 i915_gem_wait_for_error(struct i915_gpu_error *error)
 102 {
 103         int ret;
 104
 105 #define EXIT_COND (!i915_reset_in_progress(error) || \
 106                    i915_terminally_wedged(error))
 107         if (EXIT_COND)
 108                 return 0;
 109
 110         /*
 111          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 112          * userspace. If it takes that long something really bad is going on and
 113          * we should simply try to bail out and fail as gracefully as possible.
 114          */
 115         ret = wait_event_interruptible_timeout(error->reset_queue,
 116                                                EXIT_COND,
 117                                                10*HZ);
 118         if (ret == 0) {
 119                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 120                 return -EIO;
 121         } else if (ret < 0) {
 122                 return ret;
 123         }
 124 #undef EXIT_COND
 125
 126         return 0;
 127 }
 128
 129 int i915_mutex_lock_interruptible(struct drm_device *dev)
 130 {
 131         struct drm_i915_private *dev_priv = dev->dev_private;
 132         int ret;
 133
 134         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 135         if (ret)
 136                 return ret;
 137
 138         ret = mutex_lock_interruptible(&dev->struct_mutex);
 139         if (ret)
 140                 return ret;
 141
 142         WARN_ON(i915_verify_lists(dev));
 143         return 0;
 144 }
 145
 146 int
 147 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 148                             struct drm_file *file)
 149 {
 150         struct drm_i915_private *dev_priv = dev->dev_private;
 151         struct drm_i915_gem_get_aperture *args = data;
 152         struct i915_gtt *ggtt = &dev_priv->gtt;
 153         struct i915_vma *vma;
 154         size_t pinned;
 155
 156         pinned = 0;
 157         mutex_lock(&dev->struct_mutex);
 158         list_for_each_entry(vma, &ggtt->base.active_list, mm_list)
 159                 if (vma->pin_count)
 160                         pinned += vma->node.size;
 161         list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list)
 162                 if (vma->pin_count)
 163                         pinned += vma->node.size;
 164         mutex_unlock(&dev->struct_mutex);
 165
 166         args->aper_size = dev_priv->gtt.base.total;
 167         args->aper_available_size = args->aper_size - pinned;
 168
 169         return 0;
 170 }
 171
 172 static int
 173 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 174 {
 175         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 176         char *vaddr = obj->phys_handle->vaddr;
 177         struct sg_table *st;
 178         struct scatterlist *sg;
 179         int i;
 180
 181         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 182                 return -EINVAL;
 183
 184         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 185                 struct page *page;
 186                 char *src;
 187
 188                 page = shmem_read_mapping_page(mapping, i);
 189                 if (IS_ERR(page))
 190                         return PTR_ERR(page);
 191
 192                 src = kmap_atomic(page);
 193                 memcpy(vaddr, src, PAGE_SIZE);
 194                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 195                 kunmap_atomic(src);
 196
 197                 page_cache_release(page);
 198                 vaddr += PAGE_SIZE;
 199         }
 200
 201         i915_gem_chipset_flush(obj->base.dev);
 202
 203         st = kmalloc(sizeof(*st), GFP_KERNEL);
 204         if (st == NULL)
 205                 return -ENOMEM;
 206
 207         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 208                 kfree(st);
 209                 return -ENOMEM;
 210         }
 211
 212         sg = st->sgl;
 213         sg->offset = 0;
 214         sg->length = obj->base.size;
 215
 216         sg_dma_address(sg) = obj->phys_handle->busaddr;
 217         sg_dma_len(sg) = obj->base.size;
 218
 219         obj->pages = st;
 220         return 0;
 221 }
 222
 223 static void
 224 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 225 {
 226         int ret;
 227
 228         BUG_ON(obj->madv == __I915_MADV_PURGED);
 229
 230         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 231         if (ret) {
 232                 /* In the event of a disaster, abandon all caches and
 233                  * hope for the best.
 234                  */
 235                 WARN_ON(ret != -EIO);
 236                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 237         }
 238
 239         if (obj->madv == I915_MADV_DONTNEED)
 240                 obj->dirty = 0;
 241
 242         if (obj->dirty) {
 243                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 244                 char *vaddr = obj->phys_handle->vaddr;
 245                 int i;
 246
 247                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 248                         struct page *page;
 249                         char *dst;
 250
 251                         page = shmem_read_mapping_page(mapping, i);
 252                         if (IS_ERR(page))
 253                                 continue;
 254
 255                         dst = kmap_atomic(page);
 256                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 257                         memcpy(dst, vaddr, PAGE_SIZE);
 258                         kunmap_atomic(dst);
 259
 260                         set_page_dirty(page);
 261                         if (obj->madv == I915_MADV_WILLNEED)
 262                                 mark_page_accessed(page);
 263                         page_cache_release(page);
 264                         vaddr += PAGE_SIZE;
 265                 }
 266                 obj->dirty = 0;
 267         }
 268
 269         sg_free_table(obj->pages);
 270         kfree(obj->pages);
 271 }
 272
 273 static void
 274 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 275 {
 276         drm_pci_free(obj->base.dev, obj->phys_handle);
 277 }
 278
 279 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 280         .get_pages = i915_gem_object_get_pages_phys,
 281         .put_pages = i915_gem_object_put_pages_phys,
 282         .release = i915_gem_object_release_phys,
 283 };
 284
 285 static int
 286 drop_pages(struct drm_i915_gem_object *obj)
 287 {
 288         struct i915_vma *vma, *next;
 289         int ret;
 290
 291         drm_gem_object_reference(&obj->base);
 292         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
 293                 if (i915_vma_unbind(vma))
 294                         break;
 295
 296         ret = i915_gem_object_put_pages(obj);
 297         drm_gem_object_unreference(&obj->base);
 298
 299         return ret;
 300 }
 301
 302 int
 303 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 304                             int align)
 305 {
 306         drm_dma_handle_t *phys;
 307         int ret;
 308
 309         if (obj->phys_handle) {
 310                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 311                         return -EBUSY;
 312
 313                 return 0;
 314         }
 315
 316         if (obj->madv != I915_MADV_WILLNEED)
 317                 return -EFAULT;
 318
 319         if (obj->base.filp == NULL)
 320                 return -EINVAL;
 321
 322         ret = drop_pages(obj);
 323         if (ret)
 324                 return ret;
 325
 326         /* create a new object */
 327         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 328         if (!phys)
 329                 return -ENOMEM;
 330
 331         obj->phys_handle = phys;
 332         obj->ops = &i915_gem_phys_ops;
 333
 334         return i915_gem_object_get_pages(obj);
 335 }
 336
 337 static int
 338 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 339                      struct drm_i915_gem_pwrite *args,
 340                      struct drm_file *file_priv)
 341 {
 342         struct drm_device *dev = obj->base.dev;
 343         void *vaddr = obj->phys_handle->vaddr + args->offset;
 344         char __user *user_data = to_user_ptr(args->data_ptr);
 345         int ret = 0;
 346
 347         /* We manually control the domain here and pretend that it
 348          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 349          */
 350         ret = i915_gem_object_wait_rendering(obj, false);
 351         if (ret)
 352                 return ret;
 353
 354         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 355         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 356                 unsigned long unwritten;
 357
 358                 /* The physical object once assigned is fixed for the lifetime
 359                  * of the obj, so we can safely drop the lock and continue
 360                  * to access vaddr.
 361                  */
 362                 mutex_unlock(&dev->struct_mutex);
 363                 unwritten = copy_from_user(vaddr, user_data, args->size);
 364                 mutex_lock(&dev->struct_mutex);
 365                 if (unwritten) {
 366                         ret = -EFAULT;
 367                         goto out;
 368                 }
 369         }
 370
 371         drm_clflush_virt_range(vaddr, args->size);
 372         i915_gem_chipset_flush(dev);
 373
 374 out:
 375         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 376         return ret;
 377 }
 378
 379 void *i915_gem_object_alloc(struct drm_device *dev)
 380 {
 381         struct drm_i915_private *dev_priv = dev->dev_private;
 382         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 383 }
 384
 385 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 386 {
 387         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 388         kmem_cache_free(dev_priv->objects, obj);
 389 }
 390
 391 static int
 392 i915_gem_create(struct drm_file *file,
 393                 struct drm_device *dev,
 394                 uint64_t size,
 395                 uint32_t *handle_p)
 396 {
 397         struct drm_i915_gem_object *obj;
 398         int ret;
 399         u32 handle;
 400
 401         size = roundup(size, PAGE_SIZE);
 402         if (size == 0)
 403                 return -EINVAL;
 404
 405         /* Allocate the new object */
 406         obj = i915_gem_alloc_object(dev, size);
 407         if (obj == NULL)
 408                 return -ENOMEM;
 409
 410         ret = drm_gem_handle_create(file, &obj->base, &handle);
 411         /* drop reference from allocate - handle holds it now */
 412         drm_gem_object_unreference_unlocked(&obj->base);
 413         if (ret)
 414                 return ret;
 415
 416         *handle_p = handle;
 417         return 0;
 418 }
 419
 420 int
 421 i915_gem_dumb_create(struct drm_file *file,
 422                      struct drm_device *dev,
 423                      struct drm_mode_create_dumb *args)
 424 {
 425         /* have to work out size/pitch and return them */
 426         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 427         args->size = args->pitch * args->height;
 428         return i915_gem_create(file, dev,
 429                                args->size, &args->handle);
 430 }
 431
 432 /**
 433  * Creates a new mm object and returns a handle to it.
 434  */
 435 int
 436 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 437                       struct drm_file *file)
 438 {
 439         struct drm_i915_gem_create *args = data;
 440
 441         return i915_gem_create(file, dev,
 442                                args->size, &args->handle);
 443 }
 444
 445 static inline int
 446 __copy_to_user_swizzled(char __user *cpu_vaddr,
 447                         const char *gpu_vaddr, int gpu_offset,
 448                         int length)
 449 {
 450         int ret, cpu_offset = 0;
 451
 452         while (length > 0) {
 453                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 454                 int this_length = min(cacheline_end - gpu_offset, length);
 455                 int swizzled_gpu_offset = gpu_offset ^ 64;
 456
 457                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 458                                      gpu_vaddr + swizzled_gpu_offset,
 459                                      this_length);
 460                 if (ret)
 461                         return ret + length;
 462
 463                 cpu_offset += this_length;
 464                 gpu_offset += this_length;
 465                 length -= this_length;
 466         }
 467
 468         return 0;
 469 }
 470
 471 static inline int
 472 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 473                           const char __user *cpu_vaddr,
 474                           int length)
 475 {
 476         int ret, cpu_offset = 0;
 477
 478         while (length > 0) {
 479                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 480                 int this_length = min(cacheline_end - gpu_offset, length);
 481                 int swizzled_gpu_offset = gpu_offset ^ 64;
 482
 483                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 484                                        cpu_vaddr + cpu_offset,
 485                                        this_length);
 486                 if (ret)
 487                         return ret + length;
 488
 489                 cpu_offset += this_length;
 490                 gpu_offset += this_length;
 491                 length -= this_length;
 492         }
 493
 494         return 0;
 495 }
 496
 497 /*
 498  * Pins the specified object's pages and synchronizes the object with
 499  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 500  * flush the object from the CPU cache.
 501  */
 502 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 503                                     int *needs_clflush)
 504 {
 505         int ret;
 506
 507         *needs_clflush = 0;
 508
 509         if (!obj->base.filp)
 510                 return -EINVAL;
 511
 512         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 513                 /* If we're not in the cpu read domain, set ourself into the gtt
 514                  * read domain and manually flush cachelines (if required). This
 515                  * optimizes for the case when the gpu will dirty the data
 516                  * anyway again before the next pread happens. */
 517                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 518                                                         obj->cache_level);
 519                 ret = i915_gem_object_wait_rendering(obj, true);
 520                 if (ret)
 521                         return ret;
 522         }
 523
 524         ret = i915_gem_object_get_pages(obj);
 525         if (ret)
 526                 return ret;
 527
 528         i915_gem_object_pin_pages(obj);
 529
 530         return ret;
 531 }
 532
 533 /* Per-page copy function for the shmem pread fastpath.
 534  * Flushes invalid cachelines before reading the target if
 535  * needs_clflush is set. */
 536 static int
 537 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 538                  char __user *user_data,
 539                  bool page_do_bit17_swizzling, bool needs_clflush)
 540 {
 541         char *vaddr;
 542         int ret;
 543
 544         if (unlikely(page_do_bit17_swizzling))
 545                 return -EINVAL;
 546
 547         vaddr = kmap_atomic(page);
 548         if (needs_clflush)
 549                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 550                                        page_length);
 551         ret = __copy_to_user_inatomic(user_data,
 552                                       vaddr + shmem_page_offset,
 553                                       page_length);
 554         kunmap_atomic(vaddr);
 555
 556         return ret ? -EFAULT : 0;
 557 }
 558
 559 static void
 560 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 561                              bool swizzled)
 562 {
 563         if (unlikely(swizzled)) {
 564                 unsigned long start = (unsigned long) addr;
 565                 unsigned long end = (unsigned long) addr + length;
 566
 567                 /* For swizzling simply ensure that we always flush both
 568                  * channels. Lame, but simple and it works. Swizzled
 569                  * pwrite/pread is far from a hotpath - current userspace
 570                  * doesn't use it at all. */
 571                 start = round_down(start, 128);
 572                 end = round_up(end, 128);
 573
 574                 drm_clflush_virt_range((void *)start, end - start);
 575         } else {
 576                 drm_clflush_virt_range(addr, length);
 577         }
 578
 579 }
 580
 581 /* Only difference to the fast-path function is that this can handle bit17
 582  * and uses non-atomic copy and kmap functions. */
 583 static int
 584 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 585                  char __user *user_data,
 586                  bool page_do_bit17_swizzling, bool needs_clflush)
 587 {
 588         char *vaddr;
 589         int ret;
 590
 591         vaddr = kmap(page);
 592         if (needs_clflush)
 593                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 594                                              page_length,
 595                                              page_do_bit17_swizzling);
 596
 597         if (page_do_bit17_swizzling)
 598                 ret = __copy_to_user_swizzled(user_data,
 599                                               vaddr, shmem_page_offset,
 600                                               page_length);
 601         else
 602                 ret = __copy_to_user(user_data,
 603                                      vaddr + shmem_page_offset,
 604                                      page_length);
 605         kunmap(page);
 606
 607         return ret ? - EFAULT : 0;
 608 }
 609
 610 static int
 611 i915_gem_shmem_pread(struct drm_device *dev,
 612                      struct drm_i915_gem_object *obj,
 613                      struct drm_i915_gem_pread *args,
 614                      struct drm_file *file)
 615 {
 616         char __user *user_data;
 617         ssize_t remain;
 618         loff_t offset;
 619         int shmem_page_offset, page_length, ret = 0;
 620         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 621         int prefaulted = 0;
 622         int needs_clflush = 0;
 623         struct sg_page_iter sg_iter;
 624
 625         user_data = to_user_ptr(args->data_ptr);
 626         remain = args->size;
 627
 628         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 629
 630         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 631         if (ret)
 632                 return ret;
 633
 634         offset = args->offset;
 635
 636         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 637                          offset >> PAGE_SHIFT) {
 638                 struct page *page = sg_page_iter_page(&sg_iter);
 639
 640                 if (remain <= 0)
 641                         break;
 642
 643                 /* Operation in this page
 644                  *
 645                  * shmem_page_offset = offset within page in shmem file
 646                  * page_length = bytes to copy for this page
 647                  */
 648                 shmem_page_offset = offset_in_page(offset);
 649                 page_length = remain;
 650                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 651                         page_length = PAGE_SIZE - shmem_page_offset;
 652
 653                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 654                         (page_to_phys(page) & (1 << 17)) != 0;
 655
 656                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 657                                        user_data, page_do_bit17_swizzling,
 658                                        needs_clflush);
 659                 if (ret == 0)
 660                         goto next_page;
 661
 662                 mutex_unlock(&dev->struct_mutex);
 663
 664                 if (likely(!i915.prefault_disable) && !prefaulted) {
 665                         ret = fault_in_multipages_writeable(user_data, remain);
 666                         /* Userspace is tricking us, but we've already clobbered
 667                          * its pages with the prefault and promised to write the
 668                          * data up to the first fault. Hence ignore any errors
 669                          * and just continue. */
 670                         (void)ret;
 671                         prefaulted = 1;
 672                 }
 673
 674                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 675                                        user_data, page_do_bit17_swizzling,
 676                                        needs_clflush);
 677
 678                 mutex_lock(&dev->struct_mutex);
 679
 680                 if (ret)
 681                         goto out;
 682
 683 next_page:
 684                 remain -= page_length;
 685                 user_data += page_length;
 686                 offset += page_length;
 687         }
 688
 689 out:
 690         i915_gem_object_unpin_pages(obj);
 691
 692         return ret;
 693 }
 694
 695 /**
 696  * Reads data from the object referenced by handle.
 697  *
 698  * On error, the contents of *data are undefined.
 699  */
 700 int
 701 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 702                      struct drm_file *file)
 703 {
 704         struct drm_i915_gem_pread *args = data;
 705         struct drm_i915_gem_object *obj;
 706         int ret = 0;
 707
 708         if (args->size == 0)
 709                 return 0;
 710
 711         if (!access_ok(VERIFY_WRITE,
 712                        to_user_ptr(args->data_ptr),
 713                        args->size))
 714                 return -EFAULT;
 715
 716         ret = i915_mutex_lock_interruptible(dev);
 717         if (ret)
 718                 return ret;
 719
 720         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 721         if (&obj->base == NULL) {
 722                 ret = -ENOENT;
 723                 goto unlock;
 724         }
 725
 726         /* Bounds check source.  */
 727         if (args->offset > obj->base.size ||
 728             args->size > obj->base.size - args->offset) {
 729                 ret = -EINVAL;
 730                 goto out;
 731         }
 732
 733         /* prime objects have no backing filp to GEM pread/pwrite
 734          * pages from.
 735          */
 736         if (!obj->base.filp) {
 737                 ret = -EINVAL;
 738                 goto out;
 739         }
 740
 741         trace_i915_gem_object_pread(obj, args->offset, args->size);
 742
 743         ret = i915_gem_shmem_pread(dev, obj, args, file);
 744
 745 out:
 746         drm_gem_object_unreference(&obj->base);
 747 unlock:
 748         mutex_unlock(&dev->struct_mutex);
 749         return ret;
 750 }
 751
 752 /* This is the fast write path which cannot handle
 753  * page faults in the source data
 754  */
 755
 756 static inline int
 757 fast_user_write(struct io_mapping *mapping,
 758                 loff_t page_base, int page_offset,
 759                 char __user *user_data,
 760                 int length)
 761 {
 762         void __iomem *vaddr_atomic;
 763         void *vaddr;
 764         unsigned long unwritten;
 765
 766         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 767         /* We can use the cpu mem copy function because this is X86. */
 768         vaddr = (void __force*)vaddr_atomic + page_offset;
 769         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 770                                                       user_data, length);
 771         io_mapping_unmap_atomic(vaddr_atomic);
 772         return unwritten;
 773 }
 774
 775 /**
 776  * This is the fast pwrite path, where we copy the data directly from the
 777  * user into the GTT, uncached.
 778  */
 779 static int
 780 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 781                          struct drm_i915_gem_object *obj,
 782                          struct drm_i915_gem_pwrite *args,
 783                          struct drm_file *file)
 784 {
 785         struct drm_i915_private *dev_priv = dev->dev_private;
 786         ssize_t remain;
 787         loff_t offset, page_base;
 788         char __user *user_data;
 789         int page_offset, page_length, ret;
 790
 791         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 792         if (ret)
 793                 goto out;
 794
 795         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 796         if (ret)
 797                 goto out_unpin;
 798
 799         ret = i915_gem_object_put_fence(obj);
 800         if (ret)
 801                 goto out_unpin;
 802
 803         user_data = to_user_ptr(args->data_ptr);
 804         remain = args->size;
 805
 806         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
 807
 808         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
 809
 810         while (remain > 0) {
 811                 /* Operation in this page
 812                  *
 813                  * page_base = page offset within aperture
 814                  * page_offset = offset within page
 815                  * page_length = bytes to copy for this page
 816                  */
 817                 page_base = offset & PAGE_MASK;
 818                 page_offset = offset_in_page(offset);
 819                 page_length = remain;
 820                 if ((page_offset + remain) > PAGE_SIZE)
 821                         page_length = PAGE_SIZE - page_offset;
 822
 823                 /* If we get a fault while copying data, then (presumably) our
 824                  * source page isn't available.  Return the error and we'll
 825                  * retry in the slow path.
 826                  */
 827                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
 828                                     page_offset, user_data, page_length)) {
 829                         ret = -EFAULT;
 830                         goto out_flush;
 831                 }
 832
 833                 remain -= page_length;
 834                 user_data += page_length;
 835                 offset += page_length;
 836         }
 837
 838 out_flush:
 839         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
 840 out_unpin:
 841         i915_gem_object_ggtt_unpin(obj);
 842 out:
 843         return ret;
 844 }
 845
 846 /* Per-page copy function for the shmem pwrite fastpath.
 847  * Flushes invalid cachelines before writing to the target if
 848  * needs_clflush_before is set and flushes out any written cachelines after
 849  * writing if needs_clflush is set. */
 850 static int
 851 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 852                   char __user *user_data,
 853                   bool page_do_bit17_swizzling,
 854                   bool needs_clflush_before,
 855                   bool needs_clflush_after)
 856 {
 857         char *vaddr;
 858         int ret;
 859
 860         if (unlikely(page_do_bit17_swizzling))
 861                 return -EINVAL;
 862
 863         vaddr = kmap_atomic(page);
 864         if (needs_clflush_before)
 865                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 866                                        page_length);
 867         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
 868                                         user_data, page_length);
 869         if (needs_clflush_after)
 870                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 871                                        page_length);
 872         kunmap_atomic(vaddr);
 873
 874         return ret ? -EFAULT : 0;
 875 }
 876
 877 /* Only difference to the fast-path function is that this can handle bit17
 878  * and uses non-atomic copy and kmap functions. */
 879 static int
 880 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 881                   char __user *user_data,
 882                   bool page_do_bit17_swizzling,
 883                   bool needs_clflush_before,
 884                   bool needs_clflush_after)
 885 {
 886         char *vaddr;
 887         int ret;
 888
 889         vaddr = kmap(page);
 890         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 891                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 892                                              page_length,
 893                                              page_do_bit17_swizzling);
 894         if (page_do_bit17_swizzling)
 895                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 896                                                 user_data,
 897                                                 page_length);
 898         else
 899                 ret = __copy_from_user(vaddr + shmem_page_offset,
 900                                        user_data,
 901                                        page_length);
 902         if (needs_clflush_after)
 903                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 904                                              page_length,
 905                                              page_do_bit17_swizzling);
 906         kunmap(page);
 907
 908         return ret ? -EFAULT : 0;
 909 }
 910
 911 static int
 912 i915_gem_shmem_pwrite(struct drm_device *dev,
 913                       struct drm_i915_gem_object *obj,
 914                       struct drm_i915_gem_pwrite *args,
 915                       struct drm_file *file)
 916 {
 917         ssize_t remain;
 918         loff_t offset;
 919         char __user *user_data;
 920         int shmem_page_offset, page_length, ret = 0;
 921         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 922         int hit_slowpath = 0;
 923         int needs_clflush_after = 0;
 924         int needs_clflush_before = 0;
 925         struct sg_page_iter sg_iter;
 926
 927         user_data = to_user_ptr(args->data_ptr);
 928         remain = args->size;
 929
 930         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 931
 932         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 933                 /* If we're not in the cpu write domain, set ourself into the gtt
 934                  * write domain and manually flush cachelines (if required). This
 935                  * optimizes for the case when the gpu will use the data
 936                  * right away and we therefore have to clflush anyway. */
 937                 needs_clflush_after = cpu_write_needs_clflush(obj);
 938                 ret = i915_gem_object_wait_rendering(obj, false);
 939                 if (ret)
 940                         return ret;
 941         }
 942         /* Same trick applies to invalidate partially written cachelines read
 943          * before writing. */
 944         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
 945                 needs_clflush_before =
 946                         !cpu_cache_is_coherent(dev, obj->cache_level);
 947
 948         ret = i915_gem_object_get_pages(obj);
 949         if (ret)
 950                 return ret;
 951
 952         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 953
 954         i915_gem_object_pin_pages(obj);
 955
 956         offset = args->offset;
 957         obj->dirty = 1;
 958
 959         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 960                          offset >> PAGE_SHIFT) {
 961                 struct page *page = sg_page_iter_page(&sg_iter);
 962                 int partial_cacheline_write;
 963
 964                 if (remain <= 0)
 965                         break;
 966
 967                 /* Operation in this page
 968                  *
 969                  * shmem_page_offset = offset within page in shmem file
 970                  * page_length = bytes to copy for this page
 971                  */
 972                 shmem_page_offset = offset_in_page(offset);
 973
 974                 page_length = remain;
 975                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 976                         page_length = PAGE_SIZE - shmem_page_offset;
 977
 978                 /* If we don't overwrite a cacheline completely we need to be
 979                  * careful to have up-to-date data by first clflushing. Don't
 980                  * overcomplicate things and flush the entire patch. */
 981                 partial_cacheline_write = needs_clflush_before &&
 982                         ((shmem_page_offset | page_length)
 983                                 & (boot_cpu_data.x86_clflush_size - 1));
 984
 985                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 986                         (page_to_phys(page) & (1 << 17)) != 0;
 987
 988                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 989                                         user_data, page_do_bit17_swizzling,
 990                                         partial_cacheline_write,
 991                                         needs_clflush_after);
 992                 if (ret == 0)
 993                         goto next_page;
 994
 995                 hit_slowpath = 1;
 996                 mutex_unlock(&dev->struct_mutex);
 997                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 998                                         user_data, page_do_bit17_swizzling,
 999                                         partial_cacheline_write,
1000                                         needs_clflush_after);
1001
1002                 mutex_lock(&dev->struct_mutex);
1003
1004                 if (ret)
1005                         goto out;
1006
1007 next_page:
1008                 remain -= page_length;
1009                 user_data += page_length;
1010                 offset += page_length;
1011         }
1012
1013 out:
1014         i915_gem_object_unpin_pages(obj);
1015
1016         if (hit_slowpath) {
1017                 /*
1018                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1019                  * cachelines in-line while writing and the object moved
1020                  * out of the cpu write domain while we've dropped the lock.
1021                  */
1022                 if (!needs_clflush_after &&
1023                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1024                         if (i915_gem_clflush_object(obj, obj->pin_display))
1025                                 i915_gem_chipset_flush(dev);
1026                 }
1027         }
1028
1029         if (needs_clflush_after)
1030                 i915_gem_chipset_flush(dev);
1031
1032         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1033         return ret;
1034 }
1035
1036 /**
1037  * Writes data to the object referenced by handle.
1038  *
1039  * On error, the contents of the buffer that were to be modified are undefined.
1040  */
1041 int
1042 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1043                       struct drm_file *file)
1044 {
1045         struct drm_i915_private *dev_priv = dev->dev_private;
1046         struct drm_i915_gem_pwrite *args = data;
1047         struct drm_i915_gem_object *obj;
1048         int ret;
1049
1050         if (args->size == 0)
1051                 return 0;
1052
1053         if (!access_ok(VERIFY_READ,
1054                        to_user_ptr(args->data_ptr),
1055                        args->size))
1056                 return -EFAULT;
1057
1058         if (likely(!i915.prefault_disable)) {
1059                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1060                                                    args->size);
1061                 if (ret)
1062                         return -EFAULT;
1063         }
1064
1065         intel_runtime_pm_get(dev_priv);
1066
1067         ret = i915_mutex_lock_interruptible(dev);
1068         if (ret)
1069                 goto put_rpm;
1070
1071         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1072         if (&obj->base == NULL) {
1073                 ret = -ENOENT;
1074                 goto unlock;
1075         }
1076
1077         /* Bounds check destination. */
1078         if (args->offset > obj->base.size ||
1079             args->size > obj->base.size - args->offset) {
1080                 ret = -EINVAL;
1081                 goto out;
1082         }
1083
1084         /* prime objects have no backing filp to GEM pread/pwrite
1085          * pages from.
1086          */
1087         if (!obj->base.filp) {
1088                 ret = -EINVAL;
1089                 goto out;
1090         }
1091
1092         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1093
1094         ret = -EFAULT;
1095         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1096          * it would end up going through the fenced access, and we'll get
1097          * different detiling behavior between reading and writing.
1098          * pread/pwrite currently are reading and writing from the CPU
1099          * perspective, requiring manual detiling by the client.
1100          */
1101         if (obj->tiling_mode == I915_TILING_NONE &&
1102             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1103             cpu_write_needs_clflush(obj)) {
1104                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1105                 /* Note that the gtt paths might fail with non-page-backed user
1106                  * pointers (e.g. gtt mappings when moving data between
1107                  * textures). Fallback to the shmem path in that case. */
1108         }
1109
1110         if (ret == -EFAULT || ret == -ENOSPC) {
1111                 if (obj->phys_handle)
1112                         ret = i915_gem_phys_pwrite(obj, args, file);
1113                 else
1114                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1115         }
1116
1117 out:
1118         drm_gem_object_unreference(&obj->base);
1119 unlock:
1120         mutex_unlock(&dev->struct_mutex);
1121 put_rpm:
1122         intel_runtime_pm_put(dev_priv);
1123
1124         return ret;
1125 }
1126
1127 int
1128 i915_gem_check_wedge(struct i915_gpu_error *error,
1129                      bool interruptible)
1130 {
1131         if (i915_reset_in_progress(error)) {
1132                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1133                  * -EIO unconditionally for these. */
1134                 if (!interruptible)
1135                         return -EIO;
1136
1137                 /* Recovery complete, but the reset failed ... */
1138                 if (i915_terminally_wedged(error))
1139                         return -EIO;
1140
1141                 /*
1142                  * Check if GPU Reset is in progress - we need intel_ring_begin
1143                  * to work properly to reinit the hw state while the gpu is
1144                  * still marked as reset-in-progress. Handle this with a flag.
1145                  */
1146                 if (!error->reload_in_reset)
1147                         return -EAGAIN;
1148         }
1149
1150         return 0;
1151 }
1152
1153 static void fake_irq(unsigned long data)
1154 {
1155         wake_up_process((struct task_struct *)data);
1156 }
1157
1158 static bool missed_irq(struct drm_i915_private *dev_priv,
1159                        struct intel_engine_cs *ring)
1160 {
1161         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1162 }
1163
1164 static int __i915_spin_request(struct drm_i915_gem_request *req)
1165 {
1166         unsigned long timeout;
1167
1168         if (i915_gem_request_get_ring(req)->irq_refcount)
1169                 return -EBUSY;
1170
1171         timeout = jiffies + 1;
1172         while (!need_resched()) {
1173                 if (i915_gem_request_completed(req, true))
1174                         return 0;
1175
1176                 if (time_after_eq(jiffies, timeout))
1177                         break;
1178
1179                 cpu_relax_lowlatency();
1180         }
1181         if (i915_gem_request_completed(req, false))
1182                 return 0;
1183
1184         return -EAGAIN;
1185 }
1186
1187 /**
1188  * __i915_wait_request - wait until execution of request has finished
1189  * @req: duh!
1190  * @reset_counter: reset sequence associated with the given request
1191  * @interruptible: do an interruptible wait (normally yes)
1192  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1193  *
1194  * Note: It is of utmost importance that the passed in seqno and reset_counter
1195  * values have been read by the caller in an smp safe manner. Where read-side
1196  * locks are involved, it is sufficient to read the reset_counter before
1197  * unlocking the lock that protects the seqno. For lockless tricks, the
1198  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1199  * inserted.
1200  *
1201  * Returns 0 if the request was found within the alloted time. Else returns the
1202  * errno with remaining time filled in timeout argument.
1203  */
1204 int __i915_wait_request(struct drm_i915_gem_request *req,
1205                         unsigned reset_counter,
1206                         bool interruptible,
1207                         s64 *timeout,
1208                         struct intel_rps_client *rps)
1209 {
1210         struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1211         struct drm_device *dev = ring->dev;
1212         struct drm_i915_private *dev_priv = dev->dev_private;
1213         const bool irq_test_in_progress =
1214                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1215         DEFINE_WAIT(wait);
1216         unsigned long timeout_expire;
1217         s64 before, now;
1218         int ret;
1219
1220         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1221
1222         if (list_empty(&req->list))
1223                 return 0;
1224
1225         if (i915_gem_request_completed(req, true))
1226                 return 0;
1227
1228         timeout_expire = timeout ?
1229                 jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
1230
1231         if (INTEL_INFO(dev_priv)->gen >= 6)
1232                 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
1233
1234         /* Record current time in case interrupted by signal, or wedged */
1235         trace_i915_gem_request_wait_begin(req);
1236         before = ktime_get_raw_ns();
1237
1238         /* Optimistic spin for the next jiffie before touching IRQs */
1239         ret = __i915_spin_request(req);
1240         if (ret == 0)
1241                 goto out;
1242
1243         if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
1244                 ret = -ENODEV;
1245                 goto out;
1246         }
1247
1248         for (;;) {
1249                 struct timer_list timer;
1250
1251                 prepare_to_wait(&ring->irq_queue, &wait,
1252                                 interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1253
1254                 /* We need to check whether any gpu reset happened in between
1255                  * the caller grabbing the seqno and now ... */
1256                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1257                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1258                          * is truely gone. */
1259                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1260                         if (ret == 0)
1261                                 ret = -EAGAIN;
1262                         break;
1263                 }
1264
1265                 if (i915_gem_request_completed(req, false)) {
1266                         ret = 0;
1267                         break;
1268                 }
1269
1270                 if (interruptible && signal_pending(current)) {
1271                         ret = -ERESTARTSYS;
1272                         break;
1273                 }
1274
1275                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1276                         ret = -ETIME;
1277                         break;
1278                 }
1279
1280                 timer.function = NULL;
1281                 if (timeout || missed_irq(dev_priv, ring)) {
1282                         unsigned long expire;
1283
1284                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1285                         expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1286                         mod_timer(&timer, expire);
1287                 }
1288
1289                 io_schedule();
1290
1291                 if (timer.function) {
1292                         del_singleshot_timer_sync(&timer);
1293                         destroy_timer_on_stack(&timer);
1294                 }
1295         }
1296         if (!irq_test_in_progress)
1297                 ring->irq_put(ring);
1298
1299         finish_wait(&ring->irq_queue, &wait);
1300
1301 out:
1302         now = ktime_get_raw_ns();
1303         trace_i915_gem_request_wait_end(req);
1304
1305         if (timeout) {
1306                 s64 tres = *timeout - (now - before);
1307
1308                 *timeout = tres < 0 ? 0 : tres;
1309
1310                 /*
1311                  * Apparently ktime isn't accurate enough and occasionally has a
1312                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1313                  * things up to make the test happy. We allow up to 1 jiffy.
1314                  *
1315                  * This is a regrssion from the timespec->ktime conversion.
1316                  */
1317                 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1318                         *timeout = 0;
1319         }
1320
1321         return ret;
1322 }
1323
1324 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
1325                                    struct drm_file *file)
1326 {
1327         struct drm_i915_private *dev_private;
1328         struct drm_i915_file_private *file_priv;
1329
1330         WARN_ON(!req || !file || req->file_priv);
1331
1332         if (!req || !file)
1333                 return -EINVAL;
1334
1335         if (req->file_priv)
1336                 return -EINVAL;
1337
1338         dev_private = req->ring->dev->dev_private;
1339         file_priv = file->driver_priv;
1340
1341         spin_lock(&file_priv->mm.lock);
1342         req->file_priv = file_priv;
1343         list_add_tail(&req->client_list, &file_priv->mm.request_list);
1344         spin_unlock(&file_priv->mm.lock);
1345
1346         req->pid = get_pid(task_pid(current));
1347
1348         return 0;
1349 }
1350
1351 static inline void
1352 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1353 {
1354         struct drm_i915_file_private *file_priv = request->file_priv;
1355
1356         if (!file_priv)
1357                 return;
1358
1359         spin_lock(&file_priv->mm.lock);
1360         list_del(&request->client_list);
1361         request->file_priv = NULL;
1362         spin_unlock(&file_priv->mm.lock);
1363
1364         put_pid(request->pid);
1365         request->pid = NULL;
1366 }
1367
1368 static void i915_gem_request_retire(struct drm_i915_gem_request *request)
1369 {
1370         trace_i915_gem_request_retire(request);
1371
1372         /* We know the GPU must have read the request to have
1373          * sent us the seqno + interrupt, so use the position
1374          * of tail of the request to update the last known position
1375          * of the GPU head.
1376          *
1377          * Note this requires that we are always called in request
1378          * completion order.
1379          */
1380         request->ringbuf->last_retired_head = request->postfix;
1381
1382         list_del_init(&request->list);
1383         i915_gem_request_remove_from_client(request);
1384
1385         i915_gem_request_unreference(request);
1386 }
1387
1388 static void
1389 __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
1390 {
1391         struct intel_engine_cs *engine = req->ring;
1392         struct drm_i915_gem_request *tmp;
1393
1394         lockdep_assert_held(&engine->dev->struct_mutex);
1395
1396         if (list_empty(&req->list))
1397                 return;
1398
1399         do {
1400                 tmp = list_first_entry(&engine->request_list,
1401                                        typeof(*tmp), list);
1402
1403                 i915_gem_request_retire(tmp);
1404         } while (tmp != req);
1405
1406         WARN_ON(i915_verify_lists(engine->dev));
1407 }
1408
1409 /**
1410  * Waits for a request to be signaled, and cleans up the
1411  * request and object lists appropriately for that event.
1412  */
1413 int
1414 i915_wait_request(struct drm_i915_gem_request *req)
1415 {
1416         struct drm_device *dev;
1417         struct drm_i915_private *dev_priv;
1418         bool interruptible;
1419         int ret;
1420
1421         BUG_ON(req == NULL);
1422
1423         dev = req->ring->dev;
1424         dev_priv = dev->dev_private;
1425         interruptible = dev_priv->mm.interruptible;
1426
1427         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1428
1429         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1430         if (ret)
1431                 return ret;
1432
1433         ret = __i915_wait_request(req,
1434                                   atomic_read(&dev_priv->gpu_error.reset_counter),
1435                                   interruptible, NULL, NULL);
1436         if (ret)
1437                 return ret;
1438
1439         __i915_gem_request_retire__upto(req);
1440         return 0;
1441 }
1442
1443 /**
1444  * Ensures that all rendering to the object has completed and the object is
1445  * safe to unbind from the GTT or access from the CPU.
1446  */
1447 int
1448 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1449                                bool readonly)
1450 {
1451         int ret, i;
1452
1453         if (!obj->active)
1454                 return 0;
1455
1456         if (readonly) {
1457                 if (obj->last_write_req != NULL) {
1458                         ret = i915_wait_request(obj->last_write_req);
1459                         if (ret)
1460                                 return ret;
1461
1462                         i = obj->last_write_req->ring->id;
1463                         if (obj->last_read_req[i] == obj->last_write_req)
1464                                 i915_gem_object_retire__read(obj, i);
1465                         else
1466                                 i915_gem_object_retire__write(obj);
1467                 }
1468         } else {
1469                 for (i = 0; i < I915_NUM_RINGS; i++) {
1470                         if (obj->last_read_req[i] == NULL)
1471                                 continue;
1472
1473                         ret = i915_wait_request(obj->last_read_req[i]);
1474                         if (ret)
1475                                 return ret;
1476
1477                         i915_gem_object_retire__read(obj, i);
1478                 }
1479                 RQ_BUG_ON(obj->active);
1480         }
1481
1482         return 0;
1483 }
1484
1485 static void
1486 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1487                                struct drm_i915_gem_request *req)
1488 {
1489         int ring = req->ring->id;
1490
1491         if (obj->last_read_req[ring] == req)
1492                 i915_gem_object_retire__read(obj, ring);
1493         else if (obj->last_write_req == req)
1494                 i915_gem_object_retire__write(obj);
1495
1496         __i915_gem_request_retire__upto(req);
1497 }
1498
1499 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1500  * as the object state may change during this call.
1501  */
1502 static __must_check int
1503 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1504                                             struct intel_rps_client *rps,
1505                                             bool readonly)
1506 {
1507         struct drm_device *dev = obj->base.dev;
1508         struct drm_i915_private *dev_priv = dev->dev_private;
1509         struct drm_i915_gem_request *requests[I915_NUM_RINGS];
1510         unsigned reset_counter;
1511         int ret, i, n = 0;
1512
1513         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1514         BUG_ON(!dev_priv->mm.interruptible);
1515
1516         if (!obj->active)
1517                 return 0;
1518
1519         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1520         if (ret)
1521                 return ret;
1522
1523         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1524
1525         if (readonly) {
1526                 struct drm_i915_gem_request *req;
1527
1528                 req = obj->last_write_req;
1529                 if (req == NULL)
1530                         return 0;
1531
1532                 requests[n++] = i915_gem_request_reference(req);
1533         } else {
1534                 for (i = 0; i < I915_NUM_RINGS; i++) {
1535                         struct drm_i915_gem_request *req;
1536
1537                         req = obj->last_read_req[i];
1538                         if (req == NULL)
1539                                 continue;
1540
1541                         requests[n++] = i915_gem_request_reference(req);
1542                 }
1543         }
1544
1545         mutex_unlock(&dev->struct_mutex);
1546         for (i = 0; ret == 0 && i < n; i++)
1547                 ret = __i915_wait_request(requests[i], reset_counter, true,
1548                                           NULL, rps);
1549         mutex_lock(&dev->struct_mutex);
1550
1551         for (i = 0; i < n; i++) {
1552                 if (ret == 0)
1553                         i915_gem_object_retire_request(obj, requests[i]);
1554                 i915_gem_request_unreference(requests[i]);
1555         }
1556
1557         return ret;
1558 }
1559
1560 static struct intel_rps_client *to_rps_client(struct drm_file *file)
1561 {
1562         struct drm_i915_file_private *fpriv = file->driver_priv;
1563         return &fpriv->rps;
1564 }
1565
1566 /**
1567  * Called when user space prepares to use an object with the CPU, either
1568  * through the mmap ioctl's mapping or a GTT mapping.
1569  */
1570 int
1571 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1572                           struct drm_file *file)
1573 {
1574         struct drm_i915_gem_set_domain *args = data;
1575         struct drm_i915_gem_object *obj;
1576         uint32_t read_domains = args->read_domains;
1577         uint32_t write_domain = args->write_domain;
1578         int ret;
1579
1580         /* Only handle setting domains to types used by the CPU. */
1581         if (write_domain & I915_GEM_GPU_DOMAINS)
1582                 return -EINVAL;
1583
1584         if (read_domains & I915_GEM_GPU_DOMAINS)
1585                 return -EINVAL;
1586
1587         /* Having something in the write domain implies it's in the read
1588          * domain, and only that read domain.  Enforce that in the request.
1589          */
1590         if (write_domain != 0 && read_domains != write_domain)
1591                 return -EINVAL;
1592
1593         ret = i915_mutex_lock_interruptible(dev);
1594         if (ret)
1595                 return ret;
1596
1597         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1598         if (&obj->base == NULL) {
1599                 ret = -ENOENT;
1600                 goto unlock;
1601         }
1602
1603         /* Try to flush the object off the GPU without holding the lock.
1604          * We will repeat the flush holding the lock in the normal manner
1605          * to catch cases where we are gazumped.
1606          */
1607         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1608                                                           to_rps_client(file),
1609                                                           !write_domain);
1610         if (ret)
1611                 goto unref;
1612
1613         if (read_domains & I915_GEM_DOMAIN_GTT)
1614                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1615         else
1616                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1617
1618         if (write_domain != 0)
1619                 intel_fb_obj_invalidate(obj,
1620                                         write_domain == I915_GEM_DOMAIN_GTT ?
1621                                         ORIGIN_GTT : ORIGIN_CPU);
1622
1623 unref:
1624         drm_gem_object_unreference(&obj->base);
1625 unlock:
1626         mutex_unlock(&dev->struct_mutex);
1627         return ret;
1628 }
1629
1630 /**
1631  * Called when user space has done writes to this buffer
1632  */
1633 int
1634 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1635                          struct drm_file *file)
1636 {
1637         struct drm_i915_gem_sw_finish *args = data;
1638         struct drm_i915_gem_object *obj;
1639         int ret = 0;
1640
1641         ret = i915_mutex_lock_interruptible(dev);
1642         if (ret)
1643                 return ret;
1644
1645         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1646         if (&obj->base == NULL) {
1647                 ret = -ENOENT;
1648                 goto unlock;
1649         }
1650
1651         /* Pinned buffers may be scanout, so flush the cache */
1652         if (obj->pin_display)
1653                 i915_gem_object_flush_cpu_write_domain(obj);
1654
1655         drm_gem_object_unreference(&obj->base);
1656 unlock:
1657         mutex_unlock(&dev->struct_mutex);
1658         return ret;
1659 }
1660
1661 /**
1662  * Maps the contents of an object, returning the address it is mapped
1663  * into.
1664  *
1665  * While the mapping holds a reference on the contents of the object, it doesn't
1666  * imply a ref on the object itself.
1667  *
1668  * IMPORTANT:
1669  *
1670  * DRM driver writers who look a this function as an example for how to do GEM
1671  * mmap support, please don't implement mmap support like here. The modern way
1672  * to implement DRM mmap support is with an mmap offset ioctl (like
1673  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1674  * That way debug tooling like valgrind will understand what's going on, hiding
1675  * the mmap call in a driver private ioctl will break that. The i915 driver only
1676  * does cpu mmaps this way because we didn't know better.
1677  */
1678 int
1679 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1680                     struct drm_file *file)
1681 {
1682         struct drm_i915_gem_mmap *args = data;
1683         struct drm_gem_object *obj;
1684         unsigned long addr;
1685
1686         if (args->flags & ~(I915_MMAP_WC))
1687                 return -EINVAL;
1688
1689         if (args->flags & I915_MMAP_WC && !cpu_has_pat)
1690                 return -ENODEV;
1691
1692         obj = drm_gem_object_lookup(dev, file, args->handle);
1693         if (obj == NULL)
1694                 return -ENOENT;
1695
1696         /* prime objects have no backing filp to GEM mmap
1697          * pages from.
1698          */
1699         if (!obj->filp) {
1700                 drm_gem_object_unreference_unlocked(obj);
1701                 return -EINVAL;
1702         }
1703
1704         addr = vm_mmap(obj->filp, 0, args->size,
1705                        PROT_READ | PROT_WRITE, MAP_SHARED,
1706                        args->offset);
1707         if (args->flags & I915_MMAP_WC) {
1708                 struct mm_struct *mm = current->mm;
1709                 struct vm_area_struct *vma;
1710
1711                 down_write(&mm->mmap_sem);
1712                 vma = find_vma(mm, addr);
1713                 if (vma)
1714                         vma->vm_page_prot =
1715                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1716                 else
1717                         addr = -ENOMEM;
1718                 up_write(&mm->mmap_sem);
1719         }
1720         drm_gem_object_unreference_unlocked(obj);
1721         if (IS_ERR((void *)addr))
1722                 return addr;
1723
1724         args->addr_ptr = (uint64_t) addr;
1725
1726         return 0;
1727 }
1728
1729 /**
1730  * i915_gem_fault - fault a page into the GTT
1731  * vma: VMA in question
1732  * vmf: fault info
1733  *
1734  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1735  * from userspace.  The fault handler takes care of binding the object to
1736  * the GTT (if needed), allocating and programming a fence register (again,
1737  * only if needed based on whether the old reg is still valid or the object
1738  * is tiled) and inserting a new PTE into the faulting process.
1739  *
1740  * Note that the faulting process may involve evicting existing objects
1741  * from the GTT and/or fence registers to make room.  So performance may
1742  * suffer if the GTT working set is large or there are few fence registers
1743  * left.
1744  */
1745 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1746 {
1747         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1748         struct drm_device *dev = obj->base.dev;
1749         struct drm_i915_private *dev_priv = dev->dev_private;
1750         struct i915_ggtt_view view = i915_ggtt_view_normal;
1751         pgoff_t page_offset;
1752         unsigned long pfn;
1753         int ret = 0;
1754         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1755
1756         intel_runtime_pm_get(dev_priv);
1757
1758         /* We don't use vmf->pgoff since that has the fake offset */
1759         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1760                 PAGE_SHIFT;
1761
1762         ret = i915_mutex_lock_interruptible(dev);
1763         if (ret)
1764                 goto out;
1765
1766         trace_i915_gem_object_fault(obj, page_offset, true, write);
1767
1768         /* Try to flush the object off the GPU first without holding the lock.
1769          * Upon reacquiring the lock, we will perform our sanity checks and then
1770          * repeat the flush holding the lock in the normal manner to catch cases
1771          * where we are gazumped.
1772          */
1773         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1774         if (ret)
1775                 goto unlock;
1776
1777         /* Access to snoopable pages through the GTT is incoherent. */
1778         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1779                 ret = -EFAULT;
1780                 goto unlock;
1781         }
1782
1783         /* Use a partial view if the object is bigger than the aperture. */
1784         if (obj->base.size >= dev_priv->gtt.mappable_end &&
1785             obj->tiling_mode == I915_TILING_NONE) {
1786                 static const unsigned int chunk_size = 256; // 1 MiB
1787
1788                 memset(&view, 0, sizeof(view));
1789                 view.type = I915_GGTT_VIEW_PARTIAL;
1790                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1791                 view.params.partial.size =
1792                         min_t(unsigned int,
1793                               chunk_size,
1794                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1795                               view.params.partial.offset);
1796         }
1797
1798         /* Now pin it into the GTT if needed */
1799         ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1800         if (ret)
1801                 goto unlock;
1802
1803         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1804         if (ret)
1805                 goto unpin;
1806
1807         ret = i915_gem_object_get_fence(obj);
1808         if (ret)
1809                 goto unpin;
1810
1811         /* Finally, remap it using the new GTT offset */
1812         pfn = dev_priv->gtt.mappable_base +
1813                 i915_gem_obj_ggtt_offset_view(obj, &view);
1814         pfn >>= PAGE_SHIFT;
1815
1816         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1817                 /* Overriding existing pages in partial view does not cause
1818                  * us any trouble as TLBs are still valid because the fault
1819                  * is due to userspace losing part of the mapping or never
1820                  * having accessed it before (at this partials' range).
1821                  */
1822                 unsigned long base = vma->vm_start +
1823                                      (view.params.partial.offset << PAGE_SHIFT);
1824                 unsigned int i;
1825
1826                 for (i = 0; i < view.params.partial.size; i++) {
1827                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1828                         if (ret)
1829                                 break;
1830                 }
1831
1832                 obj->fault_mappable = true;
1833         } else {
1834                 if (!obj->fault_mappable) {
1835                         unsigned long size = min_t(unsigned long,
1836                                                    vma->vm_end - vma->vm_start,
1837                                                    obj->base.size);
1838                         int i;
1839
1840                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
1841                                 ret = vm_insert_pfn(vma,
1842                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
1843                                                     pfn + i);
1844                                 if (ret)
1845                                         break;
1846                         }
1847
1848                         obj->fault_mappable = true;
1849                 } else
1850                         ret = vm_insert_pfn(vma,
1851                                             (unsigned long)vmf->virtual_address,
1852                                             pfn + page_offset);
1853         }
1854 unpin:
1855         i915_gem_object_ggtt_unpin_view(obj, &view);
1856 unlock:
1857         mutex_unlock(&dev->struct_mutex);
1858 out:
1859         switch (ret) {
1860         case -EIO:
1861                 /*
1862                  * We eat errors when the gpu is terminally wedged to avoid
1863                  * userspace unduly crashing (gl has no provisions for mmaps to
1864                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1865                  * and so needs to be reported.
1866                  */
1867                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1868                         ret = VM_FAULT_SIGBUS;
1869                         break;
1870                 }
1871         case -EAGAIN:
1872                 /*
1873                  * EAGAIN means the gpu is hung and we'll wait for the error
1874                  * handler to reset everything when re-faulting in
1875                  * i915_mutex_lock_interruptible.
1876                  */
1877         case 0:
1878         case -ERESTARTSYS:
1879         case -EINTR:
1880         case -EBUSY:
1881                 /*
1882                  * EBUSY is ok: this just means that another thread
1883                  * already did the job.
1884                  */
1885                 ret = VM_FAULT_NOPAGE;
1886                 break;
1887         case -ENOMEM:
1888                 ret = VM_FAULT_OOM;
1889                 break;
1890         case -ENOSPC:
1891         case -EFAULT:
1892                 ret = VM_FAULT_SIGBUS;
1893                 break;
1894         default:
1895                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1896                 ret = VM_FAULT_SIGBUS;
1897                 break;
1898         }
1899
1900         intel_runtime_pm_put(dev_priv);
1901         return ret;
1902 }
1903
1904 /**
1905  * i915_gem_release_mmap - remove physical page mappings
1906  * @obj: obj in question
1907  *
1908  * Preserve the reservation of the mmapping with the DRM core code, but
1909  * relinquish ownership of the pages back to the system.
1910  *
1911  * It is vital that we remove the page mapping if we have mapped a tiled
1912  * object through the GTT and then lose the fence register due to
1913  * resource pressure. Similarly if the object has been moved out of the
1914  * aperture, than pages mapped into userspace must be revoked. Removing the
1915  * mapping will then trigger a page fault on the next user access, allowing
1916  * fixup by i915_gem_fault().
1917  */
1918 void
1919 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1920 {
1921         if (!obj->fault_mappable)
1922                 return;
1923
1924         drm_vma_node_unmap(&obj->base.vma_node,
1925                            obj->base.dev->anon_inode->i_mapping);
1926         obj->fault_mappable = false;
1927 }
1928
1929 void
1930 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1931 {
1932         struct drm_i915_gem_object *obj;
1933
1934         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1935                 i915_gem_release_mmap(obj);
1936 }
1937
1938 uint32_t
1939 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1940 {
1941         uint32_t gtt_size;
1942
1943         if (INTEL_INFO(dev)->gen >= 4 ||
1944             tiling_mode == I915_TILING_NONE)
1945                 return size;
1946
1947         /* Previous chips need a power-of-two fence region when tiling */
1948         if (INTEL_INFO(dev)->gen == 3)
1949                 gtt_size = 1024*1024;
1950         else
1951                 gtt_size = 512*1024;
1952
1953         while (gtt_size < size)
1954                 gtt_size <<= 1;
1955
1956         return gtt_size;
1957 }
1958
1959 /**
1960  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1961  * @obj: object to check
1962  *
1963  * Return the required GTT alignment for an object, taking into account
1964  * potential fence register mapping.
1965  */
1966 uint32_t
1967 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1968                            int tiling_mode, bool fenced)
1969 {
1970         /*
1971          * Minimum alignment is 4k (GTT page size), but might be greater
1972          * if a fence register is needed for the object.
1973          */
1974         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1975             tiling_mode == I915_TILING_NONE)
1976                 return 4096;
1977
1978         /*
1979          * Previous chips need to be aligned to the size of the smallest
1980          * fence register that can contain the object.
1981          */
1982         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1983 }
1984
1985 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1986 {
1987         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1988         int ret;
1989
1990         if (drm_vma_node_has_offset(&obj->base.vma_node))
1991                 return 0;
1992
1993         dev_priv->mm.shrinker_no_lock_stealing = true;
1994
1995         ret = drm_gem_create_mmap_offset(&obj->base);
1996         if (ret != -ENOSPC)
1997                 goto out;
1998
1999         /* Badly fragmented mmap space? The only way we can recover
2000          * space is by destroying unwanted objects. We can't randomly release
2001          * mmap_offsets as userspace expects them to be persistent for the
2002          * lifetime of the objects. The closest we can is to release the
2003          * offsets on purgeable objects by truncating it and marking it purged,
2004          * which prevents userspace from ever using that object again.
2005          */
2006         i915_gem_shrink(dev_priv,
2007                         obj->base.size >> PAGE_SHIFT,
2008                         I915_SHRINK_BOUND |
2009                         I915_SHRINK_UNBOUND |
2010                         I915_SHRINK_PURGEABLE);
2011         ret = drm_gem_create_mmap_offset(&obj->base);
2012         if (ret != -ENOSPC)
2013                 goto out;
2014
2015         i915_gem_shrink_all(dev_priv);
2016         ret = drm_gem_create_mmap_offset(&obj->base);
2017 out:
2018         dev_priv->mm.shrinker_no_lock_stealing = false;
2019
2020         return ret;
2021 }
2022
2023 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2024 {
2025         drm_gem_free_mmap_offset(&obj->base);
2026 }
2027
2028 int
2029 i915_gem_mmap_gtt(struct drm_file *file,
2030                   struct drm_device *dev,
2031                   uint32_t handle,
2032                   uint64_t *offset)
2033 {
2034         struct drm_i915_gem_object *obj;
2035         int ret;
2036
2037         ret = i915_mutex_lock_interruptible(dev);
2038         if (ret)
2039                 return ret;
2040
2041         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
2042         if (&obj->base == NULL) {
2043                 ret = -ENOENT;
2044                 goto unlock;
2045         }
2046
2047         if (obj->madv != I915_MADV_WILLNEED) {
2048                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
2049                 ret = -EFAULT;
2050                 goto out;
2051         }
2052
2053         ret = i915_gem_object_create_mmap_offset(obj);
2054         if (ret)
2055                 goto out;
2056
2057         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2058
2059 out:
2060         drm_gem_object_unreference(&obj->base);
2061 unlock:
2062         mutex_unlock(&dev->struct_mutex);
2063         return ret;
2064 }
2065
2066 /**
2067  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2068  * @dev: DRM device
2069  * @data: GTT mapping ioctl data
2070  * @file: GEM object info
2071  *
2072  * Simply returns the fake offset to userspace so it can mmap it.
2073  * The mmap call will end up in drm_gem_mmap(), which will set things
2074  * up so we can get faults in the handler above.
2075  *
2076  * The fault handler will take care of binding the object into the GTT
2077  * (since it may have been evicted to make room for something), allocating
2078  * a fence register, and mapping the appropriate aperture address into
2079  * userspace.
2080  */
2081 int
2082 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2083                         struct drm_file *file)
2084 {
2085         struct drm_i915_gem_mmap_gtt *args = data;
2086
2087         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2088 }
2089
2090 /* Immediately discard the backing storage */
2091 static void
2092 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2093 {
2094         i915_gem_object_free_mmap_offset(obj);
2095
2096         if (obj->base.filp == NULL)
2097                 return;
2098
2099         /* Our goal here is to return as much of the memory as
2100          * is possible back to the system as we are called from OOM.
2101          * To do this we must instruct the shmfs to drop all of its
2102          * backing pages, *now*.
2103          */
2104         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2105         obj->madv = __I915_MADV_PURGED;
2106 }
2107
2108 /* Try to discard unwanted pages */
2109 static void
2110 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2111 {
2112         struct address_space *mapping;
2113
2114         switch (obj->madv) {
2115         case I915_MADV_DONTNEED:
2116                 i915_gem_object_truncate(obj);
2117         case __I915_MADV_PURGED:
2118                 return;
2119         }
2120
2121         if (obj->base.filp == NULL)
2122                 return;
2123
2124         mapping = file_inode(obj->base.filp)->i_mapping,
2125         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2126 }
2127
2128 static void
2129 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2130 {
2131         struct sg_page_iter sg_iter;
2132         int ret;
2133
2134         BUG_ON(obj->madv == __I915_MADV_PURGED);
2135
2136         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2137         if (ret) {
2138                 /* In the event of a disaster, abandon all caches and
2139                  * hope for the best.
2140                  */
2141                 WARN_ON(ret != -EIO);
2142                 i915_gem_clflush_object(obj, true);
2143                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2144         }
2145
2146         i915_gem_gtt_finish_object(obj);
2147
2148         if (i915_gem_object_needs_bit17_swizzle(obj))
2149                 i915_gem_object_save_bit_17_swizzle(obj);
2150
2151         if (obj->madv == I915_MADV_DONTNEED)
2152                 obj->dirty = 0;
2153
2154         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2155                 struct page *page = sg_page_iter_page(&sg_iter);
2156
2157                 if (obj->dirty)
2158                         set_page_dirty(page);
2159
2160                 if (obj->madv == I915_MADV_WILLNEED)
2161                         mark_page_accessed(page);
2162
2163                 page_cache_release(page);
2164         }
2165         obj->dirty = 0;
2166
2167         sg_free_table(obj->pages);
2168         kfree(obj->pages);
2169 }
2170
2171 int
2172 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2173 {
2174         const struct drm_i915_gem_object_ops *ops = obj->ops;
2175
2176         if (obj->pages == NULL)
2177                 return 0;
2178
2179         if (obj->pages_pin_count)
2180                 return -EBUSY;
2181
2182         BUG_ON(i915_gem_obj_bound_any(obj));
2183
2184         /* ->put_pages might need to allocate memory for the bit17 swizzle
2185          * array, hence protect them from being reaped by removing them from gtt
2186          * lists early. */
2187         list_del(&obj->global_list);
2188
2189         ops->put_pages(obj);
2190         obj->pages = NULL;
2191
2192         i915_gem_object_invalidate(obj);
2193
2194         return 0;
2195 }
2196
2197 static int
2198 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2199 {
2200         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2201         int page_count, i;
2202         struct address_space *mapping;
2203         struct sg_table *st;
2204         struct scatterlist *sg;
2205         struct sg_page_iter sg_iter;
2206         struct page *page;
2207         unsigned long last_pfn = 0;     /* suppress gcc warning */
2208         int ret;
2209         gfp_t gfp;
2210
2211         /* Assert that the object is not currently in any GPU domain. As it
2212          * wasn't in the GTT, there shouldn't be any way it could have been in
2213          * a GPU cache
2214          */
2215         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2216         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2217
2218         st = kmalloc(sizeof(*st), GFP_KERNEL);
2219         if (st == NULL)
2220                 return -ENOMEM;
2221
2222         page_count = obj->base.size / PAGE_SIZE;
2223         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2224                 kfree(st);
2225                 return -ENOMEM;
2226         }
2227
2228         /* Get the list of pages out of our struct file.  They'll be pinned
2229          * at this point until we release them.
2230          *
2231          * Fail silently without starting the shrinker
2232          */
2233         mapping = file_inode(obj->base.filp)->i_mapping;
2234         gfp = mapping_gfp_mask(mapping);
2235         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
2236         gfp &= ~(__GFP_IO | __GFP_WAIT);
2237         sg = st->sgl;
2238         st->nents = 0;
2239         for (i = 0; i < page_count; i++) {
2240                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2241                 if (IS_ERR(page)) {
2242                         i915_gem_shrink(dev_priv,
2243                                         page_count,
2244                                         I915_SHRINK_BOUND |
2245                                         I915_SHRINK_UNBOUND |
2246                                         I915_SHRINK_PURGEABLE);
2247                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2248                 }
2249                 if (IS_ERR(page)) {
2250                         /* We've tried hard to allocate the memory by reaping
2251                          * our own buffer, now let the real VM do its job and
2252                          * go down in flames if truly OOM.
2253                          */
2254                         i915_gem_shrink_all(dev_priv);
2255                         page = shmem_read_mapping_page(mapping, i);
2256                         if (IS_ERR(page)) {
2257                                 ret = PTR_ERR(page);
2258                                 goto err_pages;
2259                         }
2260                 }
2261 #ifdef CONFIG_SWIOTLB
2262                 if (swiotlb_nr_tbl()) {
2263                         st->nents++;
2264                         sg_set_page(sg, page, PAGE_SIZE, 0);
2265                         sg = sg_next(sg);
2266                         continue;
2267                 }
2268 #endif
2269                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2270                         if (i)
2271                                 sg = sg_next(sg);
2272                         st->nents++;
2273                         sg_set_page(sg, page, PAGE_SIZE, 0);
2274                 } else {
2275                         sg->length += PAGE_SIZE;
2276                 }
2277                 last_pfn = page_to_pfn(page);
2278
2279                 /* Check that the i965g/gm workaround works. */
2280                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2281         }
2282 #ifdef CONFIG_SWIOTLB
2283         if (!swiotlb_nr_tbl())
2284 #endif
2285                 sg_mark_end(sg);
2286         obj->pages = st;
2287
2288         ret = i915_gem_gtt_prepare_object(obj);
2289         if (ret)
2290                 goto err_pages;
2291
2292         if (i915_gem_object_needs_bit17_swizzle(obj))
2293                 i915_gem_object_do_bit_17_swizzle(obj);
2294
2295         if (obj->tiling_mode != I915_TILING_NONE &&
2296             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2297                 i915_gem_object_pin_pages(obj);
2298
2299         return 0;
2300
2301 err_pages:
2302         sg_mark_end(sg);
2303         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2304                 page_cache_release(sg_page_iter_page(&sg_iter));
2305         sg_free_table(st);
2306         kfree(st);
2307
2308         /* shmemfs first checks if there is enough memory to allocate the page
2309          * and reports ENOSPC should there be insufficient, along with the usual
2310          * ENOMEM for a genuine allocation failure.
2311          *
2312          * We use ENOSPC in our driver to mean that we have run out of aperture
2313          * space and so want to translate the error from shmemfs back to our
2314          * usual understanding of ENOMEM.
2315          */
2316         if (ret == -ENOSPC)
2317                 ret = -ENOMEM;
2318
2319         return ret;
2320 }
2321
2322 /* Ensure that the associated pages are gathered from the backing storage
2323  * and pinned into our object. i915_gem_object_get_pages() may be called
2324  * multiple times before they are released by a single call to
2325  * i915_gem_object_put_pages() - once the pages are no longer referenced
2326  * either as a result of memory pressure (reaping pages under the shrinker)
2327  * or as the object is itself released.
2328  */
2329 int
2330 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2331 {
2332         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2333         const struct drm_i915_gem_object_ops *ops = obj->ops;
2334         int ret;
2335
2336         if (obj->pages)
2337                 return 0;
2338
2339         if (obj->madv != I915_MADV_WILLNEED) {
2340                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2341                 return -EFAULT;
2342         }
2343
2344         BUG_ON(obj->pages_pin_count);
2345
2346         ret = ops->get_pages(obj);
2347         if (ret)
2348                 return ret;
2349
2350         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2351
2352         obj->get_page.sg = obj->pages->sgl;
2353         obj->get_page.last = 0;
2354
2355         return 0;
2356 }
2357
2358 void i915_vma_move_to_active(struct i915_vma *vma,
2359                              struct drm_i915_gem_request *req)
2360 {
2361         struct drm_i915_gem_object *obj = vma->obj;
2362         struct intel_engine_cs *ring;
2363
2364         ring = i915_gem_request_get_ring(req);
2365
2366         /* Add a reference if we're newly entering the active list. */
2367         if (obj->active == 0)
2368                 drm_gem_object_reference(&obj->base);
2369         obj->active |= intel_ring_flag(ring);
2370
2371         list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
2372         i915_gem_request_assign(&obj->last_read_req[ring->id], req);
2373
2374         list_move_tail(&vma->mm_list, &vma->vm->active_list);
2375 }
2376
2377 static void
2378 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2379 {
2380         RQ_BUG_ON(obj->last_write_req == NULL);
2381         RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
2382
2383         i915_gem_request_assign(&obj->last_write_req, NULL);
2384         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2385 }
2386
2387 static void
2388 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2389 {
2390         struct i915_vma *vma;
2391
2392         RQ_BUG_ON(obj->last_read_req[ring] == NULL);
2393         RQ_BUG_ON(!(obj->active & (1 << ring)));
2394
2395         list_del_init(&obj->ring_list[ring]);
2396         i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2397
2398         if (obj->last_write_req && obj->last_write_req->ring->id == ring)
2399                 i915_gem_object_retire__write(obj);
2400
2401         obj->active &= ~(1 << ring);
2402         if (obj->active)
2403                 return;
2404
2405         list_for_each_entry(vma, &obj->vma_list, vma_link) {
2406                 if (!list_empty(&vma->mm_list))
2407                         list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
2408         }
2409
2410         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2411         drm_gem_object_unreference(&obj->base);
2412 }
2413
2414 static int
2415 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2416 {
2417         struct drm_i915_private *dev_priv = dev->dev_private;
2418         struct intel_engine_cs *ring;
2419         int ret, i, j;
2420
2421         /* Carefully retire all requests without writing to the rings */
2422         for_each_ring(ring, dev_priv, i) {
2423                 ret = intel_ring_idle(ring);
2424                 if (ret)
2425                         return ret;
2426         }
2427         i915_gem_retire_requests(dev);
2428
2429         /* Finally reset hw state */
2430         for_each_ring(ring, dev_priv, i) {
2431                 intel_ring_init_seqno(ring, seqno);
2432
2433                 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
2434                         ring->semaphore.sync_seqno[j] = 0;
2435         }
2436
2437         return 0;
2438 }
2439
2440 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2441 {
2442         struct drm_i915_private *dev_priv = dev->dev_private;
2443         int ret;
2444
2445         if (seqno == 0)
2446                 return -EINVAL;
2447
2448         /* HWS page needs to be set less than what we
2449          * will inject to ring
2450          */
2451         ret = i915_gem_init_seqno(dev, seqno - 1);
2452         if (ret)
2453                 return ret;
2454
2455         /* Carefully set the last_seqno value so that wrap
2456          * detection still works
2457          */
2458         dev_priv->next_seqno = seqno;
2459         dev_priv->last_seqno = seqno - 1;
2460         if (dev_priv->last_seqno == 0)
2461                 dev_priv->last_seqno--;
2462
2463         return 0;
2464 }
2465
2466 int
2467 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2468 {
2469         struct drm_i915_private *dev_priv = dev->dev_private;
2470
2471         /* reserve 0 for non-seqno */
2472         if (dev_priv->next_seqno == 0) {
2473                 int ret = i915_gem_init_seqno(dev, 0);
2474                 if (ret)
2475                         return ret;
2476
2477                 dev_priv->next_seqno = 1;
2478         }
2479
2480         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2481         return 0;
2482 }
2483
2484 /*
2485  * NB: This function is not allowed to fail. Doing so would mean the the
2486  * request is not being tracked for completion but the work itself is
2487  * going to happen on the hardware. This would be a Bad Thing(tm).
2488  */
2489 void __i915_add_request(struct drm_i915_gem_request *request,
2490                         struct drm_i915_gem_object *obj,
2491                         bool flush_caches)
2492 {
2493         struct intel_engine_cs *ring;
2494         struct drm_i915_private *dev_priv;
2495         struct intel_ringbuffer *ringbuf;
2496         u32 request_start;
2497         int ret;
2498
2499         if (WARN_ON(request == NULL))
2500                 return;
2501
2502         ring = request->ring;
2503         dev_priv = ring->dev->dev_private;
2504         ringbuf = request->ringbuf;
2505
2506         /*
2507          * To ensure that this call will not fail, space for its emissions
2508          * should already have been reserved in the ring buffer. Let the ring
2509          * know that it is time to use that space up.
2510          */
2511         intel_ring_reserved_space_use(ringbuf);
2512
2513         request_start = intel_ring_get_tail(ringbuf);
2514         /*
2515          * Emit any outstanding flushes - execbuf can fail to emit the flush
2516          * after having emitted the batchbuffer command. Hence we need to fix
2517          * things up similar to emitting the lazy request. The difference here
2518          * is that the flush _must_ happen before the next request, no matter
2519          * what.
2520          */
2521         if (flush_caches) {
2522                 if (i915.enable_execlists)
2523                         ret = logical_ring_flush_all_caches(request);
2524                 else
2525                         ret = intel_ring_flush_all_caches(request);
2526                 /* Not allowed to fail! */
2527                 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
2528         }
2529
2530         /* Record the position of the start of the request so that
2531          * should we detect the updated seqno part-way through the
2532          * GPU processing the request, we never over-estimate the
2533          * position of the head.
2534          */
2535         request->postfix = intel_ring_get_tail(ringbuf);
2536
2537         if (i915.enable_execlists)
2538                 ret = ring->emit_request(request);
2539         else {
2540                 ret = ring->add_request(request);
2541
2542                 request->tail = intel_ring_get_tail(ringbuf);
2543         }
2544         /* Not allowed to fail! */
2545         WARN(ret, "emit|add_request failed: %d!\n", ret);
2546
2547         request->head = request_start;
2548
2549         /* Whilst this request exists, batch_obj will be on the
2550          * active_list, and so will hold the active reference. Only when this
2551          * request is retired will the the batch_obj be moved onto the
2552          * inactive_list and lose its active reference. Hence we do not need
2553          * to explicitly hold another reference here.
2554          */
2555         request->batch_obj = obj;
2556
2557         request->emitted_jiffies = jiffies;
2558         ring->last_submitted_seqno = request->seqno;
2559         list_add_tail(&request->list, &ring->request_list);
2560
2561         trace_i915_gem_request_add(request);
2562
2563         i915_queue_hangcheck(ring->dev);
2564
2565         queue_delayed_work(dev_priv->wq,
2566                            &dev_priv->mm.retire_work,
2567                            round_jiffies_up_relative(HZ));
2568         intel_mark_busy(dev_priv->dev);
2569
2570         /* Sanity check that the reserved size was large enough. */
2571         intel_ring_reserved_space_end(ringbuf);
2572 }
2573
2574 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2575                                    const struct intel_context *ctx)
2576 {
2577         unsigned long elapsed;
2578
2579         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2580
2581         if (ctx->hang_stats.banned)
2582                 return true;
2583
2584         if (ctx->hang_stats.ban_period_seconds &&
2585             elapsed <= ctx->hang_stats.ban_period_seconds) {
2586                 if (!i915_gem_context_is_default(ctx)) {
2587                         DRM_DEBUG("context hanging too fast, banning!\n");
2588                         return true;
2589                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2590                         if (i915_stop_ring_allow_warn(dev_priv))
2591                                 DRM_ERROR("gpu hanging too fast, banning!\n");
2592                         return true;
2593                 }
2594         }
2595
2596         return false;
2597 }
2598
2599 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2600                                   struct intel_context *ctx,
2601                                   const bool guilty)
2602 {
2603         struct i915_ctx_hang_stats *hs;
2604
2605         if (WARN_ON(!ctx))
2606                 return;
2607
2608         hs = &ctx->hang_stats;
2609
2610         if (guilty) {
2611                 hs->banned = i915_context_is_banned(dev_priv, ctx);
2612                 hs->batch_active++;
2613                 hs->guilty_ts = get_seconds();
2614         } else {
2615                 hs->batch_pending++;
2616         }
2617 }
2618
2619 void i915_gem_request_free(struct kref *req_ref)
2620 {
2621         struct drm_i915_gem_request *req = container_of(req_ref,
2622                                                  typeof(*req), ref);
2623         struct intel_context *ctx = req->ctx;
2624
2625         if (req->file_priv)
2626                 i915_gem_request_remove_from_client(req);
2627
2628         if (ctx) {
2629                 if (i915.enable_execlists) {
2630                         if (ctx != req->ring->default_context)
2631                                 intel_lr_context_unpin(req);
2632                 }
2633
2634                 i915_gem_context_unreference(ctx);
2635         }
2636
2637         kmem_cache_free(req->i915->requests, req);
2638 }
2639
2640 int i915_gem_request_alloc(struct intel_engine_cs *ring,
2641                            struct intel_context *ctx,
2642                            struct drm_i915_gem_request **req_out)
2643 {
2644         struct drm_i915_private *dev_priv = to_i915(ring->dev);
2645         struct drm_i915_gem_request *req;
2646         int ret;
2647
2648         if (!req_out)
2649                 return -EINVAL;
2650
2651         *req_out = NULL;
2652
2653         req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
2654         if (req == NULL)
2655                 return -ENOMEM;
2656
2657         ret = i915_gem_get_seqno(ring->dev, &req->seqno);
2658         if (ret)
2659                 goto err;
2660
2661         kref_init(&req->ref);
2662         req->i915 = dev_priv;
2663         req->ring = ring;
2664         req->ctx  = ctx;
2665         i915_gem_context_reference(req->ctx);
2666
2667         if (i915.enable_execlists)
2668                 ret = intel_logical_ring_alloc_request_extras(req);
2669         else
2670                 ret = intel_ring_alloc_request_extras(req);
2671         if (ret) {
2672                 i915_gem_context_unreference(req->ctx);
2673                 goto err;
2674         }
2675
2676         /*
2677          * Reserve space in the ring buffer for all the commands required to
2678          * eventually emit this request. This is to guarantee that the
2679          * i915_add_request() call can't fail. Note that the reserve may need
2680          * to be redone if the request is not actually submitted straight
2681          * away, e.g. because a GPU scheduler has deferred it.
2682          */
2683         if (i915.enable_execlists)
2684                 ret = intel_logical_ring_reserve_space(req);
2685         else
2686                 ret = intel_ring_reserve_space(req);
2687         if (ret) {
2688                 /*
2689                  * At this point, the request is fully allocated even if not
2690                  * fully prepared. Thus it can be cleaned up using the proper
2691                  * free code.
2692                  */
2693                 i915_gem_request_cancel(req);
2694                 return ret;
2695         }
2696
2697         *req_out = req;
2698         return 0;
2699
2700 err:
2701         kmem_cache_free(dev_priv->requests, req);
2702         return ret;
2703 }
2704
2705 void i915_gem_request_cancel(struct drm_i915_gem_request *req)
2706 {
2707         intel_ring_reserved_space_cancel(req->ringbuf);
2708
2709         i915_gem_request_unreference(req);
2710 }
2711
2712 struct drm_i915_gem_request *
2713 i915_gem_find_active_request(struct intel_engine_cs *ring)
2714 {
2715         struct drm_i915_gem_request *request;
2716
2717         list_for_each_entry(request, &ring->request_list, list) {
2718                 if (i915_gem_request_completed(request, false))
2719                         continue;
2720
2721                 return request;
2722         }
2723
2724         return NULL;
2725 }
2726
2727 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2728                                        struct intel_engine_cs *ring)
2729 {
2730         struct drm_i915_gem_request *request;
2731         bool ring_hung;
2732
2733         request = i915_gem_find_active_request(ring);
2734
2735         if (request == NULL)
2736                 return;
2737
2738         ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2739
2740         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2741
2742         list_for_each_entry_continue(request, &ring->request_list, list)
2743                 i915_set_reset_status(dev_priv, request->ctx, false);
2744 }
2745
2746 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2747                                         struct intel_engine_cs *ring)
2748 {
2749         while (!list_empty(&ring->active_list)) {
2750                 struct drm_i915_gem_object *obj;
2751
2752                 obj = list_first_entry(&ring->active_list,
2753                                        struct drm_i915_gem_object,
2754                                        ring_list[ring->id]);
2755
2756                 i915_gem_object_retire__read(obj, ring->id);
2757         }
2758
2759         /*
2760          * Clear the execlists queue up before freeing the requests, as those
2761          * are the ones that keep the context and ringbuffer backing objects
2762          * pinned in place.
2763          */
2764         while (!list_empty(&ring->execlist_queue)) {
2765                 struct drm_i915_gem_request *submit_req;
2766
2767                 submit_req = list_first_entry(&ring->execlist_queue,
2768                                 struct drm_i915_gem_request,
2769                                 execlist_link);
2770                 list_del(&submit_req->execlist_link);
2771
2772                 if (submit_req->ctx != ring->default_context)
2773                         intel_lr_context_unpin(submit_req);
2774
2775                 i915_gem_request_unreference(submit_req);
2776         }
2777
2778         /*
2779          * We must free the requests after all the corresponding objects have
2780          * been moved off active lists. Which is the same order as the normal
2781          * retire_requests function does. This is important if object hold
2782          * implicit references on things like e.g. ppgtt address spaces through
2783          * the request.
2784          */
2785         while (!list_empty(&ring->request_list)) {
2786                 struct drm_i915_gem_request *request;
2787
2788                 request = list_first_entry(&ring->request_list,
2789                                            struct drm_i915_gem_request,
2790                                            list);
2791
2792                 i915_gem_request_retire(request);
2793         }
2794 }
2795
2796 void i915_gem_restore_fences(struct drm_device *dev)
2797 {
2798         struct drm_i915_private *dev_priv = dev->dev_private;
2799         int i;
2800
2801         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2802                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2803
2804                 /*
2805                  * Commit delayed tiling changes if we have an object still
2806                  * attached to the fence, otherwise just clear the fence.
2807                  */
2808                 if (reg->obj) {
2809                         i915_gem_object_update_fence(reg->obj, reg,
2810                                                      reg->obj->tiling_mode);
2811                 } else {
2812                         i915_gem_write_fence(dev, i, NULL);
2813                 }
2814         }
2815 }
2816
2817 void i915_gem_reset(struct drm_device *dev)
2818 {
2819         struct drm_i915_private *dev_priv = dev->dev_private;
2820         struct intel_engine_cs *ring;
2821         int i;
2822
2823         /*
2824          * Before we free the objects from the requests, we need to inspect
2825          * them for finding the guilty party. As the requests only borrow
2826          * their reference to the objects, the inspection must be done first.
2827          */
2828         for_each_ring(ring, dev_priv, i)
2829                 i915_gem_reset_ring_status(dev_priv, ring);
2830
2831         for_each_ring(ring, dev_priv, i)
2832                 i915_gem_reset_ring_cleanup(dev_priv, ring);
2833
2834         i915_gem_context_reset(dev);
2835
2836         i915_gem_restore_fences(dev);
2837
2838         WARN_ON(i915_verify_lists(dev));
2839 }
2840
2841 /**
2842  * This function clears the request list as sequence numbers are passed.
2843  */
2844 void
2845 i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2846 {
2847         WARN_ON(i915_verify_lists(ring->dev));
2848
2849         /* Retire requests first as we use it above for the early return.
2850          * If we retire requests last, we may use a later seqno and so clear
2851          * the requests lists without clearing the active list, leading to
2852          * confusion.
2853          */
2854         while (!list_empty(&ring->request_list)) {
2855                 struct drm_i915_gem_request *request;
2856
2857                 request = list_first_entry(&ring->request_list,
2858                                            struct drm_i915_gem_request,
2859                                            list);
2860
2861                 if (!i915_gem_request_completed(request, true))
2862                         break;
2863
2864                 i915_gem_request_retire(request);
2865         }
2866
2867         /* Move any buffers on the active list that are no longer referenced
2868          * by the ringbuffer to the flushing/inactive lists as appropriate,
2869          * before we free the context associated with the requests.
2870          */
2871         while (!list_empty(&ring->active_list)) {
2872                 struct drm_i915_gem_object *obj;
2873
2874                 obj = list_first_entry(&ring->active_list,
2875                                       struct drm_i915_gem_object,
2876                                       ring_list[ring->id]);
2877
2878                 if (!list_empty(&obj->last_read_req[ring->id]->list))
2879                         break;
2880
2881                 i915_gem_object_retire__read(obj, ring->id);
2882         }
2883
2884         if (unlikely(ring->trace_irq_req &&
2885                      i915_gem_request_completed(ring->trace_irq_req, true))) {
2886                 ring->irq_put(ring);
2887                 i915_gem_request_assign(&ring->trace_irq_req, NULL);
2888         }
2889
2890         WARN_ON(i915_verify_lists(ring->dev));
2891 }
2892
2893 bool
2894 i915_gem_retire_requests(struct drm_device *dev)
2895 {
2896         struct drm_i915_private *dev_priv = dev->dev_private;
2897         struct intel_engine_cs *ring;
2898         bool idle = true;
2899         int i;
2900
2901         for_each_ring(ring, dev_priv, i) {
2902                 i915_gem_retire_requests_ring(ring);
2903                 idle &= list_empty(&ring->request_list);
2904                 if (i915.enable_execlists) {
2905                         unsigned long flags;
2906
2907                         spin_lock_irqsave(&ring->execlist_lock, flags);
2908                         idle &= list_empty(&ring->execlist_queue);
2909                         spin_unlock_irqrestore(&ring->execlist_lock, flags);
2910
2911                         intel_execlists_retire_requests(ring);
2912                 }
2913         }
2914
2915         if (idle)
2916                 mod_delayed_work(dev_priv->wq,
2917                                    &dev_priv->mm.idle_work,
2918                                    msecs_to_jiffies(100));
2919
2920         return idle;
2921 }
2922
2923 static void
2924 i915_gem_retire_work_handler(struct work_struct *work)
2925 {
2926         struct drm_i915_private *dev_priv =
2927                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2928         struct drm_device *dev = dev_priv->dev;
2929         bool idle;
2930
2931         /* Come back later if the device is busy... */
2932         idle = false;
2933         if (mutex_trylock(&dev->struct_mutex)) {
2934                 idle = i915_gem_retire_requests(dev);
2935                 mutex_unlock(&dev->struct_mutex);
2936         }
2937         if (!idle)
2938                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2939                                    round_jiffies_up_relative(HZ));
2940 }
2941
2942 static void
2943 i915_gem_idle_work_handler(struct work_struct *work)
2944 {
2945         struct drm_i915_private *dev_priv =
2946                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
2947         struct drm_device *dev = dev_priv->dev;
2948         struct intel_engine_cs *ring;
2949         int i;
2950
2951         for_each_ring(ring, dev_priv, i)
2952                 if (!list_empty(&ring->request_list))
2953                         return;
2954
2955         intel_mark_idle(dev);
2956
2957         if (mutex_trylock(&dev->struct_mutex)) {
2958                 struct intel_engine_cs *ring;
2959                 int i;
2960
2961                 for_each_ring(ring, dev_priv, i)
2962                         i915_gem_batch_pool_fini(&ring->batch_pool);
2963
2964                 mutex_unlock(&dev->struct_mutex);
2965         }
2966 }
2967
2968 /**
2969  * Ensures that an object will eventually get non-busy by flushing any required
2970  * write domains, emitting any outstanding lazy request and retiring and
2971  * completed requests.
2972  */
2973 static int
2974 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2975 {
2976         int i;
2977
2978         if (!obj->active)
2979                 return 0;
2980
2981         for (i = 0; i < I915_NUM_RINGS; i++) {
2982                 struct drm_i915_gem_request *req;
2983
2984                 req = obj->last_read_req[i];
2985                 if (req == NULL)
2986                         continue;
2987
2988                 if (list_empty(&req->list))
2989                         goto retire;
2990
2991                 if (i915_gem_request_completed(req, true)) {
2992                         __i915_gem_request_retire__upto(req);
2993 retire:
2994                         i915_gem_object_retire__read(obj, i);
2995                 }
2996         }
2997
2998         return 0;
2999 }
3000
3001 /**
3002  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3003  * @DRM_IOCTL_ARGS: standard ioctl arguments
3004  *
3005  * Returns 0 if successful, else an error is returned with the remaining time in
3006  * the timeout parameter.
3007  *  -ETIME: object is still busy after timeout
3008  *  -ERESTARTSYS: signal interrupted the wait
3009  *  -ENONENT: object doesn't exist
3010  * Also possible, but rare:
3011  *  -EAGAIN: GPU wedged
3012  *  -ENOMEM: damn
3013  *  -ENODEV: Internal IRQ fail
3014  *  -E?: The add request failed
3015  *
3016  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3017  * non-zero timeout parameter the wait ioctl will wait for the given number of
3018  * nanoseconds on an object becoming unbusy. Since the wait itself does so
3019  * without holding struct_mutex the object may become re-busied before this
3020  * function completes. A similar but shorter * race condition exists in the busy
3021  * ioctl
3022  */
3023 int
3024 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3025 {
3026         struct drm_i915_private *dev_priv = dev->dev_private;
3027         struct drm_i915_gem_wait *args = data;
3028         struct drm_i915_gem_object *obj;
3029         struct drm_i915_gem_request *req[I915_NUM_RINGS];
3030         unsigned reset_counter;
3031         int i, n = 0;
3032         int ret;
3033
3034         if (args->flags != 0)
3035                 return -EINVAL;
3036
3037         ret = i915_mutex_lock_interruptible(dev);
3038         if (ret)
3039                 return ret;
3040
3041         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
3042         if (&obj->base == NULL) {
3043                 mutex_unlock(&dev->struct_mutex);
3044                 return -ENOENT;
3045         }
3046
3047         /* Need to make sure the object gets inactive eventually. */
3048         ret = i915_gem_object_flush_active(obj);
3049         if (ret)
3050                 goto out;
3051
3052         if (!obj->active)
3053                 goto out;
3054
3055         /* Do this after OLR check to make sure we make forward progress polling
3056          * on this IOCTL with a timeout == 0 (like busy ioctl)
3057          */
3058         if (args->timeout_ns == 0) {
3059                 ret = -ETIME;
3060                 goto out;
3061         }
3062
3063         drm_gem_object_unreference(&obj->base);
3064         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3065
3066         for (i = 0; i < I915_NUM_RINGS; i++) {
3067                 if (obj->last_read_req[i] == NULL)
3068                         continue;
3069
3070                 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
3071         }
3072
3073         mutex_unlock(&dev->struct_mutex);
3074
3075         for (i = 0; i < n; i++) {
3076                 if (ret == 0)
3077                         ret = __i915_wait_request(req[i], reset_counter, true,
3078                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
3079                                                   file->driver_priv);
3080                 i915_gem_request_unreference__unlocked(req[i]);
3081         }
3082         return ret;
3083
3084 out:
3085         drm_gem_object_unreference(&obj->base);
3086         mutex_unlock(&dev->struct_mutex);
3087         return ret;
3088 }
3089
3090 static int
3091 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
3092                        struct intel_engine_cs *to,
3093                        struct drm_i915_gem_request *from_req,
3094                        struct drm_i915_gem_request **to_req)
3095 {
3096         struct intel_engine_cs *from;
3097         int ret;
3098
3099         from = i915_gem_request_get_ring(from_req);
3100         if (to == from)
3101                 return 0;
3102
3103         if (i915_gem_request_completed(from_req, true))
3104                 return 0;
3105
3106         if (!i915_semaphore_is_enabled(obj->base.dev)) {
3107                 struct drm_i915_private *i915 = to_i915(obj->base.dev);
3108                 ret = __i915_wait_request(from_req,
3109                                           atomic_read(&i915->gpu_error.reset_counter),
3110                                           i915->mm.interruptible,
3111                                           NULL,
3112                                           &i915->rps.semaphores);
3113                 if (ret)
3114                         return ret;
3115
3116                 i915_gem_object_retire_request(obj, from_req);
3117         } else {
3118                 int idx = intel_ring_sync_index(from, to);
3119                 u32 seqno = i915_gem_request_get_seqno(from_req);
3120
3121                 WARN_ON(!to_req);
3122
3123                 if (seqno <= from->semaphore.sync_seqno[idx])
3124                         return 0;
3125
3126                 if (*to_req == NULL) {
3127                         ret = i915_gem_request_alloc(to, to->default_context, to_req);
3128                         if (ret)
3129                                 return ret;
3130                 }
3131
3132                 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
3133                 ret = to->semaphore.sync_to(*to_req, from, seqno);
3134                 if (ret)
3135                         return ret;
3136
3137                 /* We use last_read_req because sync_to()
3138                  * might have just caused seqno wrap under
3139                  * the radar.
3140                  */
3141                 from->semaphore.sync_seqno[idx] =
3142                         i915_gem_request_get_seqno(obj->last_read_req[from->id]);
3143         }
3144
3145         return 0;
3146 }
3147
3148 /**
3149  * i915_gem_object_sync - sync an object to a ring.
3150  *
3151  * @obj: object which may be in use on another ring.
3152  * @to: ring we wish to use the object on. May be NULL.
3153  * @to_req: request we wish to use the object for. See below.
3154  *          This will be allocated and returned if a request is
3155  *          required but not passed in.
3156  *
3157  * This code is meant to abstract object synchronization with the GPU.
3158  * Calling with NULL implies synchronizing the object with the CPU
3159  * rather than a particular GPU ring. Conceptually we serialise writes
3160  * between engines inside the GPU. We only allow one engine to write
3161  * into a buffer at any time, but multiple readers. To ensure each has
3162  * a coherent view of memory, we must:
3163  *
3164  * - If there is an outstanding write request to the object, the new
3165  *   request must wait for it to complete (either CPU or in hw, requests
3166  *   on the same ring will be naturally ordered).
3167  *
3168  * - If we are a write request (pending_write_domain is set), the new
3169  *   request must wait for outstanding read requests to complete.
3170  *
3171  * For CPU synchronisation (NULL to) no request is required. For syncing with
3172  * rings to_req must be non-NULL. However, a request does not have to be
3173  * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
3174  * request will be allocated automatically and returned through *to_req. Note
3175  * that it is not guaranteed that commands will be emitted (because the system
3176  * might already be idle). Hence there is no need to create a request that
3177  * might never have any work submitted. Note further that if a request is
3178  * returned in *to_req, it is the responsibility of the caller to submit
3179  * that request (after potentially adding more work to it).
3180  *
3181  * Returns 0 if successful, else propagates up the lower layer error.
3182  */
3183 int
3184 i915_gem_object_sync(struct drm_i915_gem_object *obj,
3185                      struct intel_engine_cs *to,
3186                      struct drm_i915_gem_request **to_req)
3187 {
3188         const bool readonly = obj->base.pending_write_domain == 0;
3189         struct drm_i915_gem_request *req[I915_NUM_RINGS];
3190         int ret, i, n;
3191
3192         if (!obj->active)
3193                 return 0;
3194
3195         if (to == NULL)
3196                 return i915_gem_object_wait_rendering(obj, readonly);
3197
3198         n = 0;
3199         if (readonly) {
3200                 if (obj->last_write_req)
3201                         req[n++] = obj->last_write_req;
3202         } else {
3203                 for (i = 0; i < I915_NUM_RINGS; i++)
3204                         if (obj->last_read_req[i])
3205                                 req[n++] = obj->last_read_req[i];
3206         }
3207         for (i = 0; i < n; i++) {
3208                 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
3209                 if (ret)
3210                         return ret;
3211         }
3212
3213         return 0;
3214 }
3215
3216 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3217 {
3218         u32 old_write_domain, old_read_domains;
3219
3220         /* Force a pagefault for domain tracking on next user access */
3221         i915_gem_release_mmap(obj);
3222
3223         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3224                 return;
3225
3226         /* Wait for any direct GTT access to complete */
3227         mb();
3228
3229         old_read_domains = obj->base.read_domains;
3230         old_write_domain = obj->base.write_domain;
3231
3232         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3233         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3234
3235         trace_i915_gem_object_change_domain(obj,
3236                                             old_read_domains,
3237                                             old_write_domain);
3238 }
3239
3240 int i915_vma_unbind(struct i915_vma *vma)
3241 {
3242         struct drm_i915_gem_object *obj = vma->obj;
3243         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3244         int ret;
3245
3246         if (list_empty(&vma->vma_link))
3247                 return 0;
3248
3249         if (!drm_mm_node_allocated(&vma->node)) {
3250                 i915_gem_vma_destroy(vma);
3251                 return 0;
3252         }
3253
3254         if (vma->pin_count)
3255                 return -EBUSY;
3256
3257         BUG_ON(obj->pages == NULL);
3258
3259         ret = i915_gem_object_wait_rendering(obj, false);
3260         if (ret)
3261                 return ret;
3262         /* Continue on if we fail due to EIO, the GPU is hung so we
3263          * should be safe and we need to cleanup or else we might
3264          * cause memory corruption through use-after-free.
3265          */
3266
3267         if (i915_is_ggtt(vma->vm) &&
3268             vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3269                 i915_gem_object_finish_gtt(obj);
3270
3271                 /* release the fence reg _after_ flushing */
3272                 ret = i915_gem_object_put_fence(obj);
3273                 if (ret)
3274                         return ret;
3275         }
3276
3277         trace_i915_vma_unbind(vma);
3278
3279         vma->vm->unbind_vma(vma);
3280         vma->bound = 0;
3281
3282         list_del_init(&vma->mm_list);
3283         if (i915_is_ggtt(vma->vm)) {
3284                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3285                         obj->map_and_fenceable = false;
3286                 } else if (vma->ggtt_view.pages) {
3287                         sg_free_table(vma->ggtt_view.pages);
3288                         kfree(vma->ggtt_view.pages);
3289                 }
3290                 vma->ggtt_view.pages = NULL;
3291         }
3292
3293         drm_mm_remove_node(&vma->node);
3294         i915_gem_vma_destroy(vma);
3295
3296         /* Since the unbound list is global, only move to that list if
3297          * no more VMAs exist. */
3298         if (list_empty(&obj->vma_list))
3299                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3300
3301         /* And finally now the object is completely decoupled from this vma,
3302          * we can drop its hold on the backing storage and allow it to be
3303          * reaped by the shrinker.
3304          */
3305         i915_gem_object_unpin_pages(obj);
3306
3307         return 0;
3308 }
3309
3310 int i915_gpu_idle(struct drm_device *dev)
3311 {
3312         struct drm_i915_private *dev_priv = dev->dev_private;
3313         struct intel_engine_cs *ring;
3314         int ret, i;
3315
3316         /* Flush everything onto the inactive list. */
3317         for_each_ring(ring, dev_priv, i) {
3318                 if (!i915.enable_execlists) {
3319                         struct drm_i915_gem_request *req;
3320
3321                         ret = i915_gem_request_alloc(ring, ring->default_context, &req);
3322                         if (ret)
3323                                 return ret;
3324
3325                         ret = i915_switch_context(req);
3326                         if (ret) {
3327                                 i915_gem_request_cancel(req);
3328                                 return ret;
3329                         }
3330
3331                         i915_add_request_no_flush(req);
3332                 }
3333
3334                 ret = intel_ring_idle(ring);
3335                 if (ret)
3336                         return ret;
3337         }
3338
3339         WARN_ON(i915_verify_lists(dev));
3340         return 0;
3341 }
3342
3343 static void i965_write_fence_reg(struct drm_device *dev, int reg,
3344                                  struct drm_i915_gem_object *obj)
3345 {
3346         struct drm_i915_private *dev_priv = dev->dev_private;
3347         int fence_reg;
3348         int fence_pitch_shift;
3349
3350         if (INTEL_INFO(dev)->gen >= 6) {
3351                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
3352                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
3353         } else {
3354                 fence_reg = FENCE_REG_965_0;
3355                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
3356         }
3357
3358         fence_reg += reg * 8;
3359
3360         /* To w/a incoherency with non-atomic 64-bit register updates,
3361          * we split the 64-bit update into two 32-bit writes. In order
3362          * for a partial fence not to be evaluated between writes, we
3363          * precede the update with write to turn off the fence register,
3364          * and only enable the fence as the last step.
3365          *
3366          * For extra levels of paranoia, we make sure each step lands
3367          * before applying the next step.
3368          */
3369         I915_WRITE(fence_reg, 0);
3370         POSTING_READ(fence_reg);
3371
3372         if (obj) {
3373                 u32 size = i915_gem_obj_ggtt_size(obj);
3374                 uint64_t val;
3375
3376                 /* Adjust fence size to match tiled area */
3377                 if (obj->tiling_mode != I915_TILING_NONE) {
3378                         uint32_t row_size = obj->stride *
3379                                 (obj->tiling_mode == I915_TILING_Y ? 32 : 8);
3380                         size = (size / row_size) * row_size;
3381                 }
3382
3383                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3384                                  0xfffff000) << 32;
3385                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3386                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3387                 if (obj->tiling_mode == I915_TILING_Y)
3388                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3389                 val |= I965_FENCE_REG_VALID;
3390
3391                 I915_WRITE(fence_reg + 4, val >> 32);
3392                 POSTING_READ(fence_reg + 4);
3393
3394                 I915_WRITE(fence_reg + 0, val);
3395                 POSTING_READ(fence_reg);
3396         } else {
3397                 I915_WRITE(fence_reg + 4, 0);
3398                 POSTING_READ(fence_reg + 4);
3399         }
3400 }
3401
3402 static void i915_write_fence_reg(struct drm_device *dev, int reg,
3403                                  struct drm_i915_gem_object *obj)
3404 {
3405         struct drm_i915_private *dev_priv = dev->dev_private;
3406         u32 val;
3407
3408         if (obj) {
3409                 u32 size = i915_gem_obj_ggtt_size(obj);
3410                 int pitch_val;
3411                 int tile_width;
3412
3413                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
3414                      (size & -size) != size ||
3415                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3416                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
3417                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
3418
3419                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
3420                         tile_width = 128;
3421                 else
3422                         tile_width = 512;
3423
3424                 /* Note: pitch better be a power of two tile widths */
3425                 pitch_val = obj->stride / tile_width;
3426                 pitch_val = ffs(pitch_val) - 1;
3427
3428                 val = i915_gem_obj_ggtt_offset(obj);
3429                 if (obj->tiling_mode == I915_TILING_Y)
3430                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3431                 val |= I915_FENCE_SIZE_BITS(size);
3432                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3433                 val |= I830_FENCE_REG_VALID;
3434         } else
3435                 val = 0;
3436
3437         if (reg < 8)
3438                 reg = FENCE_REG_830_0 + reg * 4;
3439         else
3440                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
3441
3442         I915_WRITE(reg, val);
3443         POSTING_READ(reg);
3444 }
3445
3446 static void i830_write_fence_reg(struct drm_device *dev, int reg,
3447                                 struct drm_i915_gem_object *obj)
3448 {
3449         struct drm_i915_private *dev_priv = dev->dev_private;
3450         uint32_t val;
3451
3452         if (obj) {
3453                 u32 size = i915_gem_obj_ggtt_size(obj);
3454                 uint32_t pitch_val;
3455
3456                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
3457                      (size & -size) != size ||
3458                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3459                      "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
3460                      i915_gem_obj_ggtt_offset(obj), size);
3461
3462                 pitch_val = obj->stride / 128;
3463                 pitch_val = ffs(pitch_val) - 1;
3464
3465                 val = i915_gem_obj_ggtt_offset(obj);
3466                 if (obj->tiling_mode == I915_TILING_Y)
3467                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3468                 val |= I830_FENCE_SIZE_BITS(size);
3469                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3470                 val |= I830_FENCE_REG_VALID;
3471         } else
3472                 val = 0;
3473
3474         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
3475         POSTING_READ(FENCE_REG_830_0 + reg * 4);
3476 }
3477
3478 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
3479 {
3480         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
3481 }
3482
3483 static void i915_gem_write_fence(struct drm_device *dev, int reg,
3484                                  struct drm_i915_gem_object *obj)
3485 {
3486         struct drm_i915_private *dev_priv = dev->dev_private;
3487
3488         /* Ensure that all CPU reads are completed before installing a fence
3489          * and all writes before removing the fence.
3490          */
3491         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
3492                 mb();
3493
3494         WARN(obj && (!obj->stride || !obj->tiling_mode),
3495              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
3496              obj->stride, obj->tiling_mode);
3497
3498         if (IS_GEN2(dev))
3499                 i830_write_fence_reg(dev, reg, obj);
3500         else if (IS_GEN3(dev))
3501                 i915_write_fence_reg(dev, reg, obj);
3502         else if (INTEL_INFO(dev)->gen >= 4)
3503                 i965_write_fence_reg(dev, reg, obj);
3504
3505         /* And similarly be paranoid that no direct access to this region
3506          * is reordered to before the fence is installed.
3507          */
3508         if (i915_gem_object_needs_mb(obj))
3509                 mb();
3510 }
3511
3512 static inline int fence_number(struct drm_i915_private *dev_priv,
3513                                struct drm_i915_fence_reg *fence)
3514 {
3515         return fence - dev_priv->fence_regs;
3516 }
3517
3518 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
3519                                          struct drm_i915_fence_reg *fence,
3520                                          bool enable)
3521 {
3522         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3523         int reg = fence_number(dev_priv, fence);
3524
3525         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3526
3527         if (enable) {
3528                 obj->fence_reg = reg;
3529                 fence->obj = obj;
3530                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
3531         } else {
3532                 obj->fence_reg = I915_FENCE_REG_NONE;
3533                 fence->obj = NULL;
3534                 list_del_init(&fence->lru_list);
3535         }
3536         obj->fence_dirty = false;
3537 }
3538
3539 static int
3540 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3541 {
3542         if (obj->last_fenced_req) {
3543                 int ret = i915_wait_request(obj->last_fenced_req);
3544                 if (ret)
3545                         return ret;
3546
3547                 i915_gem_request_assign(&obj->last_fenced_req, NULL);
3548         }
3549
3550         return 0;
3551 }
3552
3553 int
3554 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3555 {
3556         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3557         struct drm_i915_fence_reg *fence;
3558         int ret;
3559
3560         ret = i915_gem_object_wait_fence(obj);
3561         if (ret)
3562                 return ret;
3563
3564         if (obj->fence_reg == I915_FENCE_REG_NONE)
3565                 return 0;
3566
3567         fence = &dev_priv->fence_regs[obj->fence_reg];
3568
3569         if (WARN_ON(fence->pin_count))
3570                 return -EBUSY;
3571
3572         i915_gem_object_fence_lost(obj);
3573         i915_gem_object_update_fence(obj, fence, false);
3574
3575         return 0;
3576 }
3577
3578 static struct drm_i915_fence_reg *
3579 i915_find_fence_reg(struct drm_device *dev)
3580 {
3581         struct drm_i915_private *dev_priv = dev->dev_private;
3582         struct drm_i915_fence_reg *reg, *avail;
3583         int i;
3584
3585         /* First try to find a free reg */
3586         avail = NULL;
3587         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3588                 reg = &dev_priv->fence_regs[i];
3589                 if (!reg->obj)
3590                         return reg;
3591
3592                 if (!reg->pin_count)
3593                         avail = reg;
3594         }
3595
3596         if (avail == NULL)
3597                 goto deadlock;
3598
3599         /* None available, try to steal one or wait for a user to finish */
3600         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3601                 if (reg->pin_count)
3602                         continue;
3603
3604                 return reg;
3605         }
3606
3607 deadlock:
3608         /* Wait for completion of pending flips which consume fences */
3609         if (intel_has_pending_fb_unpin(dev))
3610                 return ERR_PTR(-EAGAIN);
3611
3612         return ERR_PTR(-EDEADLK);
3613 }
3614
3615 /**
3616  * i915_gem_object_get_fence - set up fencing for an object
3617  * @obj: object to map through a fence reg
3618  *
3619  * When mapping objects through the GTT, userspace wants to be able to write
3620  * to them without having to worry about swizzling if the object is tiled.
3621  * This function walks the fence regs looking for a free one for @obj,
3622  * stealing one if it can't find any.
3623  *
3624  * It then sets up the reg based on the object's properties: address, pitch
3625  * and tiling format.
3626  *
3627  * For an untiled surface, this removes any existing fence.
3628  */
3629 int
3630 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3631 {
3632         struct drm_device *dev = obj->base.dev;
3633         struct drm_i915_private *dev_priv = dev->dev_private;
3634         bool enable = obj->tiling_mode != I915_TILING_NONE;
3635         struct drm_i915_fence_reg *reg;
3636         int ret;
3637
3638         /* Have we updated the tiling parameters upon the object and so
3639          * will need to serialise the write to the associated fence register?
3640          */
3641         if (obj->fence_dirty) {
3642                 ret = i915_gem_object_wait_fence(obj);
3643                 if (ret)
3644                         return ret;
3645         }
3646
3647         /* Just update our place in the LRU if our fence is getting reused. */
3648         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3649                 reg = &dev_priv->fence_regs[obj->fence_reg];
3650                 if (!obj->fence_dirty) {
3651                         list_move_tail(&reg->lru_list,
3652                                        &dev_priv->mm.fence_list);
3653                         return 0;
3654                 }
3655         } else if (enable) {
3656                 if (WARN_ON(!obj->map_and_fenceable))
3657                         return -EINVAL;
3658
3659                 reg = i915_find_fence_reg(dev);
3660                 if (IS_ERR(reg))
3661                         return PTR_ERR(reg);
3662
3663                 if (reg->obj) {
3664                         struct drm_i915_gem_object *old = reg->obj;
3665
3666                         ret = i915_gem_object_wait_fence(old);
3667                         if (ret)
3668                                 return ret;
3669
3670                         i915_gem_object_fence_lost(old);
3671                 }
3672         } else
3673                 return 0;
3674
3675         i915_gem_object_update_fence(obj, reg, enable);
3676
3677         return 0;
3678 }
3679
3680 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3681                                      unsigned long cache_level)
3682 {
3683         struct drm_mm_node *gtt_space = &vma->node;
3684         struct drm_mm_node *other;
3685
3686         /*
3687          * On some machines we have to be careful when putting differing types
3688          * of snoopable memory together to avoid the prefetcher crossing memory
3689          * domains and dying. During vm initialisation, we decide whether or not
3690          * these constraints apply and set the drm_mm.color_adjust
3691          * appropriately.
3692          */
3693         if (vma->vm->mm.color_adjust == NULL)
3694                 return true;
3695
3696         if (!drm_mm_node_allocated(gtt_space))
3697                 return true;
3698
3699         if (list_empty(&gtt_space->node_list))
3700                 return true;
3701
3702         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3703         if (other->allocated && !other->hole_follows && other->color != cache_level)
3704                 return false;
3705
3706         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3707         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3708                 return false;
3709
3710         return true;
3711 }
3712
3713 /**
3714  * Finds free space in the GTT aperture and binds the object or a view of it
3715  * there.
3716  */
3717 static struct i915_vma *
3718 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3719                            struct i915_address_space *vm,
3720                            const struct i915_ggtt_view *ggtt_view,
3721                            unsigned alignment,
3722                            uint64_t flags)
3723 {
3724         struct drm_device *dev = obj->base.dev;
3725         struct drm_i915_private *dev_priv = dev->dev_private;
3726         u32 size, fence_size, fence_alignment, unfenced_alignment;
3727         u64 start =
3728                 flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3729         u64 end =
3730                 flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
3731         struct i915_vma *vma;
3732         int ret;
3733
3734         if (i915_is_ggtt(vm)) {
3735                 u32 view_size;
3736
3737                 if (WARN_ON(!ggtt_view))
3738                         return ERR_PTR(-EINVAL);
3739
3740                 view_size = i915_ggtt_view_size(obj, ggtt_view);
3741
3742                 fence_size = i915_gem_get_gtt_size(dev,
3743                                                    view_size,
3744                                                    obj->tiling_mode);
3745                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3746                                                              view_size,
3747                                                              obj->tiling_mode,
3748                                                              true);
3749                 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3750                                                                 view_size,
3751                                                                 obj->tiling_mode,
3752                                                                 false);
3753                 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3754         } else {
3755                 fence_size = i915_gem_get_gtt_size(dev,
3756                                                    obj->base.size,
3757                                                    obj->tiling_mode);
3758                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3759                                                              obj->base.size,
3760                                                              obj->tiling_mode,
3761                                                              true);
3762                 unfenced_alignment =
3763                         i915_gem_get_gtt_alignment(dev,
3764                                                    obj->base.size,
3765                                                    obj->tiling_mode,
3766                                                    false);
3767                 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3768         }
3769
3770         if (alignment == 0)
3771                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3772                                                 unfenced_alignment;
3773         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3774                 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3775                           ggtt_view ? ggtt_view->type : 0,
3776                           alignment);
3777                 return ERR_PTR(-EINVAL);
3778         }
3779
3780         /* If binding the object/GGTT view requires more space than the entire
3781          * aperture has, reject it early before evicting everything in a vain
3782          * attempt to find space.
3783          */
3784         if (size > end) {
3785                 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%u > %s aperture=%llu\n",
3786                           ggtt_view ? ggtt_view->type : 0,
3787                           size,
3788                           flags & PIN_MAPPABLE ? "mappable" : "total",
3789                           end);
3790                 return ERR_PTR(-E2BIG);
3791         }
3792
3793         ret = i915_gem_object_get_pages(obj);
3794         if (ret)
3795                 return ERR_PTR(ret);
3796
3797         i915_gem_object_pin_pages(obj);
3798
3799         vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3800                           i915_gem_obj_lookup_or_create_vma(obj, vm);
3801
3802         if (IS_ERR(vma))
3803                 goto err_unpin;
3804
3805 search_free:
3806         ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3807                                                   size, alignment,
3808                                                   obj->cache_level,
3809                                                   start, end,
3810                                                   DRM_MM_SEARCH_DEFAULT,
3811                                                   DRM_MM_CREATE_DEFAULT);
3812         if (ret) {
3813                 ret = i915_gem_evict_something(dev, vm, size, alignment,
3814                                                obj->cache_level,
3815                                                start, end,
3816                                                flags);
3817                 if (ret == 0)
3818                         goto search_free;
3819
3820                 goto err_free_vma;
3821         }
3822         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3823                 ret = -EINVAL;
3824                 goto err_remove_node;
3825         }
3826
3827         trace_i915_vma_bind(vma, flags);
3828         ret = i915_vma_bind(vma, obj->cache_level, flags);
3829         if (ret)
3830                 goto err_remove_node;
3831
3832         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3833         list_add_tail(&vma->mm_list, &vm->inactive_list);
3834
3835         return vma;
3836
3837 err_remove_node:
3838         drm_mm_remove_node(&vma->node);
3839 err_free_vma:
3840         i915_gem_vma_destroy(vma);
3841         vma = ERR_PTR(ret);
3842 err_unpin:
3843         i915_gem_object_unpin_pages(obj);
3844         return vma;
3845 }
3846
3847 bool
3848 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3849                         bool force)
3850 {
3851         /* If we don't have a page list set up, then we're not pinned
3852          * to GPU, and we can ignore the cache flush because it'll happen
3853          * again at bind time.
3854          */
3855         if (obj->pages == NULL)
3856                 return false;
3857
3858         /*
3859          * Stolen memory is always coherent with the GPU as it is explicitly
3860          * marked as wc by the system, or the system is cache-coherent.
3861          */
3862         if (obj->stolen || obj->phys_handle)
3863                 return false;
3864
3865         /* If the GPU is snooping the contents of the CPU cache,
3866          * we do not need to manually clear the CPU cache lines.  However,
3867          * the caches are only snooped when the render cache is
3868          * flushed/invalidated.  As we always have to emit invalidations
3869          * and flushes when moving into and out of the RENDER domain, correct
3870          * snooping behaviour occurs naturally as the result of our domain
3871          * tracking.
3872          */
3873         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3874                 obj->cache_dirty = true;
3875                 return false;
3876         }
3877
3878         trace_i915_gem_object_clflush(obj);
3879         drm_clflush_sg(obj->pages);
3880         obj->cache_dirty = false;
3881
3882         return true;
3883 }
3884
3885 /** Flushes the GTT write domain for the object if it's dirty. */
3886 static void
3887 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3888 {
3889         uint32_t old_write_domain;
3890
3891         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3892                 return;
3893
3894         /* No actual flushing is required for the GTT write domain.  Writes
3895          * to it immediately go to main memory as far as we know, so there's
3896          * no chipset flush.  It also doesn't land in render cache.
3897          *
3898          * However, we do have to enforce the order so that all writes through
3899          * the GTT land before any writes to the device, such as updates to
3900          * the GATT itself.
3901          */
3902         wmb();
3903
3904         old_write_domain = obj->base.write_domain;
3905         obj->base.write_domain = 0;
3906
3907         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3908
3909         trace_i915_gem_object_change_domain(obj,
3910                                             obj->base.read_domains,
3911                                             old_write_domain);
3912 }
3913
3914 /** Flushes the CPU write domain for the object if it's dirty. */
3915 static void
3916 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3917 {
3918         uint32_t old_write_domain;
3919
3920         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3921                 return;
3922
3923         if (i915_gem_clflush_object(obj, obj->pin_display))
3924                 i915_gem_chipset_flush(obj->base.dev);
3925
3926         old_write_domain = obj->base.write_domain;
3927         obj->base.write_domain = 0;
3928
3929         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3930
3931         trace_i915_gem_object_change_domain(obj,
3932                                             obj->base.read_domains,
3933                                             old_write_domain);
3934 }
3935
3936 /**
3937  * Moves a single object to the GTT read, and possibly write domain.
3938  *
3939  * This function returns when the move is complete, including waiting on
3940  * flushes to occur.
3941  */
3942 int
3943 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3944 {
3945         uint32_t old_write_domain, old_read_domains;
3946         struct i915_vma *vma;
3947         int ret;
3948
3949         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3950                 return 0;
3951
3952         ret = i915_gem_object_wait_rendering(obj, !write);
3953         if (ret)
3954                 return ret;
3955
3956         /* Flush and acquire obj->pages so that we are coherent through
3957          * direct access in memory with previous cached writes through
3958          * shmemfs and that our cache domain tracking remains valid.
3959          * For example, if the obj->filp was moved to swap without us
3960          * being notified and releasing the pages, we would mistakenly
3961          * continue to assume that the obj remained out of the CPU cached
3962          * domain.
3963          */
3964         ret = i915_gem_object_get_pages(obj);
3965         if (ret)
3966                 return ret;
3967
3968         i915_gem_object_flush_cpu_write_domain(obj);
3969
3970         /* Serialise direct access to this object with the barriers for
3971          * coherent writes from the GPU, by effectively invalidating the
3972          * GTT domain upon first access.
3973          */
3974         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3975                 mb();
3976
3977         old_write_domain = obj->base.write_domain;
3978         old_read_domains = obj->base.read_domains;
3979
3980         /* It should now be out of any other write domains, and we can update
3981          * the domain values for our changes.
3982          */
3983         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3984         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3985         if (write) {
3986                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3987                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3988                 obj->dirty = 1;
3989         }
3990
3991         trace_i915_gem_object_change_domain(obj,
3992                                             old_read_domains,
3993                                             old_write_domain);
3994
3995         /* And bump the LRU for this access */
3996         vma = i915_gem_obj_to_ggtt(obj);
3997         if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3998                 list_move_tail(&vma->mm_list,
3999                                &to_i915(obj->base.dev)->gtt.base.inactive_list);
4000
4001         return 0;
4002 }
4003
4004 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
4005                                     enum i915_cache_level cache_level)
4006 {
4007         struct drm_device *dev = obj->base.dev;
4008         struct i915_vma *vma, *next;
4009         int ret;
4010
4011         if (obj->cache_level == cache_level)
4012                 return 0;
4013
4014         if (i915_gem_obj_is_pinned(obj)) {
4015                 DRM_DEBUG("can not change the cache level of pinned objects\n");
4016                 return -EBUSY;
4017         }
4018
4019         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4020                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
4021                         ret = i915_vma_unbind(vma);
4022                         if (ret)
4023                                 return ret;
4024                 }
4025         }
4026
4027         if (i915_gem_obj_bound_any(obj)) {
4028                 ret = i915_gem_object_wait_rendering(obj, false);
4029                 if (ret)
4030                         return ret;
4031
4032                 i915_gem_object_finish_gtt(obj);
4033
4034                 /* Before SandyBridge, you could not use tiling or fence
4035                  * registers with snooped memory, so relinquish any fences
4036                  * currently pointing to our region in the aperture.
4037                  */
4038                 if (INTEL_INFO(dev)->gen < 6) {
4039                         ret = i915_gem_object_put_fence(obj);
4040                         if (ret)
4041                                 return ret;
4042                 }
4043
4044                 list_for_each_entry(vma, &obj->vma_list, vma_link)
4045                         if (drm_mm_node_allocated(&vma->node)) {
4046                                 ret = i915_vma_bind(vma, cache_level,
4047                                                     PIN_UPDATE);
4048                                 if (ret)
4049                                         return ret;
4050                         }
4051         }
4052
4053         list_for_each_entry(vma, &obj->vma_list, vma_link)
4054                 vma->node.color = cache_level;
4055         obj->cache_level = cache_level;
4056
4057         if (obj->cache_dirty &&
4058             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
4059             cpu_write_needs_clflush(obj)) {
4060                 if (i915_gem_clflush_object(obj, true))
4061                         i915_gem_chipset_flush(obj->base.dev);
4062         }
4063
4064         return 0;
4065 }
4066
4067 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
4068                                struct drm_file *file)
4069 {
4070         struct drm_i915_gem_caching *args = data;
4071         struct drm_i915_gem_object *obj;
4072
4073         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4074         if (&obj->base == NULL)
4075                 return -ENOENT;
4076
4077         switch (obj->cache_level) {
4078         case I915_CACHE_LLC:
4079         case I915_CACHE_L3_LLC:
4080                 args->caching = I915_CACHING_CACHED;
4081                 break;
4082
4083         case I915_CACHE_WT:
4084                 args->caching = I915_CACHING_DISPLAY;
4085                 break;
4086
4087         default:
4088                 args->caching = I915_CACHING_NONE;
4089                 break;
4090         }
4091
4092         drm_gem_object_unreference_unlocked(&obj->base);
4093         return 0;
4094 }
4095
4096 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
4097                                struct drm_file *file)
4098 {
4099         struct drm_i915_gem_caching *args = data;
4100         struct drm_i915_gem_object *obj;
4101         enum i915_cache_level level;
4102         int ret;
4103
4104         switch (args->caching) {
4105         case I915_CACHING_NONE:
4106                 level = I915_CACHE_NONE;
4107                 break;
4108         case I915_CACHING_CACHED:
4109                 level = I915_CACHE_LLC;
4110                 break;
4111         case I915_CACHING_DISPLAY:
4112                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
4113                 break;
4114         default:
4115                 return -EINVAL;
4116         }
4117
4118         ret = i915_mutex_lock_interruptible(dev);
4119         if (ret)
4120                 return ret;
4121
4122         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4123         if (&obj->base == NULL) {
4124                 ret = -ENOENT;
4125                 goto unlock;
4126         }
4127
4128         ret = i915_gem_object_set_cache_level(obj, level);
4129
4130         drm_gem_object_unreference(&obj->base);
4131 unlock:
4132         mutex_unlock(&dev->struct_mutex);
4133         return ret;
4134 }
4135
4136 /*
4137  * Prepare buffer for display plane (scanout, cursors, etc).
4138  * Can be called from an uninterruptible phase (modesetting) and allows
4139  * any flushes to be pipelined (for pageflips).
4140  */
4141 int
4142 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4143                                      u32 alignment,
4144                                      struct intel_engine_cs *pipelined,
4145                                      struct drm_i915_gem_request **pipelined_request,
4146                                      const struct i915_ggtt_view *view)
4147 {
4148         u32 old_read_domains, old_write_domain;
4149         int ret;
4150
4151         ret = i915_gem_object_sync(obj, pipelined, pipelined_request);
4152         if (ret)
4153                 return ret;
4154
4155         /* Mark the pin_display early so that we account for the
4156          * display coherency whilst setting up the cache domains.
4157          */
4158         obj->pin_display++;
4159
4160         /* The display engine is not coherent with the LLC cache on gen6.  As
4161          * a result, we make sure that the pinning that is about to occur is
4162          * done with uncached PTEs. This is lowest common denominator for all
4163          * chipsets.
4164          *
4165          * However for gen6+, we could do better by using the GFDT bit instead
4166          * of uncaching, which would allow us to flush all the LLC-cached data
4167          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4168          */
4169         ret = i915_gem_object_set_cache_level(obj,
4170                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
4171         if (ret)
4172                 goto err_unpin_display;
4173
4174         /* As the user may map the buffer once pinned in the display plane
4175          * (e.g. libkms for the bootup splash), we have to ensure that we
4176          * always use map_and_fenceable for all scanout buffers.
4177          */
4178         ret = i915_gem_object_ggtt_pin(obj, view, alignment,
4179                                        view->type == I915_GGTT_VIEW_NORMAL ?
4180                                        PIN_MAPPABLE : 0);
4181         if (ret)
4182                 goto err_unpin_display;
4183
4184         i915_gem_object_flush_cpu_write_domain(obj);
4185
4186         old_write_domain = obj->base.write_domain;
4187         old_read_domains = obj->base.read_domains;
4188
4189         /* It should now be out of any other write domains, and we can update
4190          * the domain values for our changes.
4191          */
4192         obj->base.write_domain = 0;
4193         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4194
4195         trace_i915_gem_object_change_domain(obj,
4196                                             old_read_domains,
4197                                             old_write_domain);
4198
4199         return 0;
4200
4201 err_unpin_display:
4202         obj->pin_display--;
4203         return ret;
4204 }
4205
4206 void
4207 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
4208                                          const struct i915_ggtt_view *view)
4209 {
4210         if (WARN_ON(obj->pin_display == 0))
4211                 return;
4212
4213         i915_gem_object_ggtt_unpin_view(obj, view);
4214
4215         obj->pin_display--;
4216 }
4217
4218 /**
4219  * Moves a single object to the CPU read, and possibly write domain.
4220  *
4221  * This function returns when the move is complete, including waiting on
4222  * flushes to occur.
4223  */
4224 int
4225 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4226 {
4227         uint32_t old_write_domain, old_read_domains;
4228         int ret;
4229
4230         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4231                 return 0;
4232
4233         ret = i915_gem_object_wait_rendering(obj, !write);
4234         if (ret)
4235                 return ret;
4236
4237         i915_gem_object_flush_gtt_write_domain(obj);
4238
4239         old_write_domain = obj->base.write_domain;
4240         old_read_domains = obj->base.read_domains;
4241
4242         /* Flush the CPU cache if it's still invalid. */
4243         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4244                 i915_gem_clflush_object(obj, false);
4245
4246                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4247         }
4248
4249         /* It should now be out of any other write domains, and we can update
4250          * the domain values for our changes.
4251          */
4252         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4253
4254         /* If we're writing through the CPU, then the GPU read domains will
4255          * need to be invalidated at next use.
4256          */
4257         if (write) {
4258                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4259                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4260         }
4261
4262         trace_i915_gem_object_change_domain(obj,
4263                                             old_read_domains,
4264                                             old_write_domain);
4265
4266         return 0;
4267 }
4268
4269 /* Throttle our rendering by waiting until the ring has completed our requests
4270  * emitted over 20 msec ago.
4271  *
4272  * Note that if we were to use the current jiffies each time around the loop,
4273  * we wouldn't escape the function with any frames outstanding if the time to
4274  * render a frame was over 20ms.
4275  *
4276  * This should get us reasonable parallelism between CPU and GPU but also
4277  * relatively low latency when blocking on a particular request to finish.
4278  */
4279 static int
4280 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4281 {
4282         struct drm_i915_private *dev_priv = dev->dev_private;
4283         struct drm_i915_file_private *file_priv = file->driver_priv;
4284         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4285         struct drm_i915_gem_request *request, *target = NULL;
4286         unsigned reset_counter;
4287         int ret;
4288
4289         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4290         if (ret)
4291                 return ret;
4292
4293         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
4294         if (ret)
4295                 return ret;
4296
4297         spin_lock(&file_priv->mm.lock);
4298         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4299                 if (time_after_eq(request->emitted_jiffies, recent_enough))
4300                         break;
4301
4302                 /*
4303                  * Note that the request might not have been submitted yet.
4304                  * In which case emitted_jiffies will be zero.
4305                  */
4306                 if (!request->emitted_jiffies)
4307                         continue;
4308
4309                 target = request;
4310         }
4311         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
4312         if (target)
4313                 i915_gem_request_reference(target);
4314         spin_unlock(&file_priv->mm.lock);
4315
4316         if (target == NULL)
4317                 return 0;
4318
4319         ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
4320         if (ret == 0)
4321                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4322
4323         i915_gem_request_unreference__unlocked(target);
4324
4325         return ret;
4326 }
4327
4328 static bool
4329 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4330 {
4331         struct drm_i915_gem_object *obj = vma->obj;
4332
4333         if (alignment &&
4334             vma->node.start & (alignment - 1))
4335                 return true;
4336
4337         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4338                 return true;
4339
4340         if (flags & PIN_OFFSET_BIAS &&
4341             vma->node.start < (flags & PIN_OFFSET_MASK))
4342                 return true;
4343
4344         return false;
4345 }
4346
4347 static int
4348 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
4349                        struct i915_address_space *vm,
4350                        const struct i915_ggtt_view *ggtt_view,
4351                        uint32_t alignment,
4352                        uint64_t flags)
4353 {
4354         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4355         struct i915_vma *vma;
4356         unsigned bound;
4357         int ret;
4358
4359         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4360                 return -ENODEV;
4361
4362         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4363                 return -EINVAL;
4364
4365         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4366                 return -EINVAL;
4367
4368         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
4369                 return -EINVAL;
4370
4371         vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
4372                           i915_gem_obj_to_vma(obj, vm);
4373
4374         if (IS_ERR(vma))
4375                 return PTR_ERR(vma);
4376
4377         if (vma) {
4378                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4379                         return -EBUSY;
4380
4381                 if (i915_vma_misplaced(vma, alignment, flags)) {
4382                         unsigned long offset;
4383                         offset = ggtt_view ? i915_gem_obj_ggtt_offset_view(obj, ggtt_view) :
4384                                              i915_gem_obj_offset(obj, vm);
4385                         WARN(vma->pin_count,
4386                              "bo is already pinned in %s with incorrect alignment:"
4387                              " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
4388                              " obj->map_and_fenceable=%d\n",
4389                              ggtt_view ? "ggtt" : "ppgtt",
4390                              offset,
4391                              alignment,
4392                              !!(flags & PIN_MAPPABLE),
4393                              obj->map_and_fenceable);
4394                         ret = i915_vma_unbind(vma);
4395                         if (ret)
4396                                 return ret;
4397
4398                         vma = NULL;
4399                 }
4400         }
4401
4402         bound = vma ? vma->bound : 0;
4403         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4404                 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
4405                                                  flags);
4406                 if (IS_ERR(vma))
4407                         return PTR_ERR(vma);
4408         } else {
4409                 ret = i915_vma_bind(vma, obj->cache_level, flags);
4410                 if (ret)
4411                         return ret;
4412         }
4413
4414         if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
4415             (bound ^ vma->bound) & GLOBAL_BIND) {
4416                 bool mappable, fenceable;
4417                 u32 fence_size, fence_alignment;
4418
4419                 fence_size = i915_gem_get_gtt_size(obj->base.dev,
4420                                                    obj->base.size,
4421                                                    obj->tiling_mode);
4422                 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4423                                                              obj->base.size,
4424                                                              obj->tiling_mode,
4425                                                              true);
4426
4427                 fenceable = (vma->node.size == fence_size &&
4428                              (vma->node.start & (fence_alignment - 1)) == 0);
4429
4430                 mappable = (vma->node.start + fence_size <=
4431                             dev_priv->gtt.mappable_end);
4432
4433                 obj->map_and_fenceable = mappable && fenceable;
4434
4435                 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4436         }
4437
4438         vma->pin_count++;
4439         return 0;
4440 }
4441
4442 int
4443 i915_gem_object_pin(struct drm_i915_gem_object *obj,
4444                     struct i915_address_space *vm,
4445                     uint32_t alignment,
4446                     uint64_t flags)
4447 {
4448         return i915_gem_object_do_pin(obj, vm,
4449                                       i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
4450                                       alignment, flags);
4451 }
4452
4453 int
4454 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4455                          const struct i915_ggtt_view *view,
4456                          uint32_t alignment,
4457                          uint64_t flags)
4458 {
4459         if (WARN_ONCE(!view, "no view specified"))
4460                 return -EINVAL;
4461
4462         return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
4463                                       alignment, flags | PIN_GLOBAL);
4464 }
4465
4466 void
4467 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4468                                 const struct i915_ggtt_view *view)
4469 {
4470         struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4471
4472         BUG_ON(!vma);
4473         WARN_ON(vma->pin_count == 0);
4474         WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
4475
4476         --vma->pin_count;
4477 }
4478
4479 bool
4480 i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
4481 {
4482         if (obj->fence_reg != I915_FENCE_REG_NONE) {
4483                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4484                 struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj);
4485
4486                 WARN_ON(!ggtt_vma ||
4487                         dev_priv->fence_regs[obj->fence_reg].pin_count >
4488                         ggtt_vma->pin_count);
4489                 dev_priv->fence_regs[obj->fence_reg].pin_count++;
4490                 return true;
4491         } else
4492                 return false;
4493 }
4494
4495 void
4496 i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
4497 {
4498         if (obj->fence_reg != I915_FENCE_REG_NONE) {
4499                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4500                 WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
4501                 dev_priv->fence_regs[obj->fence_reg].pin_count--;
4502         }
4503 }
4504
4505 int
4506 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4507                     struct drm_file *file)
4508 {
4509         struct drm_i915_gem_busy *args = data;
4510         struct drm_i915_gem_object *obj;
4511         int ret;
4512
4513         ret = i915_mutex_lock_interruptible(dev);
4514         if (ret)
4515                 return ret;
4516
4517         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4518         if (&obj->base == NULL) {
4519                 ret = -ENOENT;
4520                 goto unlock;
4521         }
4522
4523         /* Count all active objects as busy, even if they are currently not used
4524          * by the gpu. Users of this interface expect objects to eventually
4525          * become non-busy without any further actions, therefore emit any
4526          * necessary flushes here.
4527          */
4528         ret = i915_gem_object_flush_active(obj);
4529         if (ret)
4530                 goto unref;
4531
4532         BUILD_BUG_ON(I915_NUM_RINGS > 16);
4533         args->busy = obj->active << 16;
4534         if (obj->last_write_req)
4535                 args->busy |= obj->last_write_req->ring->id;
4536
4537 unref:
4538         drm_gem_object_unreference(&obj->base);
4539 unlock:
4540         mutex_unlock(&dev->struct_mutex);
4541         return ret;
4542 }
4543
4544 int
4545 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4546                         struct drm_file *file_priv)
4547 {
4548         return i915_gem_ring_throttle(dev, file_priv);
4549 }
4550
4551 int
4552 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4553                        struct drm_file *file_priv)
4554 {
4555         struct drm_i915_private *dev_priv = dev->dev_private;
4556         struct drm_i915_gem_madvise *args = data;
4557         struct drm_i915_gem_object *obj;
4558         int ret;
4559
4560         switch (args->madv) {
4561         case I915_MADV_DONTNEED:
4562         case I915_MADV_WILLNEED:
4563             break;
4564         default:
4565             return -EINVAL;
4566         }
4567
4568         ret = i915_mutex_lock_interruptible(dev);
4569         if (ret)
4570                 return ret;
4571
4572         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4573         if (&obj->base == NULL) {
4574                 ret = -ENOENT;
4575                 goto unlock;
4576         }
4577
4578         if (i915_gem_obj_is_pinned(obj)) {
4579                 ret = -EINVAL;
4580                 goto out;
4581         }
4582
4583         if (obj->pages &&
4584             obj->tiling_mode != I915_TILING_NONE &&
4585             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4586                 if (obj->madv == I915_MADV_WILLNEED)
4587                         i915_gem_object_unpin_pages(obj);
4588                 if (args->madv == I915_MADV_WILLNEED)
4589                         i915_gem_object_pin_pages(obj);
4590         }
4591
4592         if (obj->madv != __I915_MADV_PURGED)
4593                 obj->madv = args->madv;
4594
4595         /* if the object is no longer attached, discard its backing storage */
4596         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4597                 i915_gem_object_truncate(obj);
4598
4599         args->retained = obj->madv != __I915_MADV_PURGED;
4600
4601 out:
4602         drm_gem_object_unreference(&obj->base);
4603 unlock:
4604         mutex_unlock(&dev->struct_mutex);
4605         return ret;
4606 }
4607
4608 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4609                           const struct drm_i915_gem_object_ops *ops)
4610 {
4611         int i;
4612
4613         INIT_LIST_HEAD(&obj->global_list);
4614         for (i = 0; i < I915_NUM_RINGS; i++)
4615                 INIT_LIST_HEAD(&obj->ring_list[i]);
4616         INIT_LIST_HEAD(&obj->obj_exec_link);
4617         INIT_LIST_HEAD(&obj->vma_list);
4618         INIT_LIST_HEAD(&obj->batch_pool_link);
4619
4620         obj->ops = ops;
4621
4622         obj->fence_reg = I915_FENCE_REG_NONE;
4623         obj->madv = I915_MADV_WILLNEED;
4624
4625         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4626 }
4627
4628 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4629         .get_pages = i915_gem_object_get_pages_gtt,
4630         .put_pages = i915_gem_object_put_pages_gtt,
4631 };
4632
4633 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4634                                                   size_t size)
4635 {
4636         struct drm_i915_gem_object *obj;
4637         struct address_space *mapping;
4638         gfp_t mask;
4639
4640         obj = i915_gem_object_alloc(dev);
4641         if (obj == NULL)
4642                 return NULL;
4643
4644         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4645                 i915_gem_object_free(obj);
4646                 return NULL;
4647         }
4648
4649         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4650         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4651                 /* 965gm cannot relocate objects above 4GiB. */
4652                 mask &= ~__GFP_HIGHMEM;
4653                 mask |= __GFP_DMA32;
4654         }
4655
4656         mapping = file_inode(obj->base.filp)->i_mapping;
4657         mapping_set_gfp_mask(mapping, mask);
4658
4659         i915_gem_object_init(obj, &i915_gem_object_ops);
4660
4661         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4662         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4663
4664         if (HAS_LLC(dev)) {
4665                 /* On some devices, we can have the GPU use the LLC (the CPU
4666                  * cache) for about a 10% performance improvement
4667                  * compared to uncached.  Graphics requests other than
4668                  * display scanout are coherent with the CPU in
4669                  * accessing this cache.  This means in this mode we
4670                  * don't need to clflush on the CPU side, and on the
4671                  * GPU side we only need to flush internal caches to
4672                  * get data visible to the CPU.
4673                  *
4674                  * However, we maintain the display planes as UC, and so
4675                  * need to rebind when first used as such.
4676                  */
4677                 obj->cache_level = I915_CACHE_LLC;
4678         } else
4679                 obj->cache_level = I915_CACHE_NONE;
4680
4681         trace_i915_gem_object_create(obj);
4682
4683         return obj;
4684 }
4685
4686 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4687 {
4688         /* If we are the last user of the backing storage (be it shmemfs
4689          * pages or stolen etc), we know that the pages are going to be
4690          * immediately released. In this case, we can then skip copying
4691          * back the contents from the GPU.
4692          */
4693
4694         if (obj->madv != I915_MADV_WILLNEED)
4695                 return false;
4696
4697         if (obj->base.filp == NULL)
4698                 return true;
4699
4700         /* At first glance, this looks racy, but then again so would be
4701          * userspace racing mmap against close. However, the first external
4702          * reference to the filp can only be obtained through the
4703          * i915_gem_mmap_ioctl() which safeguards us against the user
4704          * acquiring such a reference whilst we are in the middle of
4705          * freeing the object.
4706          */
4707         return atomic_long_read(&obj->base.filp->f_count) == 1;
4708 }
4709
4710 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4711 {
4712         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4713         struct drm_device *dev = obj->base.dev;
4714         struct drm_i915_private *dev_priv = dev->dev_private;
4715         struct i915_vma *vma, *next;
4716
4717         intel_runtime_pm_get(dev_priv);
4718
4719         trace_i915_gem_object_destroy(obj);
4720
4721         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4722                 int ret;
4723
4724                 vma->pin_count = 0;
4725                 ret = i915_vma_unbind(vma);
4726                 if (WARN_ON(ret == -ERESTARTSYS)) {
4727                         bool was_interruptible;
4728
4729                         was_interruptible = dev_priv->mm.interruptible;
4730                         dev_priv->mm.interruptible = false;
4731
4732                         WARN_ON(i915_vma_unbind(vma));
4733
4734                         dev_priv->mm.interruptible = was_interruptible;
4735                 }
4736         }
4737
4738         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4739          * before progressing. */
4740         if (obj->stolen)
4741                 i915_gem_object_unpin_pages(obj);
4742
4743         WARN_ON(obj->frontbuffer_bits);
4744
4745         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4746             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4747             obj->tiling_mode != I915_TILING_NONE)
4748                 i915_gem_object_unpin_pages(obj);
4749
4750         if (WARN_ON(obj->pages_pin_count))
4751                 obj->pages_pin_count = 0;
4752         if (discard_backing_storage(obj))
4753                 obj->madv = I915_MADV_DONTNEED;
4754         i915_gem_object_put_pages(obj);
4755         i915_gem_object_free_mmap_offset(obj);
4756
4757         BUG_ON(obj->pages);
4758
4759         if (obj->base.import_attach)
4760                 drm_prime_gem_destroy(&obj->base, NULL);
4761
4762         if (obj->ops->release)
4763                 obj->ops->release(obj);
4764
4765         drm_gem_object_release(&obj->base);
4766         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4767
4768         kfree(obj->bit_17);
4769         i915_gem_object_free(obj);
4770
4771         intel_runtime_pm_put(dev_priv);
4772 }
4773
4774 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4775                                      struct i915_address_space *vm)
4776 {
4777         struct i915_vma *vma;
4778         list_for_each_entry(vma, &obj->vma_list, vma_link) {
4779                 if (i915_is_ggtt(vma->vm) &&
4780                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4781                         continue;
4782                 if (vma->vm == vm)
4783                         return vma;
4784         }
4785         return NULL;
4786 }
4787
4788 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4789                                            const struct i915_ggtt_view *view)
4790 {
4791         struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
4792         struct i915_vma *vma;
4793
4794         if (WARN_ONCE(!view, "no view specified"))
4795                 return ERR_PTR(-EINVAL);
4796
4797         list_for_each_entry(vma, &obj->vma_list, vma_link)
4798                 if (vma->vm == ggtt &&
4799                     i915_ggtt_view_equal(&vma->ggtt_view, view))
4800                         return vma;
4801         return NULL;
4802 }
4803
4804 void i915_gem_vma_destroy(struct i915_vma *vma)
4805 {
4806         struct i915_address_space *vm = NULL;
4807         WARN_ON(vma->node.allocated);
4808
4809         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4810         if (!list_empty(&vma->exec_list))
4811                 return;
4812
4813         vm = vma->vm;
4814
4815         if (!i915_is_ggtt(vm))
4816                 i915_ppgtt_put(i915_vm_to_ppgtt(vm));
4817
4818         list_del(&vma->vma_link);
4819
4820         kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4821 }
4822
4823 static void
4824 i915_gem_stop_ringbuffers(struct drm_device *dev)
4825 {
4826         struct drm_i915_private *dev_priv = dev->dev_private;
4827         struct intel_engine_cs *ring;
4828         int i;
4829
4830         for_each_ring(ring, dev_priv, i)
4831                 dev_priv->gt.stop_ring(ring);
4832 }
4833
4834 int
4835 i915_gem_suspend(struct drm_device *dev)
4836 {
4837         struct drm_i915_private *dev_priv = dev->dev_private;
4838         int ret = 0;
4839
4840         mutex_lock(&dev->struct_mutex);
4841         ret = i915_gpu_idle(dev);
4842         if (ret)
4843                 goto err;
4844
4845         i915_gem_retire_requests(dev);
4846
4847         i915_gem_stop_ringbuffers(dev);
4848         mutex_unlock(&dev->struct_mutex);
4849
4850         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4851         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4852         flush_delayed_work(&dev_priv->mm.idle_work);
4853
4854         /* Assert that we sucessfully flushed all the work and
4855          * reset the GPU back to its idle, low power state.
4856          */
4857         WARN_ON(dev_priv->mm.busy);
4858
4859         return 0;
4860
4861 err:
4862         mutex_unlock(&dev->struct_mutex);
4863         return ret;
4864 }
4865
4866 int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
4867 {
4868         struct intel_engine_cs *ring = req->ring;
4869         struct drm_device *dev = ring->dev;
4870         struct drm_i915_private *dev_priv = dev->dev_private;
4871         u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
4872         u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4873         int i, ret;
4874
4875         if (!HAS_L3_DPF(dev) || !remap_info)
4876                 return 0;
4877
4878         ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
4879         if (ret)
4880                 return ret;
4881
4882         /*
4883          * Note: We do not worry about the concurrent register cacheline hang
4884          * here because no other code should access these registers other than
4885          * at initialization time.
4886          */
4887         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4888                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4889                 intel_ring_emit(ring, reg_base + i);
4890                 intel_ring_emit(ring, remap_info[i/4]);
4891         }
4892
4893         intel_ring_advance(ring);
4894
4895         return ret;
4896 }
4897
4898 void i915_gem_init_swizzling(struct drm_device *dev)
4899 {
4900         struct drm_i915_private *dev_priv = dev->dev_private;
4901
4902         if (INTEL_INFO(dev)->gen < 5 ||
4903             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4904                 return;
4905
4906         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4907                                  DISP_TILE_SURFACE_SWIZZLING);
4908
4909         if (IS_GEN5(dev))
4910                 return;
4911
4912         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4913         if (IS_GEN6(dev))
4914                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4915         else if (IS_GEN7(dev))
4916                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4917         else if (IS_GEN8(dev))
4918                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4919         else
4920                 BUG();
4921 }
4922
4923 static bool
4924 intel_enable_blt(struct drm_device *dev)
4925 {
4926         if (!HAS_BLT(dev))
4927                 return false;
4928
4929         /* The blitter was dysfunctional on early prototypes */
4930         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
4931                 DRM_INFO("BLT not supported on this pre-production hardware;"
4932                          " graphics performance will be degraded.\n");
4933                 return false;
4934         }
4935
4936         return true;
4937 }
4938
4939 static void init_unused_ring(struct drm_device *dev, u32 base)
4940 {
4941         struct drm_i915_private *dev_priv = dev->dev_private;
4942
4943         I915_WRITE(RING_CTL(base), 0);
4944         I915_WRITE(RING_HEAD(base), 0);
4945         I915_WRITE(RING_TAIL(base), 0);
4946         I915_WRITE(RING_START(base), 0);
4947 }
4948
4949 static void init_unused_rings(struct drm_device *dev)
4950 {
4951         if (IS_I830(dev)) {
4952                 init_unused_ring(dev, PRB1_BASE);
4953                 init_unused_ring(dev, SRB0_BASE);
4954                 init_unused_ring(dev, SRB1_BASE);
4955                 init_unused_ring(dev, SRB2_BASE);
4956                 init_unused_ring(dev, SRB3_BASE);
4957         } else if (IS_GEN2(dev)) {
4958                 init_unused_ring(dev, SRB0_BASE);
4959                 init_unused_ring(dev, SRB1_BASE);
4960         } else if (IS_GEN3(dev)) {
4961                 init_unused_ring(dev, PRB1_BASE);
4962                 init_unused_ring(dev, PRB2_BASE);
4963         }
4964 }
4965
4966 int i915_gem_init_rings(struct drm_device *dev)
4967 {
4968         struct drm_i915_private *dev_priv = dev->dev_private;
4969         int ret;
4970
4971         ret = intel_init_render_ring_buffer(dev);
4972         if (ret)
4973                 return ret;
4974
4975         if (HAS_BSD(dev)) {
4976                 ret = intel_init_bsd_ring_buffer(dev);
4977                 if (ret)
4978                         goto cleanup_render_ring;
4979         }
4980
4981         if (intel_enable_blt(dev)) {
4982                 ret = intel_init_blt_ring_buffer(dev);
4983                 if (ret)
4984                         goto cleanup_bsd_ring;
4985         }
4986
4987         if (HAS_VEBOX(dev)) {
4988                 ret = intel_init_vebox_ring_buffer(dev);
4989                 if (ret)
4990                         goto cleanup_blt_ring;
4991         }
4992
4993         if (HAS_BSD2(dev)) {
4994                 ret = intel_init_bsd2_ring_buffer(dev);
4995                 if (ret)
4996                         goto cleanup_vebox_ring;
4997         }
4998
4999         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
5000         if (ret)
5001                 goto cleanup_bsd2_ring;
5002
5003         return 0;
5004
5005 cleanup_bsd2_ring:
5006         intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]);
5007 cleanup_vebox_ring:
5008         intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
5009 cleanup_blt_ring:
5010         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
5011 cleanup_bsd_ring:
5012         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
5013 cleanup_render_ring:
5014         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
5015
5016         return ret;
5017 }
5018
5019 int
5020 i915_gem_init_hw(struct drm_device *dev)
5021 {
5022         struct drm_i915_private *dev_priv = dev->dev_private;
5023         struct intel_engine_cs *ring;
5024         int ret, i, j;
5025
5026         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
5027                 return -EIO;
5028
5029         /* Double layer security blanket, see i915_gem_init() */
5030         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5031
5032         if (dev_priv->ellc_size)
5033                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
5034
5035         if (IS_HASWELL(dev))
5036                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
5037                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
5038
5039         if (HAS_PCH_NOP(dev)) {
5040                 if (IS_IVYBRIDGE(dev)) {
5041                         u32 temp = I915_READ(GEN7_MSG_CTL);
5042                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
5043                         I915_WRITE(GEN7_MSG_CTL, temp);
5044                 } else if (INTEL_INFO(dev)->gen >= 7) {
5045                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
5046                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
5047                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
5048                 }
5049         }
5050
5051         i915_gem_init_swizzling(dev);
5052
5053         /*
5054          * At least 830 can leave some of the unused rings
5055          * "active" (ie. head != tail) after resume which
5056          * will prevent c3 entry. Makes sure all unused rings
5057          * are totally idle.
5058          */
5059         init_unused_rings(dev);
5060
5061         BUG_ON(!dev_priv->ring[RCS].default_context);
5062
5063         ret = i915_ppgtt_init_hw(dev);
5064         if (ret) {
5065                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
5066                 goto out;
5067         }
5068
5069         /* Need to do basic initialisation of all rings first: */
5070         for_each_ring(ring, dev_priv, i) {
5071                 ret = ring->init_hw(ring);
5072                 if (ret)
5073                         goto out;
5074         }
5075
5076         /* Now it is safe to go back round and do everything else: */
5077         for_each_ring(ring, dev_priv, i) {
5078                 struct drm_i915_gem_request *req;
5079
5080                 WARN_ON(!ring->default_context);
5081
5082                 ret = i915_gem_request_alloc(ring, ring->default_context, &req);
5083                 if (ret) {
5084                         i915_gem_cleanup_ringbuffer(dev);
5085                         goto out;
5086                 }
5087
5088                 if (ring->id == RCS) {
5089                         for (j = 0; j < NUM_L3_SLICES(dev); j++)
5090                                 i915_gem_l3_remap(req, j);
5091                 }
5092
5093                 ret = i915_ppgtt_init_ring(req);
5094                 if (ret && ret != -EIO) {
5095                         DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
5096                         i915_gem_request_cancel(req);
5097                         i915_gem_cleanup_ringbuffer(dev);
5098                         goto out;
5099                 }
5100
5101                 ret = i915_gem_context_enable(req);
5102                 if (ret && ret != -EIO) {
5103                         DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
5104                         i915_gem_request_cancel(req);
5105                         i915_gem_cleanup_ringbuffer(dev);
5106                         goto out;
5107                 }
5108
5109                 i915_add_request_no_flush(req);
5110         }
5111
5112 out:
5113         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5114         return ret;
5115 }
5116
5117 int i915_gem_init(struct drm_device *dev)
5118 {
5119         struct drm_i915_private *dev_priv = dev->dev_private;
5120         int ret;
5121
5122         i915.enable_execlists = intel_sanitize_enable_execlists(dev,
5123                         i915.enable_execlists);
5124
5125         mutex_lock(&dev->struct_mutex);
5126
5127         if (IS_VALLEYVIEW(dev)) {
5128                 /* VLVA0 (potential hack), BIOS isn't actually waking us */
5129                 I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
5130                 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
5131                               VLV_GTLC_ALLOWWAKEACK), 10))
5132                         DRM_DEBUG_DRIVER("allow wake ack timed out\n");
5133         }
5134
5135         if (!i915.enable_execlists) {
5136                 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
5137                 dev_priv->gt.init_rings = i915_gem_init_rings;
5138                 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
5139                 dev_priv->gt.stop_ring = intel_stop_ring_buffer;
5140         } else {
5141                 dev_priv->gt.execbuf_submit = intel_execlists_submission;
5142                 dev_priv->gt.init_rings = intel_logical_rings_init;
5143                 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
5144                 dev_priv->gt.stop_ring = intel_logical_ring_stop;
5145         }
5146
5147         /* This is just a security blanket to placate dragons.
5148          * On some systems, we very sporadically observe that the first TLBs
5149          * used by the CS may be stale, despite us poking the TLB reset. If
5150          * we hold the forcewake during initialisation these problems
5151          * just magically go away.
5152          */
5153         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5154
5155         ret = i915_gem_init_userptr(dev);
5156         if (ret)
5157                 goto out_unlock;
5158
5159         i915_gem_init_global_gtt(dev);
5160
5161         ret = i915_gem_context_init(dev);
5162         if (ret)
5163                 goto out_unlock;
5164
5165         ret = dev_priv->gt.init_rings(dev);
5166         if (ret)
5167                 goto out_unlock;
5168
5169         ret = i915_gem_init_hw(dev);
5170         if (ret == -EIO) {
5171                 /* Allow ring initialisation to fail by marking the GPU as
5172                  * wedged. But we only want to do this where the GPU is angry,
5173                  * for all other failure, such as an allocation failure, bail.
5174                  */
5175                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
5176                 atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
5177                 ret = 0;
5178         }
5179
5180 out_unlock:
5181         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5182         mutex_unlock(&dev->struct_mutex);
5183
5184         return ret;
5185 }
5186
5187 void
5188 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
5189 {
5190         struct drm_i915_private *dev_priv = dev->dev_private;
5191         struct intel_engine_cs *ring;
5192         int i;
5193
5194         for_each_ring(ring, dev_priv, i)
5195                 dev_priv->gt.cleanup_ring(ring);
5196
5197     if (i915.enable_execlists)
5198             /*
5199              * Neither the BIOS, ourselves or any other kernel
5200              * expects the system to be in execlists mode on startup,
5201              * so we need to reset the GPU back to legacy mode.
5202              */
5203             intel_gpu_reset(dev);
5204 }
5205
5206 static void
5207 init_ring_lists(struct intel_engine_cs *ring)
5208 {
5209         INIT_LIST_HEAD(&ring->active_list);
5210         INIT_LIST_HEAD(&ring->request_list);
5211 }
5212
5213 void i915_init_vm(struct drm_i915_private *dev_priv,
5214                   struct i915_address_space *vm)
5215 {
5216         if (!i915_is_ggtt(vm))
5217                 drm_mm_init(&vm->mm, vm->start, vm->total);
5218         vm->dev = dev_priv->dev;
5219         INIT_LIST_HEAD(&vm->active_list);
5220         INIT_LIST_HEAD(&vm->inactive_list);
5221         INIT_LIST_HEAD(&vm->global_link);
5222         list_add_tail(&vm->global_link, &dev_priv->vm_list);
5223 }
5224
5225 void
5226 i915_gem_load(struct drm_device *dev)
5227 {
5228         struct drm_i915_private *dev_priv = dev->dev_private;
5229         int i;
5230
5231         dev_priv->objects =
5232                 kmem_cache_create("i915_gem_object",
5233                                   sizeof(struct drm_i915_gem_object), 0,
5234                                   SLAB_HWCACHE_ALIGN,
5235                                   NULL);
5236         dev_priv->vmas =
5237                 kmem_cache_create("i915_gem_vma",
5238                                   sizeof(struct i915_vma), 0,
5239                                   SLAB_HWCACHE_ALIGN,
5240                                   NULL);
5241         dev_priv->requests =
5242                 kmem_cache_create("i915_gem_request",
5243                                   sizeof(struct drm_i915_gem_request), 0,
5244                                   SLAB_HWCACHE_ALIGN,
5245                                   NULL);
5246
5247         INIT_LIST_HEAD(&dev_priv->vm_list);
5248         i915_init_vm(dev_priv, &dev_priv->gtt.base);
5249
5250         INIT_LIST_HEAD(&dev_priv->context_list);
5251         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5252         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5253         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5254         for (i = 0; i < I915_NUM_RINGS; i++)
5255                 init_ring_lists(&dev_priv->ring[i]);
5256         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5257                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5258         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5259                           i915_gem_retire_work_handler);
5260         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5261                           i915_gem_idle_work_handler);
5262         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5263
5264         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5265
5266         if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
5267                 dev_priv->num_fence_regs = 32;
5268         else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5269                 dev_priv->num_fence_regs = 16;
5270         else
5271                 dev_priv->num_fence_regs = 8;
5272
5273         if (intel_vgpu_active(dev))
5274                 dev_priv->num_fence_regs =
5275                                 I915_READ(vgtif_reg(avail_rs.fence_num));
5276
5277         /* Initialize fence registers to zero */
5278         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5279         i915_gem_restore_fences(dev);
5280
5281         i915_gem_detect_bit_6_swizzle(dev);
5282         init_waitqueue_head(&dev_priv->pending_flip_queue);
5283
5284         dev_priv->mm.interruptible = true;
5285
5286         i915_gem_shrinker_init(dev_priv);
5287
5288         mutex_init(&dev_priv->fb_tracking.lock);
5289 }
5290
5291 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5292 {
5293         struct drm_i915_file_private *file_priv = file->driver_priv;
5294
5295         /* Clean up our request list when the client is going away, so that
5296          * later retire_requests won't dereference our soon-to-be-gone
5297          * file_priv.
5298          */
5299         spin_lock(&file_priv->mm.lock);
5300         while (!list_empty(&file_priv->mm.request_list)) {
5301                 struct drm_i915_gem_request *request;
5302
5303                 request = list_first_entry(&file_priv->mm.request_list,
5304                                            struct drm_i915_gem_request,
5305                                            client_list);
5306                 list_del(&request->client_list);
5307                 request->file_priv = NULL;
5308         }
5309         spin_unlock(&file_priv->mm.lock);
5310
5311         if (!list_empty(&file_priv->rps.link)) {
5312                 spin_lock(&to_i915(dev)->rps.client_lock);
5313                 list_del(&file_priv->rps.link);
5314                 spin_unlock(&to_i915(dev)->rps.client_lock);
5315         }
5316 }
5317
5318 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5319 {
5320         struct drm_i915_file_private *file_priv;
5321         int ret;
5322
5323         DRM_DEBUG_DRIVER("\n");
5324
5325         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5326         if (!file_priv)
5327                 return -ENOMEM;
5328
5329         file->driver_priv = file_priv;
5330         file_priv->dev_priv = dev->dev_private;
5331         file_priv->file = file;
5332         INIT_LIST_HEAD(&file_priv->rps.link);
5333
5334         spin_lock_init(&file_priv->mm.lock);
5335         INIT_LIST_HEAD(&file_priv->mm.request_list);
5336
5337         ret = i915_gem_context_open(dev, file);
5338         if (ret)
5339                 kfree(file_priv);
5340
5341         return ret;
5342 }
5343
5344 /**
5345  * i915_gem_track_fb - update frontbuffer tracking
5346  * old: current GEM buffer for the frontbuffer slots
5347  * new: new GEM buffer for the frontbuffer slots
5348  * frontbuffer_bits: bitmask of frontbuffer slots
5349  *
5350  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5351  * from @old and setting them in @new. Both @old and @new can be NULL.
5352  */
5353 void i915_gem_track_fb(struct drm_i915_gem_object *old,
5354                        struct drm_i915_gem_object *new,
5355                        unsigned frontbuffer_bits)
5356 {
5357         if (old) {
5358                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5359                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5360                 old->frontbuffer_bits &= ~frontbuffer_bits;
5361         }
5362
5363         if (new) {
5364                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5365                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5366                 new->frontbuffer_bits |= frontbuffer_bits;
5367         }
5368 }
5369
5370 /* All the new VM stuff */
5371 unsigned long
5372 i915_gem_obj_offset(struct drm_i915_gem_object *o,
5373                     struct i915_address_space *vm)
5374 {
5375         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5376         struct i915_vma *vma;
5377
5378         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5379
5380         list_for_each_entry(vma, &o->vma_list, vma_link) {
5381                 if (i915_is_ggtt(vma->vm) &&
5382                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5383                         continue;
5384                 if (vma->vm == vm)
5385                         return vma->node.start;
5386         }
5387
5388         WARN(1, "%s vma for this object not found.\n",
5389              i915_is_ggtt(vm) ? "global" : "ppgtt");
5390         return -1;
5391 }
5392
5393 unsigned long
5394 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
5395                               const struct i915_ggtt_view *view)
5396 {
5397         struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5398         struct i915_vma *vma;
5399
5400         list_for_each_entry(vma, &o->vma_list, vma_link)
5401                 if (vma->vm == ggtt &&
5402                     i915_ggtt_view_equal(&vma->ggtt_view, view))
5403                         return vma->node.start;
5404
5405         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
5406         return -1;
5407 }
5408
5409 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5410                         struct i915_address_space *vm)
5411 {
5412         struct i915_vma *vma;
5413
5414         list_for_each_entry(vma, &o->vma_list, vma_link) {
5415                 if (i915_is_ggtt(vma->vm) &&
5416                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5417                         continue;
5418                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5419                         return true;
5420         }
5421
5422         return false;
5423 }
5424
5425 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
5426                                   const struct i915_ggtt_view *view)
5427 {
5428         struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5429         struct i915_vma *vma;
5430
5431         list_for_each_entry(vma, &o->vma_list, vma_link)
5432                 if (vma->vm == ggtt &&
5433                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
5434                     drm_mm_node_allocated(&vma->node))
5435                         return true;
5436
5437         return false;
5438 }
5439
5440 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5441 {
5442         struct i915_vma *vma;
5443
5444         list_for_each_entry(vma, &o->vma_list, vma_link)
5445                 if (drm_mm_node_allocated(&vma->node))
5446                         return true;
5447
5448         return false;
5449 }
5450
5451 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5452                                 struct i915_address_space *vm)
5453 {
5454         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5455         struct i915_vma *vma;
5456
5457         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5458
5459         BUG_ON(list_empty(&o->vma_list));
5460
5461         list_for_each_entry(vma, &o->vma_list, vma_link) {
5462                 if (i915_is_ggtt(vma->vm) &&
5463                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5464                         continue;
5465                 if (vma->vm == vm)
5466                         return vma->node.size;
5467         }
5468         return 0;
5469 }
5470
5471 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
5472 {
5473         struct i915_vma *vma;
5474         list_for_each_entry(vma, &obj->vma_list, vma_link)
5475                 if (vma->pin_count > 0)
5476                         return true;
5477
5478         return false;
5479 }