drivers/xen/grant-table.c

   1 /******************************************************************************
   2  * grant_table.c
   3  *
   4  * Granting foreign access to our memory reservation.
   5  *
   6  * Copyright (c) 2005-2006, Christopher Clark
   7  * Copyright (c) 2004-2005, K A Fraser
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License version 2
  11  * as published by the Free Software Foundation; or, when distributed
  12  * separately from the Linux kernel or incorporated into other
  13  * software packages, subject to the following license:
  14  *
  15  * Permission is hereby granted, free of charge, to any person obtaining a copy
  16  * of this source file (the "Software"), to deal in the Software without
  17  * restriction, including without limitation the rights to use, copy, modify,
  18  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19  * and to permit persons to whom the Software is furnished to do so, subject to
  20  * the following conditions:
  21  *
  22  * The above copyright notice and this permission notice shall be included in
  23  * all copies or substantial portions of the Software.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31  * IN THE SOFTWARE.
  32  */
  33
  34 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  35
  36 #include <linux/module.h>
  37 #include <linux/sched.h>
  38 #include <linux/mm.h>
  39 #include <linux/slab.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/uaccess.h>
  42 #include <linux/io.h>
  43 #include <linux/delay.h>
  44 #include <linux/hardirq.h>
  45
  46 #include <xen/xen.h>
  47 #include <xen/interface/xen.h>
  48 #include <xen/page.h>
  49 #include <xen/grant_table.h>
  50 #include <xen/interface/memory.h>
  51 #include <xen/hvc-console.h>
  52 #include <xen/swiotlb-xen.h>
  53 #include <asm/xen/hypercall.h>
  54 #include <asm/xen/interface.h>
  55
  56 #include <asm/pgtable.h>
  57 #include <asm/sync_bitops.h>
  58
  59 /* External tools reserve first few grant table entries. */
  60 #define NR_RESERVED_ENTRIES 8
  61 #define GNTTAB_LIST_END 0xffffffff
  62
  63 static grant_ref_t **gnttab_list;
  64 static unsigned int nr_grant_frames;
  65 static unsigned int boot_max_nr_grant_frames;
  66 static int gnttab_free_count;
  67 static grant_ref_t gnttab_free_head;
  68 static DEFINE_SPINLOCK(gnttab_list_lock);
  69 unsigned long xen_hvm_resume_frames;
  70 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
  71
  72 static union {
  73         struct grant_entry_v1 *v1;
  74         union grant_entry_v2 *v2;
  75         void *addr;
  76 } gnttab_shared;
  77
  78 /*This is a structure of function pointers for grant table*/
  79 struct gnttab_ops {
  80         /*
  81          * Mapping a list of frames for storing grant entries. Frames parameter
  82          * is used to store grant table address when grant table being setup,
  83          * nr_gframes is the number of frames to map grant table. Returning
  84          * GNTST_okay means success and negative value means failure.
  85          */
  86         int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
  87         /*
  88          * Release a list of frames which are mapped in map_frames for grant
  89          * entry status.
  90          */
  91         void (*unmap_frames)(void);
  92         /*
  93          * Introducing a valid entry into the grant table, granting the frame of
  94          * this grant entry to domain for accessing or transfering. Ref
  95          * parameter is reference of this introduced grant entry, domid is id of
  96          * granted domain, frame is the page frame to be granted, and flags is
  97          * status of the grant entry to be updated.
  98          */
  99         void (*update_entry)(grant_ref_t ref, domid_t domid,
 100                              unsigned long frame, unsigned flags);
 101         /*
 102          * Stop granting a grant entry to domain for accessing. Ref parameter is
 103          * reference of a grant entry whose grant access will be stopped,
 104          * readonly is not in use in this function. If the grant entry is
 105          * currently mapped for reading or writing, just return failure(==0)
 106          * directly and don't tear down the grant access. Otherwise, stop grant
 107          * access for this entry and return success(==1).
 108          */
 109         int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
 110         /*
 111          * Stop granting a grant entry to domain for transfer. Ref parameter is
 112          * reference of a grant entry whose grant transfer will be stopped. If
 113          * tranfer has not started, just reclaim the grant entry and return
 114          * failure(==0). Otherwise, wait for the transfer to complete and then
 115          * return the frame.
 116          */
 117         unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
 118         /*
 119          * Query the status of a grant entry. Ref parameter is reference of
 120          * queried grant entry, return value is the status of queried entry.
 121          * Detailed status(writing/reading) can be gotten from the return value
 122          * by bit operations.
 123          */
 124         int (*query_foreign_access)(grant_ref_t ref);
 125         /*
 126          * Grant a domain to access a range of bytes within the page referred by
 127          * an available grant entry. Ref parameter is reference of a grant entry
 128          * which will be sub-page accessed, domid is id of grantee domain, frame
 129          * is frame address of subpage grant, flags is grant type and flag
 130          * information, page_off is offset of the range of bytes, and length is
 131          * length of bytes to be accessed.
 132          */
 133         void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
 134                                      unsigned long frame, int flags,
 135                                      unsigned page_off, unsigned length);
 136         /*
 137          * Redirect an available grant entry on domain A to another grant
 138          * reference of domain B, then allow domain C to use grant reference
 139          * of domain B transitively. Ref parameter is an available grant entry
 140          * reference on domain A, domid is id of domain C which accesses grant
 141          * entry transitively, flags is grant type and flag information,
 142          * trans_domid is id of domain B whose grant entry is finally accessed
 143          * transitively, trans_gref is grant entry transitive reference of
 144          * domain B.
 145          */
 146         void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
 147                                    domid_t trans_domid, grant_ref_t trans_gref);
 148 };
 149
 150 static struct gnttab_ops *gnttab_interface;
 151
 152 /*This reflects status of grant entries, so act as a global value*/
 153 static grant_status_t *grstatus;
 154
 155 static int grant_table_version;
 156 static int grefs_per_grant_frame;
 157
 158 static struct gnttab_free_callback *gnttab_free_callback_list;
 159
 160 static int gnttab_expand(unsigned int req_entries);
 161
 162 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 163 #define SPP (PAGE_SIZE / sizeof(grant_status_t))
 164
 165 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 166 {
 167         return &gnttab_list[(entry) / RPP][(entry) % RPP];
 168 }
 169 /* This can be used as an l-value */
 170 #define gnttab_entry(entry) (*__gnttab_entry(entry))
 171
 172 static int get_free_entries(unsigned count)
 173 {
 174         unsigned long flags;
 175         int ref, rc = 0;
 176         grant_ref_t head;
 177
 178         spin_lock_irqsave(&gnttab_list_lock, flags);
 179
 180         if ((gnttab_free_count < count) &&
 181             ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 182                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 183                 return rc;
 184         }
 185
 186         ref = head = gnttab_free_head;
 187         gnttab_free_count -= count;
 188         while (count-- > 1)
 189                 head = gnttab_entry(head);
 190         gnttab_free_head = gnttab_entry(head);
 191         gnttab_entry(head) = GNTTAB_LIST_END;
 192
 193         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 194
 195         return ref;
 196 }
 197
 198 static void do_free_callbacks(void)
 199 {
 200         struct gnttab_free_callback *callback, *next;
 201
 202         callback = gnttab_free_callback_list;
 203         gnttab_free_callback_list = NULL;
 204
 205         while (callback != NULL) {
 206                 next = callback->next;
 207                 if (gnttab_free_count >= callback->count) {
 208                         callback->next = NULL;
 209                         callback->fn(callback->arg);
 210                 } else {
 211                         callback->next = gnttab_free_callback_list;
 212                         gnttab_free_callback_list = callback;
 213                 }
 214                 callback = next;
 215         }
 216 }
 217
 218 static inline void check_free_callbacks(void)
 219 {
 220         if (unlikely(gnttab_free_callback_list))
 221                 do_free_callbacks();
 222 }
 223
 224 static void put_free_entry(grant_ref_t ref)
 225 {
 226         unsigned long flags;
 227         spin_lock_irqsave(&gnttab_list_lock, flags);
 228         gnttab_entry(ref) = gnttab_free_head;
 229         gnttab_free_head = ref;
 230         gnttab_free_count++;
 231         check_free_callbacks();
 232         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 233 }
 234
 235 /*
 236  * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
 237  * Introducing a valid entry into the grant table:
 238  *  1. Write ent->domid.
 239  *  2. Write ent->frame:
 240  *      GTF_permit_access:   Frame to which access is permitted.
 241  *      GTF_accept_transfer: Pseudo-phys frame slot being filled by new
 242  *                           frame, or zero if none.
 243  *  3. Write memory barrier (WMB).
 244  *  4. Write ent->flags, inc. valid type.
 245  */
 246 static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 247                                    unsigned long frame, unsigned flags)
 248 {
 249         gnttab_shared.v1[ref].domid = domid;
 250         gnttab_shared.v1[ref].frame = frame;
 251         wmb();
 252         gnttab_shared.v1[ref].flags = flags;
 253 }
 254
 255 static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
 256                                    unsigned long frame, unsigned flags)
 257 {
 258         gnttab_shared.v2[ref].hdr.domid = domid;
 259         gnttab_shared.v2[ref].full_page.frame = frame;
 260         wmb();
 261         gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
 262 }
 263
 264 /*
 265  * Public grant-issuing interface functions
 266  */
 267 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 268                                      unsigned long frame, int readonly)
 269 {
 270         gnttab_interface->update_entry(ref, domid, frame,
 271                            GTF_permit_access | (readonly ? GTF_readonly : 0));
 272 }
 273 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 274
 275 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 276                                 int readonly)
 277 {
 278         int ref;
 279
 280         ref = get_free_entries(1);
 281         if (unlikely(ref < 0))
 282                 return -ENOSPC;
 283
 284         gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 285
 286         return ref;
 287 }
 288 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 289
 290 static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
 291                                            unsigned long frame, int flags,
 292                                            unsigned page_off, unsigned length)
 293 {
 294         gnttab_shared.v2[ref].sub_page.frame = frame;
 295         gnttab_shared.v2[ref].sub_page.page_off = page_off;
 296         gnttab_shared.v2[ref].sub_page.length = length;
 297         gnttab_shared.v2[ref].hdr.domid = domid;
 298         wmb();
 299         gnttab_shared.v2[ref].hdr.flags =
 300                                 GTF_permit_access | GTF_sub_page | flags;
 301 }
 302
 303 int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
 304                                             unsigned long frame, int flags,
 305                                             unsigned page_off,
 306                                             unsigned length)
 307 {
 308         if (flags & (GTF_accept_transfer | GTF_reading |
 309                      GTF_writing | GTF_transitive))
 310                 return -EPERM;
 311
 312         if (gnttab_interface->update_subpage_entry == NULL)
 313                 return -ENOSYS;
 314
 315         gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
 316                                                page_off, length);
 317
 318         return 0;
 319 }
 320 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
 321
 322 int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
 323                                         int flags, unsigned page_off,
 324                                         unsigned length)
 325 {
 326         int ref, rc;
 327
 328         ref = get_free_entries(1);
 329         if (unlikely(ref < 0))
 330                 return -ENOSPC;
 331
 332         rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
 333                                                      page_off, length);
 334         if (rc < 0) {
 335                 put_free_entry(ref);
 336                 return rc;
 337         }
 338
 339         return ref;
 340 }
 341 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
 342
 343 bool gnttab_subpage_grants_available(void)
 344 {
 345         return gnttab_interface->update_subpage_entry != NULL;
 346 }
 347 EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
 348
 349 static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
 350                                          int flags, domid_t trans_domid,
 351                                          grant_ref_t trans_gref)
 352 {
 353         gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
 354         gnttab_shared.v2[ref].transitive.gref = trans_gref;
 355         gnttab_shared.v2[ref].hdr.domid = domid;
 356         wmb();
 357         gnttab_shared.v2[ref].hdr.flags =
 358                                 GTF_permit_access | GTF_transitive | flags;
 359 }
 360
 361 int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
 362                                           int flags, domid_t trans_domid,
 363                                           grant_ref_t trans_gref)
 364 {
 365         if (flags & (GTF_accept_transfer | GTF_reading |
 366                      GTF_writing | GTF_sub_page))
 367                 return -EPERM;
 368
 369         if (gnttab_interface->update_trans_entry == NULL)
 370                 return -ENOSYS;
 371
 372         gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
 373                                              trans_gref);
 374
 375         return 0;
 376 }
 377 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
 378
 379 int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
 380                                       domid_t trans_domid,
 381                                       grant_ref_t trans_gref)
 382 {
 383         int ref, rc;
 384
 385         ref = get_free_entries(1);
 386         if (unlikely(ref < 0))
 387                 return -ENOSPC;
 388
 389         rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
 390                                                    trans_domid, trans_gref);
 391         if (rc < 0) {
 392                 put_free_entry(ref);
 393                 return rc;
 394         }
 395
 396         return ref;
 397 }
 398 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
 399
 400 bool gnttab_trans_grants_available(void)
 401 {
 402         return gnttab_interface->update_trans_entry != NULL;
 403 }
 404 EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
 405
 406 static int gnttab_query_foreign_access_v1(grant_ref_t ref)
 407 {
 408         return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
 409 }
 410
 411 static int gnttab_query_foreign_access_v2(grant_ref_t ref)
 412 {
 413         return grstatus[ref] & (GTF_reading|GTF_writing);
 414 }
 415
 416 int gnttab_query_foreign_access(grant_ref_t ref)
 417 {
 418         return gnttab_interface->query_foreign_access(ref);
 419 }
 420 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
 421
 422 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
 423 {
 424         u16 flags, nflags;
 425         u16 *pflags;
 426
 427         pflags = &gnttab_shared.v1[ref].flags;
 428         nflags = *pflags;
 429         do {
 430                 flags = nflags;
 431                 if (flags & (GTF_reading|GTF_writing))
 432                         return 0;
 433         } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
 434
 435         return 1;
 436 }
 437
 438 static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
 439 {
 440         gnttab_shared.v2[ref].hdr.flags = 0;
 441         mb();
 442         if (grstatus[ref] & (GTF_reading|GTF_writing)) {
 443                 return 0;
 444         } else {
 445                 /* The read of grstatus needs to have acquire
 446                 semantics.  On x86, reads already have
 447                 that, and we just need to protect against
 448                 compiler reorderings.  On other
 449                 architectures we may need a full
 450                 barrier. */
 451 #ifdef CONFIG_X86
 452                 barrier();
 453 #else
 454                 mb();
 455 #endif
 456         }
 457
 458         return 1;
 459 }
 460
 461 static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 462 {
 463         return gnttab_interface->end_foreign_access_ref(ref, readonly);
 464 }
 465
 466 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 467 {
 468         if (_gnttab_end_foreign_access_ref(ref, readonly))
 469                 return 1;
 470         pr_warn("WARNING: g.e. %#x still in use!\n", ref);
 471         return 0;
 472 }
 473 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 474
 475 struct deferred_entry {
 476         struct list_head list;
 477         grant_ref_t ref;
 478         bool ro;
 479         uint16_t warn_delay;
 480         struct page *page;
 481 };
 482 static LIST_HEAD(deferred_list);
 483 static void gnttab_handle_deferred(unsigned long);
 484 static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
 485
 486 static void gnttab_handle_deferred(unsigned long unused)
 487 {
 488         unsigned int nr = 10;
 489         struct deferred_entry *first = NULL;
 490         unsigned long flags;
 491
 492         spin_lock_irqsave(&gnttab_list_lock, flags);
 493         while (nr--) {
 494                 struct deferred_entry *entry
 495                         = list_first_entry(&deferred_list,
 496                                            struct deferred_entry, list);
 497
 498                 if (entry == first)
 499                         break;
 500                 list_del(&entry->list);
 501                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 502                 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
 503                         put_free_entry(entry->ref);
 504                         if (entry->page) {
 505                                 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
 506                                          entry->ref, page_to_pfn(entry->page));
 507                                 __free_page(entry->page);
 508                         } else
 509                                 pr_info("freeing g.e. %#x\n", entry->ref);
 510                         kfree(entry);
 511                         entry = NULL;
 512                 } else {
 513                         if (!--entry->warn_delay)
 514                                 pr_info("g.e. %#x still pending\n", entry->ref);
 515                         if (!first)
 516                                 first = entry;
 517                 }
 518                 spin_lock_irqsave(&gnttab_list_lock, flags);
 519                 if (entry)
 520                         list_add_tail(&entry->list, &deferred_list);
 521                 else if (list_empty(&deferred_list))
 522                         break;
 523         }
 524         if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
 525                 deferred_timer.expires = jiffies + HZ;
 526                 add_timer(&deferred_timer);
 527         }
 528         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 529 }
 530
 531 static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
 532                                 struct page *page)
 533 {
 534         struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 535         const char *what = KERN_WARNING "leaking";
 536
 537         if (entry) {
 538                 unsigned long flags;
 539
 540                 entry->ref = ref;
 541                 entry->ro = readonly;
 542                 entry->page = page;
 543                 entry->warn_delay = 60;
 544                 spin_lock_irqsave(&gnttab_list_lock, flags);
 545                 list_add_tail(&entry->list, &deferred_list);
 546                 if (!timer_pending(&deferred_timer)) {
 547                         deferred_timer.expires = jiffies + HZ;
 548                         add_timer(&deferred_timer);
 549                 }
 550                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 551                 what = KERN_DEBUG "deferring";
 552         }
 553         printk("%s g.e. %#x (pfn %#lx)\n",
 554                what, ref, page ? page_to_pfn(page) : -1);
 555 }
 556
 557 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
 558                                unsigned long page)
 559 {
 560         if (gnttab_end_foreign_access_ref(ref, readonly)) {
 561                 put_free_entry(ref);
 562                 if (page != 0)
 563                         free_page(page);
 564         } else
 565                 gnttab_add_deferred(ref, readonly,
 566                                     page ? virt_to_page(page) : NULL);
 567 }
 568 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 569
 570 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 571 {
 572         int ref;
 573
 574         ref = get_free_entries(1);
 575         if (unlikely(ref < 0))
 576                 return -ENOSPC;
 577         gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 578
 579         return ref;
 580 }
 581 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
 582
 583 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
 584                                        unsigned long pfn)
 585 {
 586         gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
 587 }
 588 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
 589
 590 static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
 591 {
 592         unsigned long frame;
 593         u16           flags;
 594         u16          *pflags;
 595
 596         pflags = &gnttab_shared.v1[ref].flags;
 597
 598         /*
 599          * If a transfer is not even yet started, try to reclaim the grant
 600          * reference and return failure (== 0).
 601          */
 602         while (!((flags = *pflags) & GTF_transfer_committed)) {
 603                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 604                         return 0;
 605                 cpu_relax();
 606         }
 607
 608         /* If a transfer is in progress then wait until it is completed. */
 609         while (!(flags & GTF_transfer_completed)) {
 610                 flags = *pflags;
 611                 cpu_relax();
 612         }
 613
 614         rmb();  /* Read the frame number /after/ reading completion status. */
 615         frame = gnttab_shared.v1[ref].frame;
 616         BUG_ON(frame == 0);
 617
 618         return frame;
 619 }
 620
 621 static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
 622 {
 623         unsigned long frame;
 624         u16           flags;
 625         u16          *pflags;
 626
 627         pflags = &gnttab_shared.v2[ref].hdr.flags;
 628
 629         /*
 630          * If a transfer is not even yet started, try to reclaim the grant
 631          * reference and return failure (== 0).
 632          */
 633         while (!((flags = *pflags) & GTF_transfer_committed)) {
 634                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 635                         return 0;
 636                 cpu_relax();
 637         }
 638
 639         /* If a transfer is in progress then wait until it is completed. */
 640         while (!(flags & GTF_transfer_completed)) {
 641                 flags = *pflags;
 642                 cpu_relax();
 643         }
 644
 645         rmb();  /* Read the frame number /after/ reading completion status. */
 646         frame = gnttab_shared.v2[ref].full_page.frame;
 647         BUG_ON(frame == 0);
 648
 649         return frame;
 650 }
 651
 652 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 653 {
 654         return gnttab_interface->end_foreign_transfer_ref(ref);
 655 }
 656 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
 657
 658 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
 659 {
 660         unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
 661         put_free_entry(ref);
 662         return frame;
 663 }
 664 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
 665
 666 void gnttab_free_grant_reference(grant_ref_t ref)
 667 {
 668         put_free_entry(ref);
 669 }
 670 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 671
 672 void gnttab_free_grant_references(grant_ref_t head)
 673 {
 674         grant_ref_t ref;
 675         unsigned long flags;
 676         int count = 1;
 677         if (head == GNTTAB_LIST_END)
 678                 return;
 679         spin_lock_irqsave(&gnttab_list_lock, flags);
 680         ref = head;
 681         while (gnttab_entry(ref) != GNTTAB_LIST_END) {
 682                 ref = gnttab_entry(ref);
 683                 count++;
 684         }
 685         gnttab_entry(ref) = gnttab_free_head;
 686         gnttab_free_head = head;
 687         gnttab_free_count += count;
 688         check_free_callbacks();
 689         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 690 }
 691 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 692
 693 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 694 {
 695         int h = get_free_entries(count);
 696
 697         if (h < 0)
 698                 return -ENOSPC;
 699
 700         *head = h;
 701
 702         return 0;
 703 }
 704 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 705
 706 int gnttab_empty_grant_references(const grant_ref_t *private_head)
 707 {
 708         return (*private_head == GNTTAB_LIST_END);
 709 }
 710 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 711
 712 int gnttab_claim_grant_reference(grant_ref_t *private_head)
 713 {
 714         grant_ref_t g = *private_head;
 715         if (unlikely(g == GNTTAB_LIST_END))
 716                 return -ENOSPC;
 717         *private_head = gnttab_entry(g);
 718         return g;
 719 }
 720 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 721
 722 void gnttab_release_grant_reference(grant_ref_t *private_head,
 723                                     grant_ref_t release)
 724 {
 725         gnttab_entry(release) = *private_head;
 726         *private_head = release;
 727 }
 728 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 729
 730 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 731                                   void (*fn)(void *), void *arg, u16 count)
 732 {
 733         unsigned long flags;
 734         struct gnttab_free_callback *cb;
 735
 736         spin_lock_irqsave(&gnttab_list_lock, flags);
 737
 738         /* Check if the callback is already on the list */
 739         cb = gnttab_free_callback_list;
 740         while (cb) {
 741                 if (cb == callback)
 742                         goto out;
 743                 cb = cb->next;
 744         }
 745
 746         callback->fn = fn;
 747         callback->arg = arg;
 748         callback->count = count;
 749         callback->next = gnttab_free_callback_list;
 750         gnttab_free_callback_list = callback;
 751         check_free_callbacks();
 752 out:
 753         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 754 }
 755 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 756
 757 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 758 {
 759         struct gnttab_free_callback **pcb;
 760         unsigned long flags;
 761
 762         spin_lock_irqsave(&gnttab_list_lock, flags);
 763         for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 764                 if (*pcb == callback) {
 765                         *pcb = callback->next;
 766                         break;
 767                 }
 768         }
 769         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 770 }
 771 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 772
 773 static int grow_gnttab_list(unsigned int more_frames)
 774 {
 775         unsigned int new_nr_grant_frames, extra_entries, i;
 776         unsigned int nr_glist_frames, new_nr_glist_frames;
 777
 778         BUG_ON(grefs_per_grant_frame == 0);
 779
 780         new_nr_grant_frames = nr_grant_frames + more_frames;
 781         extra_entries       = more_frames * grefs_per_grant_frame;
 782
 783         nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
 784         new_nr_glist_frames =
 785                 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
 786         for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 787                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 788                 if (!gnttab_list[i])
 789                         goto grow_nomem;
 790         }
 791
 792
 793         for (i = grefs_per_grant_frame * nr_grant_frames;
 794              i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++)
 795                 gnttab_entry(i) = i + 1;
 796
 797         gnttab_entry(i) = gnttab_free_head;
 798         gnttab_free_head = grefs_per_grant_frame * nr_grant_frames;
 799         gnttab_free_count += extra_entries;
 800
 801         nr_grant_frames = new_nr_grant_frames;
 802
 803         check_free_callbacks();
 804
 805         return 0;
 806
 807 grow_nomem:
 808         for ( ; i >= nr_glist_frames; i--)
 809                 free_page((unsigned long) gnttab_list[i]);
 810         return -ENOMEM;
 811 }
 812
 813 static unsigned int __max_nr_grant_frames(void)
 814 {
 815         struct gnttab_query_size query;
 816         int rc;
 817
 818         query.dom = DOMID_SELF;
 819
 820         rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 821         if ((rc < 0) || (query.status != GNTST_okay))
 822                 return 4; /* Legacy max supported number of frames */
 823
 824         return query.max_nr_frames;
 825 }
 826
 827 unsigned int gnttab_max_grant_frames(void)
 828 {
 829         unsigned int xen_max = __max_nr_grant_frames();
 830
 831         if (xen_max > boot_max_nr_grant_frames)
 832                 return boot_max_nr_grant_frames;
 833         return xen_max;
 834 }
 835 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 836
 837 /* Handling of paged out grant targets (GNTST_eagain) */
 838 #define MAX_DELAY 256
 839 static inline void
 840 gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
 841                                                 const char *func)
 842 {
 843         unsigned delay = 1;
 844
 845         do {
 846                 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
 847                 if (*status == GNTST_eagain)
 848                         msleep(delay++);
 849         } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
 850
 851         if (delay >= MAX_DELAY) {
 852                 pr_err("%s: %s eagain grant\n", func, current->comm);
 853                 *status = GNTST_bad_page;
 854         }
 855 }
 856
 857 void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
 858 {
 859         struct gnttab_map_grant_ref *op;
 860
 861         if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
 862                 BUG();
 863         for (op = batch; op < batch + count; op++)
 864                 if (op->status == GNTST_eagain)
 865                         gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
 866                                                 &op->status, __func__);
 867 }
 868 EXPORT_SYMBOL_GPL(gnttab_batch_map);
 869
 870 void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
 871 {
 872         struct gnttab_copy *op;
 873
 874         if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
 875                 BUG();
 876         for (op = batch; op < batch + count; op++)
 877                 if (op->status == GNTST_eagain)
 878                         gnttab_retry_eagain_gop(GNTTABOP_copy, op,
 879                                                 &op->status, __func__);
 880 }
 881 EXPORT_SYMBOL_GPL(gnttab_batch_copy);
 882
 883 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 884                     struct gnttab_map_grant_ref *kmap_ops,
 885                     struct page **pages, unsigned int count)
 886 {
 887         int i, ret;
 888         bool lazy = false;
 889         pte_t *pte;
 890         unsigned long mfn;
 891
 892         ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
 893         if (ret)
 894                 return ret;
 895
 896         /* Retry eagain maps */
 897         for (i = 0; i < count; i++)
 898                 if (map_ops[i].status == GNTST_eagain)
 899                         gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
 900                                                 &map_ops[i].status, __func__);
 901
 902         /* this is basically a nop on x86 */
 903         if (xen_feature(XENFEAT_auto_translated_physmap)) {
 904                 for (i = 0; i < count; i++) {
 905                         if (map_ops[i].status)
 906                                 continue;
 907                         set_phys_to_machine(map_ops[i].host_addr >> PAGE_SHIFT,
 908                                         map_ops[i].dev_bus_addr >> PAGE_SHIFT);
 909                 }
 910                 return ret;
 911         }
 912
 913         if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 914                 arch_enter_lazy_mmu_mode();
 915                 lazy = true;
 916         }
 917
 918         for (i = 0; i < count; i++) {
 919                 /* Do not add to override if the map failed. */
 920                 if (map_ops[i].status)
 921                         continue;
 922
 923                 if (map_ops[i].flags & GNTMAP_contains_pte) {
 924                         pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
 925                                 (map_ops[i].host_addr & ~PAGE_MASK));
 926                         mfn = pte_mfn(*pte);
 927                 } else {
 928                         mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
 929                 }
 930                 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
 931                                        &kmap_ops[i] : NULL);
 932                 if (ret)
 933                         goto out;
 934         }
 935
 936  out:
 937         if (lazy)
 938                 arch_leave_lazy_mmu_mode();
 939
 940         return ret;
 941 }
 942 EXPORT_SYMBOL_GPL(gnttab_map_refs);
 943
 944 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
 945                       struct gnttab_map_grant_ref *kmap_ops,
 946                       struct page **pages, unsigned int count)
 947 {
 948         int i, ret;
 949         bool lazy = false;
 950
 951         ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
 952         if (ret)
 953                 return ret;
 954
 955         /* this is basically a nop on x86 */
 956         if (xen_feature(XENFEAT_auto_translated_physmap)) {
 957                 for (i = 0; i < count; i++) {
 958                         set_phys_to_machine(unmap_ops[i].host_addr >> PAGE_SHIFT,
 959                                         INVALID_P2M_ENTRY);
 960                 }
 961                 return ret;
 962         }
 963
 964         if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 965                 arch_enter_lazy_mmu_mode();
 966                 lazy = true;
 967         }
 968
 969         for (i = 0; i < count; i++) {
 970                 ret = m2p_remove_override(pages[i], kmap_ops ?
 971                                        &kmap_ops[i] : NULL);
 972                 if (ret)
 973                         goto out;
 974         }
 975
 976  out:
 977         if (lazy)
 978                 arch_leave_lazy_mmu_mode();
 979
 980         return ret;
 981 }
 982 EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
 983
 984 static unsigned nr_status_frames(unsigned nr_grant_frames)
 985 {
 986         BUG_ON(grefs_per_grant_frame == 0);
 987         return (nr_grant_frames * grefs_per_grant_frame + SPP - 1) / SPP;
 988 }
 989
 990 static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
 991 {
 992         int rc;
 993
 994         rc = arch_gnttab_map_shared(frames, nr_gframes,
 995                                     gnttab_max_grant_frames(),
 996                                     &gnttab_shared.addr);
 997         BUG_ON(rc);
 998
 999         return 0;
1000 }
1001
1002 static void gnttab_unmap_frames_v1(void)
1003 {
1004         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1005 }
1006
1007 static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
1008 {
1009         uint64_t *sframes;
1010         unsigned int nr_sframes;
1011         struct gnttab_get_status_frames getframes;
1012         int rc;
1013
1014         nr_sframes = nr_status_frames(nr_gframes);
1015
1016         /* No need for kzalloc as it is initialized in following hypercall
1017          * GNTTABOP_get_status_frames.
1018          */
1019         sframes = kmalloc(nr_sframes  * sizeof(uint64_t), GFP_ATOMIC);
1020         if (!sframes)
1021                 return -ENOMEM;
1022
1023         getframes.dom        = DOMID_SELF;
1024         getframes.nr_frames  = nr_sframes;
1025         set_xen_guest_handle(getframes.frame_list, sframes);
1026
1027         rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1028                                        &getframes, 1);
1029         if (rc == -ENOSYS) {
1030                 kfree(sframes);
1031                 return -ENOSYS;
1032         }
1033
1034         BUG_ON(rc || getframes.status);
1035
1036         rc = arch_gnttab_map_status(sframes, nr_sframes,
1037                                     nr_status_frames(gnttab_max_grant_frames()),
1038                                     &grstatus);
1039         BUG_ON(rc);
1040         kfree(sframes);
1041
1042         rc = arch_gnttab_map_shared(frames, nr_gframes,
1043                                     gnttab_max_grant_frames(),
1044                                     &gnttab_shared.addr);
1045         BUG_ON(rc);
1046
1047         return 0;
1048 }
1049
1050 static void gnttab_unmap_frames_v2(void)
1051 {
1052         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1053         arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
1054 }
1055
1056 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1057 {
1058         struct gnttab_setup_table setup;
1059         xen_pfn_t *frames;
1060         unsigned int nr_gframes = end_idx + 1;
1061         int rc;
1062
1063         if (xen_hvm_domain()) {
1064                 struct xen_add_to_physmap xatp;
1065                 unsigned int i = end_idx;
1066                 rc = 0;
1067                 /*
1068                  * Loop backwards, so that the first hypercall has the largest
1069                  * index, ensuring that the table will grow only once.
1070                  */
1071                 do {
1072                         xatp.domid = DOMID_SELF;
1073                         xatp.idx = i;
1074                         xatp.space = XENMAPSPACE_grant_table;
1075                         xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
1076                         rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1077                         if (rc != 0) {
1078                                 pr_warn("grant table add_to_physmap failed, err=%d\n",
1079                                         rc);
1080                                 break;
1081                         }
1082                 } while (i-- > start_idx);
1083
1084                 return rc;
1085         }
1086
1087         /* No need for kzalloc as it is initialized in following hypercall
1088          * GNTTABOP_setup_table.
1089          */
1090         frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1091         if (!frames)
1092                 return -ENOMEM;
1093
1094         setup.dom        = DOMID_SELF;
1095         setup.nr_frames  = nr_gframes;
1096         set_xen_guest_handle(setup.frame_list, frames);
1097
1098         rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1099         if (rc == -ENOSYS) {
1100                 kfree(frames);
1101                 return -ENOSYS;
1102         }
1103
1104         BUG_ON(rc || setup.status);
1105
1106         rc = gnttab_interface->map_frames(frames, nr_gframes);
1107
1108         kfree(frames);
1109
1110         return rc;
1111 }
1112
1113 static struct gnttab_ops gnttab_v1_ops = {
1114         .map_frames                     = gnttab_map_frames_v1,
1115         .unmap_frames                   = gnttab_unmap_frames_v1,
1116         .update_entry                   = gnttab_update_entry_v1,
1117         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v1,
1118         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v1,
1119         .query_foreign_access           = gnttab_query_foreign_access_v1,
1120 };
1121
1122 static struct gnttab_ops gnttab_v2_ops = {
1123         .map_frames                     = gnttab_map_frames_v2,
1124         .unmap_frames                   = gnttab_unmap_frames_v2,
1125         .update_entry                   = gnttab_update_entry_v2,
1126         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v2,
1127         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v2,
1128         .query_foreign_access           = gnttab_query_foreign_access_v2,
1129         .update_subpage_entry           = gnttab_update_subpage_entry_v2,
1130         .update_trans_entry             = gnttab_update_trans_entry_v2,
1131 };
1132
1133 static void gnttab_request_version(void)
1134 {
1135         int rc;
1136         struct gnttab_set_version gsv;
1137
1138         if (xen_hvm_domain())
1139                 gsv.version = 1;
1140         else
1141                 gsv.version = 2;
1142         rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1143         if (rc == 0 && gsv.version == 2) {
1144                 grant_table_version = 2;
1145                 grefs_per_grant_frame = PAGE_SIZE / sizeof(union grant_entry_v2);
1146                 gnttab_interface = &gnttab_v2_ops;
1147         } else if (grant_table_version == 2) {
1148                 /*
1149                  * If we've already used version 2 features,
1150                  * but then suddenly discover that they're not
1151                  * available (e.g. migrating to an older
1152                  * version of Xen), almost unbounded badness
1153                  * can happen.
1154                  */
1155                 panic("we need grant tables version 2, but only version 1 is available");
1156         } else {
1157                 grant_table_version = 1;
1158                 grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1);
1159                 gnttab_interface = &gnttab_v1_ops;
1160         }
1161         pr_info("Grant tables using version %d layout\n", grant_table_version);
1162 }
1163
1164 static int gnttab_setup(void)
1165 {
1166         unsigned int max_nr_gframes;
1167
1168         max_nr_gframes = gnttab_max_grant_frames();
1169         if (max_nr_gframes < nr_grant_frames)
1170                 return -ENOSYS;
1171
1172         if (xen_pv_domain())
1173                 return gnttab_map(0, nr_grant_frames - 1);
1174
1175         if (gnttab_shared.addr == NULL) {
1176                 gnttab_shared.addr = xen_remap(xen_hvm_resume_frames,
1177                                                 PAGE_SIZE * max_nr_gframes);
1178                 if (gnttab_shared.addr == NULL) {
1179                         pr_warn("Failed to ioremap gnttab share frames!\n");
1180                         return -ENOMEM;
1181                 }
1182         }
1183
1184         gnttab_map(0, nr_grant_frames - 1);
1185
1186         return 0;
1187 }
1188
1189 int gnttab_resume(void)
1190 {
1191         gnttab_request_version();
1192         return gnttab_setup();
1193 }
1194
1195 int gnttab_suspend(void)
1196 {
1197         gnttab_interface->unmap_frames();
1198         return 0;
1199 }
1200
1201 static int gnttab_expand(unsigned int req_entries)
1202 {
1203         int rc;
1204         unsigned int cur, extra;
1205
1206         BUG_ON(grefs_per_grant_frame == 0);
1207         cur = nr_grant_frames;
1208         extra = ((req_entries + (grefs_per_grant_frame-1)) /
1209                  grefs_per_grant_frame);
1210         if (cur + extra > gnttab_max_grant_frames())
1211                 return -ENOSPC;
1212
1213         rc = gnttab_map(cur, cur + extra - 1);
1214         if (rc == 0)
1215                 rc = grow_gnttab_list(extra);
1216
1217         return rc;
1218 }
1219
1220 int gnttab_init(void)
1221 {
1222         int i;
1223         unsigned int max_nr_glist_frames, nr_glist_frames;
1224         unsigned int nr_init_grefs;
1225         int ret;
1226
1227         gnttab_request_version();
1228         nr_grant_frames = 1;
1229         boot_max_nr_grant_frames = __max_nr_grant_frames();
1230
1231         /* Determine the maximum number of frames required for the
1232          * grant reference free list on the current hypervisor.
1233          */
1234         BUG_ON(grefs_per_grant_frame == 0);
1235         max_nr_glist_frames = (boot_max_nr_grant_frames *
1236                                grefs_per_grant_frame / RPP);
1237
1238         gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1239                               GFP_KERNEL);
1240         if (gnttab_list == NULL)
1241                 return -ENOMEM;
1242
1243         nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
1244         for (i = 0; i < nr_glist_frames; i++) {
1245                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1246                 if (gnttab_list[i] == NULL) {
1247                         ret = -ENOMEM;
1248                         goto ini_nomem;
1249                 }
1250         }
1251
1252         if (gnttab_setup() < 0) {
1253                 ret = -ENODEV;
1254                 goto ini_nomem;
1255         }
1256
1257         nr_init_grefs = nr_grant_frames * grefs_per_grant_frame;
1258
1259         for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1260                 gnttab_entry(i) = i + 1;
1261
1262         gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1263         gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1264         gnttab_free_head  = NR_RESERVED_ENTRIES;
1265
1266         printk("Grant table initialized\n");
1267         return 0;
1268
1269  ini_nomem:
1270         for (i--; i >= 0; i--)
1271                 free_page((unsigned long)gnttab_list[i]);
1272         kfree(gnttab_list);
1273         return ret;
1274 }
1275 EXPORT_SYMBOL_GPL(gnttab_init);
1276
1277 static int __gnttab_init(void)
1278 {
1279         /* Delay grant-table initialization in the PV on HVM case */
1280         if (xen_hvm_domain())
1281                 return 0;
1282
1283         if (!xen_pv_domain())
1284                 return -ENODEV;
1285
1286         return gnttab_init();
1287 }
1288
1289 core_initcall(__gnttab_init);