drivers/staging/rdma/hfi1/diag.c

   1 /*
   2  *
   3  * This file is provided under a dual BSD/GPLv2 license.  When using or
   4  * redistributing this file, you may do so under either license.
   5  *
   6  * GPL LICENSE SUMMARY
   7  *
   8  * Copyright(c) 2015 Intel Corporation.
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of version 2 of the GNU General Public License as
  12  * published by the Free Software Foundation.
  13  *
  14  * This program is distributed in the hope that it will be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * General Public License for more details.
  18  *
  19  * BSD LICENSE
  20  *
  21  * Copyright(c) 2015 Intel Corporation.
  22  *
  23  * Redistribution and use in source and binary forms, with or without
  24  * modification, are permitted provided that the following conditions
  25  * are met:
  26  *
  27  *  - Redistributions of source code must retain the above copyright
  28  *    notice, this list of conditions and the following disclaimer.
  29  *  - Redistributions in binary form must reproduce the above copyright
  30  *    notice, this list of conditions and the following disclaimer in
  31  *    the documentation and/or other materials provided with the
  32  *    distribution.
  33  *  - Neither the name of Intel Corporation nor the names of its
  34  *    contributors may be used to endorse or promote products derived
  35  *    from this software without specific prior written permission.
  36  *
  37  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  38  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  39  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  40  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  41  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  42  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  43  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  44  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  45  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  46  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  47  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  48  *
  49  */
  50
  51 /*
  52  * This file contains support for diagnostic functions.  It is accessed by
  53  * opening the hfi1_diag device, normally minor number 129.  Diagnostic use
  54  * of the chip may render the chip or board unusable until the driver
  55  * is unloaded, or in some cases, until the system is rebooted.
  56  *
  57  * Accesses to the chip through this interface are not similar to going
  58  * through the /sys/bus/pci resource mmap interface.
  59  */
  60
  61 #include <linux/io.h>
  62 #include <linux/pci.h>
  63 #include <linux/poll.h>
  64 #include <linux/vmalloc.h>
  65 #include <linux/export.h>
  66 #include <linux/fs.h>
  67 #include <linux/uaccess.h>
  68 #include <linux/module.h>
  69 #include <rdma/ib_smi.h>
  70 #include "hfi.h"
  71 #include "device.h"
  72 #include "common.h"
  73 #include "trace.h"
  74
  75 #undef pr_fmt
  76 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
  77 #define snoop_dbg(fmt, ...) \
  78         hfi1_cdbg(SNOOP, fmt, ##__VA_ARGS__)
  79
  80 /* Snoop option mask */
  81 #define SNOOP_DROP_SEND (1 << 0)
  82 #define SNOOP_USE_METADATA      (1 << 1)
  83
  84 static u8 snoop_flags;
  85
  86 /*
  87  * Extract packet length from LRH header.
  88  * Why & 0x7FF? Because len is only 11 bits in case it wasn't 0'd we throw the
  89  * bogus bits away. This is in Dwords so multiply by 4 to get size in bytes
  90  */
  91 #define HFI1_GET_PKT_LEN(x)      (((be16_to_cpu((x)->lrh[2]) & 0x7FF)) << 2)
  92
  93 enum hfi1_filter_status {
  94         HFI1_FILTER_HIT,
  95         HFI1_FILTER_ERR,
  96         HFI1_FILTER_MISS
  97 };
  98
  99 /* snoop processing functions */
 100 rhf_rcv_function_ptr snoop_rhf_rcv_functions[8] = {
 101         [RHF_RCV_TYPE_EXPECTED] = snoop_recv_handler,
 102         [RHF_RCV_TYPE_EAGER]    = snoop_recv_handler,
 103         [RHF_RCV_TYPE_IB]       = snoop_recv_handler,
 104         [RHF_RCV_TYPE_ERROR]    = snoop_recv_handler,
 105         [RHF_RCV_TYPE_BYPASS]   = snoop_recv_handler,
 106         [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
 107         [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
 108         [RHF_RCV_TYPE_INVALID7] = process_receive_invalid
 109 };
 110
 111 /* Snoop packet structure */
 112 struct snoop_packet {
 113         struct list_head list;
 114         u32 total_len;
 115         u8 data[];
 116 };
 117
 118 /* Do not make these an enum or it will blow up the capture_md */
 119 #define PKT_DIR_EGRESS 0x0
 120 #define PKT_DIR_INGRESS 0x1
 121
 122 /* Packet capture metadata returned to the user with the packet. */
 123 struct capture_md {
 124         u8 port;
 125         u8 dir;
 126         u8 reserved[6];
 127         union {
 128                 u64 pbc;
 129                 u64 rhf;
 130         } u;
 131 };
 132
 133 static atomic_t diagpkt_count = ATOMIC_INIT(0);
 134 static struct cdev diagpkt_cdev;
 135 static struct device *diagpkt_device;
 136
 137 static ssize_t diagpkt_write(struct file *fp, const char __user *data,
 138                                  size_t count, loff_t *off);
 139
 140 static const struct file_operations diagpkt_file_ops = {
 141         .owner = THIS_MODULE,
 142         .write = diagpkt_write,
 143         .llseek = noop_llseek,
 144 };
 145
 146 /*
 147  * This is used for communication with user space for snoop extended IOCTLs
 148  */
 149 struct hfi1_link_info {
 150         __be64 node_guid;
 151         u8 port_mode;
 152         u8 port_state;
 153         u16 link_speed_active;
 154         u16 link_width_active;
 155         u16 vl15_init;
 156         u8 port_number;
 157         /*
 158          * Add padding to make this a full IB SMP payload. Note: changing the
 159          * size of this structure will make the IOCTLs created with _IOWR
 160          * change.
 161          * Be sure to run tests on all IOCTLs when making changes to this
 162          * structure.
 163          */
 164         u8 res[47];
 165 };
 166
 167 /*
 168  * This starts our ioctl sequence numbers *way* off from the ones
 169  * defined in ib_core.
 170  */
 171 #define SNOOP_CAPTURE_VERSION 0x1
 172
 173 #define IB_IOCTL_MAGIC          0x1b /* See Documentation/ioctl-number.txt */
 174 #define HFI1_SNOOP_IOC_MAGIC IB_IOCTL_MAGIC
 175 #define HFI1_SNOOP_IOC_BASE_SEQ 0x80
 176
 177 #define HFI1_SNOOP_IOCGETLINKSTATE \
 178         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ)
 179 #define HFI1_SNOOP_IOCSETLINKSTATE \
 180         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+1)
 181 #define HFI1_SNOOP_IOCCLEARQUEUE \
 182         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+2)
 183 #define HFI1_SNOOP_IOCCLEARFILTER \
 184         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+3)
 185 #define HFI1_SNOOP_IOCSETFILTER \
 186         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+4)
 187 #define HFI1_SNOOP_IOCGETVERSION \
 188         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+5)
 189 #define HFI1_SNOOP_IOCSET_OPTS \
 190         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+6)
 191
 192 /*
 193  * These offsets +6/+7 could change, but these are already known and used
 194  * IOCTL numbers so don't change them without a good reason.
 195  */
 196 #define HFI1_SNOOP_IOCGETLINKSTATE_EXTRA \
 197         _IOWR(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+6, \
 198                 struct hfi1_link_info)
 199 #define HFI1_SNOOP_IOCSETLINKSTATE_EXTRA \
 200         _IOWR(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ+7, \
 201                 struct hfi1_link_info)
 202
 203 static int hfi1_snoop_open(struct inode *in, struct file *fp);
 204 static ssize_t hfi1_snoop_read(struct file *fp, char __user *data,
 205                                 size_t pkt_len, loff_t *off);
 206 static ssize_t hfi1_snoop_write(struct file *fp, const char __user *data,
 207                                  size_t count, loff_t *off);
 208 static long hfi1_ioctl(struct file *fp, unsigned int cmd, unsigned long arg);
 209 static unsigned int hfi1_snoop_poll(struct file *fp,
 210                                         struct poll_table_struct *wait);
 211 static int hfi1_snoop_release(struct inode *in, struct file *fp);
 212
 213 struct hfi1_packet_filter_command {
 214         int opcode;
 215         int length;
 216         void *value_ptr;
 217 };
 218
 219 /* Can't re-use PKT_DIR_*GRESS here because 0 means no packets for this */
 220 #define HFI1_SNOOP_INGRESS 0x1
 221 #define HFI1_SNOOP_EGRESS  0x2
 222
 223 enum hfi1_packet_filter_opcodes {
 224         FILTER_BY_LID,
 225         FILTER_BY_DLID,
 226         FILTER_BY_MAD_MGMT_CLASS,
 227         FILTER_BY_QP_NUMBER,
 228         FILTER_BY_PKT_TYPE,
 229         FILTER_BY_SERVICE_LEVEL,
 230         FILTER_BY_PKEY,
 231         FILTER_BY_DIRECTION,
 232 };
 233
 234 static const struct file_operations snoop_file_ops = {
 235         .owner = THIS_MODULE,
 236         .open = hfi1_snoop_open,
 237         .read = hfi1_snoop_read,
 238         .unlocked_ioctl = hfi1_ioctl,
 239         .poll = hfi1_snoop_poll,
 240         .write = hfi1_snoop_write,
 241         .release = hfi1_snoop_release
 242 };
 243
 244 struct hfi1_filter_array {
 245         int (*filter)(void *, void *, void *);
 246 };
 247
 248 static int hfi1_filter_lid(void *ibhdr, void *packet_data, void *value);
 249 static int hfi1_filter_dlid(void *ibhdr, void *packet_data, void *value);
 250 static int hfi1_filter_mad_mgmt_class(void *ibhdr, void *packet_data,
 251                                       void *value);
 252 static int hfi1_filter_qp_number(void *ibhdr, void *packet_data, void *value);
 253 static int hfi1_filter_ibpacket_type(void *ibhdr, void *packet_data,
 254                                      void *value);
 255 static int hfi1_filter_ib_service_level(void *ibhdr, void *packet_data,
 256                                         void *value);
 257 static int hfi1_filter_ib_pkey(void *ibhdr, void *packet_data, void *value);
 258 static int hfi1_filter_direction(void *ibhdr, void *packet_data, void *value);
 259
 260 static struct hfi1_filter_array hfi1_filters[] = {
 261         { hfi1_filter_lid },
 262         { hfi1_filter_dlid },
 263         { hfi1_filter_mad_mgmt_class },
 264         { hfi1_filter_qp_number },
 265         { hfi1_filter_ibpacket_type },
 266         { hfi1_filter_ib_service_level },
 267         { hfi1_filter_ib_pkey },
 268         { hfi1_filter_direction },
 269 };
 270
 271 #define HFI1_MAX_FILTERS        ARRAY_SIZE(hfi1_filters)
 272 #define HFI1_DIAG_MINOR_BASE    129
 273
 274 static int hfi1_snoop_add(struct hfi1_devdata *dd, const char *name);
 275
 276 int hfi1_diag_add(struct hfi1_devdata *dd)
 277 {
 278         char name[16];
 279         int ret = 0;
 280
 281         snprintf(name, sizeof(name), "%s_diagpkt%d", class_name(),
 282                  dd->unit);
 283         /*
 284          * Do this for each device as opposed to the normal diagpkt
 285          * interface which is one per host
 286          */
 287         ret = hfi1_snoop_add(dd, name);
 288         if (ret)
 289                 dd_dev_err(dd, "Unable to init snoop/capture device");
 290
 291         snprintf(name, sizeof(name), "%s_diagpkt", class_name());
 292         if (atomic_inc_return(&diagpkt_count) == 1) {
 293                 ret = hfi1_cdev_init(HFI1_DIAGPKT_MINOR, name,
 294                                      &diagpkt_file_ops, &diagpkt_cdev,
 295                                      &diagpkt_device, false);
 296         }
 297
 298         return ret;
 299 }
 300
 301 /* this must be called w/ dd->snoop_in_lock held */
 302 static void drain_snoop_list(struct list_head *queue)
 303 {
 304         struct list_head *pos, *q;
 305         struct snoop_packet *packet;
 306
 307         list_for_each_safe(pos, q, queue) {
 308                 packet = list_entry(pos, struct snoop_packet, list);
 309                 list_del(pos);
 310                 kfree(packet);
 311         }
 312 }
 313
 314 static void hfi1_snoop_remove(struct hfi1_devdata *dd)
 315 {
 316         unsigned long flags = 0;
 317
 318         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 319         drain_snoop_list(&dd->hfi1_snoop.queue);
 320         hfi1_cdev_cleanup(&dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev);
 321         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 322 }
 323
 324 void hfi1_diag_remove(struct hfi1_devdata *dd)
 325 {
 326
 327         hfi1_snoop_remove(dd);
 328         if (atomic_dec_and_test(&diagpkt_count))
 329                 hfi1_cdev_cleanup(&diagpkt_cdev, &diagpkt_device);
 330         hfi1_cdev_cleanup(&dd->diag_cdev, &dd->diag_device);
 331 }
 332
 333
 334 /*
 335  * Allocated structure shared between the credit return mechanism and
 336  * diagpkt_send().
 337  */
 338 struct diagpkt_wait {
 339         struct completion credits_returned;
 340         int code;
 341         atomic_t count;
 342 };
 343
 344 /*
 345  * When each side is finished with the structure, they call this.
 346  * The last user frees the structure.
 347  */
 348 static void put_diagpkt_wait(struct diagpkt_wait *wait)
 349 {
 350         if (atomic_dec_and_test(&wait->count))
 351                 kfree(wait);
 352 }
 353
 354 /*
 355  * Callback from the credit return code.  Set the complete, which
 356  * will let diapkt_send() continue.
 357  */
 358 static void diagpkt_complete(void *arg, int code)
 359 {
 360         struct diagpkt_wait *wait = (struct diagpkt_wait *)arg;
 361
 362         wait->code = code;
 363         complete(&wait->credits_returned);
 364         put_diagpkt_wait(wait); /* finished with the structure */
 365 }
 366
 367 /**
 368  * diagpkt_send - send a packet
 369  * @dp: diag packet descriptor
 370  */
 371 static ssize_t diagpkt_send(struct diag_pkt *dp)
 372 {
 373         struct hfi1_devdata *dd;
 374         struct send_context *sc;
 375         struct pio_buf *pbuf;
 376         u32 *tmpbuf = NULL;
 377         ssize_t ret = 0;
 378         u32 pkt_len, total_len;
 379         pio_release_cb credit_cb = NULL;
 380         void *credit_arg = NULL;
 381         struct diagpkt_wait *wait = NULL;
 382
 383         dd = hfi1_lookup(dp->unit);
 384         if (!dd || !(dd->flags & HFI1_PRESENT) || !dd->kregbase) {
 385                 ret = -ENODEV;
 386                 goto bail;
 387         }
 388         if (!(dd->flags & HFI1_INITTED)) {
 389                 /* no hardware, freeze, etc. */
 390                 ret = -ENODEV;
 391                 goto bail;
 392         }
 393
 394         if (dp->version != _DIAG_PKT_VERS) {
 395                 dd_dev_err(dd, "Invalid version %u for diagpkt_write\n",
 396                             dp->version);
 397                 ret = -EINVAL;
 398                 goto bail;
 399         }
 400
 401         /* send count must be an exact number of dwords */
 402         if (dp->len & 3) {
 403                 ret = -EINVAL;
 404                 goto bail;
 405         }
 406
 407         /* there is only port 1 */
 408         if (dp->port != 1) {
 409                 ret = -EINVAL;
 410                 goto bail;
 411         }
 412
 413         /* need a valid context */
 414         if (dp->sw_index >= dd->num_send_contexts) {
 415                 ret = -EINVAL;
 416                 goto bail;
 417         }
 418         /* can only use kernel contexts */
 419         if (dd->send_contexts[dp->sw_index].type != SC_KERNEL) {
 420                 ret = -EINVAL;
 421                 goto bail;
 422         }
 423         /* must be allocated */
 424         sc = dd->send_contexts[dp->sw_index].sc;
 425         if (!sc) {
 426                 ret = -EINVAL;
 427                 goto bail;
 428         }
 429         /* must be enabled */
 430         if (!(sc->flags & SCF_ENABLED)) {
 431                 ret = -EINVAL;
 432                 goto bail;
 433         }
 434
 435         /* allocate a buffer and copy the data in */
 436         tmpbuf = vmalloc(dp->len);
 437         if (!tmpbuf) {
 438                 ret = -ENOMEM;
 439                 goto bail;
 440         }
 441
 442         if (copy_from_user(tmpbuf,
 443                            (const void __user *) (unsigned long) dp->data,
 444                            dp->len)) {
 445                 ret = -EFAULT;
 446                 goto bail;
 447         }
 448
 449         /*
 450          * pkt_len is how much data we have to write, includes header and data.
 451          * total_len is length of the packet in Dwords plus the PBC should not
 452          * include the CRC.
 453          */
 454         pkt_len = dp->len >> 2;
 455         total_len = pkt_len + 2; /* PBC + packet */
 456
 457         /* if 0, fill in a default */
 458         if (dp->pbc == 0) {
 459                 struct hfi1_pportdata *ppd = dd->pport;
 460
 461                 hfi1_cdbg(PKT, "Generating PBC");
 462                 dp->pbc = create_pbc(ppd, 0, 0, 0, total_len);
 463         } else {
 464                 hfi1_cdbg(PKT, "Using passed in PBC");
 465         }
 466
 467         hfi1_cdbg(PKT, "Egress PBC content is 0x%llx", dp->pbc);
 468
 469         /*
 470          * The caller wants to wait until the packet is sent and to
 471          * check for errors.  The best we can do is wait until
 472          * the buffer credits are returned and check if any packet
 473          * error has occurred.  If there are any late errors, this
 474          * could miss it.  If there are other senders who generate
 475          * an error, this may find it.  However, in general, it
 476          * should catch most.
 477          */
 478         if (dp->flags & F_DIAGPKT_WAIT) {
 479                 /* always force a credit return */
 480                 dp->pbc |= PBC_CREDIT_RETURN;
 481                 /* turn on credit return interrupts */
 482                 sc_add_credit_return_intr(sc);
 483                 wait = kmalloc(sizeof(*wait), GFP_KERNEL);
 484                 if (!wait) {
 485                         ret = -ENOMEM;
 486                         goto bail;
 487                 }
 488                 init_completion(&wait->credits_returned);
 489                 atomic_set(&wait->count, 2);
 490                 wait->code = PRC_OK;
 491
 492                 credit_cb = diagpkt_complete;
 493                 credit_arg = wait;
 494         }
 495
 496         pbuf = sc_buffer_alloc(sc, total_len, credit_cb, credit_arg);
 497         if (!pbuf) {
 498                 /*
 499                  * No send buffer means no credit callback.  Undo
 500                  * the wait set-up that was done above.  We free wait
 501                  * because the callback will never be called.
 502                  */
 503                 if (dp->flags & F_DIAGPKT_WAIT) {
 504                         sc_del_credit_return_intr(sc);
 505                         kfree(wait);
 506                         wait = NULL;
 507                 }
 508                 ret = -ENOSPC;
 509                 goto bail;
 510         }
 511
 512         pio_copy(dd, pbuf, dp->pbc, tmpbuf, pkt_len);
 513         /* no flush needed as the HW knows the packet size */
 514
 515         ret = sizeof(*dp);
 516
 517         if (dp->flags & F_DIAGPKT_WAIT) {
 518                 /* wait for credit return */
 519                 ret = wait_for_completion_interruptible(
 520                                                 &wait->credits_returned);
 521                 /*
 522                  * If the wait returns an error, the wait was interrupted,
 523                  * e.g. with a ^C in the user program.  The callback is
 524                  * still pending.  This is OK as the wait structure is
 525                  * kmalloc'ed and the structure will free itself when
 526                  * all users are done with it.
 527                  *
 528                  * A context disable occurs on a send context restart, so
 529                  * include that in the list of errors below to check for.
 530                  * NOTE: PRC_FILL_ERR is at best informational and cannot
 531                  * be depended on.
 532                  */
 533                 if (!ret && (((wait->code & PRC_STATUS_ERR)
 534                                 || (wait->code & PRC_FILL_ERR)
 535                                 || (wait->code & PRC_SC_DISABLE))))
 536                         ret = -EIO;
 537
 538                 put_diagpkt_wait(wait); /* finished with the structure */
 539                 sc_del_credit_return_intr(sc);
 540         }
 541
 542 bail:
 543         vfree(tmpbuf);
 544         return ret;
 545 }
 546
 547 static ssize_t diagpkt_write(struct file *fp, const char __user *data,
 548                                  size_t count, loff_t *off)
 549 {
 550         struct hfi1_devdata *dd;
 551         struct send_context *sc;
 552         u8 vl;
 553
 554         struct diag_pkt dp;
 555
 556         if (count != sizeof(dp))
 557                 return -EINVAL;
 558
 559         if (copy_from_user(&dp, data, sizeof(dp)))
 560                 return -EFAULT;
 561
 562         /*
 563         * The Send Context is derived from the PbcVL value
 564         * if PBC is populated
 565         */
 566         if (dp.pbc) {
 567                 dd = hfi1_lookup(dp.unit);
 568                 if (dd == NULL)
 569                         return -ENODEV;
 570                 vl = (dp.pbc >> PBC_VL_SHIFT) & PBC_VL_MASK;
 571                 sc = dd->vld[vl].sc;
 572                 if (sc) {
 573                         dp.sw_index = sc->sw_index;
 574                         hfi1_cdbg(
 575                                PKT,
 576                                "Packet sent over VL %d via Send Context %u(%u)",
 577                                vl, sc->sw_index, sc->hw_context);
 578                 }
 579         }
 580
 581         return diagpkt_send(&dp);
 582 }
 583
 584 static int hfi1_snoop_add(struct hfi1_devdata *dd, const char *name)
 585 {
 586         int ret = 0;
 587
 588         dd->hfi1_snoop.mode_flag = 0;
 589         spin_lock_init(&dd->hfi1_snoop.snoop_lock);
 590         INIT_LIST_HEAD(&dd->hfi1_snoop.queue);
 591         init_waitqueue_head(&dd->hfi1_snoop.waitq);
 592
 593         ret = hfi1_cdev_init(HFI1_SNOOP_CAPTURE_BASE + dd->unit, name,
 594                              &snoop_file_ops,
 595                              &dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev,
 596                              false);
 597
 598         if (ret) {
 599                 dd_dev_err(dd, "Couldn't create %s device: %d", name, ret);
 600                 hfi1_cdev_cleanup(&dd->hfi1_snoop.cdev,
 601                                  &dd->hfi1_snoop.class_dev);
 602         }
 603
 604         return ret;
 605 }
 606
 607 static struct hfi1_devdata *hfi1_dd_from_sc_inode(struct inode *in)
 608 {
 609         int unit = iminor(in) - HFI1_SNOOP_CAPTURE_BASE;
 610         struct hfi1_devdata *dd = NULL;
 611
 612         dd = hfi1_lookup(unit);
 613         return dd;
 614
 615 }
 616
 617 /* clear or restore send context integrity checks */
 618 static void adjust_integrity_checks(struct hfi1_devdata *dd)
 619 {
 620         struct send_context *sc;
 621         unsigned long sc_flags;
 622         int i;
 623
 624         spin_lock_irqsave(&dd->sc_lock, sc_flags);
 625         for (i = 0; i < dd->num_send_contexts; i++) {
 626                 int enable;
 627
 628                 sc = dd->send_contexts[i].sc;
 629
 630                 if (!sc)
 631                         continue;       /* not allocated */
 632
 633                 enable = likely(!HFI1_CAP_IS_KSET(NO_INTEGRITY)) &&
 634                          dd->hfi1_snoop.mode_flag != HFI1_PORT_SNOOP_MODE;
 635
 636                 set_pio_integrity(sc);
 637
 638                 if (enable) /* take HFI_CAP_* flags into account */
 639                         hfi1_init_ctxt(sc);
 640         }
 641         spin_unlock_irqrestore(&dd->sc_lock, sc_flags);
 642 }
 643
 644 static int hfi1_snoop_open(struct inode *in, struct file *fp)
 645 {
 646         int ret;
 647         int mode_flag = 0;
 648         unsigned long flags = 0;
 649         struct hfi1_devdata *dd;
 650         struct list_head *queue;
 651
 652         mutex_lock(&hfi1_mutex);
 653
 654         dd = hfi1_dd_from_sc_inode(in);
 655         if (dd == NULL) {
 656                 ret = -ENODEV;
 657                 goto bail;
 658         }
 659
 660         /*
 661          * File mode determines snoop or capture. Some existing user
 662          * applications expect the capture device to be able to be opened RDWR
 663          * because they expect a dedicated capture device. For this reason we
 664          * support a module param to force capture mode even if the file open
 665          * mode matches snoop.
 666          */
 667         if ((fp->f_flags & O_ACCMODE) == O_RDONLY) {
 668                 snoop_dbg("Capture Enabled");
 669                 mode_flag = HFI1_PORT_CAPTURE_MODE;
 670         } else if ((fp->f_flags & O_ACCMODE) == O_RDWR) {
 671                 snoop_dbg("Snoop Enabled");
 672                 mode_flag = HFI1_PORT_SNOOP_MODE;
 673         } else {
 674                 snoop_dbg("Invalid");
 675                 ret =  -EINVAL;
 676                 goto bail;
 677         }
 678         queue = &dd->hfi1_snoop.queue;
 679
 680         /*
 681          * We are not supporting snoop and capture at the same time.
 682          */
 683         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 684         if (dd->hfi1_snoop.mode_flag) {
 685                 ret = -EBUSY;
 686                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 687                 goto bail;
 688         }
 689
 690         dd->hfi1_snoop.mode_flag = mode_flag;
 691         drain_snoop_list(queue);
 692
 693         dd->hfi1_snoop.filter_callback = NULL;
 694         dd->hfi1_snoop.filter_value = NULL;
 695
 696         /*
 697          * Send side packet integrity checks are not helpful when snooping so
 698          * disable and re-enable when we stop snooping.
 699          */
 700         if (mode_flag == HFI1_PORT_SNOOP_MODE) {
 701                 /* clear after snoop mode is on */
 702                 adjust_integrity_checks(dd); /* clear */
 703
 704                 /*
 705                  * We also do not want to be doing the DLID LMC check for
 706                  * ingressed packets.
 707                  */
 708                 dd->hfi1_snoop.dcc_cfg = read_csr(dd, DCC_CFG_PORT_CONFIG1);
 709                 write_csr(dd, DCC_CFG_PORT_CONFIG1,
 710                           (dd->hfi1_snoop.dcc_cfg >> 32) << 32);
 711         }
 712
 713         /*
 714          * As soon as we set these function pointers the recv and send handlers
 715          * are active. This is a race condition so we must make sure to drain
 716          * the queue and init filter values above. Technically we should add
 717          * locking here but all that will happen is on recv a packet will get
 718          * allocated and get stuck on the snoop_lock before getting added to the
 719          * queue. Same goes for send.
 720          */
 721         dd->rhf_rcv_function_map = snoop_rhf_rcv_functions;
 722         dd->process_pio_send = snoop_send_pio_handler;
 723         dd->process_dma_send = snoop_send_pio_handler;
 724         dd->pio_inline_send = snoop_inline_pio_send;
 725
 726         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 727         ret = 0;
 728
 729 bail:
 730         mutex_unlock(&hfi1_mutex);
 731
 732         return ret;
 733 }
 734
 735 static int hfi1_snoop_release(struct inode *in, struct file *fp)
 736 {
 737         unsigned long flags = 0;
 738         struct hfi1_devdata *dd;
 739         int mode_flag;
 740
 741         dd = hfi1_dd_from_sc_inode(in);
 742         if (dd == NULL)
 743                 return -ENODEV;
 744
 745         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 746
 747         /* clear the snoop mode before re-adjusting send context CSRs */
 748         mode_flag = dd->hfi1_snoop.mode_flag;
 749         dd->hfi1_snoop.mode_flag = 0;
 750
 751         /*
 752          * Drain the queue and clear the filters we are done with it. Don't
 753          * forget to restore the packet integrity checks
 754          */
 755         drain_snoop_list(&dd->hfi1_snoop.queue);
 756         if (mode_flag == HFI1_PORT_SNOOP_MODE) {
 757                 /* restore after snoop mode is clear */
 758                 adjust_integrity_checks(dd); /* restore */
 759
 760                 /*
 761                  * Also should probably reset the DCC_CONFIG1 register for DLID
 762                  * checking on incoming packets again. Use the value saved when
 763                  * opening the snoop device.
 764                  */
 765                 write_csr(dd, DCC_CFG_PORT_CONFIG1, dd->hfi1_snoop.dcc_cfg);
 766         }
 767
 768         dd->hfi1_snoop.filter_callback = NULL;
 769         kfree(dd->hfi1_snoop.filter_value);
 770         dd->hfi1_snoop.filter_value = NULL;
 771
 772         /*
 773          * User is done snooping and capturing, return control to the normal
 774          * handler. Re-enable SDMA handling.
 775          */
 776         dd->rhf_rcv_function_map = dd->normal_rhf_rcv_functions;
 777         dd->process_pio_send = hfi1_verbs_send_pio;
 778         dd->process_dma_send = hfi1_verbs_send_dma;
 779         dd->pio_inline_send = pio_copy;
 780
 781         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 782
 783         snoop_dbg("snoop/capture device released");
 784
 785         return 0;
 786 }
 787
 788 static unsigned int hfi1_snoop_poll(struct file *fp,
 789                                     struct poll_table_struct *wait)
 790 {
 791         int ret = 0;
 792         unsigned long flags = 0;
 793
 794         struct hfi1_devdata *dd;
 795
 796         dd = hfi1_dd_from_sc_inode(fp->f_inode);
 797         if (dd == NULL)
 798                 return -ENODEV;
 799
 800         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 801
 802         poll_wait(fp, &dd->hfi1_snoop.waitq, wait);
 803         if (!list_empty(&dd->hfi1_snoop.queue))
 804                 ret |= POLLIN | POLLRDNORM;
 805
 806         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 807         return ret;
 808
 809 }
 810
 811 static ssize_t hfi1_snoop_write(struct file *fp, const char __user *data,
 812                                 size_t count, loff_t *off)
 813 {
 814         struct diag_pkt dpkt;
 815         struct hfi1_devdata *dd;
 816         size_t ret;
 817         u8 byte_two, sl, sc5, sc4, vl, byte_one;
 818         struct send_context *sc;
 819         u32 len;
 820         u64 pbc;
 821         struct hfi1_ibport *ibp;
 822         struct hfi1_pportdata *ppd;
 823
 824         dd = hfi1_dd_from_sc_inode(fp->f_inode);
 825         if (dd == NULL)
 826                 return -ENODEV;
 827
 828         ppd = dd->pport;
 829         snoop_dbg("received %lu bytes from user", count);
 830
 831         memset(&dpkt, 0, sizeof(struct diag_pkt));
 832         dpkt.version = _DIAG_PKT_VERS;
 833         dpkt.unit = dd->unit;
 834         dpkt.port = 1;
 835
 836         if (likely(!(snoop_flags & SNOOP_USE_METADATA))) {
 837                 /*
 838                 * We need to generate the PBC and not let diagpkt_send do it,
 839                 * to do this we need the VL and the length in dwords.
 840                 * The VL can be determined by using the SL and looking up the
 841                 * SC. Then the SC can be converted into VL. The exception to
 842                 * this is those packets which are from an SMI queue pair.
 843                 * Since we can't detect anything about the QP here we have to
 844                 * rely on the SC. If its 0xF then we assume its SMI and
 845                 * do not look at the SL.
 846                 */
 847                 if (copy_from_user(&byte_one, data, 1))
 848                         return -EINVAL;
 849
 850                 if (copy_from_user(&byte_two, data+1, 1))
 851                         return -EINVAL;
 852
 853                 sc4 = (byte_one >> 4) & 0xf;
 854                 if (sc4 == 0xF) {
 855                         snoop_dbg("Detected VL15 packet ignoring SL in packet");
 856                         vl = sc4;
 857                 } else {
 858                         sl = (byte_two >> 4) & 0xf;
 859                         ibp = to_iport(&dd->verbs_dev.ibdev, 1);
 860                         sc5 = ibp->sl_to_sc[sl];
 861                         vl = sc_to_vlt(dd, sc5);
 862                         if (vl != sc4) {
 863                                 snoop_dbg("VL %d does not match SC %d of packet",
 864                                           vl, sc4);
 865                                 return -EINVAL;
 866                         }
 867                 }
 868
 869                 sc = dd->vld[vl].sc; /* Look up the context based on VL */
 870                 if (sc) {
 871                         dpkt.sw_index = sc->sw_index;
 872                         snoop_dbg("Sending on context %u(%u)", sc->sw_index,
 873                                   sc->hw_context);
 874                 } else {
 875                         snoop_dbg("Could not find context for vl %d", vl);
 876                         return -EINVAL;
 877                 }
 878
 879                 len = (count >> 2) + 2; /* Add in PBC */
 880                 pbc = create_pbc(ppd, 0, 0, vl, len);
 881         } else {
 882                 if (copy_from_user(&pbc, data, sizeof(pbc)))
 883                         return -EINVAL;
 884                 vl = (pbc >> PBC_VL_SHIFT) & PBC_VL_MASK;
 885                 sc = dd->vld[vl].sc; /* Look up the context based on VL */
 886                 if (sc) {
 887                         dpkt.sw_index = sc->sw_index;
 888                 } else {
 889                         snoop_dbg("Could not find context for vl %d", vl);
 890                         return -EINVAL;
 891                 }
 892                 data += sizeof(pbc);
 893                 count -= sizeof(pbc);
 894         }
 895         dpkt.len = count;
 896         dpkt.data = (unsigned long)data;
 897
 898         snoop_dbg("PBC: vl=0x%llx Length=0x%llx",
 899                   (pbc >> 12) & 0xf,
 900                   (pbc & 0xfff));
 901
 902         dpkt.pbc = pbc;
 903         ret = diagpkt_send(&dpkt);
 904         /*
 905          * diagpkt_send only returns number of bytes in the diagpkt so patch
 906          * that up here before returning.
 907          */
 908         if (ret == sizeof(dpkt))
 909                 return count;
 910
 911         return ret;
 912 }
 913
 914 static ssize_t hfi1_snoop_read(struct file *fp, char __user *data,
 915                                size_t pkt_len, loff_t *off)
 916 {
 917         ssize_t ret = 0;
 918         unsigned long flags = 0;
 919         struct snoop_packet *packet = NULL;
 920         struct hfi1_devdata *dd;
 921
 922         dd = hfi1_dd_from_sc_inode(fp->f_inode);
 923         if (dd == NULL)
 924                 return -ENODEV;
 925
 926         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 927
 928         while (list_empty(&dd->hfi1_snoop.queue)) {
 929                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 930
 931                 if (fp->f_flags & O_NONBLOCK)
 932                         return -EAGAIN;
 933
 934                 if (wait_event_interruptible(
 935                                 dd->hfi1_snoop.waitq,
 936                                 !list_empty(&dd->hfi1_snoop.queue)))
 937                         return -EINTR;
 938
 939                 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 940         }
 941
 942         if (!list_empty(&dd->hfi1_snoop.queue)) {
 943                 packet = list_entry(dd->hfi1_snoop.queue.next,
 944                                     struct snoop_packet, list);
 945                 list_del(&packet->list);
 946                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 947                 if (pkt_len >= packet->total_len) {
 948                         if (copy_to_user(data, packet->data,
 949                                 packet->total_len))
 950                                 ret = -EFAULT;
 951                         else
 952                                 ret = packet->total_len;
 953                 } else
 954                         ret = -EINVAL;
 955
 956                 kfree(packet);
 957         } else
 958                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 959
 960         return ret;
 961 }
 962
 963 static long hfi1_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
 964 {
 965         struct hfi1_devdata *dd;
 966         void *filter_value = NULL;
 967         long ret = 0;
 968         int value = 0;
 969         u8 physState = 0;
 970         u8 linkState = 0;
 971         u16 devState = 0;
 972         unsigned long flags = 0;
 973         unsigned long *argp = NULL;
 974         struct hfi1_packet_filter_command filter_cmd = {0};
 975         int mode_flag = 0;
 976         struct hfi1_pportdata *ppd = NULL;
 977         unsigned int index;
 978         struct hfi1_link_info link_info;
 979
 980         dd = hfi1_dd_from_sc_inode(fp->f_inode);
 981         if (dd == NULL)
 982                 return -ENODEV;
 983
 984         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 985
 986         mode_flag = dd->hfi1_snoop.mode_flag;
 987
 988         if (((_IOC_DIR(cmd) & _IOC_READ)
 989             && !access_ok(VERIFY_WRITE, (void __user *)arg, _IOC_SIZE(cmd)))
 990             || ((_IOC_DIR(cmd) & _IOC_WRITE)
 991             && !access_ok(VERIFY_READ, (void __user *)arg, _IOC_SIZE(cmd)))) {
 992                 ret = -EFAULT;
 993         } else if (!capable(CAP_SYS_ADMIN)) {
 994                 ret = -EPERM;
 995         } else if ((mode_flag & HFI1_PORT_CAPTURE_MODE) &&
 996                    (cmd != HFI1_SNOOP_IOCCLEARQUEUE) &&
 997                    (cmd != HFI1_SNOOP_IOCCLEARFILTER) &&
 998                    (cmd != HFI1_SNOOP_IOCSETFILTER)) {
 999                 /* Capture devices are allowed only 3 operations
1000                  * 1.Clear capture queue
1001                  * 2.Clear capture filter
1002                  * 3.Set capture filter
1003                  * Other are invalid.
1004                  */
1005                 ret = -EINVAL;
1006         } else {
1007                 switch (cmd) {
1008                 case HFI1_SNOOP_IOCSETLINKSTATE:
1009                         snoop_dbg("HFI1_SNOOP_IOCSETLINKSTATE is not valid");
1010                         ret = -EINVAL;
1011                         break;
1012
1013                 case HFI1_SNOOP_IOCSETLINKSTATE_EXTRA:
1014                         memset(&link_info, 0, sizeof(link_info));
1015
1016                         if (copy_from_user(&link_info,
1017                                 (struct hfi1_link_info __user *)arg,
1018                                 sizeof(link_info)))
1019                                 ret = -EFAULT;
1020
1021                         value = link_info.port_state;
1022                         index = link_info.port_number;
1023                         if (index > dd->num_pports - 1) {
1024                                 ret = -EINVAL;
1025                                 break;
1026                         }
1027
1028                         ppd = &dd->pport[index];
1029                         if (!ppd) {
1030                                 ret = -EINVAL;
1031                                 break;
1032                         }
1033
1034                         /* What we want to transition to */
1035                         physState = (value >> 4) & 0xF;
1036                         linkState = value & 0xF;
1037                         snoop_dbg("Setting link state 0x%x", value);
1038
1039                         switch (linkState) {
1040                         case IB_PORT_NOP:
1041                                 if (physState == 0)
1042                                         break;
1043                                         /* fall through */
1044                         case IB_PORT_DOWN:
1045                                 switch (physState) {
1046                                 case 0:
1047                                         devState = HLS_DN_DOWNDEF;
1048                                         break;
1049                                 case 2:
1050                                         devState = HLS_DN_POLL;
1051                                         break;
1052                                 case 3:
1053                                         devState = HLS_DN_DISABLE;
1054                                         break;
1055                                 default:
1056                                         ret = -EINVAL;
1057                                         goto done;
1058                                 }
1059                                 ret = set_link_state(ppd, devState);
1060                                 break;
1061                         case IB_PORT_ARMED:
1062                                 ret = set_link_state(ppd, HLS_UP_ARMED);
1063                                 if (!ret)
1064                                         send_idle_sma(dd, SMA_IDLE_ARM);
1065                                 break;
1066                         case IB_PORT_ACTIVE:
1067                                 ret = set_link_state(ppd, HLS_UP_ACTIVE);
1068                                 if (!ret)
1069                                         send_idle_sma(dd, SMA_IDLE_ACTIVE);
1070                                 break;
1071                         default:
1072                                 ret = -EINVAL;
1073                                 break;
1074                         }
1075
1076                         if (ret)
1077                                 break;
1078                         /* fall through */
1079                 case HFI1_SNOOP_IOCGETLINKSTATE:
1080                 case HFI1_SNOOP_IOCGETLINKSTATE_EXTRA:
1081                         if (cmd == HFI1_SNOOP_IOCGETLINKSTATE_EXTRA) {
1082                                 memset(&link_info, 0, sizeof(link_info));
1083                                 if (copy_from_user(&link_info,
1084                                         (struct hfi1_link_info __user *)arg,
1085                                         sizeof(link_info)))
1086                                         ret = -EFAULT;
1087                                 index = link_info.port_number;
1088                         } else {
1089                                 ret = __get_user(index, (int __user *) arg);
1090                                 if (ret !=  0)
1091                                         break;
1092                         }
1093
1094                         if (index > dd->num_pports - 1) {
1095                                 ret = -EINVAL;
1096                                 break;
1097                         }
1098
1099                         ppd = &dd->pport[index];
1100                         if (!ppd) {
1101                                 ret = -EINVAL;
1102                                 break;
1103                         }
1104                         value = hfi1_ibphys_portstate(ppd);
1105                         value <<= 4;
1106                         value |= driver_lstate(ppd);
1107
1108                         snoop_dbg("Link port | Link State: %d", value);
1109
1110                         if ((cmd == HFI1_SNOOP_IOCGETLINKSTATE_EXTRA) ||
1111                             (cmd == HFI1_SNOOP_IOCSETLINKSTATE_EXTRA)) {
1112                                 link_info.port_state = value;
1113                                 link_info.node_guid = cpu_to_be64(ppd->guid);
1114                                 link_info.link_speed_active =
1115                                                         ppd->link_speed_active;
1116                                 link_info.link_width_active =
1117                                                         ppd->link_width_active;
1118                                 if (copy_to_user(
1119                                         (struct hfi1_link_info __user *)arg,
1120                                         &link_info, sizeof(link_info)))
1121                                         ret = -EFAULT;
1122                         } else {
1123                                 ret = __put_user(value, (int __user *)arg);
1124                         }
1125                         break;
1126
1127                 case HFI1_SNOOP_IOCCLEARQUEUE:
1128                         snoop_dbg("Clearing snoop queue");
1129                         drain_snoop_list(&dd->hfi1_snoop.queue);
1130                         break;
1131
1132                 case HFI1_SNOOP_IOCCLEARFILTER:
1133                         snoop_dbg("Clearing filter");
1134                         if (dd->hfi1_snoop.filter_callback) {
1135                                 /* Drain packets first */
1136                                 drain_snoop_list(&dd->hfi1_snoop.queue);
1137                                 dd->hfi1_snoop.filter_callback = NULL;
1138                         }
1139                         kfree(dd->hfi1_snoop.filter_value);
1140                         dd->hfi1_snoop.filter_value = NULL;
1141                         break;
1142
1143                 case HFI1_SNOOP_IOCSETFILTER:
1144                         snoop_dbg("Setting filter");
1145                         /* just copy command structure */
1146                         argp = (unsigned long *)arg;
1147                         if (copy_from_user(&filter_cmd, (void __user *)argp,
1148                                              sizeof(filter_cmd))) {
1149                                 ret = -EFAULT;
1150                                 break;
1151                         }
1152                         if (filter_cmd.opcode >= HFI1_MAX_FILTERS) {
1153                                 pr_alert("Invalid opcode in request\n");
1154                                 ret = -EINVAL;
1155                                 break;
1156                         }
1157
1158                         snoop_dbg("Opcode %d Len %d Ptr %p",
1159                                    filter_cmd.opcode, filter_cmd.length,
1160                                    filter_cmd.value_ptr);
1161
1162                         filter_value = kzalloc(
1163                                                 filter_cmd.length * sizeof(u8),
1164                                                 GFP_KERNEL);
1165                         if (!filter_value) {
1166                                 pr_alert("Not enough memory\n");
1167                                 ret = -ENOMEM;
1168                                 break;
1169                         }
1170                         /* copy remaining data from userspace */
1171                         if (copy_from_user((u8 *)filter_value,
1172                                         (void __user *)filter_cmd.value_ptr,
1173                                         filter_cmd.length)) {
1174                                 kfree(filter_value);
1175                                 ret = -EFAULT;
1176                                 break;
1177                         }
1178                         /* Drain packets first */
1179                         drain_snoop_list(&dd->hfi1_snoop.queue);
1180                         dd->hfi1_snoop.filter_callback =
1181                                 hfi1_filters[filter_cmd.opcode].filter;
1182                         /* just in case we see back to back sets */
1183                         kfree(dd->hfi1_snoop.filter_value);
1184                         dd->hfi1_snoop.filter_value = filter_value;
1185
1186                         break;
1187                 case HFI1_SNOOP_IOCGETVERSION:
1188                         value = SNOOP_CAPTURE_VERSION;
1189                         snoop_dbg("Getting version: %d", value);
1190                         ret = __put_user(value, (int __user *)arg);
1191                         break;
1192                 case HFI1_SNOOP_IOCSET_OPTS:
1193                         snoop_flags = 0;
1194                         ret = __get_user(value, (int __user *) arg);
1195                         if (ret != 0)
1196                                 break;
1197
1198                         snoop_dbg("Setting snoop option %d", value);
1199                         if (value & SNOOP_DROP_SEND)
1200                                 snoop_flags |= SNOOP_DROP_SEND;
1201                         if (value & SNOOP_USE_METADATA)
1202                                 snoop_flags |= SNOOP_USE_METADATA;
1203                         break;
1204                 default:
1205                         ret = -ENOTTY;
1206                         break;
1207                 }
1208         }
1209 done:
1210         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1211         return ret;
1212 }
1213
1214 static void snoop_list_add_tail(struct snoop_packet *packet,
1215                                 struct hfi1_devdata *dd)
1216 {
1217         unsigned long flags = 0;
1218
1219         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
1220         if (likely((dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE) ||
1221                    (dd->hfi1_snoop.mode_flag & HFI1_PORT_CAPTURE_MODE))) {
1222                 list_add_tail(&packet->list, &dd->hfi1_snoop.queue);
1223                 snoop_dbg("Added packet to list");
1224         }
1225
1226         /*
1227          * Technically we can could have closed the snoop device while waiting
1228          * on the above lock and it is gone now. The snoop mode_flag will
1229          * prevent us from adding the packet to the queue though.
1230          */
1231
1232         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1233         wake_up_interruptible(&dd->hfi1_snoop.waitq);
1234 }
1235
1236 static inline int hfi1_filter_check(void *val, const char *msg)
1237 {
1238         if (!val) {
1239                 snoop_dbg("Error invalid %s value for filter", msg);
1240                 return HFI1_FILTER_ERR;
1241         }
1242         return 0;
1243 }
1244
1245 static int hfi1_filter_lid(void *ibhdr, void *packet_data, void *value)
1246 {
1247         struct hfi1_ib_header *hdr;
1248         int ret;
1249
1250         ret = hfi1_filter_check(ibhdr, "header");
1251         if (ret)
1252                 return ret;
1253         ret = hfi1_filter_check(value, "user");
1254         if (ret)
1255                 return ret;
1256         hdr = (struct hfi1_ib_header *)ibhdr;
1257
1258         if (*((u16 *)value) == be16_to_cpu(hdr->lrh[3])) /* matches slid */
1259                 return HFI1_FILTER_HIT; /* matched */
1260
1261         return HFI1_FILTER_MISS; /* Not matched */
1262 }
1263
1264 static int hfi1_filter_dlid(void *ibhdr, void *packet_data, void *value)
1265 {
1266         struct hfi1_ib_header *hdr;
1267         int ret;
1268
1269         ret = hfi1_filter_check(ibhdr, "header");
1270         if (ret)
1271                 return ret;
1272         ret = hfi1_filter_check(value, "user");
1273         if (ret)
1274                 return ret;
1275
1276         hdr = (struct hfi1_ib_header *)ibhdr;
1277
1278         if (*((u16 *)value) == be16_to_cpu(hdr->lrh[1]))
1279                 return HFI1_FILTER_HIT;
1280
1281         return HFI1_FILTER_MISS;
1282 }
1283
1284 /* Not valid for outgoing packets, send handler passes null for data*/
1285 static int hfi1_filter_mad_mgmt_class(void *ibhdr, void *packet_data,
1286                                       void *value)
1287 {
1288         struct hfi1_ib_header *hdr;
1289         struct hfi1_other_headers *ohdr = NULL;
1290         struct ib_smp *smp = NULL;
1291         u32 qpn = 0;
1292         int ret;
1293
1294         ret = hfi1_filter_check(ibhdr, "header");
1295         if (ret)
1296                 return ret;
1297         ret = hfi1_filter_check(packet_data, "packet_data");
1298         if (ret)
1299                 return ret;
1300         ret = hfi1_filter_check(value, "user");
1301         if (ret)
1302                 return ret;
1303
1304         hdr = (struct hfi1_ib_header *)ibhdr;
1305
1306         /* Check for GRH */
1307         if ((be16_to_cpu(hdr->lrh[0]) & 3) == HFI1_LRH_BTH)
1308                 ohdr = &hdr->u.oth; /* LRH + BTH + DETH */
1309         else
1310                 ohdr = &hdr->u.l.oth; /* LRH + GRH + BTH + DETH */
1311
1312         qpn = be32_to_cpu(ohdr->bth[1]) & 0x00FFFFFF;
1313         if (qpn <= 1) {
1314                 smp = (struct ib_smp *)packet_data;
1315                 if (*((u8 *)value) == smp->mgmt_class)
1316                         return HFI1_FILTER_HIT;
1317                 else
1318                         return HFI1_FILTER_MISS;
1319         }
1320         return HFI1_FILTER_ERR;
1321 }
1322
1323 static int hfi1_filter_qp_number(void *ibhdr, void *packet_data, void *value)
1324 {
1325
1326         struct hfi1_ib_header *hdr;
1327         struct hfi1_other_headers *ohdr = NULL;
1328         int ret;
1329
1330         ret = hfi1_filter_check(ibhdr, "header");
1331         if (ret)
1332                 return ret;
1333         ret = hfi1_filter_check(value, "user");
1334         if (ret)
1335                 return ret;
1336
1337         hdr = (struct hfi1_ib_header *)ibhdr;
1338
1339         /* Check for GRH */
1340         if ((be16_to_cpu(hdr->lrh[0]) & 3) == HFI1_LRH_BTH)
1341                 ohdr = &hdr->u.oth; /* LRH + BTH + DETH */
1342         else
1343                 ohdr = &hdr->u.l.oth; /* LRH + GRH + BTH + DETH */
1344         if (*((u32 *)value) == (be32_to_cpu(ohdr->bth[1]) & 0x00FFFFFF))
1345                 return HFI1_FILTER_HIT;
1346
1347         return HFI1_FILTER_MISS;
1348 }
1349
1350 static int hfi1_filter_ibpacket_type(void *ibhdr, void *packet_data,
1351                                      void *value)
1352 {
1353         u32 lnh = 0;
1354         u8 opcode = 0;
1355         struct hfi1_ib_header *hdr;
1356         struct hfi1_other_headers *ohdr = NULL;
1357         int ret;
1358
1359         ret = hfi1_filter_check(ibhdr, "header");
1360         if (ret)
1361                 return ret;
1362         ret = hfi1_filter_check(value, "user");
1363         if (ret)
1364                 return ret;
1365
1366         hdr = (struct hfi1_ib_header *)ibhdr;
1367
1368         lnh = (be16_to_cpu(hdr->lrh[0]) & 3);
1369
1370         if (lnh == HFI1_LRH_BTH)
1371                 ohdr = &hdr->u.oth;
1372         else if (lnh == HFI1_LRH_GRH)
1373                 ohdr = &hdr->u.l.oth;
1374         else
1375                 return HFI1_FILTER_ERR;
1376
1377         opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
1378
1379         if (*((u8 *)value) == ((opcode >> 5) & 0x7))
1380                 return HFI1_FILTER_HIT;
1381
1382         return HFI1_FILTER_MISS;
1383 }
1384
1385 static int hfi1_filter_ib_service_level(void *ibhdr, void *packet_data,
1386                                         void *value)
1387 {
1388         struct hfi1_ib_header *hdr;
1389         int ret;
1390
1391         ret = hfi1_filter_check(ibhdr, "header");
1392         if (ret)
1393                 return ret;
1394         ret = hfi1_filter_check(value, "user");
1395         if (ret)
1396                 return ret;
1397
1398         hdr = (struct hfi1_ib_header *)ibhdr;
1399
1400         if ((*((u8 *)value)) == ((be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF))
1401                 return HFI1_FILTER_HIT;
1402
1403         return HFI1_FILTER_MISS;
1404 }
1405
1406 static int hfi1_filter_ib_pkey(void *ibhdr, void *packet_data, void *value)
1407 {
1408
1409         u32 lnh = 0;
1410         struct hfi1_ib_header *hdr;
1411         struct hfi1_other_headers *ohdr = NULL;
1412         int ret;
1413
1414         ret = hfi1_filter_check(ibhdr, "header");
1415         if (ret)
1416                 return ret;
1417         ret = hfi1_filter_check(value, "user");
1418         if (ret)
1419                 return ret;
1420
1421         hdr = (struct hfi1_ib_header *)ibhdr;
1422
1423         lnh = (be16_to_cpu(hdr->lrh[0]) & 3);
1424         if (lnh == HFI1_LRH_BTH)
1425                 ohdr = &hdr->u.oth;
1426         else if (lnh == HFI1_LRH_GRH)
1427                 ohdr = &hdr->u.l.oth;
1428         else
1429                 return HFI1_FILTER_ERR;
1430
1431         /* P_key is 16-bit entity, however top most bit indicates
1432          * type of membership. 0 for limited and 1 for Full.
1433          * Limited members cannot accept information from other
1434          * Limited members, but communication is allowed between
1435          * every other combination of membership.
1436          * Hence we'll omit comparing top-most bit while filtering
1437          */
1438
1439         if ((*(u16 *)value & 0x7FFF) ==
1440                 ((be32_to_cpu(ohdr->bth[0])) & 0x7FFF))
1441                 return HFI1_FILTER_HIT;
1442
1443         return HFI1_FILTER_MISS;
1444 }
1445
1446 /*
1447  * If packet_data is NULL then this is coming from one of the send functions.
1448  * Thus we know if its an ingressed or egressed packet.
1449  */
1450 static int hfi1_filter_direction(void *ibhdr, void *packet_data, void *value)
1451 {
1452         u8 user_dir = *(u8 *)value;
1453         int ret;
1454
1455         ret = hfi1_filter_check(value, "user");
1456         if (ret)
1457                 return ret;
1458
1459         if (packet_data) {
1460                 /* Incoming packet */
1461                 if (user_dir & HFI1_SNOOP_INGRESS)
1462                         return HFI1_FILTER_HIT;
1463         } else {
1464                 /* Outgoing packet */
1465                 if (user_dir & HFI1_SNOOP_EGRESS)
1466                         return HFI1_FILTER_HIT;
1467         }
1468
1469         return HFI1_FILTER_MISS;
1470 }
1471
1472 /*
1473  * Allocate a snoop packet. The structure that is stored in the ring buffer, not
1474  * to be confused with an hfi packet type.
1475  */
1476 static struct snoop_packet *allocate_snoop_packet(u32 hdr_len,
1477                                                   u32 data_len,
1478                                                   u32 md_len)
1479 {
1480
1481         struct snoop_packet *packet = NULL;
1482
1483         packet = kzalloc(sizeof(struct snoop_packet) + hdr_len + data_len
1484                          + md_len,
1485                          GFP_ATOMIC | __GFP_NOWARN);
1486         if (likely(packet))
1487                 INIT_LIST_HEAD(&packet->list);
1488
1489
1490         return packet;
1491 }
1492
1493 /*
1494  * Instead of having snoop and capture code intermixed with the recv functions,
1495  * both the interrupt handler and hfi1_ib_rcv() we are going to hijack the call
1496  * and land in here for snoop/capture but if not enabled the call will go
1497  * through as before. This gives us a single point to constrain all of the snoop
1498  * snoop recv logic. There is nothing special that needs to happen for bypass
1499  * packets. This routine should not try to look into the packet. It just copied
1500  * it. There is no guarantee for filters when it comes to bypass packets as
1501  * there is no specific support. Bottom line is this routine does now even know
1502  * what a bypass packet is.
1503  */
1504 int snoop_recv_handler(struct hfi1_packet *packet)
1505 {
1506         struct hfi1_pportdata *ppd = packet->rcd->ppd;
1507         struct hfi1_ib_header *hdr = packet->hdr;
1508         int header_size = packet->hlen;
1509         void *data = packet->ebuf;
1510         u32 tlen = packet->tlen;
1511         struct snoop_packet *s_packet = NULL;
1512         int ret;
1513         int snoop_mode = 0;
1514         u32 md_len = 0;
1515         struct capture_md md;
1516
1517         snoop_dbg("PACKET IN: hdr size %d tlen %d data %p", header_size, tlen,
1518                   data);
1519
1520         trace_snoop_capture(ppd->dd, header_size, hdr, tlen - header_size,
1521                             data);
1522
1523         if (!ppd->dd->hfi1_snoop.filter_callback) {
1524                 snoop_dbg("filter not set");
1525                 ret = HFI1_FILTER_HIT;
1526         } else {
1527                 ret = ppd->dd->hfi1_snoop.filter_callback(hdr, data,
1528                                         ppd->dd->hfi1_snoop.filter_value);
1529         }
1530
1531         switch (ret) {
1532         case HFI1_FILTER_ERR:
1533                 snoop_dbg("Error in filter call");
1534                 break;
1535         case HFI1_FILTER_MISS:
1536                 snoop_dbg("Filter Miss");
1537                 break;
1538         case HFI1_FILTER_HIT:
1539
1540                 if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1541                         snoop_mode = 1;
1542                 if ((snoop_mode == 0) ||
1543                     unlikely(snoop_flags & SNOOP_USE_METADATA))
1544                         md_len = sizeof(struct capture_md);
1545
1546
1547                 s_packet = allocate_snoop_packet(header_size,
1548                                                  tlen - header_size,
1549                                                  md_len);
1550
1551                 if (unlikely(s_packet == NULL)) {
1552                         dd_dev_warn_ratelimited(ppd->dd, "Unable to allocate snoop/capture packet\n");
1553                         break;
1554                 }
1555
1556                 if (md_len > 0) {
1557                         memset(&md, 0, sizeof(struct capture_md));
1558                         md.port = 1;
1559                         md.dir = PKT_DIR_INGRESS;
1560                         md.u.rhf = packet->rhf;
1561                         memcpy(s_packet->data, &md, md_len);
1562                 }
1563
1564                 /* We should always have a header */
1565                 if (hdr) {
1566                         memcpy(s_packet->data + md_len, hdr, header_size);
1567                 } else {
1568                         dd_dev_err(ppd->dd, "Unable to copy header to snoop/capture packet\n");
1569                         kfree(s_packet);
1570                         break;
1571                 }
1572
1573                 /*
1574                  * Packets with no data are possible. If there is no data needed
1575                  * to take care of the last 4 bytes which are normally included
1576                  * with data buffers and are included in tlen.  Since we kzalloc
1577                  * the buffer we do not need to set any values but if we decide
1578                  * not to use kzalloc we should zero them.
1579                  */
1580                 if (data)
1581                         memcpy(s_packet->data + header_size + md_len, data,
1582                                tlen - header_size);
1583
1584                 s_packet->total_len = tlen + md_len;
1585                 snoop_list_add_tail(s_packet, ppd->dd);
1586
1587                 /*
1588                  * If we are snooping the packet not capturing then throw away
1589                  * after adding to the list.
1590                  */
1591                 snoop_dbg("Capturing packet");
1592                 if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE) {
1593                         snoop_dbg("Throwing packet away");
1594                         /*
1595                          * If we are dropping the packet we still may need to
1596                          * handle the case where error flags are set, this is
1597                          * normally done by the type specific handler but that
1598                          * won't be called in this case.
1599                          */
1600                         if (unlikely(rhf_err_flags(packet->rhf)))
1601                                 handle_eflags(packet);
1602
1603                         /* throw the packet on the floor */
1604                         return RHF_RCV_CONTINUE;
1605                 }
1606                 break;
1607         default:
1608                 break;
1609         }
1610
1611         /*
1612          * We do not care what type of packet came in here - just pass it off
1613          * to the normal handler.
1614          */
1615         return ppd->dd->normal_rhf_rcv_functions[rhf_rcv_type(packet->rhf)]
1616                         (packet);
1617 }
1618
1619 /*
1620  * Handle snooping and capturing packets when sdma is being used.
1621  */
1622 int snoop_send_dma_handler(struct hfi1_qp *qp, struct ahg_ib_header *ibhdr,
1623                            u32 hdrwords, struct hfi1_sge_state *ss, u32 len,
1624                            u32 plen, u32 dwords, u64 pbc)
1625 {
1626         pr_alert("Snooping/Capture of  Send DMA Packets Is Not Supported!\n");
1627         snoop_dbg("Unsupported Operation");
1628         return hfi1_verbs_send_dma(qp, ibhdr, hdrwords, ss, len, plen, dwords,
1629                                   0);
1630 }
1631
1632 /*
1633  * Handle snooping and capturing packets when pio is being used. Does not handle
1634  * bypass packets. The only way to send a bypass packet currently is to use the
1635  * diagpkt interface. When that interface is enable snoop/capture is not.
1636  */
1637 int snoop_send_pio_handler(struct hfi1_qp *qp, struct ahg_ib_header *ahdr,
1638                            u32 hdrwords, struct hfi1_sge_state *ss, u32 len,
1639                            u32 plen, u32 dwords, u64 pbc)
1640 {
1641         struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
1642         struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
1643         struct snoop_packet *s_packet = NULL;
1644         u32 *hdr = (u32 *)&ahdr->ibh;
1645         u32 length = 0;
1646         struct hfi1_sge_state temp_ss;
1647         void *data = NULL;
1648         void *data_start = NULL;
1649         int ret;
1650         int snoop_mode = 0;
1651         int md_len = 0;
1652         struct capture_md md;
1653         u32 vl;
1654         u32 hdr_len = hdrwords << 2;
1655         u32 tlen = HFI1_GET_PKT_LEN(&ahdr->ibh);
1656
1657         md.u.pbc = 0;
1658
1659         snoop_dbg("PACKET OUT: hdrword %u len %u plen %u dwords %u tlen %u",
1660                   hdrwords, len, plen, dwords, tlen);
1661         if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1662                 snoop_mode = 1;
1663         if ((snoop_mode == 0) ||
1664             unlikely(snoop_flags & SNOOP_USE_METADATA))
1665                 md_len = sizeof(struct capture_md);
1666
1667         /* not using ss->total_len as arg 2 b/c that does not count CRC */
1668         s_packet = allocate_snoop_packet(hdr_len, tlen - hdr_len, md_len);
1669
1670         if (unlikely(s_packet == NULL)) {
1671                 dd_dev_warn_ratelimited(ppd->dd, "Unable to allocate snoop/capture packet\n");
1672                 goto out;
1673         }
1674
1675         s_packet->total_len = tlen + md_len;
1676
1677         if (md_len > 0) {
1678                 memset(&md, 0, sizeof(struct capture_md));
1679                 md.port = 1;
1680                 md.dir = PKT_DIR_EGRESS;
1681                 if (likely(pbc == 0)) {
1682                         vl = be16_to_cpu(ahdr->ibh.lrh[0]) >> 12;
1683                         md.u.pbc = create_pbc(ppd, 0, qp->s_srate, vl, plen);
1684                 } else {
1685                         md.u.pbc = 0;
1686                 }
1687                 memcpy(s_packet->data, &md, md_len);
1688         } else {
1689                 md.u.pbc = pbc;
1690         }
1691
1692         /* Copy header */
1693         if (likely(hdr)) {
1694                 memcpy(s_packet->data + md_len, hdr, hdr_len);
1695         } else {
1696                 dd_dev_err(ppd->dd,
1697                            "Unable to copy header to snoop/capture packet\n");
1698                 kfree(s_packet);
1699                 goto out;
1700         }
1701
1702         if (ss) {
1703                 data = s_packet->data + hdr_len + md_len;
1704                 data_start = data;
1705
1706                 /*
1707                  * Copy SGE State
1708                  * The update_sge() function below will not modify the
1709                  * individual SGEs in the array. It will make a copy each time
1710                  * and operate on that. So we only need to copy this instance
1711                  * and it won't impact PIO.
1712                  */
1713                 temp_ss = *ss;
1714                 length = len;
1715
1716                 snoop_dbg("Need to copy %d bytes", length);
1717                 while (length) {
1718                         void *addr = temp_ss.sge.vaddr;
1719                         u32 slen = temp_ss.sge.length;
1720
1721                         if (slen > length) {
1722                                 slen = length;
1723                                 snoop_dbg("slen %d > len %d", slen, length);
1724                         }
1725                         snoop_dbg("copy %d to %p", slen, addr);
1726                         memcpy(data, addr, slen);
1727                         update_sge(&temp_ss, slen);
1728                         length -= slen;
1729                         data += slen;
1730                         snoop_dbg("data is now %p bytes left %d", data, length);
1731                 }
1732                 snoop_dbg("Completed SGE copy");
1733         }
1734
1735         /*
1736          * Why do the filter check down here? Because the event tracing has its
1737          * own filtering and we need to have the walked the SGE list.
1738          */
1739         if (!ppd->dd->hfi1_snoop.filter_callback) {
1740                 snoop_dbg("filter not set\n");
1741                 ret = HFI1_FILTER_HIT;
1742         } else {
1743                 ret = ppd->dd->hfi1_snoop.filter_callback(
1744                                         &ahdr->ibh,
1745                                         NULL,
1746                                         ppd->dd->hfi1_snoop.filter_value);
1747         }
1748
1749         switch (ret) {
1750         case HFI1_FILTER_ERR:
1751                 snoop_dbg("Error in filter call");
1752                 /* fall through */
1753         case HFI1_FILTER_MISS:
1754                 snoop_dbg("Filter Miss");
1755                 kfree(s_packet);
1756                 break;
1757         case HFI1_FILTER_HIT:
1758                 snoop_dbg("Capturing packet");
1759                 snoop_list_add_tail(s_packet, ppd->dd);
1760
1761                 if (unlikely((snoop_flags & SNOOP_DROP_SEND) &&
1762                              (ppd->dd->hfi1_snoop.mode_flag &
1763                               HFI1_PORT_SNOOP_MODE))) {
1764                         unsigned long flags;
1765
1766                         snoop_dbg("Dropping packet");
1767                         if (qp->s_wqe) {
1768                                 spin_lock_irqsave(&qp->s_lock, flags);
1769                                 hfi1_send_complete(
1770                                         qp,
1771                                         qp->s_wqe,
1772                                         IB_WC_SUCCESS);
1773                                 spin_unlock_irqrestore(&qp->s_lock, flags);
1774                         } else if (qp->ibqp.qp_type == IB_QPT_RC) {
1775                                 spin_lock_irqsave(&qp->s_lock, flags);
1776                                 hfi1_rc_send_complete(qp, &ahdr->ibh);
1777                                 spin_unlock_irqrestore(&qp->s_lock, flags);
1778                         }
1779                         return 0;
1780                 }
1781                 break;
1782         default:
1783                 kfree(s_packet);
1784                 break;
1785         }
1786 out:
1787         return hfi1_verbs_send_pio(qp, ahdr, hdrwords, ss, len, plen, dwords,
1788                                   md.u.pbc);
1789 }
1790
1791 /*
1792  * Callers of this must pass a hfi1_ib_header type for the from ptr. Currently
1793  * this can be used anywhere, but the intention is for inline ACKs for RC and
1794  * CCA packets. We don't restrict this usage though.
1795  */
1796 void snoop_inline_pio_send(struct hfi1_devdata *dd, struct pio_buf *pbuf,
1797                            u64 pbc, const void *from, size_t count)
1798 {
1799         int snoop_mode = 0;
1800         int md_len = 0;
1801         struct capture_md md;
1802         struct snoop_packet *s_packet = NULL;
1803
1804         /*
1805          * count is in dwords so we need to convert to bytes.
1806          * We also need to account for CRC which would be tacked on by hardware.
1807          */
1808         int packet_len = (count << 2) + 4;
1809         int ret;
1810
1811         snoop_dbg("ACK OUT: len %d", packet_len);
1812
1813         if (!dd->hfi1_snoop.filter_callback) {
1814                 snoop_dbg("filter not set");
1815                 ret = HFI1_FILTER_HIT;
1816         } else {
1817                 ret = dd->hfi1_snoop.filter_callback(
1818                                 (struct hfi1_ib_header *)from,
1819                                 NULL,
1820                                 dd->hfi1_snoop.filter_value);
1821         }
1822
1823         switch (ret) {
1824         case HFI1_FILTER_ERR:
1825                 snoop_dbg("Error in filter call");
1826                 /* fall through */
1827         case HFI1_FILTER_MISS:
1828                 snoop_dbg("Filter Miss");
1829                 break;
1830         case HFI1_FILTER_HIT:
1831                 snoop_dbg("Capturing packet");
1832                 if (dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1833                         snoop_mode = 1;
1834                 if ((snoop_mode == 0) ||
1835                     unlikely(snoop_flags & SNOOP_USE_METADATA))
1836                         md_len = sizeof(struct capture_md);
1837
1838                 s_packet = allocate_snoop_packet(packet_len, 0, md_len);
1839
1840                 if (unlikely(s_packet == NULL)) {
1841                         dd_dev_warn_ratelimited(dd, "Unable to allocate snoop/capture packet\n");
1842                         goto inline_pio_out;
1843                 }
1844
1845                 s_packet->total_len = packet_len + md_len;
1846
1847                 /* Fill in the metadata for the packet */
1848                 if (md_len > 0) {
1849                         memset(&md, 0, sizeof(struct capture_md));
1850                         md.port = 1;
1851                         md.dir = PKT_DIR_EGRESS;
1852                         md.u.pbc = pbc;
1853                         memcpy(s_packet->data, &md, md_len);
1854                 }
1855
1856                 /* Add the packet data which is a single buffer */
1857                 memcpy(s_packet->data + md_len, from, packet_len);
1858
1859                 snoop_list_add_tail(s_packet, dd);
1860
1861                 if (unlikely((snoop_flags & SNOOP_DROP_SEND) && snoop_mode)) {
1862                         snoop_dbg("Dropping packet");
1863                         return;
1864                 }
1865                 break;
1866         default:
1867                 break;
1868         }
1869
1870 inline_pio_out:
1871         pio_copy(dd, pbuf, pbc, from, count);
1872
1873 }