drivers/crypto/ccp/sp-pci.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Secure Processor device driver
   4  *
   5  * Copyright (C) 2013,2019 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Tom Lendacky <[email protected]>
   8  * Author: Gary R Hook <[email protected]>
   9  */
  10
  11 #include <linux/bitfield.h>
  12 #include <linux/module.h>
  13 #include <linux/kernel.h>
  14 #include <linux/device.h>
  15 #include <linux/pci.h>
  16 #include <linux/pci_ids.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/kthread.h>
  19 #include <linux/sched.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/delay.h>
  23 #include <linux/ccp.h>
  24
  25 #include "ccp-dev.h"
  26 #include "psp-dev.h"
  27
  28 /* used for version string AA.BB.CC.DD */
  29 #define AA                              GENMASK(31, 24)
  30 #define BB                              GENMASK(23, 16)
  31 #define CC                              GENMASK(15, 8)
  32 #define DD                              GENMASK(7, 0)
  33
  34 #define MSIX_VECTORS                    2
  35
  36 struct sp_pci {
  37         int msix_count;
  38         struct msix_entry msix_entry[MSIX_VECTORS];
  39 };
  40 static struct sp_device *sp_dev_master;
  41
  42 #define security_attribute_show(name, def)                                      \
  43 static ssize_t name##_show(struct device *d, struct device_attribute *attr,     \
  44                            char *buf)                                           \
  45 {                                                                               \
  46         struct sp_device *sp = dev_get_drvdata(d);                              \
  47         struct psp_device *psp = sp->psp_data;                                  \
  48         int bit = PSP_SECURITY_##def << PSP_CAPABILITY_PSP_SECURITY_OFFSET;     \
  49         return sysfs_emit(buf, "%d\n", (psp->capability & bit) > 0);            \
  50 }
  51
  52 security_attribute_show(fused_part, FUSED_PART)
  53 static DEVICE_ATTR_RO(fused_part);
  54 security_attribute_show(debug_lock_on, DEBUG_LOCK_ON)
  55 static DEVICE_ATTR_RO(debug_lock_on);
  56 security_attribute_show(tsme_status, TSME_STATUS)
  57 static DEVICE_ATTR_RO(tsme_status);
  58 security_attribute_show(anti_rollback_status, ANTI_ROLLBACK_STATUS)
  59 static DEVICE_ATTR_RO(anti_rollback_status);
  60 security_attribute_show(rpmc_production_enabled, RPMC_PRODUCTION_ENABLED)
  61 static DEVICE_ATTR_RO(rpmc_production_enabled);
  62 security_attribute_show(rpmc_spirom_available, RPMC_SPIROM_AVAILABLE)
  63 static DEVICE_ATTR_RO(rpmc_spirom_available);
  64 security_attribute_show(hsp_tpm_available, HSP_TPM_AVAILABLE)
  65 static DEVICE_ATTR_RO(hsp_tpm_available);
  66 security_attribute_show(rom_armor_enforced, ROM_ARMOR_ENFORCED)
  67 static DEVICE_ATTR_RO(rom_armor_enforced);
  68
  69 static struct attribute *psp_security_attrs[] = {
  70         &dev_attr_fused_part.attr,
  71         &dev_attr_debug_lock_on.attr,
  72         &dev_attr_tsme_status.attr,
  73         &dev_attr_anti_rollback_status.attr,
  74         &dev_attr_rpmc_production_enabled.attr,
  75         &dev_attr_rpmc_spirom_available.attr,
  76         &dev_attr_hsp_tpm_available.attr,
  77         &dev_attr_rom_armor_enforced.attr,
  78         NULL
  79 };
  80
  81 static umode_t psp_security_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
  82 {
  83         struct device *dev = kobj_to_dev(kobj);
  84         struct sp_device *sp = dev_get_drvdata(dev);
  85         struct psp_device *psp = sp->psp_data;
  86
  87         if (psp && (psp->capability & PSP_CAPABILITY_PSP_SECURITY_REPORTING))
  88                 return 0444;
  89
  90         return 0;
  91 }
  92
  93 static struct attribute_group psp_security_attr_group = {
  94         .attrs = psp_security_attrs,
  95         .is_visible = psp_security_is_visible,
  96 };
  97
  98 #define version_attribute_show(name, _offset)                                   \
  99 static ssize_t name##_show(struct device *d, struct device_attribute *attr,     \
 100                            char *buf)                                           \
 101 {                                                                               \
 102         struct sp_device *sp = dev_get_drvdata(d);                              \
 103         struct psp_device *psp = sp->psp_data;                                  \
 104         unsigned int val = ioread32(psp->io_regs + _offset);                    \
 105         return sysfs_emit(buf, "%02lx.%02lx.%02lx.%02lx\n",                     \
 106                           FIELD_GET(AA, val),                   \
 107                           FIELD_GET(BB, val),                   \
 108                           FIELD_GET(CC, val),                   \
 109                           FIELD_GET(DD, val));                  \
 110 }
 111
 112 version_attribute_show(bootloader_version, psp->vdata->bootloader_info_reg)
 113 static DEVICE_ATTR_RO(bootloader_version);
 114 version_attribute_show(tee_version, psp->vdata->tee->info_reg)
 115 static DEVICE_ATTR_RO(tee_version);
 116
 117 static struct attribute *psp_firmware_attrs[] = {
 118         &dev_attr_bootloader_version.attr,
 119         &dev_attr_tee_version.attr,
 120         NULL,
 121 };
 122
 123 static umode_t psp_firmware_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
 124 {
 125         struct device *dev = kobj_to_dev(kobj);
 126         struct sp_device *sp = dev_get_drvdata(dev);
 127         struct psp_device *psp = sp->psp_data;
 128         unsigned int val = 0xffffffff;
 129
 130         if (!psp)
 131                 return 0;
 132
 133         if (attr == &dev_attr_bootloader_version.attr &&
 134             psp->vdata->bootloader_info_reg)
 135                 val = ioread32(psp->io_regs + psp->vdata->bootloader_info_reg);
 136
 137         if (attr == &dev_attr_tee_version.attr &&
 138             psp->capability & PSP_CAPABILITY_TEE &&
 139             psp->vdata->tee->info_reg)
 140                 val = ioread32(psp->io_regs + psp->vdata->tee->info_reg);
 141
 142         /* If platform disallows accessing this register it will be all f's */
 143         if (val != 0xffffffff)
 144                 return 0444;
 145
 146         return 0;
 147 }
 148
 149 static struct attribute_group psp_firmware_attr_group = {
 150         .attrs = psp_firmware_attrs,
 151         .is_visible = psp_firmware_is_visible,
 152 };
 153
 154 static const struct attribute_group *psp_groups[] = {
 155         &psp_security_attr_group,
 156         &psp_firmware_attr_group,
 157         NULL,
 158 };
 159
 160 static int sp_get_msix_irqs(struct sp_device *sp)
 161 {
 162         struct sp_pci *sp_pci = sp->dev_specific;
 163         struct device *dev = sp->dev;
 164         struct pci_dev *pdev = to_pci_dev(dev);
 165         int v, ret;
 166
 167         for (v = 0; v < ARRAY_SIZE(sp_pci->msix_entry); v++)
 168                 sp_pci->msix_entry[v].entry = v;
 169
 170         ret = pci_enable_msix_range(pdev, sp_pci->msix_entry, 1, v);
 171         if (ret < 0)
 172                 return ret;
 173
 174         sp_pci->msix_count = ret;
 175         sp->use_tasklet = true;
 176
 177         sp->psp_irq = sp_pci->msix_entry[0].vector;
 178         sp->ccp_irq = (sp_pci->msix_count > 1) ? sp_pci->msix_entry[1].vector
 179                                                : sp_pci->msix_entry[0].vector;
 180         return 0;
 181 }
 182
 183 static int sp_get_msi_irq(struct sp_device *sp)
 184 {
 185         struct device *dev = sp->dev;
 186         struct pci_dev *pdev = to_pci_dev(dev);
 187         int ret;
 188
 189         ret = pci_enable_msi(pdev);
 190         if (ret)
 191                 return ret;
 192
 193         sp->ccp_irq = pdev->irq;
 194         sp->psp_irq = pdev->irq;
 195
 196         return 0;
 197 }
 198
 199 static int sp_get_irqs(struct sp_device *sp)
 200 {
 201         struct device *dev = sp->dev;
 202         int ret;
 203
 204         ret = sp_get_msix_irqs(sp);
 205         if (!ret)
 206                 return 0;
 207
 208         /* Couldn't get MSI-X vectors, try MSI */
 209         dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
 210         ret = sp_get_msi_irq(sp);
 211         if (!ret)
 212                 return 0;
 213
 214         /* Couldn't get MSI interrupt */
 215         dev_notice(dev, "could not enable MSI (%d)\n", ret);
 216
 217         return ret;
 218 }
 219
 220 static void sp_free_irqs(struct sp_device *sp)
 221 {
 222         struct sp_pci *sp_pci = sp->dev_specific;
 223         struct device *dev = sp->dev;
 224         struct pci_dev *pdev = to_pci_dev(dev);
 225
 226         if (sp_pci->msix_count)
 227                 pci_disable_msix(pdev);
 228         else if (sp->psp_irq)
 229                 pci_disable_msi(pdev);
 230
 231         sp->ccp_irq = 0;
 232         sp->psp_irq = 0;
 233 }
 234
 235 static bool sp_pci_is_master(struct sp_device *sp)
 236 {
 237         struct device *dev_cur, *dev_new;
 238         struct pci_dev *pdev_cur, *pdev_new;
 239
 240         dev_new = sp->dev;
 241         dev_cur = sp_dev_master->dev;
 242
 243         pdev_new = to_pci_dev(dev_new);
 244         pdev_cur = to_pci_dev(dev_cur);
 245
 246         if (pdev_new->bus->number < pdev_cur->bus->number)
 247                 return true;
 248
 249         if (PCI_SLOT(pdev_new->devfn) < PCI_SLOT(pdev_cur->devfn))
 250                 return true;
 251
 252         if (PCI_FUNC(pdev_new->devfn) < PCI_FUNC(pdev_cur->devfn))
 253                 return true;
 254
 255         return false;
 256 }
 257
 258 static void psp_set_master(struct sp_device *sp)
 259 {
 260         if (!sp_dev_master) {
 261                 sp_dev_master = sp;
 262                 return;
 263         }
 264
 265         if (sp_pci_is_master(sp))
 266                 sp_dev_master = sp;
 267 }
 268
 269 static struct sp_device *psp_get_master(void)
 270 {
 271         return sp_dev_master;
 272 }
 273
 274 static void psp_clear_master(struct sp_device *sp)
 275 {
 276         if (sp == sp_dev_master) {
 277                 sp_dev_master = NULL;
 278                 dev_dbg(sp->dev, "Cleared sp_dev_master\n");
 279         }
 280 }
 281
 282 static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 283 {
 284         struct sp_device *sp;
 285         struct sp_pci *sp_pci;
 286         struct device *dev = &pdev->dev;
 287         void __iomem * const *iomap_table;
 288         int bar_mask;
 289         int ret;
 290
 291         ret = -ENOMEM;
 292         sp = sp_alloc_struct(dev);
 293         if (!sp)
 294                 goto e_err;
 295
 296         sp_pci = devm_kzalloc(dev, sizeof(*sp_pci), GFP_KERNEL);
 297         if (!sp_pci)
 298                 goto e_err;
 299
 300         sp->dev_specific = sp_pci;
 301         sp->dev_vdata = (struct sp_dev_vdata *)id->driver_data;
 302         if (!sp->dev_vdata) {
 303                 ret = -ENODEV;
 304                 dev_err(dev, "missing driver data\n");
 305                 goto e_err;
 306         }
 307
 308         ret = pcim_enable_device(pdev);
 309         if (ret) {
 310                 dev_err(dev, "pcim_enable_device failed (%d)\n", ret);
 311                 goto e_err;
 312         }
 313
 314         bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
 315         ret = pcim_iomap_regions(pdev, bar_mask, "ccp");
 316         if (ret) {
 317                 dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret);
 318                 goto e_err;
 319         }
 320
 321         iomap_table = pcim_iomap_table(pdev);
 322         if (!iomap_table) {
 323                 dev_err(dev, "pcim_iomap_table failed\n");
 324                 ret = -ENOMEM;
 325                 goto e_err;
 326         }
 327
 328         sp->io_map = iomap_table[sp->dev_vdata->bar];
 329         if (!sp->io_map) {
 330                 dev_err(dev, "ioremap failed\n");
 331                 ret = -ENOMEM;
 332                 goto e_err;
 333         }
 334
 335         ret = sp_get_irqs(sp);
 336         if (ret)
 337                 goto e_err;
 338
 339         pci_set_master(pdev);
 340         sp->set_psp_master_device = psp_set_master;
 341         sp->get_psp_master_device = psp_get_master;
 342         sp->clear_psp_master_device = psp_clear_master;
 343
 344         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
 345         if (ret) {
 346                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 347                 if (ret) {
 348                         dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
 349                                 ret);
 350                         goto free_irqs;
 351                 }
 352         }
 353
 354         dev_set_drvdata(dev, sp);
 355
 356         ret = sp_init(sp);
 357         if (ret)
 358                 goto free_irqs;
 359
 360         return 0;
 361
 362 free_irqs:
 363         sp_free_irqs(sp);
 364 e_err:
 365         dev_notice(dev, "initialization failed\n");
 366         return ret;
 367 }
 368
 369 static void sp_pci_shutdown(struct pci_dev *pdev)
 370 {
 371         struct device *dev = &pdev->dev;
 372         struct sp_device *sp = dev_get_drvdata(dev);
 373
 374         if (!sp)
 375                 return;
 376
 377         sp_destroy(sp);
 378 }
 379
 380 static void sp_pci_remove(struct pci_dev *pdev)
 381 {
 382         struct device *dev = &pdev->dev;
 383         struct sp_device *sp = dev_get_drvdata(dev);
 384
 385         if (!sp)
 386                 return;
 387
 388         sp_destroy(sp);
 389
 390         sp_free_irqs(sp);
 391 }
 392
 393 static int __maybe_unused sp_pci_suspend(struct device *dev)
 394 {
 395         struct sp_device *sp = dev_get_drvdata(dev);
 396
 397         return sp_suspend(sp);
 398 }
 399
 400 static int __maybe_unused sp_pci_resume(struct device *dev)
 401 {
 402         struct sp_device *sp = dev_get_drvdata(dev);
 403
 404         return sp_resume(sp);
 405 }
 406
 407 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 408 static const struct sev_vdata sevv1 = {
 409         .cmdresp_reg            = 0x10580,      /* C2PMSG_32 */
 410         .cmdbuff_addr_lo_reg    = 0x105e0,      /* C2PMSG_56 */
 411         .cmdbuff_addr_hi_reg    = 0x105e4,      /* C2PMSG_57 */
 412 };
 413
 414 static const struct sev_vdata sevv2 = {
 415         .cmdresp_reg            = 0x10980,      /* C2PMSG_32 */
 416         .cmdbuff_addr_lo_reg    = 0x109e0,      /* C2PMSG_56 */
 417         .cmdbuff_addr_hi_reg    = 0x109e4,      /* C2PMSG_57 */
 418 };
 419
 420 static const struct tee_vdata teev1 = {
 421         .cmdresp_reg            = 0x10544,      /* C2PMSG_17 */
 422         .cmdbuff_addr_lo_reg    = 0x10548,      /* C2PMSG_18 */
 423         .cmdbuff_addr_hi_reg    = 0x1054c,      /* C2PMSG_19 */
 424         .ring_wptr_reg          = 0x10550,      /* C2PMSG_20 */
 425         .ring_rptr_reg          = 0x10554,      /* C2PMSG_21 */
 426         .info_reg               = 0x109e8,      /* C2PMSG_58 */
 427 };
 428
 429 static const struct tee_vdata teev2 = {
 430         .cmdresp_reg            = 0x10944,      /* C2PMSG_17 */
 431         .cmdbuff_addr_lo_reg    = 0x10948,      /* C2PMSG_18 */
 432         .cmdbuff_addr_hi_reg    = 0x1094c,      /* C2PMSG_19 */
 433         .ring_wptr_reg          = 0x10950,      /* C2PMSG_20 */
 434         .ring_rptr_reg          = 0x10954,      /* C2PMSG_21 */
 435 };
 436
 437 static const struct platform_access_vdata pa_v1 = {
 438         .cmdresp_reg            = 0x10570,      /* C2PMSG_28 */
 439         .cmdbuff_addr_lo_reg    = 0x10574,      /* C2PMSG_29 */
 440         .cmdbuff_addr_hi_reg    = 0x10578,      /* C2PMSG_30 */
 441         .doorbell_button_reg    = 0x10a24,      /* C2PMSG_73 */
 442         .doorbell_cmd_reg       = 0x10a40,      /* C2PMSG_80 */
 443 };
 444
 445 static const struct platform_access_vdata pa_v2 = {
 446         .doorbell_button_reg    = 0x10a24,      /* C2PMSG_73 */
 447         .doorbell_cmd_reg       = 0x10a40,      /* C2PMSG_80 */
 448 };
 449
 450 static const struct psp_vdata pspv1 = {
 451         .sev                    = &sevv1,
 452         .bootloader_info_reg    = 0x105ec,      /* C2PMSG_59 */
 453         .feature_reg            = 0x105fc,      /* C2PMSG_63 */
 454         .inten_reg              = 0x10610,      /* P2CMSG_INTEN */
 455         .intsts_reg             = 0x10614,      /* P2CMSG_INTSTS */
 456 };
 457
 458 static const struct psp_vdata pspv2 = {
 459         .sev                    = &sevv2,
 460         .bootloader_info_reg    = 0x109ec,      /* C2PMSG_59 */
 461         .feature_reg            = 0x109fc,      /* C2PMSG_63 */
 462         .inten_reg              = 0x10690,      /* P2CMSG_INTEN */
 463         .intsts_reg             = 0x10694,      /* P2CMSG_INTSTS */
 464 };
 465
 466 static const struct psp_vdata pspv3 = {
 467         .tee                    = &teev1,
 468         .platform_access        = &pa_v1,
 469         .bootloader_info_reg    = 0x109ec,      /* C2PMSG_59 */
 470         .feature_reg            = 0x109fc,      /* C2PMSG_63 */
 471         .inten_reg              = 0x10690,      /* P2CMSG_INTEN */
 472         .intsts_reg             = 0x10694,      /* P2CMSG_INTSTS */
 473         .platform_features      = PLATFORM_FEATURE_DBC,
 474 };
 475
 476 static const struct psp_vdata pspv4 = {
 477         .sev                    = &sevv2,
 478         .tee                    = &teev1,
 479         .bootloader_info_reg    = 0x109ec,      /* C2PMSG_59 */
 480         .feature_reg            = 0x109fc,      /* C2PMSG_63 */
 481         .inten_reg              = 0x10690,      /* P2CMSG_INTEN */
 482         .intsts_reg             = 0x10694,      /* P2CMSG_INTSTS */
 483 };
 484
 485 static const struct psp_vdata pspv5 = {
 486         .tee                    = &teev2,
 487         .platform_access        = &pa_v2,
 488         .feature_reg            = 0x109fc,      /* C2PMSG_63 */
 489         .inten_reg              = 0x10510,      /* P2CMSG_INTEN */
 490         .intsts_reg             = 0x10514,      /* P2CMSG_INTSTS */
 491 };
 492
 493 static const struct psp_vdata pspv6 = {
 494         .sev                    = &sevv2,
 495         .tee                    = &teev2,
 496         .feature_reg            = 0x109fc,      /* C2PMSG_63 */
 497         .inten_reg              = 0x10510,      /* P2CMSG_INTEN */
 498         .intsts_reg             = 0x10514,      /* P2CMSG_INTSTS */
 499 };
 500
 501 #endif
 502
 503 static const struct sp_dev_vdata dev_vdata[] = {
 504         {       /* 0 */
 505                 .bar = 2,
 506 #ifdef CONFIG_CRYPTO_DEV_SP_CCP
 507                 .ccp_vdata = &ccpv3,
 508 #endif
 509         },
 510         {       /* 1 */
 511                 .bar = 2,
 512 #ifdef CONFIG_CRYPTO_DEV_SP_CCP
 513                 .ccp_vdata = &ccpv5a,
 514 #endif
 515 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 516                 .psp_vdata = &pspv1,
 517 #endif
 518         },
 519         {       /* 2 */
 520                 .bar = 2,
 521 #ifdef CONFIG_CRYPTO_DEV_SP_CCP
 522                 .ccp_vdata = &ccpv5b,
 523 #endif
 524         },
 525         {       /* 3 */
 526                 .bar = 2,
 527 #ifdef CONFIG_CRYPTO_DEV_SP_CCP
 528                 .ccp_vdata = &ccpv5a,
 529 #endif
 530 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 531                 .psp_vdata = &pspv2,
 532 #endif
 533         },
 534         {       /* 4 */
 535                 .bar = 2,
 536 #ifdef CONFIG_CRYPTO_DEV_SP_CCP
 537                 .ccp_vdata = &ccpv5a,
 538 #endif
 539 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 540                 .psp_vdata = &pspv3,
 541 #endif
 542         },
 543         {       /* 5 */
 544                 .bar = 2,
 545 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 546                 .psp_vdata = &pspv4,
 547 #endif
 548         },
 549         {       /* 6 */
 550                 .bar = 2,
 551 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 552                 .psp_vdata = &pspv3,
 553 #endif
 554         },
 555         {       /* 7 */
 556                 .bar = 2,
 557 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 558                 .psp_vdata = &pspv5,
 559 #endif
 560         },
 561         {       /* 8 */
 562                 .bar = 2,
 563 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 564                 .psp_vdata = &pspv6,
 565 #endif
 566         },
 567 };
 568 static const struct pci_device_id sp_pci_table[] = {
 569         { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] },
 570         { PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&dev_vdata[1] },
 571         { PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] },
 572         { PCI_VDEVICE(AMD, 0x1486), (kernel_ulong_t)&dev_vdata[3] },
 573         { PCI_VDEVICE(AMD, 0x15DF), (kernel_ulong_t)&dev_vdata[4] },
 574         { PCI_VDEVICE(AMD, 0x14CA), (kernel_ulong_t)&dev_vdata[5] },
 575         { PCI_VDEVICE(AMD, 0x15C7), (kernel_ulong_t)&dev_vdata[6] },
 576         { PCI_VDEVICE(AMD, 0x1649), (kernel_ulong_t)&dev_vdata[6] },
 577         { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] },
 578         { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] },
 579         /* Last entry must be zero */
 580         { 0, }
 581 };
 582 MODULE_DEVICE_TABLE(pci, sp_pci_table);
 583
 584 static SIMPLE_DEV_PM_OPS(sp_pci_pm_ops, sp_pci_suspend, sp_pci_resume);
 585
 586 static struct pci_driver sp_pci_driver = {
 587         .name = "ccp",
 588         .id_table = sp_pci_table,
 589         .probe = sp_pci_probe,
 590         .remove = sp_pci_remove,
 591         .shutdown = sp_pci_shutdown,
 592         .driver.pm = &sp_pci_pm_ops,
 593         .dev_groups = psp_groups,
 594 };
 595
 596 int sp_pci_init(void)
 597 {
 598         return pci_register_driver(&sp_pci_driver);
 599 }
 600
 601 void sp_pci_exit(void)
 602 {
 603         pci_unregister_driver(&sp_pci_driver);
 604 }