tools/testing/selftests/iommu/iommufd_utils.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */
   3 #ifndef __SELFTEST_IOMMUFD_UTILS
   4 #define __SELFTEST_IOMMUFD_UTILS
   5
   6 #include <unistd.h>
   7 #include <stddef.h>
   8 #include <sys/fcntl.h>
   9 #include <sys/ioctl.h>
  10 #include <stdint.h>
  11 #include <assert.h>
  12
  13 #include "../kselftest_harness.h"
  14 #include "../../../../drivers/iommu/iommufd/iommufd_test.h"
  15
  16 /* Hack to make assertions more readable */
  17 #define _IOMMU_TEST_CMD(x) IOMMU_TEST_CMD
  18
  19 /* Imported from include/asm-generic/bitops/generic-non-atomic.h */
  20 #define BITS_PER_BYTE 8
  21 #define BITS_PER_LONG __BITS_PER_LONG
  22 #define BIT_MASK(nr) (1UL << ((nr) % __BITS_PER_LONG))
  23 #define BIT_WORD(nr) ((nr) / __BITS_PER_LONG)
  24
  25 enum {
  26         IOPT_PAGES_ACCOUNT_NONE = 0,
  27         IOPT_PAGES_ACCOUNT_USER = 1,
  28         IOPT_PAGES_ACCOUNT_MM = 2,
  29 };
  30
  31 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
  32
  33 static inline void set_bit(unsigned int nr, unsigned long *addr)
  34 {
  35         unsigned long mask = BIT_MASK(nr);
  36         unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
  37
  38         *p |= mask;
  39 }
  40
  41 static inline bool test_bit(unsigned int nr, unsigned long *addr)
  42 {
  43         return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)));
  44 }
  45
  46 static void *buffer;
  47 static unsigned long BUFFER_SIZE;
  48
  49 static void *mfd_buffer;
  50 static int mfd;
  51
  52 static unsigned long PAGE_SIZE;
  53
  54 #define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
  55 #define offsetofend(TYPE, MEMBER) \
  56         (offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
  57
  58 static inline void *memfd_mmap(size_t length, int prot, int flags, int *mfd_p)
  59 {
  60         int mfd_flags = (flags & MAP_HUGETLB) ? MFD_HUGETLB : 0;
  61         int mfd = memfd_create("buffer", mfd_flags);
  62
  63         if (mfd <= 0)
  64                 return MAP_FAILED;
  65         if (ftruncate(mfd, length))
  66                 return MAP_FAILED;
  67         *mfd_p = mfd;
  68         return mmap(0, length, prot, flags, mfd, 0);
  69 }
  70
  71 /*
  72  * Have the kernel check the refcount on pages. I don't know why a freshly
  73  * mmap'd anon non-compound page starts out with a ref of 3
  74  */
  75 #define check_refs(_ptr, _length, _refs)                                      \
  76         ({                                                                    \
  77                 struct iommu_test_cmd test_cmd = {                            \
  78                         .size = sizeof(test_cmd),                             \
  79                         .op = IOMMU_TEST_OP_MD_CHECK_REFS,                    \
  80                         .check_refs = { .length = _length,                    \
  81                                         .uptr = (uintptr_t)(_ptr),            \
  82                                         .refs = _refs },                      \
  83                 };                                                            \
  84                 ASSERT_EQ(0,                                                  \
  85                           ioctl(self->fd,                                     \
  86                                 _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS), \
  87                                 &test_cmd));                                  \
  88         })
  89
  90 static int _test_cmd_mock_domain(int fd, unsigned int ioas_id, __u32 *stdev_id,
  91                                  __u32 *hwpt_id, __u32 *idev_id)
  92 {
  93         struct iommu_test_cmd cmd = {
  94                 .size = sizeof(cmd),
  95                 .op = IOMMU_TEST_OP_MOCK_DOMAIN,
  96                 .id = ioas_id,
  97                 .mock_domain = {},
  98         };
  99         int ret;
 100
 101         ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
 102         if (ret)
 103                 return ret;
 104         if (stdev_id)
 105                 *stdev_id = cmd.mock_domain.out_stdev_id;
 106         assert(cmd.id != 0);
 107         if (hwpt_id)
 108                 *hwpt_id = cmd.mock_domain.out_hwpt_id;
 109         if (idev_id)
 110                 *idev_id = cmd.mock_domain.out_idev_id;
 111         return 0;
 112 }
 113 #define test_cmd_mock_domain(ioas_id, stdev_id, hwpt_id, idev_id)       \
 114         ASSERT_EQ(0, _test_cmd_mock_domain(self->fd, ioas_id, stdev_id, \
 115                                            hwpt_id, idev_id))
 116 #define test_err_mock_domain(_errno, ioas_id, stdev_id, hwpt_id)      \
 117         EXPECT_ERRNO(_errno, _test_cmd_mock_domain(self->fd, ioas_id, \
 118                                                    stdev_id, hwpt_id, NULL))
 119
 120 static int _test_cmd_mock_domain_flags(int fd, unsigned int ioas_id,
 121                                        __u32 stdev_flags, __u32 *stdev_id,
 122                                        __u32 *hwpt_id, __u32 *idev_id)
 123 {
 124         struct iommu_test_cmd cmd = {
 125                 .size = sizeof(cmd),
 126                 .op = IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS,
 127                 .id = ioas_id,
 128                 .mock_domain_flags = { .dev_flags = stdev_flags },
 129         };
 130         int ret;
 131
 132         ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
 133         if (ret)
 134                 return ret;
 135         if (stdev_id)
 136                 *stdev_id = cmd.mock_domain_flags.out_stdev_id;
 137         assert(cmd.id != 0);
 138         if (hwpt_id)
 139                 *hwpt_id = cmd.mock_domain_flags.out_hwpt_id;
 140         if (idev_id)
 141                 *idev_id = cmd.mock_domain_flags.out_idev_id;
 142         return 0;
 143 }
 144 #define test_cmd_mock_domain_flags(ioas_id, flags, stdev_id, hwpt_id, idev_id) \
 145         ASSERT_EQ(0, _test_cmd_mock_domain_flags(self->fd, ioas_id, flags,     \
 146                                                  stdev_id, hwpt_id, idev_id))
 147 #define test_err_mock_domain_flags(_errno, ioas_id, flags, stdev_id, hwpt_id) \
 148         EXPECT_ERRNO(_errno,                                                  \
 149                      _test_cmd_mock_domain_flags(self->fd, ioas_id, flags,    \
 150                                                  stdev_id, hwpt_id, NULL))
 151
 152 static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id,
 153                                          __u32 *hwpt_id)
 154 {
 155         struct iommu_test_cmd cmd = {
 156                 .size = sizeof(cmd),
 157                 .op = IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE,
 158                 .id = stdev_id,
 159                 .mock_domain_replace = {
 160                         .pt_id = pt_id,
 161                 },
 162         };
 163         int ret;
 164
 165         ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
 166         if (ret)
 167                 return ret;
 168         if (hwpt_id)
 169                 *hwpt_id = cmd.mock_domain_replace.pt_id;
 170         return 0;
 171 }
 172
 173 #define test_cmd_mock_domain_replace(stdev_id, pt_id)                         \
 174         ASSERT_EQ(0, _test_cmd_mock_domain_replace(self->fd, stdev_id, pt_id, \
 175                                                    NULL))
 176 #define test_err_mock_domain_replace(_errno, stdev_id, pt_id)                  \
 177         EXPECT_ERRNO(_errno, _test_cmd_mock_domain_replace(self->fd, stdev_id, \
 178                                                            pt_id, NULL))
 179
 180 static int _test_cmd_hwpt_alloc(int fd, __u32 device_id, __u32 pt_id, __u32 ft_id,
 181                                 __u32 flags, __u32 *hwpt_id, __u32 data_type,
 182                                 void *data, size_t data_len)
 183 {
 184         struct iommu_hwpt_alloc cmd = {
 185                 .size = sizeof(cmd),
 186                 .flags = flags,
 187                 .dev_id = device_id,
 188                 .pt_id = pt_id,
 189                 .data_type = data_type,
 190                 .data_len = data_len,
 191                 .data_uptr = (uint64_t)data,
 192                 .fault_id = ft_id,
 193         };
 194         int ret;
 195
 196         ret = ioctl(fd, IOMMU_HWPT_ALLOC, &cmd);
 197         if (ret)
 198                 return ret;
 199         if (hwpt_id)
 200                 *hwpt_id = cmd.out_hwpt_id;
 201         return 0;
 202 }
 203
 204 #define test_cmd_hwpt_alloc(device_id, pt_id, flags, hwpt_id)                  \
 205         ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags,   \
 206                                           hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, \
 207                                           0))
 208 #define test_err_hwpt_alloc(_errno, device_id, pt_id, flags, hwpt_id)   \
 209         EXPECT_ERRNO(_errno, _test_cmd_hwpt_alloc(                      \
 210                                      self->fd, device_id, pt_id, 0, flags, \
 211                                      hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, 0))
 212
 213 #define test_cmd_hwpt_alloc_nested(device_id, pt_id, flags, hwpt_id,         \
 214                                    data_type, data, data_len)                \
 215         ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
 216                                           hwpt_id, data_type, data, data_len))
 217 #define test_err_hwpt_alloc_nested(_errno, device_id, pt_id, flags, hwpt_id, \
 218                                    data_type, data, data_len)                \
 219         EXPECT_ERRNO(_errno,                                                 \
 220                      _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \
 221                                           hwpt_id, data_type, data, data_len))
 222
 223 #define test_cmd_hwpt_alloc_iopf(device_id, pt_id, fault_id, flags, hwpt_id,    \
 224                                    data_type, data, data_len)                   \
 225         ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
 226                                           flags, hwpt_id, data_type, data,      \
 227                                           data_len))
 228 #define test_err_hwpt_alloc_iopf(_errno, device_id, pt_id, fault_id, flags,     \
 229                                  hwpt_id, data_type, data, data_len)            \
 230         EXPECT_ERRNO(_errno,                                                    \
 231                      _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, fault_id, \
 232                                           flags, hwpt_id, data_type, data,      \
 233                                           data_len))
 234
 235 #define test_cmd_hwpt_check_iotlb(hwpt_id, iotlb_id, expected)                 \
 236         ({                                                                     \
 237                 struct iommu_test_cmd test_cmd = {                             \
 238                         .size = sizeof(test_cmd),                              \
 239                         .op = IOMMU_TEST_OP_MD_CHECK_IOTLB,                    \
 240                         .id = hwpt_id,                                         \
 241                         .check_iotlb = {                                       \
 242                                 .id = iotlb_id,                                \
 243                                 .iotlb = expected,                             \
 244                         },                                                     \
 245                 };                                                             \
 246                 ASSERT_EQ(0,                                                   \
 247                           ioctl(self->fd,                                      \
 248                                 _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_IOTLB), \
 249                                 &test_cmd));                                   \
 250         })
 251
 252 #define test_cmd_hwpt_check_iotlb_all(hwpt_id, expected)                       \
 253         ({                                                                     \
 254                 int i;                                                         \
 255                 for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++)             \
 256                         test_cmd_hwpt_check_iotlb(hwpt_id, i, expected);       \
 257         })
 258
 259 #define test_cmd_dev_check_cache(device_id, cache_id, expected)                \
 260         ({                                                                     \
 261                 struct iommu_test_cmd test_cmd = {                             \
 262                         .size = sizeof(test_cmd),                              \
 263                         .op = IOMMU_TEST_OP_DEV_CHECK_CACHE,                   \
 264                         .id = device_id,                                       \
 265                         .check_dev_cache = {                                   \
 266                                 .id = cache_id,                                \
 267                                 .cache = expected,                             \
 268                         },                                                     \
 269                 };                                                             \
 270                 ASSERT_EQ(0, ioctl(self->fd,                                   \
 271                                    _IOMMU_TEST_CMD(                            \
 272                                            IOMMU_TEST_OP_DEV_CHECK_CACHE),     \
 273                                    &test_cmd));                                \
 274         })
 275
 276 #define test_cmd_dev_check_cache_all(device_id, expected)                      \
 277         ({                                                                     \
 278                 int c;                                                         \
 279                 for (c = 0; c < MOCK_DEV_CACHE_NUM; c++)                       \
 280                         test_cmd_dev_check_cache(device_id, c, expected);      \
 281         })
 282
 283 static int _test_cmd_hwpt_invalidate(int fd, __u32 hwpt_id, void *reqs,
 284                                      uint32_t data_type, uint32_t lreq,
 285                                      uint32_t *nreqs)
 286 {
 287         struct iommu_hwpt_invalidate cmd = {
 288                 .size = sizeof(cmd),
 289                 .hwpt_id = hwpt_id,
 290                 .data_type = data_type,
 291                 .data_uptr = (uint64_t)reqs,
 292                 .entry_len = lreq,
 293                 .entry_num = *nreqs,
 294         };
 295         int rc = ioctl(fd, IOMMU_HWPT_INVALIDATE, &cmd);
 296         *nreqs = cmd.entry_num;
 297         return rc;
 298 }
 299
 300 #define test_cmd_hwpt_invalidate(hwpt_id, reqs, data_type, lreq, nreqs)       \
 301         ({                                                                    \
 302                 ASSERT_EQ(0,                                                  \
 303                           _test_cmd_hwpt_invalidate(self->fd, hwpt_id, reqs,  \
 304                                                     data_type, lreq, nreqs)); \
 305         })
 306 #define test_err_hwpt_invalidate(_errno, hwpt_id, reqs, data_type, lreq, \
 307                                  nreqs)                                  \
 308         ({                                                               \
 309                 EXPECT_ERRNO(_errno, _test_cmd_hwpt_invalidate(          \
 310                                              self->fd, hwpt_id, reqs,    \
 311                                              data_type, lreq, nreqs));   \
 312         })
 313
 314 static int _test_cmd_viommu_invalidate(int fd, __u32 viommu_id, void *reqs,
 315                                        uint32_t data_type, uint32_t lreq,
 316                                        uint32_t *nreqs)
 317 {
 318         struct iommu_hwpt_invalidate cmd = {
 319                 .size = sizeof(cmd),
 320                 .hwpt_id = viommu_id,
 321                 .data_type = data_type,
 322                 .data_uptr = (uint64_t)reqs,
 323                 .entry_len = lreq,
 324                 .entry_num = *nreqs,
 325         };
 326         int rc = ioctl(fd, IOMMU_HWPT_INVALIDATE, &cmd);
 327         *nreqs = cmd.entry_num;
 328         return rc;
 329 }
 330
 331 #define test_cmd_viommu_invalidate(viommu, reqs, lreq, nreqs)                  \
 332         ({                                                                     \
 333                 ASSERT_EQ(0,                                                   \
 334                           _test_cmd_viommu_invalidate(self->fd, viommu, reqs,  \
 335                                         IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, \
 336                                         lreq, nreqs));                         \
 337         })
 338 #define test_err_viommu_invalidate(_errno, viommu_id, reqs, data_type, lreq,   \
 339                                  nreqs)                                        \
 340         ({                                                                     \
 341                 EXPECT_ERRNO(_errno, _test_cmd_viommu_invalidate(              \
 342                                              self->fd, viommu_id, reqs,        \
 343                                              data_type, lreq, nreqs));         \
 344         })
 345
 346 static int _test_cmd_access_replace_ioas(int fd, __u32 access_id,
 347                                          unsigned int ioas_id)
 348 {
 349         struct iommu_test_cmd cmd = {
 350                 .size = sizeof(cmd),
 351                 .op = IOMMU_TEST_OP_ACCESS_REPLACE_IOAS,
 352                 .id = access_id,
 353                 .access_replace_ioas = { .ioas_id = ioas_id },
 354         };
 355         int ret;
 356
 357         ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
 358         if (ret)
 359                 return ret;
 360         return 0;
 361 }
 362 #define test_cmd_access_replace_ioas(access_id, ioas_id) \
 363         ASSERT_EQ(0, _test_cmd_access_replace_ioas(self->fd, access_id, ioas_id))
 364
 365 static int _test_cmd_set_dirty_tracking(int fd, __u32 hwpt_id, bool enabled)
 366 {
 367         struct iommu_hwpt_set_dirty_tracking cmd = {
 368                 .size = sizeof(cmd),
 369                 .flags = enabled ? IOMMU_HWPT_DIRTY_TRACKING_ENABLE : 0,
 370                 .hwpt_id = hwpt_id,
 371         };
 372         int ret;
 373
 374         ret = ioctl(fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &cmd);
 375         if (ret)
 376                 return -errno;
 377         return 0;
 378 }
 379 #define test_cmd_set_dirty_tracking(hwpt_id, enabled) \
 380         ASSERT_EQ(0, _test_cmd_set_dirty_tracking(self->fd, hwpt_id, enabled))
 381
 382 static int _test_cmd_get_dirty_bitmap(int fd, __u32 hwpt_id, size_t length,
 383                                       __u64 iova, size_t page_size,
 384                                       __u64 *bitmap, __u32 flags)
 385 {
 386         struct iommu_hwpt_get_dirty_bitmap cmd = {
 387                 .size = sizeof(cmd),
 388                 .hwpt_id = hwpt_id,
 389                 .flags = flags,
 390                 .iova = iova,
 391                 .length = length,
 392                 .page_size = page_size,
 393                 .data = (uintptr_t)bitmap,
 394         };
 395         int ret;
 396
 397         ret = ioctl(fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &cmd);
 398         if (ret)
 399                 return ret;
 400         return 0;
 401 }
 402
 403 #define test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size,    \
 404                                   bitmap, flags)                           \
 405         ASSERT_EQ(0, _test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, \
 406                                                 page_size, bitmap, flags))
 407
 408 static int _test_cmd_mock_domain_set_dirty(int fd, __u32 hwpt_id, size_t length,
 409                                            __u64 iova, size_t page_size,
 410                                            __u64 *bitmap, __u64 *dirty)
 411 {
 412         struct iommu_test_cmd cmd = {
 413                 .size = sizeof(cmd),
 414                 .op = IOMMU_TEST_OP_DIRTY,
 415                 .id = hwpt_id,
 416                 .dirty = {
 417                         .iova = iova,
 418                         .length = length,
 419                         .page_size = page_size,
 420                         .uptr = (uintptr_t)bitmap,
 421                 }
 422         };
 423         int ret;
 424
 425         ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_DIRTY), &cmd);
 426         if (ret)
 427                 return -ret;
 428         if (dirty)
 429                 *dirty = cmd.dirty.out_nr_dirty;
 430         return 0;
 431 }
 432
 433 #define test_cmd_mock_domain_set_dirty(fd, hwpt_id, length, iova, page_size, \
 434                                        bitmap, nr)                           \
 435         ASSERT_EQ(0,                                                         \
 436                   _test_cmd_mock_domain_set_dirty(fd, hwpt_id, length, iova, \
 437                                                   page_size, bitmap, nr))
 438
 439 static int _test_mock_dirty_bitmaps(int fd, __u32 hwpt_id, size_t length,
 440                                     __u64 iova, size_t page_size,
 441                                     size_t pte_page_size, __u64 *bitmap,
 442                                     __u64 nbits, __u32 flags,
 443                                     struct __test_metadata *_metadata)
 444 {
 445         unsigned long npte = pte_page_size / page_size, pteset = 2 * npte;
 446         unsigned long j, i, nr = nbits / pteset ?: 1;
 447         unsigned long bitmap_size = DIV_ROUND_UP(nbits, BITS_PER_BYTE);
 448         __u64 out_dirty = 0;
 449
 450         /* Mark all even bits as dirty in the mock domain */
 451         memset(bitmap, 0, bitmap_size);
 452         for (i = 0; i < nbits; i += pteset)
 453                 set_bit(i, (unsigned long *)bitmap);
 454
 455         test_cmd_mock_domain_set_dirty(fd, hwpt_id, length, iova, page_size,
 456                                        bitmap, &out_dirty);
 457         ASSERT_EQ(nr, out_dirty);
 458
 459         /* Expect all even bits as dirty in the user bitmap */
 460         memset(bitmap, 0, bitmap_size);
 461         test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size, bitmap,
 462                                   flags);
 463         /* Beware ASSERT_EQ() is two statements -- braces are not redundant! */
 464         for (i = 0; i < nbits; i += pteset) {
 465                 for (j = 0; j < pteset; j++) {
 466                         ASSERT_EQ(j < npte,
 467                                   test_bit(i + j, (unsigned long *)bitmap));
 468                 }
 469                 ASSERT_EQ(!(i % pteset), test_bit(i, (unsigned long *)bitmap));
 470         }
 471
 472         memset(bitmap, 0, bitmap_size);
 473         test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size, bitmap,
 474                                   flags);
 475
 476         /* It as read already -- expect all zeroes */
 477         for (i = 0; i < nbits; i += pteset) {
 478                 for (j = 0; j < pteset; j++) {
 479                         ASSERT_EQ(
 480                                 (j < npte) &&
 481                                         (flags &
 482                                          IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR),
 483                                 test_bit(i + j, (unsigned long *)bitmap));
 484                 }
 485         }
 486
 487         return 0;
 488 }
 489 #define test_mock_dirty_bitmaps(hwpt_id, length, iova, page_size, pte_size,\
 490                                 bitmap, bitmap_size, flags, _metadata)     \
 491         ASSERT_EQ(0, _test_mock_dirty_bitmaps(self->fd, hwpt_id, length, iova, \
 492                                               page_size, pte_size, bitmap,     \
 493                                               bitmap_size, flags, _metadata))
 494
 495 static int _test_cmd_create_access(int fd, unsigned int ioas_id,
 496                                    __u32 *access_id, unsigned int flags)
 497 {
 498         struct iommu_test_cmd cmd = {
 499                 .size = sizeof(cmd),
 500                 .op = IOMMU_TEST_OP_CREATE_ACCESS,
 501                 .id = ioas_id,
 502                 .create_access = { .flags = flags },
 503         };
 504         int ret;
 505
 506         ret = ioctl(fd, IOMMU_TEST_CMD, &cmd);
 507         if (ret)
 508                 return ret;
 509         *access_id = cmd.create_access.out_access_fd;
 510         return 0;
 511 }
 512 #define test_cmd_create_access(ioas_id, access_id, flags)                  \
 513         ASSERT_EQ(0, _test_cmd_create_access(self->fd, ioas_id, access_id, \
 514                                              flags))
 515
 516 static int _test_cmd_destroy_access(unsigned int access_id)
 517 {
 518         return close(access_id);
 519 }
 520 #define test_cmd_destroy_access(access_id) \
 521         ASSERT_EQ(0, _test_cmd_destroy_access(access_id))
 522
 523 static int _test_cmd_destroy_access_pages(int fd, unsigned int access_id,
 524                                           unsigned int access_pages_id)
 525 {
 526         struct iommu_test_cmd cmd = {
 527                 .size = sizeof(cmd),
 528                 .op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES,
 529                 .id = access_id,
 530                 .destroy_access_pages = { .access_pages_id = access_pages_id },
 531         };
 532         return ioctl(fd, IOMMU_TEST_CMD, &cmd);
 533 }
 534 #define test_cmd_destroy_access_pages(access_id, access_pages_id)        \
 535         ASSERT_EQ(0, _test_cmd_destroy_access_pages(self->fd, access_id, \
 536                                                     access_pages_id))
 537 #define test_err_destroy_access_pages(_errno, access_id, access_pages_id) \
 538         EXPECT_ERRNO(_errno, _test_cmd_destroy_access_pages(              \
 539                                      self->fd, access_id, access_pages_id))
 540
 541 static int _test_ioctl_destroy(int fd, unsigned int id)
 542 {
 543         struct iommu_destroy cmd = {
 544                 .size = sizeof(cmd),
 545                 .id = id,
 546         };
 547         return ioctl(fd, IOMMU_DESTROY, &cmd);
 548 }
 549 #define test_ioctl_destroy(id) ASSERT_EQ(0, _test_ioctl_destroy(self->fd, id))
 550
 551 static int _test_ioctl_ioas_alloc(int fd, __u32 *id)
 552 {
 553         struct iommu_ioas_alloc cmd = {
 554                 .size = sizeof(cmd),
 555         };
 556         int ret;
 557
 558         ret = ioctl(fd, IOMMU_IOAS_ALLOC, &cmd);
 559         if (ret)
 560                 return ret;
 561         *id = cmd.out_ioas_id;
 562         return 0;
 563 }
 564 #define test_ioctl_ioas_alloc(id)                                   \
 565         ({                                                          \
 566                 ASSERT_EQ(0, _test_ioctl_ioas_alloc(self->fd, id)); \
 567                 ASSERT_NE(0, *(id));                                \
 568         })
 569
 570 static int _test_ioctl_ioas_map(int fd, unsigned int ioas_id, void *buffer,
 571                                 size_t length, __u64 *iova, unsigned int flags)
 572 {
 573         struct iommu_ioas_map cmd = {
 574                 .size = sizeof(cmd),
 575                 .flags = flags,
 576                 .ioas_id = ioas_id,
 577                 .user_va = (uintptr_t)buffer,
 578                 .length = length,
 579         };
 580         int ret;
 581
 582         if (flags & IOMMU_IOAS_MAP_FIXED_IOVA)
 583                 cmd.iova = *iova;
 584
 585         ret = ioctl(fd, IOMMU_IOAS_MAP, &cmd);
 586         *iova = cmd.iova;
 587         return ret;
 588 }
 589 #define test_ioctl_ioas_map(buffer, length, iova_p)                        \
 590         ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
 591                                           length, iova_p,                  \
 592                                           IOMMU_IOAS_MAP_WRITEABLE |       \
 593                                                   IOMMU_IOAS_MAP_READABLE))
 594
 595 #define test_err_ioctl_ioas_map(_errno, buffer, length, iova_p)            \
 596         EXPECT_ERRNO(_errno,                                               \
 597                      _test_ioctl_ioas_map(self->fd, self->ioas_id, buffer, \
 598                                           length, iova_p,                  \
 599                                           IOMMU_IOAS_MAP_WRITEABLE |       \
 600                                                   IOMMU_IOAS_MAP_READABLE))
 601
 602 #define test_ioctl_ioas_map_id(ioas_id, buffer, length, iova_p)              \
 603         ASSERT_EQ(0, _test_ioctl_ioas_map(self->fd, ioas_id, buffer, length, \
 604                                           iova_p,                            \
 605                                           IOMMU_IOAS_MAP_WRITEABLE |         \
 606                                                   IOMMU_IOAS_MAP_READABLE))
 607
 608 #define test_ioctl_ioas_map_fixed(buffer, length, iova)                       \
 609         ({                                                                    \
 610                 __u64 __iova = iova;                                          \
 611                 ASSERT_EQ(0, _test_ioctl_ioas_map(                            \
 612                                      self->fd, self->ioas_id, buffer, length, \
 613                                      &__iova,                                 \
 614                                      IOMMU_IOAS_MAP_FIXED_IOVA |              \
 615                                              IOMMU_IOAS_MAP_WRITEABLE |       \
 616                                              IOMMU_IOAS_MAP_READABLE));       \
 617         })
 618
 619 #define test_ioctl_ioas_map_fixed_id(ioas_id, buffer, length, iova)           \
 620         ({                                                                    \
 621                 __u64 __iova = iova;                                          \
 622                 ASSERT_EQ(0,                                                  \
 623                           _test_ioctl_ioas_map(                               \
 624                                   self->fd, ioas_id, buffer, length, &__iova, \
 625                                   IOMMU_IOAS_MAP_FIXED_IOVA |                 \
 626                                           IOMMU_IOAS_MAP_WRITEABLE |          \
 627                                           IOMMU_IOAS_MAP_READABLE));          \
 628         })
 629
 630 #define test_err_ioctl_ioas_map_fixed(_errno, buffer, length, iova)           \
 631         ({                                                                    \
 632                 __u64 __iova = iova;                                          \
 633                 EXPECT_ERRNO(_errno,                                          \
 634                              _test_ioctl_ioas_map(                            \
 635                                      self->fd, self->ioas_id, buffer, length, \
 636                                      &__iova,                                 \
 637                                      IOMMU_IOAS_MAP_FIXED_IOVA |              \
 638                                              IOMMU_IOAS_MAP_WRITEABLE |       \
 639                                              IOMMU_IOAS_MAP_READABLE));       \
 640         })
 641
 642 static int _test_ioctl_ioas_unmap(int fd, unsigned int ioas_id, uint64_t iova,
 643                                   size_t length, uint64_t *out_len)
 644 {
 645         struct iommu_ioas_unmap cmd = {
 646                 .size = sizeof(cmd),
 647                 .ioas_id = ioas_id,
 648                 .iova = iova,
 649                 .length = length,
 650         };
 651         int ret;
 652
 653         ret = ioctl(fd, IOMMU_IOAS_UNMAP, &cmd);
 654         if (out_len)
 655                 *out_len = cmd.length;
 656         return ret;
 657 }
 658 #define test_ioctl_ioas_unmap(iova, length)                                \
 659         ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, iova, \
 660                                             length, NULL))
 661
 662 #define test_ioctl_ioas_unmap_id(ioas_id, iova, length)                      \
 663         ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, ioas_id, iova, length, \
 664                                             NULL))
 665
 666 #define test_err_ioctl_ioas_unmap(_errno, iova, length)                      \
 667         EXPECT_ERRNO(_errno, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, \
 668                                                     iova, length, NULL))
 669
 670 static int _test_ioctl_ioas_map_file(int fd, unsigned int ioas_id, int mfd,
 671                                      size_t start, size_t length, __u64 *iova,
 672                                      unsigned int flags)
 673 {
 674         struct iommu_ioas_map_file cmd = {
 675                 .size = sizeof(cmd),
 676                 .flags = flags,
 677                 .ioas_id = ioas_id,
 678                 .fd = mfd,
 679                 .start = start,
 680                 .length = length,
 681         };
 682         int ret;
 683
 684         if (flags & IOMMU_IOAS_MAP_FIXED_IOVA)
 685                 cmd.iova = *iova;
 686
 687         ret = ioctl(fd, IOMMU_IOAS_MAP_FILE, &cmd);
 688         *iova = cmd.iova;
 689         return ret;
 690 }
 691
 692 #define test_ioctl_ioas_map_file(mfd, start, length, iova_p)                   \
 693         ASSERT_EQ(0,                                                           \
 694                   _test_ioctl_ioas_map_file(                                   \
 695                           self->fd, self->ioas_id, mfd, start, length, iova_p, \
 696                           IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))
 697
 698 #define test_err_ioctl_ioas_map_file(_errno, mfd, start, length, iova_p)     \
 699         EXPECT_ERRNO(                                                        \
 700                 _errno,                                                      \
 701                 _test_ioctl_ioas_map_file(                                   \
 702                         self->fd, self->ioas_id, mfd, start, length, iova_p, \
 703                         IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))
 704
 705 #define test_ioctl_ioas_map_id_file(ioas_id, mfd, start, length, iova_p)     \
 706         ASSERT_EQ(0,                                                         \
 707                   _test_ioctl_ioas_map_file(                                 \
 708                           self->fd, ioas_id, mfd, start, length, iova_p,     \
 709                           IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))
 710
 711 static int _test_ioctl_set_temp_memory_limit(int fd, unsigned int limit)
 712 {
 713         struct iommu_test_cmd memlimit_cmd = {
 714                 .size = sizeof(memlimit_cmd),
 715                 .op = IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT,
 716                 .memory_limit = { .limit = limit },
 717         };
 718
 719         return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT),
 720                      &memlimit_cmd);
 721 }
 722
 723 #define test_ioctl_set_temp_memory_limit(limit) \
 724         ASSERT_EQ(0, _test_ioctl_set_temp_memory_limit(self->fd, limit))
 725
 726 #define test_ioctl_set_default_memory_limit() \
 727         test_ioctl_set_temp_memory_limit(65536)
 728
 729 static void teardown_iommufd(int fd, struct __test_metadata *_metadata)
 730 {
 731         struct iommu_test_cmd test_cmd = {
 732                 .size = sizeof(test_cmd),
 733                 .op = IOMMU_TEST_OP_MD_CHECK_REFS,
 734                 .check_refs = { .length = BUFFER_SIZE,
 735                                 .uptr = (uintptr_t)buffer },
 736         };
 737
 738         if (fd == -1)
 739                 return;
 740
 741         EXPECT_EQ(0, close(fd));
 742
 743         fd = open("/dev/iommu", O_RDWR);
 744         EXPECT_NE(-1, fd);
 745         EXPECT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_REFS),
 746                            &test_cmd));
 747         EXPECT_EQ(0, close(fd));
 748 }
 749
 750 #define EXPECT_ERRNO(expected_errno, cmd)         \
 751         ({                                        \
 752                 ASSERT_EQ(-1, cmd);               \
 753                 EXPECT_EQ(expected_errno, errno); \
 754         })
 755
 756 #endif
 757
 758 /* @data can be NULL */
 759 static int _test_cmd_get_hw_info(int fd, __u32 device_id, void *data,
 760                                  size_t data_len, uint32_t *capabilities)
 761 {
 762         struct iommu_test_hw_info *info = (struct iommu_test_hw_info *)data;
 763         struct iommu_hw_info cmd = {
 764                 .size = sizeof(cmd),
 765                 .dev_id = device_id,
 766                 .data_len = data_len,
 767                 .data_uptr = (uint64_t)data,
 768                 .out_capabilities = 0,
 769         };
 770         int ret;
 771
 772         ret = ioctl(fd, IOMMU_GET_HW_INFO, &cmd);
 773         if (ret)
 774                 return ret;
 775
 776         assert(cmd.out_data_type == IOMMU_HW_INFO_TYPE_SELFTEST);
 777
 778         /*
 779          * The struct iommu_test_hw_info should be the one defined
 780          * by the current kernel.
 781          */
 782         assert(cmd.data_len == sizeof(struct iommu_test_hw_info));
 783
 784         /*
 785          * Trailing bytes should be 0 if user buffer is larger than
 786          * the data that kernel reports.
 787          */
 788         if (data_len > cmd.data_len) {
 789                 char *ptr = (char *)(data + cmd.data_len);
 790                 int idx = 0;
 791
 792                 while (idx < data_len - cmd.data_len) {
 793                         assert(!*(ptr + idx));
 794                         idx++;
 795                 }
 796         }
 797
 798         if (info) {
 799                 if (data_len >= offsetofend(struct iommu_test_hw_info, test_reg))
 800                         assert(info->test_reg == IOMMU_HW_INFO_SELFTEST_REGVAL);
 801                 if (data_len >= offsetofend(struct iommu_test_hw_info, flags))
 802                         assert(!info->flags);
 803         }
 804
 805         if (capabilities)
 806                 *capabilities = cmd.out_capabilities;
 807
 808         return 0;
 809 }
 810
 811 #define test_cmd_get_hw_info(device_id, data, data_len)               \
 812         ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, data, \
 813                                            data_len, NULL))
 814
 815 #define test_err_get_hw_info(_errno, device_id, data, data_len)               \
 816         EXPECT_ERRNO(_errno, _test_cmd_get_hw_info(self->fd, device_id, data, \
 817                                                    data_len, NULL))
 818
 819 #define test_cmd_get_hw_capabilities(device_id, caps, mask) \
 820         ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, 0, &caps))
 821
 822 static int _test_ioctl_fault_alloc(int fd, __u32 *fault_id, __u32 *fault_fd)
 823 {
 824         struct iommu_fault_alloc cmd = {
 825                 .size = sizeof(cmd),
 826         };
 827         int ret;
 828
 829         ret = ioctl(fd, IOMMU_FAULT_QUEUE_ALLOC, &cmd);
 830         if (ret)
 831                 return ret;
 832         *fault_id = cmd.out_fault_id;
 833         *fault_fd = cmd.out_fault_fd;
 834         return 0;
 835 }
 836
 837 #define test_ioctl_fault_alloc(fault_id, fault_fd)                       \
 838         ({                                                               \
 839                 ASSERT_EQ(0, _test_ioctl_fault_alloc(self->fd, fault_id, \
 840                                                      fault_fd));         \
 841                 ASSERT_NE(0, *(fault_id));                               \
 842                 ASSERT_NE(0, *(fault_fd));                               \
 843         })
 844
 845 static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 fault_fd)
 846 {
 847         struct iommu_test_cmd trigger_iopf_cmd = {
 848                 .size = sizeof(trigger_iopf_cmd),
 849                 .op = IOMMU_TEST_OP_TRIGGER_IOPF,
 850                 .trigger_iopf = {
 851                         .dev_id = device_id,
 852                         .pasid = 0x1,
 853                         .grpid = 0x2,
 854                         .perm = IOMMU_PGFAULT_PERM_READ | IOMMU_PGFAULT_PERM_WRITE,
 855                         .addr = 0xdeadbeaf,
 856                 },
 857         };
 858         struct iommu_hwpt_page_response response = {
 859                 .code = IOMMUFD_PAGE_RESP_SUCCESS,
 860         };
 861         struct iommu_hwpt_pgfault fault = {};
 862         ssize_t bytes;
 863         int ret;
 864
 865         ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_TRIGGER_IOPF), &trigger_iopf_cmd);
 866         if (ret)
 867                 return ret;
 868
 869         bytes = read(fault_fd, &fault, sizeof(fault));
 870         if (bytes <= 0)
 871                 return -EIO;
 872
 873         response.cookie = fault.cookie;
 874
 875         bytes = write(fault_fd, &response, sizeof(response));
 876         if (bytes <= 0)
 877                 return -EIO;
 878
 879         return 0;
 880 }
 881
 882 #define test_cmd_trigger_iopf(device_id, fault_fd) \
 883         ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, fault_fd))
 884
 885 static int _test_cmd_viommu_alloc(int fd, __u32 device_id, __u32 hwpt_id,
 886                                   __u32 type, __u32 flags, __u32 *viommu_id)
 887 {
 888         struct iommu_viommu_alloc cmd = {
 889                 .size = sizeof(cmd),
 890                 .flags = flags,
 891                 .type = type,
 892                 .dev_id = device_id,
 893                 .hwpt_id = hwpt_id,
 894         };
 895         int ret;
 896
 897         ret = ioctl(fd, IOMMU_VIOMMU_ALLOC, &cmd);
 898         if (ret)
 899                 return ret;
 900         if (viommu_id)
 901                 *viommu_id = cmd.out_viommu_id;
 902         return 0;
 903 }
 904
 905 #define test_cmd_viommu_alloc(device_id, hwpt_id, type, viommu_id)        \
 906         ASSERT_EQ(0, _test_cmd_viommu_alloc(self->fd, device_id, hwpt_id, \
 907                                             type, 0, viommu_id))
 908 #define test_err_viommu_alloc(_errno, device_id, hwpt_id, type, viommu_id) \
 909         EXPECT_ERRNO(_errno,                                               \
 910                      _test_cmd_viommu_alloc(self->fd, device_id, hwpt_id,  \
 911                                             type, 0, viommu_id))
 912
 913 static int _test_cmd_vdevice_alloc(int fd, __u32 viommu_id, __u32 idev_id,
 914                                    __u64 virt_id, __u32 *vdev_id)
 915 {
 916         struct iommu_vdevice_alloc cmd = {
 917                 .size = sizeof(cmd),
 918                 .dev_id = idev_id,
 919                 .viommu_id = viommu_id,
 920                 .virt_id = virt_id,
 921         };
 922         int ret;
 923
 924         ret = ioctl(fd, IOMMU_VDEVICE_ALLOC, &cmd);
 925         if (ret)
 926                 return ret;
 927         if (vdev_id)
 928                 *vdev_id = cmd.out_vdevice_id;
 929         return 0;
 930 }
 931
 932 #define test_cmd_vdevice_alloc(viommu_id, idev_id, virt_id, vdev_id)       \
 933         ASSERT_EQ(0, _test_cmd_vdevice_alloc(self->fd, viommu_id, idev_id, \
 934                                              virt_id, vdev_id))
 935 #define test_err_vdevice_alloc(_errno, viommu_id, idev_id, virt_id, vdev_id) \
 936         EXPECT_ERRNO(_errno,                                                 \
 937                      _test_cmd_vdevice_alloc(self->fd, viommu_id, idev_id,   \
 938                                              virt_id, vdev_id))