crypto/testmgr.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Algorithm testing framework and tests.
   4  *
   5  * Copyright (c) 2002 James Morris <[email protected]>
   6  * Copyright (c) 2002 Jean-Francois Dive <[email protected]>
   7  * Copyright (c) 2007 Nokia Siemens Networks
   8  * Copyright (c) 2008 Herbert Xu <[email protected]>
   9  * Copyright (c) 2019 Google LLC
  10  *
  11  * Updated RFC4106 AES-GCM testing.
  12  *    Authors: Aidan O'Mahony ([email protected])
  13  *             Adrian Hoban <[email protected]>
  14  *             Gabriele Paoloni <[email protected]>
  15  *             Tadeusz Struk ([email protected])
  16  *    Copyright (c) 2010, Intel Corporation.
  17  */
  18
  19 #include <crypto/aead.h>
  20 #include <crypto/hash.h>
  21 #include <crypto/skcipher.h>
  22 #include <linux/err.h>
  23 #include <linux/fips.h>
  24 #include <linux/module.h>
  25 #include <linux/once.h>
  26 #include <linux/prandom.h>
  27 #include <linux/scatterlist.h>
  28 #include <linux/slab.h>
  29 #include <linux/string.h>
  30 #include <linux/uio.h>
  31 #include <crypto/rng.h>
  32 #include <crypto/drbg.h>
  33 #include <crypto/akcipher.h>
  34 #include <crypto/kpp.h>
  35 #include <crypto/acompress.h>
  36 #include <crypto/sig.h>
  37 #include <crypto/internal/cipher.h>
  38 #include <crypto/internal/simd.h>
  39
  40 #include "internal.h"
  41
  42 MODULE_IMPORT_NS("CRYPTO_INTERNAL");
  43
  44 static bool notests;
  45 module_param(notests, bool, 0644);
  46 MODULE_PARM_DESC(notests, "disable crypto self-tests");
  47
  48 static bool panic_on_fail;
  49 module_param(panic_on_fail, bool, 0444);
  50
  51 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
  52 static bool noextratests;
  53 module_param(noextratests, bool, 0644);
  54 MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests");
  55
  56 static unsigned int fuzz_iterations = 100;
  57 module_param(fuzz_iterations, uint, 0644);
  58 MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
  59 #endif
  60
  61 #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS
  62
  63 /* a perfect nop */
  64 int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
  65 {
  66         return 0;
  67 }
  68
  69 #else
  70
  71 #include "testmgr.h"
  72
  73 /*
  74  * Need slab memory for testing (size in number of pages).
  75  */
  76 #define XBUFSIZE        8
  77
  78 /*
  79 * Used by test_cipher()
  80 */
  81 #define ENCRYPT 1
  82 #define DECRYPT 0
  83
  84 struct aead_test_suite {
  85         const struct aead_testvec *vecs;
  86         unsigned int count;
  87
  88         /*
  89          * Set if trying to decrypt an inauthentic ciphertext with this
  90          * algorithm might result in EINVAL rather than EBADMSG, due to other
  91          * validation the algorithm does on the inputs such as length checks.
  92          */
  93         unsigned int einval_allowed : 1;
  94
  95         /*
  96          * Set if this algorithm requires that the IV be located at the end of
  97          * the AAD buffer, in addition to being given in the normal way.  The
  98          * behavior when the two IV copies differ is implementation-defined.
  99          */
 100         unsigned int aad_iv : 1;
 101 };
 102
 103 struct cipher_test_suite {
 104         const struct cipher_testvec *vecs;
 105         unsigned int count;
 106 };
 107
 108 struct comp_test_suite {
 109         struct {
 110                 const struct comp_testvec *vecs;
 111                 unsigned int count;
 112         } comp, decomp;
 113 };
 114
 115 struct hash_test_suite {
 116         const struct hash_testvec *vecs;
 117         unsigned int count;
 118 };
 119
 120 struct cprng_test_suite {
 121         const struct cprng_testvec *vecs;
 122         unsigned int count;
 123 };
 124
 125 struct drbg_test_suite {
 126         const struct drbg_testvec *vecs;
 127         unsigned int count;
 128 };
 129
 130 struct akcipher_test_suite {
 131         const struct akcipher_testvec *vecs;
 132         unsigned int count;
 133 };
 134
 135 struct sig_test_suite {
 136         const struct sig_testvec *vecs;
 137         unsigned int count;
 138 };
 139
 140 struct kpp_test_suite {
 141         const struct kpp_testvec *vecs;
 142         unsigned int count;
 143 };
 144
 145 struct alg_test_desc {
 146         const char *alg;
 147         const char *generic_driver;
 148         int (*test)(const struct alg_test_desc *desc, const char *driver,
 149                     u32 type, u32 mask);
 150         int fips_allowed;       /* set if alg is allowed in fips mode */
 151
 152         union {
 153                 struct aead_test_suite aead;
 154                 struct cipher_test_suite cipher;
 155                 struct comp_test_suite comp;
 156                 struct hash_test_suite hash;
 157                 struct cprng_test_suite cprng;
 158                 struct drbg_test_suite drbg;
 159                 struct akcipher_test_suite akcipher;
 160                 struct sig_test_suite sig;
 161                 struct kpp_test_suite kpp;
 162         } suite;
 163 };
 164
 165 static void hexdump(unsigned char *buf, unsigned int len)
 166 {
 167         print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 168                         16, 1,
 169                         buf, len, false);
 170 }
 171
 172 static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)
 173 {
 174         int i;
 175
 176         for (i = 0; i < XBUFSIZE; i++) {
 177                 buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);
 178                 if (!buf[i])
 179                         goto err_free_buf;
 180         }
 181
 182         return 0;
 183
 184 err_free_buf:
 185         while (i-- > 0)
 186                 free_pages((unsigned long)buf[i], order);
 187
 188         return -ENOMEM;
 189 }
 190
 191 static int testmgr_alloc_buf(char *buf[XBUFSIZE])
 192 {
 193         return __testmgr_alloc_buf(buf, 0);
 194 }
 195
 196 static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)
 197 {
 198         int i;
 199
 200         for (i = 0; i < XBUFSIZE; i++)
 201                 free_pages((unsigned long)buf[i], order);
 202 }
 203
 204 static void testmgr_free_buf(char *buf[XBUFSIZE])
 205 {
 206         __testmgr_free_buf(buf, 0);
 207 }
 208
 209 #define TESTMGR_POISON_BYTE     0xfe
 210 #define TESTMGR_POISON_LEN      16
 211
 212 static inline void testmgr_poison(void *addr, size_t len)
 213 {
 214         memset(addr, TESTMGR_POISON_BYTE, len);
 215 }
 216
 217 /* Is the memory region still fully poisoned? */
 218 static inline bool testmgr_is_poison(const void *addr, size_t len)
 219 {
 220         return memchr_inv(addr, TESTMGR_POISON_BYTE, len) == NULL;
 221 }
 222
 223 /* flush type for hash algorithms */
 224 enum flush_type {
 225         /* merge with update of previous buffer(s) */
 226         FLUSH_TYPE_NONE = 0,
 227
 228         /* update with previous buffer(s) before doing this one */
 229         FLUSH_TYPE_FLUSH,
 230
 231         /* likewise, but also export and re-import the intermediate state */
 232         FLUSH_TYPE_REIMPORT,
 233 };
 234
 235 /* finalization function for hash algorithms */
 236 enum finalization_type {
 237         FINALIZATION_TYPE_FINAL,        /* use final() */
 238         FINALIZATION_TYPE_FINUP,        /* use finup() */
 239         FINALIZATION_TYPE_DIGEST,       /* use digest() */
 240 };
 241
 242 /*
 243  * Whether the crypto operation will occur in-place, and if so whether the
 244  * source and destination scatterlist pointers will coincide (req->src ==
 245  * req->dst), or whether they'll merely point to two separate scatterlists
 246  * (req->src != req->dst) that reference the same underlying memory.
 247  *
 248  * This is only relevant for algorithm types that support in-place operation.
 249  */
 250 enum inplace_mode {
 251         OUT_OF_PLACE,
 252         INPLACE_ONE_SGLIST,
 253         INPLACE_TWO_SGLISTS,
 254 };
 255
 256 #define TEST_SG_TOTAL   10000
 257
 258 /**
 259  * struct test_sg_division - description of a scatterlist entry
 260  *
 261  * This struct describes one entry of a scatterlist being constructed to check a
 262  * crypto test vector.
 263  *
 264  * @proportion_of_total: length of this chunk relative to the total length,
 265  *                       given as a proportion out of TEST_SG_TOTAL so that it
 266  *                       scales to fit any test vector
 267  * @offset: byte offset into a 2-page buffer at which this chunk will start
 268  * @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the
 269  *                                @offset
 270  * @flush_type: for hashes, whether an update() should be done now vs.
 271  *              continuing to accumulate data
 272  * @nosimd: if doing the pending update(), do it with SIMD disabled?
 273  */
 274 struct test_sg_division {
 275         unsigned int proportion_of_total;
 276         unsigned int offset;
 277         bool offset_relative_to_alignmask;
 278         enum flush_type flush_type;
 279         bool nosimd;
 280 };
 281
 282 /**
 283  * struct testvec_config - configuration for testing a crypto test vector
 284  *
 285  * This struct describes the data layout and other parameters with which each
 286  * crypto test vector can be tested.
 287  *
 288  * @name: name of this config, logged for debugging purposes if a test fails
 289  * @inplace_mode: whether and how to operate on the data in-place, if applicable
 290  * @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP
 291  * @src_divs: description of how to arrange the source scatterlist
 292  * @dst_divs: description of how to arrange the dst scatterlist, if applicable
 293  *            for the algorithm type.  Defaults to @src_divs if unset.
 294  * @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],
 295  *             where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary
 296  * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 297  *                                   the @iv_offset
 298  * @key_offset: misalignment of the key, where 0 is default alignment
 299  * @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 300  *                                    the @key_offset
 301  * @finalization_type: what finalization function to use for hashes
 302  * @nosimd: execute with SIMD disabled?  Requires !CRYPTO_TFM_REQ_MAY_SLEEP.
 303  *          This applies to the parts of the operation that aren't controlled
 304  *          individually by @nosimd_setkey or @src_divs[].nosimd.
 305  * @nosimd_setkey: set the key (if applicable) with SIMD disabled?  Requires
 306  *                 !CRYPTO_TFM_REQ_MAY_SLEEP.
 307  */
 308 struct testvec_config {
 309         const char *name;
 310         enum inplace_mode inplace_mode;
 311         u32 req_flags;
 312         struct test_sg_division src_divs[XBUFSIZE];
 313         struct test_sg_division dst_divs[XBUFSIZE];
 314         unsigned int iv_offset;
 315         unsigned int key_offset;
 316         bool iv_offset_relative_to_alignmask;
 317         bool key_offset_relative_to_alignmask;
 318         enum finalization_type finalization_type;
 319         bool nosimd;
 320         bool nosimd_setkey;
 321 };
 322
 323 #define TESTVEC_CONFIG_NAMELEN  192
 324
 325 /*
 326  * The following are the lists of testvec_configs to test for each algorithm
 327  * type when the basic crypto self-tests are enabled, i.e. when
 328  * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset.  They aim to provide good test
 329  * coverage, while keeping the test time much shorter than the full fuzz tests
 330  * so that the basic tests can be enabled in a wider range of circumstances.
 331  */
 332
 333 /* Configs for skciphers and aeads */
 334 static const struct testvec_config default_cipher_testvec_configs[] = {
 335         {
 336                 .name = "in-place (one sglist)",
 337                 .inplace_mode = INPLACE_ONE_SGLIST,
 338                 .src_divs = { { .proportion_of_total = 10000 } },
 339         }, {
 340                 .name = "in-place (two sglists)",
 341                 .inplace_mode = INPLACE_TWO_SGLISTS,
 342                 .src_divs = { { .proportion_of_total = 10000 } },
 343         }, {
 344                 .name = "out-of-place",
 345                 .inplace_mode = OUT_OF_PLACE,
 346                 .src_divs = { { .proportion_of_total = 10000 } },
 347         }, {
 348                 .name = "unaligned buffer, offset=1",
 349                 .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 350                 .iv_offset = 1,
 351                 .key_offset = 1,
 352         }, {
 353                 .name = "buffer aligned only to alignmask",
 354                 .src_divs = {
 355                         {
 356                                 .proportion_of_total = 10000,
 357                                 .offset = 1,
 358                                 .offset_relative_to_alignmask = true,
 359                         },
 360                 },
 361                 .iv_offset = 1,
 362                 .iv_offset_relative_to_alignmask = true,
 363                 .key_offset = 1,
 364                 .key_offset_relative_to_alignmask = true,
 365         }, {
 366                 .name = "two even aligned splits",
 367                 .src_divs = {
 368                         { .proportion_of_total = 5000 },
 369                         { .proportion_of_total = 5000 },
 370                 },
 371         }, {
 372                 .name = "one src, two even splits dst",
 373                 .inplace_mode = OUT_OF_PLACE,
 374                 .src_divs = { { .proportion_of_total = 10000 } },
 375                 .dst_divs = {
 376                         { .proportion_of_total = 5000 },
 377                         { .proportion_of_total = 5000 },
 378                  },
 379         }, {
 380                 .name = "uneven misaligned splits, may sleep",
 381                 .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 382                 .src_divs = {
 383                         { .proportion_of_total = 1900, .offset = 33 },
 384                         { .proportion_of_total = 3300, .offset = 7  },
 385                         { .proportion_of_total = 4800, .offset = 18 },
 386                 },
 387                 .iv_offset = 3,
 388                 .key_offset = 3,
 389         }, {
 390                 .name = "misaligned splits crossing pages, inplace",
 391                 .inplace_mode = INPLACE_ONE_SGLIST,
 392                 .src_divs = {
 393                         {
 394                                 .proportion_of_total = 7500,
 395                                 .offset = PAGE_SIZE - 32
 396                         }, {
 397                                 .proportion_of_total = 2500,
 398                                 .offset = PAGE_SIZE - 7
 399                         },
 400                 },
 401         }
 402 };
 403
 404 static const struct testvec_config default_hash_testvec_configs[] = {
 405         {
 406                 .name = "init+update+final aligned buffer",
 407                 .src_divs = { { .proportion_of_total = 10000 } },
 408                 .finalization_type = FINALIZATION_TYPE_FINAL,
 409         }, {
 410                 .name = "init+finup aligned buffer",
 411                 .src_divs = { { .proportion_of_total = 10000 } },
 412                 .finalization_type = FINALIZATION_TYPE_FINUP,
 413         }, {
 414                 .name = "digest aligned buffer",
 415                 .src_divs = { { .proportion_of_total = 10000 } },
 416                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 417         }, {
 418                 .name = "init+update+final misaligned buffer",
 419                 .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 420                 .finalization_type = FINALIZATION_TYPE_FINAL,
 421                 .key_offset = 1,
 422         }, {
 423                 .name = "digest misaligned buffer",
 424                 .src_divs = {
 425                         {
 426                                 .proportion_of_total = 10000,
 427                                 .offset = 1,
 428                         },
 429                 },
 430                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 431                 .key_offset = 1,
 432         }, {
 433                 .name = "init+update+update+final two even splits",
 434                 .src_divs = {
 435                         { .proportion_of_total = 5000 },
 436                         {
 437                                 .proportion_of_total = 5000,
 438                                 .flush_type = FLUSH_TYPE_FLUSH,
 439                         },
 440                 },
 441                 .finalization_type = FINALIZATION_TYPE_FINAL,
 442         }, {
 443                 .name = "digest uneven misaligned splits, may sleep",
 444                 .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 445                 .src_divs = {
 446                         { .proportion_of_total = 1900, .offset = 33 },
 447                         { .proportion_of_total = 3300, .offset = 7  },
 448                         { .proportion_of_total = 4800, .offset = 18 },
 449                 },
 450                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 451         }, {
 452                 .name = "digest misaligned splits crossing pages",
 453                 .src_divs = {
 454                         {
 455                                 .proportion_of_total = 7500,
 456                                 .offset = PAGE_SIZE - 32,
 457                         }, {
 458                                 .proportion_of_total = 2500,
 459                                 .offset = PAGE_SIZE - 7,
 460                         },
 461                 },
 462                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 463         }, {
 464                 .name = "import/export",
 465                 .src_divs = {
 466                         {
 467                                 .proportion_of_total = 6500,
 468                                 .flush_type = FLUSH_TYPE_REIMPORT,
 469                         }, {
 470                                 .proportion_of_total = 3500,
 471                                 .flush_type = FLUSH_TYPE_REIMPORT,
 472                         },
 473                 },
 474                 .finalization_type = FINALIZATION_TYPE_FINAL,
 475         }
 476 };
 477
 478 static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)
 479 {
 480         unsigned int remaining = TEST_SG_TOTAL;
 481         unsigned int ndivs = 0;
 482
 483         do {
 484                 remaining -= divs[ndivs++].proportion_of_total;
 485         } while (remaining);
 486
 487         return ndivs;
 488 }
 489
 490 #define SGDIVS_HAVE_FLUSHES     BIT(0)
 491 #define SGDIVS_HAVE_NOSIMD      BIT(1)
 492
 493 static bool valid_sg_divisions(const struct test_sg_division *divs,
 494                                unsigned int count, int *flags_ret)
 495 {
 496         unsigned int total = 0;
 497         unsigned int i;
 498
 499         for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {
 500                 if (divs[i].proportion_of_total <= 0 ||
 501                     divs[i].proportion_of_total > TEST_SG_TOTAL - total)
 502                         return false;
 503                 total += divs[i].proportion_of_total;
 504                 if (divs[i].flush_type != FLUSH_TYPE_NONE)
 505                         *flags_ret |= SGDIVS_HAVE_FLUSHES;
 506                 if (divs[i].nosimd)
 507                         *flags_ret |= SGDIVS_HAVE_NOSIMD;
 508         }
 509         return total == TEST_SG_TOTAL &&
 510                 memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL;
 511 }
 512
 513 /*
 514  * Check whether the given testvec_config is valid.  This isn't strictly needed
 515  * since every testvec_config should be valid, but check anyway so that people
 516  * don't unknowingly add broken configs that don't do what they wanted.
 517  */
 518 static bool valid_testvec_config(const struct testvec_config *cfg)
 519 {
 520         int flags = 0;
 521
 522         if (cfg->name == NULL)
 523                 return false;
 524
 525         if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs),
 526                                 &flags))
 527                 return false;
 528
 529         if (cfg->dst_divs[0].proportion_of_total) {
 530                 if (!valid_sg_divisions(cfg->dst_divs,
 531                                         ARRAY_SIZE(cfg->dst_divs), &flags))
 532                         return false;
 533         } else {
 534                 if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs)))
 535                         return false;
 536                 /* defaults to dst_divs=src_divs */
 537         }
 538
 539         if (cfg->iv_offset +
 540             (cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >
 541             MAX_ALGAPI_ALIGNMASK + 1)
 542                 return false;
 543
 544         if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) &&
 545             cfg->finalization_type == FINALIZATION_TYPE_DIGEST)
 546                 return false;
 547
 548         if ((cfg->nosimd || cfg->nosimd_setkey ||
 549              (flags & SGDIVS_HAVE_NOSIMD)) &&
 550             (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))
 551                 return false;
 552
 553         return true;
 554 }
 555
 556 struct test_sglist {
 557         char *bufs[XBUFSIZE];
 558         struct scatterlist sgl[XBUFSIZE];
 559         struct scatterlist sgl_saved[XBUFSIZE];
 560         struct scatterlist *sgl_ptr;
 561         unsigned int nents;
 562 };
 563
 564 static int init_test_sglist(struct test_sglist *tsgl)
 565 {
 566         return __testmgr_alloc_buf(tsgl->bufs, 1 /* two pages per buffer */);
 567 }
 568
 569 static void destroy_test_sglist(struct test_sglist *tsgl)
 570 {
 571         return __testmgr_free_buf(tsgl->bufs, 1 /* two pages per buffer */);
 572 }
 573
 574 /**
 575  * build_test_sglist() - build a scatterlist for a crypto test
 576  *
 577  * @tsgl: the scatterlist to build.  @tsgl->bufs[] contains an array of 2-page
 578  *        buffers which the scatterlist @tsgl->sgl[] will be made to point into.
 579  * @divs: the layout specification on which the scatterlist will be based
 580  * @alignmask: the algorithm's alignmask
 581  * @total_len: the total length of the scatterlist to build in bytes
 582  * @data: if non-NULL, the buffers will be filled with this data until it ends.
 583  *        Otherwise the buffers will be poisoned.  In both cases, some bytes
 584  *        past the end of each buffer will be poisoned to help detect overruns.
 585  * @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry
 586  *            corresponds will be returned here.  This will match @divs except
 587  *            that divisions resolving to a length of 0 are omitted as they are
 588  *            not included in the scatterlist.
 589  *
 590  * Return: 0 or a -errno value
 591  */
 592 static int build_test_sglist(struct test_sglist *tsgl,
 593                              const struct test_sg_division *divs,
 594                              const unsigned int alignmask,
 595                              const unsigned int total_len,
 596                              struct iov_iter *data,
 597                              const struct test_sg_division *out_divs[XBUFSIZE])
 598 {
 599         struct {
 600                 const struct test_sg_division *div;
 601                 size_t length;
 602         } partitions[XBUFSIZE];
 603         const unsigned int ndivs = count_test_sg_divisions(divs);
 604         unsigned int len_remaining = total_len;
 605         unsigned int i;
 606
 607         BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));
 608         if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))
 609                 return -EINVAL;
 610
 611         /* Calculate the (div, length) pairs */
 612         tsgl->nents = 0;
 613         for (i = 0; i < ndivs; i++) {
 614                 unsigned int len_this_sg =
 615                         min(len_remaining,
 616                             (total_len * divs[i].proportion_of_total +
 617                              TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);
 618
 619                 if (len_this_sg != 0) {
 620                         partitions[tsgl->nents].div = &divs[i];
 621                         partitions[tsgl->nents].length = len_this_sg;
 622                         tsgl->nents++;
 623                         len_remaining -= len_this_sg;
 624                 }
 625         }
 626         if (tsgl->nents == 0) {
 627                 partitions[tsgl->nents].div = &divs[0];
 628                 partitions[tsgl->nents].length = 0;
 629                 tsgl->nents++;
 630         }
 631         partitions[tsgl->nents - 1].length += len_remaining;
 632
 633         /* Set up the sgl entries and fill the data or poison */
 634         sg_init_table(tsgl->sgl, tsgl->nents);
 635         for (i = 0; i < tsgl->nents; i++) {
 636                 unsigned int offset = partitions[i].div->offset;
 637                 void *addr;
 638
 639                 if (partitions[i].div->offset_relative_to_alignmask)
 640                         offset += alignmask;
 641
 642                 while (offset + partitions[i].length + TESTMGR_POISON_LEN >
 643                        2 * PAGE_SIZE) {
 644                         if (WARN_ON(offset <= 0))
 645                                 return -EINVAL;
 646                         offset /= 2;
 647                 }
 648
 649                 addr = &tsgl->bufs[i][offset];
 650                 sg_set_buf(&tsgl->sgl[i], addr, partitions[i].length);
 651
 652                 if (out_divs)
 653                         out_divs[i] = partitions[i].div;
 654
 655                 if (data) {
 656                         size_t copy_len, copied;
 657
 658                         copy_len = min(partitions[i].length, data->count);
 659                         copied = copy_from_iter(addr, copy_len, data);
 660                         if (WARN_ON(copied != copy_len))
 661                                 return -EINVAL;
 662                         testmgr_poison(addr + copy_len, partitions[i].length +
 663                                        TESTMGR_POISON_LEN - copy_len);
 664                 } else {
 665                         testmgr_poison(addr, partitions[i].length +
 666                                        TESTMGR_POISON_LEN);
 667                 }
 668         }
 669
 670         sg_mark_end(&tsgl->sgl[tsgl->nents - 1]);
 671         tsgl->sgl_ptr = tsgl->sgl;
 672         memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));
 673         return 0;
 674 }
 675
 676 /*
 677  * Verify that a scatterlist crypto operation produced the correct output.
 678  *
 679  * @tsgl: scatterlist containing the actual output
 680  * @expected_output: buffer containing the expected output
 681  * @len_to_check: length of @expected_output in bytes
 682  * @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result
 683  * @check_poison: verify that the poison bytes after each chunk are intact?
 684  *
 685  * Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.
 686  */
 687 static int verify_correct_output(const struct test_sglist *tsgl,
 688                                  const char *expected_output,
 689                                  unsigned int len_to_check,
 690                                  unsigned int unchecked_prefix_len,
 691                                  bool check_poison)
 692 {
 693         unsigned int i;
 694
 695         for (i = 0; i < tsgl->nents; i++) {
 696                 struct scatterlist *sg = &tsgl->sgl_ptr[i];
 697                 unsigned int len = sg->length;
 698                 unsigned int offset = sg->offset;
 699                 const char *actual_output;
 700
 701                 if (unchecked_prefix_len) {
 702                         if (unchecked_prefix_len >= len) {
 703                                 unchecked_prefix_len -= len;
 704                                 continue;
 705                         }
 706                         offset += unchecked_prefix_len;
 707                         len -= unchecked_prefix_len;
 708                         unchecked_prefix_len = 0;
 709                 }
 710                 len = min(len, len_to_check);
 711                 actual_output = page_address(sg_page(sg)) + offset;
 712                 if (memcmp(expected_output, actual_output, len) != 0)
 713                         return -EINVAL;
 714                 if (check_poison &&
 715                     !testmgr_is_poison(actual_output + len, TESTMGR_POISON_LEN))
 716                         return -EOVERFLOW;
 717                 len_to_check -= len;
 718                 expected_output += len;
 719         }
 720         if (WARN_ON(len_to_check != 0))
 721                 return -EINVAL;
 722         return 0;
 723 }
 724
 725 static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)
 726 {
 727         unsigned int i;
 728
 729         for (i = 0; i < tsgl->nents; i++) {
 730                 if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)
 731                         return true;
 732                 if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)
 733                         return true;
 734                 if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)
 735                         return true;
 736         }
 737         return false;
 738 }
 739
 740 struct cipher_test_sglists {
 741         struct test_sglist src;
 742         struct test_sglist dst;
 743 };
 744
 745 static struct cipher_test_sglists *alloc_cipher_test_sglists(void)
 746 {
 747         struct cipher_test_sglists *tsgls;
 748
 749         tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);
 750         if (!tsgls)
 751                 return NULL;
 752
 753         if (init_test_sglist(&tsgls->src) != 0)
 754                 goto fail_kfree;
 755         if (init_test_sglist(&tsgls->dst) != 0)
 756                 goto fail_destroy_src;
 757
 758         return tsgls;
 759
 760 fail_destroy_src:
 761         destroy_test_sglist(&tsgls->src);
 762 fail_kfree:
 763         kfree(tsgls);
 764         return NULL;
 765 }
 766
 767 static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)
 768 {
 769         if (tsgls) {
 770                 destroy_test_sglist(&tsgls->src);
 771                 destroy_test_sglist(&tsgls->dst);
 772                 kfree(tsgls);
 773         }
 774 }
 775
 776 /* Build the src and dst scatterlists for an skcipher or AEAD test */
 777 static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,
 778                                      const struct testvec_config *cfg,
 779                                      unsigned int alignmask,
 780                                      unsigned int src_total_len,
 781                                      unsigned int dst_total_len,
 782                                      const struct kvec *inputs,
 783                                      unsigned int nr_inputs)
 784 {
 785         struct iov_iter input;
 786         int err;
 787
 788         iov_iter_kvec(&input, ITER_SOURCE, inputs, nr_inputs, src_total_len);
 789         err = build_test_sglist(&tsgls->src, cfg->src_divs, alignmask,
 790                                 cfg->inplace_mode != OUT_OF_PLACE ?
 791                                         max(dst_total_len, src_total_len) :
 792                                         src_total_len,
 793                                 &input, NULL);
 794         if (err)
 795                 return err;
 796
 797         /*
 798          * In-place crypto operations can use the same scatterlist for both the
 799          * source and destination (req->src == req->dst), or can use separate
 800          * scatterlists (req->src != req->dst) which point to the same
 801          * underlying memory.  Make sure to test both cases.
 802          */
 803         if (cfg->inplace_mode == INPLACE_ONE_SGLIST) {
 804                 tsgls->dst.sgl_ptr = tsgls->src.sgl;
 805                 tsgls->dst.nents = tsgls->src.nents;
 806                 return 0;
 807         }
 808         if (cfg->inplace_mode == INPLACE_TWO_SGLISTS) {
 809                 /*
 810                  * For now we keep it simple and only test the case where the
 811                  * two scatterlists have identical entries, rather than
 812                  * different entries that split up the same memory differently.
 813                  */
 814                 memcpy(tsgls->dst.sgl, tsgls->src.sgl,
 815                        tsgls->src.nents * sizeof(tsgls->src.sgl[0]));
 816                 memcpy(tsgls->dst.sgl_saved, tsgls->src.sgl,
 817                        tsgls->src.nents * sizeof(tsgls->src.sgl[0]));
 818                 tsgls->dst.sgl_ptr = tsgls->dst.sgl;
 819                 tsgls->dst.nents = tsgls->src.nents;
 820                 return 0;
 821         }
 822         /* Out of place */
 823         return build_test_sglist(&tsgls->dst,
 824                                  cfg->dst_divs[0].proportion_of_total ?
 825                                         cfg->dst_divs : cfg->src_divs,
 826                                  alignmask, dst_total_len, NULL, NULL);
 827 }
 828
 829 /*
 830  * Support for testing passing a misaligned key to setkey():
 831  *
 832  * If cfg->key_offset is set, copy the key into a new buffer at that offset,
 833  * optionally adding alignmask.  Else, just use the key directly.
 834  */
 835 static int prepare_keybuf(const u8 *key, unsigned int ksize,
 836                           const struct testvec_config *cfg,
 837                           unsigned int alignmask,
 838                           const u8 **keybuf_ret, const u8 **keyptr_ret)
 839 {
 840         unsigned int key_offset = cfg->key_offset;
 841         u8 *keybuf = NULL, *keyptr = (u8 *)key;
 842
 843         if (key_offset != 0) {
 844                 if (cfg->key_offset_relative_to_alignmask)
 845                         key_offset += alignmask;
 846                 keybuf = kmalloc(key_offset + ksize, GFP_KERNEL);
 847                 if (!keybuf)
 848                         return -ENOMEM;
 849                 keyptr = keybuf + key_offset;
 850                 memcpy(keyptr, key, ksize);
 851         }
 852         *keybuf_ret = keybuf;
 853         *keyptr_ret = keyptr;
 854         return 0;
 855 }
 856
 857 /*
 858  * Like setkey_f(tfm, key, ksize), but sometimes misalign the key.
 859  * In addition, run the setkey function in no-SIMD context if requested.
 860  */
 861 #define do_setkey(setkey_f, tfm, key, ksize, cfg, alignmask)            \
 862 ({                                                                      \
 863         const u8 *keybuf, *keyptr;                                      \
 864         int err;                                                        \
 865                                                                         \
 866         err = prepare_keybuf((key), (ksize), (cfg), (alignmask),        \
 867                              &keybuf, &keyptr);                         \
 868         if (err == 0) {                                                 \
 869                 if ((cfg)->nosimd_setkey)                               \
 870                         crypto_disable_simd_for_test();                 \
 871                 err = setkey_f((tfm), keyptr, (ksize));                 \
 872                 if ((cfg)->nosimd_setkey)                               \
 873                         crypto_reenable_simd_for_test();                \
 874                 kfree(keybuf);                                          \
 875         }                                                               \
 876         err;                                                            \
 877 })
 878
 879 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
 880
 881 /*
 882  * The fuzz tests use prandom instead of the normal Linux RNG since they don't
 883  * need cryptographically secure random numbers.  This greatly improves the
 884  * performance of these tests, especially if they are run before the Linux RNG
 885  * has been initialized or if they are run on a lockdep-enabled kernel.
 886  */
 887
 888 static inline void init_rnd_state(struct rnd_state *rng)
 889 {
 890         prandom_seed_state(rng, get_random_u64());
 891 }
 892
 893 static inline u8 prandom_u8(struct rnd_state *rng)
 894 {
 895         return prandom_u32_state(rng);
 896 }
 897
 898 static inline u32 prandom_u32_below(struct rnd_state *rng, u32 ceil)
 899 {
 900         /*
 901          * This is slightly biased for non-power-of-2 values of 'ceil', but this
 902          * isn't important here.
 903          */
 904         return prandom_u32_state(rng) % ceil;
 905 }
 906
 907 static inline bool prandom_bool(struct rnd_state *rng)
 908 {
 909         return prandom_u32_below(rng, 2);
 910 }
 911
 912 static inline u32 prandom_u32_inclusive(struct rnd_state *rng,
 913                                         u32 floor, u32 ceil)
 914 {
 915         return floor + prandom_u32_below(rng, ceil - floor + 1);
 916 }
 917
 918 /* Generate a random length in range [0, max_len], but prefer smaller values */
 919 static unsigned int generate_random_length(struct rnd_state *rng,
 920                                            unsigned int max_len)
 921 {
 922         unsigned int len = prandom_u32_below(rng, max_len + 1);
 923
 924         switch (prandom_u32_below(rng, 4)) {
 925         case 0:
 926                 len %= 64;
 927                 break;
 928         case 1:
 929                 len %= 256;
 930                 break;
 931         case 2:
 932                 len %= 1024;
 933                 break;
 934         default:
 935                 break;
 936         }
 937         if (len && prandom_u32_below(rng, 4) == 0)
 938                 len = rounddown_pow_of_two(len);
 939         return len;
 940 }
 941
 942 /* Flip a random bit in the given nonempty data buffer */
 943 static void flip_random_bit(struct rnd_state *rng, u8 *buf, size_t size)
 944 {
 945         size_t bitpos;
 946
 947         bitpos = prandom_u32_below(rng, size * 8);
 948         buf[bitpos / 8] ^= 1 << (bitpos % 8);
 949 }
 950
 951 /* Flip a random byte in the given nonempty data buffer */
 952 static void flip_random_byte(struct rnd_state *rng, u8 *buf, size_t size)
 953 {
 954         buf[prandom_u32_below(rng, size)] ^= 0xff;
 955 }
 956
 957 /* Sometimes make some random changes to the given nonempty data buffer */
 958 static void mutate_buffer(struct rnd_state *rng, u8 *buf, size_t size)
 959 {
 960         size_t num_flips;
 961         size_t i;
 962
 963         /* Sometimes flip some bits */
 964         if (prandom_u32_below(rng, 4) == 0) {
 965                 num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8),
 966                                   size * 8);
 967                 for (i = 0; i < num_flips; i++)
 968                         flip_random_bit(rng, buf, size);
 969         }
 970
 971         /* Sometimes flip some bytes */
 972         if (prandom_u32_below(rng, 4) == 0) {
 973                 num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8), size);
 974                 for (i = 0; i < num_flips; i++)
 975                         flip_random_byte(rng, buf, size);
 976         }
 977 }
 978
 979 /* Randomly generate 'count' bytes, but sometimes make them "interesting" */
 980 static void generate_random_bytes(struct rnd_state *rng, u8 *buf, size_t count)
 981 {
 982         u8 b;
 983         u8 increment;
 984         size_t i;
 985
 986         if (count == 0)
 987                 return;
 988
 989         switch (prandom_u32_below(rng, 8)) { /* Choose a generation strategy */
 990         case 0:
 991         case 1:
 992                 /* All the same byte, plus optional mutations */
 993                 switch (prandom_u32_below(rng, 4)) {
 994                 case 0:
 995                         b = 0x00;
 996                         break;
 997                 case 1:
 998                         b = 0xff;
 999                         break;
1000                 default:
1001                         b = prandom_u8(rng);
1002                         break;
1003                 }
1004                 memset(buf, b, count);
1005                 mutate_buffer(rng, buf, count);
1006                 break;
1007         case 2:
1008                 /* Ascending or descending bytes, plus optional mutations */
1009                 increment = prandom_u8(rng);
1010                 b = prandom_u8(rng);
1011                 for (i = 0; i < count; i++, b += increment)
1012                         buf[i] = b;
1013                 mutate_buffer(rng, buf, count);
1014                 break;
1015         default:
1016                 /* Fully random bytes */
1017                 prandom_bytes_state(rng, buf, count);
1018         }
1019 }
1020
1021 static char *generate_random_sgl_divisions(struct rnd_state *rng,
1022                                            struct test_sg_division *divs,
1023                                            size_t max_divs, char *p, char *end,
1024                                            bool gen_flushes, u32 req_flags)
1025 {
1026         struct test_sg_division *div = divs;
1027         unsigned int remaining = TEST_SG_TOTAL;
1028
1029         do {
1030                 unsigned int this_len;
1031                 const char *flushtype_str;
1032
1033                 if (div == &divs[max_divs - 1] || prandom_bool(rng))
1034                         this_len = remaining;
1035                 else if (prandom_u32_below(rng, 4) == 0)
1036                         this_len = (remaining + 1) / 2;
1037                 else
1038                         this_len = prandom_u32_inclusive(rng, 1, remaining);
1039                 div->proportion_of_total = this_len;
1040
1041                 if (prandom_u32_below(rng, 4) == 0)
1042                         div->offset = prandom_u32_inclusive(rng,
1043                                                             PAGE_SIZE - 128,
1044                                                             PAGE_SIZE - 1);
1045                 else if (prandom_bool(rng))
1046                         div->offset = prandom_u32_below(rng, 32);
1047                 else
1048                         div->offset = prandom_u32_below(rng, PAGE_SIZE);
1049                 if (prandom_u32_below(rng, 8) == 0)
1050                         div->offset_relative_to_alignmask = true;
1051
1052                 div->flush_type = FLUSH_TYPE_NONE;
1053                 if (gen_flushes) {
1054                         switch (prandom_u32_below(rng, 4)) {
1055                         case 0:
1056                                 div->flush_type = FLUSH_TYPE_REIMPORT;
1057                                 break;
1058                         case 1:
1059                                 div->flush_type = FLUSH_TYPE_FLUSH;
1060                                 break;
1061                         }
1062                 }
1063
1064                 if (div->flush_type != FLUSH_TYPE_NONE &&
1065                     !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
1066                     prandom_bool(rng))
1067                         div->nosimd = true;
1068
1069                 switch (div->flush_type) {
1070                 case FLUSH_TYPE_FLUSH:
1071                         if (div->nosimd)
1072                                 flushtype_str = "<flush,nosimd>";
1073                         else
1074                                 flushtype_str = "<flush>";
1075                         break;
1076                 case FLUSH_TYPE_REIMPORT:
1077                         if (div->nosimd)
1078                                 flushtype_str = "<reimport,nosimd>";
1079                         else
1080                                 flushtype_str = "<reimport>";
1081                         break;
1082                 default:
1083                         flushtype_str = "";
1084                         break;
1085                 }
1086
1087                 BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */
1088                 p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str,
1089                                this_len / 100, this_len % 100,
1090                                div->offset_relative_to_alignmask ?
1091                                         "alignmask" : "",
1092                                div->offset, this_len == remaining ? "" : ", ");
1093                 remaining -= this_len;
1094                 div++;
1095         } while (remaining);
1096
1097         return p;
1098 }
1099
1100 /* Generate a random testvec_config for fuzz testing */
1101 static void generate_random_testvec_config(struct rnd_state *rng,
1102                                            struct testvec_config *cfg,
1103                                            char *name, size_t max_namelen)
1104 {
1105         char *p = name;
1106         char * const end = name + max_namelen;
1107
1108         memset(cfg, 0, sizeof(*cfg));
1109
1110         cfg->name = name;
1111
1112         p += scnprintf(p, end - p, "random:");
1113
1114         switch (prandom_u32_below(rng, 4)) {
1115         case 0:
1116         case 1:
1117                 cfg->inplace_mode = OUT_OF_PLACE;
1118                 break;
1119         case 2:
1120                 cfg->inplace_mode = INPLACE_ONE_SGLIST;
1121                 p += scnprintf(p, end - p, " inplace_one_sglist");
1122                 break;
1123         default:
1124                 cfg->inplace_mode = INPLACE_TWO_SGLISTS;
1125                 p += scnprintf(p, end - p, " inplace_two_sglists");
1126                 break;
1127         }
1128
1129         if (prandom_bool(rng)) {
1130                 cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
1131                 p += scnprintf(p, end - p, " may_sleep");
1132         }
1133
1134         switch (prandom_u32_below(rng, 4)) {
1135         case 0:
1136                 cfg->finalization_type = FINALIZATION_TYPE_FINAL;
1137                 p += scnprintf(p, end - p, " use_final");
1138                 break;
1139         case 1:
1140                 cfg->finalization_type = FINALIZATION_TYPE_FINUP;
1141                 p += scnprintf(p, end - p, " use_finup");
1142                 break;
1143         default:
1144                 cfg->finalization_type = FINALIZATION_TYPE_DIGEST;
1145                 p += scnprintf(p, end - p, " use_digest");
1146                 break;
1147         }
1148
1149         if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) {
1150                 if (prandom_bool(rng)) {
1151                         cfg->nosimd = true;
1152                         p += scnprintf(p, end - p, " nosimd");
1153                 }
1154                 if (prandom_bool(rng)) {
1155                         cfg->nosimd_setkey = true;
1156                         p += scnprintf(p, end - p, " nosimd_setkey");
1157                 }
1158         }
1159
1160         p += scnprintf(p, end - p, " src_divs=[");
1161         p = generate_random_sgl_divisions(rng, cfg->src_divs,
1162                                           ARRAY_SIZE(cfg->src_divs), p, end,
1163                                           (cfg->finalization_type !=
1164                                            FINALIZATION_TYPE_DIGEST),
1165                                           cfg->req_flags);
1166         p += scnprintf(p, end - p, "]");
1167
1168         if (cfg->inplace_mode == OUT_OF_PLACE && prandom_bool(rng)) {
1169                 p += scnprintf(p, end - p, " dst_divs=[");
1170                 p = generate_random_sgl_divisions(rng, cfg->dst_divs,
1171                                                   ARRAY_SIZE(cfg->dst_divs),
1172                                                   p, end, false,
1173                                                   cfg->req_flags);
1174                 p += scnprintf(p, end - p, "]");
1175         }
1176
1177         if (prandom_bool(rng)) {
1178                 cfg->iv_offset = prandom_u32_inclusive(rng, 1,
1179                                                        MAX_ALGAPI_ALIGNMASK);
1180                 p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);
1181         }
1182
1183         if (prandom_bool(rng)) {
1184                 cfg->key_offset = prandom_u32_inclusive(rng, 1,
1185                                                         MAX_ALGAPI_ALIGNMASK);
1186                 p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset);
1187         }
1188
1189         WARN_ON_ONCE(!valid_testvec_config(cfg));
1190 }
1191
1192 static void crypto_disable_simd_for_test(void)
1193 {
1194         migrate_disable();
1195         __this_cpu_write(crypto_simd_disabled_for_test, true);
1196 }
1197
1198 static void crypto_reenable_simd_for_test(void)
1199 {
1200         __this_cpu_write(crypto_simd_disabled_for_test, false);
1201         migrate_enable();
1202 }
1203
1204 /*
1205  * Given an algorithm name, build the name of the generic implementation of that
1206  * algorithm, assuming the usual naming convention.  Specifically, this appends
1207  * "-generic" to every part of the name that is not a template name.  Examples:
1208  *
1209  *      aes => aes-generic
1210  *      cbc(aes) => cbc(aes-generic)
1211  *      cts(cbc(aes)) => cts(cbc(aes-generic))
1212  *      rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)
1213  *
1214  * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long
1215  */
1216 static int build_generic_driver_name(const char *algname,
1217                                      char driver_name[CRYPTO_MAX_ALG_NAME])
1218 {
1219         const char *in = algname;
1220         char *out = driver_name;
1221         size_t len = strlen(algname);
1222
1223         if (len >= CRYPTO_MAX_ALG_NAME)
1224                 goto too_long;
1225         do {
1226                 const char *in_saved = in;
1227
1228                 while (*in && *in != '(' && *in != ')' && *in != ',')
1229                         *out++ = *in++;
1230                 if (*in != '(' && in > in_saved) {
1231                         len += 8;
1232                         if (len >= CRYPTO_MAX_ALG_NAME)
1233                                 goto too_long;
1234                         memcpy(out, "-generic", 8);
1235                         out += 8;
1236                 }
1237         } while ((*out++ = *in++) != '\0');
1238         return 0;
1239
1240 too_long:
1241         pr_err("alg: generic driver name for \"%s\" would be too long\n",
1242                algname);
1243         return -ENAMETOOLONG;
1244 }
1245 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1246 static void crypto_disable_simd_for_test(void)
1247 {
1248 }
1249
1250 static void crypto_reenable_simd_for_test(void)
1251 {
1252 }
1253 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1254
1255 static int build_hash_sglist(struct test_sglist *tsgl,
1256                              const struct hash_testvec *vec,
1257                              const struct testvec_config *cfg,
1258                              unsigned int alignmask,
1259                              const struct test_sg_division *divs[XBUFSIZE])
1260 {
1261         struct kvec kv;
1262         struct iov_iter input;
1263
1264         kv.iov_base = (void *)vec->plaintext;
1265         kv.iov_len = vec->psize;
1266         iov_iter_kvec(&input, ITER_SOURCE, &kv, 1, vec->psize);
1267         return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
1268                                  &input, divs);
1269 }
1270
1271 static int check_hash_result(const char *type,
1272                              const u8 *result, unsigned int digestsize,
1273                              const struct hash_testvec *vec,
1274                              const char *vec_name,
1275                              const char *driver,
1276                              const struct testvec_config *cfg)
1277 {
1278         if (memcmp(result, vec->digest, digestsize) != 0) {
1279                 pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
1280                        type, driver, vec_name, cfg->name);
1281                 return -EINVAL;
1282         }
1283         if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
1284                 pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
1285                        type, driver, vec_name, cfg->name);
1286                 return -EOVERFLOW;
1287         }
1288         return 0;
1289 }
1290
1291 static inline int check_shash_op(const char *op, int err,
1292                                  const char *driver, const char *vec_name,
1293                                  const struct testvec_config *cfg)
1294 {
1295         if (err)
1296                 pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1297                        driver, op, err, vec_name, cfg->name);
1298         return err;
1299 }
1300
1301 /* Test one hash test vector in one configuration, using the shash API */
1302 static int test_shash_vec_cfg(const struct hash_testvec *vec,
1303                               const char *vec_name,
1304                               const struct testvec_config *cfg,
1305                               struct shash_desc *desc,
1306                               struct test_sglist *tsgl,
1307                               u8 *hashstate)
1308 {
1309         struct crypto_shash *tfm = desc->tfm;
1310         const unsigned int digestsize = crypto_shash_digestsize(tfm);
1311         const unsigned int statesize = crypto_shash_statesize(tfm);
1312         const char *driver = crypto_shash_driver_name(tfm);
1313         const struct test_sg_division *divs[XBUFSIZE];
1314         unsigned int i;
1315         u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1316         int err;
1317
1318         /* Set the key, if specified */
1319         if (vec->ksize) {
1320                 err = do_setkey(crypto_shash_setkey, tfm, vec->key, vec->ksize,
1321                                 cfg, 0);
1322                 if (err) {
1323                         if (err == vec->setkey_error)
1324                                 return 0;
1325                         pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1326                                driver, vec_name, vec->setkey_error, err,
1327                                crypto_shash_get_flags(tfm));
1328                         return err;
1329                 }
1330                 if (vec->setkey_error) {
1331                         pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1332                                driver, vec_name, vec->setkey_error);
1333                         return -EINVAL;
1334                 }
1335         }
1336
1337         /* Build the scatterlist for the source data */
1338         err = build_hash_sglist(tsgl, vec, cfg, 0, divs);
1339         if (err) {
1340                 pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1341                        driver, vec_name, cfg->name);
1342                 return err;
1343         }
1344
1345         /* Do the actual hashing */
1346
1347         testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1348         testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1349
1350         if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1351             vec->digest_error) {
1352                 /* Just using digest() */
1353                 if (tsgl->nents != 1)
1354                         return 0;
1355                 if (cfg->nosimd)
1356                         crypto_disable_simd_for_test();
1357                 err = crypto_shash_digest(desc, sg_virt(&tsgl->sgl[0]),
1358                                           tsgl->sgl[0].length, result);
1359                 if (cfg->nosimd)
1360                         crypto_reenable_simd_for_test();
1361                 if (err) {
1362                         if (err == vec->digest_error)
1363                                 return 0;
1364                         pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1365                                driver, vec_name, vec->digest_error, err,
1366                                cfg->name);
1367                         return err;
1368                 }
1369                 if (vec->digest_error) {
1370                         pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1371                                driver, vec_name, vec->digest_error, cfg->name);
1372                         return -EINVAL;
1373                 }
1374                 goto result_ready;
1375         }
1376
1377         /* Using init(), zero or more update(), then final() or finup() */
1378
1379         if (cfg->nosimd)
1380                 crypto_disable_simd_for_test();
1381         err = crypto_shash_init(desc);
1382         if (cfg->nosimd)
1383                 crypto_reenable_simd_for_test();
1384         err = check_shash_op("init", err, driver, vec_name, cfg);
1385         if (err)
1386                 return err;
1387
1388         for (i = 0; i < tsgl->nents; i++) {
1389                 if (i + 1 == tsgl->nents &&
1390                     cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
1391                         if (divs[i]->nosimd)
1392                                 crypto_disable_simd_for_test();
1393                         err = crypto_shash_finup(desc, sg_virt(&tsgl->sgl[i]),
1394                                                  tsgl->sgl[i].length, result);
1395                         if (divs[i]->nosimd)
1396                                 crypto_reenable_simd_for_test();
1397                         err = check_shash_op("finup", err, driver, vec_name,
1398                                              cfg);
1399                         if (err)
1400                                 return err;
1401                         goto result_ready;
1402                 }
1403                 if (divs[i]->nosimd)
1404                         crypto_disable_simd_for_test();
1405                 err = crypto_shash_update(desc, sg_virt(&tsgl->sgl[i]),
1406                                           tsgl->sgl[i].length);
1407                 if (divs[i]->nosimd)
1408                         crypto_reenable_simd_for_test();
1409                 err = check_shash_op("update", err, driver, vec_name, cfg);
1410                 if (err)
1411                         return err;
1412                 if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1413                         /* Test ->export() and ->import() */
1414                         testmgr_poison(hashstate + statesize,
1415                                        TESTMGR_POISON_LEN);
1416                         err = crypto_shash_export(desc, hashstate);
1417                         err = check_shash_op("export", err, driver, vec_name,
1418                                              cfg);
1419                         if (err)
1420                                 return err;
1421                         if (!testmgr_is_poison(hashstate + statesize,
1422                                                TESTMGR_POISON_LEN)) {
1423                                 pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1424                                        driver, vec_name, cfg->name);
1425                                 return -EOVERFLOW;
1426                         }
1427                         testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1428                         err = crypto_shash_import(desc, hashstate);
1429                         err = check_shash_op("import", err, driver, vec_name,
1430                                              cfg);
1431                         if (err)
1432                                 return err;
1433                 }
1434         }
1435
1436         if (cfg->nosimd)
1437                 crypto_disable_simd_for_test();
1438         err = crypto_shash_final(desc, result);
1439         if (cfg->nosimd)
1440                 crypto_reenable_simd_for_test();
1441         err = check_shash_op("final", err, driver, vec_name, cfg);
1442         if (err)
1443                 return err;
1444 result_ready:
1445         return check_hash_result("shash", result, digestsize, vec, vec_name,
1446                                  driver, cfg);
1447 }
1448
1449 static int do_ahash_op(int (*op)(struct ahash_request *req),
1450                        struct ahash_request *req,
1451                        struct crypto_wait *wait, bool nosimd)
1452 {
1453         int err;
1454
1455         if (nosimd)
1456                 crypto_disable_simd_for_test();
1457
1458         err = op(req);
1459
1460         if (nosimd)
1461                 crypto_reenable_simd_for_test();
1462
1463         return crypto_wait_req(err, wait);
1464 }
1465
1466 static int check_nonfinal_ahash_op(const char *op, int err,
1467                                    u8 *result, unsigned int digestsize,
1468                                    const char *driver, const char *vec_name,
1469                                    const struct testvec_config *cfg)
1470 {
1471         if (err) {
1472                 pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1473                        driver, op, err, vec_name, cfg->name);
1474                 return err;
1475         }
1476         if (!testmgr_is_poison(result, digestsize)) {
1477                 pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
1478                        driver, op, vec_name, cfg->name);
1479                 return -EINVAL;
1480         }
1481         return 0;
1482 }
1483
1484 /* Test one hash test vector in one configuration, using the ahash API */
1485 static int test_ahash_vec_cfg(const struct hash_testvec *vec,
1486                               const char *vec_name,
1487                               const struct testvec_config *cfg,
1488                               struct ahash_request *req,
1489                               struct test_sglist *tsgl,
1490                               u8 *hashstate)
1491 {
1492         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1493         const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1494         const unsigned int statesize = crypto_ahash_statesize(tfm);
1495         const char *driver = crypto_ahash_driver_name(tfm);
1496         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1497         const struct test_sg_division *divs[XBUFSIZE];
1498         DECLARE_CRYPTO_WAIT(wait);
1499         unsigned int i;
1500         struct scatterlist *pending_sgl;
1501         unsigned int pending_len;
1502         u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1503         int err;
1504
1505         /* Set the key, if specified */
1506         if (vec->ksize) {
1507                 err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize,
1508                                 cfg, 0);
1509                 if (err) {
1510                         if (err == vec->setkey_error)
1511                                 return 0;
1512                         pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1513                                driver, vec_name, vec->setkey_error, err,
1514                                crypto_ahash_get_flags(tfm));
1515                         return err;
1516                 }
1517                 if (vec->setkey_error) {
1518                         pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1519                                driver, vec_name, vec->setkey_error);
1520                         return -EINVAL;
1521                 }
1522         }
1523
1524         /* Build the scatterlist for the source data */
1525         err = build_hash_sglist(tsgl, vec, cfg, 0, divs);
1526         if (err) {
1527                 pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1528                        driver, vec_name, cfg->name);
1529                 return err;
1530         }
1531
1532         /* Do the actual hashing */
1533
1534         testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1535         testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1536
1537         if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1538             vec->digest_error) {
1539                 /* Just using digest() */
1540                 ahash_request_set_callback(req, req_flags, crypto_req_done,
1541                                            &wait);
1542                 ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize);
1543                 err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd);
1544                 if (err) {
1545                         if (err == vec->digest_error)
1546                                 return 0;
1547                         pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1548                                driver, vec_name, vec->digest_error, err,
1549                                cfg->name);
1550                         return err;
1551                 }
1552                 if (vec->digest_error) {
1553                         pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1554                                driver, vec_name, vec->digest_error, cfg->name);
1555                         return -EINVAL;
1556                 }
1557                 goto result_ready;
1558         }
1559
1560         /* Using init(), zero or more update(), then final() or finup() */
1561
1562         ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1563         ahash_request_set_crypt(req, NULL, result, 0);
1564         err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
1565         err = check_nonfinal_ahash_op("init", err, result, digestsize,
1566                                       driver, vec_name, cfg);
1567         if (err)
1568                 return err;
1569
1570         pending_sgl = NULL;
1571         pending_len = 0;
1572         for (i = 0; i < tsgl->nents; i++) {
1573                 if (divs[i]->flush_type != FLUSH_TYPE_NONE &&
1574                     pending_sgl != NULL) {
1575                         /* update() with the pending data */
1576                         ahash_request_set_callback(req, req_flags,
1577                                                    crypto_req_done, &wait);
1578                         ahash_request_set_crypt(req, pending_sgl, result,
1579                                                 pending_len);
1580                         err = do_ahash_op(crypto_ahash_update, req, &wait,
1581                                           divs[i]->nosimd);
1582                         err = check_nonfinal_ahash_op("update", err,
1583                                                       result, digestsize,
1584                                                       driver, vec_name, cfg);
1585                         if (err)
1586                                 return err;
1587                         pending_sgl = NULL;
1588                         pending_len = 0;
1589                 }
1590                 if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1591                         /* Test ->export() and ->import() */
1592                         testmgr_poison(hashstate + statesize,
1593                                        TESTMGR_POISON_LEN);
1594                         err = crypto_ahash_export(req, hashstate);
1595                         err = check_nonfinal_ahash_op("export", err,
1596                                                       result, digestsize,
1597                                                       driver, vec_name, cfg);
1598                         if (err)
1599                                 return err;
1600                         if (!testmgr_is_poison(hashstate + statesize,
1601                                                TESTMGR_POISON_LEN)) {
1602                                 pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1603                                        driver, vec_name, cfg->name);
1604                                 return -EOVERFLOW;
1605                         }
1606
1607                         testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1608                         err = crypto_ahash_import(req, hashstate);
1609                         err = check_nonfinal_ahash_op("import", err,
1610                                                       result, digestsize,
1611                                                       driver, vec_name, cfg);
1612                         if (err)
1613                                 return err;
1614                 }
1615                 if (pending_sgl == NULL)
1616                         pending_sgl = &tsgl->sgl[i];
1617                 pending_len += tsgl->sgl[i].length;
1618         }
1619
1620         ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1621         ahash_request_set_crypt(req, pending_sgl, result, pending_len);
1622         if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
1623                 /* finish with update() and final() */
1624                 err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
1625                 err = check_nonfinal_ahash_op("update", err, result, digestsize,
1626                                               driver, vec_name, cfg);
1627                 if (err)
1628                         return err;
1629                 err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
1630                 if (err) {
1631                         pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
1632                                driver, err, vec_name, cfg->name);
1633                         return err;
1634                 }
1635         } else {
1636                 /* finish with finup() */
1637                 err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
1638                 if (err) {
1639                         pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
1640                                driver, err, vec_name, cfg->name);
1641                         return err;
1642                 }
1643         }
1644
1645 result_ready:
1646         return check_hash_result("ahash", result, digestsize, vec, vec_name,
1647                                  driver, cfg);
1648 }
1649
1650 static int test_hash_vec_cfg(const struct hash_testvec *vec,
1651                              const char *vec_name,
1652                              const struct testvec_config *cfg,
1653                              struct ahash_request *req,
1654                              struct shash_desc *desc,
1655                              struct test_sglist *tsgl,
1656                              u8 *hashstate)
1657 {
1658         int err;
1659
1660         /*
1661          * For algorithms implemented as "shash", most bugs will be detected by
1662          * both the shash and ahash tests.  Test the shash API first so that the
1663          * failures involve less indirection, so are easier to debug.
1664          */
1665
1666         if (desc) {
1667                 err = test_shash_vec_cfg(vec, vec_name, cfg, desc, tsgl,
1668                                          hashstate);
1669                 if (err)
1670                         return err;
1671         }
1672
1673         return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate);
1674 }
1675
1676 static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num,
1677                          struct ahash_request *req, struct shash_desc *desc,
1678                          struct test_sglist *tsgl, u8 *hashstate)
1679 {
1680         char vec_name[16];
1681         unsigned int i;
1682         int err;
1683
1684         sprintf(vec_name, "%u", vec_num);
1685
1686         for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
1687                 err = test_hash_vec_cfg(vec, vec_name,
1688                                         &default_hash_testvec_configs[i],
1689                                         req, desc, tsgl, hashstate);
1690                 if (err)
1691                         return err;
1692         }
1693
1694 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1695         if (!noextratests) {
1696                 struct rnd_state rng;
1697                 struct testvec_config cfg;
1698                 char cfgname[TESTVEC_CONFIG_NAMELEN];
1699
1700                 init_rnd_state(&rng);
1701
1702                 for (i = 0; i < fuzz_iterations; i++) {
1703                         generate_random_testvec_config(&rng, &cfg, cfgname,
1704                                                        sizeof(cfgname));
1705                         err = test_hash_vec_cfg(vec, vec_name, &cfg,
1706                                                 req, desc, tsgl, hashstate);
1707                         if (err)
1708                                 return err;
1709                         cond_resched();
1710                 }
1711         }
1712 #endif
1713         return 0;
1714 }
1715
1716 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1717 /*
1718  * Generate a hash test vector from the given implementation.
1719  * Assumes the buffers in 'vec' were already allocated.
1720  */
1721 static void generate_random_hash_testvec(struct rnd_state *rng,
1722                                          struct shash_desc *desc,
1723                                          struct hash_testvec *vec,
1724                                          unsigned int maxkeysize,
1725                                          unsigned int maxdatasize,
1726                                          char *name, size_t max_namelen)
1727 {
1728         /* Data */
1729         vec->psize = generate_random_length(rng, maxdatasize);
1730         generate_random_bytes(rng, (u8 *)vec->plaintext, vec->psize);
1731
1732         /*
1733          * Key: length in range [1, maxkeysize], but usually choose maxkeysize.
1734          * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.
1735          */
1736         vec->setkey_error = 0;
1737         vec->ksize = 0;
1738         if (maxkeysize) {
1739                 vec->ksize = maxkeysize;
1740                 if (prandom_u32_below(rng, 4) == 0)
1741                         vec->ksize = prandom_u32_inclusive(rng, 1, maxkeysize);
1742                 generate_random_bytes(rng, (u8 *)vec->key, vec->ksize);
1743
1744                 vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,
1745                                                         vec->ksize);
1746                 /* If the key couldn't be set, no need to continue to digest. */
1747                 if (vec->setkey_error)
1748                         goto done;
1749         }
1750
1751         /* Digest */
1752         vec->digest_error = crypto_shash_digest(desc, vec->plaintext,
1753                                                 vec->psize, (u8 *)vec->digest);
1754 done:
1755         snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"",
1756                  vec->psize, vec->ksize);
1757 }
1758
1759 /*
1760  * Test the hash algorithm represented by @req against the corresponding generic
1761  * implementation, if one is available.
1762  */
1763 static int test_hash_vs_generic_impl(const char *generic_driver,
1764                                      unsigned int maxkeysize,
1765                                      struct ahash_request *req,
1766                                      struct shash_desc *desc,
1767                                      struct test_sglist *tsgl,
1768                                      u8 *hashstate)
1769 {
1770         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1771         const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1772         const unsigned int blocksize = crypto_ahash_blocksize(tfm);
1773         const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
1774         const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
1775         const char *driver = crypto_ahash_driver_name(tfm);
1776         struct rnd_state rng;
1777         char _generic_driver[CRYPTO_MAX_ALG_NAME];
1778         struct crypto_shash *generic_tfm = NULL;
1779         struct shash_desc *generic_desc = NULL;
1780         unsigned int i;
1781         struct hash_testvec vec = { 0 };
1782         char vec_name[64];
1783         struct testvec_config *cfg;
1784         char cfgname[TESTVEC_CONFIG_NAMELEN];
1785         int err;
1786
1787         if (noextratests)
1788                 return 0;
1789
1790         init_rnd_state(&rng);
1791
1792         if (!generic_driver) { /* Use default naming convention? */
1793                 err = build_generic_driver_name(algname, _generic_driver);
1794                 if (err)
1795                         return err;
1796                 generic_driver = _generic_driver;
1797         }
1798
1799         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
1800                 return 0;
1801
1802         generic_tfm = crypto_alloc_shash(generic_driver, 0, 0);
1803         if (IS_ERR(generic_tfm)) {
1804                 err = PTR_ERR(generic_tfm);
1805                 if (err == -ENOENT) {
1806                         pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",
1807                                 driver, generic_driver);
1808                         return 0;
1809                 }
1810                 pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",
1811                        generic_driver, algname, err);
1812                 return err;
1813         }
1814
1815         cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1816         if (!cfg) {
1817                 err = -ENOMEM;
1818                 goto out;
1819         }
1820
1821         generic_desc = kzalloc(sizeof(*desc) +
1822                                crypto_shash_descsize(generic_tfm), GFP_KERNEL);
1823         if (!generic_desc) {
1824                 err = -ENOMEM;
1825                 goto out;
1826         }
1827         generic_desc->tfm = generic_tfm;
1828
1829         /* Check the algorithm properties for consistency. */
1830
1831         if (digestsize != crypto_shash_digestsize(generic_tfm)) {
1832                 pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",
1833                        driver, digestsize,
1834                        crypto_shash_digestsize(generic_tfm));
1835                 err = -EINVAL;
1836                 goto out;
1837         }
1838
1839         if (blocksize != crypto_shash_blocksize(generic_tfm)) {
1840                 pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",
1841                        driver, blocksize, crypto_shash_blocksize(generic_tfm));
1842                 err = -EINVAL;
1843                 goto out;
1844         }
1845
1846         /*
1847          * Now generate test vectors using the generic implementation, and test
1848          * the other implementation against them.
1849          */
1850
1851         vec.key = kmalloc(maxkeysize, GFP_KERNEL);
1852         vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);
1853         vec.digest = kmalloc(digestsize, GFP_KERNEL);
1854         if (!vec.key || !vec.plaintext || !vec.digest) {
1855                 err = -ENOMEM;
1856                 goto out;
1857         }
1858
1859         for (i = 0; i < fuzz_iterations * 8; i++) {
1860                 generate_random_hash_testvec(&rng, generic_desc, &vec,
1861                                              maxkeysize, maxdatasize,
1862                                              vec_name, sizeof(vec_name));
1863                 generate_random_testvec_config(&rng, cfg, cfgname,
1864                                                sizeof(cfgname));
1865
1866                 err = test_hash_vec_cfg(&vec, vec_name, cfg,
1867                                         req, desc, tsgl, hashstate);
1868                 if (err)
1869                         goto out;
1870                 cond_resched();
1871         }
1872         err = 0;
1873 out:
1874         kfree(cfg);
1875         kfree(vec.key);
1876         kfree(vec.plaintext);
1877         kfree(vec.digest);
1878         crypto_free_shash(generic_tfm);
1879         kfree_sensitive(generic_desc);
1880         return err;
1881 }
1882 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1883 static int test_hash_vs_generic_impl(const char *generic_driver,
1884                                      unsigned int maxkeysize,
1885                                      struct ahash_request *req,
1886                                      struct shash_desc *desc,
1887                                      struct test_sglist *tsgl,
1888                                      u8 *hashstate)
1889 {
1890         return 0;
1891 }
1892 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1893
1894 static int alloc_shash(const char *driver, u32 type, u32 mask,
1895                        struct crypto_shash **tfm_ret,
1896                        struct shash_desc **desc_ret)
1897 {
1898         struct crypto_shash *tfm;
1899         struct shash_desc *desc;
1900
1901         tfm = crypto_alloc_shash(driver, type, mask);
1902         if (IS_ERR(tfm)) {
1903                 if (PTR_ERR(tfm) == -ENOENT) {
1904                         /*
1905                          * This algorithm is only available through the ahash
1906                          * API, not the shash API, so skip the shash tests.
1907                          */
1908                         return 0;
1909                 }
1910                 pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
1911                        driver, PTR_ERR(tfm));
1912                 return PTR_ERR(tfm);
1913         }
1914
1915         desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
1916         if (!desc) {
1917                 crypto_free_shash(tfm);
1918                 return -ENOMEM;
1919         }
1920         desc->tfm = tfm;
1921
1922         *tfm_ret = tfm;
1923         *desc_ret = desc;
1924         return 0;
1925 }
1926
1927 static int __alg_test_hash(const struct hash_testvec *vecs,
1928                            unsigned int num_vecs, const char *driver,
1929                            u32 type, u32 mask,
1930                            const char *generic_driver, unsigned int maxkeysize)
1931 {
1932         struct crypto_ahash *atfm = NULL;
1933         struct ahash_request *req = NULL;
1934         struct crypto_shash *stfm = NULL;
1935         struct shash_desc *desc = NULL;
1936         struct test_sglist *tsgl = NULL;
1937         u8 *hashstate = NULL;
1938         unsigned int statesize;
1939         unsigned int i;
1940         int err;
1941
1942         /*
1943          * Always test the ahash API.  This works regardless of whether the
1944          * algorithm is implemented as ahash or shash.
1945          */
1946
1947         atfm = crypto_alloc_ahash(driver, type, mask);
1948         if (IS_ERR(atfm)) {
1949                 if (PTR_ERR(atfm) == -ENOENT)
1950                         return 0;
1951                 pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
1952                        driver, PTR_ERR(atfm));
1953                 return PTR_ERR(atfm);
1954         }
1955         driver = crypto_ahash_driver_name(atfm);
1956
1957         req = ahash_request_alloc(atfm, GFP_KERNEL);
1958         if (!req) {
1959                 pr_err("alg: hash: failed to allocate request for %s\n",
1960                        driver);
1961                 err = -ENOMEM;
1962                 goto out;
1963         }
1964
1965         /*
1966          * If available also test the shash API, to cover corner cases that may
1967          * be missed by testing the ahash API only.
1968          */
1969         err = alloc_shash(driver, type, mask, &stfm, &desc);
1970         if (err)
1971                 goto out;
1972
1973         tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
1974         if (!tsgl || init_test_sglist(tsgl) != 0) {
1975                 pr_err("alg: hash: failed to allocate test buffers for %s\n",
1976                        driver);
1977                 kfree(tsgl);
1978                 tsgl = NULL;
1979                 err = -ENOMEM;
1980                 goto out;
1981         }
1982
1983         statesize = crypto_ahash_statesize(atfm);
1984         if (stfm)
1985                 statesize = max(statesize, crypto_shash_statesize(stfm));
1986         hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
1987         if (!hashstate) {
1988                 pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
1989                        driver);
1990                 err = -ENOMEM;
1991                 goto out;
1992         }
1993
1994         for (i = 0; i < num_vecs; i++) {
1995                 if (fips_enabled && vecs[i].fips_skip)
1996                         continue;
1997
1998                 err = test_hash_vec(&vecs[i], i, req, desc, tsgl, hashstate);
1999                 if (err)
2000                         goto out;
2001                 cond_resched();
2002         }
2003         err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req,
2004                                         desc, tsgl, hashstate);
2005 out:
2006         kfree(hashstate);
2007         if (tsgl) {
2008                 destroy_test_sglist(tsgl);
2009                 kfree(tsgl);
2010         }
2011         kfree(desc);
2012         crypto_free_shash(stfm);
2013         ahash_request_free(req);
2014         crypto_free_ahash(atfm);
2015         return err;
2016 }
2017
2018 static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
2019                          u32 type, u32 mask)
2020 {
2021         const struct hash_testvec *template = desc->suite.hash.vecs;
2022         unsigned int tcount = desc->suite.hash.count;
2023         unsigned int nr_unkeyed, nr_keyed;
2024         unsigned int maxkeysize = 0;
2025         int err;
2026
2027         /*
2028          * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
2029          * first, before setting a key on the tfm.  To make this easier, we
2030          * require that the unkeyed test vectors (if any) are listed first.
2031          */
2032
2033         for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
2034                 if (template[nr_unkeyed].ksize)
2035                         break;
2036         }
2037         for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
2038                 if (!template[nr_unkeyed + nr_keyed].ksize) {
2039                         pr_err("alg: hash: test vectors for %s out of order, "
2040                                "unkeyed ones must come first\n", desc->alg);
2041                         return -EINVAL;
2042                 }
2043                 maxkeysize = max_t(unsigned int, maxkeysize,
2044                                    template[nr_unkeyed + nr_keyed].ksize);
2045         }
2046
2047         err = 0;
2048         if (nr_unkeyed) {
2049                 err = __alg_test_hash(template, nr_unkeyed, driver, type, mask,
2050                                       desc->generic_driver, maxkeysize);
2051                 template += nr_unkeyed;
2052         }
2053
2054         if (!err && nr_keyed)
2055                 err = __alg_test_hash(template, nr_keyed, driver, type, mask,
2056                                       desc->generic_driver, maxkeysize);
2057
2058         return err;
2059 }
2060
2061 static int test_aead_vec_cfg(int enc, const struct aead_testvec *vec,
2062                              const char *vec_name,
2063                              const struct testvec_config *cfg,
2064                              struct aead_request *req,
2065                              struct cipher_test_sglists *tsgls)
2066 {
2067         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2068         const unsigned int alignmask = crypto_aead_alignmask(tfm);
2069         const unsigned int ivsize = crypto_aead_ivsize(tfm);
2070         const unsigned int authsize = vec->clen - vec->plen;
2071         const char *driver = crypto_aead_driver_name(tfm);
2072         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2073         const char *op = enc ? "encryption" : "decryption";
2074         DECLARE_CRYPTO_WAIT(wait);
2075         u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2076         u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2077                  cfg->iv_offset +
2078                  (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2079         struct kvec input[2];
2080         int err;
2081
2082         /* Set the key */
2083         if (vec->wk)
2084                 crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2085         else
2086                 crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2087
2088         err = do_setkey(crypto_aead_setkey, tfm, vec->key, vec->klen,
2089                         cfg, alignmask);
2090         if (err && err != vec->setkey_error) {
2091                 pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2092                        driver, vec_name, vec->setkey_error, err,
2093                        crypto_aead_get_flags(tfm));
2094                 return err;
2095         }
2096         if (!err && vec->setkey_error) {
2097                 pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2098                        driver, vec_name, vec->setkey_error);
2099                 return -EINVAL;
2100         }
2101
2102         /* Set the authentication tag size */
2103         err = crypto_aead_setauthsize(tfm, authsize);
2104         if (err && err != vec->setauthsize_error) {
2105                 pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",
2106                        driver, vec_name, vec->setauthsize_error, err);
2107                 return err;
2108         }
2109         if (!err && vec->setauthsize_error) {
2110                 pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",
2111                        driver, vec_name, vec->setauthsize_error);
2112                 return -EINVAL;
2113         }
2114
2115         if (vec->setkey_error || vec->setauthsize_error)
2116                 return 0;
2117
2118         /* The IV must be copied to a buffer, as the algorithm may modify it */
2119         if (WARN_ON(ivsize > MAX_IVLEN))
2120                 return -EINVAL;
2121         if (vec->iv)
2122                 memcpy(iv, vec->iv, ivsize);
2123         else
2124                 memset(iv, 0, ivsize);
2125
2126         /* Build the src/dst scatterlists */
2127         input[0].iov_base = (void *)vec->assoc;
2128         input[0].iov_len = vec->alen;
2129         input[1].iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2130         input[1].iov_len = enc ? vec->plen : vec->clen;
2131         err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2132                                         vec->alen + (enc ? vec->plen :
2133                                                      vec->clen),
2134                                         vec->alen + (enc ? vec->clen :
2135                                                      vec->plen),
2136                                         input, 2);
2137         if (err) {
2138                 pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2139                        driver, op, vec_name, cfg->name);
2140                 return err;
2141         }
2142
2143         /* Do the actual encryption or decryption */
2144         testmgr_poison(req->__ctx, crypto_aead_reqsize(tfm));
2145         aead_request_set_callback(req, req_flags, crypto_req_done, &wait);
2146         aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2147                                enc ? vec->plen : vec->clen, iv);
2148         aead_request_set_ad(req, vec->alen);
2149         if (cfg->nosimd)
2150                 crypto_disable_simd_for_test();
2151         err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
2152         if (cfg->nosimd)
2153                 crypto_reenable_simd_for_test();
2154         err = crypto_wait_req(err, &wait);
2155
2156         /* Check that the algorithm didn't overwrite things it shouldn't have */
2157         if (req->cryptlen != (enc ? vec->plen : vec->clen) ||
2158             req->assoclen != vec->alen ||
2159             req->iv != iv ||
2160             req->src != tsgls->src.sgl_ptr ||
2161             req->dst != tsgls->dst.sgl_ptr ||
2162             crypto_aead_reqtfm(req) != tfm ||
2163             req->base.complete != crypto_req_done ||
2164             req->base.flags != req_flags ||
2165             req->base.data != &wait) {
2166                 pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2167                        driver, op, vec_name, cfg->name);
2168                 if (req->cryptlen != (enc ? vec->plen : vec->clen))
2169                         pr_err("alg: aead: changed 'req->cryptlen'\n");
2170                 if (req->assoclen != vec->alen)
2171                         pr_err("alg: aead: changed 'req->assoclen'\n");
2172                 if (req->iv != iv)
2173                         pr_err("alg: aead: changed 'req->iv'\n");
2174                 if (req->src != tsgls->src.sgl_ptr)
2175                         pr_err("alg: aead: changed 'req->src'\n");
2176                 if (req->dst != tsgls->dst.sgl_ptr)
2177                         pr_err("alg: aead: changed 'req->dst'\n");
2178                 if (crypto_aead_reqtfm(req) != tfm)
2179                         pr_err("alg: aead: changed 'req->base.tfm'\n");
2180                 if (req->base.complete != crypto_req_done)
2181                         pr_err("alg: aead: changed 'req->base.complete'\n");
2182                 if (req->base.flags != req_flags)
2183                         pr_err("alg: aead: changed 'req->base.flags'\n");
2184                 if (req->base.data != &wait)
2185                         pr_err("alg: aead: changed 'req->base.data'\n");
2186                 return -EINVAL;
2187         }
2188         if (is_test_sglist_corrupted(&tsgls->src)) {
2189                 pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2190                        driver, op, vec_name, cfg->name);
2191                 return -EINVAL;
2192         }
2193         if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2194             is_test_sglist_corrupted(&tsgls->dst)) {
2195                 pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2196                        driver, op, vec_name, cfg->name);
2197                 return -EINVAL;
2198         }
2199
2200         /* Check for unexpected success or failure, or wrong error code */
2201         if ((err == 0 && vec->novrfy) ||
2202             (err != vec->crypt_error && !(err == -EBADMSG && vec->novrfy))) {
2203                 char expected_error[32];
2204
2205                 if (vec->novrfy &&
2206                     vec->crypt_error != 0 && vec->crypt_error != -EBADMSG)
2207                         sprintf(expected_error, "-EBADMSG or %d",
2208                                 vec->crypt_error);
2209                 else if (vec->novrfy)
2210                         sprintf(expected_error, "-EBADMSG");
2211                 else
2212                         sprintf(expected_error, "%d", vec->crypt_error);
2213                 if (err) {
2214                         pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%s, actual_error=%d, cfg=\"%s\"\n",
2215                                driver, op, vec_name, expected_error, err,
2216                                cfg->name);
2217                         return err;
2218                 }
2219                 pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%s, cfg=\"%s\"\n",
2220                        driver, op, vec_name, expected_error, cfg->name);
2221                 return -EINVAL;
2222         }
2223         if (err) /* Expectedly failed. */
2224                 return 0;
2225
2226         /* Check for the correct output (ciphertext or plaintext) */
2227         err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2228                                     enc ? vec->clen : vec->plen,
2229                                     vec->alen,
2230                                     enc || cfg->inplace_mode == OUT_OF_PLACE);
2231         if (err == -EOVERFLOW) {
2232                 pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2233                        driver, op, vec_name, cfg->name);
2234                 return err;
2235         }
2236         if (err) {
2237                 pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2238                        driver, op, vec_name, cfg->name);
2239                 return err;
2240         }
2241
2242         return 0;
2243 }
2244
2245 static int test_aead_vec(int enc, const struct aead_testvec *vec,
2246                          unsigned int vec_num, struct aead_request *req,
2247                          struct cipher_test_sglists *tsgls)
2248 {
2249         char vec_name[16];
2250         unsigned int i;
2251         int err;
2252
2253         if (enc && vec->novrfy)
2254                 return 0;
2255
2256         sprintf(vec_name, "%u", vec_num);
2257
2258         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2259                 err = test_aead_vec_cfg(enc, vec, vec_name,
2260                                         &default_cipher_testvec_configs[i],
2261                                         req, tsgls);
2262                 if (err)
2263                         return err;
2264         }
2265
2266 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2267         if (!noextratests) {
2268                 struct rnd_state rng;
2269                 struct testvec_config cfg;
2270                 char cfgname[TESTVEC_CONFIG_NAMELEN];
2271
2272                 init_rnd_state(&rng);
2273
2274                 for (i = 0; i < fuzz_iterations; i++) {
2275                         generate_random_testvec_config(&rng, &cfg, cfgname,
2276                                                        sizeof(cfgname));
2277                         err = test_aead_vec_cfg(enc, vec, vec_name,
2278                                                 &cfg, req, tsgls);
2279                         if (err)
2280                                 return err;
2281                         cond_resched();
2282                 }
2283         }
2284 #endif
2285         return 0;
2286 }
2287
2288 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2289
2290 struct aead_extra_tests_ctx {
2291         struct rnd_state rng;
2292         struct aead_request *req;
2293         struct crypto_aead *tfm;
2294         const struct alg_test_desc *test_desc;
2295         struct cipher_test_sglists *tsgls;
2296         unsigned int maxdatasize;
2297         unsigned int maxkeysize;
2298
2299         struct aead_testvec vec;
2300         char vec_name[64];
2301         char cfgname[TESTVEC_CONFIG_NAMELEN];
2302         struct testvec_config cfg;
2303 };
2304
2305 /*
2306  * Make at least one random change to a (ciphertext, AAD) pair.  "Ciphertext"
2307  * here means the full ciphertext including the authentication tag.  The
2308  * authentication tag (and hence also the ciphertext) is assumed to be nonempty.
2309  */
2310 static void mutate_aead_message(struct rnd_state *rng,
2311                                 struct aead_testvec *vec, bool aad_iv,
2312                                 unsigned int ivsize)
2313 {
2314         const unsigned int aad_tail_size = aad_iv ? ivsize : 0;
2315         const unsigned int authsize = vec->clen - vec->plen;
2316
2317         if (prandom_bool(rng) && vec->alen > aad_tail_size) {
2318                  /* Mutate the AAD */
2319                 flip_random_bit(rng, (u8 *)vec->assoc,
2320                                 vec->alen - aad_tail_size);
2321                 if (prandom_bool(rng))
2322                         return;
2323         }
2324         if (prandom_bool(rng)) {
2325                 /* Mutate auth tag (assuming it's at the end of ciphertext) */
2326                 flip_random_bit(rng, (u8 *)vec->ctext + vec->plen, authsize);
2327         } else {
2328                 /* Mutate any part of the ciphertext */
2329                 flip_random_bit(rng, (u8 *)vec->ctext, vec->clen);
2330         }
2331 }
2332
2333 /*
2334  * Minimum authentication tag size in bytes at which we assume that we can
2335  * reliably generate inauthentic messages, i.e. not generate an authentic
2336  * message by chance.
2337  */
2338 #define MIN_COLLISION_FREE_AUTHSIZE 8
2339
2340 static void generate_aead_message(struct rnd_state *rng,
2341                                   struct aead_request *req,
2342                                   const struct aead_test_suite *suite,
2343                                   struct aead_testvec *vec,
2344                                   bool prefer_inauthentic)
2345 {
2346         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2347         const unsigned int ivsize = crypto_aead_ivsize(tfm);
2348         const unsigned int authsize = vec->clen - vec->plen;
2349         const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&
2350                                  (prefer_inauthentic ||
2351                                   prandom_u32_below(rng, 4) == 0);
2352
2353         /* Generate the AAD. */
2354         generate_random_bytes(rng, (u8 *)vec->assoc, vec->alen);
2355         if (suite->aad_iv && vec->alen >= ivsize)
2356                 /* Avoid implementation-defined behavior. */
2357                 memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);
2358
2359         if (inauthentic && prandom_bool(rng)) {
2360                 /* Generate a random ciphertext. */
2361                 generate_random_bytes(rng, (u8 *)vec->ctext, vec->clen);
2362         } else {
2363                 int i = 0;
2364                 struct scatterlist src[2], dst;
2365                 u8 iv[MAX_IVLEN];
2366                 DECLARE_CRYPTO_WAIT(wait);
2367
2368                 /* Generate a random plaintext and encrypt it. */
2369                 sg_init_table(src, 2);
2370                 if (vec->alen)
2371                         sg_set_buf(&src[i++], vec->assoc, vec->alen);
2372                 if (vec->plen) {
2373                         generate_random_bytes(rng, (u8 *)vec->ptext, vec->plen);
2374                         sg_set_buf(&src[i++], vec->ptext, vec->plen);
2375                 }
2376                 sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);
2377                 memcpy(iv, vec->iv, ivsize);
2378                 aead_request_set_callback(req, 0, crypto_req_done, &wait);
2379                 aead_request_set_crypt(req, src, &dst, vec->plen, iv);
2380                 aead_request_set_ad(req, vec->alen);
2381                 vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req),
2382                                                    &wait);
2383                 /* If encryption failed, we're done. */
2384                 if (vec->crypt_error != 0)
2385                         return;
2386                 memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);
2387                 if (!inauthentic)
2388                         return;
2389                 /*
2390                  * Mutate the authentic (ciphertext, AAD) pair to get an
2391                  * inauthentic one.
2392                  */
2393                 mutate_aead_message(rng, vec, suite->aad_iv, ivsize);
2394         }
2395         vec->novrfy = 1;
2396         if (suite->einval_allowed)
2397                 vec->crypt_error = -EINVAL;
2398 }
2399
2400 /*
2401  * Generate an AEAD test vector 'vec' using the implementation specified by
2402  * 'req'.  The buffers in 'vec' must already be allocated.
2403  *
2404  * If 'prefer_inauthentic' is true, then this function will generate inauthentic
2405  * test vectors (i.e. vectors with 'vec->novrfy=1') more often.
2406  */
2407 static void generate_random_aead_testvec(struct rnd_state *rng,
2408                                          struct aead_request *req,
2409                                          struct aead_testvec *vec,
2410                                          const struct aead_test_suite *suite,
2411                                          unsigned int maxkeysize,
2412                                          unsigned int maxdatasize,
2413                                          char *name, size_t max_namelen,
2414                                          bool prefer_inauthentic)
2415 {
2416         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2417         const unsigned int ivsize = crypto_aead_ivsize(tfm);
2418         const unsigned int maxauthsize = crypto_aead_maxauthsize(tfm);
2419         unsigned int authsize;
2420         unsigned int total_len;
2421
2422         /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2423         vec->klen = maxkeysize;
2424         if (prandom_u32_below(rng, 4) == 0)
2425                 vec->klen = prandom_u32_below(rng, maxkeysize + 1);
2426         generate_random_bytes(rng, (u8 *)vec->key, vec->klen);
2427         vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);
2428
2429         /* IV */
2430         generate_random_bytes(rng, (u8 *)vec->iv, ivsize);
2431
2432         /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
2433         authsize = maxauthsize;
2434         if (prandom_u32_below(rng, 4) == 0)
2435                 authsize = prandom_u32_below(rng, maxauthsize + 1);
2436         if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)
2437                 authsize = MIN_COLLISION_FREE_AUTHSIZE;
2438         if (WARN_ON(authsize > maxdatasize))
2439                 authsize = maxdatasize;
2440         maxdatasize -= authsize;
2441         vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);
2442
2443         /* AAD, plaintext, and ciphertext lengths */
2444         total_len = generate_random_length(rng, maxdatasize);
2445         if (prandom_u32_below(rng, 4) == 0)
2446                 vec->alen = 0;
2447         else
2448                 vec->alen = generate_random_length(rng, total_len);
2449         vec->plen = total_len - vec->alen;
2450         vec->clen = vec->plen + authsize;
2451
2452         /*
2453          * Generate the AAD, plaintext, and ciphertext.  Not applicable if the
2454          * key or the authentication tag size couldn't be set.
2455          */
2456         vec->novrfy = 0;
2457         vec->crypt_error = 0;
2458         if (vec->setkey_error == 0 && vec->setauthsize_error == 0)
2459                 generate_aead_message(rng, req, suite, vec, prefer_inauthentic);
2460         snprintf(name, max_namelen,
2461                  "\"random: alen=%u plen=%u authsize=%u klen=%u novrfy=%d\"",
2462                  vec->alen, vec->plen, authsize, vec->klen, vec->novrfy);
2463 }
2464
2465 static void try_to_generate_inauthentic_testvec(
2466                                         struct aead_extra_tests_ctx *ctx)
2467 {
2468         int i;
2469
2470         for (i = 0; i < 10; i++) {
2471                 generate_random_aead_testvec(&ctx->rng, ctx->req, &ctx->vec,
2472                                              &ctx->test_desc->suite.aead,
2473                                              ctx->maxkeysize, ctx->maxdatasize,
2474                                              ctx->vec_name,
2475                                              sizeof(ctx->vec_name), true);
2476                 if (ctx->vec.novrfy)
2477                         return;
2478         }
2479 }
2480
2481 /*
2482  * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the
2483  * result of an encryption with the key) and verify that decryption fails.
2484  */
2485 static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx)
2486 {
2487         unsigned int i;
2488         int err;
2489
2490         for (i = 0; i < fuzz_iterations * 8; i++) {
2491                 /*
2492                  * Since this part of the tests isn't comparing the
2493                  * implementation to another, there's no point in testing any
2494                  * test vectors other than inauthentic ones (vec.novrfy=1) here.
2495                  *
2496                  * If we're having trouble generating such a test vector, e.g.
2497                  * if the algorithm keeps rejecting the generated keys, don't
2498                  * retry forever; just continue on.
2499                  */
2500                 try_to_generate_inauthentic_testvec(ctx);
2501                 if (ctx->vec.novrfy) {
2502                         generate_random_testvec_config(&ctx->rng, &ctx->cfg,
2503                                                        ctx->cfgname,
2504                                                        sizeof(ctx->cfgname));
2505                         err = test_aead_vec_cfg(DECRYPT, &ctx->vec,
2506                                                 ctx->vec_name, &ctx->cfg,
2507                                                 ctx->req, ctx->tsgls);
2508                         if (err)
2509                                 return err;
2510                 }
2511                 cond_resched();
2512         }
2513         return 0;
2514 }
2515
2516 /*
2517  * Test the AEAD algorithm against the corresponding generic implementation, if
2518  * one is available.
2519  */
2520 static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx)
2521 {
2522         struct crypto_aead *tfm = ctx->tfm;
2523         const char *algname = crypto_aead_alg(tfm)->base.cra_name;
2524         const char *driver = crypto_aead_driver_name(tfm);
2525         const char *generic_driver = ctx->test_desc->generic_driver;
2526         char _generic_driver[CRYPTO_MAX_ALG_NAME];
2527         struct crypto_aead *generic_tfm = NULL;
2528         struct aead_request *generic_req = NULL;
2529         unsigned int i;
2530         int err;
2531
2532         if (!generic_driver) { /* Use default naming convention? */
2533                 err = build_generic_driver_name(algname, _generic_driver);
2534                 if (err)
2535                         return err;
2536                 generic_driver = _generic_driver;
2537         }
2538
2539         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2540                 return 0;
2541
2542         generic_tfm = crypto_alloc_aead(generic_driver, 0, 0);
2543         if (IS_ERR(generic_tfm)) {
2544                 err = PTR_ERR(generic_tfm);
2545                 if (err == -ENOENT) {
2546                         pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",
2547                                 driver, generic_driver);
2548                         return 0;
2549                 }
2550                 pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",
2551                        generic_driver, algname, err);
2552                 return err;
2553         }
2554
2555         generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
2556         if (!generic_req) {
2557                 err = -ENOMEM;
2558                 goto out;
2559         }
2560
2561         /* Check the algorithm properties for consistency. */
2562
2563         if (crypto_aead_maxauthsize(tfm) !=
2564             crypto_aead_maxauthsize(generic_tfm)) {
2565                 pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",
2566                        driver, crypto_aead_maxauthsize(tfm),
2567                        crypto_aead_maxauthsize(generic_tfm));
2568                 err = -EINVAL;
2569                 goto out;
2570         }
2571
2572         if (crypto_aead_ivsize(tfm) != crypto_aead_ivsize(generic_tfm)) {
2573                 pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",
2574                        driver, crypto_aead_ivsize(tfm),
2575                        crypto_aead_ivsize(generic_tfm));
2576                 err = -EINVAL;
2577                 goto out;
2578         }
2579
2580         if (crypto_aead_blocksize(tfm) != crypto_aead_blocksize(generic_tfm)) {
2581                 pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",
2582                        driver, crypto_aead_blocksize(tfm),
2583                        crypto_aead_blocksize(generic_tfm));
2584                 err = -EINVAL;
2585                 goto out;
2586         }
2587
2588         /*
2589          * Now generate test vectors using the generic implementation, and test
2590          * the other implementation against them.
2591          */
2592         for (i = 0; i < fuzz_iterations * 8; i++) {
2593                 generate_random_aead_testvec(&ctx->rng, generic_req, &ctx->vec,
2594                                              &ctx->test_desc->suite.aead,
2595                                              ctx->maxkeysize, ctx->maxdatasize,
2596                                              ctx->vec_name,
2597                                              sizeof(ctx->vec_name), false);
2598                 generate_random_testvec_config(&ctx->rng, &ctx->cfg,
2599                                                ctx->cfgname,
2600                                                sizeof(ctx->cfgname));
2601                 if (!ctx->vec.novrfy) {
2602                         err = test_aead_vec_cfg(ENCRYPT, &ctx->vec,
2603                                                 ctx->vec_name, &ctx->cfg,
2604                                                 ctx->req, ctx->tsgls);
2605                         if (err)
2606                                 goto out;
2607                 }
2608                 if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) {
2609                         err = test_aead_vec_cfg(DECRYPT, &ctx->vec,
2610                                                 ctx->vec_name, &ctx->cfg,
2611                                                 ctx->req, ctx->tsgls);
2612                         if (err)
2613                                 goto out;
2614                 }
2615                 cond_resched();
2616         }
2617         err = 0;
2618 out:
2619         crypto_free_aead(generic_tfm);
2620         aead_request_free(generic_req);
2621         return err;
2622 }
2623
2624 static int test_aead_extra(const struct alg_test_desc *test_desc,
2625                            struct aead_request *req,
2626                            struct cipher_test_sglists *tsgls)
2627 {
2628         struct aead_extra_tests_ctx *ctx;
2629         unsigned int i;
2630         int err;
2631
2632         if (noextratests)
2633                 return 0;
2634
2635         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2636         if (!ctx)
2637                 return -ENOMEM;
2638         init_rnd_state(&ctx->rng);
2639         ctx->req = req;
2640         ctx->tfm = crypto_aead_reqtfm(req);
2641         ctx->test_desc = test_desc;
2642         ctx->tsgls = tsgls;
2643         ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2644         ctx->maxkeysize = 0;
2645         for (i = 0; i < test_desc->suite.aead.count; i++)
2646                 ctx->maxkeysize = max_t(unsigned int, ctx->maxkeysize,
2647                                         test_desc->suite.aead.vecs[i].klen);
2648
2649         ctx->vec.key = kmalloc(ctx->maxkeysize, GFP_KERNEL);
2650         ctx->vec.iv = kmalloc(crypto_aead_ivsize(ctx->tfm), GFP_KERNEL);
2651         ctx->vec.assoc = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2652         ctx->vec.ptext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2653         ctx->vec.ctext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2654         if (!ctx->vec.key || !ctx->vec.iv || !ctx->vec.assoc ||
2655             !ctx->vec.ptext || !ctx->vec.ctext) {
2656                 err = -ENOMEM;
2657                 goto out;
2658         }
2659
2660         err = test_aead_vs_generic_impl(ctx);
2661         if (err)
2662                 goto out;
2663
2664         err = test_aead_inauthentic_inputs(ctx);
2665 out:
2666         kfree(ctx->vec.key);
2667         kfree(ctx->vec.iv);
2668         kfree(ctx->vec.assoc);
2669         kfree(ctx->vec.ptext);
2670         kfree(ctx->vec.ctext);
2671         kfree(ctx);
2672         return err;
2673 }
2674 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2675 static int test_aead_extra(const struct alg_test_desc *test_desc,
2676                            struct aead_request *req,
2677                            struct cipher_test_sglists *tsgls)
2678 {
2679         return 0;
2680 }
2681 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2682
2683 static int test_aead(int enc, const struct aead_test_suite *suite,
2684                      struct aead_request *req,
2685                      struct cipher_test_sglists *tsgls)
2686 {
2687         unsigned int i;
2688         int err;
2689
2690         for (i = 0; i < suite->count; i++) {
2691                 err = test_aead_vec(enc, &suite->vecs[i], i, req, tsgls);
2692                 if (err)
2693                         return err;
2694                 cond_resched();
2695         }
2696         return 0;
2697 }
2698
2699 static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
2700                          u32 type, u32 mask)
2701 {
2702         const struct aead_test_suite *suite = &desc->suite.aead;
2703         struct crypto_aead *tfm;
2704         struct aead_request *req = NULL;
2705         struct cipher_test_sglists *tsgls = NULL;
2706         int err;
2707
2708         if (suite->count <= 0) {
2709                 pr_err("alg: aead: empty test suite for %s\n", driver);
2710                 return -EINVAL;
2711         }
2712
2713         tfm = crypto_alloc_aead(driver, type, mask);
2714         if (IS_ERR(tfm)) {
2715                 if (PTR_ERR(tfm) == -ENOENT)
2716                         return 0;
2717                 pr_err("alg: aead: failed to allocate transform for %s: %ld\n",
2718                        driver, PTR_ERR(tfm));
2719                 return PTR_ERR(tfm);
2720         }
2721         driver = crypto_aead_driver_name(tfm);
2722
2723         req = aead_request_alloc(tfm, GFP_KERNEL);
2724         if (!req) {
2725                 pr_err("alg: aead: failed to allocate request for %s\n",
2726                        driver);
2727                 err = -ENOMEM;
2728                 goto out;
2729         }
2730
2731         tsgls = alloc_cipher_test_sglists();
2732         if (!tsgls) {
2733                 pr_err("alg: aead: failed to allocate test buffers for %s\n",
2734                        driver);
2735                 err = -ENOMEM;
2736                 goto out;
2737         }
2738
2739         err = test_aead(ENCRYPT, suite, req, tsgls);
2740         if (err)
2741                 goto out;
2742
2743         err = test_aead(DECRYPT, suite, req, tsgls);
2744         if (err)
2745                 goto out;
2746
2747         err = test_aead_extra(desc, req, tsgls);
2748 out:
2749         free_cipher_test_sglists(tsgls);
2750         aead_request_free(req);
2751         crypto_free_aead(tfm);
2752         return err;
2753 }
2754
2755 static int test_cipher(struct crypto_cipher *tfm, int enc,
2756                        const struct cipher_testvec *template,
2757                        unsigned int tcount)
2758 {
2759         const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm));
2760         unsigned int i, j, k;
2761         char *q;
2762         const char *e;
2763         const char *input, *result;
2764         void *data;
2765         char *xbuf[XBUFSIZE];
2766         int ret = -ENOMEM;
2767
2768         if (testmgr_alloc_buf(xbuf))
2769                 goto out_nobuf;
2770
2771         if (enc == ENCRYPT)
2772                 e = "encryption";
2773         else
2774                 e = "decryption";
2775
2776         j = 0;
2777         for (i = 0; i < tcount; i++) {
2778
2779                 if (fips_enabled && template[i].fips_skip)
2780                         continue;
2781
2782                 input  = enc ? template[i].ptext : template[i].ctext;
2783                 result = enc ? template[i].ctext : template[i].ptext;
2784                 j++;
2785
2786                 ret = -EINVAL;
2787                 if (WARN_ON(template[i].len > PAGE_SIZE))
2788                         goto out;
2789
2790                 data = xbuf[0];
2791                 memcpy(data, input, template[i].len);
2792
2793                 crypto_cipher_clear_flags(tfm, ~0);
2794                 if (template[i].wk)
2795                         crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2796
2797                 ret = crypto_cipher_setkey(tfm, template[i].key,
2798                                            template[i].klen);
2799                 if (ret) {
2800                         if (ret == template[i].setkey_error)
2801                                 continue;
2802                         pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",
2803                                algo, j, template[i].setkey_error, ret,
2804                                crypto_cipher_get_flags(tfm));
2805                         goto out;
2806                 }
2807                 if (template[i].setkey_error) {
2808                         pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",
2809                                algo, j, template[i].setkey_error);
2810                         ret = -EINVAL;
2811                         goto out;
2812                 }
2813
2814                 for (k = 0; k < template[i].len;
2815                      k += crypto_cipher_blocksize(tfm)) {
2816                         if (enc)
2817                                 crypto_cipher_encrypt_one(tfm, data + k,
2818                                                           data + k);
2819                         else
2820                                 crypto_cipher_decrypt_one(tfm, data + k,
2821                                                           data + k);
2822                 }
2823
2824                 q = data;
2825                 if (memcmp(q, result, template[i].len)) {
2826                         printk(KERN_ERR "alg: cipher: Test %d failed "
2827                                "on %s for %s\n", j, e, algo);
2828                         hexdump(q, template[i].len);
2829                         ret = -EINVAL;
2830                         goto out;
2831                 }
2832         }
2833
2834         ret = 0;
2835
2836 out:
2837         testmgr_free_buf(xbuf);
2838 out_nobuf:
2839         return ret;
2840 }
2841
2842 static int test_skcipher_vec_cfg(int enc, const struct cipher_testvec *vec,
2843                                  const char *vec_name,
2844                                  const struct testvec_config *cfg,
2845                                  struct skcipher_request *req,
2846                                  struct cipher_test_sglists *tsgls)
2847 {
2848         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2849         const unsigned int alignmask = crypto_skcipher_alignmask(tfm);
2850         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2851         const char *driver = crypto_skcipher_driver_name(tfm);
2852         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2853         const char *op = enc ? "encryption" : "decryption";
2854         DECLARE_CRYPTO_WAIT(wait);
2855         u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2856         u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2857                  cfg->iv_offset +
2858                  (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2859         struct kvec input;
2860         int err;
2861
2862         /* Set the key */
2863         if (vec->wk)
2864                 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2865         else
2866                 crypto_skcipher_clear_flags(tfm,
2867                                             CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2868         err = do_setkey(crypto_skcipher_setkey, tfm, vec->key, vec->klen,
2869                         cfg, alignmask);
2870         if (err) {
2871                 if (err == vec->setkey_error)
2872                         return 0;
2873                 pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2874                        driver, vec_name, vec->setkey_error, err,
2875                        crypto_skcipher_get_flags(tfm));
2876                 return err;
2877         }
2878         if (vec->setkey_error) {
2879                 pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2880                        driver, vec_name, vec->setkey_error);
2881                 return -EINVAL;
2882         }
2883
2884         /* The IV must be copied to a buffer, as the algorithm may modify it */
2885         if (ivsize) {
2886                 if (WARN_ON(ivsize > MAX_IVLEN))
2887                         return -EINVAL;
2888                 if (vec->generates_iv && !enc)
2889                         memcpy(iv, vec->iv_out, ivsize);
2890                 else if (vec->iv)
2891                         memcpy(iv, vec->iv, ivsize);
2892                 else
2893                         memset(iv, 0, ivsize);
2894         } else {
2895                 if (vec->generates_iv) {
2896                         pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n",
2897                                driver, vec_name);
2898                         return -EINVAL;
2899                 }
2900                 iv = NULL;
2901         }
2902
2903         /* Build the src/dst scatterlists */
2904         input.iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2905         input.iov_len = vec->len;
2906         err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2907                                         vec->len, vec->len, &input, 1);
2908         if (err) {
2909                 pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2910                        driver, op, vec_name, cfg->name);
2911                 return err;
2912         }
2913
2914         /* Do the actual encryption or decryption */
2915         testmgr_poison(req->__ctx, crypto_skcipher_reqsize(tfm));
2916         skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait);
2917         skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2918                                    vec->len, iv);
2919         if (cfg->nosimd)
2920                 crypto_disable_simd_for_test();
2921         err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
2922         if (cfg->nosimd)
2923                 crypto_reenable_simd_for_test();
2924         err = crypto_wait_req(err, &wait);
2925
2926         /* Check that the algorithm didn't overwrite things it shouldn't have */
2927         if (req->cryptlen != vec->len ||
2928             req->iv != iv ||
2929             req->src != tsgls->src.sgl_ptr ||
2930             req->dst != tsgls->dst.sgl_ptr ||
2931             crypto_skcipher_reqtfm(req) != tfm ||
2932             req->base.complete != crypto_req_done ||
2933             req->base.flags != req_flags ||
2934             req->base.data != &wait) {
2935                 pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2936                        driver, op, vec_name, cfg->name);
2937                 if (req->cryptlen != vec->len)
2938                         pr_err("alg: skcipher: changed 'req->cryptlen'\n");
2939                 if (req->iv != iv)
2940                         pr_err("alg: skcipher: changed 'req->iv'\n");
2941                 if (req->src != tsgls->src.sgl_ptr)
2942                         pr_err("alg: skcipher: changed 'req->src'\n");
2943                 if (req->dst != tsgls->dst.sgl_ptr)
2944                         pr_err("alg: skcipher: changed 'req->dst'\n");
2945                 if (crypto_skcipher_reqtfm(req) != tfm)
2946                         pr_err("alg: skcipher: changed 'req->base.tfm'\n");
2947                 if (req->base.complete != crypto_req_done)
2948                         pr_err("alg: skcipher: changed 'req->base.complete'\n");
2949                 if (req->base.flags != req_flags)
2950                         pr_err("alg: skcipher: changed 'req->base.flags'\n");
2951                 if (req->base.data != &wait)
2952                         pr_err("alg: skcipher: changed 'req->base.data'\n");
2953                 return -EINVAL;
2954         }
2955         if (is_test_sglist_corrupted(&tsgls->src)) {
2956                 pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2957                        driver, op, vec_name, cfg->name);
2958                 return -EINVAL;
2959         }
2960         if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2961             is_test_sglist_corrupted(&tsgls->dst)) {
2962                 pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2963                        driver, op, vec_name, cfg->name);
2964                 return -EINVAL;
2965         }
2966
2967         /* Check for success or failure */
2968         if (err) {
2969                 if (err == vec->crypt_error)
2970                         return 0;
2971                 pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
2972                        driver, op, vec_name, vec->crypt_error, err, cfg->name);
2973                 return err;
2974         }
2975         if (vec->crypt_error) {
2976                 pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
2977                        driver, op, vec_name, vec->crypt_error, cfg->name);
2978                 return -EINVAL;
2979         }
2980
2981         /* Check for the correct output (ciphertext or plaintext) */
2982         err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2983                                     vec->len, 0, true);
2984         if (err == -EOVERFLOW) {
2985                 pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2986                        driver, op, vec_name, cfg->name);
2987                 return err;
2988         }
2989         if (err) {
2990                 pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2991                        driver, op, vec_name, cfg->name);
2992                 return err;
2993         }
2994
2995         /* If applicable, check that the algorithm generated the correct IV */
2996         if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) {
2997                 pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",
2998                        driver, op, vec_name, cfg->name);
2999                 hexdump(iv, ivsize);
3000                 return -EINVAL;
3001         }
3002
3003         return 0;
3004 }
3005
3006 static int test_skcipher_vec(int enc, const struct cipher_testvec *vec,
3007                              unsigned int vec_num,
3008                              struct skcipher_request *req,
3009                              struct cipher_test_sglists *tsgls)
3010 {
3011         char vec_name[16];
3012         unsigned int i;
3013         int err;
3014
3015         if (fips_enabled && vec->fips_skip)
3016                 return 0;
3017
3018         sprintf(vec_name, "%u", vec_num);
3019
3020         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
3021                 err = test_skcipher_vec_cfg(enc, vec, vec_name,
3022                                             &default_cipher_testvec_configs[i],
3023                                             req, tsgls);
3024                 if (err)
3025                         return err;
3026         }
3027
3028 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
3029         if (!noextratests) {
3030                 struct rnd_state rng;
3031                 struct testvec_config cfg;
3032                 char cfgname[TESTVEC_CONFIG_NAMELEN];
3033
3034                 init_rnd_state(&rng);
3035
3036                 for (i = 0; i < fuzz_iterations; i++) {
3037                         generate_random_testvec_config(&rng, &cfg, cfgname,
3038                                                        sizeof(cfgname));
3039                         err = test_skcipher_vec_cfg(enc, vec, vec_name,
3040                                                     &cfg, req, tsgls);
3041                         if (err)
3042                                 return err;
3043                         cond_resched();
3044                 }
3045         }
3046 #endif
3047         return 0;
3048 }
3049
3050 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
3051 /*
3052  * Generate a symmetric cipher test vector from the given implementation.
3053  * Assumes the buffers in 'vec' were already allocated.
3054  */
3055 static void generate_random_cipher_testvec(struct rnd_state *rng,
3056                                            struct skcipher_request *req,
3057                                            struct cipher_testvec *vec,
3058                                            unsigned int maxdatasize,
3059                                            char *name, size_t max_namelen)
3060 {
3061         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3062         const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
3063         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
3064         struct scatterlist src, dst;
3065         u8 iv[MAX_IVLEN];
3066         DECLARE_CRYPTO_WAIT(wait);
3067
3068         /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
3069         vec->klen = maxkeysize;
3070         if (prandom_u32_below(rng, 4) == 0)
3071                 vec->klen = prandom_u32_below(rng, maxkeysize + 1);
3072         generate_random_bytes(rng, (u8 *)vec->key, vec->klen);
3073         vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
3074
3075         /* IV */
3076         generate_random_bytes(rng, (u8 *)vec->iv, ivsize);
3077
3078         /* Plaintext */
3079         vec->len = generate_random_length(rng, maxdatasize);
3080         generate_random_bytes(rng, (u8 *)vec->ptext, vec->len);
3081
3082         /* If the key couldn't be set, no need to continue to encrypt. */
3083         if (vec->setkey_error)
3084                 goto done;
3085
3086         /* Ciphertext */
3087         sg_init_one(&src, vec->ptext, vec->len);
3088         sg_init_one(&dst, vec->ctext, vec->len);
3089         memcpy(iv, vec->iv, ivsize);
3090         skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
3091         skcipher_request_set_crypt(req, &src, &dst, vec->len, iv);
3092         vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
3093         if (vec->crypt_error != 0) {
3094                 /*
3095                  * The only acceptable error here is for an invalid length, so
3096                  * skcipher decryption should fail with the same error too.
3097                  * We'll test for this.  But to keep the API usage well-defined,
3098                  * explicitly initialize the ciphertext buffer too.
3099                  */
3100                 memset((u8 *)vec->ctext, 0, vec->len);
3101         }
3102 done:
3103         snprintf(name, max_namelen, "\"random: len=%u klen=%u\"",
3104                  vec->len, vec->klen);
3105 }
3106
3107 /*
3108  * Test the skcipher algorithm represented by @req against the corresponding
3109  * generic implementation, if one is available.
3110  */
3111 static int test_skcipher_vs_generic_impl(const char *generic_driver,
3112                                          struct skcipher_request *req,
3113                                          struct cipher_test_sglists *tsgls)
3114 {
3115         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3116         const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
3117         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
3118         const unsigned int blocksize = crypto_skcipher_blocksize(tfm);
3119         const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
3120         const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;
3121         const char *driver = crypto_skcipher_driver_name(tfm);
3122         struct rnd_state rng;
3123         char _generic_driver[CRYPTO_MAX_ALG_NAME];
3124         struct crypto_skcipher *generic_tfm = NULL;
3125         struct skcipher_request *generic_req = NULL;
3126         unsigned int i;
3127         struct cipher_testvec vec = { 0 };
3128         char vec_name[64];
3129         struct testvec_config *cfg;
3130         char cfgname[TESTVEC_CONFIG_NAMELEN];
3131         int err;
3132
3133         if (noextratests)
3134                 return 0;
3135
3136         /* Keywrap isn't supported here yet as it handles its IV differently. */
3137         if (strncmp(algname, "kw(", 3) == 0)
3138                 return 0;
3139
3140         init_rnd_state(&rng);
3141
3142         if (!generic_driver) { /* Use default naming convention? */
3143                 err = build_generic_driver_name(algname, _generic_driver);
3144                 if (err)
3145                         return err;
3146                 generic_driver = _generic_driver;
3147         }
3148
3149         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
3150                 return 0;
3151
3152         generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0);
3153         if (IS_ERR(generic_tfm)) {
3154                 err = PTR_ERR(generic_tfm);
3155                 if (err == -ENOENT) {
3156                         pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",
3157                                 driver, generic_driver);
3158                         return 0;
3159                 }
3160                 pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",
3161                        generic_driver, algname, err);
3162                 return err;
3163         }
3164
3165         cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
3166         if (!cfg) {
3167                 err = -ENOMEM;
3168                 goto out;
3169         }
3170
3171         generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
3172         if (!generic_req) {
3173                 err = -ENOMEM;
3174                 goto out;
3175         }
3176
3177         /* Check the algorithm properties for consistency. */
3178
3179         if (crypto_skcipher_min_keysize(tfm) !=
3180             crypto_skcipher_min_keysize(generic_tfm)) {
3181                 pr_err("alg: skcipher: min keysize for %s (%u) doesn't match generic impl (%u)\n",
3182                        driver, crypto_skcipher_min_keysize(tfm),
3183                        crypto_skcipher_min_keysize(generic_tfm));
3184                 err = -EINVAL;
3185                 goto out;
3186         }
3187
3188         if (maxkeysize != crypto_skcipher_max_keysize(generic_tfm)) {
3189                 pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",
3190                        driver, maxkeysize,
3191                        crypto_skcipher_max_keysize(generic_tfm));
3192                 err = -EINVAL;
3193                 goto out;
3194         }
3195
3196         if (ivsize != crypto_skcipher_ivsize(generic_tfm)) {
3197                 pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",
3198                        driver, ivsize, crypto_skcipher_ivsize(generic_tfm));
3199                 err = -EINVAL;
3200                 goto out;
3201         }
3202
3203         if (blocksize != crypto_skcipher_blocksize(generic_tfm)) {
3204                 pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",
3205                        driver, blocksize,
3206                        crypto_skcipher_blocksize(generic_tfm));
3207                 err = -EINVAL;
3208                 goto out;
3209         }
3210
3211         /*
3212          * Now generate test vectors using the generic implementation, and test
3213          * the other implementation against them.
3214          */
3215
3216         vec.key = kmalloc(maxkeysize, GFP_KERNEL);
3217         vec.iv = kmalloc(ivsize, GFP_KERNEL);
3218         vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
3219         vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
3220         if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) {
3221                 err = -ENOMEM;
3222                 goto out;
3223         }
3224
3225         for (i = 0; i < fuzz_iterations * 8; i++) {
3226                 generate_random_cipher_testvec(&rng, generic_req, &vec,
3227                                                maxdatasize,
3228                                                vec_name, sizeof(vec_name));
3229                 generate_random_testvec_config(&rng, cfg, cfgname,
3230                                                sizeof(cfgname));
3231
3232                 err = test_skcipher_vec_cfg(ENCRYPT, &vec, vec_name,
3233                                             cfg, req, tsgls);
3234                 if (err)
3235                         goto out;
3236                 err = test_skcipher_vec_cfg(DECRYPT, &vec, vec_name,
3237                                             cfg, req, tsgls);
3238                 if (err)
3239                         goto out;
3240                 cond_resched();
3241         }
3242         err = 0;
3243 out:
3244         kfree(cfg);
3245         kfree(vec.key);
3246         kfree(vec.iv);
3247         kfree(vec.ptext);
3248         kfree(vec.ctext);
3249         crypto_free_skcipher(generic_tfm);
3250         skcipher_request_free(generic_req);
3251         return err;
3252 }
3253 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
3254 static int test_skcipher_vs_generic_impl(const char *generic_driver,
3255                                          struct skcipher_request *req,
3256                                          struct cipher_test_sglists *tsgls)
3257 {
3258         return 0;
3259 }
3260 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
3261
3262 static int test_skcipher(int enc, const struct cipher_test_suite *suite,
3263                          struct skcipher_request *req,
3264                          struct cipher_test_sglists *tsgls)
3265 {
3266         unsigned int i;
3267         int err;
3268
3269         for (i = 0; i < suite->count; i++) {
3270                 err = test_skcipher_vec(enc, &suite->vecs[i], i, req, tsgls);
3271                 if (err)
3272                         return err;
3273                 cond_resched();
3274         }
3275         return 0;
3276 }
3277
3278 static int alg_test_skcipher(const struct alg_test_desc *desc,
3279                              const char *driver, u32 type, u32 mask)
3280 {
3281         const struct cipher_test_suite *suite = &desc->suite.cipher;
3282         struct crypto_skcipher *tfm;
3283         struct skcipher_request *req = NULL;
3284         struct cipher_test_sglists *tsgls = NULL;
3285         int err;
3286
3287         if (suite->count <= 0) {
3288                 pr_err("alg: skcipher: empty test suite for %s\n", driver);
3289                 return -EINVAL;
3290         }
3291
3292         tfm = crypto_alloc_skcipher(driver, type, mask);
3293         if (IS_ERR(tfm)) {
3294                 if (PTR_ERR(tfm) == -ENOENT)
3295                         return 0;
3296                 pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",
3297                        driver, PTR_ERR(tfm));
3298                 return PTR_ERR(tfm);
3299         }
3300         driver = crypto_skcipher_driver_name(tfm);
3301
3302         req = skcipher_request_alloc(tfm, GFP_KERNEL);
3303         if (!req) {
3304                 pr_err("alg: skcipher: failed to allocate request for %s\n",
3305                        driver);
3306                 err = -ENOMEM;
3307                 goto out;
3308         }
3309
3310         tsgls = alloc_cipher_test_sglists();
3311         if (!tsgls) {
3312                 pr_err("alg: skcipher: failed to allocate test buffers for %s\n",
3313                        driver);
3314                 err = -ENOMEM;
3315                 goto out;
3316         }
3317
3318         err = test_skcipher(ENCRYPT, suite, req, tsgls);
3319         if (err)
3320                 goto out;
3321
3322         err = test_skcipher(DECRYPT, suite, req, tsgls);
3323         if (err)
3324                 goto out;
3325
3326         err = test_skcipher_vs_generic_impl(desc->generic_driver, req, tsgls);
3327 out:
3328         free_cipher_test_sglists(tsgls);
3329         skcipher_request_free(req);
3330         crypto_free_skcipher(tfm);
3331         return err;
3332 }
3333
3334 static int test_comp(struct crypto_comp *tfm,
3335                      const struct comp_testvec *ctemplate,
3336                      const struct comp_testvec *dtemplate,
3337                      int ctcount, int dtcount)
3338 {
3339         const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm));
3340         char *output, *decomp_output;
3341         unsigned int i;
3342         int ret;
3343
3344         output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3345         if (!output)
3346                 return -ENOMEM;
3347
3348         decomp_output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3349         if (!decomp_output) {
3350                 kfree(output);
3351                 return -ENOMEM;
3352         }
3353
3354         for (i = 0; i < ctcount; i++) {
3355                 int ilen;
3356                 unsigned int dlen = COMP_BUF_SIZE;
3357
3358                 memset(output, 0, COMP_BUF_SIZE);
3359                 memset(decomp_output, 0, COMP_BUF_SIZE);
3360
3361                 ilen = ctemplate[i].inlen;
3362                 ret = crypto_comp_compress(tfm, ctemplate[i].input,
3363                                            ilen, output, &dlen);
3364                 if (ret) {
3365                         printk(KERN_ERR "alg: comp: compression failed "
3366                                "on test %d for %s: ret=%d\n", i + 1, algo,
3367                                -ret);
3368                         goto out;
3369                 }
3370
3371                 ilen = dlen;
3372                 dlen = COMP_BUF_SIZE;
3373                 ret = crypto_comp_decompress(tfm, output,
3374                                              ilen, decomp_output, &dlen);
3375                 if (ret) {
3376                         pr_err("alg: comp: compression failed: decompress: on test %d for %s failed: ret=%d\n",
3377                                i + 1, algo, -ret);
3378                         goto out;
3379                 }
3380
3381                 if (dlen != ctemplate[i].inlen) {
3382                         printk(KERN_ERR "alg: comp: Compression test %d "
3383                                "failed for %s: output len = %d\n", i + 1, algo,
3384                                dlen);
3385                         ret = -EINVAL;
3386                         goto out;
3387                 }
3388
3389                 if (memcmp(decomp_output, ctemplate[i].input,
3390                            ctemplate[i].inlen)) {
3391                         pr_err("alg: comp: compression failed: output differs: on test %d for %s\n",
3392                                i + 1, algo);
3393                         hexdump(decomp_output, dlen);
3394                         ret = -EINVAL;
3395                         goto out;
3396                 }
3397         }
3398
3399         for (i = 0; i < dtcount; i++) {
3400                 int ilen;
3401                 unsigned int dlen = COMP_BUF_SIZE;
3402
3403                 memset(decomp_output, 0, COMP_BUF_SIZE);
3404
3405                 ilen = dtemplate[i].inlen;
3406                 ret = crypto_comp_decompress(tfm, dtemplate[i].input,
3407                                              ilen, decomp_output, &dlen);
3408                 if (ret) {
3409                         printk(KERN_ERR "alg: comp: decompression failed "
3410                                "on test %d for %s: ret=%d\n", i + 1, algo,
3411                                -ret);
3412                         goto out;
3413                 }
3414
3415                 if (dlen != dtemplate[i].outlen) {
3416                         printk(KERN_ERR "alg: comp: Decompression test %d "
3417                                "failed for %s: output len = %d\n", i + 1, algo,
3418                                dlen);
3419                         ret = -EINVAL;
3420                         goto out;
3421                 }
3422
3423                 if (memcmp(decomp_output, dtemplate[i].output, dlen)) {
3424                         printk(KERN_ERR "alg: comp: Decompression test %d "
3425                                "failed for %s\n", i + 1, algo);
3426                         hexdump(decomp_output, dlen);
3427                         ret = -EINVAL;
3428                         goto out;
3429                 }
3430         }
3431
3432         ret = 0;
3433
3434 out:
3435         kfree(decomp_output);
3436         kfree(output);
3437         return ret;
3438 }
3439
3440 static int test_acomp(struct crypto_acomp *tfm,
3441                       const struct comp_testvec *ctemplate,
3442                       const struct comp_testvec *dtemplate,
3443                       int ctcount, int dtcount)
3444 {
3445         const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
3446         unsigned int i;
3447         char *output, *decomp_out;
3448         int ret;
3449         struct scatterlist src, dst;
3450         struct acomp_req *req;
3451         struct crypto_wait wait;
3452
3453         output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3454         if (!output)
3455                 return -ENOMEM;
3456
3457         decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3458         if (!decomp_out) {
3459                 kfree(output);
3460                 return -ENOMEM;
3461         }
3462
3463         for (i = 0; i < ctcount; i++) {
3464                 unsigned int dlen = COMP_BUF_SIZE;
3465                 int ilen = ctemplate[i].inlen;
3466                 void *input_vec;
3467
3468                 input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);
3469                 if (!input_vec) {
3470                         ret = -ENOMEM;
3471                         goto out;
3472                 }
3473
3474                 memset(output, 0, dlen);
3475                 crypto_init_wait(&wait);
3476                 sg_init_one(&src, input_vec, ilen);
3477                 sg_init_one(&dst, output, dlen);
3478
3479                 req = acomp_request_alloc(tfm);
3480                 if (!req) {
3481                         pr_err("alg: acomp: request alloc failed for %s\n",
3482                                algo);
3483                         kfree(input_vec);
3484                         ret = -ENOMEM;
3485                         goto out;
3486                 }
3487
3488                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
3489                 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3490                                            crypto_req_done, &wait);
3491
3492                 ret = crypto_wait_req(crypto_acomp_compress(req), &wait);
3493                 if (ret) {
3494                         pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3495                                i + 1, algo, -ret);
3496                         kfree(input_vec);
3497                         acomp_request_free(req);
3498                         goto out;
3499                 }
3500
3501                 ilen = req->dlen;
3502                 dlen = COMP_BUF_SIZE;
3503                 sg_init_one(&src, output, ilen);
3504                 sg_init_one(&dst, decomp_out, dlen);
3505                 crypto_init_wait(&wait);
3506                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
3507
3508                 ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3509                 if (ret) {
3510                         pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3511                                i + 1, algo, -ret);
3512                         kfree(input_vec);
3513                         acomp_request_free(req);
3514                         goto out;
3515                 }
3516
3517                 if (req->dlen != ctemplate[i].inlen) {
3518                         pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",
3519                                i + 1, algo, req->dlen);
3520                         ret = -EINVAL;
3521                         kfree(input_vec);
3522                         acomp_request_free(req);
3523                         goto out;
3524                 }
3525
3526                 if (memcmp(input_vec, decomp_out, req->dlen)) {
3527                         pr_err("alg: acomp: Compression test %d failed for %s\n",
3528                                i + 1, algo);
3529                         hexdump(output, req->dlen);
3530                         ret = -EINVAL;
3531                         kfree(input_vec);
3532                         acomp_request_free(req);
3533                         goto out;
3534                 }
3535
3536 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
3537                 crypto_init_wait(&wait);
3538                 sg_init_one(&src, input_vec, ilen);
3539                 acomp_request_set_params(req, &src, NULL, ilen, 0);
3540
3541                 ret = crypto_wait_req(crypto_acomp_compress(req), &wait);
3542                 if (ret) {
3543                         pr_err("alg: acomp: compression failed on NULL dst buffer test %d for %s: ret=%d\n",
3544                                i + 1, algo, -ret);
3545                         kfree(input_vec);
3546                         acomp_request_free(req);
3547                         goto out;
3548                 }
3549 #endif
3550
3551                 kfree(input_vec);
3552                 acomp_request_free(req);
3553         }
3554
3555         for (i = 0; i < dtcount; i++) {
3556                 unsigned int dlen = COMP_BUF_SIZE;
3557                 int ilen = dtemplate[i].inlen;
3558                 void *input_vec;
3559
3560                 input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);
3561                 if (!input_vec) {
3562                         ret = -ENOMEM;
3563                         goto out;
3564                 }
3565
3566                 memset(output, 0, dlen);
3567                 crypto_init_wait(&wait);
3568                 sg_init_one(&src, input_vec, ilen);
3569                 sg_init_one(&dst, output, dlen);
3570
3571                 req = acomp_request_alloc(tfm);
3572                 if (!req) {
3573                         pr_err("alg: acomp: request alloc failed for %s\n",
3574                                algo);
3575                         kfree(input_vec);
3576                         ret = -ENOMEM;
3577                         goto out;
3578                 }
3579
3580                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
3581                 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3582                                            crypto_req_done, &wait);
3583
3584                 ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3585                 if (ret) {
3586                         pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",
3587                                i + 1, algo, -ret);
3588                         kfree(input_vec);
3589                         acomp_request_free(req);
3590                         goto out;
3591                 }
3592
3593                 if (req->dlen != dtemplate[i].outlen) {
3594                         pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",
3595                                i + 1, algo, req->dlen);
3596                         ret = -EINVAL;
3597                         kfree(input_vec);
3598                         acomp_request_free(req);
3599                         goto out;
3600                 }
3601
3602                 if (memcmp(output, dtemplate[i].output, req->dlen)) {
3603                         pr_err("alg: acomp: Decompression test %d failed for %s\n",
3604                                i + 1, algo);
3605                         hexdump(output, req->dlen);
3606                         ret = -EINVAL;
3607                         kfree(input_vec);
3608                         acomp_request_free(req);
3609                         goto out;
3610                 }
3611
3612 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
3613                 crypto_init_wait(&wait);
3614                 acomp_request_set_params(req, &src, NULL, ilen, 0);
3615
3616                 ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3617                 if (ret) {
3618                         pr_err("alg: acomp: decompression failed on NULL dst buffer test %d for %s: ret=%d\n",
3619                                i + 1, algo, -ret);
3620                         kfree(input_vec);
3621                         acomp_request_free(req);
3622                         goto out;
3623                 }
3624 #endif
3625
3626                 kfree(input_vec);
3627                 acomp_request_free(req);
3628         }
3629
3630         ret = 0;
3631
3632 out:
3633         kfree(decomp_out);
3634         kfree(output);
3635         return ret;
3636 }
3637
3638 static int test_cprng(struct crypto_rng *tfm,
3639                       const struct cprng_testvec *template,
3640                       unsigned int tcount)
3641 {
3642         const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm));
3643         int err = 0, i, j, seedsize;
3644         u8 *seed;
3645         char result[32];
3646
3647         seedsize = crypto_rng_seedsize(tfm);
3648
3649         seed = kmalloc(seedsize, GFP_KERNEL);
3650         if (!seed) {
3651                 printk(KERN_ERR "alg: cprng: Failed to allocate seed space "
3652                        "for %s\n", algo);
3653                 return -ENOMEM;
3654         }
3655
3656         for (i = 0; i < tcount; i++) {
3657                 memset(result, 0, 32);
3658
3659                 memcpy(seed, template[i].v, template[i].vlen);
3660                 memcpy(seed + template[i].vlen, template[i].key,
3661                        template[i].klen);
3662                 memcpy(seed + template[i].vlen + template[i].klen,
3663                        template[i].dt, template[i].dtlen);
3664
3665                 err = crypto_rng_reset(tfm, seed, seedsize);
3666                 if (err) {
3667                         printk(KERN_ERR "alg: cprng: Failed to reset rng "
3668                                "for %s\n", algo);
3669                         goto out;
3670                 }
3671
3672                 for (j = 0; j < template[i].loops; j++) {
3673                         err = crypto_rng_get_bytes(tfm, result,
3674                                                    template[i].rlen);
3675                         if (err < 0) {
3676                                 printk(KERN_ERR "alg: cprng: Failed to obtain "
3677                                        "the correct amount of random data for "
3678                                        "%s (requested %d)\n", algo,
3679                                        template[i].rlen);
3680                                 goto out;
3681                         }
3682                 }
3683
3684                 err = memcmp(result, template[i].result,
3685                              template[i].rlen);
3686                 if (err) {
3687                         printk(KERN_ERR "alg: cprng: Test %d failed for %s\n",
3688                                i, algo);
3689                         hexdump(result, template[i].rlen);
3690                         err = -EINVAL;
3691                         goto out;
3692                 }
3693         }
3694
3695 out:
3696         kfree(seed);
3697         return err;
3698 }
3699
3700 static int alg_test_cipher(const struct alg_test_desc *desc,
3701                            const char *driver, u32 type, u32 mask)
3702 {
3703         const struct cipher_test_suite *suite = &desc->suite.cipher;
3704         struct crypto_cipher *tfm;
3705         int err;
3706
3707         tfm = crypto_alloc_cipher(driver, type, mask);
3708         if (IS_ERR(tfm)) {
3709                 if (PTR_ERR(tfm) == -ENOENT)
3710                         return 0;
3711                 printk(KERN_ERR "alg: cipher: Failed to load transform for "
3712                        "%s: %ld\n", driver, PTR_ERR(tfm));
3713                 return PTR_ERR(tfm);
3714         }
3715
3716         err = test_cipher(tfm, ENCRYPT, suite->vecs, suite->count);
3717         if (!err)
3718                 err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);
3719
3720         crypto_free_cipher(tfm);
3721         return err;
3722 }
3723
3724 static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
3725                          u32 type, u32 mask)
3726 {
3727         struct crypto_comp *comp;
3728         struct crypto_acomp *acomp;
3729         int err;
3730         u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK;
3731
3732         if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) {
3733                 acomp = crypto_alloc_acomp(driver, type, mask);
3734                 if (IS_ERR(acomp)) {
3735                         if (PTR_ERR(acomp) == -ENOENT)
3736                                 return 0;
3737                         pr_err("alg: acomp: Failed to load transform for %s: %ld\n",
3738                                driver, PTR_ERR(acomp));
3739                         return PTR_ERR(acomp);
3740                 }
3741                 err = test_acomp(acomp, desc->suite.comp.comp.vecs,
3742                                  desc->suite.comp.decomp.vecs,
3743                                  desc->suite.comp.comp.count,
3744                                  desc->suite.comp.decomp.count);
3745                 crypto_free_acomp(acomp);
3746         } else {
3747                 comp = crypto_alloc_comp(driver, type, mask);
3748                 if (IS_ERR(comp)) {
3749                         if (PTR_ERR(comp) == -ENOENT)
3750                                 return 0;
3751                         pr_err("alg: comp: Failed to load transform for %s: %ld\n",
3752                                driver, PTR_ERR(comp));
3753                         return PTR_ERR(comp);
3754                 }
3755
3756                 err = test_comp(comp, desc->suite.comp.comp.vecs,
3757                                 desc->suite.comp.decomp.vecs,
3758                                 desc->suite.comp.comp.count,
3759                                 desc->suite.comp.decomp.count);
3760
3761                 crypto_free_comp(comp);
3762         }
3763         return err;
3764 }
3765
3766 static int alg_test_crc32c(const struct alg_test_desc *desc,
3767                            const char *driver, u32 type, u32 mask)
3768 {
3769         struct crypto_shash *tfm;
3770         __le32 val;
3771         int err;
3772
3773         err = alg_test_hash(desc, driver, type, mask);
3774         if (err)
3775                 return err;
3776
3777         tfm = crypto_alloc_shash(driver, type, mask);
3778         if (IS_ERR(tfm)) {
3779                 if (PTR_ERR(tfm) == -ENOENT) {
3780                         /*
3781                          * This crc32c implementation is only available through
3782                          * ahash API, not the shash API, so the remaining part
3783                          * of the test is not applicable to it.
3784                          */
3785                         return 0;
3786                 }
3787                 printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "
3788                        "%ld\n", driver, PTR_ERR(tfm));
3789                 return PTR_ERR(tfm);
3790         }
3791         driver = crypto_shash_driver_name(tfm);
3792
3793         do {
3794                 SHASH_DESC_ON_STACK(shash, tfm);
3795                 u32 *ctx = (u32 *)shash_desc_ctx(shash);
3796
3797                 shash->tfm = tfm;
3798
3799                 *ctx = 420553207;
3800                 err = crypto_shash_final(shash, (u8 *)&val);
3801                 if (err) {
3802                         printk(KERN_ERR "alg: crc32c: Operation failed for "
3803                                "%s: %d\n", driver, err);
3804                         break;
3805                 }
3806
3807                 if (val != cpu_to_le32(~420553207)) {
3808                         pr_err("alg: crc32c: Test failed for %s: %u\n",
3809                                driver, le32_to_cpu(val));
3810                         err = -EINVAL;
3811                 }
3812         } while (0);
3813
3814         crypto_free_shash(tfm);
3815
3816         return err;
3817 }
3818
3819 static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,
3820                           u32 type, u32 mask)
3821 {
3822         struct crypto_rng *rng;
3823         int err;
3824
3825         rng = crypto_alloc_rng(driver, type, mask);
3826         if (IS_ERR(rng)) {
3827                 if (PTR_ERR(rng) == -ENOENT)
3828                         return 0;
3829                 printk(KERN_ERR "alg: cprng: Failed to load transform for %s: "
3830                        "%ld\n", driver, PTR_ERR(rng));
3831                 return PTR_ERR(rng);
3832         }
3833
3834         err = test_cprng(rng, desc->suite.cprng.vecs, desc->suite.cprng.count);
3835
3836         crypto_free_rng(rng);
3837
3838         return err;
3839 }
3840
3841
3842 static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
3843                           const char *driver, u32 type, u32 mask)
3844 {
3845         int ret = -EAGAIN;
3846         struct crypto_rng *drng;
3847         struct drbg_test_data test_data;
3848         struct drbg_string addtl, pers, testentropy;
3849         unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);
3850
3851         if (!buf)
3852                 return -ENOMEM;
3853
3854         drng = crypto_alloc_rng(driver, type, mask);
3855         if (IS_ERR(drng)) {
3856                 kfree_sensitive(buf);
3857                 if (PTR_ERR(drng) == -ENOENT)
3858                         return 0;
3859                 printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
3860                        "%s\n", driver);
3861                 return PTR_ERR(drng);
3862         }
3863
3864         test_data.testentropy = &testentropy;
3865         drbg_string_fill(&testentropy, test->entropy, test->entropylen);
3866         drbg_string_fill(&pers, test->pers, test->perslen);
3867         ret = crypto_drbg_reset_test(drng, &pers, &test_data);
3868         if (ret) {
3869                 printk(KERN_ERR "alg: drbg: Failed to reset rng\n");
3870                 goto outbuf;
3871         }
3872
3873         drbg_string_fill(&addtl, test->addtla, test->addtllen);
3874         if (pr) {
3875                 drbg_string_fill(&testentropy, test->entpra, test->entprlen);
3876                 ret = crypto_drbg_get_bytes_addtl_test(drng,
3877                         buf, test->expectedlen, &addtl, &test_data);
3878         } else {
3879                 ret = crypto_drbg_get_bytes_addtl(drng,
3880                         buf, test->expectedlen, &addtl);
3881         }
3882         if (ret < 0) {
3883                 printk(KERN_ERR "alg: drbg: could not obtain random data for "
3884                        "driver %s\n", driver);
3885                 goto outbuf;
3886         }
3887
3888         drbg_string_fill(&addtl, test->addtlb, test->addtllen);
3889         if (pr) {
3890                 drbg_string_fill(&testentropy, test->entprb, test->entprlen);
3891                 ret = crypto_drbg_get_bytes_addtl_test(drng,
3892                         buf, test->expectedlen, &addtl, &test_data);
3893         } else {
3894                 ret = crypto_drbg_get_bytes_addtl(drng,
3895                         buf, test->expectedlen, &addtl);
3896         }
3897         if (ret < 0) {
3898                 printk(KERN_ERR "alg: drbg: could not obtain random data for "
3899                        "driver %s\n", driver);
3900                 goto outbuf;
3901         }
3902
3903         ret = memcmp(test->expected, buf, test->expectedlen);
3904
3905 outbuf:
3906         crypto_free_rng(drng);
3907         kfree_sensitive(buf);
3908         return ret;
3909 }
3910
3911
3912 static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,
3913                          u32 type, u32 mask)
3914 {
3915         int err = 0;
3916         int pr = 0;
3917         int i = 0;
3918         const struct drbg_testvec *template = desc->suite.drbg.vecs;
3919         unsigned int tcount = desc->suite.drbg.count;
3920
3921         if (0 == memcmp(driver, "drbg_pr_", 8))
3922                 pr = 1;
3923
3924         for (i = 0; i < tcount; i++) {
3925                 err = drbg_cavs_test(&template[i], pr, driver, type, mask);
3926                 if (err) {
3927                         printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",
3928                                i, driver);
3929                         err = -EINVAL;
3930                         break;
3931                 }
3932         }
3933         return err;
3934
3935 }
3936
3937 static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
3938                        const char *alg)
3939 {
3940         struct kpp_request *req;
3941         void *input_buf = NULL;
3942         void *output_buf = NULL;
3943         void *a_public = NULL;
3944         void *a_ss = NULL;
3945         void *shared_secret = NULL;
3946         struct crypto_wait wait;
3947         unsigned int out_len_max;
3948         int err = -ENOMEM;
3949         struct scatterlist src, dst;
3950
3951         req = kpp_request_alloc(tfm, GFP_KERNEL);
3952         if (!req)
3953                 return err;
3954
3955         crypto_init_wait(&wait);
3956
3957         err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size);
3958         if (err < 0)
3959                 goto free_req;
3960
3961         out_len_max = crypto_kpp_maxsize(tfm);
3962         output_buf = kzalloc(out_len_max, GFP_KERNEL);
3963         if (!output_buf) {
3964                 err = -ENOMEM;
3965                 goto free_req;
3966         }
3967
3968         /* Use appropriate parameter as base */
3969         kpp_request_set_input(req, NULL, 0);
3970         sg_init_one(&dst, output_buf, out_len_max);
3971         kpp_request_set_output(req, &dst, out_len_max);
3972         kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3973                                  crypto_req_done, &wait);
3974
3975         /* Compute party A's public key */
3976         err = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait);
3977         if (err) {
3978                 pr_err("alg: %s: Party A: generate public key test failed. err %d\n",
3979                        alg, err);
3980                 goto free_output;
3981         }
3982
3983         if (vec->genkey) {
3984                 /* Save party A's public key */
3985                 a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);
3986                 if (!a_public) {
3987                         err = -ENOMEM;
3988                         goto free_output;
3989                 }
3990         } else {
3991                 /* Verify calculated public key */
3992                 if (memcmp(vec->expected_a_public, sg_virt(req->dst),
3993                            vec->expected_a_public_size)) {
3994                         pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",
3995                                alg);
3996                         err = -EINVAL;
3997                         goto free_output;
3998                 }
3999         }
4000
4001         /* Calculate shared secret key by using counter part (b) public key. */
4002         input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);
4003         if (!input_buf) {
4004                 err = -ENOMEM;
4005                 goto free_output;
4006         }
4007
4008         sg_init_one(&src, input_buf, vec->b_public_size);
4009         sg_init_one(&dst, output_buf, out_len_max);
4010         kpp_request_set_input(req, &src, vec->b_public_size);
4011         kpp_request_set_output(req, &dst, out_len_max);
4012         kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
4013                                  crypto_req_done, &wait);
4014         err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait);
4015         if (err) {
4016                 pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",
4017                        alg, err);
4018                 goto free_all;
4019         }
4020
4021         if (vec->genkey) {
4022                 /* Save the shared secret obtained by party A */
4023                 a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);
4024                 if (!a_ss) {
4025                         err = -ENOMEM;
4026                         goto free_all;
4027                 }
4028
4029                 /*
4030                  * Calculate party B's shared secret by using party A's
4031                  * public key.
4032                  */
4033                 err = crypto_kpp_set_secret(tfm, vec->b_secret,
4034                                             vec->b_secret_size);
4035                 if (err < 0)
4036                         goto free_all;
4037
4038                 sg_init_one(&src, a_public, vec->expected_a_public_size);
4039                 sg_init_one(&dst, output_buf, out_len_max);
4040                 kpp_request_set_input(req, &src, vec->expected_a_public_size);
4041                 kpp_request_set_output(req, &dst, out_len_max);
4042                 kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
4043                                          crypto_req_done, &wait);
4044                 err = crypto_wait_req(crypto_kpp_compute_shared_secret(req),
4045                                       &wait);
4046                 if (err) {
4047                         pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",
4048                                alg, err);
4049                         goto free_all;
4050                 }
4051
4052                 shared_secret = a_ss;
4053         } else {
4054                 shared_secret = (void *)vec->expected_ss;
4055         }
4056
4057         /*
4058          * verify shared secret from which the user will derive
4059          * secret key by executing whatever hash it has chosen
4060          */
4061         if (memcmp(shared_secret, sg_virt(req->dst),
4062                    vec->expected_ss_size)) {
4063                 pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
4064                        alg);
4065                 err = -EINVAL;
4066         }
4067
4068 free_all:
4069         kfree(a_ss);
4070         kfree(input_buf);
4071 free_output:
4072         kfree(a_public);
4073         kfree(output_buf);
4074 free_req:
4075         kpp_request_free(req);
4076         return err;
4077 }
4078
4079 static int test_kpp(struct crypto_kpp *tfm, const char *alg,
4080                     const struct kpp_testvec *vecs, unsigned int tcount)
4081 {
4082         int ret, i;
4083
4084         for (i = 0; i < tcount; i++) {
4085                 ret = do_test_kpp(tfm, vecs++, alg);
4086                 if (ret) {
4087                         pr_err("alg: %s: test failed on vector %d, err=%d\n",
4088                                alg, i + 1, ret);
4089                         return ret;
4090                 }
4091         }
4092         return 0;
4093 }
4094
4095 static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
4096                         u32 type, u32 mask)
4097 {
4098         struct crypto_kpp *tfm;
4099         int err = 0;
4100
4101         tfm = crypto_alloc_kpp(driver, type, mask);
4102         if (IS_ERR(tfm)) {
4103                 if (PTR_ERR(tfm) == -ENOENT)
4104                         return 0;
4105                 pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
4106                        driver, PTR_ERR(tfm));
4107                 return PTR_ERR(tfm);
4108         }
4109         if (desc->suite.kpp.vecs)
4110                 err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs,
4111                                desc->suite.kpp.count);
4112
4113         crypto_free_kpp(tfm);
4114         return err;
4115 }
4116
4117 static u8 *test_pack_u32(u8 *dst, u32 val)
4118 {
4119         memcpy(dst, &val, sizeof(val));
4120         return dst + sizeof(val);
4121 }
4122
4123 static int test_akcipher_one(struct crypto_akcipher *tfm,
4124                              const struct akcipher_testvec *vecs)
4125 {
4126         char *xbuf[XBUFSIZE];
4127         struct akcipher_request *req;
4128         void *outbuf_enc = NULL;
4129         void *outbuf_dec = NULL;
4130         struct crypto_wait wait;
4131         unsigned int out_len_max, out_len = 0;
4132         int err = -ENOMEM;
4133         struct scatterlist src, dst, src_tab[2];
4134         const char *c;
4135         unsigned int c_size;
4136
4137         if (testmgr_alloc_buf(xbuf))
4138                 return err;
4139
4140         req = akcipher_request_alloc(tfm, GFP_KERNEL);
4141         if (!req)
4142                 goto free_xbuf;
4143
4144         crypto_init_wait(&wait);
4145
4146         if (vecs->public_key_vec)
4147                 err = crypto_akcipher_set_pub_key(tfm, vecs->key,
4148                                                   vecs->key_len);
4149         else
4150                 err = crypto_akcipher_set_priv_key(tfm, vecs->key,
4151                                                    vecs->key_len);
4152         if (err)
4153                 goto free_req;
4154
4155         /* First run encrypt test which does not require a private key */
4156         err = -ENOMEM;
4157         out_len_max = crypto_akcipher_maxsize(tfm);
4158         outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
4159         if (!outbuf_enc)
4160                 goto free_req;
4161
4162         c = vecs->c;
4163         c_size = vecs->c_size;
4164
4165         err = -E2BIG;
4166         if (WARN_ON(vecs->m_size > PAGE_SIZE))
4167                 goto free_all;
4168         memcpy(xbuf[0], vecs->m, vecs->m_size);
4169
4170         sg_init_table(src_tab, 2);
4171         sg_set_buf(&src_tab[0], xbuf[0], 8);
4172         sg_set_buf(&src_tab[1], xbuf[0] + 8, vecs->m_size - 8);
4173         sg_init_one(&dst, outbuf_enc, out_len_max);
4174         akcipher_request_set_crypt(req, src_tab, &dst, vecs->m_size,
4175                                    out_len_max);
4176         akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
4177                                       crypto_req_done, &wait);
4178
4179         err = crypto_wait_req(crypto_akcipher_encrypt(req), &wait);
4180         if (err) {
4181                 pr_err("alg: akcipher: encrypt test failed. err %d\n", err);
4182                 goto free_all;
4183         }
4184         if (c) {
4185                 if (req->dst_len != c_size) {
4186                         pr_err("alg: akcipher: encrypt test failed. Invalid output len\n");
4187                         err = -EINVAL;
4188                         goto free_all;
4189                 }
4190                 /* verify that encrypted message is equal to expected */
4191                 if (memcmp(c, outbuf_enc, c_size) != 0) {
4192                         pr_err("alg: akcipher: encrypt test failed. Invalid output\n");
4193                         hexdump(outbuf_enc, c_size);
4194                         err = -EINVAL;
4195                         goto free_all;
4196                 }
4197         }
4198
4199         /*
4200          * Don't invoke decrypt test which requires a private key
4201          * for vectors with only a public key.
4202          */
4203         if (vecs->public_key_vec) {
4204                 err = 0;
4205                 goto free_all;
4206         }
4207         outbuf_dec = kzalloc(out_len_max, GFP_KERNEL);
4208         if (!outbuf_dec) {
4209                 err = -ENOMEM;
4210                 goto free_all;
4211         }
4212
4213         if (!c) {
4214                 c = outbuf_enc;
4215                 c_size = req->dst_len;
4216         }
4217
4218         err = -E2BIG;
4219         if (WARN_ON(c_size > PAGE_SIZE))
4220                 goto free_all;
4221         memcpy(xbuf[0], c, c_size);
4222
4223         sg_init_one(&src, xbuf[0], c_size);
4224         sg_init_one(&dst, outbuf_dec, out_len_max);
4225         crypto_init_wait(&wait);
4226         akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max);
4227
4228         err = crypto_wait_req(crypto_akcipher_decrypt(req), &wait);
4229         if (err) {
4230                 pr_err("alg: akcipher: decrypt test failed. err %d\n", err);
4231                 goto free_all;
4232         }
4233         out_len = req->dst_len;
4234         if (out_len < vecs->m_size) {
4235                 pr_err("alg: akcipher: decrypt test failed. Invalid output len %u\n",
4236                        out_len);
4237                 err = -EINVAL;
4238                 goto free_all;
4239         }
4240         /* verify that decrypted message is equal to the original msg */
4241         if (memchr_inv(outbuf_dec, 0, out_len - vecs->m_size) ||
4242             memcmp(vecs->m, outbuf_dec + out_len - vecs->m_size,
4243                    vecs->m_size)) {
4244                 pr_err("alg: akcipher: decrypt test failed. Invalid output\n");
4245                 hexdump(outbuf_dec, out_len);
4246                 err = -EINVAL;
4247         }
4248 free_all:
4249         kfree(outbuf_dec);
4250         kfree(outbuf_enc);
4251 free_req:
4252         akcipher_request_free(req);
4253 free_xbuf:
4254         testmgr_free_buf(xbuf);
4255         return err;
4256 }
4257
4258 static int test_akcipher(struct crypto_akcipher *tfm, const char *alg,
4259                          const struct akcipher_testvec *vecs,
4260                          unsigned int tcount)
4261 {
4262         const char *algo =
4263                 crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm));
4264         int ret, i;
4265
4266         for (i = 0; i < tcount; i++) {
4267                 ret = test_akcipher_one(tfm, vecs++);
4268                 if (!ret)
4269                         continue;
4270
4271                 pr_err("alg: akcipher: test %d failed for %s, err=%d\n",
4272                        i + 1, algo, ret);
4273                 return ret;
4274         }
4275         return 0;
4276 }
4277
4278 static int alg_test_akcipher(const struct alg_test_desc *desc,
4279                              const char *driver, u32 type, u32 mask)
4280 {
4281         struct crypto_akcipher *tfm;
4282         int err = 0;
4283
4284         tfm = crypto_alloc_akcipher(driver, type, mask);
4285         if (IS_ERR(tfm)) {
4286                 if (PTR_ERR(tfm) == -ENOENT)
4287                         return 0;
4288                 pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",
4289                        driver, PTR_ERR(tfm));
4290                 return PTR_ERR(tfm);
4291         }
4292         if (desc->suite.akcipher.vecs)
4293                 err = test_akcipher(tfm, desc->alg, desc->suite.akcipher.vecs,
4294                                     desc->suite.akcipher.count);
4295
4296         crypto_free_akcipher(tfm);
4297         return err;
4298 }
4299
4300 static int test_sig_one(struct crypto_sig *tfm, const struct sig_testvec *vecs)
4301 {
4302         u8 *ptr, *key __free(kfree);
4303         int err, sig_size;
4304
4305         key = kmalloc(vecs->key_len + 2 * sizeof(u32) + vecs->param_len,
4306                       GFP_KERNEL);
4307         if (!key)
4308                 return -ENOMEM;
4309
4310         /* ecrdsa expects additional parameters appended to the key */
4311         memcpy(key, vecs->key, vecs->key_len);
4312         ptr = key + vecs->key_len;
4313         ptr = test_pack_u32(ptr, vecs->algo);
4314         ptr = test_pack_u32(ptr, vecs->param_len);
4315         memcpy(ptr, vecs->params, vecs->param_len);
4316
4317         if (vecs->public_key_vec)
4318                 err = crypto_sig_set_pubkey(tfm, key, vecs->key_len);
4319         else
4320                 err = crypto_sig_set_privkey(tfm, key, vecs->key_len);
4321         if (err)
4322                 return err;
4323
4324         /*
4325          * Run asymmetric signature verification first
4326          * (which does not require a private key)
4327          */
4328         err = crypto_sig_verify(tfm, vecs->c, vecs->c_size,
4329                                 vecs->m, vecs->m_size);
4330         if (err) {
4331                 pr_err("alg: sig: verify test failed: err %d\n", err);
4332                 return err;
4333         }
4334
4335         /*
4336          * Don't invoke sign test (which requires a private key)
4337          * for vectors with only a public key.
4338          */
4339         if (vecs->public_key_vec)
4340                 return 0;
4341
4342         sig_size = crypto_sig_keysize(tfm);
4343         if (sig_size < vecs->c_size) {
4344                 pr_err("alg: sig: invalid maxsize %u\n", sig_size);
4345                 return -EINVAL;
4346         }
4347
4348         u8 *sig __free(kfree) = kzalloc(sig_size, GFP_KERNEL);
4349         if (!sig)
4350                 return -ENOMEM;
4351
4352         /* Run asymmetric signature generation */
4353         err = crypto_sig_sign(tfm, vecs->m, vecs->m_size, sig, sig_size);
4354         if (err) {
4355                 pr_err("alg: sig: sign test failed: err %d\n", err);
4356                 return err;
4357         }
4358
4359         /* Verify that generated signature equals cooked signature */
4360         if (memcmp(sig, vecs->c, vecs->c_size) ||
4361             memchr_inv(sig + vecs->c_size, 0, sig_size - vecs->c_size)) {
4362                 pr_err("alg: sig: sign test failed: invalid output\n");
4363                 hexdump(sig, sig_size);
4364                 return -EINVAL;
4365         }
4366
4367         return 0;
4368 }
4369
4370 static int test_sig(struct crypto_sig *tfm, const char *alg,
4371                     const struct sig_testvec *vecs, unsigned int tcount)
4372 {
4373         const char *algo = crypto_tfm_alg_driver_name(crypto_sig_tfm(tfm));
4374         int ret, i;
4375
4376         for (i = 0; i < tcount; i++) {
4377                 ret = test_sig_one(tfm, vecs++);
4378                 if (ret) {
4379                         pr_err("alg: sig: test %d failed for %s: err %d\n",
4380                                i + 1, algo, ret);
4381                         return ret;
4382                 }
4383         }
4384         return 0;
4385 }
4386
4387 static int alg_test_sig(const struct alg_test_desc *desc, const char *driver,
4388                         u32 type, u32 mask)
4389 {
4390         struct crypto_sig *tfm;
4391         int err = 0;
4392
4393         tfm = crypto_alloc_sig(driver, type, mask);
4394         if (IS_ERR(tfm)) {
4395                 pr_err("alg: sig: Failed to load tfm for %s: %ld\n",
4396                        driver, PTR_ERR(tfm));
4397                 return PTR_ERR(tfm);
4398         }
4399         if (desc->suite.sig.vecs)
4400                 err = test_sig(tfm, desc->alg, desc->suite.sig.vecs,
4401                                desc->suite.sig.count);
4402
4403         crypto_free_sig(tfm);
4404         return err;
4405 }
4406
4407 static int alg_test_null(const struct alg_test_desc *desc,
4408                              const char *driver, u32 type, u32 mask)
4409 {
4410         return 0;
4411 }
4412
4413 #define ____VECS(tv)    .vecs = tv, .count = ARRAY_SIZE(tv)
4414 #define __VECS(tv)      { ____VECS(tv) }
4415
4416 /* Please keep this list sorted by algorithm name. */
4417 static const struct alg_test_desc alg_test_descs[] = {
4418         {
4419                 .alg = "adiantum(xchacha12,aes)",
4420                 .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)",
4421                 .test = alg_test_skcipher,
4422                 .suite = {
4423                         .cipher = __VECS(adiantum_xchacha12_aes_tv_template)
4424                 },
4425         }, {
4426                 .alg = "adiantum(xchacha20,aes)",
4427                 .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)",
4428                 .test = alg_test_skcipher,
4429                 .suite = {
4430                         .cipher = __VECS(adiantum_xchacha20_aes_tv_template)
4431                 },
4432         }, {
4433                 .alg = "aegis128",
4434                 .test = alg_test_aead,
4435                 .suite = {
4436                         .aead = __VECS(aegis128_tv_template)
4437                 }
4438         }, {
4439                 .alg = "ansi_cprng",
4440                 .test = alg_test_cprng,
4441                 .suite = {
4442                         .cprng = __VECS(ansi_cprng_aes_tv_template)
4443                 }
4444         }, {
4445                 .alg = "authenc(hmac(md5),ecb(cipher_null))",
4446                 .test = alg_test_aead,
4447                 .suite = {
4448                         .aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)
4449                 }
4450         }, {
4451                 .alg = "authenc(hmac(sha1),cbc(aes))",
4452                 .test = alg_test_aead,
4453                 .fips_allowed = 1,
4454                 .suite = {
4455                         .aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
4456                 }
4457         }, {
4458                 .alg = "authenc(hmac(sha1),cbc(des))",
4459                 .test = alg_test_aead,
4460                 .suite = {
4461                         .aead = __VECS(hmac_sha1_des_cbc_tv_temp)
4462                 }
4463         }, {
4464                 .alg = "authenc(hmac(sha1),cbc(des3_ede))",
4465                 .test = alg_test_aead,
4466                 .suite = {
4467                         .aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
4468                 }
4469         }, {
4470                 .alg = "authenc(hmac(sha1),ctr(aes))",
4471                 .test = alg_test_null,
4472                 .fips_allowed = 1,
4473         }, {
4474                 .alg = "authenc(hmac(sha1),ecb(cipher_null))",
4475                 .test = alg_test_aead,
4476                 .suite = {
4477                         .aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
4478                 }
4479         }, {
4480                 .alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4481                 .test = alg_test_null,
4482                 .fips_allowed = 1,
4483         }, {
4484                 .alg = "authenc(hmac(sha224),cbc(des))",
4485                 .test = alg_test_aead,
4486                 .suite = {
4487                         .aead = __VECS(hmac_sha224_des_cbc_tv_temp)
4488                 }
4489         }, {
4490                 .alg = "authenc(hmac(sha224),cbc(des3_ede))",
4491                 .test = alg_test_aead,
4492                 .suite = {
4493                         .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
4494                 }
4495         }, {
4496                 .alg = "authenc(hmac(sha256),cbc(aes))",
4497                 .test = alg_test_aead,
4498                 .fips_allowed = 1,
4499                 .suite = {
4500                         .aead = __VECS(hmac_sha256_aes_cbc_tv_temp)
4501                 }
4502         }, {
4503                 .alg = "authenc(hmac(sha256),cbc(des))",
4504                 .test = alg_test_aead,
4505                 .suite = {
4506                         .aead = __VECS(hmac_sha256_des_cbc_tv_temp)
4507                 }
4508         }, {
4509                 .alg = "authenc(hmac(sha256),cbc(des3_ede))",
4510                 .test = alg_test_aead,
4511                 .suite = {
4512                         .aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
4513                 }
4514         }, {
4515                 .alg = "authenc(hmac(sha256),ctr(aes))",
4516                 .test = alg_test_null,
4517                 .fips_allowed = 1,
4518         }, {
4519                 .alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4520                 .test = alg_test_null,
4521                 .fips_allowed = 1,
4522         }, {
4523                 .alg = "authenc(hmac(sha384),cbc(des))",
4524                 .test = alg_test_aead,
4525                 .suite = {
4526                         .aead = __VECS(hmac_sha384_des_cbc_tv_temp)
4527                 }
4528         }, {
4529                 .alg = "authenc(hmac(sha384),cbc(des3_ede))",
4530                 .test = alg_test_aead,
4531                 .suite = {
4532                         .aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
4533                 }
4534         }, {
4535                 .alg = "authenc(hmac(sha384),ctr(aes))",
4536                 .test = alg_test_null,
4537                 .fips_allowed = 1,
4538         }, {
4539                 .alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4540                 .test = alg_test_null,
4541                 .fips_allowed = 1,
4542         }, {
4543                 .alg = "authenc(hmac(sha512),cbc(aes))",
4544                 .fips_allowed = 1,
4545                 .test = alg_test_aead,
4546                 .suite = {
4547                         .aead = __VECS(hmac_sha512_aes_cbc_tv_temp)
4548                 }
4549         }, {
4550                 .alg = "authenc(hmac(sha512),cbc(des))",
4551                 .test = alg_test_aead,
4552                 .suite = {
4553                         .aead = __VECS(hmac_sha512_des_cbc_tv_temp)
4554                 }
4555         }, {
4556                 .alg = "authenc(hmac(sha512),cbc(des3_ede))",
4557                 .test = alg_test_aead,
4558                 .suite = {
4559                         .aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
4560                 }
4561         }, {
4562                 .alg = "authenc(hmac(sha512),ctr(aes))",
4563                 .test = alg_test_null,
4564                 .fips_allowed = 1,
4565         }, {
4566                 .alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4567                 .test = alg_test_null,
4568                 .fips_allowed = 1,
4569         }, {
4570                 .alg = "blake2b-160",
4571                 .test = alg_test_hash,
4572                 .fips_allowed = 0,
4573                 .suite = {
4574                         .hash = __VECS(blake2b_160_tv_template)
4575                 }
4576         }, {
4577                 .alg = "blake2b-256",
4578                 .test = alg_test_hash,
4579                 .fips_allowed = 0,
4580                 .suite = {
4581                         .hash = __VECS(blake2b_256_tv_template)
4582                 }
4583         }, {
4584                 .alg = "blake2b-384",
4585                 .test = alg_test_hash,
4586                 .fips_allowed = 0,
4587                 .suite = {
4588                         .hash = __VECS(blake2b_384_tv_template)
4589                 }
4590         }, {
4591                 .alg = "blake2b-512",
4592                 .test = alg_test_hash,
4593                 .fips_allowed = 0,
4594                 .suite = {
4595                         .hash = __VECS(blake2b_512_tv_template)
4596                 }
4597         }, {
4598                 .alg = "cbc(aes)",
4599                 .test = alg_test_skcipher,
4600                 .fips_allowed = 1,
4601                 .suite = {
4602                         .cipher = __VECS(aes_cbc_tv_template)
4603                 },
4604         }, {
4605                 .alg = "cbc(anubis)",
4606                 .test = alg_test_skcipher,
4607                 .suite = {
4608                         .cipher = __VECS(anubis_cbc_tv_template)
4609                 },
4610         }, {
4611                 .alg = "cbc(aria)",
4612                 .test = alg_test_skcipher,
4613                 .suite = {
4614                         .cipher = __VECS(aria_cbc_tv_template)
4615                 },
4616         }, {
4617                 .alg = "cbc(blowfish)",
4618                 .test = alg_test_skcipher,
4619                 .suite = {
4620                         .cipher = __VECS(bf_cbc_tv_template)
4621                 },
4622         }, {
4623                 .alg = "cbc(camellia)",
4624                 .test = alg_test_skcipher,
4625                 .suite = {
4626                         .cipher = __VECS(camellia_cbc_tv_template)
4627                 },
4628         }, {
4629                 .alg = "cbc(cast5)",
4630                 .test = alg_test_skcipher,
4631                 .suite = {
4632                         .cipher = __VECS(cast5_cbc_tv_template)
4633                 },
4634         }, {
4635                 .alg = "cbc(cast6)",
4636                 .test = alg_test_skcipher,
4637                 .suite = {
4638                         .cipher = __VECS(cast6_cbc_tv_template)
4639                 },
4640         }, {
4641                 .alg = "cbc(des)",
4642                 .test = alg_test_skcipher,
4643                 .suite = {
4644                         .cipher = __VECS(des_cbc_tv_template)
4645                 },
4646         }, {
4647                 .alg = "cbc(des3_ede)",
4648                 .test = alg_test_skcipher,
4649                 .suite = {
4650                         .cipher = __VECS(des3_ede_cbc_tv_template)
4651                 },
4652         }, {
4653                 /* Same as cbc(aes) except the key is stored in
4654                  * hardware secure memory which we reference by index
4655                  */
4656                 .alg = "cbc(paes)",
4657                 .test = alg_test_null,
4658                 .fips_allowed = 1,
4659         }, {
4660                 /* Same as cbc(sm4) except the key is stored in
4661                  * hardware secure memory which we reference by index
4662                  */
4663                 .alg = "cbc(psm4)",
4664                 .test = alg_test_null,
4665         }, {
4666                 .alg = "cbc(serpent)",
4667                 .test = alg_test_skcipher,
4668                 .suite = {
4669                         .cipher = __VECS(serpent_cbc_tv_template)
4670                 },
4671         }, {
4672                 .alg = "cbc(sm4)",
4673                 .test = alg_test_skcipher,
4674                 .suite = {
4675                         .cipher = __VECS(sm4_cbc_tv_template)
4676                 }
4677         }, {
4678                 .alg = "cbc(twofish)",
4679                 .test = alg_test_skcipher,
4680                 .suite = {
4681                         .cipher = __VECS(tf_cbc_tv_template)
4682                 },
4683         }, {
4684 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4685                 .alg = "cbc-paes-s390",
4686                 .fips_allowed = 1,
4687                 .test = alg_test_skcipher,
4688                 .suite = {
4689                         .cipher = __VECS(aes_cbc_tv_template)
4690                 }
4691         }, {
4692 #endif
4693                 .alg = "cbcmac(aes)",
4694                 .test = alg_test_hash,
4695                 .suite = {
4696                         .hash = __VECS(aes_cbcmac_tv_template)
4697                 }
4698         }, {
4699                 .alg = "cbcmac(sm4)",
4700                 .test = alg_test_hash,
4701                 .suite = {
4702                         .hash = __VECS(sm4_cbcmac_tv_template)
4703                 }
4704         }, {
4705                 .alg = "ccm(aes)",
4706                 .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",
4707                 .test = alg_test_aead,
4708                 .fips_allowed = 1,
4709                 .suite = {
4710                         .aead = {
4711                                 ____VECS(aes_ccm_tv_template),
4712                                 .einval_allowed = 1,
4713                         }
4714                 }
4715         }, {
4716                 .alg = "ccm(sm4)",
4717                 .generic_driver = "ccm_base(ctr(sm4-generic),cbcmac(sm4-generic))",
4718                 .test = alg_test_aead,
4719                 .suite = {
4720                         .aead = {
4721                                 ____VECS(sm4_ccm_tv_template),
4722                                 .einval_allowed = 1,
4723                         }
4724                 }
4725         }, {
4726                 .alg = "chacha20",
4727                 .test = alg_test_skcipher,
4728                 .suite = {
4729                         .cipher = __VECS(chacha20_tv_template)
4730                 },
4731         }, {
4732                 .alg = "cmac(aes)",
4733                 .fips_allowed = 1,
4734                 .test = alg_test_hash,
4735                 .suite = {
4736                         .hash = __VECS(aes_cmac128_tv_template)
4737                 }
4738         }, {
4739                 .alg = "cmac(camellia)",
4740                 .test = alg_test_hash,
4741                 .suite = {
4742                         .hash = __VECS(camellia_cmac128_tv_template)
4743                 }
4744         }, {
4745                 .alg = "cmac(des3_ede)",
4746                 .test = alg_test_hash,
4747                 .suite = {
4748                         .hash = __VECS(des3_ede_cmac64_tv_template)
4749                 }
4750         }, {
4751                 .alg = "cmac(sm4)",
4752                 .test = alg_test_hash,
4753                 .suite = {
4754                         .hash = __VECS(sm4_cmac128_tv_template)
4755                 }
4756         }, {
4757                 .alg = "compress_null",
4758                 .test = alg_test_null,
4759         }, {
4760                 .alg = "crc32",
4761                 .test = alg_test_hash,
4762                 .fips_allowed = 1,
4763                 .suite = {
4764                         .hash = __VECS(crc32_tv_template)
4765                 }
4766         }, {
4767                 .alg = "crc32c",
4768                 .test = alg_test_crc32c,
4769                 .fips_allowed = 1,
4770                 .suite = {
4771                         .hash = __VECS(crc32c_tv_template)
4772                 }
4773         }, {
4774                 .alg = "crc64-rocksoft",
4775                 .test = alg_test_hash,
4776                 .fips_allowed = 1,
4777                 .suite = {
4778                         .hash = __VECS(crc64_rocksoft_tv_template)
4779                 }
4780         }, {
4781                 .alg = "crct10dif",
4782                 .test = alg_test_hash,
4783                 .fips_allowed = 1,
4784                 .suite = {
4785                         .hash = __VECS(crct10dif_tv_template)
4786                 }
4787         }, {
4788                 .alg = "ctr(aes)",
4789                 .test = alg_test_skcipher,
4790                 .fips_allowed = 1,
4791                 .suite = {
4792                         .cipher = __VECS(aes_ctr_tv_template)
4793                 }
4794         }, {
4795                 .alg = "ctr(aria)",
4796                 .test = alg_test_skcipher,
4797                 .suite = {
4798                         .cipher = __VECS(aria_ctr_tv_template)
4799                 }
4800         }, {
4801                 .alg = "ctr(blowfish)",
4802                 .test = alg_test_skcipher,
4803                 .suite = {
4804                         .cipher = __VECS(bf_ctr_tv_template)
4805                 }
4806         }, {
4807                 .alg = "ctr(camellia)",
4808                 .test = alg_test_skcipher,
4809                 .suite = {
4810                         .cipher = __VECS(camellia_ctr_tv_template)
4811                 }
4812         }, {
4813                 .alg = "ctr(cast5)",
4814                 .test = alg_test_skcipher,
4815                 .suite = {
4816                         .cipher = __VECS(cast5_ctr_tv_template)
4817                 }
4818         }, {
4819                 .alg = "ctr(cast6)",
4820                 .test = alg_test_skcipher,
4821                 .suite = {
4822                         .cipher = __VECS(cast6_ctr_tv_template)
4823                 }
4824         }, {
4825                 .alg = "ctr(des)",
4826                 .test = alg_test_skcipher,
4827                 .suite = {
4828                         .cipher = __VECS(des_ctr_tv_template)
4829                 }
4830         }, {
4831                 .alg = "ctr(des3_ede)",
4832                 .test = alg_test_skcipher,
4833                 .suite = {
4834                         .cipher = __VECS(des3_ede_ctr_tv_template)
4835                 }
4836         }, {
4837                 /* Same as ctr(aes) except the key is stored in
4838                  * hardware secure memory which we reference by index
4839                  */
4840                 .alg = "ctr(paes)",
4841                 .test = alg_test_null,
4842                 .fips_allowed = 1,
4843         }, {
4844
4845                 /* Same as ctr(sm4) except the key is stored in
4846                  * hardware secure memory which we reference by index
4847                  */
4848                 .alg = "ctr(psm4)",
4849                 .test = alg_test_null,
4850         }, {
4851                 .alg = "ctr(serpent)",
4852                 .test = alg_test_skcipher,
4853                 .suite = {
4854                         .cipher = __VECS(serpent_ctr_tv_template)
4855                 }
4856         }, {
4857                 .alg = "ctr(sm4)",
4858                 .test = alg_test_skcipher,
4859                 .suite = {
4860                         .cipher = __VECS(sm4_ctr_tv_template)
4861                 }
4862         }, {
4863                 .alg = "ctr(twofish)",
4864                 .test = alg_test_skcipher,
4865                 .suite = {
4866                         .cipher = __VECS(tf_ctr_tv_template)
4867                 }
4868         }, {
4869 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4870                 .alg = "ctr-paes-s390",
4871                 .fips_allowed = 1,
4872                 .test = alg_test_skcipher,
4873                 .suite = {
4874                         .cipher = __VECS(aes_ctr_tv_template)
4875                 }
4876         }, {
4877 #endif
4878                 .alg = "cts(cbc(aes))",
4879                 .test = alg_test_skcipher,
4880                 .fips_allowed = 1,
4881                 .suite = {
4882                         .cipher = __VECS(cts_mode_tv_template)
4883                 }
4884         }, {
4885                 /* Same as cts(cbc((aes)) except the key is stored in
4886                  * hardware secure memory which we reference by index
4887                  */
4888                 .alg = "cts(cbc(paes))",
4889                 .test = alg_test_null,
4890                 .fips_allowed = 1,
4891         }, {
4892                 .alg = "cts(cbc(sm4))",
4893                 .test = alg_test_skcipher,
4894                 .suite = {
4895                         .cipher = __VECS(sm4_cts_tv_template)
4896                 }
4897         }, {
4898                 .alg = "curve25519",
4899                 .test = alg_test_kpp,
4900                 .suite = {
4901                         .kpp = __VECS(curve25519_tv_template)
4902                 }
4903         }, {
4904                 .alg = "deflate",
4905                 .test = alg_test_comp,
4906                 .fips_allowed = 1,
4907                 .suite = {
4908                         .comp = {
4909                                 .comp = __VECS(deflate_comp_tv_template),
4910                                 .decomp = __VECS(deflate_decomp_tv_template)
4911                         }
4912                 }
4913         }, {
4914                 .alg = "deflate-iaa",
4915                 .test = alg_test_comp,
4916                 .fips_allowed = 1,
4917                 .suite = {
4918                         .comp = {
4919                                 .comp = __VECS(deflate_comp_tv_template),
4920                                 .decomp = __VECS(deflate_decomp_tv_template)
4921                         }
4922                 }
4923         }, {
4924                 .alg = "dh",
4925                 .test = alg_test_kpp,
4926                 .suite = {
4927                         .kpp = __VECS(dh_tv_template)
4928                 }
4929         }, {
4930                 .alg = "digest_null",
4931                 .test = alg_test_null,
4932         }, {
4933                 .alg = "drbg_nopr_ctr_aes128",
4934                 .test = alg_test_drbg,
4935                 .fips_allowed = 1,
4936                 .suite = {
4937                         .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
4938                 }
4939         }, {
4940                 .alg = "drbg_nopr_ctr_aes192",
4941                 .test = alg_test_drbg,
4942                 .fips_allowed = 1,
4943                 .suite = {
4944                         .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
4945                 }
4946         }, {
4947                 .alg = "drbg_nopr_ctr_aes256",
4948                 .test = alg_test_drbg,
4949                 .fips_allowed = 1,
4950                 .suite = {
4951                         .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
4952                 }
4953         }, {
4954                 .alg = "drbg_nopr_hmac_sha256",
4955                 .test = alg_test_drbg,
4956                 .fips_allowed = 1,
4957                 .suite = {
4958                         .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
4959                 }
4960         }, {
4961                 /*
4962                  * There is no need to specifically test the DRBG with every
4963                  * backend cipher -- covered by drbg_nopr_hmac_sha512 test
4964                  */
4965                 .alg = "drbg_nopr_hmac_sha384",
4966                 .test = alg_test_null,
4967         }, {
4968                 .alg = "drbg_nopr_hmac_sha512",
4969                 .test = alg_test_drbg,
4970                 .fips_allowed = 1,
4971                 .suite = {
4972                         .drbg = __VECS(drbg_nopr_hmac_sha512_tv_template)
4973                 }
4974         }, {
4975                 .alg = "drbg_nopr_sha256",
4976                 .test = alg_test_drbg,
4977                 .fips_allowed = 1,
4978                 .suite = {
4979                         .drbg = __VECS(drbg_nopr_sha256_tv_template)
4980                 }
4981         }, {
4982                 /* covered by drbg_nopr_sha256 test */
4983                 .alg = "drbg_nopr_sha384",
4984                 .test = alg_test_null,
4985         }, {
4986                 .alg = "drbg_nopr_sha512",
4987                 .fips_allowed = 1,
4988                 .test = alg_test_null,
4989         }, {
4990                 .alg = "drbg_pr_ctr_aes128",
4991                 .test = alg_test_drbg,
4992                 .fips_allowed = 1,
4993                 .suite = {
4994                         .drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
4995                 }
4996         }, {
4997                 /* covered by drbg_pr_ctr_aes128 test */
4998                 .alg = "drbg_pr_ctr_aes192",
4999                 .fips_allowed = 1,
5000                 .test = alg_test_null,
5001         }, {
5002                 .alg = "drbg_pr_ctr_aes256",
5003                 .fips_allowed = 1,
5004                 .test = alg_test_null,
5005         }, {
5006                 .alg = "drbg_pr_hmac_sha256",
5007                 .test = alg_test_drbg,
5008                 .fips_allowed = 1,
5009                 .suite = {
5010                         .drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
5011                 }
5012         }, {
5013                 /* covered by drbg_pr_hmac_sha256 test */
5014                 .alg = "drbg_pr_hmac_sha384",
5015                 .test = alg_test_null,
5016         }, {
5017                 .alg = "drbg_pr_hmac_sha512",
5018                 .test = alg_test_null,
5019                 .fips_allowed = 1,
5020         }, {
5021                 .alg = "drbg_pr_sha256",
5022                 .test = alg_test_drbg,
5023                 .fips_allowed = 1,
5024                 .suite = {
5025                         .drbg = __VECS(drbg_pr_sha256_tv_template)
5026                 }
5027         }, {
5028                 /* covered by drbg_pr_sha256 test */
5029                 .alg = "drbg_pr_sha384",
5030                 .test = alg_test_null,
5031         }, {
5032                 .alg = "drbg_pr_sha512",
5033                 .fips_allowed = 1,
5034                 .test = alg_test_null,
5035         }, {
5036                 .alg = "ecb(aes)",
5037                 .test = alg_test_skcipher,
5038                 .fips_allowed = 1,
5039                 .suite = {
5040                         .cipher = __VECS(aes_tv_template)
5041                 }
5042         }, {
5043                 .alg = "ecb(anubis)",
5044                 .test = alg_test_skcipher,
5045                 .suite = {
5046                         .cipher = __VECS(anubis_tv_template)
5047                 }
5048         }, {
5049                 .alg = "ecb(arc4)",
5050                 .generic_driver = "arc4-generic",
5051                 .test = alg_test_skcipher,
5052                 .suite = {
5053                         .cipher = __VECS(arc4_tv_template)
5054                 }
5055         }, {
5056                 .alg = "ecb(aria)",
5057                 .test = alg_test_skcipher,
5058                 .suite = {
5059                         .cipher = __VECS(aria_tv_template)
5060                 }
5061         }, {
5062                 .alg = "ecb(blowfish)",
5063                 .test = alg_test_skcipher,
5064                 .suite = {
5065                         .cipher = __VECS(bf_tv_template)
5066                 }
5067         }, {
5068                 .alg = "ecb(camellia)",
5069                 .test = alg_test_skcipher,
5070                 .suite = {
5071                         .cipher = __VECS(camellia_tv_template)
5072                 }
5073         }, {
5074                 .alg = "ecb(cast5)",
5075                 .test = alg_test_skcipher,
5076                 .suite = {
5077                         .cipher = __VECS(cast5_tv_template)
5078                 }
5079         }, {
5080                 .alg = "ecb(cast6)",
5081                 .test = alg_test_skcipher,
5082                 .suite = {
5083                         .cipher = __VECS(cast6_tv_template)
5084                 }
5085         }, {
5086                 .alg = "ecb(cipher_null)",
5087                 .test = alg_test_null,
5088                 .fips_allowed = 1,
5089         }, {
5090                 .alg = "ecb(des)",
5091                 .test = alg_test_skcipher,
5092                 .suite = {
5093                         .cipher = __VECS(des_tv_template)
5094                 }
5095         }, {
5096                 .alg = "ecb(des3_ede)",
5097                 .test = alg_test_skcipher,
5098                 .suite = {
5099                         .cipher = __VECS(des3_ede_tv_template)
5100                 }
5101         }, {
5102                 .alg = "ecb(fcrypt)",
5103                 .test = alg_test_skcipher,
5104                 .suite = {
5105                         .cipher = {
5106                                 .vecs = fcrypt_pcbc_tv_template,
5107                                 .count = 1
5108                         }
5109                 }
5110         }, {
5111                 .alg = "ecb(khazad)",
5112                 .test = alg_test_skcipher,
5113                 .suite = {
5114                         .cipher = __VECS(khazad_tv_template)
5115                 }
5116         }, {
5117                 /* Same as ecb(aes) except the key is stored in
5118                  * hardware secure memory which we reference by index
5119                  */
5120                 .alg = "ecb(paes)",
5121                 .test = alg_test_null,
5122                 .fips_allowed = 1,
5123         }, {
5124                 .alg = "ecb(seed)",
5125                 .test = alg_test_skcipher,
5126                 .suite = {
5127                         .cipher = __VECS(seed_tv_template)
5128                 }
5129         }, {
5130                 .alg = "ecb(serpent)",
5131                 .test = alg_test_skcipher,
5132                 .suite = {
5133                         .cipher = __VECS(serpent_tv_template)
5134                 }
5135         }, {
5136                 .alg = "ecb(sm4)",
5137                 .test = alg_test_skcipher,
5138                 .suite = {
5139                         .cipher = __VECS(sm4_tv_template)
5140                 }
5141         }, {
5142                 .alg = "ecb(tea)",
5143                 .test = alg_test_skcipher,
5144                 .suite = {
5145                         .cipher = __VECS(tea_tv_template)
5146                 }
5147         }, {
5148                 .alg = "ecb(twofish)",
5149                 .test = alg_test_skcipher,
5150                 .suite = {
5151                         .cipher = __VECS(tf_tv_template)
5152                 }
5153         }, {
5154                 .alg = "ecb(xeta)",
5155                 .test = alg_test_skcipher,
5156                 .suite = {
5157                         .cipher = __VECS(xeta_tv_template)
5158                 }
5159         }, {
5160                 .alg = "ecb(xtea)",
5161                 .test = alg_test_skcipher,
5162                 .suite = {
5163                         .cipher = __VECS(xtea_tv_template)
5164                 }
5165         }, {
5166 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
5167                 .alg = "ecb-paes-s390",
5168                 .fips_allowed = 1,
5169                 .test = alg_test_skcipher,
5170                 .suite = {
5171                         .cipher = __VECS(aes_tv_template)
5172                 }
5173         }, {
5174 #endif
5175                 .alg = "ecdh-nist-p192",
5176                 .test = alg_test_kpp,
5177                 .suite = {
5178                         .kpp = __VECS(ecdh_p192_tv_template)
5179                 }
5180         }, {
5181                 .alg = "ecdh-nist-p256",
5182                 .test = alg_test_kpp,
5183                 .fips_allowed = 1,
5184                 .suite = {
5185                         .kpp = __VECS(ecdh_p256_tv_template)
5186                 }
5187         }, {
5188                 .alg = "ecdh-nist-p384",
5189                 .test = alg_test_kpp,
5190                 .fips_allowed = 1,
5191                 .suite = {
5192                         .kpp = __VECS(ecdh_p384_tv_template)
5193                 }
5194         }, {
5195                 .alg = "ecdsa-nist-p192",
5196                 .test = alg_test_sig,
5197                 .suite = {
5198                         .sig = __VECS(ecdsa_nist_p192_tv_template)
5199                 }
5200         }, {
5201                 .alg = "ecdsa-nist-p256",
5202                 .test = alg_test_sig,
5203                 .fips_allowed = 1,
5204                 .suite = {
5205                         .sig = __VECS(ecdsa_nist_p256_tv_template)
5206                 }
5207         }, {
5208                 .alg = "ecdsa-nist-p384",
5209                 .test = alg_test_sig,
5210                 .fips_allowed = 1,
5211                 .suite = {
5212                         .sig = __VECS(ecdsa_nist_p384_tv_template)
5213                 }
5214         }, {
5215                 .alg = "ecdsa-nist-p521",
5216                 .test = alg_test_sig,
5217                 .fips_allowed = 1,
5218                 .suite = {
5219                         .sig = __VECS(ecdsa_nist_p521_tv_template)
5220                 }
5221         }, {
5222                 .alg = "ecrdsa",
5223                 .test = alg_test_sig,
5224                 .suite = {
5225                         .sig = __VECS(ecrdsa_tv_template)
5226                 }
5227         }, {
5228                 .alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",
5229                 .test = alg_test_aead,
5230                 .fips_allowed = 1,
5231                 .suite = {
5232                         .aead = __VECS(essiv_hmac_sha256_aes_cbc_tv_temp)
5233                 }
5234         }, {
5235                 .alg = "essiv(cbc(aes),sha256)",
5236                 .test = alg_test_skcipher,
5237                 .fips_allowed = 1,
5238                 .suite = {
5239                         .cipher = __VECS(essiv_aes_cbc_tv_template)
5240                 }
5241         }, {
5242 #if IS_ENABLED(CONFIG_CRYPTO_DH_RFC7919_GROUPS)
5243                 .alg = "ffdhe2048(dh)",
5244                 .test = alg_test_kpp,
5245                 .fips_allowed = 1,
5246                 .suite = {
5247                         .kpp = __VECS(ffdhe2048_dh_tv_template)
5248                 }
5249         }, {
5250                 .alg = "ffdhe3072(dh)",
5251                 .test = alg_test_kpp,
5252                 .fips_allowed = 1,
5253                 .suite = {
5254                         .kpp = __VECS(ffdhe3072_dh_tv_template)
5255                 }
5256         }, {
5257                 .alg = "ffdhe4096(dh)",
5258                 .test = alg_test_kpp,
5259                 .fips_allowed = 1,
5260                 .suite = {
5261                         .kpp = __VECS(ffdhe4096_dh_tv_template)
5262                 }
5263         }, {
5264                 .alg = "ffdhe6144(dh)",
5265                 .test = alg_test_kpp,
5266                 .fips_allowed = 1,
5267                 .suite = {
5268                         .kpp = __VECS(ffdhe6144_dh_tv_template)
5269                 }
5270         }, {
5271                 .alg = "ffdhe8192(dh)",
5272                 .test = alg_test_kpp,
5273                 .fips_allowed = 1,
5274                 .suite = {
5275                         .kpp = __VECS(ffdhe8192_dh_tv_template)
5276                 }
5277         }, {
5278 #endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */
5279                 .alg = "gcm(aes)",
5280                 .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",
5281                 .test = alg_test_aead,
5282                 .fips_allowed = 1,
5283                 .suite = {
5284                         .aead = __VECS(aes_gcm_tv_template)
5285                 }
5286         }, {
5287                 .alg = "gcm(aria)",
5288                 .generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)",
5289                 .test = alg_test_aead,
5290                 .suite = {
5291                         .aead = __VECS(aria_gcm_tv_template)
5292                 }
5293         }, {
5294                 .alg = "gcm(sm4)",
5295                 .generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)",
5296                 .test = alg_test_aead,
5297                 .suite = {
5298                         .aead = __VECS(sm4_gcm_tv_template)
5299                 }
5300         }, {
5301                 .alg = "ghash",
5302                 .test = alg_test_hash,
5303                 .suite = {
5304                         .hash = __VECS(ghash_tv_template)
5305                 }
5306         }, {
5307                 .alg = "hctr2(aes)",
5308                 .generic_driver =
5309                     "hctr2_base(xctr(aes-generic),polyval-generic)",
5310                 .test = alg_test_skcipher,
5311                 .suite = {
5312                         .cipher = __VECS(aes_hctr2_tv_template)
5313                 }
5314         }, {
5315                 .alg = "hmac(md5)",
5316                 .test = alg_test_hash,
5317                 .suite = {
5318                         .hash = __VECS(hmac_md5_tv_template)
5319                 }
5320         }, {
5321                 .alg = "hmac(rmd160)",
5322                 .test = alg_test_hash,
5323                 .suite = {
5324                         .hash = __VECS(hmac_rmd160_tv_template)
5325                 }
5326         }, {
5327                 .alg = "hmac(sha1)",
5328                 .test = alg_test_hash,
5329                 .fips_allowed = 1,
5330                 .suite = {
5331                         .hash = __VECS(hmac_sha1_tv_template)
5332                 }
5333         }, {
5334                 .alg = "hmac(sha224)",
5335                 .test = alg_test_hash,
5336                 .fips_allowed = 1,
5337                 .suite = {
5338                         .hash = __VECS(hmac_sha224_tv_template)
5339                 }
5340         }, {
5341                 .alg = "hmac(sha256)",
5342                 .test = alg_test_hash,
5343                 .fips_allowed = 1,
5344                 .suite = {
5345                         .hash = __VECS(hmac_sha256_tv_template)
5346                 }
5347         }, {
5348                 .alg = "hmac(sha3-224)",
5349                 .test = alg_test_hash,
5350                 .fips_allowed = 1,
5351                 .suite = {
5352                         .hash = __VECS(hmac_sha3_224_tv_template)
5353                 }
5354         }, {
5355                 .alg = "hmac(sha3-256)",
5356                 .test = alg_test_hash,
5357                 .fips_allowed = 1,
5358                 .suite = {
5359                         .hash = __VECS(hmac_sha3_256_tv_template)
5360                 }
5361         }, {
5362                 .alg = "hmac(sha3-384)",
5363                 .test = alg_test_hash,
5364                 .fips_allowed = 1,
5365                 .suite = {
5366                         .hash = __VECS(hmac_sha3_384_tv_template)
5367                 }
5368         }, {
5369                 .alg = "hmac(sha3-512)",
5370                 .test = alg_test_hash,
5371                 .fips_allowed = 1,
5372                 .suite = {
5373                         .hash = __VECS(hmac_sha3_512_tv_template)
5374                 }
5375         }, {
5376                 .alg = "hmac(sha384)",
5377                 .test = alg_test_hash,
5378                 .fips_allowed = 1,
5379                 .suite = {
5380                         .hash = __VECS(hmac_sha384_tv_template)
5381                 }
5382         }, {
5383                 .alg = "hmac(sha512)",
5384                 .test = alg_test_hash,
5385                 .fips_allowed = 1,
5386                 .suite = {
5387                         .hash = __VECS(hmac_sha512_tv_template)
5388                 }
5389         }, {
5390                 .alg = "hmac(sm3)",
5391                 .test = alg_test_hash,
5392                 .suite = {
5393                         .hash = __VECS(hmac_sm3_tv_template)
5394                 }
5395         }, {
5396                 .alg = "hmac(streebog256)",
5397                 .test = alg_test_hash,
5398                 .suite = {
5399                         .hash = __VECS(hmac_streebog256_tv_template)
5400                 }
5401         }, {
5402                 .alg = "hmac(streebog512)",
5403                 .test = alg_test_hash,
5404                 .suite = {
5405                         .hash = __VECS(hmac_streebog512_tv_template)
5406                 }
5407         }, {
5408                 .alg = "jitterentropy_rng",
5409                 .fips_allowed = 1,
5410                 .test = alg_test_null,
5411         }, {
5412                 .alg = "kw(aes)",
5413                 .test = alg_test_skcipher,
5414                 .fips_allowed = 1,
5415                 .suite = {
5416                         .cipher = __VECS(aes_kw_tv_template)
5417                 }
5418         }, {
5419                 .alg = "lrw(aes)",
5420                 .generic_driver = "lrw(ecb(aes-generic))",
5421                 .test = alg_test_skcipher,
5422                 .suite = {
5423                         .cipher = __VECS(aes_lrw_tv_template)
5424                 }
5425         }, {
5426                 .alg = "lrw(camellia)",
5427                 .generic_driver = "lrw(ecb(camellia-generic))",
5428                 .test = alg_test_skcipher,
5429                 .suite = {
5430                         .cipher = __VECS(camellia_lrw_tv_template)
5431                 }
5432         }, {
5433                 .alg = "lrw(cast6)",
5434                 .generic_driver = "lrw(ecb(cast6-generic))",
5435                 .test = alg_test_skcipher,
5436                 .suite = {
5437                         .cipher = __VECS(cast6_lrw_tv_template)
5438                 }
5439         }, {
5440                 .alg = "lrw(serpent)",
5441                 .generic_driver = "lrw(ecb(serpent-generic))",
5442                 .test = alg_test_skcipher,
5443                 .suite = {
5444                         .cipher = __VECS(serpent_lrw_tv_template)
5445                 }
5446         }, {
5447                 .alg = "lrw(twofish)",
5448                 .generic_driver = "lrw(ecb(twofish-generic))",
5449                 .test = alg_test_skcipher,
5450                 .suite = {
5451                         .cipher = __VECS(tf_lrw_tv_template)
5452                 }
5453         }, {
5454                 .alg = "lz4",
5455                 .test = alg_test_comp,
5456                 .fips_allowed = 1,
5457                 .suite = {
5458                         .comp = {
5459                                 .comp = __VECS(lz4_comp_tv_template),
5460                                 .decomp = __VECS(lz4_decomp_tv_template)
5461                         }
5462                 }
5463         }, {
5464                 .alg = "lz4hc",
5465                 .test = alg_test_comp,
5466                 .fips_allowed = 1,
5467                 .suite = {
5468                         .comp = {
5469                                 .comp = __VECS(lz4hc_comp_tv_template),
5470                                 .decomp = __VECS(lz4hc_decomp_tv_template)
5471                         }
5472                 }
5473         }, {
5474                 .alg = "lzo",
5475                 .test = alg_test_comp,
5476                 .fips_allowed = 1,
5477                 .suite = {
5478                         .comp = {
5479                                 .comp = __VECS(lzo_comp_tv_template),
5480                                 .decomp = __VECS(lzo_decomp_tv_template)
5481                         }
5482                 }
5483         }, {
5484                 .alg = "lzo-rle",
5485                 .test = alg_test_comp,
5486                 .fips_allowed = 1,
5487                 .suite = {
5488                         .comp = {
5489                                 .comp = __VECS(lzorle_comp_tv_template),
5490                                 .decomp = __VECS(lzorle_decomp_tv_template)
5491                         }
5492                 }
5493         }, {
5494                 .alg = "md4",
5495                 .test = alg_test_hash,
5496                 .suite = {
5497                         .hash = __VECS(md4_tv_template)
5498                 }
5499         }, {
5500                 .alg = "md5",
5501                 .test = alg_test_hash,
5502                 .suite = {
5503                         .hash = __VECS(md5_tv_template)
5504                 }
5505         }, {
5506                 .alg = "michael_mic",
5507                 .test = alg_test_hash,
5508                 .suite = {
5509                         .hash = __VECS(michael_mic_tv_template)
5510                 }
5511         }, {
5512                 .alg = "nhpoly1305",
5513                 .test = alg_test_hash,
5514                 .suite = {
5515                         .hash = __VECS(nhpoly1305_tv_template)
5516                 }
5517         }, {
5518                 .alg = "p1363(ecdsa-nist-p192)",
5519                 .test = alg_test_null,
5520         }, {
5521                 .alg = "p1363(ecdsa-nist-p256)",
5522                 .test = alg_test_sig,
5523                 .fips_allowed = 1,
5524                 .suite = {
5525                         .sig = __VECS(p1363_ecdsa_nist_p256_tv_template)
5526                 }
5527         }, {
5528                 .alg = "p1363(ecdsa-nist-p384)",
5529                 .test = alg_test_null,
5530                 .fips_allowed = 1,
5531         }, {
5532                 .alg = "p1363(ecdsa-nist-p521)",
5533                 .test = alg_test_null,
5534                 .fips_allowed = 1,
5535         }, {
5536                 .alg = "pcbc(fcrypt)",
5537                 .test = alg_test_skcipher,
5538                 .suite = {
5539                         .cipher = __VECS(fcrypt_pcbc_tv_template)
5540                 }
5541         }, {
5542                 .alg = "pkcs1(rsa,none)",
5543                 .test = alg_test_sig,
5544                 .suite = {
5545                         .sig = __VECS(pkcs1_rsa_none_tv_template)
5546                 }
5547         }, {
5548                 .alg = "pkcs1(rsa,sha224)",
5549                 .test = alg_test_null,
5550                 .fips_allowed = 1,
5551         }, {
5552                 .alg = "pkcs1(rsa,sha256)",
5553                 .test = alg_test_sig,
5554                 .fips_allowed = 1,
5555                 .suite = {
5556                         .sig = __VECS(pkcs1_rsa_tv_template)
5557                 }
5558         }, {
5559                 .alg = "pkcs1(rsa,sha3-256)",
5560                 .test = alg_test_null,
5561                 .fips_allowed = 1,
5562         }, {
5563                 .alg = "pkcs1(rsa,sha3-384)",
5564                 .test = alg_test_null,
5565                 .fips_allowed = 1,
5566         }, {
5567                 .alg = "pkcs1(rsa,sha3-512)",
5568                 .test = alg_test_null,
5569                 .fips_allowed = 1,
5570         }, {
5571                 .alg = "pkcs1(rsa,sha384)",
5572                 .test = alg_test_null,
5573                 .fips_allowed = 1,
5574         }, {
5575                 .alg = "pkcs1(rsa,sha512)",
5576                 .test = alg_test_null,
5577                 .fips_allowed = 1,
5578         }, {
5579                 .alg = "pkcs1pad(rsa)",
5580                 .test = alg_test_null,
5581                 .fips_allowed = 1,
5582         }, {
5583                 .alg = "poly1305",
5584                 .test = alg_test_hash,
5585                 .suite = {
5586                         .hash = __VECS(poly1305_tv_template)
5587                 }
5588         }, {
5589                 .alg = "polyval",
5590                 .test = alg_test_hash,
5591                 .suite = {
5592                         .hash = __VECS(polyval_tv_template)
5593                 }
5594         }, {
5595                 .alg = "rfc3686(ctr(aes))",
5596                 .test = alg_test_skcipher,
5597                 .fips_allowed = 1,
5598                 .suite = {
5599                         .cipher = __VECS(aes_ctr_rfc3686_tv_template)
5600                 }
5601         }, {
5602                 .alg = "rfc3686(ctr(sm4))",
5603                 .test = alg_test_skcipher,
5604                 .suite = {
5605                         .cipher = __VECS(sm4_ctr_rfc3686_tv_template)
5606                 }
5607         }, {
5608                 .alg = "rfc4106(gcm(aes))",
5609                 .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",
5610                 .test = alg_test_aead,
5611                 .fips_allowed = 1,
5612                 .suite = {
5613                         .aead = {
5614                                 ____VECS(aes_gcm_rfc4106_tv_template),
5615                                 .einval_allowed = 1,
5616                                 .aad_iv = 1,
5617                         }
5618                 }
5619         }, {
5620                 .alg = "rfc4309(ccm(aes))",
5621                 .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",
5622                 .test = alg_test_aead,
5623                 .fips_allowed = 1,
5624                 .suite = {
5625                         .aead = {
5626                                 ____VECS(aes_ccm_rfc4309_tv_template),
5627                                 .einval_allowed = 1,
5628                                 .aad_iv = 1,
5629                         }
5630                 }
5631         }, {
5632                 .alg = "rfc4543(gcm(aes))",
5633                 .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",
5634                 .test = alg_test_aead,
5635                 .suite = {
5636                         .aead = {
5637                                 ____VECS(aes_gcm_rfc4543_tv_template),
5638                                 .einval_allowed = 1,
5639                                 .aad_iv = 1,
5640                         }
5641                 }
5642         }, {
5643                 .alg = "rfc7539(chacha20,poly1305)",
5644                 .test = alg_test_aead,
5645                 .suite = {
5646                         .aead = __VECS(rfc7539_tv_template)
5647                 }
5648         }, {
5649                 .alg = "rfc7539esp(chacha20,poly1305)",
5650                 .test = alg_test_aead,
5651                 .suite = {
5652                         .aead = {
5653                                 ____VECS(rfc7539esp_tv_template),
5654                                 .einval_allowed = 1,
5655                                 .aad_iv = 1,
5656                         }
5657                 }
5658         }, {
5659                 .alg = "rmd160",
5660                 .test = alg_test_hash,
5661                 .suite = {
5662                         .hash = __VECS(rmd160_tv_template)
5663                 }
5664         }, {
5665                 .alg = "rsa",
5666                 .test = alg_test_akcipher,
5667                 .fips_allowed = 1,
5668                 .suite = {
5669                         .akcipher = __VECS(rsa_tv_template)
5670                 }
5671         }, {
5672                 .alg = "sha1",
5673                 .test = alg_test_hash,
5674                 .fips_allowed = 1,
5675                 .suite = {
5676                         .hash = __VECS(sha1_tv_template)
5677                 }
5678         }, {
5679                 .alg = "sha224",
5680                 .test = alg_test_hash,
5681                 .fips_allowed = 1,
5682                 .suite = {
5683                         .hash = __VECS(sha224_tv_template)
5684                 }
5685         }, {
5686                 .alg = "sha256",
5687                 .test = alg_test_hash,
5688                 .fips_allowed = 1,
5689                 .suite = {
5690                         .hash = __VECS(sha256_tv_template)
5691                 }
5692         }, {
5693                 .alg = "sha3-224",
5694                 .test = alg_test_hash,
5695                 .fips_allowed = 1,
5696                 .suite = {
5697                         .hash = __VECS(sha3_224_tv_template)
5698                 }
5699         }, {
5700                 .alg = "sha3-256",
5701                 .test = alg_test_hash,
5702                 .fips_allowed = 1,
5703                 .suite = {
5704                         .hash = __VECS(sha3_256_tv_template)
5705                 }
5706         }, {
5707                 .alg = "sha3-384",
5708                 .test = alg_test_hash,
5709                 .fips_allowed = 1,
5710                 .suite = {
5711                         .hash = __VECS(sha3_384_tv_template)
5712                 }
5713         }, {
5714                 .alg = "sha3-512",
5715                 .test = alg_test_hash,
5716                 .fips_allowed = 1,
5717                 .suite = {
5718                         .hash = __VECS(sha3_512_tv_template)
5719                 }
5720         }, {
5721                 .alg = "sha384",
5722                 .test = alg_test_hash,
5723                 .fips_allowed = 1,
5724                 .suite = {
5725                         .hash = __VECS(sha384_tv_template)
5726                 }
5727         }, {
5728                 .alg = "sha512",
5729                 .test = alg_test_hash,
5730                 .fips_allowed = 1,
5731                 .suite = {
5732                         .hash = __VECS(sha512_tv_template)
5733                 }
5734         }, {
5735                 .alg = "sm3",
5736                 .test = alg_test_hash,
5737                 .suite = {
5738                         .hash = __VECS(sm3_tv_template)
5739                 }
5740         }, {
5741                 .alg = "streebog256",
5742                 .test = alg_test_hash,
5743                 .suite = {
5744                         .hash = __VECS(streebog256_tv_template)
5745                 }
5746         }, {
5747                 .alg = "streebog512",
5748                 .test = alg_test_hash,
5749                 .suite = {
5750                         .hash = __VECS(streebog512_tv_template)
5751                 }
5752         }, {
5753                 .alg = "vmac64(aes)",
5754                 .test = alg_test_hash,
5755                 .suite = {
5756                         .hash = __VECS(vmac64_aes_tv_template)
5757                 }
5758         }, {
5759                 .alg = "wp256",
5760                 .test = alg_test_hash,
5761                 .suite = {
5762                         .hash = __VECS(wp256_tv_template)
5763                 }
5764         }, {
5765                 .alg = "wp384",
5766                 .test = alg_test_hash,
5767                 .suite = {
5768                         .hash = __VECS(wp384_tv_template)
5769                 }
5770         }, {
5771                 .alg = "wp512",
5772                 .test = alg_test_hash,
5773                 .suite = {
5774                         .hash = __VECS(wp512_tv_template)
5775                 }
5776         }, {
5777                 .alg = "x962(ecdsa-nist-p192)",
5778                 .test = alg_test_sig,
5779                 .suite = {
5780                         .sig = __VECS(x962_ecdsa_nist_p192_tv_template)
5781                 }
5782         }, {
5783                 .alg = "x962(ecdsa-nist-p256)",
5784                 .test = alg_test_sig,
5785                 .fips_allowed = 1,
5786                 .suite = {
5787                         .sig = __VECS(x962_ecdsa_nist_p256_tv_template)
5788                 }
5789         }, {
5790                 .alg = "x962(ecdsa-nist-p384)",
5791                 .test = alg_test_sig,
5792                 .fips_allowed = 1,
5793                 .suite = {
5794                         .sig = __VECS(x962_ecdsa_nist_p384_tv_template)
5795                 }
5796         }, {
5797                 .alg = "x962(ecdsa-nist-p521)",
5798                 .test = alg_test_sig,
5799                 .fips_allowed = 1,
5800                 .suite = {
5801                         .sig = __VECS(x962_ecdsa_nist_p521_tv_template)
5802                 }
5803         }, {
5804                 .alg = "xcbc(aes)",
5805                 .test = alg_test_hash,
5806                 .suite = {
5807                         .hash = __VECS(aes_xcbc128_tv_template)
5808                 }
5809         }, {
5810                 .alg = "xcbc(sm4)",
5811                 .test = alg_test_hash,
5812                 .suite = {
5813                         .hash = __VECS(sm4_xcbc128_tv_template)
5814                 }
5815         }, {
5816                 .alg = "xchacha12",
5817                 .test = alg_test_skcipher,
5818                 .suite = {
5819                         .cipher = __VECS(xchacha12_tv_template)
5820                 },
5821         }, {
5822                 .alg = "xchacha20",
5823                 .test = alg_test_skcipher,
5824                 .suite = {
5825                         .cipher = __VECS(xchacha20_tv_template)
5826                 },
5827         }, {
5828                 .alg = "xctr(aes)",
5829                 .test = alg_test_skcipher,
5830                 .suite = {
5831                         .cipher = __VECS(aes_xctr_tv_template)
5832                 }
5833         }, {
5834                 .alg = "xts(aes)",
5835                 .generic_driver = "xts(ecb(aes-generic))",
5836                 .test = alg_test_skcipher,
5837                 .fips_allowed = 1,
5838                 .suite = {
5839                         .cipher = __VECS(aes_xts_tv_template)
5840                 }
5841         }, {
5842                 .alg = "xts(camellia)",
5843                 .generic_driver = "xts(ecb(camellia-generic))",
5844                 .test = alg_test_skcipher,
5845                 .suite = {
5846                         .cipher = __VECS(camellia_xts_tv_template)
5847                 }
5848         }, {
5849                 .alg = "xts(cast6)",
5850                 .generic_driver = "xts(ecb(cast6-generic))",
5851                 .test = alg_test_skcipher,
5852                 .suite = {
5853                         .cipher = __VECS(cast6_xts_tv_template)
5854                 }
5855         }, {
5856                 /* Same as xts(aes) except the key is stored in
5857                  * hardware secure memory which we reference by index
5858                  */
5859                 .alg = "xts(paes)",
5860                 .test = alg_test_null,
5861                 .fips_allowed = 1,
5862         }, {
5863                 .alg = "xts(serpent)",
5864                 .generic_driver = "xts(ecb(serpent-generic))",
5865                 .test = alg_test_skcipher,
5866                 .suite = {
5867                         .cipher = __VECS(serpent_xts_tv_template)
5868                 }
5869         }, {
5870                 .alg = "xts(sm4)",
5871                 .generic_driver = "xts(ecb(sm4-generic))",
5872                 .test = alg_test_skcipher,
5873                 .suite = {
5874                         .cipher = __VECS(sm4_xts_tv_template)
5875                 }
5876         }, {
5877                 .alg = "xts(twofish)",
5878                 .generic_driver = "xts(ecb(twofish-generic))",
5879                 .test = alg_test_skcipher,
5880                 .suite = {
5881                         .cipher = __VECS(tf_xts_tv_template)
5882                 }
5883         }, {
5884 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
5885                 .alg = "xts-paes-s390",
5886                 .fips_allowed = 1,
5887                 .test = alg_test_skcipher,
5888                 .suite = {
5889                         .cipher = __VECS(aes_xts_tv_template)
5890                 }
5891         }, {
5892 #endif
5893                 .alg = "xxhash64",
5894                 .test = alg_test_hash,
5895                 .fips_allowed = 1,
5896                 .suite = {
5897                         .hash = __VECS(xxhash64_tv_template)
5898                 }
5899         }, {
5900                 .alg = "zstd",
5901                 .test = alg_test_comp,
5902                 .fips_allowed = 1,
5903                 .suite = {
5904                         .comp = {
5905                                 .comp = __VECS(zstd_comp_tv_template),
5906                                 .decomp = __VECS(zstd_decomp_tv_template)
5907                         }
5908                 }
5909         }
5910 };
5911
5912 static void alg_check_test_descs_order(void)
5913 {
5914         int i;
5915
5916         for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {
5917                 int diff = strcmp(alg_test_descs[i - 1].alg,
5918                                   alg_test_descs[i].alg);
5919
5920                 if (WARN_ON(diff > 0)) {
5921                         pr_warn("testmgr: alg_test_descs entries in wrong order: '%s' before '%s'\n",
5922                                 alg_test_descs[i - 1].alg,
5923                                 alg_test_descs[i].alg);
5924                 }
5925
5926                 if (WARN_ON(diff == 0)) {
5927                         pr_warn("testmgr: duplicate alg_test_descs entry: '%s'\n",
5928                                 alg_test_descs[i].alg);
5929                 }
5930         }
5931 }
5932
5933 static void alg_check_testvec_configs(void)
5934 {
5935         int i;
5936
5937         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)
5938                 WARN_ON(!valid_testvec_config(
5939                                 &default_cipher_testvec_configs[i]));
5940
5941         for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)
5942                 WARN_ON(!valid_testvec_config(
5943                                 &default_hash_testvec_configs[i]));
5944 }
5945
5946 static void testmgr_onetime_init(void)
5947 {
5948         alg_check_test_descs_order();
5949         alg_check_testvec_configs();
5950
5951 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
5952         pr_warn("alg: extra crypto tests enabled.  This is intended for developer use only.\n");
5953 #endif
5954 }
5955
5956 static int alg_find_test(const char *alg)
5957 {
5958         int start = 0;
5959         int end = ARRAY_SIZE(alg_test_descs);
5960
5961         while (start < end) {
5962                 int i = (start + end) / 2;
5963                 int diff = strcmp(alg_test_descs[i].alg, alg);
5964
5965                 if (diff > 0) {
5966                         end = i;
5967                         continue;
5968                 }
5969
5970                 if (diff < 0) {
5971                         start = i + 1;
5972                         continue;
5973                 }
5974
5975                 return i;
5976         }
5977
5978         return -1;
5979 }
5980
5981 static int alg_fips_disabled(const char *driver, const char *alg)
5982 {
5983         pr_info("alg: %s (%s) is disabled due to FIPS\n", alg, driver);
5984
5985         return -ECANCELED;
5986 }
5987
5988 int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
5989 {
5990         int i;
5991         int j;
5992         int rc;
5993
5994         if (!fips_enabled && notests) {
5995                 printk_once(KERN_INFO "alg: self-tests disabled\n");
5996                 return 0;
5997         }
5998
5999         DO_ONCE(testmgr_onetime_init);
6000
6001         if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) {
6002                 char nalg[CRYPTO_MAX_ALG_NAME];
6003
6004                 if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=
6005                     sizeof(nalg))
6006                         return -ENAMETOOLONG;
6007
6008                 i = alg_find_test(nalg);
6009                 if (i < 0)
6010                         goto notest;
6011
6012                 if (fips_enabled && !alg_test_descs[i].fips_allowed)
6013                         goto non_fips_alg;
6014
6015                 rc = alg_test_cipher(alg_test_descs + i, driver, type, mask);
6016                 goto test_done;
6017         }
6018
6019         i = alg_find_test(alg);
6020         j = alg_find_test(driver);
6021         if (i < 0 && j < 0)
6022                 goto notest;
6023
6024         if (fips_enabled) {
6025                 if (j >= 0 && !alg_test_descs[j].fips_allowed)
6026                         return -EINVAL;
6027
6028                 if (i >= 0 && !alg_test_descs[i].fips_allowed)
6029                         goto non_fips_alg;
6030         }
6031
6032         rc = 0;
6033         if (i >= 0)
6034                 rc |= alg_test_descs[i].test(alg_test_descs + i, driver,
6035                                              type, mask);
6036         if (j >= 0 && j != i)
6037                 rc |= alg_test_descs[j].test(alg_test_descs + j, driver,
6038                                              type, mask);
6039
6040 test_done:
6041         if (rc) {
6042                 if (fips_enabled || panic_on_fail) {
6043                         fips_fail_notify();
6044                         panic("alg: self-tests for %s (%s) failed in %s mode!\n",
6045                               driver, alg,
6046                               fips_enabled ? "fips" : "panic_on_fail");
6047                 }
6048                 pr_warn("alg: self-tests for %s using %s failed (rc=%d)",
6049                         alg, driver, rc);
6050                 WARN(rc != -ENOENT,
6051                      "alg: self-tests for %s using %s failed (rc=%d)",
6052                      alg, driver, rc);
6053         } else {
6054                 if (fips_enabled)
6055                         pr_info("alg: self-tests for %s (%s) passed\n",
6056                                 driver, alg);
6057         }
6058
6059         return rc;
6060
6061 notest:
6062         if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_LSKCIPHER) {
6063                 char nalg[CRYPTO_MAX_ALG_NAME];
6064
6065                 if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=
6066                     sizeof(nalg))
6067                         goto notest2;
6068
6069                 i = alg_find_test(nalg);
6070                 if (i < 0)
6071                         goto notest2;
6072
6073                 if (fips_enabled && !alg_test_descs[i].fips_allowed)
6074                         goto non_fips_alg;
6075
6076                 rc = alg_test_skcipher(alg_test_descs + i, driver, type, mask);
6077                 goto test_done;
6078         }
6079
6080 notest2:
6081         printk(KERN_INFO "alg: No test for %s (%s)\n", alg, driver);
6082
6083         if (type & CRYPTO_ALG_FIPS_INTERNAL)
6084                 return alg_fips_disabled(driver, alg);
6085
6086         return 0;
6087 non_fips_alg:
6088         return alg_fips_disabled(driver, alg);
6089 }
6090
6091 #endif /* CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */
6092
6093 EXPORT_SYMBOL_GPL(alg_test);