[linux.git] / lib / raid6 / avx512.c

// SPDX-License-Identifier: GPL-2.0-or-later
/* -*- linux-c -*- --------------------------------------------------------
 *
 *   Copyright (C) 2016 Intel Corporation
 *
 *   Author: Gayatri Kammela <[email protected]>
 *   Author: Megha Dey <[email protected]>
 *
 *   Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
 *   Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
 *
 * -----------------------------------------------------------------------
 */

/*
 * AVX512 implementation of RAID-6 syndrome functions
 *
 */

#ifdef CONFIG_AS_AVX512

#include <linux/raid/pq.h>
#include "x86.h"

static const struct raid6_avx512_constants {
	u64 x1d[8];
} raid6_avx512_constants __aligned(512/8) = {
	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
};

static int raid6_have_avx512(void)
{
	return boot_cpu_has(X86_FEATURE_AVX2) &&
		boot_cpu_has(X86_FEATURE_AVX) &&
		boot_cpu_has(X86_FEATURE_AVX512F) &&
		boot_cpu_has(X86_FEATURE_AVX512BW) &&
		boot_cpu_has(X86_FEATURE_AVX512VL) &&
		boot_cpu_has(X86_FEATURE_AVX512DQ);
}

static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

	z0 = disks - 3;         /* Highest data disk */
	p = dptr[z0+1];         /* XOR parity */
	q = dptr[z0+2];         /* RS syndrome */

	kernel_fpu_begin();

	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
		     :
		     : "m" (raid6_avx512_constants.x1d[0]));

	for (d = 0; d < bytes; d += 64) {
		asm volatile("prefetchnta %0\n\t"
			     "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
			     "prefetchnta %1\n\t"
			     "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
			     "vmovdqa64 %1,%%zmm6"
			     :
			     : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
		for (z = z0-2; z >= 0; z--) {
			asm volatile("prefetchnta %0\n\t"
				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
				     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
				     "vmovdqa64 %0,%%zmm6"
				     :
				     : "m" (dptr[z][d]));
		}
		asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
			     "vpmovm2b %%k1,%%zmm5\n\t"
			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
			     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
			     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
			     "vmovntdq %%zmm2,%0\n\t"
			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
			     "vmovntdq %%zmm4,%1\n\t"
			     "vpxorq %%zmm4,%%zmm4,%%zmm4"
			     :
			     : "m" (p[d]), "m" (q[d]));
	}

	asm volatile("sfence" : : : "memory");
	kernel_fpu_end();
}

static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
				       size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

	z0 = stop;		/* P/Q right side optimization */
	p = dptr[disks-2];	/* XOR parity */
	q = dptr[disks-1];	/* RS syndrome */

	kernel_fpu_begin();

	asm volatile("vmovdqa64 %0,%%zmm0"
		     : : "m" (raid6_avx512_constants.x1d[0]));

	for (d = 0 ; d < bytes ; d += 64) {
		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
			     "vmovdqa64 %1,%%zmm2\n\t"
			     "vpxorq %%zmm4,%%zmm2,%%zmm2"
			     :
			     : "m" (dptr[z0][d]),  "m" (p[d]));
		/* P/Q data pages */
		for (z = z0-1 ; z >= start ; z--) {
			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vmovdqa64 %0,%%zmm5\n\t"
				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
				     :
				     : "m" (dptr[z][d]));
		}
		/* P/Q left side optimization */
		for (z = start-1 ; z >= 0 ; z--) {
			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
				     :
				     : );
		}
		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
		/* Don't use movntdq for r/w memory area < cache line */
			     "vmovdqa64 %%zmm4,%0\n\t"
			     "vmovdqa64 %%zmm2,%1"
			     :
			     : "m" (q[d]), "m" (p[d]));
	}

	asm volatile("sfence" : : : "memory");
	kernel_fpu_end();
}

const struct raid6_calls raid6_avx512x1 = {
	raid6_avx5121_gen_syndrome,
	raid6_avx5121_xor_syndrome,
	raid6_have_avx512,
	"avx512x1",
	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
};

/*
 * Unrolled-by-2 AVX512 implementation
 */
static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

	z0 = disks - 3;         /* Highest data disk */
	p = dptr[z0+1];         /* XOR parity */
	q = dptr[z0+2];         /* RS syndrome */

	kernel_fpu_begin();

	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
		     :
		     : "m" (raid6_avx512_constants.x1d[0]));

	/* We uniformly assume a single prefetch covers at least 64 bytes */
	for (d = 0; d < bytes; d += 128) {
		asm volatile("prefetchnta %0\n\t"
			     "prefetchnta %1\n\t"
			     "vmovdqa64 %0,%%zmm2\n\t"      /* P[0] */
			     "vmovdqa64 %1,%%zmm3\n\t"      /* P[1] */
			     "vmovdqa64 %%zmm2,%%zmm4\n\t"  /* Q[0] */
			     "vmovdqa64 %%zmm3,%%zmm6"      /* Q[1] */
			     :
			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
		for (z = z0-1; z >= 0; z--) {
			asm volatile("prefetchnta %0\n\t"
				     "prefetchnta %1\n\t"
				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
				     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpmovm2b %%k2,%%zmm7\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
				     "vmovdqa64 %0,%%zmm5\n\t"
				     "vmovdqa64 %1,%%zmm7\n\t"
				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
				     :
				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
		}
		asm volatile("vmovntdq %%zmm2,%0\n\t"
			     "vmovntdq %%zmm3,%1\n\t"
			     "vmovntdq %%zmm4,%2\n\t"
			     "vmovntdq %%zmm6,%3"
			     :
			     : "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
			       "m" (q[d+64]));
	}

	asm volatile("sfence" : : : "memory");
	kernel_fpu_end();
}

static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
				       size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

	z0 = stop;		/* P/Q right side optimization */
	p = dptr[disks-2];	/* XOR parity */
	q = dptr[disks-1];	/* RS syndrome */

	kernel_fpu_begin();

	asm volatile("vmovdqa64 %0,%%zmm0"
		     : : "m" (raid6_avx512_constants.x1d[0]));

	for (d = 0 ; d < bytes ; d += 128) {
		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
			     "vmovdqa64 %1,%%zmm6\n\t"
			     "vmovdqa64 %2,%%zmm2\n\t"
			     "vmovdqa64 %3,%%zmm3\n\t"
			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
			     "vpxorq %%zmm6,%%zmm3,%%zmm3"
			     :
			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
			       "m" (p[d]), "m" (p[d+64]));
		/* P/Q data pages */
		for (z = z0-1 ; z >= start ; z--) {
			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpmovm2b %%k2,%%zmm7\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
				     "vmovdqa64 %0,%%zmm5\n\t"
				     "vmovdqa64 %1,%%zmm7\n\t"
				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
				     :
				     : "m" (dptr[z][d]),  "m" (dptr[z][d+64]));
		}
		/* P/Q left side optimization */
		for (z = start-1 ; z >= 0 ; z--) {
			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpmovm2b %%k2,%%zmm7\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
				     :
				     : );
		}
		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
			     "vpxorq %1,%%zmm6,%%zmm6\n\t"
			     /* Don't use movntdq for r/w
			      * memory area < cache line
			      */
			     "vmovdqa64 %%zmm4,%0\n\t"
			     "vmovdqa64 %%zmm6,%1\n\t"
			     "vmovdqa64 %%zmm2,%2\n\t"
			     "vmovdqa64 %%zmm3,%3"
			     :
			     : "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
			       "m" (p[d+64]));
	}

	asm volatile("sfence" : : : "memory");
	kernel_fpu_end();
}

const struct raid6_calls raid6_avx512x2 = {
	raid6_avx5122_gen_syndrome,
	raid6_avx5122_xor_syndrome,
	raid6_have_avx512,
	"avx512x2",
	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
};

#ifdef CONFIG_X86_64

/*
 * Unrolled-by-4 AVX2 implementation
 */
static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

	z0 = disks - 3;         /* Highest data disk */
	p = dptr[z0+1];         /* XOR parity */
	q = dptr[z0+2];         /* RS syndrome */

	kernel_fpu_begin();

	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
		     "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t"       /* Zero temp */
		     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"       /* P[0] */
		     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"       /* P[1] */
		     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"       /* Q[0] */
		     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"       /* Q[1] */
		     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"    /* P[2] */
		     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"    /* P[3] */
		     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"    /* Q[2] */
		     "vpxorq %%zmm14,%%zmm14,%%zmm14"        /* Q[3] */
		     :
		     : "m" (raid6_avx512_constants.x1d[0]));

	for (d = 0; d < bytes; d += 256) {
		for (z = z0; z >= 0; z--) {
		asm volatile("prefetchnta %0\n\t"
			     "prefetchnta %1\n\t"
			     "prefetchnta %2\n\t"
			     "prefetchnta %3\n\t"
			     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
			     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
			     "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
			     "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
			     "vpmovm2b %%k1,%%zmm5\n\t"
			     "vpmovm2b %%k2,%%zmm7\n\t"
			     "vpmovm2b %%k3,%%zmm13\n\t"
			     "vpmovm2b %%k4,%%zmm15\n\t"
			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
			     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
			     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
			     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
			     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
			     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
			     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
			     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
			     "vmovdqa64 %0,%%zmm5\n\t"
			     "vmovdqa64 %1,%%zmm7\n\t"
			     "vmovdqa64 %2,%%zmm13\n\t"
			     "vmovdqa64 %3,%%zmm15\n\t"
			     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
			     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
			     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
			     "vpxorq %%zmm15,%%zmm11,%%zmm11\n"
			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
			     "vpxorq %%zmm15,%%zmm14,%%zmm14"
			     :
			     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
			       "m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
		}
		asm volatile("vmovntdq %%zmm2,%0\n\t"
			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
			     "vmovntdq %%zmm3,%1\n\t"
			     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
			     "vmovntdq %%zmm10,%2\n\t"
			     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
			     "vmovntdq %%zmm11,%3\n\t"
			     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
			     "vmovntdq %%zmm4,%4\n\t"
			     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
			     "vmovntdq %%zmm6,%5\n\t"
			     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
			     "vmovntdq %%zmm12,%6\n\t"
			     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
			     "vmovntdq %%zmm14,%7\n\t"
			     "vpxorq %%zmm14,%%zmm14,%%zmm14"
			     :
			     : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
			       "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
			       "m" (q[d+128]), "m" (q[d+192]));
	}

	asm volatile("sfence" : : : "memory");
	kernel_fpu_end();
}

static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
				       size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

	z0 = stop;		/* P/Q right side optimization */
	p = dptr[disks-2];	/* XOR parity */
	q = dptr[disks-1];	/* RS syndrome */

	kernel_fpu_begin();

	asm volatile("vmovdqa64 %0,%%zmm0"
		     :: "m" (raid6_avx512_constants.x1d[0]));

	for (d = 0 ; d < bytes ; d += 256) {
		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
			     "vmovdqa64 %1,%%zmm6\n\t"
			     "vmovdqa64 %2,%%zmm12\n\t"
			     "vmovdqa64 %3,%%zmm14\n\t"
			     "vmovdqa64 %4,%%zmm2\n\t"
			     "vmovdqa64 %5,%%zmm3\n\t"
			     "vmovdqa64 %6,%%zmm10\n\t"
			     "vmovdqa64 %7,%%zmm11\n\t"
			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
			     "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
			     "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
			     "vpxorq %%zmm14,%%zmm11,%%zmm11"
			     :
			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
			       "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
			       "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
			       "m" (p[d+192]));
		/* P/Q data pages */
		for (z = z0-1 ; z >= start ; z--) {
			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
				     "prefetchnta %0\n\t"
				     "prefetchnta %2\n\t"
				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpmovm2b %%k2,%%zmm7\n\t"
				     "vpmovm2b %%k3,%%zmm13\n\t"
				     "vpmovm2b %%k4,%%zmm15\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
				     "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
				     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
				     "vmovdqa64 %0,%%zmm5\n\t"
				     "vmovdqa64 %1,%%zmm7\n\t"
				     "vmovdqa64 %2,%%zmm13\n\t"
				     "vmovdqa64 %3,%%zmm15\n\t"
				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
				     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
				     "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
				     :
				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
				       "m" (dptr[z][d+128]),
				       "m" (dptr[z][d+192]));
		}
		asm volatile("prefetchnta %0\n\t"
			     "prefetchnta %1\n\t"
			     :
			     : "m" (q[d]), "m" (q[d+128]));
		/* P/Q left side optimization */
		for (z = start-1 ; z >= 0 ; z--) {
			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpmovm2b %%k2,%%zmm7\n\t"
				     "vpmovm2b %%k3,%%zmm13\n\t"
				     "vpmovm2b %%k4,%%zmm15\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
				     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
				     :
				     : );
		}
		asm volatile("vmovntdq %%zmm2,%0\n\t"
			     "vmovntdq %%zmm3,%1\n\t"
			     "vmovntdq %%zmm10,%2\n\t"
			     "vmovntdq %%zmm11,%3\n\t"
			     "vpxorq %4,%%zmm4,%%zmm4\n\t"
			     "vpxorq %5,%%zmm6,%%zmm6\n\t"
			     "vpxorq %6,%%zmm12,%%zmm12\n\t"
			     "vpxorq %7,%%zmm14,%%zmm14\n\t"
			     "vmovntdq %%zmm4,%4\n\t"
			     "vmovntdq %%zmm6,%5\n\t"
			     "vmovntdq %%zmm12,%6\n\t"
			     "vmovntdq %%zmm14,%7"
			     :
			     : "m" (p[d]),  "m" (p[d+64]), "m" (p[d+128]),
			       "m" (p[d+192]), "m" (q[d]),  "m" (q[d+64]),
			       "m" (q[d+128]), "m" (q[d+192]));
	}
	asm volatile("sfence" : : : "memory");
	kernel_fpu_end();
}
const struct raid6_calls raid6_avx512x4 = {
	raid6_avx5124_gen_syndrome,
	raid6_avx5124_xor_syndrome,
	raid6_have_avx512,
	"avx512x4",
	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
};
#endif

#endif /* CONFIG_AS_AVX512 */
Commit	Line	Data
dd165a65	1	// SPDX-License-Identifier: GPL-2.0-or-later
e0a491c1 GK	2	/* -- linux-c -- --------------------------------------------------------
	3	*
	4	* Copyright (C) 2016 Intel Corporation
	5	*
	6	* Author: Gayatri Kammela <[email protected]>
	7	* Author: Megha Dey <[email protected]>
	8	*
	9	* Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
	10	* Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
	11	*
e0a491c1 GK	12	* -----------------------------------------------------------------------
	13	*/
	14
	15	/*
	16	* AVX512 implementation of RAID-6 syndrome functions
	17	*
	18	*/
	19
	20	#ifdef CONFIG_AS_AVX512
	21
	22	#include <linux/raid/pq.h>
	23	#include "x86.h"
	24
	25	static const struct raid6_avx512_constants {
	26	u64 x1d[8];
b5e0fff1	27	} raid6_avx512_constants __aligned(512/8) = {
e0a491c1 GK	28	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	29	0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	30	0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	31	0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
	32	};
	33
	34	static int raid6_have_avx512(void)
	35	{
	36	return boot_cpu_has(X86_FEATURE_AVX2) &&
	37	boot_cpu_has(X86_FEATURE_AVX) &&
	38	boot_cpu_has(X86_FEATURE_AVX512F) &&
	39	boot_cpu_has(X86_FEATURE_AVX512BW) &&
	40	boot_cpu_has(X86_FEATURE_AVX512VL) &&
	41	boot_cpu_has(X86_FEATURE_AVX512DQ);
	42	}
	43
	44	static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
	45	{
	46	u8 dptr = (u8 )ptrs;
	47	u8 p, q;
	48	int d, z, z0;
	49
	50	z0 = disks - 3; /* Highest data disk */
	51	p = dptr[z0+1]; /* XOR parity */
	52	q = dptr[z0+2]; /* RS syndrome */
	53
	54	kernel_fpu_begin();
	55
	56	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
	57	"vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
	58	:
	59	: "m" (raid6_avx512_constants.x1d[0]));
	60
	61	for (d = 0; d < bytes; d += 64) {
	62	asm volatile("prefetchnta %0\n\t"
	63	"vmovdqa64 %0,%%zmm2\n\t" /* P[0] */
	64	"prefetchnta %1\n\t"
	65	"vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
	66	"vmovdqa64 %1,%%zmm6"
	67	:
	68	: "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
	69	for (z = z0-2; z >= 0; z--) {
	70	asm volatile("prefetchnta %0\n\t"
	71	"vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
	72	"vpmovm2b %%k1,%%zmm5\n\t"
	73	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	74	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	75	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	76	"vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
	77	"vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
	78	"vmovdqa64 %0,%%zmm6"
	79	:
	80	: "m" (dptr[z][d]));
	81	}
	82	asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
	83	"vpmovm2b %%k1,%%zmm5\n\t"
	84	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	85	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	86	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	87	"vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
	88	"vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
	89	"vmovntdq %%zmm2,%0\n\t"
	90	"vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
	91	"vmovntdq %%zmm4,%1\n\t"
92	"vpxorq %%zmm4,%%zmm4,%%zmm4"
93	:
94	: "m" (p[d]), "m" (q[d]));
95	}
96
97	asm volatile("sfence" : : : "memory");
98	kernel_fpu_end();
99	}
100
694dda62 GK	101	static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
	102	size_t bytes, void **ptrs)
	103	{
	104	u8 dptr = (u8 )ptrs;
	105	u8 p, q;
	106	int d, z, z0;
	107
	108	z0 = stop; /* P/Q right side optimization */
	109	p = dptr[disks-2]; /* XOR parity */
	110	q = dptr[disks-1]; /* RS syndrome */
	111
	112	kernel_fpu_begin();
	113
	114	asm volatile("vmovdqa64 %0,%%zmm0"
	115	: : "m" (raid6_avx512_constants.x1d[0]));
	116
	117	for (d = 0 ; d < bytes ; d += 64) {
	118	asm volatile("vmovdqa64 %0,%%zmm4\n\t"
	119	"vmovdqa64 %1,%%zmm2\n\t"
	120	"vpxorq %%zmm4,%%zmm2,%%zmm2"
	121	:
	122	: "m" (dptr[z0][d]), "m" (p[d]));
	123	/* P/Q data pages */
	124	for (z = z0-1 ; z >= start ; z--) {
	125	asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
	126	"vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
	127	"vpmovm2b %%k1,%%zmm5\n\t"
	128	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	129	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	130	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	131	"vmovdqa64 %0,%%zmm5\n\t"
	132	"vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
	133	"vpxorq %%zmm5,%%zmm4,%%zmm4"
	134	:
	135	: "m" (dptr[z][d]));
	136	}
	137	/* P/Q left side optimization */
	138	for (z = start-1 ; z >= 0 ; z--) {
	139	asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
	140	"vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
	141	"vpmovm2b %%k1,%%zmm5\n\t"
	142	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	143	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	144	"vpxorq %%zmm5,%%zmm4,%%zmm4"
	145	:
	146	: );
	147	}
	148	asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
	149	/* Don't use movntdq for r/w memory area < cache line */
	150	"vmovdqa64 %%zmm4,%0\n\t"
	151	"vmovdqa64 %%zmm2,%1"
	152	:
	153	: "m" (q[d]), "m" (p[d]));
	154	}
	155
	156	asm volatile("sfence" : : : "memory");
	157	kernel_fpu_end();
	158	}
	159
e0a491c1 GK	160	const struct raid6_calls raid6_avx512x1 = {
e0a491c1 GK	161	raid6_avx5121_gen_syndrome,
694dda62	162	raid6_avx5121_xor_syndrome,
e0a491c1 GK	163	raid6_have_avx512,
e0a491c1 GK	164	"avx512x1",
36dacddb	165	.priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
e0a491c1 GK	166	};
	167
	168	/*
	169	* Unrolled-by-2 AVX512 implementation
	170	*/
	171	static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
	172	{
	173	u8 dptr = (u8 )ptrs;
	174	u8 p, q;
	175	int d, z, z0;
	176
	177	z0 = disks - 3; /* Highest data disk */
	178	p = dptr[z0+1]; /* XOR parity */
	179	q = dptr[z0+2]; /* RS syndrome */
	180
	181	kernel_fpu_begin();
	182
	183	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
	184	"vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
	185	:
	186	: "m" (raid6_avx512_constants.x1d[0]));
	187
	188	/* We uniformly assume a single prefetch covers at least 64 bytes */
	189	for (d = 0; d < bytes; d += 128) {
	190	asm volatile("prefetchnta %0\n\t"
	191	"prefetchnta %1\n\t"
	192	"vmovdqa64 %0,%%zmm2\n\t" /* P[0] */
	193	"vmovdqa64 %1,%%zmm3\n\t" /* P[1] */
	194	"vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
	195	"vmovdqa64 %%zmm3,%%zmm6" /* Q[1] */
	196	:
	197	: "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
	198	for (z = z0-1; z >= 0; z--) {
	199	asm volatile("prefetchnta %0\n\t"
	200	"prefetchnta %1\n\t"
	201	"vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
	202	"vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
	203	"vpmovm2b %%k1,%%zmm5\n\t"
	204	"vpmovm2b %%k2,%%zmm7\n\t"
	205	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	206	"vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
	207	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	208	"vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
	209	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	210	"vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
	211	"vmovdqa64 %0,%%zmm5\n\t"
	212	"vmovdqa64 %1,%%zmm7\n\t"
	213	"vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
	214	"vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
	215	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	216	"vpxorq %%zmm7,%%zmm6,%%zmm6"
	217	:
	218	: "m" (dptr[z][d]), "m" (dptr[z][d+64]));
	219	}
	220	asm volatile("vmovntdq %%zmm2,%0\n\t"
	221	"vmovntdq %%zmm3,%1\n\t"
	222	"vmovntdq %%zmm4,%2\n\t"
	223	"vmovntdq %%zmm6,%3"
	224	:
	225	: "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
	226	"m" (q[d+64]));
	227	}
	228
	229	asm volatile("sfence" : : : "memory");
230	kernel_fpu_end();
231	}
232
694dda62 GK	233	static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
	234	size_t bytes, void **ptrs)
	235	{
	236	u8 dptr = (u8 )ptrs;
	237	u8 p, q;
	238	int d, z, z0;
	239
	240	z0 = stop; /* P/Q right side optimization */
	241	p = dptr[disks-2]; /* XOR parity */
	242	q = dptr[disks-1]; /* RS syndrome */
	243
	244	kernel_fpu_begin();
	245
	246	asm volatile("vmovdqa64 %0,%%zmm0"
	247	: : "m" (raid6_avx512_constants.x1d[0]));
	248
	249	for (d = 0 ; d < bytes ; d += 128) {
	250	asm volatile("vmovdqa64 %0,%%zmm4\n\t"
	251	"vmovdqa64 %1,%%zmm6\n\t"
	252	"vmovdqa64 %2,%%zmm2\n\t"
	253	"vmovdqa64 %3,%%zmm3\n\t"
	254	"vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
	255	"vpxorq %%zmm6,%%zmm3,%%zmm3"
	256	:
	257	: "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
	258	"m" (p[d]), "m" (p[d+64]));
	259	/* P/Q data pages */
	260	for (z = z0-1 ; z >= start ; z--) {
	261	asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
	262	"vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
	263	"vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
	264	"vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
	265	"vpmovm2b %%k1,%%zmm5\n\t"
	266	"vpmovm2b %%k2,%%zmm7\n\t"
	267	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	268	"vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
	269	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	270	"vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
	271	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	272	"vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
	273	"vmovdqa64 %0,%%zmm5\n\t"
	274	"vmovdqa64 %1,%%zmm7\n\t"
	275	"vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
	276	"vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
	277	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	278	"vpxorq %%zmm7,%%zmm6,%%zmm6"
	279	:
	280	: "m" (dptr[z][d]), "m" (dptr[z][d+64]));
	281	}
	282	/* P/Q left side optimization */
	283	for (z = start-1 ; z >= 0 ; z--) {
	284	asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
	285	"vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
	286	"vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
	287	"vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
	288	"vpmovm2b %%k1,%%zmm5\n\t"
	289	"vpmovm2b %%k2,%%zmm7\n\t"
	290	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	291	"vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
	292	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	293	"vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
	294	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	295	"vpxorq %%zmm7,%%zmm6,%%zmm6"
	296	:
297	: );
298	}
299	asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
300	"vpxorq %1,%%zmm6,%%zmm6\n\t"
301	/* Don't use movntdq for r/w
302	* memory area < cache line
303	*/
304	"vmovdqa64 %%zmm4,%0\n\t"
305	"vmovdqa64 %%zmm6,%1\n\t"
306	"vmovdqa64 %%zmm2,%2\n\t"
307	"vmovdqa64 %%zmm3,%3"
308	:
309	: "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
310	"m" (p[d+64]));
311	}
312
313	asm volatile("sfence" : : : "memory");
314	kernel_fpu_end();
315	}
316
e0a491c1 GK	317	const struct raid6_calls raid6_avx512x2 = {
e0a491c1 GK	318	raid6_avx5122_gen_syndrome,
694dda62	319	raid6_avx5122_xor_syndrome,
e0a491c1 GK	320	raid6_have_avx512,
e0a491c1 GK	321	"avx512x2",
36dacddb	322	.priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
e0a491c1 GK	323	};
	324
	325	#ifdef CONFIG_X86_64
	326
	327	/*
	328	* Unrolled-by-4 AVX2 implementation
	329	*/
	330	static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
	331	{
	332	u8 dptr = (u8 )ptrs;
	333	u8 p, q;
	334	int d, z, z0;
	335
	336	z0 = disks - 3; /* Highest data disk */
	337	p = dptr[z0+1]; /* XOR parity */
	338	q = dptr[z0+2]; /* RS syndrome */
	339
	340	kernel_fpu_begin();
	341
	342	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
	343	"vpxorq %%zmm1,%%zmm1,%%zmm1\n\t" /* Zero temp */
	344	"vpxorq %%zmm2,%%zmm2,%%zmm2\n\t" /* P[0] */
	345	"vpxorq %%zmm3,%%zmm3,%%zmm3\n\t" /* P[1] */
	346	"vpxorq %%zmm4,%%zmm4,%%zmm4\n\t" /* Q[0] */
	347	"vpxorq %%zmm6,%%zmm6,%%zmm6\n\t" /* Q[1] */
	348	"vpxorq %%zmm10,%%zmm10,%%zmm10\n\t" /* P[2] */
	349	"vpxorq %%zmm11,%%zmm11,%%zmm11\n\t" /* P[3] */
	350	"vpxorq %%zmm12,%%zmm12,%%zmm12\n\t" /* Q[2] */
	351	"vpxorq %%zmm14,%%zmm14,%%zmm14" /* Q[3] */
	352	:
	353	: "m" (raid6_avx512_constants.x1d[0]));
	354
	355	for (d = 0; d < bytes; d += 256) {
	356	for (z = z0; z >= 0; z--) {
	357	asm volatile("prefetchnta %0\n\t"
	358	"prefetchnta %1\n\t"
	359	"prefetchnta %2\n\t"
	360	"prefetchnta %3\n\t"
	361	"vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
	362	"vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
	363	"vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
	364	"vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
	365	"vpmovm2b %%k1,%%zmm5\n\t"
	366	"vpmovm2b %%k2,%%zmm7\n\t"
	367	"vpmovm2b %%k3,%%zmm13\n\t"
	368	"vpmovm2b %%k4,%%zmm15\n\t"
	369	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	370	"vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
	371	"vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
	372	"vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
	373	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	374	"vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
	375	"vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
	376	"vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
	377	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	378	"vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
	379	"vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
	380	"vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
	381	"vmovdqa64 %0,%%zmm5\n\t"
	382	"vmovdqa64 %1,%%zmm7\n\t"
	383	"vmovdqa64 %2,%%zmm13\n\t"
	384	"vmovdqa64 %3,%%zmm15\n\t"
	385	"vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
	386	"vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
387	"vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
388	"vpxorq %%zmm15,%%zmm11,%%zmm11\n"
389	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
390	"vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
391	"vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
392	"vpxorq %%zmm15,%%zmm14,%%zmm14"
393	:
394	: "m" (dptr[z][d]), "m" (dptr[z][d+64]),
395	"m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
396	}
397	asm volatile("vmovntdq %%zmm2,%0\n\t"
398	"vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
399	"vmovntdq %%zmm3,%1\n\t"
400	"vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
401	"vmovntdq %%zmm10,%2\n\t"
402	"vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
403	"vmovntdq %%zmm11,%3\n\t"
404	"vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
405	"vmovntdq %%zmm4,%4\n\t"
406	"vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
407	"vmovntdq %%zmm6,%5\n\t"
408	"vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
409	"vmovntdq %%zmm12,%6\n\t"
410	"vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
411	"vmovntdq %%zmm14,%7\n\t"
412	"vpxorq %%zmm14,%%zmm14,%%zmm14"
413	:
414	: "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
415	"m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
416	"m" (q[d+128]), "m" (q[d+192]));
417	}
418
419	asm volatile("sfence" : : : "memory");
420	kernel_fpu_end();
421	}
422
694dda62 GK	423	static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
	424	size_t bytes, void **ptrs)
	425	{
	426	u8 dptr = (u8 )ptrs;
	427	u8 p, q;
	428	int d, z, z0;
	429
	430	z0 = stop; /* P/Q right side optimization */
	431	p = dptr[disks-2]; /* XOR parity */
	432	q = dptr[disks-1]; /* RS syndrome */
	433
	434	kernel_fpu_begin();
	435
	436	asm volatile("vmovdqa64 %0,%%zmm0"
	437	:: "m" (raid6_avx512_constants.x1d[0]));
	438
	439	for (d = 0 ; d < bytes ; d += 256) {
	440	asm volatile("vmovdqa64 %0,%%zmm4\n\t"
	441	"vmovdqa64 %1,%%zmm6\n\t"
	442	"vmovdqa64 %2,%%zmm12\n\t"
	443	"vmovdqa64 %3,%%zmm14\n\t"
	444	"vmovdqa64 %4,%%zmm2\n\t"
	445	"vmovdqa64 %5,%%zmm3\n\t"
	446	"vmovdqa64 %6,%%zmm10\n\t"
	447	"vmovdqa64 %7,%%zmm11\n\t"
	448	"vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
	449	"vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
	450	"vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
	451	"vpxorq %%zmm14,%%zmm11,%%zmm11"
	452	:
	453	: "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
	454	"m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
	455	"m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
	456	"m" (p[d+192]));
	457	/* P/Q data pages */
	458	for (z = z0-1 ; z >= start ; z--) {
	459	asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
	460	"vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
	461	"vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
	462	"vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
	463	"prefetchnta %0\n\t"
	464	"prefetchnta %2\n\t"
	465	"vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
	466	"vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
	467	"vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
	468	"vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
	469	"vpmovm2b %%k1,%%zmm5\n\t"
	470	"vpmovm2b %%k2,%%zmm7\n\t"
	471	"vpmovm2b %%k3,%%zmm13\n\t"
	472	"vpmovm2b %%k4,%%zmm15\n\t"
	473	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
	474	"vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
	475	"vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
	476	"vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
	477	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
	478	"vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
	479	"vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
	480	"vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
	481	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
	482	"vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
	483	"vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
	484	"vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
	485	"vmovdqa64 %0,%%zmm5\n\t"
	486	"vmovdqa64 %1,%%zmm7\n\t"
487	"vmovdqa64 %2,%%zmm13\n\t"
488	"vmovdqa64 %3,%%zmm15\n\t"
489	"vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
490	"vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
491	"vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
492	"vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
493	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
494	"vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
495	"vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
496	"vpxorq %%zmm15,%%zmm14,%%zmm14"
497	:
498	: "m" (dptr[z][d]), "m" (dptr[z][d+64]),
499	"m" (dptr[z][d+128]),
500	"m" (dptr[z][d+192]));
501	}
502	asm volatile("prefetchnta %0\n\t"
503	"prefetchnta %1\n\t"
504	:
505	: "m" (q[d]), "m" (q[d+128]));
506	/* P/Q left side optimization */
507	for (z = start-1 ; z >= 0 ; z--) {
508	asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
509	"vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
510	"vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
511	"vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
512	"vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
513	"vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
514	"vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
515	"vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
516	"vpmovm2b %%k1,%%zmm5\n\t"
517	"vpmovm2b %%k2,%%zmm7\n\t"
518	"vpmovm2b %%k3,%%zmm13\n\t"
519	"vpmovm2b %%k4,%%zmm15\n\t"
520	"vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
521	"vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
522	"vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
523	"vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
524	"vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
525	"vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
526	"vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
527	"vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
528	"vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
529	"vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
530	"vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
531	"vpxorq %%zmm15,%%zmm14,%%zmm14"
532	:
533	: );
534	}
535	asm volatile("vmovntdq %%zmm2,%0\n\t"
536	"vmovntdq %%zmm3,%1\n\t"
537	"vmovntdq %%zmm10,%2\n\t"
538	"vmovntdq %%zmm11,%3\n\t"
539	"vpxorq %4,%%zmm4,%%zmm4\n\t"
540	"vpxorq %5,%%zmm6,%%zmm6\n\t"
541	"vpxorq %6,%%zmm12,%%zmm12\n\t"
542	"vpxorq %7,%%zmm14,%%zmm14\n\t"
543	"vmovntdq %%zmm4,%4\n\t"
544	"vmovntdq %%zmm6,%5\n\t"
545	"vmovntdq %%zmm12,%6\n\t"
546	"vmovntdq %%zmm14,%7"
547	:
548	: "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
549	"m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
550	"m" (q[d+128]), "m" (q[d+192]));
551	}
552	asm volatile("sfence" : : : "memory");
553	kernel_fpu_end();
554	}
e0a491c1 GK	555	const struct raid6_calls raid6_avx512x4 = {
e0a491c1 GK	556	raid6_avx5124_gen_syndrome,
694dda62	557	raid6_avx5124_xor_syndrome,
e0a491c1 GK	558	raid6_have_avx512,
e0a491c1 GK	559	"avx512x4",
36dacddb	560	.priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
e0a491c1 GK	561	};
	562	#endif
	563
	564	#endif /* CONFIG_AS_AVX512 */