[linux.git] / drivers / gpu / drm / amd / amdgpu / vega10_ih.c

/*
 * Copyright 2016 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */
#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "soc15.h"

#include "oss/osssys_4_0_offset.h"
#include "oss/osssys_4_0_sh_mask.h"

#include "soc15_common.h"
#include "vega10_ih.h"


static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);

/**
 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Enable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
	adev->irq.ih.enabled = true;
}

/**
 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Disable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
	/* set rptr, wptr to 0 */
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
	adev->irq.ih.enabled = false;
	adev->irq.ih.rptr = 0;
}

/**
 * vega10_ih_irq_init - init and enable the interrupt ring
 *
 * @adev: amdgpu_device pointer
 *
 * Allocate a ring buffer for the interrupt controller,
 * enable the RLC, disable interrupts, enable the IH
 * ring buffer and enable it (VI).
 * Called at device load and reume.
 * Returns 0 for success, errors for failure.
 */
static int vega10_ih_irq_init(struct amdgpu_device *adev)
{
	int ret = 0;
	int rb_bufsz;
	u32 ih_rb_cntl, ih_doorbell_rtpr;
	u32 tmp;
	u64 wptr_off;

	/* disable irqs */
	vega10_ih_disable_interrupts(adev);

	adev->nbio_funcs->ih_control(adev);

	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
	if (adev->irq.ih.use_bus_addr) {
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
	} else {
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	}
	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);

	if (adev->irq.msi_enabled)
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);

	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);

	/* set the writeback address whether it's enabled or not */
	if (adev->irq.ih.use_bus_addr)
		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	else
		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFFFF);

	/* set rptr, wptr to 0 */
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);

	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
	if (adev->irq.ih.use_doorbell) {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 OFFSET, adev->irq.ih.doorbell_index);
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 1);
	} else {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 0);
	}
	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
					    adev->irq.ih.doorbell_index);

	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
			    CLIENT18_IS_STORM_CLIENT, 1);
	WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);

	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
	WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);

	pci_set_master(adev->pdev);

	/* enable interrupts */
	vega10_ih_enable_interrupts(adev);

	return ret;
}

/**
 * vega10_ih_irq_disable - disable interrupts
 *
 * @adev: amdgpu_device pointer
 *
 * Disable interrupts on the hw (VEGA10).
 */
static void vega10_ih_irq_disable(struct amdgpu_device *adev)
{
	vega10_ih_disable_interrupts(adev);

	/* Wait and acknowledge irq */
	mdelay(1);
}

/**
 * vega10_ih_get_wptr - get the IH ring buffer wptr
 *
 * @adev: amdgpu_device pointer
 *
 * Get the IH ring buffer wptr from either the register
 * or the writeback memory buffer (VEGA10).  Also check for
 * ring buffer overflow and deal with it.
 * Returns the value of the wptr.
 */
static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
{
	u32 wptr, tmp;

	if (adev->irq.ih.use_bus_addr)
		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	else
		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);

	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
		wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);

		/* When a ring buffer overflow happen start parsing interrupt
		 * from the last not overwritten vector (wptr + 32). Hopefully
		 * this should allow us to catchup.
		 */
		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
			wptr, adev->irq.ih.rptr, tmp);
		adev->irq.ih.rptr = tmp;

		tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
		WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
	}
	return (wptr & adev->irq.ih.ptr_mask);
}

/**
 * vega10_ih_prescreen_iv - prescreen an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Returns true if the interrupt vector should be further processed.
 */
static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
{
	u32 ring_index = adev->irq.ih.rptr >> 2;
	u32 dw0, dw3, dw4, dw5;
	u16 pasid;
	u64 addr, key;
	struct amdgpu_vm *vm;
	int r;

	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);

	/* Filter retry page faults, let only the first one pass. If
	 * there are too many outstanding faults, ignore them until
	 * some faults get cleared.
	 */
	switch (dw0 & 0xff) {
	case SOC15_IH_CLIENTID_VMC:
	case SOC15_IH_CLIENTID_UTCL2:
		break;
	default:
		/* Not a VM fault */
		return true;
	}

	pasid = dw3 & 0xffff;
	/* No PASID, can't identify faulting process */
	if (!pasid)
		return true;

	/* Not a retry fault, check fault credit */
	if (!(dw5 & 0x80)) {
		if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
			goto ignore_iv;
		return true;
	}

	/* Track retry faults in per-VM fault FIFO. */
	spin_lock(&adev->vm_manager.pasid_lock);
	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
	addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
	key = AMDGPU_VM_FAULT(pasid, addr);
	if (!vm) {
		/* VM not found, process it normally */
		spin_unlock(&adev->vm_manager.pasid_lock);
		return true;
	} else {
		r = amdgpu_vm_add_fault(vm->fault_hash, key);

		/* Hash table is full or the fault is already being processed,
		 * ignore further page faults
		 */
		if (r != 0) {
			spin_unlock(&adev->vm_manager.pasid_lock);
			goto ignore_iv;
		}
	}
	/* No locking required with single writer and single reader */
	r = kfifo_put(&vm->faults, key);
	if (!r) {
		/* FIFO is full. Ignore it until there is space */
		amdgpu_vm_clear_fault(vm->fault_hash, key);
		spin_unlock(&adev->vm_manager.pasid_lock);
		goto ignore_iv;
	}

	spin_unlock(&adev->vm_manager.pasid_lock);
	/* It's the first fault for this address, process it normally */
	return true;

ignore_iv:
	adev->irq.ih.rptr += 32;
	return false;
}

/**
 * vega10_ih_decode_iv - decode an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Decodes the interrupt vector at the current rptr
 * position and also advance the position.
 */
static void vega10_ih_decode_iv(struct amdgpu_device *adev,
				 struct amdgpu_iv_entry *entry)
{
	/* wptr/rptr are in bytes! */
	u32 ring_index = adev->irq.ih.rptr >> 2;
	uint32_t dw[8];

	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);

	entry->client_id = dw[0] & 0xff;
	entry->src_id = (dw[0] >> 8) & 0xff;
	entry->ring_id = (dw[0] >> 16) & 0xff;
	entry->vmid = (dw[0] >> 24) & 0xf;
	entry->vmid_src = (dw[0] >> 31);
	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
	entry->timestamp_src = dw[2] >> 31;
	entry->pasid = dw[3] & 0xffff;
	entry->pasid_src = dw[3] >> 31;
	entry->src_data[0] = dw[4];
	entry->src_data[1] = dw[5];
	entry->src_data[2] = dw[6];
	entry->src_data[3] = dw[7];


	/* wptr/rptr are in bytes! */
	adev->irq.ih.rptr += 32;
}

/**
 * vega10_ih_set_rptr - set the IH ring buffer rptr
 *
 * @adev: amdgpu_device pointer
 *
 * Set the IH ring buffer rptr.
 */
static void vega10_ih_set_rptr(struct amdgpu_device *adev)
{
	if (adev->irq.ih.use_doorbell) {
		/* XXX check if swapping is necessary on BE */
		if (adev->irq.ih.use_bus_addr)
			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		else
			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	} else {
		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
	}
}

static int vega10_ih_early_init(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_set_interrupt_funcs(adev);
	return 0;
}

static int vega10_ih_sw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
	if (r)
		return r;

	adev->irq.ih.use_doorbell = true;
	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;

	r = amdgpu_irq_init(adev);

	return r;
}

static int vega10_ih_sw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	amdgpu_irq_fini(adev);
	amdgpu_ih_ring_fini(adev, &adev->irq.ih);

	return 0;
}

static int vega10_ih_hw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = vega10_ih_irq_init(adev);
	if (r)
		return r;

	return 0;
}

static int vega10_ih_hw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_irq_disable(adev);

	return 0;
}

static int vega10_ih_suspend(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_fini(adev);
}

static int vega10_ih_resume(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_init(adev);
}

static bool vega10_ih_is_idle(void *handle)
{
	/* todo */
	return true;
}

static int vega10_ih_wait_for_idle(void *handle)
{
	/* todo */
	return -ETIMEDOUT;
}

static int vega10_ih_soft_reset(void *handle)
{
	/* todo */

	return 0;
}

static int vega10_ih_set_clockgating_state(void *handle,
					  enum amd_clockgating_state state)
{
	return 0;
}

static int vega10_ih_set_powergating_state(void *handle,
					  enum amd_powergating_state state)
{
	return 0;
}

const struct amd_ip_funcs vega10_ih_ip_funcs = {
	.name = "vega10_ih",
	.early_init = vega10_ih_early_init,
	.late_init = NULL,
	.sw_init = vega10_ih_sw_init,
	.sw_fini = vega10_ih_sw_fini,
	.hw_init = vega10_ih_hw_init,
	.hw_fini = vega10_ih_hw_fini,
	.suspend = vega10_ih_suspend,
	.resume = vega10_ih_resume,
	.is_idle = vega10_ih_is_idle,
	.wait_for_idle = vega10_ih_wait_for_idle,
	.soft_reset = vega10_ih_soft_reset,
	.set_clockgating_state = vega10_ih_set_clockgating_state,
	.set_powergating_state = vega10_ih_set_powergating_state,
};

static const struct amdgpu_ih_funcs vega10_ih_funcs = {
	.get_wptr = vega10_ih_get_wptr,
	.prescreen_iv = vega10_ih_prescreen_iv,
	.decode_iv = vega10_ih_decode_iv,
	.set_rptr = vega10_ih_set_rptr
};

static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
{
	adev->irq.ih_funcs = &vega10_ih_funcs;
}

const struct amdgpu_ip_block_version vega10_ih_ip_block =
{
	.type = AMD_IP_BLOCK_TYPE_IH,
	.major = 4,
	.minor = 0,
	.rev = 0,
	.funcs = &vega10_ih_ip_funcs,
};
Commit	Line	Data
282aae55 KW	1	/*
	2	* Copyright 2016 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
248a1d6f	23	#include <drm/drmP.h>
282aae55 KW	24	#include "amdgpu.h"
	25	#include "amdgpu_ih.h"
	26	#include "soc15.h"
	27
8af7454e FX	28	#include "oss/osssys_4_0_offset.h"
8af7454e FX	29	#include "oss/osssys_4_0_sh_mask.h"
282aae55 KW	30
	31	#include "soc15_common.h"
	32	#include "vega10_ih.h"
	33
	34
	35
	36	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
	37
	38	/**
	39	* vega10_ih_enable_interrupts - Enable the interrupt ring buffer
	40	*
	41	* @adev: amdgpu_device pointer
	42	*
	43	* Enable the interrupt ring buffer (VEGA10).
	44	*/
	45	static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
	46	{
b2b7e457	47	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
282aae55 KW	48
	49	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	50	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
b2b7e457	51	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
282aae55 KW	52	adev->irq.ih.enabled = true;
	53	}
	54
	55	/**
	56	* vega10_ih_disable_interrupts - Disable the interrupt ring buffer
	57	*
	58	* @adev: amdgpu_device pointer
	59	*
	60	* Disable the interrupt ring buffer (VEGA10).
	61	*/
	62	static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
	63	{
b2b7e457	64	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
282aae55 KW	65
	66	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	67	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
b2b7e457	68	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
282aae55	69	/* set rptr, wptr to 0 */
b2b7e457 HZ	70	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
b2b7e457 HZ	71	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
282aae55 KW	72	adev->irq.ih.enabled = false;
	73	adev->irq.ih.rptr = 0;
	74	}
	75
	76	/**
	77	* vega10_ih_irq_init - init and enable the interrupt ring
	78	*
	79	* @adev: amdgpu_device pointer
	80	*
	81	* Allocate a ring buffer for the interrupt controller,
	82	* enable the RLC, disable interrupts, enable the IH
	83	* ring buffer and enable it (VI).
	84	* Called at device load and reume.
	85	* Returns 0 for success, errors for failure.
	86	*/
	87	static int vega10_ih_irq_init(struct amdgpu_device *adev)
	88	{
	89	int ret = 0;
	90	int rb_bufsz;
	91	u32 ih_rb_cntl, ih_doorbell_rtpr;
	92	u32 tmp;
	93	u64 wptr_off;
	94
	95	/* disable irqs */
	96	vega10_ih_disable_interrupts(adev);
	97
bf383fb6	98	adev->nbio_funcs->ih_control(adev);
282aae55	99
b2b7e457	100	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
282aae55 KW	101	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
282aae55 KW	102	if (adev->irq.ih.use_bus_addr) {
b2b7e457 HZ	103	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
b2b7e457 HZ	104	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
282aae55 KW	105	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
282aae55 KW	106	} else {
b2b7e457 HZ	107	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
b2b7e457 HZ	108	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
282aae55 KW	109	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	110	}
	111	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	112	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	113	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	114	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	115	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	116	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	117	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	118	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	119	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
	120
	121	if (adev->irq.msi_enabled)
	122	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
	123
b2b7e457	124	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
282aae55 KW	125
	126	/* set the writeback address whether it's enabled or not */
	127	if (adev->irq.ih.use_bus_addr)
	128	wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	129	else
	130	wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
b2b7e457	131	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
c837243f	132	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFFFF);
282aae55 KW	133
282aae55 KW	134	/* set rptr, wptr to 0 */
b2b7e457 HZ	135	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
b2b7e457 HZ	136	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
282aae55	137
b2b7e457	138	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
282aae55 KW	139	if (adev->irq.ih.use_doorbell) {
	140	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	141	OFFSET, adev->irq.ih.doorbell_index);
	142	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	143	ENABLE, 1);
	144	} else {
	145	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	146	ENABLE, 0);
	147	}
b2b7e457	148	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
bf383fb6 AD	149	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
bf383fb6 AD	150	adev->irq.ih.doorbell_index);
282aae55	151
b2b7e457	152	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
282aae55 KW	153	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
282aae55 KW	154	CLIENT18_IS_STORM_CLIENT, 1);
b2b7e457	155	WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
282aae55	156
b2b7e457	157	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
282aae55	158	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
b2b7e457	159	WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
282aae55 KW	160
	161	pci_set_master(adev->pdev);
	162
	163	/* enable interrupts */
	164	vega10_ih_enable_interrupts(adev);
	165
	166	return ret;
	167	}
	168
	169	/**
	170	* vega10_ih_irq_disable - disable interrupts
	171	*
	172	* @adev: amdgpu_device pointer
	173	*
	174	* Disable interrupts on the hw (VEGA10).
	175	*/
	176	static void vega10_ih_irq_disable(struct amdgpu_device *adev)
	177	{
	178	vega10_ih_disable_interrupts(adev);
	179
	180	/* Wait and acknowledge irq */
	181	mdelay(1);
	182	}
	183
	184	/**
	185	* vega10_ih_get_wptr - get the IH ring buffer wptr
	186	*
	187	* @adev: amdgpu_device pointer
	188	*
	189	* Get the IH ring buffer wptr from either the register
	190	* or the writeback memory buffer (VEGA10). Also check for
	191	* ring buffer overflow and deal with it.
	192	* Returns the value of the wptr.
	193	*/
	194	static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
	195	{
	196	u32 wptr, tmp;
	197
	198	if (adev->irq.ih.use_bus_addr)
	199	wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	200	else
	201	wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
	202
	203	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
	204	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
	205
	206	/* When a ring buffer overflow happen start parsing interrupt
	207	* from the last not overwritten vector (wptr + 32). Hopefully
	208	* this should allow us to catchup.
	209	*/
	210	tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
	211	dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
	212	wptr, adev->irq.ih.rptr, tmp);
	213	adev->irq.ih.rptr = tmp;
	214
7c3f2167	215	tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
282aae55	216	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
7c3f2167	217	WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
282aae55 KW	218	}
	219	return (wptr & adev->irq.ih.ptr_mask);
	220	}
	221
00ecd8a2 FK	222	/**
	223	* vega10_ih_prescreen_iv - prescreen an interrupt vector
	224	*
	225	* @adev: amdgpu_device pointer
	226	*
	227	* Returns true if the interrupt vector should be further processed.
	228	*/
	229	static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
	230	{
a2f14820 FK	231	u32 ring_index = adev->irq.ih.rptr >> 2;
	232	u32 dw0, dw3, dw4, dw5;
	233	u16 pasid;
	234	u64 addr, key;
	235	struct amdgpu_vm *vm;
	236	int r;
	237
	238	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	239	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	240	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	241	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	242
	243	/* Filter retry page faults, let only the first one pass. If
	244	* there are too many outstanding faults, ignore them until
	245	* some faults get cleared.
	246	*/
	247	switch (dw0 & 0xff) {
3760f76c OZ	248	case SOC15_IH_CLIENTID_VMC:
3760f76c OZ	249	case SOC15_IH_CLIENTID_UTCL2:
a2f14820 FK	250	break;
	251	default:
	252	/* Not a VM fault */
	253	return true;
	254	}
	255
a2f14820 FK	256	pasid = dw3 & 0xffff;
	257	/* No PASID, can't identify faulting process */
	258	if (!pasid)
	259	return true;
	260
c98171cc FK	261	/* Not a retry fault, check fault credit */
	262	if (!(dw5 & 0x80)) {
	263	if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
	264	goto ignore_iv;
	265	return true;
	266	}
	267
a2f14820 FK	268	/* Track retry faults in per-VM fault FIFO. */
	269	spin_lock(&adev->vm_manager.pasid_lock);
	270	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
240cd9a6 OZ	271	addr = ((u64)(dw5 & 0xf) << 44) \| ((u64)dw4 << 12);
240cd9a6 OZ	272	key = AMDGPU_VM_FAULT(pasid, addr);
147f2558	273	if (!vm) {
a2f14820	274	/* VM not found, process it normally */
153b9e1b	275	spin_unlock(&adev->vm_manager.pasid_lock);
a2f14820	276	return true;
240cd9a6 OZ	277	} else {
	278	r = amdgpu_vm_add_fault(vm->fault_hash, key);
	279
	280	/* Hash table is full or the fault is already being processed,
	281	* ignore further page faults
	282	*/
	283	if (r != 0) {
	284	spin_unlock(&adev->vm_manager.pasid_lock);
	285	goto ignore_iv;
	286	}
a2f14820 FK	287	}
	288	/* No locking required with single writer and single reader */
	289	r = kfifo_put(&vm->faults, key);
	290	if (!r) {
	291	/* FIFO is full. Ignore it until there is space */
240cd9a6	292	amdgpu_vm_clear_fault(vm->fault_hash, key);
153b9e1b	293	spin_unlock(&adev->vm_manager.pasid_lock);
a2f14820 FK	294	goto ignore_iv;
	295	}
	296
240cd9a6	297	spin_unlock(&adev->vm_manager.pasid_lock);
a2f14820	298	/* It's the first fault for this address, process it normally */
00ecd8a2	299	return true;
a2f14820 FK	300
	301	ignore_iv:
	302	adev->irq.ih.rptr += 32;
	303	return false;
00ecd8a2 FK	304	}
00ecd8a2 FK	305
282aae55 KW	306	/**
	307	* vega10_ih_decode_iv - decode an interrupt vector
	308	*
	309	* @adev: amdgpu_device pointer
	310	*
	311	* Decodes the interrupt vector at the current rptr
	312	* position and also advance the position.
	313	*/
	314	static void vega10_ih_decode_iv(struct amdgpu_device *adev,
	315	struct amdgpu_iv_entry *entry)
	316	{
	317	/* wptr/rptr are in bytes! */
	318	u32 ring_index = adev->irq.ih.rptr >> 2;
	319	uint32_t dw[8];
	320
	321	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	322	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	323	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	324	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	325	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	326	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	327	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	328	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);
	329
	330	entry->client_id = dw[0] & 0xff;
	331	entry->src_id = (dw[0] >> 8) & 0xff;
	332	entry->ring_id = (dw[0] >> 16) & 0xff;
c4f46f22 CK	333	entry->vmid = (dw[0] >> 24) & 0xf;
c4f46f22 CK	334	entry->vmid_src = (dw[0] >> 31);
282aae55 KW	335	entry->timestamp = dw[1] \| ((u64)(dw[2] & 0xffff) << 32);
282aae55 KW	336	entry->timestamp_src = dw[2] >> 31;
3816e42f	337	entry->pasid = dw[3] & 0xffff;
282aae55 KW	338	entry->pasid_src = dw[3] >> 31;
	339	entry->src_data[0] = dw[4];
	340	entry->src_data[1] = dw[5];
	341	entry->src_data[2] = dw[6];
	342	entry->src_data[3] = dw[7];
	343
	344
	345	/* wptr/rptr are in bytes! */
	346	adev->irq.ih.rptr += 32;
	347	}
	348
	349	/**
	350	* vega10_ih_set_rptr - set the IH ring buffer rptr
	351	*
	352	* @adev: amdgpu_device pointer
	353	*
	354	* Set the IH ring buffer rptr.
	355	*/
	356	static void vega10_ih_set_rptr(struct amdgpu_device *adev)
	357	{
	358	if (adev->irq.ih.use_doorbell) {
	359	/* XXX check if swapping is necessary on BE */
	360	if (adev->irq.ih.use_bus_addr)
	361	adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	362	else
	363	adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	364	WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	365	} else {
b2b7e457	366	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
282aae55 KW	367	}
	368	}
	369
	370	static int vega10_ih_early_init(void *handle)
	371	{
	372	struct amdgpu_device adev = (struct amdgpu_device )handle;
	373
	374	vega10_ih_set_interrupt_funcs(adev);
	375	return 0;
	376	}
	377
	378	static int vega10_ih_sw_init(void *handle)
	379	{
	380	int r;
	381	struct amdgpu_device adev = (struct amdgpu_device )handle;
	382
425c3143	383	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
282aae55 KW	384	if (r)
	385	return r;
	386
	387	adev->irq.ih.use_doorbell = true;
	388	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
	389
	390	r = amdgpu_irq_init(adev);
	391
	392	return r;
	393	}
	394
	395	static int vega10_ih_sw_fini(void *handle)
	396	{
	397	struct amdgpu_device adev = (struct amdgpu_device )handle;
	398
	399	amdgpu_irq_fini(adev);
425c3143	400	amdgpu_ih_ring_fini(adev, &adev->irq.ih);
282aae55 KW	401
	402	return 0;
	403	}
	404
	405	static int vega10_ih_hw_init(void *handle)
	406	{
	407	int r;
	408	struct amdgpu_device adev = (struct amdgpu_device )handle;
	409
	410	r = vega10_ih_irq_init(adev);
	411	if (r)
	412	return r;
	413
	414	return 0;
	415	}
	416
	417	static int vega10_ih_hw_fini(void *handle)
	418	{
	419	struct amdgpu_device adev = (struct amdgpu_device )handle;
	420
	421	vega10_ih_irq_disable(adev);
	422
	423	return 0;
	424	}
	425
	426	static int vega10_ih_suspend(void *handle)
	427	{
	428	struct amdgpu_device adev = (struct amdgpu_device )handle;
	429
	430	return vega10_ih_hw_fini(adev);
	431	}
	432
	433	static int vega10_ih_resume(void *handle)
	434	{
	435	struct amdgpu_device adev = (struct amdgpu_device )handle;
	436
	437	return vega10_ih_hw_init(adev);
	438	}
	439
	440	static bool vega10_ih_is_idle(void *handle)
	441	{
	442	/* todo */
	443	return true;
	444	}
	445
	446	static int vega10_ih_wait_for_idle(void *handle)
	447	{
	448	/* todo */
	449	return -ETIMEDOUT;
	450	}
	451
	452	static int vega10_ih_soft_reset(void *handle)
	453	{
	454	/* todo */
	455
	456	return 0;
	457	}
	458
	459	static int vega10_ih_set_clockgating_state(void *handle,
	460	enum amd_clockgating_state state)
	461	{
	462	return 0;
	463	}
	464
465	static int vega10_ih_set_powergating_state(void *handle,
466	enum amd_powergating_state state)
467	{
468	return 0;
469	}
470
471	const struct amd_ip_funcs vega10_ih_ip_funcs = {
472	.name = "vega10_ih",
473	.early_init = vega10_ih_early_init,
474	.late_init = NULL,
475	.sw_init = vega10_ih_sw_init,
476	.sw_fini = vega10_ih_sw_fini,
477	.hw_init = vega10_ih_hw_init,
478	.hw_fini = vega10_ih_hw_fini,
479	.suspend = vega10_ih_suspend,
480	.resume = vega10_ih_resume,
481	.is_idle = vega10_ih_is_idle,
482	.wait_for_idle = vega10_ih_wait_for_idle,
483	.soft_reset = vega10_ih_soft_reset,
484	.set_clockgating_state = vega10_ih_set_clockgating_state,
485	.set_powergating_state = vega10_ih_set_powergating_state,
486	};
487
488	static const struct amdgpu_ih_funcs vega10_ih_funcs = {
489	.get_wptr = vega10_ih_get_wptr,
00ecd8a2	490	.prescreen_iv = vega10_ih_prescreen_iv,
282aae55 KW	491	.decode_iv = vega10_ih_decode_iv,
	492	.set_rptr = vega10_ih_set_rptr
	493	};
	494
	495	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
	496	{
f54b30d7	497	adev->irq.ih_funcs = &vega10_ih_funcs;
282aae55 KW	498	}
	499
	500	const struct amdgpu_ip_block_version vega10_ih_ip_block =
	501	{
	502	.type = AMD_IP_BLOCK_TYPE_IH,
	503	.major = 4,
	504	.minor = 0,
	505	.rev = 0,
	506	.funcs = &vega10_ih_ip_funcs,
	507	};