From: Joerg Roedel <joerg.roedel@amd.com>
Date: Tue, 10 May 2011 08:25:23 +0000 (+0200)
Subject: Merge branches 'dma-debug/next', 'amd-iommu/command-cleanups', 'amd-iommu/ats' and... 
X-Git-Tag: v3.0-rc1~407^2~1^2~1
X-Git-Url: https://repo.jachan.dev/linux.git/commitdiff_plain/604c307bf47350c74bb36507b86a08726c7c2075

Merge branches 'dma-debug/next', 'amd-iommu/command-cleanups', 'amd-iommu/ats' and 'amd-iommu/extended-features' into iommu/2.6.40

Conflicts:
	arch/x86/include/asm/amd_iommu_types.h
	arch/x86/kernel/amd_iommu.c
	arch/x86/kernel/amd_iommu_init.c
---

604c307bf47350c74bb36507b86a08726c7c2075
diff --cc arch/x86/include/asm/amd_iommu_types.h
index e3509fc303bf,e3509fc303bf,7434377b2ab9,df62d26ed2ab..4c9982995414
--- a/arch/x86/include/asm/amd_iommu_types.h
+++ b/arch/x86/include/asm/amd_iommu_types.h
@@@@@ -113,7 -113,7 -113,8 -126,8 +126,9 @@@@@
    /* command specific defines */
    #define CMD_COMPL_WAIT          0x01
    #define CMD_INV_DEV_ENTRY       0x02
-- -#define CMD_INV_IOMMU_PAGES     0x03
++ +#define CMD_INV_IOMMU_PAGES	0x03
++ +#define CMD_INV_IOTLB_PAGES	0x04
+++ #define CMD_INV_ALL		0x08
    
    #define CMD_COMPL_WAIT_STORE_MASK	0x01
    #define CMD_COMPL_WAIT_INT_MASK		0x02
diff --cc arch/x86/kernel/amd_iommu.c
index 57ca77787220,57ca77787220,e4791f66aa38,d6192bcf9f09..dc5dddafe5c2
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@@@@ -383,31 -383,31 -382,122 -381,97 +382,128 @@@@@ irqreturn_t amd_iommu_int_handler(int i
     *
     ****************************************************************************/
    
--  /*
--   * Writes the command to the IOMMUs command buffer and informs the
--   * hardware about the new command. Must be called with iommu->lock held.
--   */
--  static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
++  static int wait_on_sem(volatile u64 *sem)
++  {
++  	int i = 0;
++  
++  	while (*sem == 0 && i < LOOP_TIMEOUT) {
++  		udelay(1);
++  		i += 1;
++  	}
++  
++  	if (i == LOOP_TIMEOUT) {
++  		pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
++  		return -EIO;
++  	}
++  
++  	return 0;
++  }
++  
++  static void copy_cmd_to_buffer(struct amd_iommu *iommu,
++  			       struct iommu_cmd *cmd,
++  			       u32 tail)
    {
--  	u32 tail, head;
    	u8 *target;
    
--  	WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
--  	tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
    	target = iommu->cmd_buf + tail;
--  	memcpy_toio(target, cmd, sizeof(*cmd));
--  	tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
--  	head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
--  	if (tail == head)
--  		return -ENOMEM;
++  	tail   = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
++  
++  	/* Copy command to buffer */
++  	memcpy(target, cmd, sizeof(*cmd));
++  
++  	/* Tell the IOMMU about it */
    	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
++  }
    
--  	return 0;
++  static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
++  {
++  	WARN_ON(address & 0x7ULL);
++  
++  	memset(cmd, 0, sizeof(*cmd));
++  	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
++  	cmd->data[1] = upper_32_bits(__pa(address));
++  	cmd->data[2] = 1;
++  	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
++  }
++  
++  static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
++  {
++  	memset(cmd, 0, sizeof(*cmd));
++  	cmd->data[0] = devid;
++  	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
++  }
++  
++  static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
++  				  size_t size, u16 domid, int pde)
++  {
++  	u64 pages;
++  	int s;
++  
++  	pages = iommu_num_pages(address, size, PAGE_SIZE);
++  	s     = 0;
++  
++  	if (pages > 1) {
++  		/*
++  		 * If we have to flush more than one page, flush all
++  		 * TLB entries for this domain
++  		 */
++  		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
++  		s = 1;
++  	}
++  
++  	address &= PAGE_MASK;
++  
++  	memset(cmd, 0, sizeof(*cmd));
++  	cmd->data[1] |= domid;
++  	cmd->data[2]  = lower_32_bits(address);
++  	cmd->data[3]  = upper_32_bits(address);
++  	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
++  	if (s) /* size bit - we flush more than one 4kb page */
++  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
++  	if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
++  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
++  }
++  
++ +static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
++ +				  u64 address, size_t size)
++ +{
++ +	u64 pages;
++ +	int s;
++ +
++ +	pages = iommu_num_pages(address, size, PAGE_SIZE);
++ +	s     = 0;
++ +
++ +	if (pages > 1) {
++ +		/*
++ +		 * If we have to flush more than one page, flush all
++ +		 * TLB entries for this domain
++ +		 */
++ +		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
++ +		s = 1;
++ +	}
++ +
++ +	address &= PAGE_MASK;
++ +
++ +	memset(cmd, 0, sizeof(*cmd));
++ +	cmd->data[0]  = devid;
++ +	cmd->data[0] |= (qdep & 0xff) << 24;
++ +	cmd->data[1]  = devid;
++ +	cmd->data[2]  = lower_32_bits(address);
++ +	cmd->data[3]  = upper_32_bits(address);
++ +	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
++ +	if (s)
++ +		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
++ +}
++ +
+++ static void build_inv_all(struct iommu_cmd *cmd)
+++ {
+++ 	memset(cmd, 0, sizeof(*cmd));
+++ 	CMD_SET_TYPE(cmd, CMD_INV_ALL);
  + }
  + 
    /*
--   * General queuing function for commands. Takes iommu->lock and calls
--   * __iommu_queue_command().
++   * Writes the command to the IOMMUs command buffer and informs the
++   * hardware about the new command.
     */
    static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
    {
@@@@@ -454,117 -454,117 -544,109 -518,93 +550,123 @@@@@ again
     * This function queues a completion wait command into the command
     * buffer of an IOMMU
     */
--  static int __iommu_completion_wait(struct amd_iommu *iommu)
++  static int iommu_completion_wait(struct amd_iommu *iommu)
    {
    	struct iommu_cmd cmd;
++  	volatile u64 sem = 0;
++  	int ret;
    
--  	 memset(&cmd, 0, sizeof(cmd));
--  	 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
--  	 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
++  	if (!iommu->need_sync)
++  		return 0;
++  
++  	build_completion_wait(&cmd, (u64)&sem);
++  
++  	ret = iommu_queue_command(iommu, &cmd);
++  	if (ret)
++  		return ret;
    
--  	 return __iommu_queue_command(iommu, &cmd);
++  	return wait_on_sem(&sem);
    }
    
--  /*
--   * This function is called whenever we need to ensure that the IOMMU has
--   * completed execution of all commands we sent. It sends a
--   * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
--   * us about that by writing a value to a physical address we pass with
--   * the command.
--   */
--  static int iommu_completion_wait(struct amd_iommu *iommu)
++  static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
    {
--  	int ret = 0;
--  	unsigned long flags;
++  	struct iommu_cmd cmd;
    
--  	spin_lock_irqsave(&iommu->lock, flags);
++  	build_inv_dte(&cmd, devid);
    
--  	if (!iommu->need_sync)
--  		goto out;
++  	return iommu_queue_command(iommu, &cmd);
++  }
    
--  	ret = __iommu_completion_wait(iommu);
++  static void iommu_flush_dte_all(struct amd_iommu *iommu)
++  {
++  	u32 devid;
    
--  	iommu->need_sync = false;
++  	for (devid = 0; devid <= 0xffff; ++devid)
++  		iommu_flush_dte(iommu, devid);
    
--  	if (ret)
--  		goto out;
--  
--  	__iommu_wait_for_completion(iommu);
++  	iommu_completion_wait(iommu);
++  }
    
--  out:
--  	spin_unlock_irqrestore(&iommu->lock, flags);
++  /*
++   * This function uses heavy locking and may disable irqs for some time. But
++   * this is no issue because it is only called during resume.
++   */
++  static void iommu_flush_tlb_all(struct amd_iommu *iommu)
++  {
++  	u32 dom_id;
    
--  	if (iommu->reset_in_progress)
--  		reset_iommu_command_buffer(iommu);
++  	for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
++  		struct iommu_cmd cmd;
++  		build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
++  				      dom_id, 1);
++  		iommu_queue_command(iommu, &cmd);
++  	}
    
--  	return 0;
++  	iommu_completion_wait(iommu);
    }
    
--  static void iommu_flush_complete(struct protection_domain *domain)
+++ static void iommu_flush_all(struct amd_iommu *iommu)
  + {
--  	int i;
+++ 	struct iommu_cmd cmd;
  + 
--  	for (i = 0; i < amd_iommus_present; ++i) {
--  		if (!domain->dev_iommu[i])
--  			continue;
+++ 	build_inv_all(&cmd);
  + 
--  		/*
--  		 * Devices of this domain are behind this IOMMU
--  		 * We need to wait for completion of all commands.
--  		 */
--  		iommu_completion_wait(amd_iommus[i]);
+++ 	iommu_queue_command(iommu, &cmd);
+++ 	iommu_completion_wait(iommu);
+++ }
+++ 
++  void iommu_flush_all_caches(struct amd_iommu *iommu)
++  {
  - 	iommu_flush_dte_all(iommu);
  - 	iommu_flush_tlb_all(iommu);
+++ 	if (iommu_feature(iommu, FEATURE_IA)) {
+++ 		iommu_flush_all(iommu);
+++ 	} else {
+++ 		iommu_flush_dte_all(iommu);
+++ 		iommu_flush_tlb_all(iommu);
  + 	}
    }
    
   +/*
--   * Command send function for invalidating a device table entry
++ + * Command send function for flushing on-device TLB
   + */
--  static int iommu_flush_device(struct device *dev)
++ +static int device_flush_iotlb(struct device *dev, u64 address, size_t size)
   +{
++ +	struct pci_dev *pdev = to_pci_dev(dev);
   +	struct amd_iommu *iommu;
   +	struct iommu_cmd cmd;
   +	u16 devid;
++ +	int qdep;
   +
++ +	qdep  = pci_ats_queue_depth(pdev);
   +	devid = get_device_id(dev);
   +	iommu = amd_iommu_rlookup_table[devid];
   +
--  	/* Build command */
--  	memset(&cmd, 0, sizeof(cmd));
--  	CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
--  	cmd.data[0] = devid;
++ +	build_inv_iotlb_pages(&cmd, devid, qdep, address, size);
   +
   +	return iommu_queue_command(iommu, &cmd);
   +}
   +
--  static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
--  					  u16 domid, int pde, int s)
--  {
--  	memset(cmd, 0, sizeof(*cmd));
--  	address &= PAGE_MASK;
--  	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
--  	cmd->data[1] |= domid;
--  	cmd->data[2] = lower_32_bits(address);
--  	cmd->data[3] = upper_32_bits(address);
--  	if (s) /* size bit - we flush more than one 4kb page */
--  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
--  	if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
--  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
--  }
--  
    /*
--   * Generic command send function for invalidaing TLB entries
++   * Command send function for invalidating a device table entry
     */
--  static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
--  		u64 address, u16 domid, int pde, int s)
++  static int device_flush_dte(struct device *dev)
    {
--  	struct iommu_cmd cmd;
++  	struct amd_iommu *iommu;
++ +	struct pci_dev *pdev;
++  	u16 devid;
   +	int ret;
    
--  	__iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
++ +	pdev  = to_pci_dev(dev);
++  	devid = get_device_id(dev);
++  	iommu = amd_iommu_rlookup_table[devid];
    
--  	ret = iommu_queue_command(iommu, &cmd);
   -	return iommu_flush_dte(iommu, devid);
++ +	ret = iommu_flush_dte(iommu, devid);
++ +	if (ret)
++ +		return ret;
++ +
++ +	if (pci_ats_enabled(pdev))
++ +		ret = device_flush_iotlb(dev, 0, ~0UL);
   +
   +	return ret;
    }
    
    /*
@@@@@ -572,23 -572,23 -654,14 -612,13 +674,14 @@@@@
     * It invalidates a single PTE if the range to flush is within a single
     * page. Otherwise it flushes the whole TLB of the IOMMU.
     */
--  static void __iommu_flush_pages(struct protection_domain *domain,
--  				u64 address, size_t size, int pde)
++  static void __domain_flush_pages(struct protection_domain *domain,
++  				 u64 address, size_t size, int pde)
    {
--  	int s = 0, i;
--  	unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE);
--  
--  	address &= PAGE_MASK;
--  
--  	if (pages > 1) {
--  		/*
--  		 * If we have to flush more than one page, flush all
--  		 * TLB entries for this domain
--  		 */
--  		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
--  		s = 1;
--  	}
++ +	struct iommu_dev_data *dev_data;
++  	struct iommu_cmd cmd;
++  	int ret = 0, i;
    
++  	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
    
    	for (i = 0; i < amd_iommus_present; ++i) {
    		if (!domain->dev_iommu[i])
@@@@@ -598,17 -598,17 -671,25 -628,16 +691,25 @@@@@
    		 * Devices of this domain are behind this IOMMU
    		 * We need a TLB flush
    		 */
--  		iommu_queue_inv_iommu_pages(amd_iommus[i], address,
--  					    domain->id, pde, s);
++  		ret |= iommu_queue_command(amd_iommus[i], &cmd);
    	}
    
--  	return;
++ +	list_for_each_entry(dev_data, &domain->dev_list, list) {
++ +		struct pci_dev *pdev = to_pci_dev(dev_data->dev);
++ +
++ +		if (!pci_ats_enabled(pdev))
++ +			continue;
++ +
++ +		ret |= device_flush_iotlb(dev_data->dev, address, size);
++ +	}
++ +
++  	WARN_ON(ret);
    }
    
--  static void iommu_flush_pages(struct protection_domain *domain,
--  			     u64 address, size_t size)
++  static void domain_flush_pages(struct protection_domain *domain,
++  			       u64 address, size_t size)
    {
--  	__iommu_flush_pages(domain, address, size, 0);
++  	__domain_flush_pages(domain, address, size, 0);
    }
    
    /* Flush the whole IO/TLB for a given protection domain */
diff --cc arch/x86/kernel/amd_iommu_init.c
index 246d727b65b7,246d727b65b7,b6c634f3dc07,047905dc3e14..28b078133688
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@@@@ -667,6 -667,6 -674,9 -687,12 +688,15 @@@@@ static void __init init_iommu_from_pci(
    					MMIO_GET_LD(range));
    	iommu->evt_msi_num = MMIO_MSI_NUM(misc);
    
++ +	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
++ +		amd_iommu_iotlb_sup = false;
++ +
+++ 	/* read extended feature bits */
+++ 	low  = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
+++ 	high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
+++ 
+++ 	iommu->features = ((u64)high << 32) | low;
+++ 
    	if (!is_rd890_iommu(iommu->dev))
    		return;