kevin/linux
1
Fork 0
Code

iommu/amd: Initial SVA support for AMD IOMMU

Browse Source 
This includes :
  - Add data structure to track per protection domain dev/pasid binding details
    protection_domain->dev_data_list will track attached list of
    dev_data/PASIDs.

  - Move 'to_pdomain()' to header file

  - Add iommu_sva_set_dev_pasid(). It will check whether PASID is supported
    or not. Also adds PASID to SVA protection domain list as well as to
    device GCR3 table.

  - Add iommu_ops.remove_dev_pasid support. It will unbind PASID from
    device. Also remove pasid data from protection domain device list.

  - Add IOMMU_SVA as dependency to AMD_IOMMU driver

For a given PASID, iommu_set_dev_pasid() will bind all devices to same
SVA protection domain (1 PASID : 1 SVA protection domain : N devices).
This protection domain is different from device protection domain (one
that's mapped in attach_device() path). IOMMU uses domain ID for caching,
invalidation, etc. In SVA mode it will use per-device-domain-ID. Hence in
invalidation path we retrieve domain ID from gcr3_info_table structure and
use that for invalidation.

Co-developed-by: Wei Huang <wei.huang2@amd.com>
Signed-off-by: Wei Huang <wei.huang2@amd.com>
Co-developed-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Signed-off-by: Vasant Hegde <vasant.hegde@amd.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20240418103400.6229-14-vasant.hegde@amd.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
This commit is contained in:
Vasant Hegde 2024-04-18 10:33:58 +00:00 committed by Joerg Roedel
parent c4cb231111
commit 1af95763e0
6 changed files with 156 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/iommu/amd/Kconfig
View File

@ -10,6 +10,7 @@ config AMD_IOMMU
select IOMMU_API
select IOMMU_IOVA
select IOMMU_IO_PGTABLE
select IOMMU_SVA
select IOMMU_IOPF
select IOMMUFD_DRIVER if IOMMUFD
depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE

2
drivers/iommu/amd/Makefile
View File

@ -1,3 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o
obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o pasid.o
obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o

11
drivers/iommu/amd/amd_iommu.h
View File

@ -44,6 +44,12 @@ extern int amd_iommu_guest_ir;
extern enum io_pgtable_fmt amd_iommu_pgtable;
extern int amd_iommu_gpt_level;
/* Protection domain ops */
int iommu_sva_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid);
void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid);
/* SVA/PASID */
bool amd_iommu_pasid_supported(void);
/* IOPF */
@ -174,6 +180,11 @@ static inline struct amd_iommu *get_amd_iommu_from_dev_data(struct iommu_dev_dat
return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu);
}
static inline struct protection_domain *to_pdomain(struct iommu_domain *dom)
{
return container_of(dom, struct protection_domain, domain);
}
bool translation_pre_enabled(struct amd_iommu *iommu);
bool amd_iommu_is_attach_deferred(struct device *dev);
int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line);

19
drivers/iommu/amd/amd_iommu_types.h
View File

@ -8,7 +8,9 @@
#ifndef _ASM_X86_AMD_IOMMU_TYPES_H
#define _ASM_X86_AMD_IOMMU_TYPES_H
#include <linux/iommu.h>
#include <linux/types.h>
#include <linux/mmu_notifier.h>
#include <linux/mutex.h>
#include <linux/msi.h>
#include <linux/list.h>
@ -511,6 +513,11 @@ extern struct kmem_cache *amd_iommu_irq_cache;
list_for_each_entry((iommu), &amd_iommu_list, list)
#define for_each_iommu_safe(iommu, next) \
list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
/* Making iterating over protection_domain->dev_data_list easier */
#define for_each_pdom_dev_data(pdom_dev_data, pdom) \
list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list)
#define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \
list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list)
struct amd_iommu;
struct iommu_domain;
@ -552,6 +559,16 @@ enum protection_domain_mode {
PD_MODE_V2,
};
/* Track dev_data/PASID list for the protection domain */
struct pdom_dev_data {
/* Points to attached device data */
struct iommu_dev_data *dev_data;
/* PASID attached to the protection domain */
ioasid_t pasid;
/* For protection_domain->dev_data_list */
struct list_head list;
};
/*
* This structure contains generic data for IOMMU protection domains
* independent of their use.
@ -568,6 +585,8 @@ struct protection_domain {
bool dirty_tracking; /* dirty tracking is enabled in the domain */
unsigned dev_cnt; /* devices assigned to this domain */
unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
struct list_head dev_data_list; /* List of pdom_dev_data */
};
/*

7
drivers/iommu/amd/iommu.c
View File

@ -194,11 +194,6 @@ static struct amd_iommu *rlookup_amd_iommu(struct device *dev)
return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));
}
static struct protection_domain *to_pdomain(struct iommu_domain *dom)
{
return container_of(dom, struct protection_domain, domain);
}
static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)
{
struct iommu_dev_data *dev_data;
@ -2345,6 +2340,7 @@ static struct protection_domain *protection_domain_alloc(unsigned int type)
spin_lock_init(&domain->lock);
INIT_LIST_HEAD(&domain->dev_list);
INIT_LIST_HEAD(&domain->dev_data_list);
domain->nid = NUMA_NO_NODE;
switch (type) {
@ -2874,6 +2870,7 @@ const struct iommu_ops amd_iommu_ops = {
.def_domain_type = amd_iommu_def_domain_type,
.dev_enable_feat = amd_iommu_dev_enable_feature,
.dev_disable_feat = amd_iommu_dev_disable_feature,
.remove_dev_pasid = amd_iommu_remove_dev_pasid,
.page_response = amd_iommu_page_response,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = amd_iommu_attach_device,

122
drivers/iommu/amd/pasid.c Normal file
View File

@ -0,0 +1,122 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2024 Advanced Micro Devices, Inc.
*/
#define pr_fmt(fmt) "AMD-Vi: " fmt
#define dev_fmt(fmt) pr_fmt(fmt)
#include <linux/iommu.h>
#include <linux/mm_types.h>
#include "amd_iommu.h"
static inline bool is_pasid_enabled(struct iommu_dev_data *dev_data)
{
if (dev_data->pasid_enabled && dev_data->max_pasids &&
dev_data->gcr3_info.gcr3_tbl != NULL)
return true;
return false;
}
static inline bool is_pasid_valid(struct iommu_dev_data *dev_data,
ioasid_t pasid)
{
if (pasid > 0 && pasid < dev_data->max_pasids)
return true;
return false;
}
static void remove_dev_pasid(struct pdom_dev_data *pdom_dev_data)
{
/* Update GCR3 table and flush IOTLB */
amd_iommu_clear_gcr3(pdom_dev_data->dev_data, pdom_dev_data->pasid);
list_del(&pdom_dev_data->list);
kfree(pdom_dev_data);
}
/* Clear PASID from device GCR3 table and remove pdom_dev_data from list */
static void remove_pdom_dev_pasid(struct protection_domain *pdom,
struct device *dev, ioasid_t pasid)
{
struct pdom_dev_data *pdom_dev_data;
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
lockdep_assert_held(&pdom->lock);
for_each_pdom_dev_data(pdom_dev_data, pdom) {
if (pdom_dev_data->dev_data == dev_data &&
pdom_dev_data->pasid == pasid) {
remove_dev_pasid(pdom_dev_data);
break;
}
}
}
int iommu_sva_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
{
struct pdom_dev_data *pdom_dev_data;
struct protection_domain *sva_pdom = to_pdomain(domain);
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
unsigned long flags;
int ret = -EINVAL;
/* PASID zero is used for requests from the I/O device without PASID */
if (!is_pasid_valid(dev_data, pasid))
return ret;
/* Make sure PASID is enabled */
if (!is_pasid_enabled(dev_data))
return ret;
/* Add PASID to protection domain pasid list */
pdom_dev_data = kzalloc(sizeof(*pdom_dev_data), GFP_KERNEL);
if (pdom_dev_data == NULL)
return ret;
pdom_dev_data->pasid = pasid;
pdom_dev_data->dev_data = dev_data;
spin_lock_irqsave(&sva_pdom->lock, flags);
/* Setup GCR3 table */
ret = amd_iommu_set_gcr3(dev_data, pasid,
iommu_virt_to_phys(domain->mm->pgd));
if (ret) {
kfree(pdom_dev_data);
goto out_unlock;
}
list_add(&pdom_dev_data->list, &sva_pdom->dev_data_list);
out_unlock:
spin_unlock_irqrestore(&sva_pdom->lock, flags);
return ret;
}
void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
{
struct protection_domain *sva_pdom;
struct iommu_domain *domain;
unsigned long flags;
if (!is_pasid_valid(dev_iommu_priv_get(dev), pasid))
return;
/* Get protection domain */
domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
if (!domain)
return;
sva_pdom = to_pdomain(domain);
spin_lock_irqsave(&sva_pdom->lock, flags);
/* Remove PASID from dev_data_list */
remove_pdom_dev_pasid(sva_pdom, dev, pasid);
spin_unlock_irqrestore(&sva_pdom->lock, flags);
}