iommu/vt-d: Factor out invalidation descriptor composition
Separate the logic for constructing IOTLB and device TLB invalidation descriptors from the qi_flush interfaces. New helpers, qi_desc(), are introduced to encapsulate this common functionality. Moving descriptor composition code to new helpers enables its reuse in the upcoming qi_batch interfaces. No functional changes are intended. Signed-off-by: Tina Zhang <tina.zhang@intel.com> Link: https://lore.kernel.org/r/20240815065221.50328-2-tina.zhang@intel.com Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com> Signed-off-by: Joerg Roedel <jroedel@suse.de>
This commit is contained in:
Tina Zhang
Joerg Roedel
parent
1f5e307ca1
commit
f701c9f36b
@ -1526,24 +1526,9 @@ void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
|
||||
void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
|
||||
unsigned int size_order, u64 type)
|
||||
{
|
||||
u8 dw = 0, dr = 0;
|
||||
|
||||
struct qi_desc desc;
|
||||
int ih = 0;
|
||||
|
||||
if (cap_write_drain(iommu->cap))
|
||||
dw = 1;
|
||||
|
||||
if (cap_read_drain(iommu->cap))
|
||||
dr = 1;
|
||||
|
||||
desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
|
||||
| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
|
||||
desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
|
||||
| QI_IOTLB_AM(size_order);
|
||||
desc.qw2 = 0;
|
||||
desc.qw3 = 0;
|
||||
|
||||
qi_desc_iotlb(iommu, did, addr, size_order, type, &desc);
|
||||
qi_submit_sync(iommu, &desc, 1, 0);
|
||||
}
|
||||
|
||||
@ -1561,20 +1546,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
|
||||
if (!(iommu->gcmd & DMA_GCMD_TE))
|
||||
return;
|
||||
|
||||
if (mask) {
|
||||
addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
|
||||
desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
|
||||
} else
|
||||
desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
|
||||
|
||||
if (qdep >= QI_DEV_IOTLB_MAX_INVS)
|
||||
qdep = 0;
|
||||
|
||||
desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
|
||||
QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
|
||||
desc.qw2 = 0;
|
||||
desc.qw3 = 0;
|
||||
|
||||
qi_desc_dev_iotlb(sid, pfsid, qdep, addr, mask, &desc);
|
||||
qi_submit_sync(iommu, &desc, 1, 0);
|
||||
}
|
||||
|
||||
@ -1594,28 +1566,7 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
|
||||
return;
|
||||
}
|
||||
|
||||
if (npages == -1) {
|
||||
desc.qw0 = QI_EIOTLB_PASID(pasid) |
|
||||
QI_EIOTLB_DID(did) |
|
||||
QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
|
||||
QI_EIOTLB_TYPE;
|
||||
desc.qw1 = 0;
|
||||
} else {
|
||||
int mask = ilog2(__roundup_pow_of_two(npages));
|
||||
unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
|
||||
|
||||
if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
|
||||
addr = ALIGN_DOWN(addr, align);
|
||||
|
||||
desc.qw0 = QI_EIOTLB_PASID(pasid) |
|
||||
QI_EIOTLB_DID(did) |
|
||||
QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
|
||||
QI_EIOTLB_TYPE;
|
||||
desc.qw1 = QI_EIOTLB_ADDR(addr) |
|
||||
QI_EIOTLB_IH(ih) |
|
||||
QI_EIOTLB_AM(mask);
|
||||
}
|
||||
|
||||
qi_desc_piotlb(did, pasid, addr, npages, ih, &desc);
|
||||
qi_submit_sync(iommu, &desc, 1, 0);
|
||||
}
|
||||
|
||||
@ -1623,7 +1574,6 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
|
||||
void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
|
||||
u32 pasid, u16 qdep, u64 addr, unsigned int size_order)
|
||||
{
|
||||
unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
|
||||
struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
|
||||
|
||||
/*
|
||||
@ -1635,40 +1585,9 @@ void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
|
||||
if (!(iommu->gcmd & DMA_GCMD_TE))
|
||||
return;
|
||||
|
||||
desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
|
||||
QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
|
||||
QI_DEV_IOTLB_PFSID(pfsid);
|
||||
|
||||
/*
|
||||
* If S bit is 0, we only flush a single page. If S bit is set,
|
||||
* The least significant zero bit indicates the invalidation address
|
||||
* range. VT-d spec 6.5.2.6.
|
||||
* e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
|
||||
* size order = 0 is PAGE_SIZE 4KB
|
||||
* Max Invs Pending (MIP) is set to 0 for now until we have DIT in
|
||||
* ECAP.
|
||||
*/
|
||||
if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
|
||||
pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
|
||||
addr, size_order);
|
||||
|
||||
/* Take page address */
|
||||
desc.qw1 = QI_DEV_EIOTLB_ADDR(addr);
|
||||
|
||||
if (size_order) {
|
||||
/*
|
||||
* Existing 0s in address below size_order may be the least
|
||||
* significant bit, we must set them to 1s to avoid having
|
||||
* smaller size than desired.
|
||||
*/
|
||||
desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
|
||||
VTD_PAGE_SHIFT);
|
||||
/* Clear size_order bit to indicate size */
|
||||
desc.qw1 &= ~mask;
|
||||
/* Set the S bit to indicate flushing more than 1 page */
|
||||
desc.qw1 |= QI_DEV_EIOTLB_SIZE;
|
||||
}
|
||||
|
||||
qi_desc_dev_iotlb_pasid(sid, pfsid, pasid,
|
||||
qdep, addr, size_order,
|
||||
&desc);
|
||||
qi_submit_sync(iommu, &desc, 1, 0);
|
||||
}
|
||||
|
||||
|
||||
@ -1066,6 +1066,115 @@ static inline unsigned long nrpages_to_size(unsigned long npages)
|
||||
return npages << VTD_PAGE_SHIFT;
|
||||
}
|
||||
|
||||
static inline void qi_desc_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
|
||||
unsigned int size_order, u64 type,
|
||||
struct qi_desc *desc)
|
||||
{
|
||||
u8 dw = 0, dr = 0;
|
||||
int ih = 0;
|
||||
|
||||
if (cap_write_drain(iommu->cap))
|
||||
dw = 1;
|
||||
|
||||
if (cap_read_drain(iommu->cap))
|
||||
dr = 1;
|
||||
|
||||
desc->qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
|
||||
| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
|
||||
desc->qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
|
||||
| QI_IOTLB_AM(size_order);
|
||||
desc->qw2 = 0;
|
||||
desc->qw3 = 0;
|
||||
}
|
||||
|
||||
static inline void qi_desc_dev_iotlb(u16 sid, u16 pfsid, u16 qdep, u64 addr,
|
||||
unsigned int mask, struct qi_desc *desc)
|
||||
{
|
||||
if (mask) {
|
||||
addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
|
||||
desc->qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
|
||||
} else {
|
||||
desc->qw1 = QI_DEV_IOTLB_ADDR(addr);
|
||||
}
|
||||
|
||||
if (qdep >= QI_DEV_IOTLB_MAX_INVS)
|
||||
qdep = 0;
|
||||
|
||||
desc->qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
|
||||
QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
|
||||
desc->qw2 = 0;
|
||||
desc->qw3 = 0;
|
||||
}
|
||||
|
||||
static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr,
|
||||
unsigned long npages, bool ih,
|
||||
struct qi_desc *desc)
|
||||
{
|
||||
if (npages == -1) {
|
||||
desc->qw0 = QI_EIOTLB_PASID(pasid) |
|
||||
QI_EIOTLB_DID(did) |
|
||||
QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
|
||||
QI_EIOTLB_TYPE;
|
||||
desc->qw1 = 0;
|
||||
} else {
|
||||
int mask = ilog2(__roundup_pow_of_two(npages));
|
||||
unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
|
||||
|
||||
if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
|
||||
addr = ALIGN_DOWN(addr, align);
|
||||
|
||||
desc->qw0 = QI_EIOTLB_PASID(pasid) |
|
||||
QI_EIOTLB_DID(did) |
|
||||
QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
|
||||
QI_EIOTLB_TYPE;
|
||||
desc->qw1 = QI_EIOTLB_ADDR(addr) |
|
||||
QI_EIOTLB_IH(ih) |
|
||||
QI_EIOTLB_AM(mask);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
|
||||
u16 qdep, u64 addr,
|
||||
unsigned int size_order,
|
||||
struct qi_desc *desc)
|
||||
{
|
||||
unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
|
||||
|
||||
desc->qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
|
||||
QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
|
||||
QI_DEV_IOTLB_PFSID(pfsid);
|
||||
|
||||
/*
|
||||
* If S bit is 0, we only flush a single page. If S bit is set,
|
||||
* The least significant zero bit indicates the invalidation address
|
||||
* range. VT-d spec 6.5.2.6.
|
||||
* e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
|
||||
* size order = 0 is PAGE_SIZE 4KB
|
||||
* Max Invs Pending (MIP) is set to 0 for now until we have DIT in
|
||||
* ECAP.
|
||||
*/
|
||||
if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
|
||||
pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
|
||||
addr, size_order);
|
||||
|
||||
/* Take page address */
|
||||
desc->qw1 = QI_DEV_EIOTLB_ADDR(addr);
|
||||
|
||||
if (size_order) {
|
||||
/*
|
||||
* Existing 0s in address below size_order may be the least
|
||||
* significant bit, we must set them to 1s to avoid having
|
||||
* smaller size than desired.
|
||||
*/
|
||||
desc->qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
|
||||
VTD_PAGE_SHIFT);
|
||||
/* Clear size_order bit to indicate size */
|
||||
desc->qw1 &= ~mask;
|
||||
/* Set the S bit to indicate flushing more than 1 page */
|
||||
desc->qw1 |= QI_DEV_EIOTLB_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
/* Convert value to context PASID directory size field coding. */
|
||||
#define context_pdts(pds) (((pds) & 0x7) << 9)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user