kevin/linux
1
Fork 0
Code

dmaengine: amd: qdma: Add AMD QDMA driver

Browse Source 
Adds driver to enable PCIe board which uses AMD QDMA (the Queue-based
Direct Memory Access) subsystem. For example, Xilinx Alveo V70 AI
Accelerator devices.
    https://www.xilinx.com/applications/data-center/v70.html

The QDMA subsystem is used in conjunction with the PCI Express IP block
to provide high performance data transfer between host memory and the
card's DMA subsystem.

            +-------+       +-------+       +-----------+
   PCIe     |       |       |       |       |           |
   Tx/Rx    |       |       |       |  AXI  |           |
 <=======>  | PCIE  | <===> | QDMA  | <====>| User Logic|
            |       |       |       |       |           |
            +-------+       +-------+       +-----------+

The primary mechanism to transfer data using the QDMA is for the QDMA
engine to operate on instructions (descriptors) provided by the host
operating system. Using the descriptors, the QDMA can move data in both
the Host to Card (H2C) direction, or the Card to Host (C2H) direction.
The QDMA provides a per-queue basis option whether DMA traffic goes
to an AXI4 memory map (MM) interface or to an AXI4-Stream interface.

The hardware detail is provided by
    https://docs.xilinx.com/r/en-US/pg302-qdma

Implements dmaengine APIs to support MM DMA transfers.
- probe the available DMA channels
- use dma_slave_map for channel lookup
- use virtual channel to manage dmaengine tx descriptors
- implement device_prep_slave_sg callback to handle host scatter gather
  list

Signed-off-by: Nishad Saraf <nishads@amd.com>
Signed-off-by: Lizhi Hou <lizhi.hou@amd.com>
Link: https://lore.kernel.org/r/20240819211948.688786-2-lizhi.hou@amd.com
Signed-off-by: Vinod Koul <vkoul@kernel.org>
This commit is contained in:
Nishad Saraf 2024-08-19 14:19:48 -07:00 committed by Vinod Koul
parent 51c42ae3d7
commit 73d5fc92a1
10 changed files with 1542 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
MAINTAINERS
View File

@ -1128,6 +1128,14 @@ L: dmaengine@vger.kernel.org
S: Maintained
F: drivers/dma/ptdma/
AMD QDMA DRIVER
M: Nishad Saraf <nishads@amd.com>
M: Lizhi Hou <lizhi.hou@amd.com>
L: dmaengine@vger.kernel.org
S: Supported
F: drivers/dma/amd/qdma/
F: include/linux/platform_data/amd_qdma.h
AMD SEATTLE DEVICE TREE SUPPORT
M: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
M: Tom Lendacky <thomas.lendacky@amd.com>

2
drivers/dma/Kconfig
View File

@ -716,6 +716,8 @@ config XILINX_ZYNQMP_DPDMA
display driver.
# driver files
source "drivers/dma/amd/Kconfig"
source "drivers/dma/bestcomm/Kconfig"
source "drivers/dma/mediatek/Kconfig"

1
drivers/dma/Makefile
View File

@ -83,6 +83,7 @@ obj-$(CONFIG_ST_FDMA) += st_fdma.o
obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/
obj-$(CONFIG_INTEL_LDMA) += lgm/
obj-y += amd/
obj-y += mediatek/
obj-y += qcom/
obj-y += stm32/

14
drivers/dma/amd/Kconfig Normal file
View File

@ -0,0 +1,14 @@
# SPDX-License-Identifier: GPL-2.0-only
config AMD_QDMA
tristate "AMD Queue-based DMA"
depends on HAS_IOMEM
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
select REGMAP_MMIO
help
Enable support for the AMD Queue-based DMA subsystem. The primary
mechanism to transfer data using the QDMA is for the QDMA engine to
operate on instructions (descriptors) provided by the host operating
system. Using the descriptors, the QDMA can move data in either the
Host to Card (H2C) direction or the Card to Host (C2H) direction.

3
drivers/dma/amd/Makefile Normal file
View File

@ -0,0 +1,3 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_AMD_QDMA) += qdma/

5
drivers/dma/amd/qdma/Makefile Normal file
View File

@ -0,0 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_AMD_QDMA) += amd-qdma.o
amd-qdma-$(CONFIG_AMD_QDMA) := qdma.o qdma-comm-regs.o

64
drivers/dma/amd/qdma/qdma-comm-regs.c Normal file
View File

@ -0,0 +1,64 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
*/
#ifndef __QDMA_REGS_DEF_H
#define __QDMA_REGS_DEF_H
#include "qdma.h"
const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX] = {
[QDMA_REGO_CTXT_DATA] = QDMA_REGO(0x804, 8),
[QDMA_REGO_CTXT_CMD] = QDMA_REGO(0x844, 1),
[QDMA_REGO_CTXT_MASK] = QDMA_REGO(0x824, 8),
[QDMA_REGO_MM_H2C_CTRL] = QDMA_REGO(0x1004, 1),
[QDMA_REGO_MM_C2H_CTRL] = QDMA_REGO(0x1204, 1),
[QDMA_REGO_QUEUE_COUNT] = QDMA_REGO(0x120, 1),
[QDMA_REGO_RING_SIZE] = QDMA_REGO(0x204, 1),
[QDMA_REGO_H2C_PIDX] = QDMA_REGO(0x18004, 1),
[QDMA_REGO_C2H_PIDX] = QDMA_REGO(0x18008, 1),
[QDMA_REGO_INTR_CIDX] = QDMA_REGO(0x18000, 1),
[QDMA_REGO_FUNC_ID] = QDMA_REGO(0x12c, 1),
[QDMA_REGO_ERR_INT] = QDMA_REGO(0xb04, 1),
[QDMA_REGO_ERR_STAT] = QDMA_REGO(0x248, 1),
};
const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX] = {
/* QDMA_REGO_CTXT_DATA fields */
[QDMA_REGF_IRQ_ENABLE] = QDMA_REGF(53, 53),
[QDMA_REGF_WBK_ENABLE] = QDMA_REGF(52, 52),
[QDMA_REGF_WBI_CHECK] = QDMA_REGF(34, 34),
[QDMA_REGF_IRQ_ARM] = QDMA_REGF(16, 16),
[QDMA_REGF_IRQ_VEC] = QDMA_REGF(138, 128),
[QDMA_REGF_IRQ_AGG] = QDMA_REGF(139, 139),
[QDMA_REGF_WBI_INTVL_ENABLE] = QDMA_REGF(35, 35),
[QDMA_REGF_MRKR_DISABLE] = QDMA_REGF(62, 62),
[QDMA_REGF_QUEUE_ENABLE] = QDMA_REGF(32, 32),
[QDMA_REGF_QUEUE_MODE] = QDMA_REGF(63, 63),
[QDMA_REGF_DESC_BASE] = QDMA_REGF(127, 64),
[QDMA_REGF_DESC_SIZE] = QDMA_REGF(49, 48),
[QDMA_REGF_RING_ID] = QDMA_REGF(47, 44),
[QDMA_REGF_QUEUE_BASE] = QDMA_REGF(11, 0),
[QDMA_REGF_QUEUE_MAX] = QDMA_REGF(44, 32),
[QDMA_REGF_FUNCTION_ID] = QDMA_REGF(24, 17),
[QDMA_REGF_INTR_AGG_BASE] = QDMA_REGF(66, 15),
[QDMA_REGF_INTR_VECTOR] = QDMA_REGF(11, 1),
[QDMA_REGF_INTR_SIZE] = QDMA_REGF(69, 67),
[QDMA_REGF_INTR_VALID] = QDMA_REGF(0, 0),
[QDMA_REGF_INTR_COLOR] = QDMA_REGF(14, 14),
[QDMA_REGF_INTR_FUNCTION_ID] = QDMA_REGF(125, 114),
/* QDMA_REGO_CTXT_CMD fields */
[QDMA_REGF_CMD_INDX] = QDMA_REGF(19, 7),
[QDMA_REGF_CMD_CMD] = QDMA_REGF(6, 5),
[QDMA_REGF_CMD_TYPE] = QDMA_REGF(4, 1),
[QDMA_REGF_CMD_BUSY] = QDMA_REGF(0, 0),
/* QDMA_REGO_QUEUE_COUNT fields */
[QDMA_REGF_QUEUE_COUNT] = QDMA_REGF(11, 0),
/* QDMA_REGO_ERR_INT fields */
[QDMA_REGF_ERR_INT_FUNC] = QDMA_REGF(11, 0),
[QDMA_REGF_ERR_INT_VEC] = QDMA_REGF(22, 12),
[QDMA_REGF_ERR_INT_ARM] = QDMA_REGF(24, 24),
};
#endif /* __QDMA_REGS_DEF_H */

1143
drivers/dma/amd/qdma/qdma.c Normal file
View File

File diff suppressed because it is too large Load Diff

266
drivers/dma/amd/qdma/qdma.h Normal file
View File

@ -0,0 +1,266 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* DMA header for AMD Queue-based DMA Subsystem
*
* Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
*/
#ifndef __QDMA_H
#define __QDMA_H
#include <linux/bitfield.h>
#include <linux/dmaengine.h>
#include <linux/kernel.h>
#include <linux/platform_device.h>
#include <linux/regmap.h>
#include "../../virt-dma.h"
#define DISABLE 0
#define ENABLE 1
#define QDMA_MIN_IRQ 3
#define QDMA_INTR_NAME_MAX_LEN 30
#define QDMA_INTR_PREFIX "amd-qdma"
#define QDMA_IDENTIFIER 0x1FD3
#define QDMA_DEFAULT_RING_SIZE (BIT(10) + 1)
#define QDMA_DEFAULT_RING_ID 0
#define QDMA_POLL_INTRVL_US 10 /* 10us */
#define QDMA_POLL_TIMEOUT_US (500 * 1000) /* 500ms */
#define QDMA_DMAP_REG_STRIDE 16
#define QDMA_CTXT_REGMAP_LEN 8 /* 8 regs */
#define QDMA_MM_DESC_SIZE 32 /* Bytes */
#define QDMA_MM_DESC_LEN_BITS 28
#define QDMA_MM_DESC_MAX_LEN (BIT(QDMA_MM_DESC_LEN_BITS) - 1)
#define QDMA_MIN_DMA_ALLOC_SIZE 4096
#define QDMA_INTR_RING_SIZE BIT(13)
#define QDMA_INTR_RING_IDX_MASK GENMASK(9, 0)
#define QDMA_INTR_RING_BASE(_addr) ((_addr) >> 12)
#define QDMA_IDENTIFIER_REGOFF 0x0
#define QDMA_IDENTIFIER_MASK GENMASK(31, 16)
#define QDMA_QUEUE_ARM_BIT BIT(16)
#define qdma_err(qdev, fmt, args...) \
dev_err(&(qdev)->pdev->dev, fmt, ##args)
#define qdma_dbg(qdev, fmt, args...) \
dev_dbg(&(qdev)->pdev->dev, fmt, ##args)
#define qdma_info(qdev, fmt, args...) \
dev_info(&(qdev)->pdev->dev, fmt, ##args)
enum qdma_reg_fields {
QDMA_REGF_IRQ_ENABLE,
QDMA_REGF_WBK_ENABLE,
QDMA_REGF_WBI_CHECK,
QDMA_REGF_IRQ_ARM,
QDMA_REGF_IRQ_VEC,
QDMA_REGF_IRQ_AGG,
QDMA_REGF_WBI_INTVL_ENABLE,
QDMA_REGF_MRKR_DISABLE,
QDMA_REGF_QUEUE_ENABLE,
QDMA_REGF_QUEUE_MODE,
QDMA_REGF_DESC_BASE,
QDMA_REGF_DESC_SIZE,
QDMA_REGF_RING_ID,
QDMA_REGF_CMD_INDX,
QDMA_REGF_CMD_CMD,
QDMA_REGF_CMD_TYPE,
QDMA_REGF_CMD_BUSY,
QDMA_REGF_QUEUE_COUNT,
QDMA_REGF_QUEUE_MAX,
QDMA_REGF_QUEUE_BASE,
QDMA_REGF_FUNCTION_ID,
QDMA_REGF_INTR_AGG_BASE,
QDMA_REGF_INTR_VECTOR,
QDMA_REGF_INTR_SIZE,
QDMA_REGF_INTR_VALID,
QDMA_REGF_INTR_COLOR,
QDMA_REGF_INTR_FUNCTION_ID,
QDMA_REGF_ERR_INT_FUNC,
QDMA_REGF_ERR_INT_VEC,
QDMA_REGF_ERR_INT_ARM,
QDMA_REGF_MAX
};
enum qdma_regs {
QDMA_REGO_CTXT_DATA,
QDMA_REGO_CTXT_CMD,
QDMA_REGO_CTXT_MASK,
QDMA_REGO_MM_H2C_CTRL,
QDMA_REGO_MM_C2H_CTRL,
QDMA_REGO_QUEUE_COUNT,
QDMA_REGO_RING_SIZE,
QDMA_REGO_H2C_PIDX,
QDMA_REGO_C2H_PIDX,
QDMA_REGO_INTR_CIDX,
QDMA_REGO_FUNC_ID,
QDMA_REGO_ERR_INT,
QDMA_REGO_ERR_STAT,
QDMA_REGO_MAX
};
struct qdma_reg_field {
u16 lsb; /* Least significant bit of field */
u16 msb; /* Most significant bit of field */
};
struct qdma_reg {
u32 off;
u32 count;
};
#define QDMA_REGF(_msb, _lsb) { \
.lsb = (_lsb), \
.msb = (_msb), \
}
#define QDMA_REGO(_off, _count) { \
.off = (_off), \
.count = (_count), \
}
enum qdma_desc_size {
QDMA_DESC_SIZE_8B,
QDMA_DESC_SIZE_16B,
QDMA_DESC_SIZE_32B,
QDMA_DESC_SIZE_64B,
};
enum qdma_queue_op_mode {
QDMA_QUEUE_OP_STREAM,
QDMA_QUEUE_OP_MM,
};
enum qdma_ctxt_type {
QDMA_CTXT_DESC_SW_C2H,
QDMA_CTXT_DESC_SW_H2C,
QDMA_CTXT_DESC_HW_C2H,
QDMA_CTXT_DESC_HW_H2C,
QDMA_CTXT_DESC_CR_C2H,
QDMA_CTXT_DESC_CR_H2C,
QDMA_CTXT_WRB,
QDMA_CTXT_PFTCH,
QDMA_CTXT_INTR_COAL,
QDMA_CTXT_RSVD,
QDMA_CTXT_HOST_PROFILE,
QDMA_CTXT_TIMER,
QDMA_CTXT_FMAP,
QDMA_CTXT_FNC_STS,
};
enum qdma_ctxt_cmd {
QDMA_CTXT_CLEAR,
QDMA_CTXT_WRITE,
QDMA_CTXT_READ,
QDMA_CTXT_INVALIDATE,
QDMA_CTXT_MAX
};
struct qdma_ctxt_sw_desc {
u64 desc_base;
u16 vec;
};
struct qdma_ctxt_intr {
u64 agg_base;
u16 vec;
u32 size;
bool valid;
bool color;
};
struct qdma_ctxt_fmap {
u16 qbase;
u16 qmax;
};
struct qdma_device;
struct qdma_mm_desc {
__le64 src_addr;
__le32 len;
__le32 reserved1;
__le64 dst_addr;
__le64 reserved2;
} __packed;
struct qdma_mm_vdesc {
struct virt_dma_desc vdesc;
struct qdma_queue *queue;
struct scatterlist *sgl;
u64 sg_off;
u32 sg_len;
u64 dev_addr;
u32 pidx;
u32 pending_descs;
struct dma_slave_config cfg;
};
#define QDMA_VDESC_QUEUED(vdesc) (!(vdesc)->sg_len)
struct qdma_queue {
struct qdma_device *qdev;
struct virt_dma_chan vchan;
enum dma_transfer_direction dir;
struct dma_slave_config cfg;
struct qdma_mm_desc *desc_base;
struct qdma_mm_vdesc *submitted_vdesc;
struct qdma_mm_vdesc *issued_vdesc;
dma_addr_t dma_desc_base;
u32 pidx_reg;
u32 cidx_reg;
u32 ring_size;
u32 idx_mask;
u16 qid;
u32 pidx;
u32 cidx;
};
struct qdma_intr_ring {
struct qdma_device *qdev;
__le64 *base;
dma_addr_t dev_base;
char msix_name[QDMA_INTR_NAME_MAX_LEN];
u32 msix_vector;
u16 msix_id;
u32 ring_size;
u16 ridx;
u16 cidx;
u8 color;
};
#define QDMA_INTR_MASK_PIDX GENMASK_ULL(15, 0)
#define QDMA_INTR_MASK_CIDX GENMASK_ULL(31, 16)
#define QDMA_INTR_MASK_DESC_COLOR GENMASK_ULL(32, 32)
#define QDMA_INTR_MASK_STATE GENMASK_ULL(34, 33)
#define QDMA_INTR_MASK_ERROR GENMASK_ULL(36, 35)
#define QDMA_INTR_MASK_TYPE GENMASK_ULL(38, 38)
#define QDMA_INTR_MASK_QID GENMASK_ULL(62, 39)
#define QDMA_INTR_MASK_COLOR GENMASK_ULL(63, 63)
struct qdma_device {
struct platform_device *pdev;
struct dma_device dma_dev;
struct regmap *regmap;
struct mutex ctxt_lock; /* protect ctxt registers */
const struct qdma_reg_field *rfields;
const struct qdma_reg *roffs;
struct qdma_queue *h2c_queues;
struct qdma_queue *c2h_queues;
struct qdma_intr_ring *qintr_rings;
u32 qintr_ring_num;
u32 qintr_ring_idx;
u32 chan_num;
u32 queue_irq_start;
u32 queue_irq_num;
u32 err_irq_idx;
u32 fid;
};
extern const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX];
extern const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX];
#endif /* __QDMA_H */

36
include/linux/platform_data/amd_qdma.h Normal file
View File

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
*/
#ifndef _PLATDATA_AMD_QDMA_H
#define _PLATDATA_AMD_QDMA_H
#include <linux/dmaengine.h>
/**
* struct qdma_queue_info - DMA queue information. This information is used to
* match queue when DMA channel is requested
* @dir: Channel transfer direction
*/
struct qdma_queue_info {
enum dma_transfer_direction dir;
};
#define QDMA_FILTER_PARAM(qinfo) ((void *)(qinfo))
struct dma_slave_map;
/**
* struct qdma_platdata - Platform specific data for QDMA engine
* @max_mm_channels: Maximum number of MM DMA channels in each direction
* @device_map: DMA slave map
* @irq_index: The index of first IRQ
*/
struct qdma_platdata {
u32 max_mm_channels;
u32 irq_index;
struct dma_slave_map *device_map;
};
#endif /* _PLATDATA_AMD_QDMA_H */