Merge branches 'arm/smmu', 'mediatek', 's390', 'ti/omap', 'riscv' and 'core' into next
Joerg Roedel
commit
ae3325f752
|
|
@ -36,10 +36,12 @@ properties:
|
||||||
items:
|
items:
|
||||||
- enum:
|
- enum:
|
||||||
- qcom,qcm2290-smmu-500
|
- qcom,qcm2290-smmu-500
|
||||||
|
- qcom,qcs615-smmu-500
|
||||||
- qcom,qcs8300-smmu-500
|
- qcom,qcs8300-smmu-500
|
||||||
- qcom,qdu1000-smmu-500
|
- qcom,qdu1000-smmu-500
|
||||||
- qcom,sa8255p-smmu-500
|
- qcom,sa8255p-smmu-500
|
||||||
- qcom,sa8775p-smmu-500
|
- qcom,sa8775p-smmu-500
|
||||||
|
- qcom,sar2130p-smmu-500
|
||||||
- qcom,sc7180-smmu-500
|
- qcom,sc7180-smmu-500
|
||||||
- qcom,sc7280-smmu-500
|
- qcom,sc7280-smmu-500
|
||||||
- qcom,sc8180x-smmu-500
|
- qcom,sc8180x-smmu-500
|
||||||
|
|
@ -88,6 +90,7 @@ properties:
|
||||||
- qcom,qcm2290-smmu-500
|
- qcom,qcm2290-smmu-500
|
||||||
- qcom,sa8255p-smmu-500
|
- qcom,sa8255p-smmu-500
|
||||||
- qcom,sa8775p-smmu-500
|
- qcom,sa8775p-smmu-500
|
||||||
|
- qcom,sar2130p-smmu-500
|
||||||
- qcom,sc7280-smmu-500
|
- qcom,sc7280-smmu-500
|
||||||
- qcom,sc8180x-smmu-500
|
- qcom,sc8180x-smmu-500
|
||||||
- qcom,sc8280xp-smmu-500
|
- qcom,sc8280xp-smmu-500
|
||||||
|
|
@ -524,6 +527,7 @@ allOf:
|
||||||
compatible:
|
compatible:
|
||||||
items:
|
items:
|
||||||
- enum:
|
- enum:
|
||||||
|
- qcom,sar2130p-smmu-500
|
||||||
- qcom,sm8550-smmu-500
|
- qcom,sm8550-smmu-500
|
||||||
- qcom,sm8650-smmu-500
|
- qcom,sm8650-smmu-500
|
||||||
- qcom,x1e80100-smmu-500
|
- qcom,x1e80100-smmu-500
|
||||||
|
|
@ -555,6 +559,7 @@ allOf:
|
||||||
- cavium,smmu-v2
|
- cavium,smmu-v2
|
||||||
- marvell,ap806-smmu-500
|
- marvell,ap806-smmu-500
|
||||||
- nvidia,smmu-500
|
- nvidia,smmu-500
|
||||||
|
- qcom,qcs615-smmu-500
|
||||||
- qcom,qcs8300-smmu-500
|
- qcom,qcs8300-smmu-500
|
||||||
- qcom,qdu1000-smmu-500
|
- qcom,qdu1000-smmu-500
|
||||||
- qcom,sa8255p-smmu-500
|
- qcom,sa8255p-smmu-500
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,147 @@
|
||||||
|
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||||
|
%YAML 1.2
|
||||||
|
---
|
||||||
|
$id: http://devicetree.org/schemas/iommu/riscv,iommu.yaml#
|
||||||
|
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||||
|
|
||||||
|
title: RISC-V IOMMU Architecture Implementation
|
||||||
|
|
||||||
|
maintainers:
|
||||||
|
- Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||||
|
|
||||||
|
description: |
|
||||||
|
The RISC-V IOMMU provides memory address translation and isolation for
|
||||||
|
input and output devices, supporting per-device translation context,
|
||||||
|
shared process address spaces including the ATS and PRI components of
|
||||||
|
the PCIe specification, two stage address translation and MSI remapping.
|
||||||
|
It supports identical translation table format to the RISC-V address
|
||||||
|
translation tables with page level access and protection attributes.
|
||||||
|
Hardware uses in-memory command and fault reporting queues with wired
|
||||||
|
interrupt or MSI notifications.
|
||||||
|
|
||||||
|
Visit https://github.com/riscv-non-isa/riscv-iommu for more details.
|
||||||
|
|
||||||
|
For information on assigning RISC-V IOMMU to its peripheral devices,
|
||||||
|
see generic IOMMU bindings.
|
||||||
|
|
||||||
|
properties:
|
||||||
|
# For PCIe IOMMU hardware compatible property should contain the vendor
|
||||||
|
# and device ID according to the PCI Bus Binding specification.
|
||||||
|
# Since PCI provides built-in identification methods, compatible is not
|
||||||
|
# actually required. For non-PCIe hardware implementations 'riscv,iommu'
|
||||||
|
# should be specified along with 'reg' property providing MMIO location.
|
||||||
|
compatible:
|
||||||
|
oneOf:
|
||||||
|
- items:
|
||||||
|
- enum:
|
||||||
|
- qemu,riscv-iommu
|
||||||
|
- const: riscv,iommu
|
||||||
|
- items:
|
||||||
|
- enum:
|
||||||
|
- pci1efd,edf1
|
||||||
|
- const: riscv,pci-iommu
|
||||||
|
|
||||||
|
reg:
|
||||||
|
maxItems: 1
|
||||||
|
description:
|
||||||
|
For non-PCI devices this represents base address and size of for the
|
||||||
|
IOMMU memory mapped registers interface.
|
||||||
|
For PCI IOMMU hardware implementation this should represent an address
|
||||||
|
of the IOMMU, as defined in the PCI Bus Binding reference.
|
||||||
|
|
||||||
|
'#iommu-cells':
|
||||||
|
const: 1
|
||||||
|
description:
|
||||||
|
The single cell describes the requester id emitted by a master to the
|
||||||
|
IOMMU.
|
||||||
|
|
||||||
|
interrupts:
|
||||||
|
minItems: 1
|
||||||
|
maxItems: 4
|
||||||
|
description:
|
||||||
|
Wired interrupt vectors available for RISC-V IOMMU to notify the
|
||||||
|
RISC-V HARTS. The cause to interrupt vector is software defined
|
||||||
|
using IVEC IOMMU register.
|
||||||
|
|
||||||
|
msi-parent: true
|
||||||
|
|
||||||
|
power-domains:
|
||||||
|
maxItems: 1
|
||||||
|
|
||||||
|
required:
|
||||||
|
- compatible
|
||||||
|
- reg
|
||||||
|
- '#iommu-cells'
|
||||||
|
|
||||||
|
additionalProperties: false
|
||||||
|
|
||||||
|
examples:
|
||||||
|
- |+
|
||||||
|
/* Example 1 (IOMMU device with wired interrupts) */
|
||||||
|
#include <dt-bindings/interrupt-controller/irq.h>
|
||||||
|
|
||||||
|
iommu1: iommu@1bccd000 {
|
||||||
|
compatible = "qemu,riscv-iommu", "riscv,iommu";
|
||||||
|
reg = <0x1bccd000 0x1000>;
|
||||||
|
interrupt-parent = <&aplic_smode>;
|
||||||
|
interrupts = <32 IRQ_TYPE_LEVEL_HIGH>,
|
||||||
|
<33 IRQ_TYPE_LEVEL_HIGH>,
|
||||||
|
<34 IRQ_TYPE_LEVEL_HIGH>,
|
||||||
|
<35 IRQ_TYPE_LEVEL_HIGH>;
|
||||||
|
#iommu-cells = <1>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Device with two IOMMU device IDs, 0 and 7 */
|
||||||
|
master1 {
|
||||||
|
iommus = <&iommu1 0>, <&iommu1 7>;
|
||||||
|
};
|
||||||
|
|
||||||
|
- |+
|
||||||
|
/* Example 2 (IOMMU device with shared wired interrupt) */
|
||||||
|
#include <dt-bindings/interrupt-controller/irq.h>
|
||||||
|
|
||||||
|
iommu2: iommu@1bccd000 {
|
||||||
|
compatible = "qemu,riscv-iommu", "riscv,iommu";
|
||||||
|
reg = <0x1bccd000 0x1000>;
|
||||||
|
interrupt-parent = <&aplic_smode>;
|
||||||
|
interrupts = <32 IRQ_TYPE_LEVEL_HIGH>;
|
||||||
|
#iommu-cells = <1>;
|
||||||
|
};
|
||||||
|
|
||||||
|
- |+
|
||||||
|
/* Example 3 (IOMMU device with MSIs) */
|
||||||
|
iommu3: iommu@1bcdd000 {
|
||||||
|
compatible = "qemu,riscv-iommu", "riscv,iommu";
|
||||||
|
reg = <0x1bccd000 0x1000>;
|
||||||
|
msi-parent = <&imsics_smode>;
|
||||||
|
#iommu-cells = <1>;
|
||||||
|
};
|
||||||
|
|
||||||
|
- |+
|
||||||
|
/* Example 4 (IOMMU PCIe device with MSIs) */
|
||||||
|
bus {
|
||||||
|
#address-cells = <2>;
|
||||||
|
#size-cells = <2>;
|
||||||
|
|
||||||
|
pcie@30000000 {
|
||||||
|
device_type = "pci";
|
||||||
|
#address-cells = <3>;
|
||||||
|
#size-cells = <2>;
|
||||||
|
reg = <0x0 0x30000000 0x0 0x1000000>;
|
||||||
|
ranges = <0x02000000 0x0 0x41000000 0x0 0x41000000 0x0 0x0f000000>;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The IOMMU manages all functions in this PCI domain except
|
||||||
|
* itself. Omit BDF 00:01.0.
|
||||||
|
*/
|
||||||
|
iommu-map = <0x0 &iommu0 0x0 0x8>,
|
||||||
|
<0x9 &iommu0 0x9 0xfff7>;
|
||||||
|
|
||||||
|
/* The IOMMU programming interface uses slot 00:01.0 */
|
||||||
|
iommu0: iommu@1,0 {
|
||||||
|
compatible = "pci1efd,edf1", "riscv,pci-iommu";
|
||||||
|
reg = <0x800 0 0 0 0>;
|
||||||
|
#iommu-cells = <1>;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
@ -19810,6 +19810,15 @@ F: arch/riscv/
|
||||||
N: riscv
|
N: riscv
|
||||||
K: riscv
|
K: riscv
|
||||||
|
|
||||||
|
RISC-V IOMMU
|
||||||
|
M: Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||||
|
L: iommu@lists.linux.dev
|
||||||
|
L: linux-riscv@lists.infradead.org
|
||||||
|
S: Maintained
|
||||||
|
T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
|
||||||
|
F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml
|
||||||
|
F: drivers/iommu/riscv/
|
||||||
|
|
||||||
RISC-V MICROCHIP FPGA SUPPORT
|
RISC-V MICROCHIP FPGA SUPPORT
|
||||||
M: Conor Dooley <conor.dooley@microchip.com>
|
M: Conor Dooley <conor.dooley@microchip.com>
|
||||||
M: Daire McNamara <daire.mcnamara@microchip.com>
|
M: Daire McNamara <daire.mcnamara@microchip.com>
|
||||||
|
|
|
||||||
|
|
@ -96,7 +96,6 @@ struct zpci_bar_struct {
|
||||||
u8 size; /* order 2 exponent */
|
u8 size; /* order 2 exponent */
|
||||||
};
|
};
|
||||||
|
|
||||||
struct s390_domain;
|
|
||||||
struct kvm_zdev;
|
struct kvm_zdev;
|
||||||
|
|
||||||
#define ZPCI_FUNCTIONS_PER_BUS 256
|
#define ZPCI_FUNCTIONS_PER_BUS 256
|
||||||
|
|
@ -181,9 +180,10 @@ struct zpci_dev {
|
||||||
struct dentry *debugfs_dev;
|
struct dentry *debugfs_dev;
|
||||||
|
|
||||||
/* IOMMU and passthrough */
|
/* IOMMU and passthrough */
|
||||||
struct s390_domain *s390_domain; /* s390 IOMMU domain data */
|
struct iommu_domain *s390_domain; /* attached IOMMU domain */
|
||||||
struct kvm_zdev *kzdev;
|
struct kvm_zdev *kzdev;
|
||||||
struct mutex kzdev_lock;
|
struct mutex kzdev_lock;
|
||||||
|
spinlock_t dom_lock; /* protect s390_domain change */
|
||||||
};
|
};
|
||||||
|
|
||||||
static inline bool zdev_enabled(struct zpci_dev *zdev)
|
static inline bool zdev_enabled(struct zpci_dev *zdev)
|
||||||
|
|
|
||||||
|
|
@ -160,6 +160,7 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
|
||||||
u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
|
u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
|
||||||
struct zpci_iommu_ctrs *ctrs;
|
struct zpci_iommu_ctrs *ctrs;
|
||||||
struct zpci_fib fib = {0};
|
struct zpci_fib fib = {0};
|
||||||
|
unsigned long flags;
|
||||||
u8 cc, status;
|
u8 cc, status;
|
||||||
|
|
||||||
if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
|
if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
|
||||||
|
|
@ -171,6 +172,7 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
|
||||||
WARN_ON((u64) zdev->fmb & 0xf);
|
WARN_ON((u64) zdev->fmb & 0xf);
|
||||||
|
|
||||||
/* reset software counters */
|
/* reset software counters */
|
||||||
|
spin_lock_irqsave(&zdev->dom_lock, flags);
|
||||||
ctrs = zpci_get_iommu_ctrs(zdev);
|
ctrs = zpci_get_iommu_ctrs(zdev);
|
||||||
if (ctrs) {
|
if (ctrs) {
|
||||||
atomic64_set(&ctrs->mapped_pages, 0);
|
atomic64_set(&ctrs->mapped_pages, 0);
|
||||||
|
|
@ -179,6 +181,7 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
|
||||||
atomic64_set(&ctrs->sync_map_rpcits, 0);
|
atomic64_set(&ctrs->sync_map_rpcits, 0);
|
||||||
atomic64_set(&ctrs->sync_rpcits, 0);
|
atomic64_set(&ctrs->sync_rpcits, 0);
|
||||||
}
|
}
|
||||||
|
spin_unlock_irqrestore(&zdev->dom_lock, flags);
|
||||||
|
|
||||||
|
|
||||||
fib.fmb_addr = virt_to_phys(zdev->fmb);
|
fib.fmb_addr = virt_to_phys(zdev->fmb);
|
||||||
|
|
|
||||||
|
|
@ -71,17 +71,23 @@ static void pci_fmb_show(struct seq_file *m, char *name[], int length,
|
||||||
|
|
||||||
static void pci_sw_counter_show(struct seq_file *m)
|
static void pci_sw_counter_show(struct seq_file *m)
|
||||||
{
|
{
|
||||||
struct zpci_iommu_ctrs *ctrs = zpci_get_iommu_ctrs(m->private);
|
struct zpci_dev *zdev = m->private;
|
||||||
|
struct zpci_iommu_ctrs *ctrs;
|
||||||
atomic64_t *counter;
|
atomic64_t *counter;
|
||||||
|
unsigned long flags;
|
||||||
int i;
|
int i;
|
||||||
|
|
||||||
|
spin_lock_irqsave(&zdev->dom_lock, flags);
|
||||||
|
ctrs = zpci_get_iommu_ctrs(m->private);
|
||||||
if (!ctrs)
|
if (!ctrs)
|
||||||
return;
|
goto unlock;
|
||||||
|
|
||||||
counter = &ctrs->mapped_pages;
|
counter = &ctrs->mapped_pages;
|
||||||
for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++)
|
for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++)
|
||||||
seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i],
|
seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i],
|
||||||
atomic64_read(counter));
|
atomic64_read(counter));
|
||||||
|
unlock:
|
||||||
|
spin_unlock_irqrestore(&zdev->dom_lock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int pci_perf_show(struct seq_file *m, void *v)
|
static int pci_perf_show(struct seq_file *m, void *v)
|
||||||
|
|
|
||||||
|
|
@ -120,8 +120,8 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev)
|
||||||
mutex_init(&tdev->iommu.mutex);
|
mutex_init(&tdev->iommu.mutex);
|
||||||
|
|
||||||
if (device_iommu_mapped(dev)) {
|
if (device_iommu_mapped(dev)) {
|
||||||
tdev->iommu.domain = iommu_domain_alloc(&platform_bus_type);
|
tdev->iommu.domain = iommu_paging_domain_alloc(dev);
|
||||||
if (!tdev->iommu.domain)
|
if (IS_ERR(tdev->iommu.domain))
|
||||||
goto error;
|
goto error;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
||||||
|
|
@ -195,6 +195,7 @@ config MSM_IOMMU
|
||||||
source "drivers/iommu/amd/Kconfig"
|
source "drivers/iommu/amd/Kconfig"
|
||||||
source "drivers/iommu/intel/Kconfig"
|
source "drivers/iommu/intel/Kconfig"
|
||||||
source "drivers/iommu/iommufd/Kconfig"
|
source "drivers/iommu/iommufd/Kconfig"
|
||||||
|
source "drivers/iommu/riscv/Kconfig"
|
||||||
|
|
||||||
config IRQ_REMAP
|
config IRQ_REMAP
|
||||||
bool "Support for Interrupt Remapping"
|
bool "Support for Interrupt Remapping"
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
# SPDX-License-Identifier: GPL-2.0
|
# SPDX-License-Identifier: GPL-2.0
|
||||||
obj-y += amd/ intel/ arm/ iommufd/
|
obj-y += amd/ intel/ arm/ iommufd/ riscv/
|
||||||
obj-$(CONFIG_IOMMU_API) += iommu.o
|
obj-$(CONFIG_IOMMU_API) += iommu.o
|
||||||
obj-$(CONFIG_IOMMU_API) += iommu-traces.o
|
obj-$(CONFIG_IOMMU_API) += iommu-traces.o
|
||||||
obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
|
obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,7 @@ extern int amd_iommu_gpt_level;
|
||||||
extern unsigned long amd_iommu_pgsize_bitmap;
|
extern unsigned long amd_iommu_pgsize_bitmap;
|
||||||
|
|
||||||
/* Protection domain ops */
|
/* Protection domain ops */
|
||||||
|
void amd_iommu_init_identity_domain(void);
|
||||||
struct protection_domain *protection_domain_alloc(unsigned int type, int nid);
|
struct protection_domain *protection_domain_alloc(unsigned int type, int nid);
|
||||||
void protection_domain_free(struct protection_domain *domain);
|
void protection_domain_free(struct protection_domain *domain);
|
||||||
struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev,
|
struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev,
|
||||||
|
|
@ -118,9 +119,14 @@ static inline bool check_feature2(u64 mask)
|
||||||
return (amd_iommu_efr2 & mask);
|
return (amd_iommu_efr2 & mask);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline bool amd_iommu_v2_pgtbl_supported(void)
|
||||||
|
{
|
||||||
|
return (check_feature(FEATURE_GIOSUP) && check_feature(FEATURE_GT));
|
||||||
|
}
|
||||||
|
|
||||||
static inline bool amd_iommu_gt_ppr_supported(void)
|
static inline bool amd_iommu_gt_ppr_supported(void)
|
||||||
{
|
{
|
||||||
return (check_feature(FEATURE_GT) &&
|
return (amd_iommu_v2_pgtbl_supported() &&
|
||||||
check_feature(FEATURE_PPR) &&
|
check_feature(FEATURE_PPR) &&
|
||||||
check_feature(FEATURE_EPHSUP));
|
check_feature(FEATURE_EPHSUP));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2070,14 +2070,6 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
|
||||||
|
|
||||||
init_iommu_perf_ctr(iommu);
|
init_iommu_perf_ctr(iommu);
|
||||||
|
|
||||||
if (amd_iommu_pgtable == AMD_IOMMU_V2) {
|
|
||||||
if (!check_feature(FEATURE_GIOSUP) ||
|
|
||||||
!check_feature(FEATURE_GT)) {
|
|
||||||
pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
|
|
||||||
amd_iommu_pgtable = AMD_IOMMU_V1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (is_rd890_iommu(iommu->dev)) {
|
if (is_rd890_iommu(iommu->dev)) {
|
||||||
int i, j;
|
int i, j;
|
||||||
|
|
||||||
|
|
@ -2172,6 +2164,9 @@ static int __init amd_iommu_init_pci(void)
|
||||||
struct amd_iommu_pci_seg *pci_seg;
|
struct amd_iommu_pci_seg *pci_seg;
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
/* Init global identity domain before registering IOMMU */
|
||||||
|
amd_iommu_init_identity_domain();
|
||||||
|
|
||||||
for_each_iommu(iommu) {
|
for_each_iommu(iommu) {
|
||||||
ret = iommu_init_pci(iommu);
|
ret = iommu_init_pci(iommu);
|
||||||
if (ret) {
|
if (ret) {
|
||||||
|
|
@ -3091,6 +3086,13 @@ static int __init early_amd_iommu_init(void)
|
||||||
FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL)
|
FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL)
|
||||||
amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
|
amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
|
||||||
|
|
||||||
|
if (amd_iommu_pgtable == AMD_IOMMU_V2) {
|
||||||
|
if (!amd_iommu_v2_pgtbl_supported()) {
|
||||||
|
pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
|
||||||
|
amd_iommu_pgtable = AMD_IOMMU_V1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Disable any previously enabled IOMMUs */
|
/* Disable any previously enabled IOMMUs */
|
||||||
if (!is_kdump_kernel() || amd_iommu_disabled)
|
if (!is_kdump_kernel() || amd_iommu_disabled)
|
||||||
disable_iommus();
|
disable_iommus();
|
||||||
|
|
|
||||||
|
|
@ -74,6 +74,9 @@ struct kmem_cache *amd_iommu_irq_cache;
|
||||||
|
|
||||||
static void detach_device(struct device *dev);
|
static void detach_device(struct device *dev);
|
||||||
|
|
||||||
|
static int amd_iommu_attach_device(struct iommu_domain *dom,
|
||||||
|
struct device *dev);
|
||||||
|
|
||||||
static void set_dte_entry(struct amd_iommu *iommu,
|
static void set_dte_entry(struct amd_iommu *iommu,
|
||||||
struct iommu_dev_data *dev_data);
|
struct iommu_dev_data *dev_data);
|
||||||
|
|
||||||
|
|
@ -2263,44 +2266,42 @@ void protection_domain_free(struct protection_domain *domain)
|
||||||
kfree(domain);
|
kfree(domain);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void protection_domain_init(struct protection_domain *domain, int nid)
|
||||||
|
{
|
||||||
|
spin_lock_init(&domain->lock);
|
||||||
|
INIT_LIST_HEAD(&domain->dev_list);
|
||||||
|
INIT_LIST_HEAD(&domain->dev_data_list);
|
||||||
|
domain->iop.pgtbl.cfg.amd.nid = nid;
|
||||||
|
}
|
||||||
|
|
||||||
struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
|
struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
|
||||||
{
|
{
|
||||||
struct io_pgtable_ops *pgtbl_ops;
|
|
||||||
struct protection_domain *domain;
|
struct protection_domain *domain;
|
||||||
int pgtable;
|
|
||||||
|
|
||||||
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
|
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
|
||||||
if (!domain)
|
if (!domain)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
domain->id = domain_id_alloc();
|
domain->id = domain_id_alloc();
|
||||||
if (!domain->id)
|
if (!domain->id) {
|
||||||
goto err_free;
|
kfree(domain);
|
||||||
|
return NULL;
|
||||||
spin_lock_init(&domain->lock);
|
|
||||||
INIT_LIST_HEAD(&domain->dev_list);
|
|
||||||
INIT_LIST_HEAD(&domain->dev_data_list);
|
|
||||||
domain->iop.pgtbl.cfg.amd.nid = nid;
|
|
||||||
|
|
||||||
switch (type) {
|
|
||||||
/* No need to allocate io pgtable ops in passthrough mode */
|
|
||||||
case IOMMU_DOMAIN_IDENTITY:
|
|
||||||
case IOMMU_DOMAIN_SVA:
|
|
||||||
return domain;
|
|
||||||
case IOMMU_DOMAIN_DMA:
|
|
||||||
pgtable = amd_iommu_pgtable;
|
|
||||||
break;
|
|
||||||
/*
|
|
||||||
* Force IOMMU v1 page table when allocating
|
|
||||||
* domain for pass-through devices.
|
|
||||||
*/
|
|
||||||
case IOMMU_DOMAIN_UNMANAGED:
|
|
||||||
pgtable = AMD_IOMMU_V1;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
goto err_id;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protection_domain_init(domain, nid);
|
||||||
|
|
||||||
|
return domain;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int pdom_setup_pgtable(struct protection_domain *domain,
|
||||||
|
unsigned int type, int pgtable)
|
||||||
|
{
|
||||||
|
struct io_pgtable_ops *pgtbl_ops;
|
||||||
|
|
||||||
|
/* No need to allocate io pgtable ops in passthrough mode */
|
||||||
|
if (!(type & __IOMMU_DOMAIN_PAGING))
|
||||||
|
return 0;
|
||||||
|
|
||||||
switch (pgtable) {
|
switch (pgtable) {
|
||||||
case AMD_IOMMU_V1:
|
case AMD_IOMMU_V1:
|
||||||
domain->pd_mode = PD_MODE_V1;
|
domain->pd_mode = PD_MODE_V1;
|
||||||
|
|
@ -2309,25 +2310,20 @@ struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
|
||||||
domain->pd_mode = PD_MODE_V2;
|
domain->pd_mode = PD_MODE_V2;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
goto err_id;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
|
|
||||||
pgtbl_ops =
|
pgtbl_ops =
|
||||||
alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl.cfg, domain);
|
alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl.cfg, domain);
|
||||||
if (!pgtbl_ops)
|
if (!pgtbl_ops)
|
||||||
goto err_id;
|
return -ENOMEM;
|
||||||
|
|
||||||
return domain;
|
return 0;
|
||||||
err_id:
|
|
||||||
domain_id_free(domain->id);
|
|
||||||
err_free:
|
|
||||||
kfree(domain);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline u64 dma_max_address(void)
|
static inline u64 dma_max_address(int pgtable)
|
||||||
{
|
{
|
||||||
if (amd_iommu_pgtable == AMD_IOMMU_V1)
|
if (pgtable == AMD_IOMMU_V1)
|
||||||
return ~0ULL;
|
return ~0ULL;
|
||||||
|
|
||||||
/* V2 with 4/5 level page table */
|
/* V2 with 4/5 level page table */
|
||||||
|
|
@ -2340,11 +2336,13 @@ static bool amd_iommu_hd_support(struct amd_iommu *iommu)
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
|
static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
|
||||||
struct device *dev, u32 flags)
|
struct device *dev,
|
||||||
|
u32 flags, int pgtable)
|
||||||
{
|
{
|
||||||
bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
|
bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
|
||||||
struct protection_domain *domain;
|
struct protection_domain *domain;
|
||||||
struct amd_iommu *iommu = NULL;
|
struct amd_iommu *iommu = NULL;
|
||||||
|
int ret;
|
||||||
|
|
||||||
if (dev)
|
if (dev)
|
||||||
iommu = get_amd_iommu_from_dev(dev);
|
iommu = get_amd_iommu_from_dev(dev);
|
||||||
|
|
@ -2356,16 +2354,20 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
|
||||||
if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY))
|
if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY))
|
||||||
return ERR_PTR(-EINVAL);
|
return ERR_PTR(-EINVAL);
|
||||||
|
|
||||||
if (dirty_tracking && !amd_iommu_hd_support(iommu))
|
|
||||||
return ERR_PTR(-EOPNOTSUPP);
|
|
||||||
|
|
||||||
domain = protection_domain_alloc(type,
|
domain = protection_domain_alloc(type,
|
||||||
dev ? dev_to_node(dev) : NUMA_NO_NODE);
|
dev ? dev_to_node(dev) : NUMA_NO_NODE);
|
||||||
if (!domain)
|
if (!domain)
|
||||||
return ERR_PTR(-ENOMEM);
|
return ERR_PTR(-ENOMEM);
|
||||||
|
|
||||||
|
ret = pdom_setup_pgtable(domain, type, pgtable);
|
||||||
|
if (ret) {
|
||||||
|
domain_id_free(domain->id);
|
||||||
|
kfree(domain);
|
||||||
|
return ERR_PTR(ret);
|
||||||
|
}
|
||||||
|
|
||||||
domain->domain.geometry.aperture_start = 0;
|
domain->domain.geometry.aperture_start = 0;
|
||||||
domain->domain.geometry.aperture_end = dma_max_address();
|
domain->domain.geometry.aperture_end = dma_max_address(pgtable);
|
||||||
domain->domain.geometry.force_aperture = true;
|
domain->domain.geometry.force_aperture = true;
|
||||||
domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap;
|
domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap;
|
||||||
|
|
||||||
|
|
@ -2383,8 +2385,16 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
|
||||||
static struct iommu_domain *amd_iommu_domain_alloc(unsigned int type)
|
static struct iommu_domain *amd_iommu_domain_alloc(unsigned int type)
|
||||||
{
|
{
|
||||||
struct iommu_domain *domain;
|
struct iommu_domain *domain;
|
||||||
|
int pgtable = amd_iommu_pgtable;
|
||||||
|
|
||||||
domain = do_iommu_domain_alloc(type, NULL, 0);
|
/*
|
||||||
|
* Force IOMMU v1 page table when allocating
|
||||||
|
* domain for pass-through devices.
|
||||||
|
*/
|
||||||
|
if (type == IOMMU_DOMAIN_UNMANAGED)
|
||||||
|
pgtable = AMD_IOMMU_V1;
|
||||||
|
|
||||||
|
domain = do_iommu_domain_alloc(type, NULL, 0, pgtable);
|
||||||
if (IS_ERR(domain))
|
if (IS_ERR(domain))
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
|
|
@ -2398,11 +2408,36 @@ amd_iommu_domain_alloc_user(struct device *dev, u32 flags,
|
||||||
|
|
||||||
{
|
{
|
||||||
unsigned int type = IOMMU_DOMAIN_UNMANAGED;
|
unsigned int type = IOMMU_DOMAIN_UNMANAGED;
|
||||||
|
struct amd_iommu *iommu = NULL;
|
||||||
|
const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
|
||||||
|
IOMMU_HWPT_ALLOC_PASID;
|
||||||
|
|
||||||
if ((flags & ~IOMMU_HWPT_ALLOC_DIRTY_TRACKING) || parent || user_data)
|
if (dev)
|
||||||
|
iommu = get_amd_iommu_from_dev(dev);
|
||||||
|
|
||||||
|
if ((flags & ~supported_flags) || parent || user_data)
|
||||||
return ERR_PTR(-EOPNOTSUPP);
|
return ERR_PTR(-EOPNOTSUPP);
|
||||||
|
|
||||||
return do_iommu_domain_alloc(type, dev, flags);
|
/* Allocate domain with v2 page table if IOMMU supports PASID. */
|
||||||
|
if (flags & IOMMU_HWPT_ALLOC_PASID) {
|
||||||
|
if (!amd_iommu_pasid_supported())
|
||||||
|
return ERR_PTR(-EOPNOTSUPP);
|
||||||
|
|
||||||
|
return do_iommu_domain_alloc(type, dev, flags, AMD_IOMMU_V2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Allocate domain with v1 page table for dirty tracking */
|
||||||
|
if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) {
|
||||||
|
if (iommu && amd_iommu_hd_support(iommu)) {
|
||||||
|
return do_iommu_domain_alloc(type, dev,
|
||||||
|
flags, AMD_IOMMU_V1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ERR_PTR(-EOPNOTSUPP);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* If nothing specific is required use the kernel commandline default */
|
||||||
|
return do_iommu_domain_alloc(type, dev, 0, amd_iommu_pgtable);
|
||||||
}
|
}
|
||||||
|
|
||||||
void amd_iommu_domain_free(struct iommu_domain *dom)
|
void amd_iommu_domain_free(struct iommu_domain *dom)
|
||||||
|
|
@ -2444,6 +2479,25 @@ static struct iommu_domain blocked_domain = {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static struct protection_domain identity_domain;
|
||||||
|
|
||||||
|
static const struct iommu_domain_ops identity_domain_ops = {
|
||||||
|
.attach_dev = amd_iommu_attach_device,
|
||||||
|
};
|
||||||
|
|
||||||
|
void amd_iommu_init_identity_domain(void)
|
||||||
|
{
|
||||||
|
struct iommu_domain *domain = &identity_domain.domain;
|
||||||
|
|
||||||
|
domain->type = IOMMU_DOMAIN_IDENTITY;
|
||||||
|
domain->ops = &identity_domain_ops;
|
||||||
|
domain->owner = &amd_iommu_ops;
|
||||||
|
|
||||||
|
identity_domain.id = domain_id_alloc();
|
||||||
|
|
||||||
|
protection_domain_init(&identity_domain, NUMA_NO_NODE);
|
||||||
|
}
|
||||||
|
|
||||||
static int amd_iommu_attach_device(struct iommu_domain *dom,
|
static int amd_iommu_attach_device(struct iommu_domain *dom,
|
||||||
struct device *dev)
|
struct device *dev)
|
||||||
{
|
{
|
||||||
|
|
@ -2842,6 +2896,7 @@ static int amd_iommu_dev_disable_feature(struct device *dev,
|
||||||
const struct iommu_ops amd_iommu_ops = {
|
const struct iommu_ops amd_iommu_ops = {
|
||||||
.capable = amd_iommu_capable,
|
.capable = amd_iommu_capable,
|
||||||
.blocked_domain = &blocked_domain,
|
.blocked_domain = &blocked_domain,
|
||||||
|
.identity_domain = &identity_domain.domain,
|
||||||
.domain_alloc = amd_iommu_domain_alloc,
|
.domain_alloc = amd_iommu_domain_alloc,
|
||||||
.domain_alloc_user = amd_iommu_domain_alloc_user,
|
.domain_alloc_user = amd_iommu_domain_alloc_user,
|
||||||
.domain_alloc_sva = amd_iommu_domain_alloc_sva,
|
.domain_alloc_sva = amd_iommu_domain_alloc_sva,
|
||||||
|
|
|
||||||
|
|
@ -3084,7 +3084,8 @@ arm_smmu_domain_alloc_user(struct device *dev, u32 flags,
|
||||||
const struct iommu_user_data *user_data)
|
const struct iommu_user_data *user_data)
|
||||||
{
|
{
|
||||||
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
|
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
|
||||||
const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
|
const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
|
||||||
|
IOMMU_HWPT_ALLOC_PASID;
|
||||||
struct arm_smmu_domain *smmu_domain;
|
struct arm_smmu_domain *smmu_domain;
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
|
@ -3093,6 +3094,9 @@ arm_smmu_domain_alloc_user(struct device *dev, u32 flags,
|
||||||
if (parent || user_data)
|
if (parent || user_data)
|
||||||
return ERR_PTR(-EOPNOTSUPP);
|
return ERR_PTR(-EOPNOTSUPP);
|
||||||
|
|
||||||
|
if (flags & IOMMU_HWPT_ALLOC_PASID)
|
||||||
|
return arm_smmu_domain_alloc_paging(dev);
|
||||||
|
|
||||||
smmu_domain = arm_smmu_domain_alloc();
|
smmu_domain = arm_smmu_domain_alloc();
|
||||||
if (IS_ERR(smmu_domain))
|
if (IS_ERR(smmu_domain))
|
||||||
return ERR_CAST(smmu_domain);
|
return ERR_CAST(smmu_domain);
|
||||||
|
|
|
||||||
|
|
@ -509,7 +509,8 @@ static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq)
|
||||||
|
|
||||||
snprintf(name, 16, "vcmdq%u", vcmdq->idx);
|
snprintf(name, 16, "vcmdq%u", vcmdq->idx);
|
||||||
|
|
||||||
q->llq.max_n_shift = VCMDQ_LOG2SIZE_MAX;
|
/* Queue size, capped to ensure natural alignment */
|
||||||
|
q->llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, VCMDQ_LOG2SIZE_MAX);
|
||||||
|
|
||||||
/* Use the common helper to init the VCMDQ, and then... */
|
/* Use the common helper to init the VCMDQ, and then... */
|
||||||
ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0,
|
ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0,
|
||||||
|
|
@ -800,7 +801,9 @@ out_fallback:
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
struct dentry *cmdqv_debugfs_dir;
|
#ifdef CONFIG_IOMMU_DEBUGFS
|
||||||
|
static struct dentry *cmdqv_debugfs_dir;
|
||||||
|
#endif
|
||||||
|
|
||||||
static struct arm_smmu_device *
|
static struct arm_smmu_device *
|
||||||
__tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
|
__tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
|
||||||
|
|
|
||||||
|
|
@ -1437,6 +1437,17 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
|
||||||
goto out_free;
|
goto out_free;
|
||||||
} else {
|
} else {
|
||||||
smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
|
smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Defer probe if the relevant SMMU instance hasn't finished
|
||||||
|
* probing yet. This is a fragile hack and we'd ideally
|
||||||
|
* avoid this race in the core code. Until that's ironed
|
||||||
|
* out, however, this is the most pragmatic option on the
|
||||||
|
* table.
|
||||||
|
*/
|
||||||
|
if (!smmu)
|
||||||
|
return ERR_PTR(dev_err_probe(dev, -EPROBE_DEFER,
|
||||||
|
"smmu dev has not bound yet\n"));
|
||||||
}
|
}
|
||||||
|
|
||||||
ret = -EINVAL;
|
ret = -EINVAL;
|
||||||
|
|
|
||||||
|
|
@ -166,7 +166,6 @@ struct arm_v7s_io_pgtable {
|
||||||
|
|
||||||
arm_v7s_iopte *pgd;
|
arm_v7s_iopte *pgd;
|
||||||
struct kmem_cache *l2_tables;
|
struct kmem_cache *l2_tables;
|
||||||
spinlock_t split_lock;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
|
static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
|
||||||
|
|
@ -363,25 +362,6 @@ static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl,
|
||||||
return pte;
|
return pte;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl)
|
|
||||||
{
|
|
||||||
int prot = IOMMU_READ;
|
|
||||||
arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl);
|
|
||||||
|
|
||||||
if (!(attr & ARM_V7S_PTE_AP_RDONLY))
|
|
||||||
prot |= IOMMU_WRITE;
|
|
||||||
if (!(attr & ARM_V7S_PTE_AP_UNPRIV))
|
|
||||||
prot |= IOMMU_PRIV;
|
|
||||||
if ((attr & (ARM_V7S_TEX_MASK << ARM_V7S_TEX_SHIFT)) == 0)
|
|
||||||
prot |= IOMMU_MMIO;
|
|
||||||
else if (pte & ARM_V7S_ATTR_C)
|
|
||||||
prot |= IOMMU_CACHE;
|
|
||||||
if (pte & ARM_V7S_ATTR_XN(lvl))
|
|
||||||
prot |= IOMMU_NOEXEC;
|
|
||||||
|
|
||||||
return prot;
|
|
||||||
}
|
|
||||||
|
|
||||||
static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)
|
static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)
|
||||||
{
|
{
|
||||||
if (lvl == 1) {
|
if (lvl == 1) {
|
||||||
|
|
@ -398,23 +378,6 @@ static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)
|
||||||
return pte;
|
return pte;
|
||||||
}
|
}
|
||||||
|
|
||||||
static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl)
|
|
||||||
{
|
|
||||||
if (lvl == 1) {
|
|
||||||
pte &= ~ARM_V7S_CONT_SECTION;
|
|
||||||
} else if (lvl == 2) {
|
|
||||||
arm_v7s_iopte xn = pte & BIT(ARM_V7S_CONT_PAGE_XN_SHIFT);
|
|
||||||
arm_v7s_iopte tex = pte & (ARM_V7S_CONT_PAGE_TEX_MASK <<
|
|
||||||
ARM_V7S_CONT_PAGE_TEX_SHIFT);
|
|
||||||
|
|
||||||
pte ^= xn | tex | ARM_V7S_PTE_TYPE_CONT_PAGE;
|
|
||||||
pte |= (xn >> ARM_V7S_CONT_PAGE_XN_SHIFT) |
|
|
||||||
(tex >> ARM_V7S_CONT_PAGE_TEX_SHIFT) |
|
|
||||||
ARM_V7S_PTE_TYPE_PAGE;
|
|
||||||
}
|
|
||||||
return pte;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl)
|
static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl)
|
||||||
{
|
{
|
||||||
if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl))
|
if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl))
|
||||||
|
|
@ -591,77 +554,6 @@ static void arm_v7s_free_pgtable(struct io_pgtable *iop)
|
||||||
kfree(data);
|
kfree(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
static arm_v7s_iopte arm_v7s_split_cont(struct arm_v7s_io_pgtable *data,
|
|
||||||
unsigned long iova, int idx, int lvl,
|
|
||||||
arm_v7s_iopte *ptep)
|
|
||||||
{
|
|
||||||
struct io_pgtable *iop = &data->iop;
|
|
||||||
arm_v7s_iopte pte;
|
|
||||||
size_t size = ARM_V7S_BLOCK_SIZE(lvl);
|
|
||||||
int i;
|
|
||||||
|
|
||||||
/* Check that we didn't lose a race to get the lock */
|
|
||||||
pte = *ptep;
|
|
||||||
if (!arm_v7s_pte_is_cont(pte, lvl))
|
|
||||||
return pte;
|
|
||||||
|
|
||||||
ptep -= idx & (ARM_V7S_CONT_PAGES - 1);
|
|
||||||
pte = arm_v7s_cont_to_pte(pte, lvl);
|
|
||||||
for (i = 0; i < ARM_V7S_CONT_PAGES; i++)
|
|
||||||
ptep[i] = pte + i * size;
|
|
||||||
|
|
||||||
__arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg);
|
|
||||||
|
|
||||||
size *= ARM_V7S_CONT_PAGES;
|
|
||||||
io_pgtable_tlb_flush_walk(iop, iova, size, size);
|
|
||||||
return pte;
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data,
|
|
||||||
struct iommu_iotlb_gather *gather,
|
|
||||||
unsigned long iova, size_t size,
|
|
||||||
arm_v7s_iopte blk_pte,
|
|
||||||
arm_v7s_iopte *ptep)
|
|
||||||
{
|
|
||||||
struct io_pgtable_cfg *cfg = &data->iop.cfg;
|
|
||||||
arm_v7s_iopte pte, *tablep;
|
|
||||||
int i, unmap_idx, num_entries, num_ptes;
|
|
||||||
|
|
||||||
tablep = __arm_v7s_alloc_table(2, GFP_ATOMIC, data);
|
|
||||||
if (!tablep)
|
|
||||||
return 0; /* Bytes unmapped */
|
|
||||||
|
|
||||||
num_ptes = ARM_V7S_PTES_PER_LVL(2, cfg);
|
|
||||||
num_entries = size >> ARM_V7S_LVL_SHIFT(2);
|
|
||||||
unmap_idx = ARM_V7S_LVL_IDX(iova, 2, cfg);
|
|
||||||
|
|
||||||
pte = arm_v7s_prot_to_pte(arm_v7s_pte_to_prot(blk_pte, 1), 2, cfg);
|
|
||||||
if (num_entries > 1)
|
|
||||||
pte = arm_v7s_pte_to_cont(pte, 2);
|
|
||||||
|
|
||||||
for (i = 0; i < num_ptes; i += num_entries, pte += size) {
|
|
||||||
/* Unmap! */
|
|
||||||
if (i == unmap_idx)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
__arm_v7s_set_pte(&tablep[i], pte, num_entries, cfg);
|
|
||||||
}
|
|
||||||
|
|
||||||
pte = arm_v7s_install_table(tablep, ptep, blk_pte, cfg);
|
|
||||||
if (pte != blk_pte) {
|
|
||||||
__arm_v7s_free_table(tablep, 2, data);
|
|
||||||
|
|
||||||
if (!ARM_V7S_PTE_IS_TABLE(pte, 1))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
tablep = iopte_deref(pte, 1, data);
|
|
||||||
return __arm_v7s_unmap(data, gather, iova, size, 2, tablep);
|
|
||||||
}
|
|
||||||
|
|
||||||
io_pgtable_tlb_add_page(&data->iop, gather, iova, size);
|
|
||||||
return size;
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
|
static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
|
||||||
struct iommu_iotlb_gather *gather,
|
struct iommu_iotlb_gather *gather,
|
||||||
unsigned long iova, size_t size, int lvl,
|
unsigned long iova, size_t size, int lvl,
|
||||||
|
|
@ -694,11 +586,8 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
|
||||||
* case in a lock for the sake of correctness and be done with it.
|
* case in a lock for the sake of correctness and be done with it.
|
||||||
*/
|
*/
|
||||||
if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) {
|
if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) {
|
||||||
unsigned long flags;
|
WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
|
||||||
|
return 0;
|
||||||
spin_lock_irqsave(&data->split_lock, flags);
|
|
||||||
pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep);
|
|
||||||
spin_unlock_irqrestore(&data->split_lock, flags);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* If the size matches this level, we're in the right place */
|
/* If the size matches this level, we're in the right place */
|
||||||
|
|
@ -721,12 +610,8 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
|
||||||
}
|
}
|
||||||
return size;
|
return size;
|
||||||
} else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) {
|
} else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) {
|
||||||
/*
|
WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
|
||||||
* Insert a table at the next level to map the old region,
|
return 0;
|
||||||
* minus the part we want to unmap
|
|
||||||
*/
|
|
||||||
return arm_v7s_split_blk_unmap(data, gather, iova, size, pte[0],
|
|
||||||
ptep);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Keep on walkin' */
|
/* Keep on walkin' */
|
||||||
|
|
@ -811,8 +696,6 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
|
||||||
if (!data)
|
if (!data)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
spin_lock_init(&data->split_lock);
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* ARM_MTK_TTBR_EXT extend the translation table base support larger
|
* ARM_MTK_TTBR_EXT extend the translation table base support larger
|
||||||
* memory address.
|
* memory address.
|
||||||
|
|
@ -936,8 +819,8 @@ static int __init arm_v7s_do_selftests(void)
|
||||||
.quirks = IO_PGTABLE_QUIRK_ARM_NS,
|
.quirks = IO_PGTABLE_QUIRK_ARM_NS,
|
||||||
.pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
|
.pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
|
||||||
};
|
};
|
||||||
unsigned int iova, size, iova_start;
|
unsigned int iova, size;
|
||||||
unsigned int i, loopnr = 0;
|
unsigned int i;
|
||||||
size_t mapped;
|
size_t mapped;
|
||||||
|
|
||||||
selftest_running = true;
|
selftest_running = true;
|
||||||
|
|
@ -985,26 +868,6 @@ static int __init arm_v7s_do_selftests(void)
|
||||||
return __FAIL(ops);
|
return __FAIL(ops);
|
||||||
|
|
||||||
iova += SZ_16M;
|
iova += SZ_16M;
|
||||||
loopnr++;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Partial unmap */
|
|
||||||
i = 1;
|
|
||||||
size = 1UL << __ffs(cfg.pgsize_bitmap);
|
|
||||||
while (i < loopnr) {
|
|
||||||
iova_start = i * SZ_16M;
|
|
||||||
if (ops->unmap_pages(ops, iova_start + size, size, 1, NULL) != size)
|
|
||||||
return __FAIL(ops);
|
|
||||||
|
|
||||||
/* Remap of partial unmap */
|
|
||||||
if (ops->map_pages(ops, iova_start + size, size, size, 1,
|
|
||||||
IOMMU_READ, GFP_KERNEL, &mapped))
|
|
||||||
return __FAIL(ops);
|
|
||||||
|
|
||||||
if (ops->iova_to_phys(ops, iova_start + size + 42)
|
|
||||||
!= (size + 42))
|
|
||||||
return __FAIL(ops);
|
|
||||||
i++;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Full unmap */
|
/* Full unmap */
|
||||||
|
|
|
||||||
|
|
@ -199,6 +199,18 @@ static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
|
||||||
return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
|
return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Convert an index returned by ARM_LPAE_PGD_IDX(), which can point into
|
||||||
|
* a concatenated PGD, into the maximum number of entries that can be
|
||||||
|
* mapped in the same table page.
|
||||||
|
*/
|
||||||
|
static inline int arm_lpae_max_entries(int i, struct arm_lpae_io_pgtable *data)
|
||||||
|
{
|
||||||
|
int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
|
||||||
|
|
||||||
|
return ptes_per_table - (i & (ptes_per_table - 1));
|
||||||
|
}
|
||||||
|
|
||||||
static bool selftest_running = false;
|
static bool selftest_running = false;
|
||||||
|
|
||||||
static dma_addr_t __arm_lpae_dma_addr(void *pages)
|
static dma_addr_t __arm_lpae_dma_addr(void *pages)
|
||||||
|
|
@ -390,7 +402,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
|
||||||
|
|
||||||
/* If we can install a leaf entry at this level, then do so */
|
/* If we can install a leaf entry at this level, then do so */
|
||||||
if (size == block_size) {
|
if (size == block_size) {
|
||||||
max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start;
|
max_entries = arm_lpae_max_entries(map_idx_start, data);
|
||||||
num_entries = min_t(int, pgcount, max_entries);
|
num_entries = min_t(int, pgcount, max_entries);
|
||||||
ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
|
ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
|
||||||
if (!ret)
|
if (!ret)
|
||||||
|
|
@ -569,66 +581,6 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop)
|
||||||
kfree(data);
|
kfree(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
|
|
||||||
struct iommu_iotlb_gather *gather,
|
|
||||||
unsigned long iova, size_t size,
|
|
||||||
arm_lpae_iopte blk_pte, int lvl,
|
|
||||||
arm_lpae_iopte *ptep, size_t pgcount)
|
|
||||||
{
|
|
||||||
struct io_pgtable_cfg *cfg = &data->iop.cfg;
|
|
||||||
arm_lpae_iopte pte, *tablep;
|
|
||||||
phys_addr_t blk_paddr;
|
|
||||||
size_t tablesz = ARM_LPAE_GRANULE(data);
|
|
||||||
size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
|
|
||||||
int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
|
|
||||||
int i, unmap_idx_start = -1, num_entries = 0, max_entries;
|
|
||||||
|
|
||||||
if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg, data->iop.cookie);
|
|
||||||
if (!tablep)
|
|
||||||
return 0; /* Bytes unmapped */
|
|
||||||
|
|
||||||
if (size == split_sz) {
|
|
||||||
unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
|
|
||||||
max_entries = ptes_per_table - unmap_idx_start;
|
|
||||||
num_entries = min_t(int, pgcount, max_entries);
|
|
||||||
}
|
|
||||||
|
|
||||||
blk_paddr = iopte_to_paddr(blk_pte, data);
|
|
||||||
pte = iopte_prot(blk_pte);
|
|
||||||
|
|
||||||
for (i = 0; i < ptes_per_table; i++, blk_paddr += split_sz) {
|
|
||||||
/* Unmap! */
|
|
||||||
if (i >= unmap_idx_start && i < (unmap_idx_start + num_entries))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
__arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);
|
|
||||||
if (pte != blk_pte) {
|
|
||||||
__arm_lpae_free_pages(tablep, tablesz, cfg, data->iop.cookie);
|
|
||||||
/*
|
|
||||||
* We may race against someone unmapping another part of this
|
|
||||||
* block, but anything else is invalid. We can't misinterpret
|
|
||||||
* a page entry here since we're never at the last level.
|
|
||||||
*/
|
|
||||||
if (iopte_type(pte) != ARM_LPAE_PTE_TYPE_TABLE)
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
tablep = iopte_deref(pte, data);
|
|
||||||
} else if (unmap_idx_start >= 0) {
|
|
||||||
for (i = 0; i < num_entries; i++)
|
|
||||||
io_pgtable_tlb_add_page(&data->iop, gather, iova + i * size, size);
|
|
||||||
|
|
||||||
return num_entries * size;
|
|
||||||
}
|
|
||||||
|
|
||||||
return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep);
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
|
static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
|
||||||
struct iommu_iotlb_gather *gather,
|
struct iommu_iotlb_gather *gather,
|
||||||
unsigned long iova, size_t size, size_t pgcount,
|
unsigned long iova, size_t size, size_t pgcount,
|
||||||
|
|
@ -650,7 +602,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
|
||||||
|
|
||||||
/* If the size matches this level, we're in the right place */
|
/* If the size matches this level, we're in the right place */
|
||||||
if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
|
if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
|
||||||
max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start;
|
max_entries = arm_lpae_max_entries(unmap_idx_start, data);
|
||||||
num_entries = min_t(int, pgcount, max_entries);
|
num_entries = min_t(int, pgcount, max_entries);
|
||||||
|
|
||||||
/* Find and handle non-leaf entries */
|
/* Find and handle non-leaf entries */
|
||||||
|
|
@ -678,12 +630,8 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
|
||||||
|
|
||||||
return i * size;
|
return i * size;
|
||||||
} else if (iopte_leaf(pte, lvl, iop->fmt)) {
|
} else if (iopte_leaf(pte, lvl, iop->fmt)) {
|
||||||
/*
|
WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
|
||||||
* Insert a table at the next level to map the old region,
|
return 0;
|
||||||
* minus the part we want to unmap
|
|
||||||
*/
|
|
||||||
return arm_lpae_split_blk_unmap(data, gather, iova, size, pte,
|
|
||||||
lvl + 1, ptep, pgcount);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Keep on walkin' */
|
/* Keep on walkin' */
|
||||||
|
|
@ -1347,19 +1295,6 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
|
||||||
iova += SZ_1G;
|
iova += SZ_1G;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Partial unmap */
|
|
||||||
size = 1UL << __ffs(cfg->pgsize_bitmap);
|
|
||||||
if (ops->unmap_pages(ops, SZ_1G + size, size, 1, NULL) != size)
|
|
||||||
return __FAIL(ops, i);
|
|
||||||
|
|
||||||
/* Remap of partial unmap */
|
|
||||||
if (ops->map_pages(ops, SZ_1G + size, size, size, 1,
|
|
||||||
IOMMU_READ, GFP_KERNEL, &mapped))
|
|
||||||
return __FAIL(ops, i);
|
|
||||||
|
|
||||||
if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
|
|
||||||
return __FAIL(ops, i);
|
|
||||||
|
|
||||||
/* Full unmap */
|
/* Full unmap */
|
||||||
iova = 0;
|
iova = 0;
|
||||||
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
|
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
|
||||||
|
|
@ -1382,6 +1317,23 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
|
||||||
iova += SZ_1G;
|
iova += SZ_1G;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Map/unmap the last largest supported page of the IAS, this can
|
||||||
|
* trigger corner cases in the concatednated page tables.
|
||||||
|
*/
|
||||||
|
mapped = 0;
|
||||||
|
size = 1UL << __fls(cfg->pgsize_bitmap);
|
||||||
|
iova = (1UL << cfg->ias) - size;
|
||||||
|
if (ops->map_pages(ops, iova, iova, size, 1,
|
||||||
|
IOMMU_READ | IOMMU_WRITE |
|
||||||
|
IOMMU_NOEXEC | IOMMU_CACHE,
|
||||||
|
GFP_KERNEL, &mapped))
|
||||||
|
return __FAIL(ops, i);
|
||||||
|
if (mapped != size)
|
||||||
|
return __FAIL(ops, i);
|
||||||
|
if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
|
||||||
|
return __FAIL(ops, i);
|
||||||
|
|
||||||
free_io_pgtable_ops(ops);
|
free_io_pgtable_ops(ops);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ static void release_device(struct device *dev)
|
||||||
kfree(dev);
|
kfree(dev);
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct class iommu_class = {
|
static const struct class iommu_class = {
|
||||||
.name = "iommu",
|
.name = "iommu",
|
||||||
.dev_release = release_device,
|
.dev_release = release_device,
|
||||||
.dev_groups = dev_groups,
|
.dev_groups = dev_groups,
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@
|
||||||
#include <trace/events/iommu.h>
|
#include <trace/events/iommu.h>
|
||||||
#include <linux/sched/mm.h>
|
#include <linux/sched/mm.h>
|
||||||
#include <linux/msi.h>
|
#include <linux/msi.h>
|
||||||
|
#include <uapi/linux/iommufd.h>
|
||||||
|
|
||||||
#include "dma-iommu.h"
|
#include "dma-iommu.h"
|
||||||
#include "iommu-priv.h"
|
#include "iommu-priv.h"
|
||||||
|
|
@ -90,15 +91,17 @@ static const char * const iommu_group_resv_type_string[] = {
|
||||||
#define IOMMU_CMD_LINE_DMA_API BIT(0)
|
#define IOMMU_CMD_LINE_DMA_API BIT(0)
|
||||||
#define IOMMU_CMD_LINE_STRICT BIT(1)
|
#define IOMMU_CMD_LINE_STRICT BIT(1)
|
||||||
|
|
||||||
|
static int bus_iommu_probe(const struct bus_type *bus);
|
||||||
static int iommu_bus_notifier(struct notifier_block *nb,
|
static int iommu_bus_notifier(struct notifier_block *nb,
|
||||||
unsigned long action, void *data);
|
unsigned long action, void *data);
|
||||||
static void iommu_release_device(struct device *dev);
|
static void iommu_release_device(struct device *dev);
|
||||||
static struct iommu_domain *
|
|
||||||
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type);
|
|
||||||
static int __iommu_attach_device(struct iommu_domain *domain,
|
static int __iommu_attach_device(struct iommu_domain *domain,
|
||||||
struct device *dev);
|
struct device *dev);
|
||||||
static int __iommu_attach_group(struct iommu_domain *domain,
|
static int __iommu_attach_group(struct iommu_domain *domain,
|
||||||
struct iommu_group *group);
|
struct iommu_group *group);
|
||||||
|
static struct iommu_domain *__iommu_paging_domain_alloc_flags(struct device *dev,
|
||||||
|
unsigned int type,
|
||||||
|
unsigned int flags);
|
||||||
|
|
||||||
enum {
|
enum {
|
||||||
IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
|
IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
|
||||||
|
|
@ -133,6 +136,8 @@ static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
|
||||||
struct device *dev);
|
struct device *dev);
|
||||||
static void __iommu_group_free_device(struct iommu_group *group,
|
static void __iommu_group_free_device(struct iommu_group *group,
|
||||||
struct group_device *grp_dev);
|
struct group_device *grp_dev);
|
||||||
|
static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
|
||||||
|
const struct iommu_ops *ops);
|
||||||
|
|
||||||
#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
|
#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
|
||||||
struct iommu_group_attribute iommu_group_attr_##_name = \
|
struct iommu_group_attribute iommu_group_attr_##_name = \
|
||||||
|
|
@ -1141,10 +1146,6 @@ map_end:
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!list_empty(&mappings) && iommu_is_dma_domain(domain))
|
|
||||||
iommu_flush_iotlb_all(domain);
|
|
||||||
|
|
||||||
out:
|
out:
|
||||||
iommu_put_resv_regions(dev, &mappings);
|
iommu_put_resv_regions(dev, &mappings);
|
||||||
|
|
||||||
|
|
@ -1586,12 +1587,59 @@ struct iommu_group *fsl_mc_device_group(struct device *dev)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(fsl_mc_device_group);
|
EXPORT_SYMBOL_GPL(fsl_mc_device_group);
|
||||||
|
|
||||||
|
static struct iommu_domain *__iommu_alloc_identity_domain(struct device *dev)
|
||||||
|
{
|
||||||
|
const struct iommu_ops *ops = dev_iommu_ops(dev);
|
||||||
|
struct iommu_domain *domain;
|
||||||
|
|
||||||
|
if (ops->identity_domain)
|
||||||
|
return ops->identity_domain;
|
||||||
|
|
||||||
|
/* Older drivers create the identity domain via ops->domain_alloc() */
|
||||||
|
if (!ops->domain_alloc)
|
||||||
|
return ERR_PTR(-EOPNOTSUPP);
|
||||||
|
|
||||||
|
domain = ops->domain_alloc(IOMMU_DOMAIN_IDENTITY);
|
||||||
|
if (IS_ERR(domain))
|
||||||
|
return domain;
|
||||||
|
if (!domain)
|
||||||
|
return ERR_PTR(-ENOMEM);
|
||||||
|
|
||||||
|
iommu_domain_init(domain, IOMMU_DOMAIN_IDENTITY, ops);
|
||||||
|
return domain;
|
||||||
|
}
|
||||||
|
|
||||||
static struct iommu_domain *
|
static struct iommu_domain *
|
||||||
__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
|
__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
|
||||||
{
|
{
|
||||||
|
struct device *dev = iommu_group_first_dev(group);
|
||||||
|
struct iommu_domain *dom;
|
||||||
|
|
||||||
if (group->default_domain && group->default_domain->type == req_type)
|
if (group->default_domain && group->default_domain->type == req_type)
|
||||||
return group->default_domain;
|
return group->default_domain;
|
||||||
return __iommu_group_domain_alloc(group, req_type);
|
|
||||||
|
/*
|
||||||
|
* When allocating the DMA API domain assume that the driver is going to
|
||||||
|
* use PASID and make sure the RID's domain is PASID compatible.
|
||||||
|
*/
|
||||||
|
if (req_type & __IOMMU_DOMAIN_PAGING) {
|
||||||
|
dom = __iommu_paging_domain_alloc_flags(dev, req_type,
|
||||||
|
dev->iommu->max_pasids ? IOMMU_HWPT_ALLOC_PASID : 0);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If driver does not support PASID feature then
|
||||||
|
* try to allocate non-PASID domain
|
||||||
|
*/
|
||||||
|
if (PTR_ERR(dom) == -EOPNOTSUPP)
|
||||||
|
dom = __iommu_paging_domain_alloc_flags(dev, req_type, 0);
|
||||||
|
|
||||||
|
return dom;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req_type == IOMMU_DOMAIN_IDENTITY)
|
||||||
|
return __iommu_alloc_identity_domain(dev);
|
||||||
|
|
||||||
|
return ERR_PTR(-EINVAL);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
@ -1795,7 +1843,7 @@ static void iommu_group_do_probe_finalize(struct device *dev)
|
||||||
ops->probe_finalize(dev);
|
ops->probe_finalize(dev);
|
||||||
}
|
}
|
||||||
|
|
||||||
int bus_iommu_probe(const struct bus_type *bus)
|
static int bus_iommu_probe(const struct bus_type *bus)
|
||||||
{
|
{
|
||||||
struct iommu_group *group, *next;
|
struct iommu_group *group, *next;
|
||||||
LIST_HEAD(group_list);
|
LIST_HEAD(group_list);
|
||||||
|
|
@ -1840,31 +1888,6 @@ int bus_iommu_probe(const struct bus_type *bus)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* iommu_present() - make platform-specific assumptions about an IOMMU
|
|
||||||
* @bus: bus to check
|
|
||||||
*
|
|
||||||
* Do not use this function. You want device_iommu_mapped() instead.
|
|
||||||
*
|
|
||||||
* Return: true if some IOMMU is present and aware of devices on the given bus;
|
|
||||||
* in general it may not be the only IOMMU, and it may not have anything to do
|
|
||||||
* with whatever device you are ultimately interested in.
|
|
||||||
*/
|
|
||||||
bool iommu_present(const struct bus_type *bus)
|
|
||||||
{
|
|
||||||
bool ret = false;
|
|
||||||
|
|
||||||
for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
|
|
||||||
if (iommu_buses[i] == bus) {
|
|
||||||
spin_lock(&iommu_device_lock);
|
|
||||||
ret = !list_empty(&iommu_device_list);
|
|
||||||
spin_unlock(&iommu_device_lock);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(iommu_present);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* device_iommu_capable() - check for a general IOMMU capability
|
* device_iommu_capable() - check for a general IOMMU capability
|
||||||
* @dev: device to which the capability would be relevant, if available
|
* @dev: device to which the capability would be relevant, if available
|
||||||
|
|
@ -1934,117 +1957,67 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
|
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
|
||||||
|
|
||||||
static struct iommu_domain *__iommu_domain_alloc(const struct iommu_ops *ops,
|
static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
|
||||||
struct device *dev,
|
const struct iommu_ops *ops)
|
||||||
unsigned int type)
|
|
||||||
{
|
{
|
||||||
struct iommu_domain *domain;
|
|
||||||
unsigned int alloc_type = type & IOMMU_DOMAIN_ALLOC_FLAGS;
|
|
||||||
|
|
||||||
if (alloc_type == IOMMU_DOMAIN_IDENTITY && ops->identity_domain)
|
|
||||||
return ops->identity_domain;
|
|
||||||
else if (alloc_type == IOMMU_DOMAIN_BLOCKED && ops->blocked_domain)
|
|
||||||
return ops->blocked_domain;
|
|
||||||
else if (type & __IOMMU_DOMAIN_PAGING && ops->domain_alloc_paging)
|
|
||||||
domain = ops->domain_alloc_paging(dev);
|
|
||||||
else if (ops->domain_alloc)
|
|
||||||
domain = ops->domain_alloc(alloc_type);
|
|
||||||
else
|
|
||||||
return ERR_PTR(-EOPNOTSUPP);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Many domain_alloc ops now return ERR_PTR, make things easier for the
|
|
||||||
* driver by accepting ERR_PTR from all domain_alloc ops instead of
|
|
||||||
* having two rules.
|
|
||||||
*/
|
|
||||||
if (IS_ERR(domain))
|
|
||||||
return domain;
|
|
||||||
if (!domain)
|
|
||||||
return ERR_PTR(-ENOMEM);
|
|
||||||
|
|
||||||
domain->type = type;
|
domain->type = type;
|
||||||
domain->owner = ops;
|
domain->owner = ops;
|
||||||
|
if (!domain->ops)
|
||||||
|
domain->ops = ops->default_domain_ops;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If not already set, assume all sizes by default; the driver
|
* If not already set, assume all sizes by default; the driver
|
||||||
* may override this later
|
* may override this later
|
||||||
*/
|
*/
|
||||||
if (!domain->pgsize_bitmap)
|
if (!domain->pgsize_bitmap)
|
||||||
domain->pgsize_bitmap = ops->pgsize_bitmap;
|
domain->pgsize_bitmap = ops->pgsize_bitmap;
|
||||||
|
|
||||||
if (!domain->ops)
|
|
||||||
domain->ops = ops->default_domain_ops;
|
|
||||||
|
|
||||||
if (iommu_is_dma_domain(domain)) {
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
rc = iommu_get_dma_cookie(domain);
|
|
||||||
if (rc) {
|
|
||||||
iommu_domain_free(domain);
|
|
||||||
return ERR_PTR(rc);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return domain;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct iommu_domain *
|
static struct iommu_domain *
|
||||||
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type)
|
__iommu_paging_domain_alloc_flags(struct device *dev, unsigned int type,
|
||||||
|
unsigned int flags)
|
||||||
{
|
{
|
||||||
struct device *dev = iommu_group_first_dev(group);
|
const struct iommu_ops *ops;
|
||||||
|
|
||||||
return __iommu_domain_alloc(dev_iommu_ops(dev), dev, type);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int __iommu_domain_alloc_dev(struct device *dev, void *data)
|
|
||||||
{
|
|
||||||
const struct iommu_ops **ops = data;
|
|
||||||
|
|
||||||
if (!dev_has_iommu(dev))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
if (WARN_ONCE(*ops && *ops != dev_iommu_ops(dev),
|
|
||||||
"Multiple IOMMU drivers present for bus %s, which the public IOMMU API can't fully support yet. You will still need to disable one or more for this to work, sorry!\n",
|
|
||||||
dev_bus_name(dev)))
|
|
||||||
return -EBUSY;
|
|
||||||
|
|
||||||
*ops = dev_iommu_ops(dev);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* The iommu ops in bus has been retired. Do not use this interface in
|
|
||||||
* new drivers.
|
|
||||||
*/
|
|
||||||
struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
|
|
||||||
{
|
|
||||||
const struct iommu_ops *ops = NULL;
|
|
||||||
int err = bus_for_each_dev(bus, NULL, &ops, __iommu_domain_alloc_dev);
|
|
||||||
struct iommu_domain *domain;
|
struct iommu_domain *domain;
|
||||||
|
|
||||||
if (err || !ops)
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
domain = __iommu_domain_alloc(ops, NULL, IOMMU_DOMAIN_UNMANAGED);
|
|
||||||
if (IS_ERR(domain))
|
|
||||||
return NULL;
|
|
||||||
return domain;
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(iommu_domain_alloc);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* iommu_paging_domain_alloc() - Allocate a paging domain
|
|
||||||
* @dev: device for which the domain is allocated
|
|
||||||
*
|
|
||||||
* Allocate a paging domain which will be managed by a kernel driver. Return
|
|
||||||
* allocated domain if successful, or a ERR pointer for failure.
|
|
||||||
*/
|
|
||||||
struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
|
|
||||||
{
|
|
||||||
if (!dev_has_iommu(dev))
|
if (!dev_has_iommu(dev))
|
||||||
return ERR_PTR(-ENODEV);
|
return ERR_PTR(-ENODEV);
|
||||||
|
|
||||||
return __iommu_domain_alloc(dev_iommu_ops(dev), dev, IOMMU_DOMAIN_UNMANAGED);
|
ops = dev_iommu_ops(dev);
|
||||||
|
|
||||||
|
if (ops->domain_alloc_paging && !flags)
|
||||||
|
domain = ops->domain_alloc_paging(dev);
|
||||||
|
else if (ops->domain_alloc_user)
|
||||||
|
domain = ops->domain_alloc_user(dev, flags, NULL, NULL);
|
||||||
|
else if (ops->domain_alloc && !flags)
|
||||||
|
domain = ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED);
|
||||||
|
else
|
||||||
|
return ERR_PTR(-EOPNOTSUPP);
|
||||||
|
|
||||||
|
if (IS_ERR(domain))
|
||||||
|
return domain;
|
||||||
|
if (!domain)
|
||||||
|
return ERR_PTR(-ENOMEM);
|
||||||
|
|
||||||
|
iommu_domain_init(domain, type, ops);
|
||||||
|
return domain;
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc);
|
|
||||||
|
/**
|
||||||
|
* iommu_paging_domain_alloc_flags() - Allocate a paging domain
|
||||||
|
* @dev: device for which the domain is allocated
|
||||||
|
* @flags: Bitmap of iommufd_hwpt_alloc_flags
|
||||||
|
*
|
||||||
|
* Allocate a paging domain which will be managed by a kernel driver. Return
|
||||||
|
* allocated domain if successful, or an ERR pointer for failure.
|
||||||
|
*/
|
||||||
|
struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
|
||||||
|
unsigned int flags)
|
||||||
|
{
|
||||||
|
return __iommu_paging_domain_alloc_flags(dev,
|
||||||
|
IOMMU_DOMAIN_UNMANAGED, flags);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc_flags);
|
||||||
|
|
||||||
void iommu_domain_free(struct iommu_domain *domain)
|
void iommu_domain_free(struct iommu_domain *domain)
|
||||||
{
|
{
|
||||||
|
|
@ -2216,8 +2189,8 @@ EXPORT_SYMBOL_GPL(iommu_attach_group);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* iommu_group_replace_domain - replace the domain that a group is attached to
|
* iommu_group_replace_domain - replace the domain that a group is attached to
|
||||||
* @new_domain: new IOMMU domain to replace with
|
|
||||||
* @group: IOMMU group that will be attached to the new domain
|
* @group: IOMMU group that will be attached to the new domain
|
||||||
|
* @new_domain: new IOMMU domain to replace with
|
||||||
*
|
*
|
||||||
* This API allows the group to switch domains without being forced to go to
|
* This API allows the group to switch domains without being forced to go to
|
||||||
* the blocking domain in-between.
|
* the blocking domain in-between.
|
||||||
|
|
@ -2586,6 +2559,20 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
|
||||||
return unmapped;
|
return unmapped;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* iommu_unmap() - Remove mappings from a range of IOVA
|
||||||
|
* @domain: Domain to manipulate
|
||||||
|
* @iova: IO virtual address to start
|
||||||
|
* @size: Length of the range starting from @iova
|
||||||
|
*
|
||||||
|
* iommu_unmap() will remove a translation created by iommu_map(). It cannot
|
||||||
|
* subdivide a mapping created by iommu_map(), so it should be called with IOVA
|
||||||
|
* ranges that match what was passed to iommu_map(). The range can aggregate
|
||||||
|
* contiguous iommu_map() calls so long as no individual range is split.
|
||||||
|
*
|
||||||
|
* Returns: Number of bytes of IOVA unmapped. iova + res will be the point
|
||||||
|
* unmapping stopped.
|
||||||
|
*/
|
||||||
size_t iommu_unmap(struct iommu_domain *domain,
|
size_t iommu_unmap(struct iommu_domain *domain,
|
||||||
unsigned long iova, size_t size)
|
unsigned long iova, size_t size)
|
||||||
{
|
{
|
||||||
|
|
@ -2965,6 +2952,14 @@ static int iommu_setup_default_domain(struct iommu_group *group,
|
||||||
if (group->default_domain == dom)
|
if (group->default_domain == dom)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
if (iommu_is_dma_domain(dom)) {
|
||||||
|
ret = iommu_get_dma_cookie(dom);
|
||||||
|
if (ret) {
|
||||||
|
iommu_domain_free(dom);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
|
* IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
|
||||||
* mapped before their device is attached, in order to guarantee
|
* mapped before their device is attached, in order to guarantee
|
||||||
|
|
@ -3152,22 +3147,25 @@ void iommu_device_unuse_default_domain(struct device *dev)
|
||||||
|
|
||||||
static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
|
static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
|
||||||
{
|
{
|
||||||
|
struct device *dev = iommu_group_first_dev(group);
|
||||||
|
const struct iommu_ops *ops = dev_iommu_ops(dev);
|
||||||
struct iommu_domain *domain;
|
struct iommu_domain *domain;
|
||||||
|
|
||||||
if (group->blocking_domain)
|
if (group->blocking_domain)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
domain = __iommu_group_domain_alloc(group, IOMMU_DOMAIN_BLOCKED);
|
if (ops->blocked_domain) {
|
||||||
if (IS_ERR(domain)) {
|
group->blocking_domain = ops->blocked_domain;
|
||||||
/*
|
return 0;
|
||||||
* For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
|
|
||||||
* create an empty domain instead.
|
|
||||||
*/
|
|
||||||
domain = __iommu_group_domain_alloc(group,
|
|
||||||
IOMMU_DOMAIN_UNMANAGED);
|
|
||||||
if (IS_ERR(domain))
|
|
||||||
return PTR_ERR(domain);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED create an
|
||||||
|
* empty PAGING domain instead.
|
||||||
|
*/
|
||||||
|
domain = iommu_paging_domain_alloc(dev);
|
||||||
|
if (IS_ERR(domain))
|
||||||
|
return PTR_ERR(domain);
|
||||||
group->blocking_domain = domain;
|
group->blocking_domain = domain;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -506,7 +506,7 @@ __adjust_overlap_range(struct iova *iova,
|
||||||
* reserve_iova - reserves an iova in the given range
|
* reserve_iova - reserves an iova in the given range
|
||||||
* @iovad: - iova domain pointer
|
* @iovad: - iova domain pointer
|
||||||
* @pfn_lo: - lower page frame address
|
* @pfn_lo: - lower page frame address
|
||||||
* @pfn_hi:- higher pfn adderss
|
* @pfn_hi:- higher pfn address
|
||||||
* This function allocates reserves the address range from pfn_lo to pfn_hi so
|
* This function allocates reserves the address range from pfn_lo to pfn_hi so
|
||||||
* that this address is not dished out as part of alloc_iova.
|
* that this address is not dished out as part of alloc_iova.
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -1599,7 +1599,7 @@ static const unsigned int mt8186_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK
|
||||||
static const struct mtk_iommu_plat_data mt8186_data_mm = {
|
static const struct mtk_iommu_plat_data mt8186_data_mm = {
|
||||||
.m4u_plat = M4U_MT8186,
|
.m4u_plat = M4U_MT8186,
|
||||||
.flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN |
|
.flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN |
|
||||||
WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM,
|
WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM | PGTABLE_PA_35_EN,
|
||||||
.larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20},
|
.larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20},
|
||||||
{MTK_INVALID_LARBID, 14, 16},
|
{MTK_INVALID_LARBID, 14, 16},
|
||||||
{MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}},
|
{MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}},
|
||||||
|
|
|
||||||
|
|
@ -1230,25 +1230,24 @@ static int omap_iommu_probe(struct platform_device *pdev)
|
||||||
if (err)
|
if (err)
|
||||||
return err;
|
return err;
|
||||||
|
|
||||||
err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);
|
|
||||||
if (err)
|
|
||||||
goto out_sysfs;
|
|
||||||
obj->has_iommu_driver = true;
|
obj->has_iommu_driver = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);
|
||||||
|
if (err)
|
||||||
|
goto out_sysfs;
|
||||||
|
|
||||||
pm_runtime_enable(obj->dev);
|
pm_runtime_enable(obj->dev);
|
||||||
|
|
||||||
omap_iommu_debugfs_add(obj);
|
omap_iommu_debugfs_add(obj);
|
||||||
|
|
||||||
dev_info(&pdev->dev, "%s registered\n", obj->name);
|
dev_info(&pdev->dev, "%s registered\n", obj->name);
|
||||||
|
|
||||||
/* Re-probe bus to probe device attached to this IOMMU */
|
|
||||||
bus_iommu_probe(&platform_bus_type);
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
out_sysfs:
|
out_sysfs:
|
||||||
iommu_device_sysfs_remove(&obj->iommu);
|
if (obj->has_iommu_driver)
|
||||||
|
iommu_device_sysfs_remove(&obj->iommu);
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1256,10 +1255,10 @@ static void omap_iommu_remove(struct platform_device *pdev)
|
||||||
{
|
{
|
||||||
struct omap_iommu *obj = platform_get_drvdata(pdev);
|
struct omap_iommu *obj = platform_get_drvdata(pdev);
|
||||||
|
|
||||||
if (obj->has_iommu_driver) {
|
if (obj->has_iommu_driver)
|
||||||
iommu_device_sysfs_remove(&obj->iommu);
|
iommu_device_sysfs_remove(&obj->iommu);
|
||||||
iommu_device_unregister(&obj->iommu);
|
|
||||||
}
|
iommu_device_unregister(&obj->iommu);
|
||||||
|
|
||||||
omap_iommu_debugfs_remove(obj);
|
omap_iommu_debugfs_remove(obj);
|
||||||
|
|
||||||
|
|
@ -1723,12 +1722,19 @@ static void omap_iommu_release_device(struct device *dev)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int omap_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
|
||||||
|
{
|
||||||
|
/* TODO: collect args->np to save re-parsing in probe above */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static const struct iommu_ops omap_iommu_ops = {
|
static const struct iommu_ops omap_iommu_ops = {
|
||||||
.identity_domain = &omap_iommu_identity_domain,
|
.identity_domain = &omap_iommu_identity_domain,
|
||||||
.domain_alloc_paging = omap_iommu_domain_alloc_paging,
|
.domain_alloc_paging = omap_iommu_domain_alloc_paging,
|
||||||
.probe_device = omap_iommu_probe_device,
|
.probe_device = omap_iommu_probe_device,
|
||||||
.release_device = omap_iommu_release_device,
|
.release_device = omap_iommu_release_device,
|
||||||
.device_group = generic_single_device_group,
|
.device_group = generic_single_device_group,
|
||||||
|
.of_xlate = omap_iommu_of_xlate,
|
||||||
.pgsize_bitmap = OMAP_IOMMU_PGSIZES,
|
.pgsize_bitmap = OMAP_IOMMU_PGSIZES,
|
||||||
.default_domain_ops = &(const struct iommu_domain_ops) {
|
.default_domain_ops = &(const struct iommu_domain_ops) {
|
||||||
.attach_dev = omap_iommu_attach_dev,
|
.attach_dev = omap_iommu_attach_dev,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
# SPDX-License-Identifier: GPL-2.0-only
|
||||||
|
# RISC-V IOMMU support
|
||||||
|
|
||||||
|
config RISCV_IOMMU
|
||||||
|
bool "RISC-V IOMMU Support"
|
||||||
|
depends on RISCV && 64BIT
|
||||||
|
default y
|
||||||
|
select IOMMU_API
|
||||||
|
help
|
||||||
|
Support for implementations of the RISC-V IOMMU architecture that
|
||||||
|
complements the RISC-V MMU capabilities, providing similar address
|
||||||
|
translation and protection functions for accesses from I/O devices.
|
||||||
|
|
||||||
|
Say Y here if your SoC includes an IOMMU device implementing
|
||||||
|
the RISC-V IOMMU architecture.
|
||||||
|
|
||||||
|
config RISCV_IOMMU_PCI
|
||||||
|
def_bool y if RISCV_IOMMU && PCI_MSI
|
||||||
|
help
|
||||||
|
Support for the PCIe implementation of RISC-V IOMMU architecture.
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
# SPDX-License-Identifier: GPL-2.0-only
|
||||||
|
obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
|
||||||
|
obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
|
||||||
|
|
@ -0,0 +1,784 @@
|
||||||
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||||
|
/*
|
||||||
|
* Copyright © 2022-2024 Rivos Inc.
|
||||||
|
* Copyright © 2023 FORTH-ICS/CARV
|
||||||
|
* Copyright © 2023 RISC-V IOMMU Task Group
|
||||||
|
*
|
||||||
|
* RISC-V IOMMU - Register Layout and Data Structures.
|
||||||
|
*
|
||||||
|
* Based on the 'RISC-V IOMMU Architecture Specification', Version 1.0
|
||||||
|
* Published at https://github.com/riscv-non-isa/riscv-iommu
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef _RISCV_IOMMU_BITS_H_
|
||||||
|
#define _RISCV_IOMMU_BITS_H_
|
||||||
|
|
||||||
|
#include <linux/types.h>
|
||||||
|
#include <linux/bitfield.h>
|
||||||
|
#include <linux/bits.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Chapter 5: Memory Mapped register interface
|
||||||
|
*/
|
||||||
|
|
||||||
|
/* Common field positions */
|
||||||
|
#define RISCV_IOMMU_PPN_FIELD GENMASK_ULL(53, 10)
|
||||||
|
#define RISCV_IOMMU_QUEUE_LOG2SZ_FIELD GENMASK_ULL(4, 0)
|
||||||
|
#define RISCV_IOMMU_QUEUE_INDEX_FIELD GENMASK_ULL(31, 0)
|
||||||
|
#define RISCV_IOMMU_QUEUE_ENABLE BIT(0)
|
||||||
|
#define RISCV_IOMMU_QUEUE_INTR_ENABLE BIT(1)
|
||||||
|
#define RISCV_IOMMU_QUEUE_MEM_FAULT BIT(8)
|
||||||
|
#define RISCV_IOMMU_QUEUE_OVERFLOW BIT(9)
|
||||||
|
#define RISCV_IOMMU_QUEUE_ACTIVE BIT(16)
|
||||||
|
#define RISCV_IOMMU_QUEUE_BUSY BIT(17)
|
||||||
|
|
||||||
|
#define RISCV_IOMMU_ATP_PPN_FIELD GENMASK_ULL(43, 0)
|
||||||
|
#define RISCV_IOMMU_ATP_MODE_FIELD GENMASK_ULL(63, 60)
|
||||||
|
|
||||||
|
/* 5.3 IOMMU Capabilities (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_CAPABILITIES 0x0000
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_VERSION GENMASK_ULL(7, 0)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV32 BIT_ULL(8)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV39 BIT_ULL(9)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV48 BIT_ULL(10)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV57 BIT_ULL(11)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SVPBMT BIT_ULL(15)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV32X4 BIT_ULL(16)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV39X4 BIT_ULL(17)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV48X4 BIT_ULL(18)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_SV57X4 BIT_ULL(19)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_AMO_MRIF BIT_ULL(21)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_MSI_FLAT BIT_ULL(22)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_MSI_MRIF BIT_ULL(23)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_AMO_HWAD BIT_ULL(24)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_ATS BIT_ULL(25)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_T2GPA BIT_ULL(26)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_END BIT_ULL(27)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_IGS GENMASK_ULL(29, 28)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_HPM BIT_ULL(30)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_DBG BIT_ULL(31)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_PAS GENMASK_ULL(37, 32)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_PD8 BIT_ULL(38)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_PD17 BIT_ULL(39)
|
||||||
|
#define RISCV_IOMMU_CAPABILITIES_PD20 BIT_ULL(40)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum riscv_iommu_igs_settings - Interrupt Generation Support Settings
|
||||||
|
* @RISCV_IOMMU_CAPABILITIES_IGS_MSI: IOMMU supports only MSI generation
|
||||||
|
* @RISCV_IOMMU_CAPABILITIES_IGS_WSI: IOMMU supports only Wired-Signaled interrupt
|
||||||
|
* @RISCV_IOMMU_CAPABILITIES_IGS_BOTH: IOMMU supports both MSI and WSI generation
|
||||||
|
* @RISCV_IOMMU_CAPABILITIES_IGS_RSRV: Reserved for standard use
|
||||||
|
*/
|
||||||
|
enum riscv_iommu_igs_settings {
|
||||||
|
RISCV_IOMMU_CAPABILITIES_IGS_MSI = 0,
|
||||||
|
RISCV_IOMMU_CAPABILITIES_IGS_WSI = 1,
|
||||||
|
RISCV_IOMMU_CAPABILITIES_IGS_BOTH = 2,
|
||||||
|
RISCV_IOMMU_CAPABILITIES_IGS_RSRV = 3
|
||||||
|
};
|
||||||
|
|
||||||
|
/* 5.4 Features control register (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_FCTL 0x0008
|
||||||
|
#define RISCV_IOMMU_FCTL_BE BIT(0)
|
||||||
|
#define RISCV_IOMMU_FCTL_WSI BIT(1)
|
||||||
|
#define RISCV_IOMMU_FCTL_GXL BIT(2)
|
||||||
|
|
||||||
|
/* 5.5 Device-directory-table pointer (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_DDTP 0x0010
|
||||||
|
#define RISCV_IOMMU_DDTP_IOMMU_MODE GENMASK_ULL(3, 0)
|
||||||
|
#define RISCV_IOMMU_DDTP_BUSY BIT_ULL(4)
|
||||||
|
#define RISCV_IOMMU_DDTP_PPN RISCV_IOMMU_PPN_FIELD
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum riscv_iommu_ddtp_modes - IOMMU translation modes
|
||||||
|
* @RISCV_IOMMU_DDTP_IOMMU_MODE_OFF: No inbound transactions allowed
|
||||||
|
* @RISCV_IOMMU_DDTP_IOMMU_MODE_BARE: Pass-through mode
|
||||||
|
* @RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL: One-level DDT
|
||||||
|
* @RISCV_IOMMU_DDTP_IOMMU_MODE_2LVL: Two-level DDT
|
||||||
|
* @RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL: Three-level DDT
|
||||||
|
* @RISCV_IOMMU_DDTP_IOMMU_MODE_MAX: Max value allowed by specification
|
||||||
|
*/
|
||||||
|
enum riscv_iommu_ddtp_modes {
|
||||||
|
RISCV_IOMMU_DDTP_IOMMU_MODE_OFF = 0,
|
||||||
|
RISCV_IOMMU_DDTP_IOMMU_MODE_BARE = 1,
|
||||||
|
RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL = 2,
|
||||||
|
RISCV_IOMMU_DDTP_IOMMU_MODE_2LVL = 3,
|
||||||
|
RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL = 4,
|
||||||
|
RISCV_IOMMU_DDTP_IOMMU_MODE_MAX = 4
|
||||||
|
};
|
||||||
|
|
||||||
|
/* 5.6 Command Queue Base (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_CQB 0x0018
|
||||||
|
#define RISCV_IOMMU_CQB_ENTRIES RISCV_IOMMU_QUEUE_LOG2SZ_FIELD
|
||||||
|
#define RISCV_IOMMU_CQB_PPN RISCV_IOMMU_PPN_FIELD
|
||||||
|
|
||||||
|
/* 5.7 Command Queue head (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_CQH 0x0020
|
||||||
|
#define RISCV_IOMMU_CQH_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||||
|
|
||||||
|
/* 5.8 Command Queue tail (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_CQT 0x0024
|
||||||
|
#define RISCV_IOMMU_CQT_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||||
|
|
||||||
|
/* 5.9 Fault Queue Base (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_FQB 0x0028
|
||||||
|
#define RISCV_IOMMU_FQB_ENTRIES RISCV_IOMMU_QUEUE_LOG2SZ_FIELD
|
||||||
|
#define RISCV_IOMMU_FQB_PPN RISCV_IOMMU_PPN_FIELD
|
||||||
|
|
||||||
|
/* 5.10 Fault Queue Head (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_FQH 0x0030
|
||||||
|
#define RISCV_IOMMU_FQH_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||||
|
|
||||||
|
/* 5.11 Fault Queue tail (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_FQT 0x0034
|
||||||
|
#define RISCV_IOMMU_FQT_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||||
|
|
||||||
|
/* 5.12 Page Request Queue base (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_PQB 0x0038
|
||||||
|
#define RISCV_IOMMU_PQB_ENTRIES RISCV_IOMMU_QUEUE_LOG2SZ_FIELD
|
||||||
|
#define RISCV_IOMMU_PQB_PPN RISCV_IOMMU_PPN_FIELD
|
||||||
|
|
||||||
|
/* 5.13 Page Request Queue head (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_PQH 0x0040
|
||||||
|
#define RISCV_IOMMU_PQH_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||||
|
|
||||||
|
/* 5.14 Page Request Queue tail (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_PQT 0x0044
|
||||||
|
#define RISCV_IOMMU_PQT_INDEX_MASK RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||||
|
|
||||||
|
/* 5.15 Command Queue CSR (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_CQCSR 0x0048
|
||||||
|
#define RISCV_IOMMU_CQCSR_CQEN RISCV_IOMMU_QUEUE_ENABLE
|
||||||
|
#define RISCV_IOMMU_CQCSR_CIE RISCV_IOMMU_QUEUE_INTR_ENABLE
|
||||||
|
#define RISCV_IOMMU_CQCSR_CQMF RISCV_IOMMU_QUEUE_MEM_FAULT
|
||||||
|
#define RISCV_IOMMU_CQCSR_CMD_TO BIT(9)
|
||||||
|
#define RISCV_IOMMU_CQCSR_CMD_ILL BIT(10)
|
||||||
|
#define RISCV_IOMMU_CQCSR_FENCE_W_IP BIT(11)
|
||||||
|
#define RISCV_IOMMU_CQCSR_CQON RISCV_IOMMU_QUEUE_ACTIVE
|
||||||
|
#define RISCV_IOMMU_CQCSR_BUSY RISCV_IOMMU_QUEUE_BUSY
|
||||||
|
|
||||||
|
/* 5.16 Fault Queue CSR (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_FQCSR 0x004C
|
||||||
|
#define RISCV_IOMMU_FQCSR_FQEN RISCV_IOMMU_QUEUE_ENABLE
|
||||||
|
#define RISCV_IOMMU_FQCSR_FIE RISCV_IOMMU_QUEUE_INTR_ENABLE
|
||||||
|
#define RISCV_IOMMU_FQCSR_FQMF RISCV_IOMMU_QUEUE_MEM_FAULT
|
||||||
|
#define RISCV_IOMMU_FQCSR_FQOF RISCV_IOMMU_QUEUE_OVERFLOW
|
||||||
|
#define RISCV_IOMMU_FQCSR_FQON RISCV_IOMMU_QUEUE_ACTIVE
|
||||||
|
#define RISCV_IOMMU_FQCSR_BUSY RISCV_IOMMU_QUEUE_BUSY
|
||||||
|
|
||||||
|
/* 5.17 Page Request Queue CSR (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_PQCSR 0x0050
|
||||||
|
#define RISCV_IOMMU_PQCSR_PQEN RISCV_IOMMU_QUEUE_ENABLE
|
||||||
|
#define RISCV_IOMMU_PQCSR_PIE RISCV_IOMMU_QUEUE_INTR_ENABLE
|
||||||
|
#define RISCV_IOMMU_PQCSR_PQMF RISCV_IOMMU_QUEUE_MEM_FAULT
|
||||||
|
#define RISCV_IOMMU_PQCSR_PQOF RISCV_IOMMU_QUEUE_OVERFLOW
|
||||||
|
#define RISCV_IOMMU_PQCSR_PQON RISCV_IOMMU_QUEUE_ACTIVE
|
||||||
|
#define RISCV_IOMMU_PQCSR_BUSY RISCV_IOMMU_QUEUE_BUSY
|
||||||
|
|
||||||
|
/* 5.18 Interrupt Pending Status (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_IPSR 0x0054
|
||||||
|
|
||||||
|
#define RISCV_IOMMU_INTR_CQ 0
|
||||||
|
#define RISCV_IOMMU_INTR_FQ 1
|
||||||
|
#define RISCV_IOMMU_INTR_PM 2
|
||||||
|
#define RISCV_IOMMU_INTR_PQ 3
|
||||||
|
#define RISCV_IOMMU_INTR_COUNT 4
|
||||||
|
|
||||||
|
#define RISCV_IOMMU_IPSR_CIP BIT(RISCV_IOMMU_INTR_CQ)
|
||||||
|
#define RISCV_IOMMU_IPSR_FIP BIT(RISCV_IOMMU_INTR_FQ)
|
||||||
|
#define RISCV_IOMMU_IPSR_PMIP BIT(RISCV_IOMMU_INTR_PM)
|
||||||
|
#define RISCV_IOMMU_IPSR_PIP BIT(RISCV_IOMMU_INTR_PQ)
|
||||||
|
|
||||||
|
/* 5.19 Performance monitoring counter overflow status (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_IOCOUNTOVF 0x0058
|
||||||
|
#define RISCV_IOMMU_IOCOUNTOVF_CY BIT(0)
|
||||||
|
#define RISCV_IOMMU_IOCOUNTOVF_HPM GENMASK_ULL(31, 1)
|
||||||
|
|
||||||
|
/* 5.20 Performance monitoring counter inhibits (32bits) */
|
||||||
|
#define RISCV_IOMMU_REG_IOCOUNTINH 0x005C
|
||||||
|
#define RISCV_IOMMU_IOCOUNTINH_CY BIT(0)
|
||||||
|
#define RISCV_IOMMU_IOCOUNTINH_HPM GENMASK(31, 1)
|
||||||
|
|
||||||
|
/* 5.21 Performance monitoring cycles counter (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_IOHPMCYCLES 0x0060
|
||||||
|
#define RISCV_IOMMU_IOHPMCYCLES_COUNTER GENMASK_ULL(62, 0)
|
||||||
|
#define RISCV_IOMMU_IOHPMCYCLES_OF BIT_ULL(63)
|
||||||
|
|
||||||
|
/* 5.22 Performance monitoring event counters (31 * 64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_IOHPMCTR_BASE 0x0068
|
||||||
|
#define RISCV_IOMMU_REG_IOHPMCTR(_n) (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
|
||||||
|
|
||||||
|
/* 5.23 Performance monitoring event selectors (31 * 64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_IOHPMEVT_BASE 0x0160
|
||||||
|
#define RISCV_IOMMU_REG_IOHPMEVT(_n) (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_EVENTID GENMASK_ULL(14, 0)
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_DMASK BIT_ULL(15)
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_PV_PSCV BIT_ULL(60)
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_DV_GSCV BIT_ULL(61)
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_IDT BIT_ULL(62)
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_OF BIT_ULL(63)
|
||||||
|
|
||||||
|
/* Number of defined performance-monitoring event selectors */
|
||||||
|
#define RISCV_IOMMU_IOHPMEVT_CNT 31
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
|
||||||
|
*
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
|
||||||
|
* @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
|
||||||
|
*/
|
||||||
|
enum riscv_iommu_hpmevent_id {
|
||||||
|
RISCV_IOMMU_HPMEVENT_INVALID = 0,
|
||||||
|
RISCV_IOMMU_HPMEVENT_URQ = 1,
|
||||||
|
RISCV_IOMMU_HPMEVENT_TRQ = 2,
|
||||||
|
RISCV_IOMMU_HPMEVENT_ATS_RQ = 3,
|
||||||
|
RISCV_IOMMU_HPMEVENT_TLB_MISS = 4,
|
||||||
|
RISCV_IOMMU_HPMEVENT_DD_WALK = 5,
|
||||||
|
RISCV_IOMMU_HPMEVENT_PD_WALK = 6,
|
||||||
|
RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
|
||||||
|
RISCV_IOMMU_HPMEVENT_G_WALKS = 8,
|
||||||
|
RISCV_IOMMU_HPMEVENT_MAX = 9
|
||||||
|
};
|
||||||
|
|
||||||
|
/* 5.24 Translation request IOVA (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_TR_REQ_IOVA 0x0258
|
||||||
|
#define RISCV_IOMMU_TR_REQ_IOVA_VPN GENMASK_ULL(63, 12)
|
||||||
|
|
||||||
|
/* 5.25 Translation request control (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_TR_REQ_CTL 0x0260
|
||||||
|
#define RISCV_IOMMU_TR_REQ_CTL_GO_BUSY BIT_ULL(0)
|
||||||
|
#define RISCV_IOMMU_TR_REQ_CTL_PRIV BIT_ULL(1)
|
||||||
|
#define RISCV_IOMMU_TR_REQ_CTL_EXE BIT_ULL(2)
|
||||||
|
#define RISCV_IOMMU_TR_REQ_CTL_NW BIT_ULL(3)
|
||||||
|
#define RISCV_IOMMU_TR_REQ_CTL_PID GENMASK_ULL(31, 12)
|
||||||
|
#define RISCV_IOMMU_TR_REQ_CTL_PV BIT_ULL(32)
|
||||||
|
#define RISCV_IOMMU_TR_REQ_CTL_DID GENMASK_ULL(63, 40)
|
||||||
|
|
||||||
|
/* 5.26 Translation request response (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_TR_RESPONSE 0x0268
|
||||||
|
#define RISCV_IOMMU_TR_RESPONSE_FAULT BIT_ULL(0)
|
||||||
|
#define RISCV_IOMMU_TR_RESPONSE_PBMT GENMASK_ULL(8, 7)
|
||||||
|
#define RISCV_IOMMU_TR_RESPONSE_SZ BIT_ULL(9)
|
||||||
|
#define RISCV_IOMMU_TR_RESPONSE_PPN RISCV_IOMMU_PPN_FIELD
|
||||||
|
|
||||||
|
/* 5.27 Interrupt cause to vector (64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_ICVEC 0x02F8
|
||||||
|
#define RISCV_IOMMU_ICVEC_CIV GENMASK_ULL(3, 0)
|
||||||
|
#define RISCV_IOMMU_ICVEC_FIV GENMASK_ULL(7, 4)
|
||||||
|
#define RISCV_IOMMU_ICVEC_PMIV GENMASK_ULL(11, 8)
|
||||||
|
#define RISCV_IOMMU_ICVEC_PIV GENMASK_ULL(15, 12)
|
||||||
|
|
||||||
|
/* 5.28 MSI Configuration table (32 * 64bits) */
|
||||||
|
#define RISCV_IOMMU_REG_MSI_CFG_TBL 0x0300
|
||||||
|
#define RISCV_IOMMU_REG_MSI_CFG_TBL_ADDR(_n) \
|
||||||
|
(RISCV_IOMMU_REG_MSI_CFG_TBL + ((_n) * 0x10))
|
||||||
|
#define RISCV_IOMMU_MSI_CFG_TBL_ADDR GENMASK_ULL(55, 2)
|
||||||
|
#define RISCV_IOMMU_REG_MSI_CFG_TBL_DATA(_n) \
|
||||||
|
(RISCV_IOMMU_REG_MSI_CFG_TBL + ((_n) * 0x10) + 0x08)
|
||||||
|
#define RISCV_IOMMU_MSI_CFG_TBL_DATA GENMASK_ULL(31, 0)
|
||||||
|
#define RISCV_IOMMU_REG_MSI_CFG_TBL_CTRL(_n) \
|
||||||
|
(RISCV_IOMMU_REG_MSI_CFG_TBL + ((_n) * 0x10) + 0x0C)
|
||||||
|
#define RISCV_IOMMU_MSI_CFG_TBL_CTRL_M BIT_ULL(0)
|
||||||
|
|
||||||
|
#define RISCV_IOMMU_REG_SIZE 0x1000
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Chapter 2: Data structures
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Device Directory Table macros for non-leaf nodes
|
||||||
|
*/
|
||||||
|
#define RISCV_IOMMU_DDTE_V BIT_ULL(0)
|
||||||
|
#define RISCV_IOMMU_DDTE_PPN RISCV_IOMMU_PPN_FIELD
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct riscv_iommu_dc - Device Context
|
||||||
|
* @tc: Translation Control
|
||||||
|
* @iohgatp: I/O Hypervisor guest address translation and protection
|
||||||
|
* (Second stage context)
|
||||||
|
* @ta: Translation Attributes
|
||||||
|
* @fsc: First stage context
|
||||||
|
* @msiptp: MSI page table pointer
|
||||||
|
* @msi_addr_mask: MSI address mask
|
||||||
|
* @msi_addr_pattern: MSI address pattern
|
||||||
|
* @_reserved: Reserved for future use, padding
|
||||||
|
*
|
||||||
|
* This structure is used for leaf nodes on the Device Directory Table,
|
||||||
|
* in case RISCV_IOMMU_CAPABILITIES_MSI_FLAT is not set, the bottom 4 fields
|
||||||
|
* are not present and are skipped with pointer arithmetic to avoid
|
||||||
|
* casting, check out riscv_iommu_get_dc().
|
||||||
|
* See section 2.1 for more details
|
||||||
|
*/
|
||||||
|
struct riscv_iommu_dc {
|
||||||
|
u64 tc;
|
||||||
|
u64 iohgatp;
|
||||||
|
u64 ta;
|
||||||
|
u64 fsc;
|
||||||
|
u64 msiptp;
|
||||||
|
u64 msi_addr_mask;
|
||||||
|
u64 msi_addr_pattern;
|
||||||
|
u64 _reserved;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Translation control fields */
|
||||||
|
#define RISCV_IOMMU_DC_TC_V BIT_ULL(0)
|
||||||
|
#define RISCV_IOMMU_DC_TC_EN_ATS BIT_ULL(1)
|
||||||
|
#define RISCV_IOMMU_DC_TC_EN_PRI BIT_ULL(2)
|
||||||
|
#define RISCV_IOMMU_DC_TC_T2GPA BIT_ULL(3)
|
||||||
|
#define RISCV_IOMMU_DC_TC_DTF BIT_ULL(4)
|
||||||
|
#define RISCV_IOMMU_DC_TC_PDTV BIT_ULL(5)
|
||||||
|
#define RISCV_IOMMU_DC_TC_PRPR BIT_ULL(6)
|
||||||
|
#define RISCV_IOMMU_DC_TC_GADE BIT_ULL(7)
|
||||||
|
#define RISCV_IOMMU_DC_TC_SADE BIT_ULL(8)
|
||||||
|
#define RISCV_IOMMU_DC_TC_DPE BIT_ULL(9)
|
||||||
|
#define RISCV_IOMMU_DC_TC_SBE BIT_ULL(10)
|
||||||
|
#define RISCV_IOMMU_DC_TC_SXL BIT_ULL(11)
|
||||||
|
|
||||||
|
/* Second-stage (aka G-stage) context fields */
|
||||||
|
#define RISCV_IOMMU_DC_IOHGATP_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||||
|
#define RISCV_IOMMU_DC_IOHGATP_GSCID GENMASK_ULL(59, 44)
|
||||||
|
#define RISCV_IOMMU_DC_IOHGATP_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum riscv_iommu_dc_iohgatp_modes - Guest address translation/protection modes
|
||||||
|
* @RISCV_IOMMU_DC_IOHGATP_MODE_BARE: No translation/protection
|
||||||
|
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4: Sv32x4 (2-bit extension of Sv32), when fctl.GXL == 1
|
||||||
|
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4: Sv39x4 (2-bit extension of Sv39), when fctl.GXL == 0
|
||||||
|
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4: Sv48x4 (2-bit extension of Sv48), when fctl.GXL == 0
|
||||||
|
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4: Sv57x4 (2-bit extension of Sv57), when fctl.GXL == 0
|
||||||
|
*/
|
||||||
|
enum riscv_iommu_dc_iohgatp_modes {
|
||||||
|
RISCV_IOMMU_DC_IOHGATP_MODE_BARE = 0,
|
||||||
|
RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4 = 8,
|
||||||
|
RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4 = 8,
|
||||||
|
RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4 = 9,
|
||||||
|
RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4 = 10
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Translation attributes fields */
|
||||||
|
#define RISCV_IOMMU_DC_TA_PSCID GENMASK_ULL(31, 12)
|
||||||
|
|
||||||
|
/* First-stage context fields */
|
||||||
|
#define RISCV_IOMMU_DC_FSC_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||||
|
#define RISCV_IOMMU_DC_FSC_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum riscv_iommu_dc_fsc_atp_modes - First stage address translation/protection modes
|
||||||
|
* @RISCV_IOMMU_DC_FSC_MODE_BARE: No translation/protection
|
||||||
|
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32: Sv32, when dc.tc.SXL == 1
|
||||||
|
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39: Sv39, when dc.tc.SXL == 0
|
||||||
|
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48: Sv48, when dc.tc.SXL == 0
|
||||||
|
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57: Sv57, when dc.tc.SXL == 0
|
||||||
|
* @RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8: 1lvl PDT, 8bit process ids
|
||||||
|
* @RISCV_IOMMU_DC_FSC_PDTP_MODE_PD17: 2lvl PDT, 17bit process ids
|
||||||
|
* @RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20: 3lvl PDT, 20bit process ids
|
||||||
|
*
|
||||||
|
* FSC holds IOSATP when RISCV_IOMMU_DC_TC_PDTV is 0 and PDTP otherwise.
|
||||||
|
* IOSATP controls the first stage address translation (same as the satp register on
|
||||||
|
* the RISC-V MMU), and PDTP holds the process directory table, used to select a
|
||||||
|
* first stage page table based on a process id (for devices that support multiple
|
||||||
|
* process ids).
|
||||||
|
*/
|
||||||
|
enum riscv_iommu_dc_fsc_atp_modes {
|
||||||
|
RISCV_IOMMU_DC_FSC_MODE_BARE = 0,
|
||||||
|
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32 = 8,
|
||||||
|
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39 = 8,
|
||||||
|
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48 = 9,
|
||||||
|
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57 = 10,
|
||||||
|
RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8 = 1,
|
||||||
|
RISCV_IOMMU_DC_FSC_PDTP_MODE_PD17 = 2,
|
||||||
|
RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20 = 3
|
||||||
|
};
|
||||||
|
|
||||||
|
/* MSI page table pointer */
|
||||||
|
#define RISCV_IOMMU_DC_MSIPTP_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||||
|
#define RISCV_IOMMU_DC_MSIPTP_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||||
|
#define RISCV_IOMMU_DC_MSIPTP_MODE_OFF 0
|
||||||
|
#define RISCV_IOMMU_DC_MSIPTP_MODE_FLAT 1
|
||||||
|
|
||||||
|
/* MSI address mask */
|
||||||
|
#define RISCV_IOMMU_DC_MSI_ADDR_MASK GENMASK_ULL(51, 0)
|
||||||
|
|
||||||
|
/* MSI address pattern */
|
||||||
|
#define RISCV_IOMMU_DC_MSI_PATTERN GENMASK_ULL(51, 0)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct riscv_iommu_pc - Process Context
|
||||||
|
* @ta: Translation Attributes
|
||||||
|
* @fsc: First stage context
|
||||||
|
*
|
||||||
|
* This structure is used for leaf nodes on the Process Directory Table
|
||||||
|
* See section 2.3 for more details
|
||||||
|
*/
|
||||||
|
struct riscv_iommu_pc {
|
||||||
|
u64 ta;
|
||||||
|
u64 fsc;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Translation attributes fields */
|
||||||
|
#define RISCV_IOMMU_PC_TA_V BIT_ULL(0)
|
||||||
|
#define RISCV_IOMMU_PC_TA_ENS BIT_ULL(1)
|
||||||
|
#define RISCV_IOMMU_PC_TA_SUM BIT_ULL(2)
|
||||||
|
#define RISCV_IOMMU_PC_TA_PSCID GENMASK_ULL(31, 12)
|
||||||
|
|
||||||
|
/* First stage context fields */
|
||||||
|
#define RISCV_IOMMU_PC_FSC_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||||
|
#define RISCV_IOMMU_PC_FSC_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Chapter 3: In-memory queue interface
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct riscv_iommu_command - Generic IOMMU command structure
|
||||||
|
* @dword0: Includes the opcode and the function identifier
|
||||||
|
* @dword1: Opcode specific data
|
||||||
|
*
|
||||||
|
* The commands are interpreted as two 64bit fields, where the first
|
||||||
|
* 7bits of the first field are the opcode which also defines the
|
||||||
|
* command's format, followed by a 3bit field that specifies the
|
||||||
|
* function invoked by that command, and the rest is opcode-specific.
|
||||||
|
* This is a generic struct which will be populated differently
|
||||||
|
* according to each command. For more infos on the commands and
|
||||||
|
* the command queue check section 3.1.
|
||||||
|
*/
|
||||||
|
struct riscv_iommu_command {
|
||||||
|
u64 dword0;
|
||||||
|
u64 dword1;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Fields on dword0, common for all commands */
|
||||||
|
#define RISCV_IOMMU_CMD_OPCODE GENMASK_ULL(6, 0)
|
||||||
|
#define RISCV_IOMMU_CMD_FUNC GENMASK_ULL(9, 7)
|
||||||
|
|
||||||
|
/* 3.1.1 IOMMU Page-table cache invalidation */
|
||||||
|
/* Fields on dword0 */
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_OPCODE 1
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA 0
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA 1
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_AV BIT_ULL(10)
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_PSCID GENMASK_ULL(31, 12)
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_PSCV BIT_ULL(32)
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_GV BIT_ULL(33)
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_GSCID GENMASK_ULL(59, 44)
|
||||||
|
/* dword1[61:10] is the 4K-aligned page address */
|
||||||
|
#define RISCV_IOMMU_CMD_IOTINVAL_ADDR GENMASK_ULL(61, 10)
|
||||||
|
|
||||||
|
/* 3.1.2 IOMMU Command Queue Fences */
|
||||||
|
/* Fields on dword0 */
|
||||||
|
#define RISCV_IOMMU_CMD_IOFENCE_OPCODE 2
|
||||||
|
#define RISCV_IOMMU_CMD_IOFENCE_FUNC_C 0
|
||||||
|
#define RISCV_IOMMU_CMD_IOFENCE_AV BIT_ULL(10)
|
||||||
|
#define RISCV_IOMMU_CMD_IOFENCE_WSI BIT_ULL(11)
|
||||||
|
#define RISCV_IOMMU_CMD_IOFENCE_PR BIT_ULL(12)
|
||||||
|
#define RISCV_IOMMU_CMD_IOFENCE_PW BIT_ULL(13)
|
||||||
|
#define RISCV_IOMMU_CMD_IOFENCE_DATA GENMASK_ULL(63, 32)
|
||||||
|
/* dword1 is the address, word-size aligned and shifted to the right by two bits. */
|
||||||
|
|
||||||
|
/* 3.1.3 IOMMU Directory cache invalidation */
|
||||||
|
/* Fields on dword0 */
|
||||||
|
#define RISCV_IOMMU_CMD_IODIR_OPCODE 3
|
||||||
|
#define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT 0
|
||||||
|
#define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT 1
|
||||||
|
#define RISCV_IOMMU_CMD_IODIR_PID GENMASK_ULL(31, 12)
|
||||||
|
#define RISCV_IOMMU_CMD_IODIR_DV BIT_ULL(33)
|
||||||
|
#define RISCV_IOMMU_CMD_IODIR_DID GENMASK_ULL(63, 40)
|
||||||
|
/* dword1 is reserved for standard use */
|
||||||
|
|
||||||
|
/* 3.1.4 IOMMU PCIe ATS */
|
||||||
|
/* Fields on dword0 */
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_OPCODE 4
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_FUNC_INVAL 0
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_FUNC_PRGR 1
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_PID GENMASK_ULL(31, 12)
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_PV BIT_ULL(32)
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_DSV BIT_ULL(33)
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_RID GENMASK_ULL(55, 40)
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_DSEG GENMASK_ULL(63, 56)
|
||||||
|
/* dword1 is the ATS payload, two different payload types for INVAL and PRGR */
|
||||||
|
|
||||||
|
/* ATS.INVAL payload*/
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_INVAL_G BIT_ULL(0)
|
||||||
|
/* Bits 1 - 10 are zeroed */
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_INVAL_S BIT_ULL(11)
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_INVAL_UADDR GENMASK_ULL(63, 12)
|
||||||
|
|
||||||
|
/* ATS.PRGR payload */
|
||||||
|
/* Bits 0 - 31 are zeroed */
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_PRGR_PRG_INDEX GENMASK_ULL(40, 32)
|
||||||
|
/* Bits 41 - 43 are zeroed */
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44)
|
||||||
|
#define RISCV_IOMMU_CMD_ATS_PRGR_DST_ID GENMASK_ULL(63, 48)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct riscv_iommu_fq_record - Fault/Event Queue Record
|
||||||
|
* @hdr: Header, includes fault/event cause, PID/DID, transaction type etc
|
||||||
|
* @_reserved: Low 32bits for custom use, high 32bits for standard use
|
||||||
|
* @iotval: Transaction-type/cause specific format
|
||||||
|
* @iotval2: Cause specific format
|
||||||
|
*
|
||||||
|
* The fault/event queue reports events and failures raised when
|
||||||
|
* processing transactions. Each record is a 32byte structure where
|
||||||
|
* the first dword has a fixed format for providing generic infos
|
||||||
|
* regarding the fault/event, and two more dwords are there for
|
||||||
|
* fault/event-specific information. For more details see section
|
||||||
|
* 3.2.
|
||||||
|
*/
|
||||||
|
struct riscv_iommu_fq_record {
|
||||||
|
u64 hdr;
|
||||||
|
u64 _reserved;
|
||||||
|
u64 iotval;
|
||||||
|
u64 iotval2;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Fields on header */
|
||||||
|
#define RISCV_IOMMU_FQ_HDR_CAUSE GENMASK_ULL(11, 0)
|
||||||
|
#define RISCV_IOMMU_FQ_HDR_PID GENMASK_ULL(31, 12)
|
||||||
|
#define RISCV_IOMMU_FQ_HDR_PV BIT_ULL(32)
|
||||||
|
#define RISCV_IOMMU_FQ_HDR_PRIV BIT_ULL(33)
|
||||||
|
#define RISCV_IOMMU_FQ_HDR_TTYP GENMASK_ULL(39, 34)
|
||||||
|
#define RISCV_IOMMU_FQ_HDR_DID GENMASK_ULL(63, 40)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum riscv_iommu_fq_causes - Fault/event cause values
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_INST_FAULT: Instruction access fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_RD_ADDR_MISALIGNED: Read address misaligned
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_RD_FAULT: Read load fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_WR_ADDR_MISALIGNED: Write/AMO address misaligned
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_WR_FAULT: Write/AMO access fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_INST_FAULT_S: Instruction page fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S: Read page fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S: Write/AMO page fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_INST_FAULT_VS: Instruction guest page fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS: Read guest page fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS: Write/AMO guest page fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED: All inbound transactions disallowed
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT: DDT entry load access fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_DDT_INVALID: DDT entry invalid
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED: DDT entry misconfigured
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_TTYP_BLOCKED: Transaction type disallowed
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT: MSI PTE load access fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_MSI_INVALID: MSI PTE invalid
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED: MSI PTE misconfigured
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_MRIF_FAULT: MRIF access fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT: PDT entry load access fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_PDT_INVALID: PDT entry invalid
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED: PDT entry misconfigured
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_DDT_CORRUPTED: DDT data corruption
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_PDT_CORRUPTED: PDT data corruption
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_MSI_PT_CORRUPTED: MSI page table data corruption
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_MRIF_CORRUIPTED: MRIF data corruption
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_INTERNAL_DP_ERROR: Internal data path error
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT: IOMMU MSI write access fault
|
||||||
|
* @RISCV_IOMMU_FQ_CAUSE_PT_CORRUPTED: First/second stage page table data corruption
|
||||||
|
*
|
||||||
|
* Values are on table 11 of the spec, encodings 275 - 2047 are reserved for standard
|
||||||
|
* use, and 2048 - 4095 for custom use.
|
||||||
|
*/
|
||||||
|
enum riscv_iommu_fq_causes {
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_INST_FAULT = 1,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_RD_ADDR_MISALIGNED = 4,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_RD_FAULT = 5,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_WR_ADDR_MISALIGNED = 6,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_WR_FAULT = 7,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_INST_FAULT_S = 12,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S = 13,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S = 15,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_INST_FAULT_VS = 20,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS = 21,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS = 23,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED = 256,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT = 257,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_DDT_INVALID = 258,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED = 259,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_TTYP_BLOCKED = 260,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT = 261,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_MSI_INVALID = 262,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED = 263,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_MRIF_FAULT = 264,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT = 265,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_PDT_INVALID = 266,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED = 267,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_DDT_CORRUPTED = 268,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_PDT_CORRUPTED = 269,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_MSI_PT_CORRUPTED = 270,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_MRIF_CORRUIPTED = 271,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_INTERNAL_DP_ERROR = 272,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT = 273,
|
||||||
|
RISCV_IOMMU_FQ_CAUSE_PT_CORRUPTED = 274
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum riscv_iommu_fq_ttypes: Fault/event transaction types
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_NONE: None. Fault not caused by an inbound transaction.
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_UADDR_INST_FETCH: Instruction fetch from untranslated address
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_UADDR_RD: Read from untranslated address
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_UADDR_WR: Write/AMO to untranslated address
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_TADDR_INST_FETCH: Instruction fetch from translated address
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_TADDR_RD: Read from translated address
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_TADDR_WR: Write/AMO to translated address
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_PCIE_ATS_REQ: PCIe ATS translation request
|
||||||
|
* @RISCV_IOMMU_FQ_TTYP_PCIE_MSG_REQ: PCIe message request
|
||||||
|
*
|
||||||
|
* Values are on table 12 of the spec, type 4 and 10 - 31 are reserved for standard use
|
||||||
|
* and 31 - 63 for custom use.
|
||||||
|
*/
|
||||||
|
enum riscv_iommu_fq_ttypes {
|
||||||
|
RISCV_IOMMU_FQ_TTYP_NONE = 0,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_UADDR_INST_FETCH = 1,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_UADDR_RD = 2,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_UADDR_WR = 3,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_TADDR_INST_FETCH = 5,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_TADDR_RD = 6,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_TADDR_WR = 7,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_PCIE_ATS_REQ = 8,
|
||||||
|
RISCV_IOMMU_FQ_TTYP_PCIE_MSG_REQ = 9,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct riscv_iommu_pq_record - PCIe Page Request record
|
||||||
|
* @hdr: Header, includes PID, DID etc
|
||||||
|
* @payload: Holds the page address, request group and permission bits
|
||||||
|
*
|
||||||
|
* For more infos on the PCIe Page Request queue see chapter 3.3.
|
||||||
|
*/
|
||||||
|
struct riscv_iommu_pq_record {
|
||||||
|
u64 hdr;
|
||||||
|
u64 payload;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Header fields */
|
||||||
|
#define RISCV_IOMMU_PQ_HDR_PID GENMASK_ULL(31, 12)
|
||||||
|
#define RISCV_IOMMU_PQ_HDR_PV BIT_ULL(32)
|
||||||
|
#define RISCV_IOMMU_PQ_HDR_PRIV BIT_ULL(33)
|
||||||
|
#define RISCV_IOMMU_PQ_HDR_EXEC BIT_ULL(34)
|
||||||
|
#define RISCV_IOMMU_PQ_HDR_DID GENMASK_ULL(63, 40)
|
||||||
|
|
||||||
|
/* Payload fields */
|
||||||
|
#define RISCV_IOMMU_PQ_PAYLOAD_R BIT_ULL(0)
|
||||||
|
#define RISCV_IOMMU_PQ_PAYLOAD_W BIT_ULL(1)
|
||||||
|
#define RISCV_IOMMU_PQ_PAYLOAD_L BIT_ULL(2)
|
||||||
|
#define RISCV_IOMMU_PQ_PAYLOAD_RWL_MASK GENMASK_ULL(2, 0)
|
||||||
|
#define RISCV_IOMMU_PQ_PAYLOAD_PRGI GENMASK_ULL(11, 3) /* Page Request Group Index */
|
||||||
|
#define RISCV_IOMMU_PQ_PAYLOAD_ADDR GENMASK_ULL(63, 12)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct riscv_iommu_msipte - MSI Page Table Entry
|
||||||
|
* @pte: MSI PTE
|
||||||
|
* @mrif_info: Memory-resident interrupt file info
|
||||||
|
*
|
||||||
|
* The MSI Page Table is used for virtualizing MSIs, so that when
|
||||||
|
* a device sends an MSI to a guest, the IOMMU can reroute it
|
||||||
|
* by translating the MSI address, either to a guest interrupt file
|
||||||
|
* or a memory resident interrupt file (MRIF). Note that this page table
|
||||||
|
* is an array of MSI PTEs, not a multi-level pt, each entry
|
||||||
|
* is a leaf entry. For more infos check out the AIA spec, chapter 9.5.
|
||||||
|
*
|
||||||
|
* Also in basic mode the mrif_info field is ignored by the IOMMU and can
|
||||||
|
* be used by software, any other reserved fields on pte must be zeroed-out
|
||||||
|
* by software.
|
||||||
|
*/
|
||||||
|
struct riscv_iommu_msipte {
|
||||||
|
u64 pte;
|
||||||
|
u64 mrif_info;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Fields on pte */
|
||||||
|
#define RISCV_IOMMU_MSIPTE_V BIT_ULL(0)
|
||||||
|
#define RISCV_IOMMU_MSIPTE_M GENMASK_ULL(2, 1)
|
||||||
|
#define RISCV_IOMMU_MSIPTE_MRIF_ADDR GENMASK_ULL(53, 7) /* When M == 1 (MRIF mode) */
|
||||||
|
#define RISCV_IOMMU_MSIPTE_PPN RISCV_IOMMU_PPN_FIELD /* When M == 3 (basic mode) */
|
||||||
|
#define RISCV_IOMMU_MSIPTE_C BIT_ULL(63)
|
||||||
|
|
||||||
|
/* Fields on mrif_info */
|
||||||
|
#define RISCV_IOMMU_MSIPTE_MRIF_NID GENMASK_ULL(9, 0)
|
||||||
|
#define RISCV_IOMMU_MSIPTE_MRIF_NPPN RISCV_IOMMU_PPN_FIELD
|
||||||
|
#define RISCV_IOMMU_MSIPTE_MRIF_NID_MSB BIT_ULL(60)
|
||||||
|
|
||||||
|
/* Helper functions: command structure builders. */
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_inval_vma(struct riscv_iommu_command *cmd)
|
||||||
|
{
|
||||||
|
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) |
|
||||||
|
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA);
|
||||||
|
cmd->dword1 = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cmd,
|
||||||
|
u64 addr)
|
||||||
|
{
|
||||||
|
cmd->dword1 = FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, phys_to_pfn(addr));
|
||||||
|
cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd,
|
||||||
|
int pscid)
|
||||||
|
{
|
||||||
|
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_PSCID, pscid) |
|
||||||
|
RISCV_IOMMU_CMD_IOTINVAL_PSCV;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_inval_set_gscid(struct riscv_iommu_command *cmd,
|
||||||
|
int gscid)
|
||||||
|
{
|
||||||
|
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_GSCID, gscid) |
|
||||||
|
RISCV_IOMMU_CMD_IOTINVAL_GV;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_iofence(struct riscv_iommu_command *cmd)
|
||||||
|
{
|
||||||
|
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
|
||||||
|
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
|
||||||
|
RISCV_IOMMU_CMD_IOFENCE_PR | RISCV_IOMMU_CMD_IOFENCE_PW;
|
||||||
|
cmd->dword1 = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_iofence_set_av(struct riscv_iommu_command *cmd,
|
||||||
|
u64 addr, u32 data)
|
||||||
|
{
|
||||||
|
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
|
||||||
|
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
|
||||||
|
FIELD_PREP(RISCV_IOMMU_CMD_IOFENCE_DATA, data) |
|
||||||
|
RISCV_IOMMU_CMD_IOFENCE_AV;
|
||||||
|
cmd->dword1 = addr >> 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_iodir_inval_ddt(struct riscv_iommu_command *cmd)
|
||||||
|
{
|
||||||
|
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
|
||||||
|
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT);
|
||||||
|
cmd->dword1 = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_iodir_inval_pdt(struct riscv_iommu_command *cmd)
|
||||||
|
{
|
||||||
|
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
|
||||||
|
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT);
|
||||||
|
cmd->dword1 = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_iodir_set_did(struct riscv_iommu_command *cmd,
|
||||||
|
unsigned int devid)
|
||||||
|
{
|
||||||
|
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_DID, devid) |
|
||||||
|
RISCV_IOMMU_CMD_IODIR_DV;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void riscv_iommu_cmd_iodir_set_pid(struct riscv_iommu_command *cmd,
|
||||||
|
unsigned int pasid)
|
||||||
|
{
|
||||||
|
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_PID, pasid);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* _RISCV_IOMMU_BITS_H_ */
|
||||||
|
|
@ -0,0 +1,120 @@
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-only
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright © 2022-2024 Rivos Inc.
|
||||||
|
* Copyright © 2023 FORTH-ICS/CARV
|
||||||
|
*
|
||||||
|
* RISCV IOMMU as a PCIe device
|
||||||
|
*
|
||||||
|
* Authors
|
||||||
|
* Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||||
|
* Nick Kossifidis <mick@ics.forth.gr>
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <linux/compiler.h>
|
||||||
|
#include <linux/init.h>
|
||||||
|
#include <linux/iommu.h>
|
||||||
|
#include <linux/kernel.h>
|
||||||
|
#include <linux/pci.h>
|
||||||
|
|
||||||
|
#include "iommu-bits.h"
|
||||||
|
#include "iommu.h"
|
||||||
|
|
||||||
|
/* QEMU RISC-V IOMMU implementation */
|
||||||
|
#define PCI_DEVICE_ID_REDHAT_RISCV_IOMMU 0x0014
|
||||||
|
|
||||||
|
/* Rivos Inc. assigned PCI Vendor and Device IDs */
|
||||||
|
#ifndef PCI_VENDOR_ID_RIVOS
|
||||||
|
#define PCI_VENDOR_ID_RIVOS 0x1efd
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA 0x0008
|
||||||
|
|
||||||
|
static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
||||||
|
{
|
||||||
|
struct device *dev = &pdev->dev;
|
||||||
|
struct riscv_iommu_device *iommu;
|
||||||
|
int rc, vec;
|
||||||
|
|
||||||
|
rc = pcim_enable_device(pdev);
|
||||||
|
if (rc)
|
||||||
|
return rc;
|
||||||
|
|
||||||
|
if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
|
||||||
|
return -ENODEV;
|
||||||
|
|
||||||
|
if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
|
||||||
|
return -ENODEV;
|
||||||
|
|
||||||
|
rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
|
||||||
|
if (rc)
|
||||||
|
return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
|
||||||
|
|
||||||
|
iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
|
||||||
|
if (!iommu)
|
||||||
|
return -ENOMEM;
|
||||||
|
|
||||||
|
iommu->dev = dev;
|
||||||
|
iommu->reg = pcim_iomap_table(pdev)[0];
|
||||||
|
|
||||||
|
pci_set_master(pdev);
|
||||||
|
dev_set_drvdata(dev, iommu);
|
||||||
|
|
||||||
|
/* Check device reported capabilities / features. */
|
||||||
|
iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
|
||||||
|
iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
|
||||||
|
|
||||||
|
/* The PCI driver only uses MSIs, make sure the IOMMU supports this */
|
||||||
|
switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
|
||||||
|
case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
|
||||||
|
case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return dev_err_probe(dev, -ENODEV,
|
||||||
|
"unable to use message-signaled interrupts\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Allocate and assign IRQ vectors for the various events */
|
||||||
|
rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
|
||||||
|
PCI_IRQ_MSIX | PCI_IRQ_MSI);
|
||||||
|
if (rc <= 0)
|
||||||
|
return dev_err_probe(dev, -ENODEV,
|
||||||
|
"unable to allocate irq vectors\n");
|
||||||
|
|
||||||
|
iommu->irqs_count = rc;
|
||||||
|
for (vec = 0; vec < iommu->irqs_count; vec++)
|
||||||
|
iommu->irqs[vec] = msi_get_virq(dev, vec);
|
||||||
|
|
||||||
|
/* Enable message-signaled interrupts, fctl.WSI */
|
||||||
|
if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
|
||||||
|
iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
|
||||||
|
riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return riscv_iommu_init(iommu);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void riscv_iommu_pci_remove(struct pci_dev *pdev)
|
||||||
|
{
|
||||||
|
struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
|
||||||
|
|
||||||
|
riscv_iommu_remove(iommu);
|
||||||
|
}
|
||||||
|
|
||||||
|
static const struct pci_device_id riscv_iommu_pci_tbl[] = {
|
||||||
|
{PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_RISCV_IOMMU), 0},
|
||||||
|
{PCI_VDEVICE(RIVOS, PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA), 0},
|
||||||
|
{0,}
|
||||||
|
};
|
||||||
|
|
||||||
|
static struct pci_driver riscv_iommu_pci_driver = {
|
||||||
|
.name = KBUILD_MODNAME,
|
||||||
|
.id_table = riscv_iommu_pci_tbl,
|
||||||
|
.probe = riscv_iommu_pci_probe,
|
||||||
|
.remove = riscv_iommu_pci_remove,
|
||||||
|
.driver = {
|
||||||
|
.suppress_bind_attrs = true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
builtin_pci_driver(riscv_iommu_pci_driver);
|
||||||
|
|
@ -0,0 +1,92 @@
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-only
|
||||||
|
/*
|
||||||
|
* RISC-V IOMMU as a platform device
|
||||||
|
*
|
||||||
|
* Copyright © 2023 FORTH-ICS/CARV
|
||||||
|
* Copyright © 2023-2024 Rivos Inc.
|
||||||
|
*
|
||||||
|
* Authors
|
||||||
|
* Nick Kossifidis <mick@ics.forth.gr>
|
||||||
|
* Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <linux/kernel.h>
|
||||||
|
#include <linux/of_platform.h>
|
||||||
|
#include <linux/platform_device.h>
|
||||||
|
|
||||||
|
#include "iommu-bits.h"
|
||||||
|
#include "iommu.h"
|
||||||
|
|
||||||
|
static int riscv_iommu_platform_probe(struct platform_device *pdev)
|
||||||
|
{
|
||||||
|
struct device *dev = &pdev->dev;
|
||||||
|
struct riscv_iommu_device *iommu = NULL;
|
||||||
|
struct resource *res = NULL;
|
||||||
|
int vec;
|
||||||
|
|
||||||
|
iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
|
||||||
|
if (!iommu)
|
||||||
|
return -ENOMEM;
|
||||||
|
|
||||||
|
iommu->dev = dev;
|
||||||
|
iommu->reg = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
|
||||||
|
if (IS_ERR(iommu->reg))
|
||||||
|
return dev_err_probe(dev, PTR_ERR(iommu->reg),
|
||||||
|
"could not map register region\n");
|
||||||
|
|
||||||
|
dev_set_drvdata(dev, iommu);
|
||||||
|
|
||||||
|
/* Check device reported capabilities / features. */
|
||||||
|
iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
|
||||||
|
iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
|
||||||
|
|
||||||
|
/* For now we only support WSI */
|
||||||
|
switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
|
||||||
|
case RISCV_IOMMU_CAPABILITIES_IGS_WSI:
|
||||||
|
case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return dev_err_probe(dev, -ENODEV,
|
||||||
|
"unable to use wire-signaled interrupts\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
iommu->irqs_count = platform_irq_count(pdev);
|
||||||
|
if (iommu->irqs_count <= 0)
|
||||||
|
return dev_err_probe(dev, -ENODEV,
|
||||||
|
"no IRQ resources provided\n");
|
||||||
|
if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT)
|
||||||
|
iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;
|
||||||
|
|
||||||
|
for (vec = 0; vec < iommu->irqs_count; vec++)
|
||||||
|
iommu->irqs[vec] = platform_get_irq(pdev, vec);
|
||||||
|
|
||||||
|
/* Enable wire-signaled interrupts, fctl.WSI */
|
||||||
|
if (!(iommu->fctl & RISCV_IOMMU_FCTL_WSI)) {
|
||||||
|
iommu->fctl |= RISCV_IOMMU_FCTL_WSI;
|
||||||
|
riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return riscv_iommu_init(iommu);
|
||||||
|
};
|
||||||
|
|
||||||
|
static void riscv_iommu_platform_remove(struct platform_device *pdev)
|
||||||
|
{
|
||||||
|
riscv_iommu_remove(dev_get_drvdata(&pdev->dev));
|
||||||
|
};
|
||||||
|
|
||||||
|
static const struct of_device_id riscv_iommu_of_match[] = {
|
||||||
|
{.compatible = "riscv,iommu",},
|
||||||
|
{},
|
||||||
|
};
|
||||||
|
|
||||||
|
static struct platform_driver riscv_iommu_platform_driver = {
|
||||||
|
.probe = riscv_iommu_platform_probe,
|
||||||
|
.remove_new = riscv_iommu_platform_remove,
|
||||||
|
.driver = {
|
||||||
|
.name = "riscv,iommu",
|
||||||
|
.of_match_table = riscv_iommu_of_match,
|
||||||
|
.suppress_bind_attrs = true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
builtin_platform_driver(riscv_iommu_platform_driver);
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,88 @@
|
||||||
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||||
|
/*
|
||||||
|
* Copyright © 2022-2024 Rivos Inc.
|
||||||
|
* Copyright © 2023 FORTH-ICS/CARV
|
||||||
|
*
|
||||||
|
* Authors
|
||||||
|
* Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||||
|
* Nick Kossifidis <mick@ics.forth.gr>
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef _RISCV_IOMMU_H_
|
||||||
|
#define _RISCV_IOMMU_H_
|
||||||
|
|
||||||
|
#include <linux/iommu.h>
|
||||||
|
#include <linux/types.h>
|
||||||
|
#include <linux/iopoll.h>
|
||||||
|
|
||||||
|
#include "iommu-bits.h"
|
||||||
|
|
||||||
|
struct riscv_iommu_device;
|
||||||
|
|
||||||
|
struct riscv_iommu_queue {
|
||||||
|
atomic_t prod; /* unbounded producer allocation index */
|
||||||
|
atomic_t head; /* unbounded shadow ring buffer consumer index */
|
||||||
|
atomic_t tail; /* unbounded shadow ring buffer producer index */
|
||||||
|
unsigned int mask; /* index mask, queue length - 1 */
|
||||||
|
unsigned int irq; /* allocated interrupt number */
|
||||||
|
struct riscv_iommu_device *iommu; /* iommu device handling the queue when active */
|
||||||
|
void *base; /* ring buffer kernel pointer */
|
||||||
|
dma_addr_t phys; /* ring buffer physical address */
|
||||||
|
u16 qbr; /* base register offset, head and tail reference */
|
||||||
|
u16 qcr; /* control and status register offset */
|
||||||
|
u8 qid; /* queue identifier, same as RISCV_IOMMU_INTR_XX */
|
||||||
|
};
|
||||||
|
|
||||||
|
struct riscv_iommu_device {
|
||||||
|
/* iommu core interface */
|
||||||
|
struct iommu_device iommu;
|
||||||
|
|
||||||
|
/* iommu hardware */
|
||||||
|
struct device *dev;
|
||||||
|
|
||||||
|
/* hardware control register space */
|
||||||
|
void __iomem *reg;
|
||||||
|
|
||||||
|
/* supported and enabled hardware capabilities */
|
||||||
|
u64 caps;
|
||||||
|
u32 fctl;
|
||||||
|
|
||||||
|
/* available interrupt numbers, MSI or WSI */
|
||||||
|
unsigned int irqs[RISCV_IOMMU_INTR_COUNT];
|
||||||
|
unsigned int irqs_count;
|
||||||
|
unsigned int icvec;
|
||||||
|
|
||||||
|
/* hardware queues */
|
||||||
|
struct riscv_iommu_queue cmdq;
|
||||||
|
struct riscv_iommu_queue fltq;
|
||||||
|
|
||||||
|
/* device directory */
|
||||||
|
unsigned int ddt_mode;
|
||||||
|
dma_addr_t ddt_phys;
|
||||||
|
u64 *ddt_root;
|
||||||
|
};
|
||||||
|
|
||||||
|
int riscv_iommu_init(struct riscv_iommu_device *iommu);
|
||||||
|
void riscv_iommu_remove(struct riscv_iommu_device *iommu);
|
||||||
|
|
||||||
|
#define riscv_iommu_readl(iommu, addr) \
|
||||||
|
readl_relaxed((iommu)->reg + (addr))
|
||||||
|
|
||||||
|
#define riscv_iommu_readq(iommu, addr) \
|
||||||
|
readq_relaxed((iommu)->reg + (addr))
|
||||||
|
|
||||||
|
#define riscv_iommu_writel(iommu, addr, val) \
|
||||||
|
writel_relaxed((val), (iommu)->reg + (addr))
|
||||||
|
|
||||||
|
#define riscv_iommu_writeq(iommu, addr, val) \
|
||||||
|
writeq_relaxed((val), (iommu)->reg + (addr))
|
||||||
|
|
||||||
|
#define riscv_iommu_readq_timeout(iommu, addr, val, cond, delay_us, timeout_us) \
|
||||||
|
readx_poll_timeout(readq_relaxed, (iommu)->reg + (addr), val, cond, \
|
||||||
|
delay_us, timeout_us)
|
||||||
|
|
||||||
|
#define riscv_iommu_readl_timeout(iommu, addr, val, cond, delay_us, timeout_us) \
|
||||||
|
readx_poll_timeout(readl_relaxed, (iommu)->reg + (addr), val, cond, \
|
||||||
|
delay_us, timeout_us)
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
@ -33,6 +33,8 @@ struct s390_domain {
|
||||||
struct rcu_head rcu;
|
struct rcu_head rcu;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static struct iommu_domain blocking_domain;
|
||||||
|
|
||||||
static inline unsigned int calc_rtx(dma_addr_t ptr)
|
static inline unsigned int calc_rtx(dma_addr_t ptr)
|
||||||
{
|
{
|
||||||
return ((unsigned long)ptr >> ZPCI_RT_SHIFT) & ZPCI_INDEX_MASK;
|
return ((unsigned long)ptr >> ZPCI_RT_SHIFT) & ZPCI_INDEX_MASK;
|
||||||
|
|
@ -369,20 +371,36 @@ static void s390_domain_free(struct iommu_domain *domain)
|
||||||
call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain);
|
call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void s390_iommu_detach_device(struct iommu_domain *domain,
|
static void zdev_s390_domain_update(struct zpci_dev *zdev,
|
||||||
struct device *dev)
|
struct iommu_domain *domain)
|
||||||
{
|
{
|
||||||
struct s390_domain *s390_domain = to_s390_domain(domain);
|
|
||||||
struct zpci_dev *zdev = to_zpci_dev(dev);
|
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
|
|
||||||
|
spin_lock_irqsave(&zdev->dom_lock, flags);
|
||||||
|
zdev->s390_domain = domain;
|
||||||
|
spin_unlock_irqrestore(&zdev->dom_lock, flags);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int blocking_domain_attach_device(struct iommu_domain *domain,
|
||||||
|
struct device *dev)
|
||||||
|
{
|
||||||
|
struct zpci_dev *zdev = to_zpci_dev(dev);
|
||||||
|
struct s390_domain *s390_domain;
|
||||||
|
unsigned long flags;
|
||||||
|
|
||||||
|
if (zdev->s390_domain->type == IOMMU_DOMAIN_BLOCKED)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
s390_domain = to_s390_domain(zdev->s390_domain);
|
||||||
spin_lock_irqsave(&s390_domain->list_lock, flags);
|
spin_lock_irqsave(&s390_domain->list_lock, flags);
|
||||||
list_del_rcu(&zdev->iommu_list);
|
list_del_rcu(&zdev->iommu_list);
|
||||||
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
|
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
|
||||||
|
|
||||||
zpci_unregister_ioat(zdev, 0);
|
zpci_unregister_ioat(zdev, 0);
|
||||||
zdev->s390_domain = NULL;
|
|
||||||
zdev->dma_table = NULL;
|
zdev->dma_table = NULL;
|
||||||
|
zdev_s390_domain_update(zdev, domain);
|
||||||
|
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int s390_iommu_attach_device(struct iommu_domain *domain,
|
static int s390_iommu_attach_device(struct iommu_domain *domain,
|
||||||
|
|
@ -401,20 +419,15 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
|
||||||
domain->geometry.aperture_end < zdev->start_dma))
|
domain->geometry.aperture_end < zdev->start_dma))
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
if (zdev->s390_domain)
|
blocking_domain_attach_device(&blocking_domain, dev);
|
||||||
s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
|
|
||||||
|
|
||||||
|
/* If we fail now DMA remains blocked via blocking domain */
|
||||||
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
|
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
|
||||||
virt_to_phys(s390_domain->dma_table), &status);
|
virt_to_phys(s390_domain->dma_table), &status);
|
||||||
/*
|
|
||||||
* If the device is undergoing error recovery the reset code
|
|
||||||
* will re-establish the new domain.
|
|
||||||
*/
|
|
||||||
if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL)
|
if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL)
|
||||||
return -EIO;
|
return -EIO;
|
||||||
|
|
||||||
zdev->dma_table = s390_domain->dma_table;
|
zdev->dma_table = s390_domain->dma_table;
|
||||||
zdev->s390_domain = s390_domain;
|
zdev_s390_domain_update(zdev, domain);
|
||||||
|
|
||||||
spin_lock_irqsave(&s390_domain->list_lock, flags);
|
spin_lock_irqsave(&s390_domain->list_lock, flags);
|
||||||
list_add_rcu(&zdev->iommu_list, &s390_domain->devices);
|
list_add_rcu(&zdev->iommu_list, &s390_domain->devices);
|
||||||
|
|
@ -466,21 +479,13 @@ static struct iommu_device *s390_iommu_probe_device(struct device *dev)
|
||||||
if (zdev->tlb_refresh)
|
if (zdev->tlb_refresh)
|
||||||
dev->iommu->shadow_on_flush = 1;
|
dev->iommu->shadow_on_flush = 1;
|
||||||
|
|
||||||
|
/* Start with DMA blocked */
|
||||||
|
spin_lock_init(&zdev->dom_lock);
|
||||||
|
zdev_s390_domain_update(zdev, &blocking_domain);
|
||||||
|
|
||||||
return &zdev->iommu_dev;
|
return &zdev->iommu_dev;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void s390_iommu_release_device(struct device *dev)
|
|
||||||
{
|
|
||||||
struct zpci_dev *zdev = to_zpci_dev(dev);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* release_device is expected to detach any domain currently attached
|
|
||||||
* to the device, but keep it attached to other devices in the group.
|
|
||||||
*/
|
|
||||||
if (zdev)
|
|
||||||
s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int zpci_refresh_all(struct zpci_dev *zdev)
|
static int zpci_refresh_all(struct zpci_dev *zdev)
|
||||||
{
|
{
|
||||||
return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
|
return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
|
||||||
|
|
@ -697,9 +702,15 @@ static size_t s390_iommu_unmap_pages(struct iommu_domain *domain,
|
||||||
|
|
||||||
struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev)
|
struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev)
|
||||||
{
|
{
|
||||||
if (!zdev || !zdev->s390_domain)
|
struct s390_domain *s390_domain;
|
||||||
|
|
||||||
|
lockdep_assert_held(&zdev->dom_lock);
|
||||||
|
|
||||||
|
if (zdev->s390_domain->type == IOMMU_DOMAIN_BLOCKED)
|
||||||
return NULL;
|
return NULL;
|
||||||
return &zdev->s390_domain->ctrs;
|
|
||||||
|
s390_domain = to_s390_domain(zdev->s390_domain);
|
||||||
|
return &s390_domain->ctrs;
|
||||||
}
|
}
|
||||||
|
|
||||||
int zpci_init_iommu(struct zpci_dev *zdev)
|
int zpci_init_iommu(struct zpci_dev *zdev)
|
||||||
|
|
@ -776,11 +787,19 @@ static int __init s390_iommu_init(void)
|
||||||
}
|
}
|
||||||
subsys_initcall(s390_iommu_init);
|
subsys_initcall(s390_iommu_init);
|
||||||
|
|
||||||
|
static struct iommu_domain blocking_domain = {
|
||||||
|
.type = IOMMU_DOMAIN_BLOCKED,
|
||||||
|
.ops = &(const struct iommu_domain_ops) {
|
||||||
|
.attach_dev = blocking_domain_attach_device,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
static const struct iommu_ops s390_iommu_ops = {
|
static const struct iommu_ops s390_iommu_ops = {
|
||||||
|
.blocked_domain = &blocking_domain,
|
||||||
|
.release_domain = &blocking_domain,
|
||||||
.capable = s390_iommu_capable,
|
.capable = s390_iommu_capable,
|
||||||
.domain_alloc_paging = s390_domain_alloc_paging,
|
.domain_alloc_paging = s390_domain_alloc_paging,
|
||||||
.probe_device = s390_iommu_probe_device,
|
.probe_device = s390_iommu_probe_device,
|
||||||
.release_device = s390_iommu_release_device,
|
|
||||||
.device_group = generic_device_group,
|
.device_group = generic_device_group,
|
||||||
.pgsize_bitmap = SZ_4K,
|
.pgsize_bitmap = SZ_4K,
|
||||||
.get_resv_regions = s390_iommu_get_resv_regions,
|
.get_resv_regions = s390_iommu_get_resv_regions,
|
||||||
|
|
|
||||||
|
|
@ -78,9 +78,10 @@ int tegra_vde_iommu_init(struct tegra_vde *vde)
|
||||||
arm_iommu_release_mapping(mapping);
|
arm_iommu_release_mapping(mapping);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
vde->domain = iommu_domain_alloc(&platform_bus_type);
|
vde->domain = iommu_paging_domain_alloc(dev);
|
||||||
if (!vde->domain) {
|
if (IS_ERR(vde->domain)) {
|
||||||
err = -ENOMEM;
|
err = PTR_ERR(vde->domain);
|
||||||
|
vde->domain = NULL;
|
||||||
goto put_group;
|
goto put_group;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -109,10 +109,10 @@ static int rproc_enable_iommu(struct rproc *rproc)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
domain = iommu_domain_alloc(dev->bus);
|
domain = iommu_paging_domain_alloc(dev);
|
||||||
if (!domain) {
|
if (IS_ERR(domain)) {
|
||||||
dev_err(dev, "can't alloc iommu domain\n");
|
dev_err(dev, "can't alloc iommu domain\n");
|
||||||
return -ENOMEM;
|
return PTR_ERR(domain);
|
||||||
}
|
}
|
||||||
|
|
||||||
iommu_set_fault_handler(domain, rproc_iommu_fault, rproc);
|
iommu_set_fault_handler(domain, rproc_iommu_fault, rproc);
|
||||||
|
|
|
||||||
|
|
@ -511,8 +511,6 @@ static inline int __iommu_copy_struct_from_user_array(
|
||||||
* the caller iommu_domain_alloc() returns.
|
* the caller iommu_domain_alloc() returns.
|
||||||
* @domain_alloc_user: Allocate an iommu domain corresponding to the input
|
* @domain_alloc_user: Allocate an iommu domain corresponding to the input
|
||||||
* parameters as defined in include/uapi/linux/iommufd.h.
|
* parameters as defined in include/uapi/linux/iommufd.h.
|
||||||
* Unlike @domain_alloc, it is called only by IOMMUFD and
|
|
||||||
* must fully initialize the new domain before return.
|
|
||||||
* Upon success, if the @user_data is valid and the @parent
|
* Upon success, if the @user_data is valid and the @parent
|
||||||
* points to a kernel-managed domain, the new domain must be
|
* points to a kernel-managed domain, the new domain must be
|
||||||
* IOMMU_DOMAIN_NESTED type; otherwise, the @parent must be
|
* IOMMU_DOMAIN_NESTED type; otherwise, the @parent must be
|
||||||
|
|
@ -784,12 +782,14 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
extern int bus_iommu_probe(const struct bus_type *bus);
|
|
||||||
extern bool iommu_present(const struct bus_type *bus);
|
extern bool iommu_present(const struct bus_type *bus);
|
||||||
extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);
|
extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);
|
||||||
extern bool iommu_group_has_isolated_msi(struct iommu_group *group);
|
extern bool iommu_group_has_isolated_msi(struct iommu_group *group);
|
||||||
extern struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus);
|
struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev, unsigned int flags);
|
||||||
struct iommu_domain *iommu_paging_domain_alloc(struct device *dev);
|
static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
|
||||||
|
{
|
||||||
|
return iommu_paging_domain_alloc_flags(dev, 0);
|
||||||
|
}
|
||||||
extern void iommu_domain_free(struct iommu_domain *domain);
|
extern void iommu_domain_free(struct iommu_domain *domain);
|
||||||
extern int iommu_attach_device(struct iommu_domain *domain,
|
extern int iommu_attach_device(struct iommu_domain *domain,
|
||||||
struct device *dev);
|
struct device *dev);
|
||||||
|
|
@ -1081,19 +1081,15 @@ struct iommu_iotlb_gather {};
|
||||||
struct iommu_dirty_bitmap {};
|
struct iommu_dirty_bitmap {};
|
||||||
struct iommu_dirty_ops {};
|
struct iommu_dirty_ops {};
|
||||||
|
|
||||||
static inline bool iommu_present(const struct bus_type *bus)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
|
static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
|
static inline struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
|
||||||
|
unsigned int flags)
|
||||||
{
|
{
|
||||||
return NULL;
|
return ERR_PTR(-ENODEV);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
|
static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
|
||||||
|
|
|
||||||
|
|
@ -359,11 +359,19 @@ struct iommu_vfio_ioas {
|
||||||
* enforced on device attachment
|
* enforced on device attachment
|
||||||
* @IOMMU_HWPT_FAULT_ID_VALID: The fault_id field of hwpt allocation data is
|
* @IOMMU_HWPT_FAULT_ID_VALID: The fault_id field of hwpt allocation data is
|
||||||
* valid.
|
* valid.
|
||||||
|
* @IOMMU_HWPT_ALLOC_PASID: Requests a domain that can be used with PASID. The
|
||||||
|
* domain can be attached to any PASID on the device.
|
||||||
|
* Any domain attached to the non-PASID part of the
|
||||||
|
* device must also be flaged, otherwise attaching a
|
||||||
|
* PASID will blocked.
|
||||||
|
* If IOMMU does not support PASID it will return
|
||||||
|
* error (-EOPNOTSUPP).
|
||||||
*/
|
*/
|
||||||
enum iommufd_hwpt_alloc_flags {
|
enum iommufd_hwpt_alloc_flags {
|
||||||
IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
|
IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
|
||||||
IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
|
IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
|
||||||
IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
|
IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
|
||||||
|
IOMMU_HWPT_ALLOC_PASID = 1 << 3,
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue