aboutsummaryrefslogtreecommitdiff
path: root/drivers/iommu/amd
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu/amd')
-rw-r--r--drivers/iommu/amd/amd_iommu.h9
-rw-r--r--drivers/iommu/amd/amd_iommu_types.h12
-rw-r--r--drivers/iommu/amd/init.c30
-rw-r--r--drivers/iommu/amd/io_pgtable.c4
-rw-r--r--drivers/iommu/amd/io_pgtable_v2.c25
-rw-r--r--drivers/iommu/amd/iommu.c17
6 files changed, 71 insertions, 26 deletions
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index c160a332ce33..e98f20a9bdd8 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -34,6 +34,7 @@ extern int amd_iommu_reenable(int);
extern int amd_iommu_enable_faulting(void);
extern int amd_iommu_guest_ir;
extern enum io_pgtable_fmt amd_iommu_pgtable;
+extern int amd_iommu_gpt_level;
/* IOMMUv2 specific functions */
struct iommu_domain;
@@ -122,6 +123,14 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev)
return PCI_SEG_DEVID_TO_SBDF(seg, devid);
}
+static inline void *alloc_pgtable_page(int nid, gfp_t gfp)
+{
+ struct page *page;
+
+ page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0);
+ return page ? page_address(page) : NULL;
+}
+
extern bool translation_pre_enabled(struct amd_iommu *iommu);
extern bool amd_iommu_is_attach_deferred(struct device *dev);
extern int __init add_special_device(u8 type, u8 id, u32 *devid,
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 3d684190b4d5..2ddbda3a4374 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -93,6 +93,8 @@
#define FEATURE_GA (1ULL<<7)
#define FEATURE_HE (1ULL<<8)
#define FEATURE_PC (1ULL<<9)
+#define FEATURE_GATS_SHIFT (12)
+#define FEATURE_GATS_MASK (3ULL)
#define FEATURE_GAM_VAPIC (1ULL<<21)
#define FEATURE_GIOSUP (1ULL<<48)
#define FEATURE_EPHSUP (1ULL<<50)
@@ -305,6 +307,9 @@
#define PAGE_MODE_6_LEVEL 0x06
#define PAGE_MODE_7_LEVEL 0x07
+#define GUEST_PGTABLE_4_LEVEL 0x00
+#define GUEST_PGTABLE_5_LEVEL 0x01
+
#define PM_LEVEL_SHIFT(x) (12 + ((x) * 9))
#define PM_LEVEL_SIZE(x) (((x) < 6) ? \
((1ULL << PM_LEVEL_SHIFT((x))) - 1): \
@@ -398,6 +403,8 @@
#define DTE_GCR3_SHIFT_B 16
#define DTE_GCR3_SHIFT_C 43
+#define DTE_GPT_LEVEL_SHIFT 54
+
#define GCR3_VALID 0x01ULL
#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
@@ -549,6 +556,7 @@ struct protection_domain {
spinlock_t lock; /* mostly used to lock the page table*/
u16 id; /* the domain id written to the device table */
int glx; /* Number of levels for GCR3 table */
+ int nid; /* Node ID */
u64 *gcr3_tbl; /* Guest CR3 table */
unsigned long flags; /* flags to find out type of domain */
unsigned dev_cnt; /* devices assigned to this domain */
@@ -1001,8 +1009,8 @@ struct amd_ir_data {
*/
struct irq_cfg *cfg;
int ga_vector;
- int ga_root_ptr;
- int ga_tag;
+ u64 ga_root_ptr;
+ u32 ga_tag;
};
struct amd_irte_ops {
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 19a46b9f7357..329a406cc37d 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -153,6 +153,8 @@ bool amd_iommu_dump;
bool amd_iommu_irq_remap __read_mostly;
enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1;
+/* Guest page table level */
+int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
@@ -306,6 +308,11 @@ static bool check_feature_on_all_iommus(u64 mask)
return !!(amd_iommu_efr & mask);
}
+static inline int check_feature_gpt_level(void)
+{
+ return ((amd_iommu_efr >> FEATURE_GATS_SHIFT) & FEATURE_GATS_MASK);
+}
+
/*
* For IVHD type 0x11/0x40, EFR is also available via IVHD.
* Default to IVHD EFR since it is available sooner
@@ -1941,7 +1948,7 @@ static ssize_t amd_iommu_show_cap(struct device *dev,
char *buf)
{
struct amd_iommu *iommu = dev_to_amd_iommu(dev);
- return sprintf(buf, "%x\n", iommu->cap);
+ return sysfs_emit(buf, "%x\n", iommu->cap);
}
static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL);
@@ -1950,7 +1957,7 @@ static ssize_t amd_iommu_show_features(struct device *dev,
char *buf)
{
struct amd_iommu *iommu = dev_to_amd_iommu(dev);
- return sprintf(buf, "%llx:%llx\n", iommu->features2, iommu->features);
+ return sysfs_emit(buf, "%llx:%llx\n", iommu->features2, iommu->features);
}
static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
@@ -2155,8 +2162,10 @@ static void print_iommu_info(void)
if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
pr_info("X2APIC enabled\n");
}
- if (amd_iommu_pgtable == AMD_IOMMU_V2)
- pr_info("V2 page table enabled\n");
+ if (amd_iommu_pgtable == AMD_IOMMU_V2) {
+ pr_info("V2 page table enabled (Paging mode : %d level)\n",
+ amd_iommu_gpt_level);
+ }
}
static int __init amd_iommu_init_pci(void)
@@ -2383,6 +2392,7 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu)
struct irq_domain *domain;
struct irq_alloc_info info;
int irq, ret;
+ int node = dev_to_node(&iommu->dev->dev);
domain = iommu_get_irqdomain();
if (!domain)
@@ -2392,7 +2402,7 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu)
info.type = X86_IRQ_ALLOC_TYPE_AMDVI;
info.data = iommu;
- irq = irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, &info);
+ irq = irq_domain_alloc_irqs(domain, 1, node, &info);
if (irq < 0) {
irq_domain_remove(domain);
return irq;
@@ -3025,6 +3035,11 @@ static int __init early_amd_iommu_init(void)
if (ret)
goto out;
+ /* 5 level guest page table */
+ if (cpu_feature_enabled(X86_FEATURE_LA57) &&
+ check_feature_gpt_level() == GUEST_PGTABLE_5_LEVEL)
+ amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
+
/* Disable any previously enabled IOMMUs */
if (!is_kdump_kernel() || amd_iommu_disabled)
disable_iommus();
@@ -3556,6 +3571,11 @@ __setup("ivrs_acpihid", parse_ivrs_acpihid);
bool amd_iommu_v2_supported(void)
{
+ /* CPU page table size should match IOMMU guest page table size */
+ if (cpu_feature_enabled(X86_FEATURE_LA57) &&
+ amd_iommu_gpt_level != PAGE_MODE_5_LEVEL)
+ return false;
+
/*
* Since DTE[Mode]=0 is prohibited on SNP-enabled system
* (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index ace0e9b8b913..1b67116882be 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -156,7 +156,7 @@ static bool increase_address_space(struct protection_domain *domain,
bool ret = true;
u64 *pte;
- pte = (void *)get_zeroed_page(gfp);
+ pte = alloc_pgtable_page(domain->nid, gfp);
if (!pte)
return false;
@@ -250,7 +250,7 @@ static u64 *alloc_pte(struct protection_domain *domain,
if (!IOMMU_PTE_PRESENT(__pte) ||
pte_level == PAGE_MODE_NONE) {
- page = (u64 *)get_zeroed_page(gfp);
+ page = alloc_pgtable_page(domain->nid, gfp);
if (!page)
return NULL;
diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c
index 8638ddf6fb3b..27c3015947e6 100644
--- a/drivers/iommu/amd/io_pgtable_v2.c
+++ b/drivers/iommu/amd/io_pgtable_v2.c
@@ -37,8 +37,7 @@
static inline int get_pgtable_level(void)
{
- /* 5 level page table is not supported */
- return PAGE_MODE_4_LEVEL;
+ return amd_iommu_gpt_level;
}
static inline bool is_large_pte(u64 pte)
@@ -46,11 +45,6 @@ static inline bool is_large_pte(u64 pte)
return (pte & IOMMU_PAGE_PSE);
}
-static inline void *alloc_pgtable_page(void)
-{
- return (void *)get_zeroed_page(GFP_KERNEL);
-}
-
static inline u64 set_pgtable_attr(u64 *page)
{
u64 prot;
@@ -138,8 +132,8 @@ static void free_pgtable(u64 *pt, int level)
}
/* Allocate page table */
-static u64 *v2_alloc_pte(u64 *pgd, unsigned long iova,
- unsigned long pg_size, bool *updated)
+static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova,
+ unsigned long pg_size, gfp_t gfp, bool *updated)
{
u64 *pte, *page;
int level, end_level;
@@ -162,7 +156,7 @@ static u64 *v2_alloc_pte(u64 *pgd, unsigned long iova,
}
if (!IOMMU_PTE_PRESENT(__pte)) {
- page = alloc_pgtable_page();
+ page = alloc_pgtable_page(nid, gfp);
if (!page)
return NULL;
@@ -262,7 +256,8 @@ static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
while (mapped_size < size) {
map_size = get_alloc_page_size(pgsize);
- pte = v2_alloc_pte(pdom->iop.pgd, iova, map_size, &updated);
+ pte = v2_alloc_pte(pdom->nid, pdom->iop.pgd,
+ iova, map_size, gfp, &updated);
if (!pte) {
ret = -EINVAL;
goto out;
@@ -383,8 +378,9 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
struct protection_domain *pdom = (struct protection_domain *)cookie;
int ret;
+ int ias = IOMMU_IN_ADDR_BIT_SIZE;
- pgtable->pgd = alloc_pgtable_page();
+ pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC);
if (!pgtable->pgd)
return NULL;
@@ -392,12 +388,15 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
if (ret)
goto err_free_pgd;
+ if (get_pgtable_level() == PAGE_MODE_5_LEVEL)
+ ias = 57;
+
pgtable->iop.ops.map_pages = iommu_v2_map_pages;
pgtable->iop.ops.unmap_pages = iommu_v2_unmap_pages;
pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys;
cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2,
- cfg->ias = IOMMU_IN_ADDR_BIT_SIZE,
+ cfg->ias = ias,
cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE,
cfg->tlb = &v2_flush_ops;
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 5a505ba5467e..4a314647d1f7 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1611,6 +1611,11 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid,
tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
flags |= tmp;
+ if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) {
+ dev_table[devid].data[2] |=
+ ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT);
+ }
+
if (domain->flags & PD_GIOV_MASK)
pte_root |= DTE_FLAG_GIOV;
}
@@ -1662,14 +1667,14 @@ static void do_attach(struct iommu_dev_data *dev_data,
dev_data->domain = domain;
list_add(&dev_data->list, &domain->dev_list);
+ /* Update NUMA Node ID */
+ if (domain->nid == NUMA_NO_NODE)
+ domain->nid = dev_to_node(dev_data->dev);
+
/* Do reference counting */
domain->dev_iommu[iommu->index] += 1;
domain->dev_cnt += 1;
- /* Override supported page sizes */
- if (domain->flags & PD_GIOV_MASK)
- domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;
-
/* Update device table */
set_dte_entry(iommu, dev_data->devid, domain,
ats, dev_data->iommu_v2);
@@ -2048,6 +2053,8 @@ static int protection_domain_init_v2(struct protection_domain *domain)
domain->flags |= PD_GIOV_MASK;
+ domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;
+
if (domain_enable_v2(domain, 1)) {
domain_id_free(domain->id);
return -ENOMEM;
@@ -2097,6 +2104,8 @@ static struct protection_domain *protection_domain_alloc(unsigned int type)
if (type == IOMMU_DOMAIN_IDENTITY)
return domain;
+ domain->nid = NUMA_NO_NODE;
+
pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain);
if (!pgtbl_ops) {
domain_id_free(domain->id);
Powered by cgit, Gruvboxed by Blaster4385.