From 895a79805c287df73142f1b424b22ea5190734c2 Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Wed, 4 May 2016 14:46:30 +1000 Subject: cxl: Handle num_of_processes larger than can fit in the SPA num_of_process is a 16 bit field, theoretically allowing an AFU to support 16K processes, however the scheduled process area currently has a maximum size of 1MB, which limits the maximum number of processes to 7704. Some AFUs may not necessarily care what the limit is and just want to be able to use the maximum by setting the field to 16K. To allow these to work, detect this situation and use the maximum size for the SPA. Downgrade the WARN_ON to a dev_warn. Signed-off-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/native.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'drivers/misc/cxl/native.c') diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c index 387fcbdf9793..b8b547a162eb 100644 --- a/drivers/misc/cxl/native.c +++ b/drivers/misc/cxl/native.c @@ -185,16 +185,25 @@ static int spa_max_procs(int spa_size) int cxl_alloc_spa(struct cxl_afu *afu) { + unsigned spa_size; + /* Work out how many pages to allocate */ afu->native->spa_order = 0; do { afu->native->spa_order++; - afu->native->spa_size = (1 << afu->native->spa_order) * PAGE_SIZE; + spa_size = (1 << afu->native->spa_order) * PAGE_SIZE; + + if (spa_size > 0x100000) { + dev_warn(&afu->dev, "num_of_processes too large for the SPA, limiting to %i (0x%x)\n", + afu->native->spa_max_procs, afu->native->spa_size); + afu->num_procs = afu->native->spa_max_procs; + break; + } + + afu->native->spa_size = spa_size; afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size); } while (afu->native->spa_max_procs < afu->num_procs); - WARN_ON(afu->native->spa_size > 0x100000); /* Max size supported by the hardware */ - if (!(afu->native->spa = (struct cxl_process_element *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) { pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n"); -- cgit From 3c206fa77aaaac8cd7d4cfcd840c82495b01b288 Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Wed, 4 May 2016 14:52:58 +1000 Subject: cxl: Ensure PSL interrupt is configured for contexts with no AFU IRQs In the cxl kernel API, it is possible to create a context and start it without allocating any interrupts. Since we assign or allocate the PSL interrupt when allocating AFU interrupts this will lead to a situation where we start the context with no means to take any faults. The user API is not affected as it always goes through the cxl interrupt allocation code paths and will have the PSL interrupt allocated or assigned, even if no AFU interrupts were requested. This checks that at least one interrupt is configured at the time of attach, and if not it will assign the multiplexed PSL interrupt for powernv, or allocate a single interrupt for PowerVM. Signed-off-by: Ian Munsie Reviewed-by: Frederic Barrat Signed-off-by: Michael Ellerman --- drivers/misc/cxl/guest.c | 12 ++++++++++++ drivers/misc/cxl/native.c | 9 +++++++++ 2 files changed, 21 insertions(+) (limited to 'drivers/misc/cxl/native.c') diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c index a83acf9f8cd9..769971c065b4 100644 --- a/drivers/misc/cxl/guest.c +++ b/drivers/misc/cxl/guest.c @@ -552,6 +552,17 @@ static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr) elem->common.sstp0 = cpu_to_be64(ctx->sstp0); elem->common.sstp1 = cpu_to_be64(ctx->sstp1); + + /* + * Ensure we have at least one interrupt allocated to take faults for + * kernel contexts that may not have allocated any AFU IRQs at all: + */ + if (ctx->irqs.range[0] == 0) { + rc = afu_register_irqs(ctx, 0); + if (rc) + goto out_free; + } + for (r = 0; r < CXL_IRQ_RANGES; r++) { for (i = 0; i < ctx->irqs.range[r]; i++) { if (r == 0 && i == 0) { @@ -597,6 +608,7 @@ static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr) enable_afu_irqs(ctx); } +out_free: free_page((u64)elem); return rc; } diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c index b8b547a162eb..5d4fb9fd84bc 100644 --- a/drivers/misc/cxl/native.c +++ b/drivers/misc/cxl/native.c @@ -534,6 +534,15 @@ static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr) ctx->elem->common.sstp0 = cpu_to_be64(ctx->sstp0); ctx->elem->common.sstp1 = cpu_to_be64(ctx->sstp1); + /* + * Ensure we have the multiplexed PSL interrupt set up to take faults + * for kernel contexts that may not have allocated any AFU IRQs at all: + */ + if (ctx->irqs.range[0] == 0) { + ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq; + ctx->irqs.range[0] = 1; + } + for (r = 0; r < CXL_IRQ_RANGES; r++) { ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]); ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]); -- cgit From 7a0d85d313c2066712e530e668bc02bb741a685c Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Fri, 6 May 2016 17:46:36 +1000 Subject: cxl: Add kernel API to allow a context to operate with relocate disabled cxl devices typically access memory using an MMU in much the same way as the CPU, and each context includes a state register much like the MSR in the CPU. Like the CPU, the state register includes a bit to enable relocation, which we currently always enable. In some cases, it may be desirable to allow a device to access memory using real addresses instead of effective addresses, so this adds a new API, cxl_set_translation_mode, that can be used to disable relocation on a given kernel context. This can allow for the creation of a special privileged context that the device can use if it needs relocation disabled, and can use regular contexts at times when it needs relocation enabled. This interface is only available to users of the kernel API for obvious reasons, and will never be supported in a virtualised environment. This will be used by the upcoming cxl support in the mlx5 driver. Signed-off-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/api.c | 19 +++++++++++++++++++ drivers/misc/cxl/cxl.h | 1 + drivers/misc/cxl/guest.c | 3 +++ drivers/misc/cxl/native.c | 5 +++-- include/misc/cxl.h | 8 ++++++++ 5 files changed, 34 insertions(+), 2 deletions(-) (limited to 'drivers/misc/cxl/native.c') diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c index 807582335a98..6d228ccd884d 100644 --- a/drivers/misc/cxl/api.c +++ b/drivers/misc/cxl/api.c @@ -183,6 +183,7 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed, ctx->pid = get_task_pid(task, PIDTYPE_PID); ctx->glpid = get_task_pid(task->group_leader, PIDTYPE_PID); kernel = false; + ctx->real_mode = false; } cxl_ctx_get(); @@ -219,6 +220,24 @@ void cxl_set_master(struct cxl_context *ctx) } EXPORT_SYMBOL_GPL(cxl_set_master); +int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode) +{ + if (ctx->status == STARTED) { + /* + * We could potentially update the PE and issue an update LLCMD + * to support this, but it doesn't seem to have a good use case + * since it's trivial to just create a second kernel context + * with different translation modes, so until someone convinces + * me otherwise: + */ + return -EBUSY; + } + + ctx->real_mode = real_mode; + return 0; +} +EXPORT_SYMBOL_GPL(cxl_set_translation_mode); + /* wrappers around afu_* file ops which are EXPORTED */ int cxl_fd_open(struct inode *inode, struct file *file) { diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h index 2823fb32fcf6..9dd3a0e7917f 100644 --- a/drivers/misc/cxl/cxl.h +++ b/drivers/misc/cxl/cxl.h @@ -514,6 +514,7 @@ struct cxl_context { bool pe_inserted; bool master; bool kernel; + bool real_mode; bool pending_irq; bool pending_fault; bool pending_afu_err; diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c index 769971c065b4..c2815b97de2e 100644 --- a/drivers/misc/cxl/guest.c +++ b/drivers/misc/cxl/guest.c @@ -617,6 +617,9 @@ static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u { pr_devel("in %s\n", __func__); + if (ctx->real_mode) + return -EPERM; + ctx->kernel = kernel; if (ctx->afu->current_mode == CXL_MODE_DIRECTED) return attach_afu_directed(ctx, wed, amr); diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c index 5d4fb9fd84bc..98f2cac45162 100644 --- a/drivers/misc/cxl/native.c +++ b/drivers/misc/cxl/native.c @@ -494,8 +494,9 @@ static u64 calculate_sr(struct cxl_context *ctx) if (mfspr(SPRN_LPCR) & LPCR_TC) sr |= CXL_PSL_SR_An_TC; if (ctx->kernel) { - sr |= CXL_PSL_SR_An_R | (mfmsr() & MSR_SF); - sr |= CXL_PSL_SR_An_HV; + if (!ctx->real_mode) + sr |= CXL_PSL_SR_An_R; + sr |= (mfmsr() & MSR_SF) | CXL_PSL_SR_An_HV; } else { sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R; sr &= ~(CXL_PSL_SR_An_HV); diff --git a/include/misc/cxl.h b/include/misc/cxl.h index 7d5e2613c7b8..56560c5781b4 100644 --- a/include/misc/cxl.h +++ b/include/misc/cxl.h @@ -126,6 +126,14 @@ int cxl_afu_reset(struct cxl_context *ctx); */ void cxl_set_master(struct cxl_context *ctx); +/* + * Sets the context to use real mode memory accesses to operate with + * translation disabled. Note that this only makes sense for kernel contexts + * under bare metal, and will not work with virtualisation. May only be + * performed on stopped contexts. + */ +int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode); + /* * Map and unmap the AFU Problem Space area. The amount and location mapped * depends on if this context is a master or slave. -- cgit