diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_drv.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_drv.c | 1753 |
1 files changed, 761 insertions, 992 deletions
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 9df65d386d11..3c512c571e60 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -36,7 +36,6 @@ #include <linux/pm_runtime.h> #include <linux/pnp.h> #include <linux/slab.h> -#include <linux/vgaarb.h> #include <linux/vga_switcheroo.h> #include <linux/vt.h> #include <acpi/video.h> @@ -47,416 +46,103 @@ #include <drm/drm_probe_helper.h> #include <drm/i915_drm.h> +#include "display/intel_acpi.h" +#include "display/intel_audio.h" +#include "display/intel_bw.h" +#include "display/intel_cdclk.h" +#include "display/intel_display_types.h" +#include "display/intel_dp.h" +#include "display/intel_fbdev.h" +#include "display/intel_hotplug.h" +#include "display/intel_overlay.h" +#include "display/intel_pipe_crc.h" +#include "display/intel_sprite.h" +#include "display/intel_vga.h" + +#include "gem/i915_gem_context.h" +#include "gem/i915_gem_ioctls.h" +#include "gt/intel_gt.h" +#include "gt/intel_gt_pm.h" +#include "gt/intel_rc6.h" + +#include "i915_debugfs.h" #include "i915_drv.h" -#include "i915_trace.h" -#include "i915_pmu.h" -#include "i915_reset.h" +#include "i915_irq.h" +#include "i915_memcpy.h" +#include "i915_perf.h" #include "i915_query.h" +#include "i915_suspend.h" +#include "i915_switcheroo.h" +#include "i915_sysfs.h" +#include "i915_trace.h" #include "i915_vgpu.h" -#include "intel_drv.h" -#include "intel_uc.h" -#include "intel_workarounds.h" +#include "intel_csr.h" +#include "intel_memory_region.h" +#include "intel_pm.h" static struct drm_driver driver; -#if IS_ENABLED(CONFIG_DRM_I915_DEBUG) -static unsigned int i915_load_fail_count; - -bool __i915_inject_load_failure(const char *func, int line) -{ - if (i915_load_fail_count >= i915_modparams.inject_load_failure) - return false; - - if (++i915_load_fail_count == i915_modparams.inject_load_failure) { - DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n", - i915_modparams.inject_load_failure, func, line); - i915_modparams.inject_load_failure = 0; - return true; - } - - return false; -} - -bool i915_error_injected(void) -{ - return i915_load_fail_count && !i915_modparams.inject_load_failure; -} - -#endif - -#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI" -#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \ - "providing the dmesg log by booting with drm.debug=0xf" - -void -__i915_printk(struct drm_i915_private *dev_priv, const char *level, - const char *fmt, ...) -{ - static bool shown_bug_once; - struct device *kdev = dev_priv->drm.dev; - bool is_error = level[1] <= KERN_ERR[1]; - bool is_debug = level[1] == KERN_DEBUG[1]; - struct va_format vaf; - va_list args; - - if (is_debug && !(drm_debug & DRM_UT_DRIVER)) - return; - - va_start(args, fmt); - - vaf.fmt = fmt; - vaf.va = &args; - - if (is_error) - dev_printk(level, kdev, "%pV", &vaf); - else - dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV", - __builtin_return_address(0), &vaf); - - va_end(args); - - if (is_error && !shown_bug_once) { - /* - * Ask the user to file a bug report for the error, except - * if they may have caused the bug by fiddling with unsafe - * module parameters. - */ - if (!test_taint(TAINT_USER)) - dev_notice(kdev, "%s", FDO_BUG_MSG); - shown_bug_once = true; - } -} - -/* Map PCH device id to PCH type, or PCH_NONE if unknown. */ -static enum intel_pch -intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) -{ - switch (id) { - case INTEL_PCH_IBX_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found Ibex Peak PCH\n"); - WARN_ON(!IS_GEN(dev_priv, 5)); - return PCH_IBX; - case INTEL_PCH_CPT_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found CougarPoint PCH\n"); - WARN_ON(!IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); - return PCH_CPT; - case INTEL_PCH_PPT_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found PantherPoint PCH\n"); - WARN_ON(!IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); - /* PantherPoint is CPT compatible */ - return PCH_CPT; - case INTEL_PCH_LPT_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found LynxPoint PCH\n"); - WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); - WARN_ON(IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)); - return PCH_LPT; - case INTEL_PCH_LPT_LP_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found LynxPoint LP PCH\n"); - WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); - WARN_ON(!IS_HSW_ULT(dev_priv) && !IS_BDW_ULT(dev_priv)); - return PCH_LPT; - case INTEL_PCH_WPT_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found WildcatPoint PCH\n"); - WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); - WARN_ON(IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)); - /* WildcatPoint is LPT compatible */ - return PCH_LPT; - case INTEL_PCH_WPT_LP_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found WildcatPoint LP PCH\n"); - WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); - WARN_ON(!IS_HSW_ULT(dev_priv) && !IS_BDW_ULT(dev_priv)); - /* WildcatPoint is LPT compatible */ - return PCH_LPT; - case INTEL_PCH_SPT_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found SunrisePoint PCH\n"); - WARN_ON(!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv)); - return PCH_SPT; - case INTEL_PCH_SPT_LP_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n"); - WARN_ON(!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv)); - return PCH_SPT; - case INTEL_PCH_KBP_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found Kaby Lake PCH (KBP)\n"); - WARN_ON(!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv) && - !IS_COFFEELAKE(dev_priv)); - return PCH_KBP; - case INTEL_PCH_CNP_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found Cannon Lake PCH (CNP)\n"); - WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv)); - return PCH_CNP; - case INTEL_PCH_CNP_LP_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found Cannon Lake LP PCH (CNP-LP)\n"); - WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv)); - return PCH_CNP; - case INTEL_PCH_ICP_DEVICE_ID_TYPE: - DRM_DEBUG_KMS("Found Ice Lake PCH\n"); - WARN_ON(!IS_ICELAKE(dev_priv)); - return PCH_ICP; - default: - return PCH_NONE; - } -} - -static bool intel_is_virt_pch(unsigned short id, - unsigned short svendor, unsigned short sdevice) -{ - return (id == INTEL_PCH_P2X_DEVICE_ID_TYPE || - id == INTEL_PCH_P3X_DEVICE_ID_TYPE || - (id == INTEL_PCH_QEMU_DEVICE_ID_TYPE && - svendor == PCI_SUBVENDOR_ID_REDHAT_QUMRANET && - sdevice == PCI_SUBDEVICE_ID_QEMU)); -} - -static unsigned short -intel_virt_detect_pch(const struct drm_i915_private *dev_priv) -{ - unsigned short id = 0; - - /* - * In a virtualized passthrough environment we can be in a - * setup where the ISA bridge is not able to be passed through. - * In this case, a south bridge can be emulated and we have to - * make an educated guess as to which PCH is really there. - */ - - if (IS_GEN(dev_priv, 5)) - id = INTEL_PCH_IBX_DEVICE_ID_TYPE; - else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) - id = INTEL_PCH_CPT_DEVICE_ID_TYPE; - else if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) - id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE; - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - id = INTEL_PCH_LPT_DEVICE_ID_TYPE; - else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) - id = INTEL_PCH_SPT_DEVICE_ID_TYPE; - else if (IS_COFFEELAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) - id = INTEL_PCH_CNP_DEVICE_ID_TYPE; - else if (IS_ICELAKE(dev_priv)) - id = INTEL_PCH_ICP_DEVICE_ID_TYPE; - - if (id) - DRM_DEBUG_KMS("Assuming PCH ID %04x\n", id); - else - DRM_DEBUG_KMS("Assuming no PCH\n"); - - return id; -} - -static void intel_detect_pch(struct drm_i915_private *dev_priv) -{ - struct pci_dev *pch = NULL; - - /* - * The reason to probe ISA bridge instead of Dev31:Fun0 is to - * make graphics device passthrough work easy for VMM, that only - * need to expose ISA bridge to let driver know the real hardware - * underneath. This is a requirement from virtualization team. - * - * In some virtualized environments (e.g. XEN), there is irrelevant - * ISA bridge in the system. To work reliably, we should scan trhough - * all the ISA bridge devices and check for the first match, instead - * of only checking the first one. - */ - while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) { - unsigned short id; - enum intel_pch pch_type; - - if (pch->vendor != PCI_VENDOR_ID_INTEL) - continue; - - id = pch->device & INTEL_PCH_DEVICE_ID_MASK; - - pch_type = intel_pch_type(dev_priv, id); - if (pch_type != PCH_NONE) { - dev_priv->pch_type = pch_type; - dev_priv->pch_id = id; - break; - } else if (intel_is_virt_pch(id, pch->subsystem_vendor, - pch->subsystem_device)) { - id = intel_virt_detect_pch(dev_priv); - pch_type = intel_pch_type(dev_priv, id); - - /* Sanity check virtual PCH id */ - if (WARN_ON(id && pch_type == PCH_NONE)) - id = 0; - - dev_priv->pch_type = pch_type; - dev_priv->pch_id = id; - break; - } - } - - /* - * Use PCH_NOP (PCH but no South Display) for PCH platforms without - * display. - */ - if (pch && !HAS_DISPLAY(dev_priv)) { - DRM_DEBUG_KMS("Display disabled, reverting to NOP PCH\n"); - dev_priv->pch_type = PCH_NOP; - dev_priv->pch_id = 0; - } - - if (!pch) - DRM_DEBUG_KMS("No PCH found.\n"); - - pci_dev_put(pch); -} - -static int i915_getparam_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - struct pci_dev *pdev = dev_priv->drm.pdev; - drm_i915_getparam_t *param = data; - int value; - - switch (param->param) { - case I915_PARAM_IRQ_ACTIVE: - case I915_PARAM_ALLOW_BATCHBUFFER: - case I915_PARAM_LAST_DISPATCH: - case I915_PARAM_HAS_EXEC_CONSTANTS: - /* Reject all old ums/dri params. */ - return -ENODEV; - case I915_PARAM_CHIPSET_ID: - value = pdev->device; - break; - case I915_PARAM_REVISION: - value = pdev->revision; - break; - case I915_PARAM_NUM_FENCES_AVAIL: - value = dev_priv->num_fence_regs; - break; - case I915_PARAM_HAS_OVERLAY: - value = dev_priv->overlay ? 1 : 0; - break; - case I915_PARAM_HAS_BSD: - value = !!dev_priv->engine[VCS]; - break; - case I915_PARAM_HAS_BLT: - value = !!dev_priv->engine[BCS]; - break; - case I915_PARAM_HAS_VEBOX: - value = !!dev_priv->engine[VECS]; - break; - case I915_PARAM_HAS_BSD2: - value = !!dev_priv->engine[VCS2]; - break; - case I915_PARAM_HAS_LLC: - value = HAS_LLC(dev_priv); - break; - case I915_PARAM_HAS_WT: - value = HAS_WT(dev_priv); - break; - case I915_PARAM_HAS_ALIASING_PPGTT: - value = min_t(int, INTEL_PPGTT(dev_priv), I915_GEM_PPGTT_FULL); - break; - case I915_PARAM_HAS_SEMAPHORES: - value = 0; - break; - case I915_PARAM_HAS_SECURE_BATCHES: - value = capable(CAP_SYS_ADMIN); - break; - case I915_PARAM_CMD_PARSER_VERSION: - value = i915_cmd_parser_get_version(dev_priv); - break; - case I915_PARAM_SUBSLICE_TOTAL: - value = sseu_subslice_total(&RUNTIME_INFO(dev_priv)->sseu); - if (!value) - return -ENODEV; - break; - case I915_PARAM_EU_TOTAL: - value = RUNTIME_INFO(dev_priv)->sseu.eu_total; - if (!value) - return -ENODEV; - break; - case I915_PARAM_HAS_GPU_RESET: - value = i915_modparams.enable_hangcheck && - intel_has_gpu_reset(dev_priv); - if (value && intel_has_reset_engine(dev_priv)) - value = 2; - break; - case I915_PARAM_HAS_RESOURCE_STREAMER: - value = 0; - break; - case I915_PARAM_HAS_POOLED_EU: - value = HAS_POOLED_EU(dev_priv); - break; - case I915_PARAM_MIN_EU_IN_POOL: - value = RUNTIME_INFO(dev_priv)->sseu.min_eu_in_pool; - break; - case I915_PARAM_HUC_STATUS: - value = intel_huc_check_status(&dev_priv->huc); - if (value < 0) - return value; - break; - case I915_PARAM_MMAP_GTT_VERSION: - /* Though we've started our numbering from 1, and so class all - * earlier versions as 0, in effect their value is undefined as - * the ioctl will report EINVAL for the unknown param! - */ - value = i915_gem_mmap_gtt_version(); - break; - case I915_PARAM_HAS_SCHEDULER: - value = dev_priv->caps.scheduler; - break; - - case I915_PARAM_MMAP_VERSION: - /* Remember to bump this if the version changes! */ - case I915_PARAM_HAS_GEM: - case I915_PARAM_HAS_PAGEFLIPPING: - case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */ - case I915_PARAM_HAS_RELAXED_FENCING: - case I915_PARAM_HAS_COHERENT_RINGS: - case I915_PARAM_HAS_RELAXED_DELTA: - case I915_PARAM_HAS_GEN7_SOL_RESET: - case I915_PARAM_HAS_WAIT_TIMEOUT: - case I915_PARAM_HAS_PRIME_VMAP_FLUSH: - case I915_PARAM_HAS_PINNED_BATCHES: - case I915_PARAM_HAS_EXEC_NO_RELOC: - case I915_PARAM_HAS_EXEC_HANDLE_LUT: - case I915_PARAM_HAS_COHERENT_PHYS_GTT: - case I915_PARAM_HAS_EXEC_SOFTPIN: - case I915_PARAM_HAS_EXEC_ASYNC: - case I915_PARAM_HAS_EXEC_FENCE: - case I915_PARAM_HAS_EXEC_CAPTURE: - case I915_PARAM_HAS_EXEC_BATCH_FIRST: - case I915_PARAM_HAS_EXEC_FENCE_ARRAY: - /* For the time being all of these are always true; - * if some supported hardware does not have one of these - * features this value needs to be provided from - * INTEL_INFO(), a feature macro, or similar. - */ - value = 1; - break; - case I915_PARAM_HAS_CONTEXT_ISOLATION: - value = intel_engines_has_context_isolation(dev_priv); - break; - case I915_PARAM_SLICE_MASK: - value = RUNTIME_INFO(dev_priv)->sseu.slice_mask; - if (!value) - return -ENODEV; - break; - case I915_PARAM_SUBSLICE_MASK: - value = RUNTIME_INFO(dev_priv)->sseu.subslice_mask[0]; - if (!value) - return -ENODEV; - break; - case I915_PARAM_CS_TIMESTAMP_FREQUENCY: - value = 1000 * RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz; - break; - case I915_PARAM_MMAP_GTT_COHERENT: - value = INTEL_INFO(dev_priv)->has_coherent_ggtt; - break; - default: - DRM_DEBUG("Unknown parameter %d\n", param->param); - return -EINVAL; - } - - if (put_user(value, param->value)) - return -EFAULT; - - return 0; -} +struct vlv_s0ix_state { + /* GAM */ + u32 wr_watermark; + u32 gfx_prio_ctrl; + u32 arb_mode; + u32 gfx_pend_tlb0; + u32 gfx_pend_tlb1; + u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; + u32 media_max_req_count; + u32 gfx_max_req_count; + u32 render_hwsp; + u32 ecochk; + u32 bsd_hwsp; + u32 blt_hwsp; + u32 tlb_rd_addr; + + /* MBC */ + u32 g3dctl; + u32 gsckgctl; + u32 mbctl; + + /* GCP */ + u32 ucgctl1; + u32 ucgctl3; + u32 rcgctl1; + u32 rcgctl2; + u32 rstctl; + u32 misccpctl; + + /* GPM */ + u32 gfxpause; + u32 rpdeuhwtc; + u32 rpdeuc; + u32 ecobus; + u32 pwrdwnupctl; + u32 rp_down_timeout; + u32 rp_deucsw; + u32 rcubmabdtmr; + u32 rcedata; + u32 spare2gh; + + /* Display 1 CZ domain */ + u32 gt_imr; + u32 gt_ier; + u32 pm_imr; + u32 pm_ier; + u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; + + /* GT SA CZ domain */ + u32 tilectl; + u32 gt_fifoctl; + u32 gtlc_wake_ctrl; + u32 gtlc_survive; + u32 pmwgicz; + + /* Display 2 CZ domain */ + u32 gu_ctl0; + u32 gu_ctl1; + u32 pcbr; + u32 clock_gate_dis2; +}; static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) { @@ -585,176 +271,97 @@ intel_teardown_mchbar(struct drm_i915_private *dev_priv) release_resource(&dev_priv->mch_res); } -/* true = enable decode, false = disable decoder */ -static unsigned int i915_vga_set_decode(void *cookie, bool state) -{ - struct drm_i915_private *dev_priv = cookie; - - intel_modeset_vga_set_state(dev_priv, state); - if (state) - return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | - VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; - else - return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; -} - -static int i915_resume_switcheroo(struct drm_device *dev); -static int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state); - -static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) -{ - struct drm_device *dev = pci_get_drvdata(pdev); - pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; - - if (state == VGA_SWITCHEROO_ON) { - pr_info("switched on\n"); - dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - /* i915 resume handler doesn't set to D0 */ - pci_set_power_state(pdev, PCI_D0); - i915_resume_switcheroo(dev); - dev->switch_power_state = DRM_SWITCH_POWER_ON; - } else { - pr_info("switched off\n"); - dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - i915_suspend_switcheroo(dev, pmm); - dev->switch_power_state = DRM_SWITCH_POWER_OFF; - } -} - -static bool i915_switcheroo_can_switch(struct pci_dev *pdev) +static int i915_driver_modeset_probe(struct drm_i915_private *i915) { - struct drm_device *dev = pci_get_drvdata(pdev); - - /* - * FIXME: open_count is protected by drm_global_mutex but that would lead to - * locking inversion with the driver load path. And the access here is - * completely racy anyway. So don't bother with locking for now. - */ - return dev->open_count == 0; -} - -static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { - .set_gpu_state = i915_switcheroo_set_state, - .reprobe = NULL, - .can_switch = i915_switcheroo_can_switch, -}; - -static int i915_load_modeset_init(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - struct pci_dev *pdev = dev_priv->drm.pdev; int ret; - if (i915_inject_load_failure()) + if (i915_inject_probe_failure(i915)) return -ENODEV; - if (HAS_DISPLAY(dev_priv)) { - ret = drm_vblank_init(&dev_priv->drm, - INTEL_INFO(dev_priv)->num_pipes); + if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) { + ret = drm_vblank_init(&i915->drm, + INTEL_NUM_PIPES(i915)); if (ret) goto out; } - intel_bios_init(dev_priv); + intel_bios_init(i915); - /* If we have > 1 VGA cards, then we need to arbitrate access - * to the common VGA resources. - * - * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA), - * then we do not take part in VGA arbitration and the - * vga_client_register() fails with -ENODEV. - */ - ret = vga_client_register(pdev, dev_priv, NULL, i915_vga_set_decode); - if (ret && ret != -ENODEV) + ret = intel_vga_register(i915); + if (ret) goto out; intel_register_dsm_handler(); - ret = vga_switcheroo_register_client(pdev, &i915_switcheroo_ops, false); + ret = i915_switcheroo_register(i915); if (ret) goto cleanup_vga_client; - /* must happen before intel_power_domains_init_hw() on VLV/CHV */ - intel_update_rawclk(dev_priv); - - intel_power_domains_init_hw(dev_priv, false); + intel_power_domains_init_hw(i915, false); - intel_csr_ucode_init(dev_priv); + intel_csr_ucode_init(i915); - ret = intel_irq_install(dev_priv); + ret = intel_irq_install(i915); if (ret) goto cleanup_csr; - intel_setup_gmbus(dev_priv); - /* Important: The output setup functions called by modeset_init need * working irqs for e.g. gmbus and dp aux transfers. */ - ret = intel_modeset_init(dev); + ret = intel_modeset_init(i915); if (ret) goto cleanup_irq; - ret = i915_gem_init(dev_priv); + ret = i915_gem_init(i915); if (ret) goto cleanup_modeset; - intel_overlay_setup(dev_priv); + intel_overlay_setup(i915); - if (!HAS_DISPLAY(dev_priv)) + if (!HAS_DISPLAY(i915) || !INTEL_DISPLAY_ENABLED(i915)) return 0; - ret = intel_fbdev_init(dev); + ret = intel_fbdev_init(&i915->drm); if (ret) goto cleanup_gem; /* Only enable hotplug handling once the fbdev is fully set up. */ - intel_hpd_init(dev_priv); + intel_hpd_init(i915); - intel_init_ipc(dev_priv); + intel_init_ipc(i915); return 0; cleanup_gem: - if (i915_gem_suspend(dev_priv)) - DRM_ERROR("failed to idle hardware; continuing to unload!\n"); - i915_gem_fini(dev_priv); + i915_gem_suspend(i915); + i915_gem_driver_remove(i915); + i915_gem_driver_release(i915); cleanup_modeset: - intel_modeset_cleanup(dev); + intel_modeset_driver_remove(i915); cleanup_irq: - drm_irq_uninstall(dev); - intel_teardown_gmbus(dev_priv); + intel_irq_uninstall(i915); cleanup_csr: - intel_csr_ucode_fini(dev_priv); - intel_power_domains_fini_hw(dev_priv); - vga_switcheroo_unregister_client(pdev); + intel_csr_ucode_fini(i915); + intel_power_domains_driver_remove(i915); + i915_switcheroo_unregister(i915); cleanup_vga_client: - vga_client_register(pdev, NULL, NULL, NULL); + intel_vga_unregister(i915); out: return ret; } -static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) +static void i915_driver_modeset_remove(struct drm_i915_private *i915) { - struct apertures_struct *ap; - struct pci_dev *pdev = dev_priv->drm.pdev; - struct i915_ggtt *ggtt = &dev_priv->ggtt; - bool primary; - int ret; + intel_modeset_driver_remove(i915); - ap = alloc_apertures(1); - if (!ap) - return -ENOMEM; - - ap->ranges[0].base = ggtt->gmadr.start; - ap->ranges[0].size = ggtt->mappable_end; + intel_irq_uninstall(i915); - primary = - pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW; + intel_bios_driver_remove(i915); - ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary); + i915_switcheroo_unregister(i915); - kfree(ap); + intel_vga_unregister(i915); - return ret; + intel_csr_ucode_fini(i915); } static void intel_init_dpio(struct drm_i915_private *dev_priv) @@ -806,15 +413,6 @@ out_err: return -ENOMEM; } -static void i915_engines_cleanup(struct drm_i915_private *i915) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, i915, id) - kfree(engine); -} - static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) { destroy_workqueue(dev_priv->hotplug.dp_wq); @@ -847,8 +445,31 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) } } +static int vlv_alloc_s0ix_state(struct drm_i915_private *i915) +{ + if (!IS_VALLEYVIEW(i915)) + return 0; + + /* we write all the values in the struct, so no need to zero it out */ + i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state), + GFP_KERNEL); + if (!i915->vlv_s0ix_state) + return -ENOMEM; + + return 0; +} + +static void vlv_free_s0ix_state(struct drm_i915_private *i915) +{ + if (!i915->vlv_s0ix_state) + return; + + kfree(i915->vlv_s0ix_state); + i915->vlv_s0ix_state = NULL; +} + /** - * i915_driver_init_early - setup state not requiring device access + * i915_driver_early_probe - setup state not requiring device access * @dev_priv: device private * * Initialize everything that is a "SW-only" state, that is state not @@ -857,46 +478,57 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) * system memory allocation, setting up device specific attributes and * function hooks not requiring accessing the device. */ -static int i915_driver_init_early(struct drm_i915_private *dev_priv) +static int i915_driver_early_probe(struct drm_i915_private *dev_priv) { int ret = 0; - if (i915_inject_load_failure()) + if (i915_inject_probe_failure(dev_priv)) return -ENODEV; + intel_device_info_subplatform_init(dev_priv); + + intel_uncore_mmio_debug_init_early(&dev_priv->mmio_debug); + intel_uncore_init_early(&dev_priv->uncore, dev_priv); + spin_lock_init(&dev_priv->irq_lock); spin_lock_init(&dev_priv->gpu_error.lock); mutex_init(&dev_priv->backlight_lock); - spin_lock_init(&dev_priv->uncore.lock); mutex_init(&dev_priv->sb_lock); + pm_qos_add_request(&dev_priv->sb_qos, + PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); + mutex_init(&dev_priv->av_mutex); mutex_init(&dev_priv->wm.wm_mutex); mutex_init(&dev_priv->pps_mutex); + mutex_init(&dev_priv->hdcp_comp_mutex); i915_memcpy_init_early(dev_priv); - intel_runtime_pm_init_early(dev_priv); + intel_runtime_pm_init_early(&dev_priv->runtime_pm); ret = i915_workqueues_init(dev_priv); if (ret < 0) - goto err_engines; + return ret; - ret = i915_gem_init_early(dev_priv); + ret = vlv_alloc_s0ix_state(dev_priv); if (ret < 0) goto err_workqueues; + intel_wopcm_init_early(&dev_priv->wopcm); + + intel_gt_init_early(&dev_priv->gt, dev_priv); + + i915_gem_init_early(dev_priv); + /* This must be called before any calls to HAS_PCH_* */ intel_detect_pch(dev_priv); - intel_wopcm_init_early(&dev_priv->wopcm); - intel_uc_init_early(dev_priv); intel_pm_setup(dev_priv); intel_init_dpio(dev_priv); ret = intel_power_domains_init(dev_priv); if (ret < 0) - goto err_uc; + goto err_gem; intel_irq_init(dev_priv); - intel_hangcheck_init(dev_priv); intel_init_display_hooks(dev_priv); intel_init_clock_gating_hooks(dev_priv); intel_init_audio_hooks(dev_priv); @@ -906,72 +538,35 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv) return 0; -err_uc: - intel_uc_cleanup_early(dev_priv); +err_gem: i915_gem_cleanup_early(dev_priv); + intel_gt_driver_late_release(&dev_priv->gt); + vlv_free_s0ix_state(dev_priv); err_workqueues: i915_workqueues_cleanup(dev_priv); -err_engines: - i915_engines_cleanup(dev_priv); return ret; } /** - * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early() + * i915_driver_late_release - cleanup the setup done in + * i915_driver_early_probe() * @dev_priv: device private */ -static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv) +static void i915_driver_late_release(struct drm_i915_private *dev_priv) { intel_irq_fini(dev_priv); intel_power_domains_cleanup(dev_priv); - intel_uc_cleanup_early(dev_priv); i915_gem_cleanup_early(dev_priv); + intel_gt_driver_late_release(&dev_priv->gt); + vlv_free_s0ix_state(dev_priv); i915_workqueues_cleanup(dev_priv); - i915_engines_cleanup(dev_priv); -} - -static int i915_mmio_setup(struct drm_i915_private *dev_priv) -{ - struct pci_dev *pdev = dev_priv->drm.pdev; - int mmio_bar; - int mmio_size; - mmio_bar = IS_GEN(dev_priv, 2) ? 1 : 0; - /* - * Before gen4, the registers and the GTT are behind different BARs. - * However, from gen4 onwards, the registers and the GTT are shared - * in the same BAR, so we want to restrict this ioremap from - * clobbering the GTT which we want ioremap_wc instead. Fortunately, - * the register BAR remains the same size for all the earlier - * generations up to Ironlake. - */ - if (INTEL_GEN(dev_priv) < 5) - mmio_size = 512 * 1024; - else - mmio_size = 2 * 1024 * 1024; - dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size); - if (dev_priv->regs == NULL) { - DRM_ERROR("failed to map registers\n"); - - return -EIO; - } - - /* Try to make sure MCHBAR is enabled before poking at it */ - intel_setup_mchbar(dev_priv); - - return 0; -} - -static void i915_mmio_cleanup(struct drm_i915_private *dev_priv) -{ - struct pci_dev *pdev = dev_priv->drm.pdev; - - intel_teardown_mchbar(dev_priv); - pci_iounmap(pdev, dev_priv->regs); + pm_qos_remove_request(&dev_priv->sb_qos); + mutex_destroy(&dev_priv->sb_lock); } /** - * i915_driver_init_mmio - setup device MMIO + * i915_driver_mmio_probe - setup device MMIO * @dev_priv: device private * * Setup minimal device state necessary for MMIO accesses later in the @@ -979,39 +574,38 @@ static void i915_mmio_cleanup(struct drm_i915_private *dev_priv) * side effects or exposing the driver via kernel internal or user space * interfaces. */ -static int i915_driver_init_mmio(struct drm_i915_private *dev_priv) +static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv) { int ret; - if (i915_inject_load_failure()) + if (i915_inject_probe_failure(dev_priv)) return -ENODEV; if (i915_get_bridge_dev(dev_priv)) return -EIO; - ret = i915_mmio_setup(dev_priv); + ret = intel_uncore_init_mmio(&dev_priv->uncore); if (ret < 0) goto err_bridge; - intel_uncore_init(dev_priv); + /* Try to make sure MCHBAR is enabled before poking at it */ + intel_setup_mchbar(dev_priv); intel_device_info_init_mmio(dev_priv); - intel_uncore_prune(dev_priv); + intel_uncore_prune_mmio_domains(&dev_priv->uncore); - intel_uc_init_mmio(dev_priv); + intel_uc_init_mmio(&dev_priv->gt.uc); - ret = intel_engines_init_mmio(dev_priv); + ret = intel_engines_init_mmio(&dev_priv->gt); if (ret) goto err_uncore; - i915_gem_init_mmio(dev_priv); - return 0; err_uncore: - intel_uncore_fini(dev_priv); - i915_mmio_cleanup(dev_priv); + intel_teardown_mchbar(dev_priv); + intel_uncore_fini_mmio(&dev_priv->uncore); err_bridge: pci_dev_put(dev_priv->bridge_dev); @@ -1019,13 +613,14 @@ err_bridge: } /** - * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio() + * i915_driver_mmio_release - cleanup the setup done in i915_driver_mmio_probe() * @dev_priv: device private */ -static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv) +static void i915_driver_mmio_release(struct drm_i915_private *dev_priv) { - intel_uncore_fini(dev_priv); - i915_mmio_cleanup(dev_priv); + intel_engines_cleanup(&dev_priv->gt); + intel_teardown_mchbar(dev_priv); + intel_uncore_fini_mmio(&dev_priv->uncore); pci_dev_put(dev_priv->bridge_dev); } @@ -1034,110 +629,180 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv) intel_gvt_sanitize_options(dev_priv); } -static enum dram_rank skl_get_dimm_rank(u8 size, u32 rank) +#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type + +static const char *intel_dram_type_str(enum intel_dram_type type) { - if (size == 0) - return I915_DRAM_RANK_INVALID; - if (rank == SKL_DRAM_RANK_SINGLE) - return I915_DRAM_RANK_SINGLE; - else if (rank == SKL_DRAM_RANK_DUAL) - return I915_DRAM_RANK_DUAL; + static const char * const str[] = { + DRAM_TYPE_STR(UNKNOWN), + DRAM_TYPE_STR(DDR3), + DRAM_TYPE_STR(DDR4), + DRAM_TYPE_STR(LPDDR3), + DRAM_TYPE_STR(LPDDR4), + }; - return I915_DRAM_RANK_INVALID; + if (type >= ARRAY_SIZE(str)) + type = INTEL_DRAM_UNKNOWN; + + return str[type]; } -static bool -skl_is_16gb_dimm(enum dram_rank rank, u8 size, u8 width) +#undef DRAM_TYPE_STR + +static int intel_dimm_num_devices(const struct dram_dimm_info *dimm) { - if (rank == I915_DRAM_RANK_SINGLE && width == 8 && size == 16) - return true; - else if (rank == I915_DRAM_RANK_DUAL && width == 8 && size == 32) - return true; - else if (rank == SKL_DRAM_RANK_SINGLE && width == 16 && size == 8) - return true; - else if (rank == SKL_DRAM_RANK_DUAL && width == 16 && size == 16) - return true; + return dimm->ranks * 64 / (dimm->width ?: 1); +} - return false; +/* Returns total GB for the whole DIMM */ +static int skl_get_dimm_size(u16 val) +{ + return val & SKL_DRAM_SIZE_MASK; } -static int -skl_dram_get_channel_info(struct dram_channel_info *ch, u32 val) +static int skl_get_dimm_width(u16 val) { - u32 tmp_l, tmp_s; - u32 s_val = val >> SKL_DRAM_S_SHIFT; + if (skl_get_dimm_size(val) == 0) + return 0; - if (!val) - return -EINVAL; + switch (val & SKL_DRAM_WIDTH_MASK) { + case SKL_DRAM_WIDTH_X8: + case SKL_DRAM_WIDTH_X16: + case SKL_DRAM_WIDTH_X32: + val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT; + return 8 << val; + default: + MISSING_CASE(val); + return 0; + } +} + +static int skl_get_dimm_ranks(u16 val) +{ + if (skl_get_dimm_size(val) == 0) + return 0; + + val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT; + + return val + 1; +} + +/* Returns total GB for the whole DIMM */ +static int cnl_get_dimm_size(u16 val) +{ + return (val & CNL_DRAM_SIZE_MASK) / 2; +} - tmp_l = val & SKL_DRAM_SIZE_MASK; - tmp_s = s_val & SKL_DRAM_SIZE_MASK; +static int cnl_get_dimm_width(u16 val) +{ + if (cnl_get_dimm_size(val) == 0) + return 0; + + switch (val & CNL_DRAM_WIDTH_MASK) { + case CNL_DRAM_WIDTH_X8: + case CNL_DRAM_WIDTH_X16: + case CNL_DRAM_WIDTH_X32: + val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT; + return 8 << val; + default: + MISSING_CASE(val); + return 0; + } +} + +static int cnl_get_dimm_ranks(u16 val) +{ + if (cnl_get_dimm_size(val) == 0) + return 0; + + val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT; + + return val + 1; +} - if (tmp_l == 0 && tmp_s == 0) +static bool +skl_is_16gb_dimm(const struct dram_dimm_info *dimm) +{ + /* Convert total GB to Gb per DRAM device */ + return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16; +} + +static void +skl_dram_get_dimm_info(struct drm_i915_private *dev_priv, + struct dram_dimm_info *dimm, + int channel, char dimm_name, u16 val) +{ + if (INTEL_GEN(dev_priv) >= 10) { + dimm->size = cnl_get_dimm_size(val); + dimm->width = cnl_get_dimm_width(val); + dimm->ranks = cnl_get_dimm_ranks(val); + } else { + dimm->size = skl_get_dimm_size(val); + dimm->width = skl_get_dimm_width(val); + dimm->ranks = skl_get_dimm_ranks(val); + } + + DRM_DEBUG_KMS("CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n", + channel, dimm_name, dimm->size, dimm->width, dimm->ranks, + yesno(skl_is_16gb_dimm(dimm))); +} + +static int +skl_dram_get_channel_info(struct drm_i915_private *dev_priv, + struct dram_channel_info *ch, + int channel, u32 val) +{ + skl_dram_get_dimm_info(dev_priv, &ch->dimm_l, + channel, 'L', val & 0xffff); + skl_dram_get_dimm_info(dev_priv, &ch->dimm_s, + channel, 'S', val >> 16); + + if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) { + DRM_DEBUG_KMS("CH%u not populated\n", channel); return -EINVAL; + } - ch->l_info.size = tmp_l; - ch->s_info.size = tmp_s; - - tmp_l = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT; - tmp_s = (s_val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT; - ch->l_info.width = (1 << tmp_l) * 8; - ch->s_info.width = (1 << tmp_s) * 8; - - tmp_l = val & SKL_DRAM_RANK_MASK; - tmp_s = s_val & SKL_DRAM_RANK_MASK; - ch->l_info.rank = skl_get_dimm_rank(ch->l_info.size, tmp_l); - ch->s_info.rank = skl_get_dimm_rank(ch->s_info.size, tmp_s); - - if (ch->l_info.rank == I915_DRAM_RANK_DUAL || - ch->s_info.rank == I915_DRAM_RANK_DUAL) - ch->rank = I915_DRAM_RANK_DUAL; - else if (ch->l_info.rank == I915_DRAM_RANK_SINGLE && - ch->s_info.rank == I915_DRAM_RANK_SINGLE) - ch->rank = I915_DRAM_RANK_DUAL; + if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2) + ch->ranks = 2; + else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1) + ch->ranks = 2; else - ch->rank = I915_DRAM_RANK_SINGLE; + ch->ranks = 1; - ch->is_16gb_dimm = skl_is_16gb_dimm(ch->l_info.rank, ch->l_info.size, - ch->l_info.width) || - skl_is_16gb_dimm(ch->s_info.rank, ch->s_info.size, - ch->s_info.width); + ch->is_16gb_dimm = + skl_is_16gb_dimm(&ch->dimm_l) || + skl_is_16gb_dimm(&ch->dimm_s); - DRM_DEBUG_KMS("(size:width:rank) L(%dGB:X%d:%s) S(%dGB:X%d:%s)\n", - ch->l_info.size, ch->l_info.width, - ch->l_info.rank ? "dual" : "single", - ch->s_info.size, ch->s_info.width, - ch->s_info.rank ? "dual" : "single"); + DRM_DEBUG_KMS("CH%u ranks: %u, 16Gb DIMMs: %s\n", + channel, ch->ranks, yesno(ch->is_16gb_dimm)); return 0; } static bool -intel_is_dram_symmetric(u32 val_ch0, u32 val_ch1, - struct dram_channel_info *ch0) +intel_is_dram_symmetric(const struct dram_channel_info *ch0, + const struct dram_channel_info *ch1) { - return (val_ch0 == val_ch1 && - (ch0->s_info.size == 0 || - (ch0->l_info.size == ch0->s_info.size && - ch0->l_info.width == ch0->s_info.width && - ch0->l_info.rank == ch0->s_info.rank))); + return !memcmp(ch0, ch1, sizeof(*ch0)) && + (ch0->dimm_s.size == 0 || + !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l))); } static int skl_dram_get_channels_info(struct drm_i915_private *dev_priv) { struct dram_info *dram_info = &dev_priv->dram_info; - struct dram_channel_info ch0, ch1; - u32 val_ch0, val_ch1; + struct dram_channel_info ch0 = {}, ch1 = {}; + u32 val; int ret; - val_ch0 = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN); - ret = skl_dram_get_channel_info(&ch0, val_ch0); + val = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN); + ret = skl_dram_get_channel_info(dev_priv, &ch0, 0, val); if (ret == 0) dram_info->num_channels++; - val_ch1 = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN); - ret = skl_dram_get_channel_info(&ch1, val_ch1); + val = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN); + ret = skl_dram_get_channel_info(dev_priv, &ch1, 1, val); if (ret == 0) dram_info->num_channels++; @@ -1151,28 +816,47 @@ skl_dram_get_channels_info(struct drm_i915_private *dev_priv) * will be same as if single rank memory, so consider single rank * memory. */ - if (ch0.rank == I915_DRAM_RANK_SINGLE || - ch1.rank == I915_DRAM_RANK_SINGLE) - dram_info->rank = I915_DRAM_RANK_SINGLE; + if (ch0.ranks == 1 || ch1.ranks == 1) + dram_info->ranks = 1; else - dram_info->rank = max(ch0.rank, ch1.rank); + dram_info->ranks = max(ch0.ranks, ch1.ranks); - if (dram_info->rank == I915_DRAM_RANK_INVALID) { + if (dram_info->ranks == 0) { DRM_INFO("couldn't get memory rank information\n"); return -EINVAL; } dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm; - dev_priv->dram_info.symmetric_memory = intel_is_dram_symmetric(val_ch0, - val_ch1, - &ch0); + dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1); - DRM_DEBUG_KMS("memory configuration is %sSymmetric memory\n", - dev_priv->dram_info.symmetric_memory ? "" : "not "); + DRM_DEBUG_KMS("Memory configuration is symmetric? %s\n", + yesno(dram_info->symmetric_memory)); return 0; } +static enum intel_dram_type +skl_get_dram_type(struct drm_i915_private *dev_priv) +{ + u32 val; + + val = I915_READ(SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN); + + switch (val & SKL_DRAM_DDR_TYPE_MASK) { + case SKL_DRAM_DDR_TYPE_DDR3: + return INTEL_DRAM_DDR3; + case SKL_DRAM_DDR_TYPE_DDR4: + return INTEL_DRAM_DDR4; + case SKL_DRAM_DDR_TYPE_LPDDR3: + return INTEL_DRAM_LPDDR3; + case SKL_DRAM_DDR_TYPE_LPDDR4: + return INTEL_DRAM_LPDDR4; + default: + MISSING_CASE(val); + return INTEL_DRAM_UNKNOWN; + } +} + static int skl_get_dram_info(struct drm_i915_private *dev_priv) { @@ -1180,6 +864,9 @@ skl_get_dram_info(struct drm_i915_private *dev_priv) u32 mem_freq_khz, val; int ret; + dram_info->type = skl_get_dram_type(dev_priv); + DRM_DEBUG_KMS("DRAM type: %s\n", intel_dram_type_str(dram_info->type)); + ret = skl_dram_get_channels_info(dev_priv); if (ret) return ret; @@ -1200,6 +887,85 @@ skl_get_dram_info(struct drm_i915_private *dev_priv) return 0; } +/* Returns Gb per DRAM device */ +static int bxt_get_dimm_size(u32 val) +{ + switch (val & BXT_DRAM_SIZE_MASK) { + case BXT_DRAM_SIZE_4GBIT: + return 4; + case BXT_DRAM_SIZE_6GBIT: + return 6; + case BXT_DRAM_SIZE_8GBIT: + return 8; + case BXT_DRAM_SIZE_12GBIT: + return 12; + case BXT_DRAM_SIZE_16GBIT: + return 16; + default: + MISSING_CASE(val); + return 0; + } +} + +static int bxt_get_dimm_width(u32 val) +{ + if (!bxt_get_dimm_size(val)) + return 0; + + val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT; + + return 8 << val; +} + +static int bxt_get_dimm_ranks(u32 val) +{ + if (!bxt_get_dimm_size(val)) + return 0; + + switch (val & BXT_DRAM_RANK_MASK) { + case BXT_DRAM_RANK_SINGLE: + return 1; + case BXT_DRAM_RANK_DUAL: + return 2; + default: + MISSING_CASE(val); + return 0; + } +} + +static enum intel_dram_type bxt_get_dimm_type(u32 val) +{ + if (!bxt_get_dimm_size(val)) + return INTEL_DRAM_UNKNOWN; + + switch (val & BXT_DRAM_TYPE_MASK) { + case BXT_DRAM_TYPE_DDR3: + return INTEL_DRAM_DDR3; + case BXT_DRAM_TYPE_LPDDR3: + return INTEL_DRAM_LPDDR3; + case BXT_DRAM_TYPE_DDR4: + return INTEL_DRAM_DDR4; + case BXT_DRAM_TYPE_LPDDR4: + return INTEL_DRAM_LPDDR4; + default: + MISSING_CASE(val); + return INTEL_DRAM_UNKNOWN; + } +} + +static void bxt_get_dimm_info(struct dram_dimm_info *dimm, + u32 val) +{ + dimm->width = bxt_get_dimm_width(val); + dimm->ranks = bxt_get_dimm_ranks(val); + + /* + * Size in register is Gb per DRAM device. Convert to total + * GB to match the way we report this for non-LP platforms. + */ + dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8; +} + static int bxt_get_dram_info(struct drm_i915_private *dev_priv) { @@ -1228,57 +994,44 @@ bxt_get_dram_info(struct drm_i915_private *dev_priv) * Now read each DUNIT8/9/10/11 to check the rank of each dimms. */ for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) { - u8 size, width; - enum dram_rank rank; - u32 tmp; + struct dram_dimm_info dimm; + enum intel_dram_type type; val = I915_READ(BXT_D_CR_DRP0_DUNIT(i)); if (val == 0xFFFFFFFF) continue; dram_info->num_channels++; - tmp = val & BXT_DRAM_RANK_MASK; - - if (tmp == BXT_DRAM_RANK_SINGLE) - rank = I915_DRAM_RANK_SINGLE; - else if (tmp == BXT_DRAM_RANK_DUAL) - rank = I915_DRAM_RANK_DUAL; - else - rank = I915_DRAM_RANK_INVALID; - - tmp = val & BXT_DRAM_SIZE_MASK; - if (tmp == BXT_DRAM_SIZE_4GB) - size = 4; - else if (tmp == BXT_DRAM_SIZE_6GB) - size = 6; - else if (tmp == BXT_DRAM_SIZE_8GB) - size = 8; - else if (tmp == BXT_DRAM_SIZE_12GB) - size = 12; - else if (tmp == BXT_DRAM_SIZE_16GB) - size = 16; - else - size = 0; - - tmp = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT; - width = (1 << tmp) * 8; - DRM_DEBUG_KMS("dram size:%dGB width:X%d rank:%s\n", size, - width, rank == I915_DRAM_RANK_SINGLE ? "single" : - rank == I915_DRAM_RANK_DUAL ? "dual" : "unknown"); + + bxt_get_dimm_info(&dimm, val); + type = bxt_get_dimm_type(val); + + WARN_ON(type != INTEL_DRAM_UNKNOWN && + dram_info->type != INTEL_DRAM_UNKNOWN && + dram_info->type != type); + + DRM_DEBUG_KMS("CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n", + i - BXT_D_CR_DRP0_DUNIT_START, + dimm.size, dimm.width, dimm.ranks, + intel_dram_type_str(type)); /* * If any of the channel is single rank channel, * worst case output will be same as if single rank * memory, so consider single rank memory. */ - if (dram_info->rank == I915_DRAM_RANK_INVALID) - dram_info->rank = rank; - else if (rank == I915_DRAM_RANK_SINGLE) - dram_info->rank = I915_DRAM_RANK_SINGLE; + if (dram_info->ranks == 0) + dram_info->ranks = dimm.ranks; + else if (dimm.ranks == 1) + dram_info->ranks = 1; + + if (type != INTEL_DRAM_UNKNOWN) + dram_info->type = type; } - if (dram_info->rank == I915_DRAM_RANK_INVALID) { - DRM_INFO("couldn't get memory rank information\n"); + if (dram_info->type == INTEL_DRAM_UNKNOWN || + dram_info->ranks == 0) { + DRM_INFO("couldn't get memory information\n"); return -EINVAL; } @@ -1290,14 +1043,8 @@ static void intel_get_dram_info(struct drm_i915_private *dev_priv) { struct dram_info *dram_info = &dev_priv->dram_info; - char bandwidth_str[32]; int ret; - dram_info->valid = false; - dram_info->rank = I915_DRAM_RANK_INVALID; - dram_info->bandwidth_kbps = 0; - dram_info->num_channels = 0; - /* * Assume 16Gb DIMMs are present until proven otherwise. * This is only used for the level 0 watermark latency @@ -1305,50 +1052,84 @@ intel_get_dram_info(struct drm_i915_private *dev_priv) */ dram_info->is_16gb_dimm = !IS_GEN9_LP(dev_priv); - if (INTEL_GEN(dev_priv) < 9 || IS_GEMINILAKE(dev_priv)) + if (INTEL_GEN(dev_priv) < 9) return; - /* Need to calculate bandwidth only for Gen9 */ - if (IS_BROXTON(dev_priv)) + if (IS_GEN9_LP(dev_priv)) ret = bxt_get_dram_info(dev_priv); - else if (IS_GEN(dev_priv, 9)) - ret = skl_get_dram_info(dev_priv); else - ret = skl_dram_get_channels_info(dev_priv); + ret = skl_get_dram_info(dev_priv); if (ret) return; - if (dram_info->bandwidth_kbps) - sprintf(bandwidth_str, "%d KBps", dram_info->bandwidth_kbps); + DRM_DEBUG_KMS("DRAM bandwidth: %u kBps, channels: %u\n", + dram_info->bandwidth_kbps, + dram_info->num_channels); + + DRM_DEBUG_KMS("DRAM ranks: %u, 16Gb DIMMs: %s\n", + dram_info->ranks, yesno(dram_info->is_16gb_dimm)); +} + +static u32 gen9_edram_size_mb(struct drm_i915_private *dev_priv, u32 cap) +{ + static const u8 ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 }; + static const u8 sets[4] = { 1, 1, 2, 2 }; + + return EDRAM_NUM_BANKS(cap) * + ways[EDRAM_WAYS_IDX(cap)] * + sets[EDRAM_SETS_IDX(cap)]; +} + +static void edram_detect(struct drm_i915_private *dev_priv) +{ + u32 edram_cap = 0; + + if (!(IS_HASWELL(dev_priv) || + IS_BROADWELL(dev_priv) || + INTEL_GEN(dev_priv) >= 9)) + return; + + edram_cap = __raw_uncore_read32(&dev_priv->uncore, HSW_EDRAM_CAP); + + /* NB: We can't write IDICR yet because we don't have gt funcs set up */ + + if (!(edram_cap & EDRAM_ENABLED)) + return; + + /* + * The needed capability bits for size calculation are not there with + * pre gen9 so return 128MB always. + */ + if (INTEL_GEN(dev_priv) < 9) + dev_priv->edram_size_mb = 128; else - sprintf(bandwidth_str, "unknown"); - DRM_DEBUG_KMS("DRAM bandwidth:%s, total-channels: %u\n", - bandwidth_str, dram_info->num_channels); - DRM_DEBUG_KMS("DRAM rank: %s rank 16GB-dimm:%s\n", - (dram_info->rank == I915_DRAM_RANK_DUAL) ? - "dual" : "single", yesno(dram_info->is_16gb_dimm)); + dev_priv->edram_size_mb = + gen9_edram_size_mb(dev_priv, edram_cap); + + dev_info(dev_priv->drm.dev, + "Found %uMB of eDRAM\n", dev_priv->edram_size_mb); } /** - * i915_driver_init_hw - setup state requiring device access + * i915_driver_hw_probe - setup state requiring device access * @dev_priv: device private * * Setup state that requires accessing the device, but doesn't require * exposing the driver via kernel internal or userspace interfaces. */ -static int i915_driver_init_hw(struct drm_i915_private *dev_priv) +static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) { struct pci_dev *pdev = dev_priv->drm.pdev; int ret; - if (i915_inject_load_failure()) + if (i915_inject_probe_failure(dev_priv)) return -ENODEV; intel_device_info_runtime_init(dev_priv); if (HAS_PPGTT(dev_priv)) { if (intel_vgpu_active(dev_priv) && - !intel_vgpu_has_full_48bit_ppgtt(dev_priv)) { + !intel_vgpu_has_full_ppgtt(dev_priv)) { i915_report_error(dev_priv, "incompatible vGPU found, support for isolated ppGTT required\n"); return -ENXIO; @@ -1371,47 +1152,50 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) intel_sanitize_options(dev_priv); + /* needs to be done before ggtt probe */ + edram_detect(dev_priv); + i915_perf_init(dev_priv); ret = i915_ggtt_probe_hw(dev_priv); if (ret) goto err_perf; - /* - * WARNING: Apparently we must kick fbdev drivers before vgacon, - * otherwise the vga fbdev driver falls over. - */ - ret = i915_kick_out_firmware_fb(dev_priv); - if (ret) { - DRM_ERROR("failed to remove conflicting framebuffer drivers\n"); + ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, "inteldrmfb"); + if (ret) goto err_ggtt; - } - ret = vga_remove_vgacon(pdev); - if (ret) { - DRM_ERROR("failed to remove conflicting VGA console\n"); + ret = i915_ggtt_init_hw(dev_priv); + if (ret) goto err_ggtt; - } - ret = i915_ggtt_init_hw(dev_priv); + ret = intel_memory_regions_hw_probe(dev_priv); if (ret) goto err_ggtt; + intel_gt_init_hw_early(&dev_priv->gt, &dev_priv->ggtt); + ret = i915_ggtt_enable_hw(dev_priv); if (ret) { DRM_ERROR("failed to enable GGTT\n"); - goto err_ggtt; + goto err_mem_regions; } pci_set_master(pdev); + /* + * We don't have a max segment size, so set it to the max so sg's + * debugging layer doesn't complain + */ + dma_set_max_seg_size(&pdev->dev, UINT_MAX); + /* overlay on gen2 is broken and can't address above 1G */ if (IS_GEN(dev_priv, 2)) { ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30)); if (ret) { DRM_ERROR("failed to set DMA mask\n"); - goto err_ggtt; + goto err_mem_regions; } } @@ -1429,17 +1213,14 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) if (ret) { DRM_ERROR("failed to set DMA mask\n"); - goto err_ggtt; + goto err_mem_regions; } } pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); - intel_uncore_sanitize(dev_priv); - intel_gt_init_workarounds(dev_priv); - i915_gem_load_init_fences(dev_priv); /* On the 945G/GM, the chipset reports the MSI capability on the * integrated graphics even though the support isn't actually there @@ -1476,6 +1257,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) */ intel_get_dram_info(dev_priv); + intel_bw_init_hw(dev_priv); return 0; @@ -1483,18 +1265,20 @@ err_msi: if (pdev->msi_enabled) pci_disable_msi(pdev); pm_qos_remove_request(&dev_priv->pm_qos); +err_mem_regions: + intel_memory_regions_driver_release(dev_priv); err_ggtt: - i915_ggtt_cleanup_hw(dev_priv); + i915_ggtt_driver_release(dev_priv); err_perf: i915_perf_fini(dev_priv); return ret; } /** - * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw() + * i915_driver_hw_remove - cleanup the setup done in i915_driver_hw_probe() * @dev_priv: device private */ -static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv) +static void i915_driver_hw_remove(struct drm_i915_private *dev_priv) { struct pci_dev *pdev = dev_priv->drm.pdev; @@ -1504,7 +1288,6 @@ static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv) pci_disable_msi(pdev); pm_qos_remove_request(&dev_priv->pm_qos); - i915_ggtt_cleanup_hw(dev_priv); } /** @@ -1518,7 +1301,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) { struct drm_device *dev = &dev_priv->drm; - i915_gem_shrinker_register(dev_priv); + i915_gem_driver_register(dev_priv); i915_pmu_register(dev_priv); /* @@ -1538,14 +1321,13 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) } else DRM_ERROR("Failed to register driver for userspace access!\n"); - if (HAS_DISPLAY(dev_priv)) { + if (HAS_DISPLAY(dev_priv) && INTEL_DISPLAY_ENABLED(dev_priv)) { /* Must be done after probing outputs */ intel_opregion_register(dev_priv); acpi_video_register(); } - if (IS_GEN(dev_priv, 5)) - intel_gpu_ips_init(dev_priv); + intel_gt_driver_register(&dev_priv->gt); intel_audio_init(dev_priv); @@ -1562,11 +1344,11 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) * We need to coordinate the hotplugs with the asynchronous fbdev * configuration, for which we use the fbdev->async_cookie. */ - if (HAS_DISPLAY(dev_priv)) + if (HAS_DISPLAY(dev_priv) && INTEL_DISPLAY_ENABLED(dev_priv)) drm_kms_helper_poll_init(dev); intel_power_domains_enable(dev_priv); - intel_runtime_pm_enable(dev_priv); + intel_runtime_pm_enable(&dev_priv->runtime_pm); } /** @@ -1575,7 +1357,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) */ static void i915_driver_unregister(struct drm_i915_private *dev_priv) { - intel_runtime_pm_disable(dev_priv); + intel_runtime_pm_disable(&dev_priv->runtime_pm); intel_power_domains_disable(dev_priv); intel_fbdev_unregister(dev_priv); @@ -1588,7 +1370,7 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv) */ drm_kms_helper_poll_fini(&dev_priv->drm); - intel_gpu_ips_teardown(); + intel_gt_driver_unregister(&dev_priv->gt); acpi_video_unregister(); intel_opregion_unregister(dev_priv); @@ -1596,9 +1378,9 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv) i915_pmu_unregister(dev_priv); i915_teardown_sysfs(dev_priv); - drm_dev_unregister(&dev_priv->drm); + drm_dev_unplug(&dev_priv->drm); - i915_gem_shrinker_unregister(dev_priv); + i915_gem_driver_unregister(dev_priv); } static void i915_welcome_messages(struct drm_i915_private *dev_priv) @@ -1606,10 +1388,12 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv) if (drm_debug & DRM_UT_DRIVER) { struct drm_printer p = drm_debug_printer("i915 device info:"); - drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s gen=%i\n", + drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n", INTEL_DEVID(dev_priv), INTEL_REVID(dev_priv), intel_platform_name(INTEL_INFO(dev_priv)->platform), + intel_subplatform(RUNTIME_INFO(dev_priv), + INTEL_INFO(dev_priv)->platform), INTEL_GEN(dev_priv)); intel_device_info_dump_flags(INTEL_INFO(dev_priv), &p); @@ -1643,17 +1427,16 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) return ERR_PTR(err); } - i915->drm.pdev = pdev; i915->drm.dev_private = i915; - pci_set_drvdata(pdev, &i915->drm); + + i915->drm.pdev = pdev; + pci_set_drvdata(pdev, i915); /* Setup the write-once "constant" device info */ device_info = mkwrite_device_info(i915); memcpy(device_info, match_info, sizeof(*device_info)); RUNTIME_INFO(i915)->device_id = pdev->device; - BUILD_BUG_ON(INTEL_MAX_PLATFORMS > - BITS_PER_TYPE(device_info->platform_mask)); BUG_ON(device_info->gen > BITS_PER_TYPE(device_info->gen_mask)); return i915; @@ -1671,17 +1454,17 @@ static void i915_driver_destroy(struct drm_i915_private *i915) } /** - * i915_driver_load - setup chip and create an initial config + * i915_driver_probe - setup chip and create an initial config * @pdev: PCI device * @ent: matching PCI ID entry * - * The driver load routine has to do several things: + * The driver probe routine has to do several things: * - drive output discovery via intel_modeset_init() * - initialize the memory manager * - allocate initial config memory * - setup the DRM framebuffer with the allocated memory */ -int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) +int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { const struct intel_device_info *match_info = (struct intel_device_info *)ent->driver_data; @@ -1696,99 +1479,124 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) if (!i915_modparams.nuclear_pageflip && match_info->gen < 5) dev_priv->drm.driver_features &= ~DRIVER_ATOMIC; + /* + * Check if we support fake LMEM -- for now we only unleash this for + * the live selftests(test-and-exit). + */ +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) + if (IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) { + if (INTEL_GEN(dev_priv) >= 9 && i915_selftest.live < 0 && + i915_modparams.fake_lmem_start) { + mkwrite_device_info(dev_priv)->memory_regions = + REGION_SMEM | REGION_LMEM | REGION_STOLEN; + mkwrite_device_info(dev_priv)->is_dgfx = true; + GEM_BUG_ON(!HAS_LMEM(dev_priv)); + GEM_BUG_ON(!IS_DGFX(dev_priv)); + } + } +#endif + ret = pci_enable_device(pdev); if (ret) goto out_fini; - ret = i915_driver_init_early(dev_priv); + ret = i915_driver_early_probe(dev_priv); if (ret < 0) goto out_pci_disable; - disable_rpm_wakeref_asserts(dev_priv); + disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); - ret = i915_driver_init_mmio(dev_priv); + i915_detect_vgpu(dev_priv); + + ret = i915_driver_mmio_probe(dev_priv); if (ret < 0) goto out_runtime_pm_put; - ret = i915_driver_init_hw(dev_priv); + ret = i915_driver_hw_probe(dev_priv); if (ret < 0) goto out_cleanup_mmio; - ret = i915_load_modeset_init(&dev_priv->drm); + ret = i915_driver_modeset_probe(dev_priv); if (ret < 0) goto out_cleanup_hw; i915_driver_register(dev_priv); - enable_rpm_wakeref_asserts(dev_priv); + enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); i915_welcome_messages(dev_priv); return 0; out_cleanup_hw: - i915_driver_cleanup_hw(dev_priv); + i915_driver_hw_remove(dev_priv); + intel_memory_regions_driver_release(dev_priv); + i915_ggtt_driver_release(dev_priv); out_cleanup_mmio: - i915_driver_cleanup_mmio(dev_priv); + i915_driver_mmio_release(dev_priv); out_runtime_pm_put: - enable_rpm_wakeref_asserts(dev_priv); - i915_driver_cleanup_early(dev_priv); + enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); + i915_driver_late_release(dev_priv); out_pci_disable: pci_disable_device(pdev); out_fini: - i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret); + i915_probe_error(dev_priv, "Device initialization failed (%d)\n", ret); i915_driver_destroy(dev_priv); return ret; } -void i915_driver_unload(struct drm_device *dev) +void i915_driver_remove(struct drm_i915_private *i915) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct pci_dev *pdev = dev_priv->drm.pdev; + disable_rpm_wakeref_asserts(&i915->runtime_pm); - disable_rpm_wakeref_asserts(dev_priv); + i915_driver_unregister(i915); - i915_driver_unregister(dev_priv); + /* + * After unregistering the device to prevent any new users, cancel + * all in-flight requests so that we can quickly unbind the active + * resources. + */ + intel_gt_set_wedged(&i915->gt); /* Flush any external code that still may be under the RCU lock */ synchronize_rcu(); - if (i915_gem_suspend(dev_priv)) - DRM_ERROR("failed to idle hardware; continuing to unload!\n"); + i915_gem_suspend(i915); - drm_atomic_helper_shutdown(dev); + drm_atomic_helper_shutdown(&i915->drm); - intel_gvt_cleanup(dev_priv); + intel_gvt_driver_remove(i915); - intel_modeset_cleanup(dev); + i915_driver_modeset_remove(i915); - intel_bios_cleanup(dev_priv); + i915_reset_error_state(i915); + i915_gem_driver_remove(i915); - vga_switcheroo_unregister_client(pdev); - vga_client_register(pdev, NULL, NULL, NULL); + intel_power_domains_driver_remove(i915); - intel_csr_ucode_fini(dev_priv); + i915_driver_hw_remove(i915); - /* Free error state after interrupts are fully disabled. */ - cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); - i915_reset_error_state(dev_priv); - - i915_gem_fini(dev_priv); - - intel_power_domains_fini_hw(dev_priv); - - i915_driver_cleanup_hw(dev_priv); - i915_driver_cleanup_mmio(dev_priv); - - enable_rpm_wakeref_asserts(dev_priv); - intel_runtime_pm_cleanup(dev_priv); + enable_rpm_wakeref_asserts(&i915->runtime_pm); } static void i915_driver_release(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; + + disable_rpm_wakeref_asserts(rpm); + + i915_gem_driver_release(dev_priv); - i915_driver_cleanup_early(dev_priv); + intel_memory_regions_driver_release(dev_priv); + i915_ggtt_driver_release(dev_priv); + + i915_driver_mmio_release(dev_priv); + + enable_rpm_wakeref_asserts(rpm); + intel_runtime_pm_driver_release(rpm); + + i915_driver_late_release(dev_priv); i915_driver_destroy(dev_priv); } @@ -1826,12 +1634,13 @@ static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; - mutex_lock(&dev->struct_mutex); i915_gem_context_close(file); i915_gem_release(dev, file); - mutex_unlock(&dev->struct_mutex); - kfree(file_priv); + kfree_rcu(file_priv, rcu); + + /* Catch up with all the deferred frees from "this" client */ + i915_gem_flush_free_objects(to_i915(dev)); } static void intel_suspend_encoders(struct drm_i915_private *dev_priv) @@ -1862,7 +1671,6 @@ static bool suspend_to_idle(struct drm_i915_private *dev_priv) static int i915_drm_prepare(struct drm_device *dev) { struct drm_i915_private *i915 = to_i915(dev); - int err; /* * NB intel_display_suspend() may issue new requests after we've @@ -1870,12 +1678,9 @@ static int i915_drm_prepare(struct drm_device *dev) * split out that work and pull it forward so that after point, * the GPU is not woken again. */ - err = i915_gem_suspend(i915); - if (err) - dev_err(&i915->drm.pdev->dev, - "GEM idle failed, suspend/resume might fail\n"); + i915_gem_suspend(i915); - return err; + return 0; } static int i915_drm_suspend(struct drm_device *dev) @@ -1884,7 +1689,7 @@ static int i915_drm_suspend(struct drm_device *dev) struct pci_dev *pdev = dev_priv->drm.pdev; pci_power_t opregion_target_state; - disable_rpm_wakeref_asserts(dev_priv); + disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); /* We do a lot of poking in a lot of registers, make sure they work * properly. */ @@ -1918,7 +1723,7 @@ static int i915_drm_suspend(struct drm_device *dev) intel_csr_ucode_suspend(dev_priv); - enable_rpm_wakeref_asserts(dev_priv); + enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); return 0; } @@ -1939,23 +1744,21 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) { struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = dev_priv->drm.pdev; - int ret; + struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; + int ret = 0; - disable_rpm_wakeref_asserts(dev_priv); + disable_rpm_wakeref_asserts(rpm); i915_gem_suspend_late(dev_priv); - intel_uncore_suspend(dev_priv); + intel_uncore_suspend(&dev_priv->uncore); intel_power_domains_suspend(dev_priv, get_suspend_mode(dev_priv, hibernation)); - ret = 0; - if (INTEL_GEN(dev_priv) >= 11 || IS_GEN9_LP(dev_priv)) - bxt_enable_dc9(dev_priv); - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - hsw_enable_pc8(dev_priv); - else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + intel_display_power_suspend_late(dev_priv); + + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ret = vlv_suspend_complete(dev_priv); if (ret) { @@ -1982,35 +1785,29 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) pci_set_power_state(pdev, PCI_D3hot); out: - enable_rpm_wakeref_asserts(dev_priv); - if (!dev_priv->uncore.user_forcewake.count) - intel_runtime_pm_cleanup(dev_priv); + enable_rpm_wakeref_asserts(rpm); + if (!dev_priv->uncore.user_forcewake_count) + intel_runtime_pm_driver_release(rpm); return ret; } -static int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state) +int i915_suspend_switcheroo(struct drm_i915_private *i915, pm_message_t state) { int error; - if (!dev) { - DRM_ERROR("dev: %p\n", dev); - DRM_ERROR("DRM not initialized, aborting suspend.\n"); - return -ENODEV; - } - if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND && state.event != PM_EVENT_FREEZE)) return -EINVAL; - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - error = i915_drm_suspend(dev); + error = i915_drm_suspend(&i915->drm); if (error) return error; - return i915_drm_suspend_late(dev, false); + return i915_drm_suspend_late(&i915->drm, false); } static int i915_drm_resume(struct drm_device *dev) @@ -2018,15 +1815,19 @@ static int i915_drm_resume(struct drm_device *dev) struct drm_i915_private *dev_priv = to_i915(dev); int ret; - disable_rpm_wakeref_asserts(dev_priv); - intel_sanitize_gt_powersave(dev_priv); + disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); - i915_gem_sanitize(dev_priv); + intel_rc6_ctx_wa_resume(&dev_priv->gt.rc6); + + intel_gt_sanitize(&dev_priv->gt, true); ret = i915_ggtt_enable_hw(dev_priv); if (ret) DRM_ERROR("failed to re-enable GGTT\n"); + i915_gem_restore_gtt_mappings(dev_priv); + i915_gem_restore_fences(&dev_priv->ggtt); + intel_csr_ucode_resume(dev_priv); i915_restore_state(dev_priv); @@ -2050,7 +1851,7 @@ static int i915_drm_resume(struct drm_device *dev) i915_gem_resume(dev_priv); - intel_modeset_init_hw(dev); + intel_modeset_init_hw(dev_priv); intel_init_clock_gating(dev_priv); spin_lock_irq(&dev_priv->irq_lock); @@ -2078,7 +1879,7 @@ static int i915_drm_resume(struct drm_device *dev) intel_power_domains_enable(dev_priv); - enable_rpm_wakeref_asserts(dev_priv); + enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); return 0; } @@ -2133,7 +1934,7 @@ static int i915_drm_resume_early(struct drm_device *dev) pci_set_master(pdev); - disable_rpm_wakeref_asserts(dev_priv); + disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ret = vlv_resume_prepare(dev_priv, false); @@ -2141,75 +1942,66 @@ static int i915_drm_resume_early(struct drm_device *dev) DRM_ERROR("Resume prepare failed: %d, continuing anyway\n", ret); - intel_uncore_resume_early(dev_priv); + intel_uncore_resume_early(&dev_priv->uncore); - if (INTEL_GEN(dev_priv) >= 11 || IS_GEN9_LP(dev_priv)) { - gen9_sanitize_dc_state(dev_priv); - bxt_disable_dc9(dev_priv); - } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - hsw_disable_pc8(dev_priv); - } + intel_gt_check_and_clear_faults(&dev_priv->gt); - intel_uncore_sanitize(dev_priv); + intel_display_power_resume_early(dev_priv); intel_power_domains_resume(dev_priv); - intel_engines_sanitize(dev_priv, true); - - enable_rpm_wakeref_asserts(dev_priv); + enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); return ret; } -static int i915_resume_switcheroo(struct drm_device *dev) +int i915_resume_switcheroo(struct drm_i915_private *i915) { int ret; - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - ret = i915_drm_resume_early(dev); + ret = i915_drm_resume_early(&i915->drm); if (ret) return ret; - return i915_drm_resume(dev); + return i915_drm_resume(&i915->drm); } static int i915_pm_prepare(struct device *kdev) { - struct pci_dev *pdev = to_pci_dev(kdev); - struct drm_device *dev = pci_get_drvdata(pdev); + struct drm_i915_private *i915 = kdev_to_i915(kdev); - if (!dev) { + if (!i915) { dev_err(kdev, "DRM not initialized, aborting suspend.\n"); return -ENODEV; } - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - return i915_drm_prepare(dev); + return i915_drm_prepare(&i915->drm); } static int i915_pm_suspend(struct device *kdev) { - struct pci_dev *pdev = to_pci_dev(kdev); - struct drm_device *dev = pci_get_drvdata(pdev); + struct drm_i915_private *i915 = kdev_to_i915(kdev); - if (!dev) { + if (!i915) { dev_err(kdev, "DRM not initialized, aborting suspend.\n"); return -ENODEV; } - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - return i915_drm_suspend(dev); + return i915_drm_suspend(&i915->drm); } static int i915_pm_suspend_late(struct device *kdev) { - struct drm_device *dev = &kdev_to_i915(kdev)->drm; + struct drm_i915_private *i915 = kdev_to_i915(kdev); /* * We have a suspend ordering issue with the snd-hda driver also @@ -2220,55 +2012,55 @@ static int i915_pm_suspend_late(struct device *kdev) * FIXME: This should be solved with a special hdmi sink device or * similar so that power domains can be employed. */ - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - return i915_drm_suspend_late(dev, false); + return i915_drm_suspend_late(&i915->drm, false); } static int i915_pm_poweroff_late(struct device *kdev) { - struct drm_device *dev = &kdev_to_i915(kdev)->drm; + struct drm_i915_private *i915 = kdev_to_i915(kdev); - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - return i915_drm_suspend_late(dev, true); + return i915_drm_suspend_late(&i915->drm, true); } static int i915_pm_resume_early(struct device *kdev) { - struct drm_device *dev = &kdev_to_i915(kdev)->drm; + struct drm_i915_private *i915 = kdev_to_i915(kdev); - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - return i915_drm_resume_early(dev); + return i915_drm_resume_early(&i915->drm); } static int i915_pm_resume(struct device *kdev) { - struct drm_device *dev = &kdev_to_i915(kdev)->drm; + struct drm_i915_private *i915 = kdev_to_i915(kdev); - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - return i915_drm_resume(dev); + return i915_drm_resume(&i915->drm); } /* freeze: before creating the hibernation_image */ static int i915_pm_freeze(struct device *kdev) { - struct drm_device *dev = &kdev_to_i915(kdev)->drm; + struct drm_i915_private *i915 = kdev_to_i915(kdev); int ret; - if (dev->switch_power_state != DRM_SWITCH_POWER_OFF) { - ret = i915_drm_suspend(dev); + if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) { + ret = i915_drm_suspend(&i915->drm); if (ret) return ret; } - ret = i915_gem_freeze(kdev_to_i915(kdev)); + ret = i915_gem_freeze(i915); if (ret) return ret; @@ -2277,16 +2069,16 @@ static int i915_pm_freeze(struct device *kdev) static int i915_pm_freeze_late(struct device *kdev) { - struct drm_device *dev = &kdev_to_i915(kdev)->drm; + struct drm_i915_private *i915 = kdev_to_i915(kdev); int ret; - if (dev->switch_power_state != DRM_SWITCH_POWER_OFF) { - ret = i915_drm_suspend_late(dev, true); + if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) { + ret = i915_drm_suspend_late(&i915->drm, true); if (ret) return ret; } - ret = i915_gem_freeze_late(kdev_to_i915(kdev)); + ret = i915_gem_freeze_late(i915); if (ret) return ret; @@ -2343,9 +2135,12 @@ static int i915_pm_restore(struct device *kdev) */ static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) { - struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; + struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state; int i; + if (!s) + return; + /* GAM 0x4000-0x4770 */ s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); @@ -2424,10 +2219,13 @@ static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) { - struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; + struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state; u32 val; int i; + if (!s) + return; + /* GAM 0x4000-0x4770 */ I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); @@ -2509,7 +2307,7 @@ static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); } -static int vlv_wait_for_pw_status(struct drm_i915_private *dev_priv, +static int vlv_wait_for_pw_status(struct drm_i915_private *i915, u32 mask, u32 val) { i915_reg_t reg = VLV_GTLC_PW_STATUS; @@ -2523,7 +2321,9 @@ static int vlv_wait_for_pw_status(struct drm_i915_private *dev_priv, * Transitioning between RC6 states should be at most 2ms (see * valleyview_enable_rps) so use a 3ms timeout. */ - ret = wait_for(((reg_value = I915_READ_NOTRACE(reg)) & mask) == val, 3); + ret = wait_for(((reg_value = + intel_uncore_read_notrace(&i915->uncore, reg)) & mask) + == val, 3); /* just trace the final value */ trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); @@ -2545,7 +2345,7 @@ int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) if (!force_on) return 0; - err = intel_wait_for_register(dev_priv, + err = intel_wait_for_register(&dev_priv->uncore, VLV_GTLC_SURVIVABILITY_REG, VLV_GFX_CLK_STATUS_BIT, VLV_GFX_CLK_STATUS_BIT, @@ -2634,8 +2434,7 @@ static int vlv_suspend_complete(struct drm_i915_private *dev_priv) if (err) goto err2; - if (!IS_CHERRYVIEW(dev_priv)) - vlv_save_gunit_s0ix_state(dev_priv); + vlv_save_gunit_s0ix_state(dev_priv); err = vlv_force_gfx_clock(dev_priv, false); if (err) @@ -2665,8 +2464,7 @@ static int vlv_resume_prepare(struct drm_i915_private *dev_priv, */ ret = vlv_force_gfx_clock(dev_priv, true); - if (!IS_CHERRYVIEW(dev_priv)) - vlv_restore_gunit_s0ix_state(dev_priv); + vlv_restore_gunit_s0ix_state(dev_priv); err = vlv_allow_gt_wake(dev_priv, true); if (!ret) @@ -2686,20 +2484,16 @@ static int vlv_resume_prepare(struct drm_i915_private *dev_priv, static int intel_runtime_suspend(struct device *kdev) { - struct pci_dev *pdev = to_pci_dev(kdev); - struct drm_device *dev = pci_get_drvdata(pdev); - struct drm_i915_private *dev_priv = to_i915(dev); - int ret; - - if (WARN_ON_ONCE(!(dev_priv->gt_pm.rc6.enabled && HAS_RC6(dev_priv)))) - return -ENODEV; + struct drm_i915_private *dev_priv = kdev_to_i915(kdev); + struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; + int ret = 0; if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) return -ENODEV; DRM_DEBUG_KMS("Suspending device\n"); - disable_rpm_wakeref_asserts(dev_priv); + disable_rpm_wakeref_asserts(rpm); /* * We are safe here against re-faults, since the fault handler takes @@ -2707,48 +2501,39 @@ static int intel_runtime_suspend(struct device *kdev) */ i915_gem_runtime_suspend(dev_priv); - intel_uc_suspend(dev_priv); + intel_gt_runtime_suspend(&dev_priv->gt); intel_runtime_pm_disable_interrupts(dev_priv); - intel_uncore_suspend(dev_priv); - - ret = 0; - if (INTEL_GEN(dev_priv) >= 11) { - icl_display_core_uninit(dev_priv); - bxt_enable_dc9(dev_priv); - } else if (IS_GEN9_LP(dev_priv)) { - bxt_display_core_uninit(dev_priv); - bxt_enable_dc9(dev_priv); - } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - hsw_enable_pc8(dev_priv); - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + intel_uncore_suspend(&dev_priv->uncore); + + intel_display_power_suspend(dev_priv); + + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ret = vlv_suspend_complete(dev_priv); - } if (ret) { DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); - intel_uncore_runtime_resume(dev_priv); + intel_uncore_runtime_resume(&dev_priv->uncore); intel_runtime_pm_enable_interrupts(dev_priv); - intel_uc_resume(dev_priv); + intel_gt_runtime_resume(&dev_priv->gt); - i915_gem_init_swizzling(dev_priv); - i915_gem_restore_fences(dev_priv); + i915_gem_restore_fences(&dev_priv->ggtt); - enable_rpm_wakeref_asserts(dev_priv); + enable_rpm_wakeref_asserts(rpm); return ret; } - enable_rpm_wakeref_asserts(dev_priv); - intel_runtime_pm_cleanup(dev_priv); + enable_rpm_wakeref_asserts(rpm); + intel_runtime_pm_driver_release(rpm); - if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv)) + if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore)) DRM_ERROR("Unclaimed access detected prior to suspending\n"); - dev_priv->runtime_pm.suspended = true; + rpm->suspended = true; /* * FIXME: We really should find a document that references the arguments @@ -2773,7 +2558,7 @@ static int intel_runtime_suspend(struct device *kdev) intel_opregion_notify_adapter(dev_priv, PCI_D1); } - assert_forcewakes_inactive(dev_priv); + assert_forcewakes_inactive(&dev_priv->uncore); if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) intel_hpd_poll_init(dev_priv); @@ -2784,9 +2569,8 @@ static int intel_runtime_suspend(struct device *kdev) static int intel_runtime_resume(struct device *kdev) { - struct pci_dev *pdev = to_pci_dev(kdev); - struct drm_device *dev = pci_get_drvdata(pdev); - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = kdev_to_i915(kdev); + struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; int ret = 0; if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) @@ -2794,49 +2578,29 @@ static int intel_runtime_resume(struct device *kdev) DRM_DEBUG_KMS("Resuming device\n"); - WARN_ON_ONCE(atomic_read(&dev_priv->runtime_pm.wakeref_count)); - disable_rpm_wakeref_asserts(dev_priv); + WARN_ON_ONCE(atomic_read(&rpm->wakeref_count)); + disable_rpm_wakeref_asserts(rpm); intel_opregion_notify_adapter(dev_priv, PCI_D0); - dev_priv->runtime_pm.suspended = false; - if (intel_uncore_unclaimed_mmio(dev_priv)) + rpm->suspended = false; + if (intel_uncore_unclaimed_mmio(&dev_priv->uncore)) DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n"); - if (INTEL_GEN(dev_priv) >= 11) { - bxt_disable_dc9(dev_priv); - icl_display_core_init(dev_priv, true); - if (dev_priv->csr.dmc_payload) { - if (dev_priv->csr.allowed_dc_mask & - DC_STATE_EN_UPTO_DC6) - skl_enable_dc6(dev_priv); - else if (dev_priv->csr.allowed_dc_mask & - DC_STATE_EN_UPTO_DC5) - gen9_enable_dc5(dev_priv); - } - } else if (IS_GEN9_LP(dev_priv)) { - bxt_disable_dc9(dev_priv); - bxt_display_core_init(dev_priv, true); - if (dev_priv->csr.dmc_payload && - (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) - gen9_enable_dc5(dev_priv); - } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - hsw_disable_pc8(dev_priv); - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + intel_display_power_resume(dev_priv); + + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ret = vlv_resume_prepare(dev_priv, true); - } - intel_uncore_runtime_resume(dev_priv); + intel_uncore_runtime_resume(&dev_priv->uncore); intel_runtime_pm_enable_interrupts(dev_priv); - intel_uc_resume(dev_priv); - /* * No point of rolling back things in case of an error, as the best * we can do is to hope that things will still work (and disable RPM). */ - i915_gem_init_swizzling(dev_priv); - i915_gem_restore_fences(dev_priv); + intel_gt_runtime_resume(&dev_priv->gt); + i915_gem_restore_fences(&dev_priv->ggtt); /* * On VLV/CHV display interrupts are part of the display @@ -2848,7 +2612,7 @@ static int intel_runtime_resume(struct device *kdev) intel_enable_ipc(dev_priv); - enable_rpm_wakeref_asserts(dev_priv); + enable_rpm_wakeref_asserts(rpm); if (ret) DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret); @@ -2930,7 +2694,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), - DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), @@ -2943,13 +2707,13 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer_ioctl, DRM_AUTH), - DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), @@ -2968,8 +2732,8 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER), DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER), DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER), - DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW), @@ -2977,9 +2741,11 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_VM_CREATE, i915_gem_vm_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW), }; static struct drm_driver driver = { @@ -2987,7 +2753,7 @@ static struct drm_driver driver = { * deal with them for Intel hardware. */ .driver_features = - DRIVER_GEM | DRIVER_PRIME | + DRIVER_GEM | DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ, .release = i915_driver_release, .open = i915_driver_open, @@ -3003,6 +2769,9 @@ static struct drm_driver driver = { .gem_prime_export = i915_gem_prime_export, .gem_prime_import = i915_gem_prime_import, + .get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos, + .get_scanout_position = i915_get_crtc_scanoutpos, + .dumb_create = i915_gem_dumb_create, .dumb_map_offset = i915_gem_mmap_gtt, .ioctls = i915_ioctls, |