diff options
Diffstat (limited to 'drivers/gpu/drm/i915/intel_pm.c')
| -rw-r--r-- | drivers/gpu/drm/i915/intel_pm.c | 1402 |
1 files changed, 679 insertions, 723 deletions
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 34e0d22d456b..0c3e63f27c29 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -33,6 +33,7 @@ #include <drm/drm_plane_helper.h> #include "display/intel_atomic.h" +#include "display/intel_atomic_plane.h" #include "display/intel_bw.h" #include "display/intel_display_types.h" #include "display/intel_fbc.h" @@ -81,24 +82,24 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) * Must match Sampler, Pixel Back End, and Media. See * WaCompressedResourceSamplerPbeMediaNewHashMode. */ - I915_WRITE(CHICKEN_PAR1_1, - I915_READ(CHICKEN_PAR1_1) | + intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1, + intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_DE_COMPRESSED_HASH_MODE); } /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */ - I915_WRITE(CHICKEN_PAR1_1, - I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); + intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1, + intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */ - I915_WRITE(GEN8_CHICKEN_DCPR_1, - I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); + intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, + intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); /* * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl * Display WA #0859: skl,bxt,kbl,glk,cfl */ - I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | DISP_FBC_MEMORY_WAKE); } @@ -107,21 +108,21 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) gen9_init_clock_gating(dev_priv); /* WaDisableSDEUnitClockGating:bxt */ - I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* * FIXME: * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only. */ - I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) | GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ); /* * Wa: Backlight PWM may stop in the asserted state, causing backlight * to stay fully on. */ - I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | + intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) | PWM1_GATING_DIS | PWM2_GATING_DIS); /* @@ -130,20 +131,20 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) * is off and a MMIO access is attempted by any privilege * application, using batch buffers or any other means. */ - I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950)); + intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950)); /* * WaFbcTurnOffFbcWatermark:bxt * Display WA #0562: bxt */ - I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | DISP_FBC_WM_DIS); /* * WaFbcHighMemBwCorruptionAvoidance:bxt * Display WA #0883: bxt */ - I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | ILK_DPFC_DISABLE_DUMMY0); } @@ -156,7 +157,7 @@ static void glk_init_clock_gating(struct drm_i915_private *dev_priv) * Backlight PWM may stop in the asserted state, causing backlight * to stay fully on. */ - I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | + intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) | PWM1_GATING_DIS | PWM2_GATING_DIS); } @@ -164,7 +165,7 @@ static void pnv_get_mem_freq(struct drm_i915_private *dev_priv) { u32 tmp; - tmp = I915_READ(CLKCFG); + tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG); switch (tmp & CLKCFG_FSB_MASK) { case CLKCFG_FSB_533: @@ -194,7 +195,7 @@ static void pnv_get_mem_freq(struct drm_i915_private *dev_priv) } /* detect pineview DDR3 setting */ - tmp = I915_READ(CSHRDDR3CTL); + tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL); dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; } @@ -365,39 +366,39 @@ static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enabl u32 val; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; - I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); - POSTING_READ(FW_BLC_SELF_VLV); + was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; + intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); + intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV); } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) { - was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; - I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); - POSTING_READ(FW_BLC_SELF); + was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; + intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); + intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF); } else if (IS_PINEVIEW(dev_priv)) { - val = I915_READ(DSPFW3); + val = intel_uncore_read(&dev_priv->uncore, DSPFW3); was_enabled = val & PINEVIEW_SELF_REFRESH_EN; if (enable) val |= PINEVIEW_SELF_REFRESH_EN; else val &= ~PINEVIEW_SELF_REFRESH_EN; - I915_WRITE(DSPFW3, val); - POSTING_READ(DSPFW3); + intel_uncore_write(&dev_priv->uncore, DSPFW3, val); + intel_uncore_posting_read(&dev_priv->uncore, DSPFW3); } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) { - was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; + was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) : _MASKED_BIT_DISABLE(FW_BLC_SELF_EN); - I915_WRITE(FW_BLC_SELF, val); - POSTING_READ(FW_BLC_SELF); + intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val); + intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF); } else if (IS_I915GM(dev_priv)) { /* * FIXME can't find a bit like this for 915G, and * and yet it does have the related watermark in * FW_BLC_SELF. What's going on? */ - was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN; + was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN; val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) : _MASKED_BIT_DISABLE(INSTPM_SELF_EN); - I915_WRITE(INSTPM, val); - POSTING_READ(INSTPM); + intel_uncore_write(&dev_priv->uncore, INSTPM, val); + intel_uncore_posting_read(&dev_priv->uncore, INSTPM); } else { return false; } @@ -493,20 +494,20 @@ static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) switch (pipe) { case PIPE_A: - dsparb = I915_READ(DSPARB); - dsparb2 = I915_READ(DSPARB2); + dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); + dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2); sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0); sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4); break; case PIPE_B: - dsparb = I915_READ(DSPARB); - dsparb2 = I915_READ(DSPARB2); + dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); + dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2); sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8); sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12); break; case PIPE_C: - dsparb2 = I915_READ(DSPARB2); - dsparb3 = I915_READ(DSPARB3); + dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2); + dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3); sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16); sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20); break; @@ -524,7 +525,7 @@ static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { - u32 dsparb = I915_READ(DSPARB); + u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); int size; size = dsparb & 0x7f; @@ -540,7 +541,7 @@ static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, static int i830_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { - u32 dsparb = I915_READ(DSPARB); + u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); int size; size = dsparb & 0x1ff; @@ -557,7 +558,7 @@ static int i830_get_fifo_size(struct drm_i915_private *dev_priv, static int i845_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { - u32 dsparb = I915_READ(DSPARB); + u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); int size; size = dsparb & 0x7f; @@ -899,49 +900,49 @@ static void pnv_update_wm(struct intel_crtc *unused_crtc) crtc = single_enabled_crtc(dev_priv); if (crtc) { - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp = fb->format->cpp[0]; - int clock = adjusted_mode->crtc_clock; + int clock = pipe_mode->crtc_clock; /* Display SR */ wm = intel_calculate_wm(clock, &pnv_display_wm, pnv_display_wm.fifo_size, cpp, latency->display_sr); - reg = I915_READ(DSPFW1); + reg = intel_uncore_read(&dev_priv->uncore, DSPFW1); reg &= ~DSPFW_SR_MASK; reg |= FW_WM(wm, SR); - I915_WRITE(DSPFW1, reg); + intel_uncore_write(&dev_priv->uncore, DSPFW1, reg); drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg); /* cursor SR */ wm = intel_calculate_wm(clock, &pnv_cursor_wm, pnv_display_wm.fifo_size, 4, latency->cursor_sr); - reg = I915_READ(DSPFW3); + reg = intel_uncore_read(&dev_priv->uncore, DSPFW3); reg &= ~DSPFW_CURSOR_SR_MASK; reg |= FW_WM(wm, CURSOR_SR); - I915_WRITE(DSPFW3, reg); + intel_uncore_write(&dev_priv->uncore, DSPFW3, reg); /* Display HPLL off SR */ wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm, pnv_display_hplloff_wm.fifo_size, cpp, latency->display_hpll_disable); - reg = I915_READ(DSPFW3); + reg = intel_uncore_read(&dev_priv->uncore, DSPFW3); reg &= ~DSPFW_HPLL_SR_MASK; reg |= FW_WM(wm, HPLL_SR); - I915_WRITE(DSPFW3, reg); + intel_uncore_write(&dev_priv->uncore, DSPFW3, reg); /* cursor HPLL off SR */ wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm, pnv_display_hplloff_wm.fifo_size, 4, latency->cursor_hpll_disable); - reg = I915_READ(DSPFW3); + reg = intel_uncore_read(&dev_priv->uncore, DSPFW3); reg &= ~DSPFW_HPLL_CURSOR_MASK; reg |= FW_WM(wm, HPLL_CURSOR); - I915_WRITE(DSPFW3, reg); + intel_uncore_write(&dev_priv->uncore, DSPFW3, reg); drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg); intel_set_memory_cxsr(dev_priv, true); @@ -975,25 +976,25 @@ static void g4x_write_wm_values(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm); - I915_WRITE(DSPFW1, + intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(wm->sr.plane, SR) | FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); - I915_WRITE(DSPFW2, + intel_uncore_write(&dev_priv->uncore, DSPFW2, (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) | FW_WM(wm->sr.fbc, FBC_SR) | FW_WM(wm->hpll.fbc, FBC_HPLL_SR) | FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) | FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); - I915_WRITE(DSPFW3, + intel_uncore_write(&dev_priv->uncore, DSPFW3, (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) | FW_WM(wm->sr.cursor, CURSOR_SR) | FW_WM(wm->hpll.cursor, HPLL_CURSOR) | FW_WM(wm->hpll.plane, HPLL_SR)); - POSTING_READ(DSPFW1); + intel_uncore_posting_read(&dev_priv->uncore, DSPFW1); } #define FW_WM_VLV(value, plane) \ @@ -1007,7 +1008,7 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) { trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm); - I915_WRITE(VLV_DDL(pipe), + intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe), (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) | (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) | (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) | @@ -1019,35 +1020,35 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv, * high order bits so that there are no out of bounds values * present in the registers during the reprogramming. */ - I915_WRITE(DSPHOWM, 0); - I915_WRITE(DSPHOWM1, 0); - I915_WRITE(DSPFW4, 0); - I915_WRITE(DSPFW5, 0); - I915_WRITE(DSPFW6, 0); + intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0); + intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0); + intel_uncore_write(&dev_priv->uncore, DSPFW4, 0); + intel_uncore_write(&dev_priv->uncore, DSPFW5, 0); + intel_uncore_write(&dev_priv->uncore, DSPFW6, 0); - I915_WRITE(DSPFW1, + intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(wm->sr.plane, SR) | FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); - I915_WRITE(DSPFW2, + intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) | FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); - I915_WRITE(DSPFW3, + intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(wm->sr.cursor, CURSOR_SR)); if (IS_CHERRYVIEW(dev_priv)) { - I915_WRITE(DSPFW7_CHV, + intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV, FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); - I915_WRITE(DSPFW8_CHV, + intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV, FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) | FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE)); - I915_WRITE(DSPFW9_CHV, + intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV, FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) | FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC)); - I915_WRITE(DSPHOWM, + intel_uncore_write(&dev_priv->uncore, DSPHOWM, FW_WM(wm->sr.plane >> 9, SR_HI) | FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) | FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) | @@ -1059,10 +1060,10 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv, FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) | FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); } else { - I915_WRITE(DSPFW7, + intel_uncore_write(&dev_priv->uncore, DSPFW7, FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); - I915_WRITE(DSPHOWM, + intel_uncore_write(&dev_priv->uncore, DSPHOWM, FW_WM(wm->sr.plane >> 9, SR_HI) | FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) | FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) | @@ -1072,7 +1073,7 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv, FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); } - POSTING_READ(DSPFW1); + intel_uncore_posting_read(&dev_priv->uncore, DSPFW1); } #undef FW_WM_VLV @@ -1135,8 +1136,8 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; unsigned int latency = dev_priv->wm.pri_latency[level] * 10; unsigned int clock, htotal, cpp, width, wm; @@ -1163,8 +1164,8 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, level != G4X_WM_LEVEL_NORMAL) cpp = max(cpp, 4u); - clock = adjusted_mode->crtc_clock; - htotal = adjusted_mode->crtc_htotal; + clock = pipe_mode->crtc_clock; + htotal = pipe_mode->crtc_htotal; width = drm_rect_width(&plane_state->uapi.dst); @@ -1660,8 +1661,8 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; unsigned int clock, htotal, cpp, width, wm; if (dev_priv->wm.pri_latency[level] == 0) @@ -1671,8 +1672,8 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, return 0; cpp = plane_state->hw.fb->format->cpp[0]; - clock = adjusted_mode->crtc_clock; - htotal = adjusted_mode->crtc_htotal; + clock = pipe_mode->crtc_clock; + htotal = pipe_mode->crtc_htotal; width = crtc_state->pipe_src_w; if (plane->id == PLANE_CURSOR) { @@ -2261,12 +2262,12 @@ static void i965_update_wm(struct intel_crtc *unused_crtc) if (crtc) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 12000; - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; - int clock = adjusted_mode->crtc_clock; - int htotal = adjusted_mode->crtc_htotal; + int clock = pipe_mode->crtc_clock; + int htotal = pipe_mode->crtc_htotal; int hdisplay = crtc->config->pipe_src_w; int cpp = fb->format->cpp[0]; int entries; @@ -2309,14 +2310,14 @@ static void i965_update_wm(struct intel_crtc *unused_crtc) srwm); /* 965 has limitations... */ - I915_WRITE(DSPFW1, FW_WM(srwm, SR) | + intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) | FW_WM(8, CURSORB) | FW_WM(8, PLANEB) | FW_WM(8, PLANEA)); - I915_WRITE(DSPFW2, FW_WM(8, CURSORA) | + intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) | FW_WM(8, PLANEC_OLD)); /* update cursor SR watermark */ - I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR)); + intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR)); if (cxsr_enabled) intel_set_memory_cxsr(dev_priv, true); @@ -2345,8 +2346,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A); crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp; @@ -2356,7 +2357,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) else cpp = fb->format->cpp[0]; - planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, + planea_wm = intel_calculate_wm(pipe_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); enabled = crtc; @@ -2372,8 +2373,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B); crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp; @@ -2383,7 +2384,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) else cpp = fb->format->cpp[0]; - planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, + planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); if (enabled == NULL) @@ -2421,12 +2422,12 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) if (HAS_FW_BLC(dev_priv) && enabled) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 6000; - const struct drm_display_mode *adjusted_mode = - &enabled->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &enabled->config->hw.pipe_mode; const struct drm_framebuffer *fb = enabled->base.primary->state->fb; - int clock = adjusted_mode->crtc_clock; - int htotal = adjusted_mode->crtc_htotal; + int clock = pipe_mode->crtc_clock; + int htotal = pipe_mode->crtc_htotal; int hdisplay = enabled->config->pipe_src_w; int cpp; int entries; @@ -2446,10 +2447,10 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) srwm = 1; if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) - I915_WRITE(FW_BLC_SELF, + intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, FW_BLC_SELF_FIFO_MASK | (srwm & 0xff)); else - I915_WRITE(FW_BLC_SELF, srwm & 0x3f); + intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f); } drm_dbg_kms(&dev_priv->drm, @@ -2463,8 +2464,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) fwater_lo = fwater_lo | (1 << 24) | (1 << 8); fwater_hi = fwater_hi | (1 << 8); - I915_WRITE(FW_BLC, fwater_lo); - I915_WRITE(FW_BLC2, fwater_hi); + intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo); + intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi); if (enabled) intel_set_memory_cxsr(dev_priv, true); @@ -2474,7 +2475,7 @@ static void i845_update_wm(struct intel_crtc *unused_crtc) { struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); struct intel_crtc *crtc; - const struct drm_display_mode *adjusted_mode; + const struct drm_display_mode *pipe_mode; u32 fwater_lo; int planea_wm; @@ -2482,18 +2483,18 @@ static void i845_update_wm(struct intel_crtc *unused_crtc) if (crtc == NULL) return; - adjusted_mode = &crtc->config->hw.adjusted_mode; - planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, + pipe_mode = &crtc->config->hw.pipe_mode; + planea_wm = intel_calculate_wm(pipe_mode->crtc_clock, &i845_wm_info, dev_priv->display.get_fifo_size(dev_priv, PLANE_A), 4, pessimal_latency_ns); - fwater_lo = I915_READ(FW_BLC) & ~0xfff; + fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff; fwater_lo |= (3<<8) | planea_wm; drm_dbg_kms(&dev_priv->drm, "Setting FIFO watermarks - A: %d\n", planea_wm); - I915_WRITE(FW_BLC, fwater_lo); + intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo); } /* latency must be in 0.1us units. */ @@ -2573,7 +2574,7 @@ static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state, return method1; method2 = ilk_wm_method2(crtc_state->pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, drm_rect_width(&plane_state->uapi.dst), cpp, mem_value); @@ -2601,7 +2602,7 @@ static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state, method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); method2 = ilk_wm_method2(crtc_state->pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, drm_rect_width(&plane_state->uapi.dst), cpp, mem_value); return min(method1, method2); @@ -2626,7 +2627,7 @@ static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state, cpp = plane_state->hw.fb->format->cpp[0]; return ilk_wm_method2(crtc_state->pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, drm_rect_width(&plane_state->uapi.dst), cpp, mem_value); } @@ -2929,7 +2930,7 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, * any underrun. If not able to get Dimm info assume 16GB dimm * to avoid any underrun. */ - if (dev_priv->dram_info.is_16gb_dimm) + if (dev_priv->dram_info.wm_lv_0_adjust_needed) wm[0] += 1; } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { @@ -3533,17 +3534,17 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) { previous->wm_lp[2] &= ~WM1_LP_SR_EN; - I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]); + intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]); changed = true; } if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) { previous->wm_lp[1] &= ~WM1_LP_SR_EN; - I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]); + intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]); changed = true; } if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) { previous->wm_lp[0] &= ~WM1_LP_SR_EN; - I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]); + intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]); changed = true; } @@ -3573,56 +3574,56 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, _ilk_disable_lp_wm(dev_priv, dirty); if (dirty & WM_DIRTY_PIPE(PIPE_A)) - I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]); + intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]); if (dirty & WM_DIRTY_PIPE(PIPE_B)) - I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]); + intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]); if (dirty & WM_DIRTY_PIPE(PIPE_C)) - I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]); + intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]); if (dirty & WM_DIRTY_DDB) { if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - val = I915_READ(WM_MISC); + val = intel_uncore_read(&dev_priv->uncore, WM_MISC); if (results->partitioning == INTEL_DDB_PART_1_2) val &= ~WM_MISC_DATA_PARTITION_5_6; else val |= WM_MISC_DATA_PARTITION_5_6; - I915_WRITE(WM_MISC, val); + intel_uncore_write(&dev_priv->uncore, WM_MISC, val); } else { - val = I915_READ(DISP_ARB_CTL2); + val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2); if (results->partitioning == INTEL_DDB_PART_1_2) val &= ~DISP_DATA_PARTITION_5_6; else val |= DISP_DATA_PARTITION_5_6; - I915_WRITE(DISP_ARB_CTL2, val); + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val); } } if (dirty & WM_DIRTY_FBC) { - val = I915_READ(DISP_ARB_CTL); + val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL); if (results->enable_fbc_wm) val &= ~DISP_FBC_WM_DIS; else val |= DISP_FBC_WM_DIS; - I915_WRITE(DISP_ARB_CTL, val); + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val); } if (dirty & WM_DIRTY_LP(1) && previous->wm_lp_spr[0] != results->wm_lp_spr[0]) - I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]); + intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]); if (INTEL_GEN(dev_priv) >= 7) { if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) - I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); + intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]); if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) - I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]); + intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]); } if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0]) - I915_WRITE(WM1_LP_ILK, results->wm_lp[0]); + intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]); if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1]) - I915_WRITE(WM2_LP_ILK, results->wm_lp[1]); + intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]); if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) - I915_WRITE(WM3_LP_ILK, results->wm_lp[2]); + intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]); dev_priv->wm.hw = *results; } @@ -3639,7 +3640,7 @@ u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv) u8 enabled_slices_mask = 0; for (i = 0; i < max_slices; i++) { - if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE) + if (intel_uncore_read(&dev_priv->uncore, DBUF_CTL_S(i)) & DBUF_POWER_STATE) enabled_slices_mask |= BIT(i); } @@ -3706,7 +3707,7 @@ skl_setup_sagv_block_time(struct drm_i915_private *dev_priv) * - All planes can enable watermarks for latencies >= SAGV engine block time * - We're not using an interlaced display configuration */ -int +static int intel_enable_sagv(struct drm_i915_private *dev_priv) { int ret; @@ -3740,7 +3741,7 @@ intel_enable_sagv(struct drm_i915_private *dev_priv) return 0; } -int +static int intel_disable_sagv(struct drm_i915_private *dev_priv) { int ret; @@ -3873,9 +3874,7 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_plane *plane; - const struct intel_plane_state *plane_state; - int level, latency; + enum plane_id plane_id; if (!intel_has_sagv(dev_priv)) return false; @@ -3883,12 +3882,13 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) if (!crtc_state->hw.active) return true; - if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) + if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE) return false; - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { + for_each_plane_id_on_crtc(crtc, plane_id) { const struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane->id]; + &crtc_state->wm.skl.optimal.planes[plane_id]; + int level; /* Skip this plane if it's not enabled */ if (!wm->wm[0].plane_en) @@ -3899,19 +3899,12 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) !wm->wm[level].plane_en; --level) { } - latency = dev_priv->wm.skl_latency[level]; - - if (skl_needs_memory_bw_wa(dev_priv) && - plane_state->uapi.fb->modifier == - I915_FORMAT_MOD_X_TILED) - latency += 15; - /* * If any of the planes on this pipe don't enable wm levels that * incur memory latencies higher than sagv_block_time_us we * can't enable SAGV. */ - if (latency < dev_priv->sagv_block_time_us) + if (!wm->wm[level].can_sagv) return false; } @@ -4024,43 +4017,48 @@ static int intel_compute_sagv_mask(struct intel_atomic_state *state) return 0; } -/* - * Calculate initial DBuf slice offset, based on slice size - * and mask(i.e if slice size is 1024 and second slice is enabled - * offset would be 1024) - */ -static unsigned int -icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask, - u32 slice_size, - u32 ddb_size) +static int intel_dbuf_size(struct drm_i915_private *dev_priv) { - unsigned int offset = 0; + int ddb_size = INTEL_INFO(dev_priv)->ddb_size; - if (!dbuf_slice_mask) - return 0; + drm_WARN_ON(&dev_priv->drm, ddb_size == 0); - offset = (ffs(dbuf_slice_mask) - 1) * slice_size; + if (INTEL_GEN(dev_priv) < 11) + return ddb_size - 4; /* 4 blocks for bypass path allocation */ - WARN_ON(offset >= ddb_size); - return offset; + return ddb_size; } -u16 intel_get_ddb_size(struct drm_i915_private *dev_priv) +static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv) { - u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size; - drm_WARN_ON(&dev_priv->drm, ddb_size == 0); + return intel_dbuf_size(dev_priv) / + INTEL_INFO(dev_priv)->num_supported_dbuf_slices; +} - if (INTEL_GEN(dev_priv) < 11) - return ddb_size - 4; /* 4 blocks for bypass path allocation */ +static void +skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask, + struct skl_ddb_entry *ddb) +{ + int slice_size = intel_dbuf_slice_size(dev_priv); - return ddb_size; + if (!slice_mask) { + ddb->start = 0; + ddb->end = 0; + return; + } + + ddb->start = (ffs(slice_mask) - 1) * slice_size; + ddb->end = fls(slice_mask) * slice_size; + + WARN_ON(ddb->start >= ddb->end); + WARN_ON(ddb->end > intel_dbuf_size(dev_priv)); } u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv, const struct skl_ddb_entry *entry) { u32 slice_mask = 0; - u16 ddb_size = intel_get_ddb_size(dev_priv); + u16 ddb_size = intel_dbuf_size(dev_priv); u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices; u16 slice_size = ddb_size / num_supported_slices; u16 start_slice; @@ -4084,116 +4082,40 @@ u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv, return slice_mask; } -static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state, - u8 active_pipes); - -static int -skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv, - const struct intel_crtc_state *crtc_state, - const u64 total_data_rate, - struct skl_ddb_entry *alloc, /* out */ - int *num_active /* out */) -{ - struct drm_atomic_state *state = crtc_state->uapi.state; - struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_crtc *for_crtc = crtc_state->uapi.crtc; - const struct intel_crtc *crtc; - u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0; - enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe; - struct intel_dbuf_state *new_dbuf_state = - intel_atomic_get_new_dbuf_state(intel_state); - const struct intel_dbuf_state *old_dbuf_state = - intel_atomic_get_old_dbuf_state(intel_state); - u8 active_pipes = new_dbuf_state->active_pipes; - u16 ddb_size; - u32 ddb_range_size; - u32 i; - u32 dbuf_slice_mask; - u32 offset; - u32 slice_size; - u32 total_slice_mask; - u32 start, end; - int ret; - - *num_active = hweight8(active_pipes); - - if (!crtc_state->hw.active) { - alloc->start = 0; - alloc->end = 0; - return 0; - } - - ddb_size = intel_get_ddb_size(dev_priv); - - slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices; +static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state) +{ + const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; + int hdisplay, vdisplay; - /* - * If the state doesn't change the active CRTC's or there is no - * modeset request, then there's no need to recalculate; - * the existing pipe allocation limits should remain unchanged. - * Note that we're safe from racing commits since any racing commit - * that changes the active CRTC list or do modeset would need to - * grab _all_ crtc locks, including the one we currently hold. - */ - if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes && - !dev_priv->wm.distrust_bios_wm) { - /* - * alloc may be cleared by clear_intel_crtc_state, - * copy from old state to be sure - * - * FIXME get rid of this mess - */ - *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb; + if (!crtc_state->hw.active) return 0; - } - - /* - * Get allowed DBuf slices for correspondent pipe and platform. - */ - dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes); - - /* - * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2 - * and slice size is 1024, the offset would be 1024 - */ - offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask, - slice_size, ddb_size); - - /* - * Figure out total size of allowed DBuf slices, which is basically - * a number of allowed slices for that pipe multiplied by slice size. - * Inside of this - * range ddb entries are still allocated in proportion to display width. - */ - ddb_range_size = hweight8(dbuf_slice_mask) * slice_size; /* * Watermark/ddb requirement highly depends upon width of the * framebuffer, So instead of allocating DDB equally among pipes * distribute DDB based on resolution/width of the display. */ - total_slice_mask = dbuf_slice_mask; - for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - enum pipe pipe = crtc->pipe; - int hdisplay, vdisplay; - u32 pipe_dbuf_slice_mask; + drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay); - if (!crtc_state->hw.active) - continue; + return hdisplay; +} - pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, - active_pipes); +static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state, + enum pipe for_pipe, + unsigned int *weight_start, + unsigned int *weight_end, + unsigned int *weight_total) +{ + struct drm_i915_private *dev_priv = + to_i915(dbuf_state->base.state->base.dev); + enum pipe pipe; - /* - * According to BSpec pipe can share one dbuf slice with another - * pipes or pipe can use multiple dbufs, in both cases we - * account for other pipes only if they have exactly same mask. - * However we need to account how many slices we should enable - * in total. - */ - total_slice_mask |= pipe_dbuf_slice_mask; + *weight_start = 0; + *weight_end = 0; + *weight_total = 0; + + for_each_pipe(dev_priv, pipe) { + int weight = dbuf_state->weight[pipe]; /* * Do not account pipes using other slice sets @@ -4202,42 +4124,78 @@ skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv, * i.e no partial intersection), so it is enough to check for * equality for now. */ - if (dbuf_slice_mask != pipe_dbuf_slice_mask) + if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe]) continue; - drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay); + *weight_total += weight; + if (pipe < for_pipe) { + *weight_start += weight; + *weight_end += weight; + } else if (pipe == for_pipe) { + *weight_end += weight; + } + } +} - total_width_in_range += hdisplay; +static int +skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + unsigned int weight_total, weight_start, weight_end; + const struct intel_dbuf_state *old_dbuf_state = + intel_atomic_get_old_dbuf_state(state); + struct intel_dbuf_state *new_dbuf_state = + intel_atomic_get_new_dbuf_state(state); + struct intel_crtc_state *crtc_state; + struct skl_ddb_entry ddb_slices; + enum pipe pipe = crtc->pipe; + u32 ddb_range_size; + u32 dbuf_slice_mask; + u32 start, end; + int ret; - if (pipe < for_pipe) - width_before_pipe_in_range += hdisplay; - else if (pipe == for_pipe) - pipe_width = hdisplay; + if (new_dbuf_state->weight[pipe] == 0) { + new_dbuf_state->ddb[pipe].start = 0; + new_dbuf_state->ddb[pipe].end = 0; + goto out; } - /* - * FIXME: For now we always enable slice S1 as per - * the Bspec display initialization sequence. - */ - new_dbuf_state->enabled_slices = total_slice_mask | BIT(DBUF_S1); + dbuf_slice_mask = new_dbuf_state->slices[pipe]; - if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) { - ret = intel_atomic_serialize_global_state(&new_dbuf_state->base); - if (ret) - return ret; - } + skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices); + ddb_range_size = skl_ddb_entry_size(&ddb_slices); + + intel_crtc_dbuf_weights(new_dbuf_state, pipe, + &weight_start, &weight_end, &weight_total); + + start = ddb_range_size * weight_start / weight_total; + end = ddb_range_size * weight_end / weight_total; - start = ddb_range_size * width_before_pipe_in_range / total_width_in_range; - end = ddb_range_size * - (width_before_pipe_in_range + pipe_width) / total_width_in_range; + new_dbuf_state->ddb[pipe].start = ddb_slices.start + start; + new_dbuf_state->ddb[pipe].end = ddb_slices.start + end; - alloc->start = offset + start; - alloc->end = offset + end; +out: + if (skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe], + &new_dbuf_state->ddb[pipe])) + return 0; + + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); + if (ret) + return ret; + + crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + crtc_state->wm.skl.ddb = new_dbuf_state->ddb[pipe]; drm_dbg_kms(&dev_priv->drm, - "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n", - for_crtc->base.id, for_crtc->name, - dbuf_slice_mask, alloc->start, alloc->end, active_pipes); + "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n", + crtc->base.base.id, crtc->base.name, + old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe], + old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end, + new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end, + old_dbuf_state->active_pipes, new_dbuf_state->active_pipes); return 0; } @@ -4307,12 +4265,12 @@ skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, /* Cursor doesn't support NV12/planar, so no extra calculation needed */ if (plane_id == PLANE_CURSOR) { - val = I915_READ(CUR_BUF_CFG(pipe)); + val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe)); skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); return; } - val = I915_READ(PLANE_CTL(pipe, plane_id)); + val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id)); /* No DDB allocated for disabled planes */ if (val & PLANE_CTL_ENABLE) @@ -4321,11 +4279,11 @@ skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, val & PLANE_CTL_ALPHA_MASK); if (INTEL_GEN(dev_priv) >= 11) { - val = I915_READ(PLANE_BUF_CFG(pipe, plane_id)); + val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id)); skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); } else { - val = I915_READ(PLANE_BUF_CFG(pipe, plane_id)); - val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id)); + val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id)); + val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id)); if (fourcc && drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc))) @@ -4639,10 +4597,8 @@ static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes) return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs); } -static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state, - u8 active_pipes) +static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; @@ -4704,50 +4660,63 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, } static u64 -skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, - u64 *plane_data_rate, - u64 *uv_plane_data_rate) +skl_get_total_relative_data_rate(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_plane *plane; + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); const struct intel_plane_state *plane_state; + struct intel_plane *plane; u64 total_data_rate = 0; + enum plane_id plane_id; + int i; /* Calculate and cache data rate for each plane */ - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { - enum plane_id plane_id = plane->id; - u64 rate; + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + if (plane->pipe != crtc->pipe) + continue; + + plane_id = plane->id; /* packed/y */ - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); - plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 0); /* uv-plane */ - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1); - uv_plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->uv_plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 1); + } + + for_each_plane_id_on_crtc(crtc, plane_id) { + total_data_rate += crtc_state->plane_data_rate[plane_id]; + total_data_rate += crtc_state->uv_plane_data_rate[plane_id]; } return total_data_rate; } static u64 -icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, - u64 *plane_data_rate) +icl_get_total_relative_data_rate(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_plane *plane; + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); const struct intel_plane_state *plane_state; + struct intel_plane *plane; u64 total_data_rate = 0; + enum plane_id plane_id; + int i; /* Calculate and cache data rate for each plane */ - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { - enum plane_id plane_id = plane->id; - u64 rate; + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + if (plane->pipe != crtc->pipe) + continue; + + plane_id = plane->id; if (!plane_state->planar_linked_plane) { - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); - plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 0); } else { enum plane_id y_plane_id; @@ -4762,17 +4731,18 @@ icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, continue; /* Y plane rate is calculated on the slave */ - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); y_plane_id = plane_state->planar_linked_plane->id; - plane_data_rate[y_plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[y_plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 0); - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1); - plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 1); } } + for_each_plane_id_on_crtc(crtc, plane_id) + total_data_rate += crtc_state->plane_data_rate[plane_id]; + return total_data_rate; } @@ -4791,71 +4761,37 @@ skl_plane_wm_level(const struct intel_crtc_state *crtc_state, } static int -skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) +skl_allocate_plane_ddb(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb; + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + const struct intel_dbuf_state *dbuf_state = + intel_atomic_get_new_dbuf_state(state); + const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe]; + int num_active = hweight8(dbuf_state->active_pipes); u16 alloc_size, start = 0; u16 total[I915_MAX_PLANES] = {}; u16 uv_total[I915_MAX_PLANES] = {}; u64 total_data_rate; enum plane_id plane_id; - int num_active; - u64 plane_data_rate[I915_MAX_PLANES] = {}; - u64 uv_plane_data_rate[I915_MAX_PLANES] = {}; u32 blocks; int level; - int ret; /* Clear the partitioning for disabled planes. */ memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y)); memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv)); - if (!crtc_state->hw.active) { - struct intel_atomic_state *state = - to_intel_atomic_state(crtc_state->uapi.state); - struct intel_dbuf_state *new_dbuf_state = - intel_atomic_get_new_dbuf_state(state); - const struct intel_dbuf_state *old_dbuf_state = - intel_atomic_get_old_dbuf_state(state); - - /* - * FIXME hack to make sure we compute this sensibly when - * turning off all the pipes. Otherwise we leave it at - * whatever we had previously, and then runtime PM will - * mess it up by turning off all but S1. Remove this - * once the dbuf state computation flow becomes sane. - */ - if (new_dbuf_state->active_pipes == 0) { - new_dbuf_state->enabled_slices = BIT(DBUF_S1); - - if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) { - ret = intel_atomic_serialize_global_state(&new_dbuf_state->base); - if (ret) - return ret; - } - } - - alloc->start = alloc->end = 0; + if (!crtc_state->hw.active) return 0; - } if (INTEL_GEN(dev_priv) >= 11) total_data_rate = - icl_get_total_relative_data_rate(crtc_state, - plane_data_rate); + icl_get_total_relative_data_rate(state, crtc); else total_data_rate = - skl_get_total_relative_data_rate(crtc_state, - plane_data_rate, - uv_plane_data_rate); - - ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, - total_data_rate, - alloc, &num_active); - if (ret) - return ret; + skl_get_total_relative_data_rate(state, crtc); alloc_size = skl_ddb_entry_size(alloc); if (alloc_size == 0) @@ -4930,7 +4866,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) if (total_data_rate == 0) break; - rate = plane_data_rate[plane_id]; + rate = crtc_state->plane_data_rate[plane_id]; extra = min_t(u16, alloc_size, DIV64_U64_ROUND_UP(alloc_size * rate, total_data_rate)); @@ -4941,7 +4877,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) if (total_data_rate == 0) break; - rate = uv_plane_data_rate[plane_id]; + rate = crtc_state->uv_plane_data_rate[plane_id]; extra = min_t(u16, alloc_size, DIV64_U64_ROUND_UP(alloc_size * rate, total_data_rate)); @@ -5093,36 +5029,12 @@ intel_get_linetime_us(const struct intel_crtc_state *crtc_state) if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0)) return u32_to_fixed16(0); - crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal; + crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal; linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate); return linetime_us; } -static u32 -skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - u64 adjusted_pixel_rate; - uint_fixed_16_16_t downscale_amount; - - /* Shouldn't reach here on disabled planes... */ - if (drm_WARN_ON(&dev_priv->drm, - !intel_wm_plane_visible(crtc_state, plane_state))) - return 0; - - /* - * Adjusted plane pixel rate is just the pipe's adjusted pixel rate - * with additional adjustments for plane-specific scaling. - */ - adjusted_pixel_rate = crtc_state->pixel_rate; - downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state); - - return mul_round_up_u32_fixed16(adjusted_pixel_rate, - downscale_amount); -} - static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, int width, const struct drm_format_info *format, @@ -5235,7 +5147,7 @@ skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state, return skl_compute_wm_params(crtc_state, width, fb->format, fb->modifier, plane_state->hw.rotation, - skl_adjusted_plane_pixel_rate(crtc_state, plane_state), + intel_plane_pixel_rate(crtc_state, plane_state), wp, color_plane); } @@ -5282,14 +5194,14 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate, wp->cpp, latency, wp->dbuf_block_size); method2 = skl_wm_method2(wp->plane_pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, latency, wp->plane_blocks_per_line); if (wp->y_tiled) { selected_result = max_fixed16(method2, wp->y_tile_minimum); } else { - if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal / + if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal / wp->dbuf_block_size < 1) && (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) { selected_result = method2; @@ -5373,6 +5285,9 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */ result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1; result->plane_en = true; + + if (INTEL_GEN(dev_priv) < 12) + result->can_sagv = latency >= dev_priv->sagv_block_time_us; } static void @@ -5478,7 +5393,7 @@ static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; struct skl_wm_params wm_params; int ret; @@ -5501,7 +5416,7 @@ static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, enum plane_id plane_id) { - struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; struct skl_wm_params wm_params; int ret; @@ -5522,10 +5437,13 @@ static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); - const struct drm_framebuffer *fb = plane_state->hw.fb; enum plane_id plane_id = plane->id; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; + const struct drm_framebuffer *fb = plane_state->hw.fb; int ret; + memset(wm, 0, sizeof(*wm)); + if (!intel_wm_plane_visible(crtc_state, plane_state)) return 0; @@ -5547,10 +5465,14 @@ static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id; + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + enum plane_id plane_id = plane->id; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; int ret; + memset(wm, 0, sizeof(*wm)); + /* Watermarks calculated in master */ if (plane_state->planar_slave) return 0; @@ -5583,22 +5505,24 @@ static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, return 0; } -static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state) +static int skl_build_pipe_wm(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; - struct intel_plane *plane; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); const struct intel_plane_state *plane_state; - int ret; - - /* - * We'll only calculate watermarks for planes that are actually - * enabled, so make sure all other planes are set as disabled. - */ - memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes)); + struct intel_plane *plane; + int ret, i; - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, - crtc_state) { + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + /* + * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc + * instead but we don't populate that correctly for NV12 Y + * planes so for now hack this. + */ + if (plane->pipe != crtc->pipe) + continue; if (INTEL_GEN(dev_priv) >= 11) ret = icl_build_plane_wm(crtc_state, plane_state); @@ -5608,6 +5532,8 @@ static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state) return ret; } + crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw; + return 0; } @@ -5737,6 +5663,18 @@ static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, return a->start < b->end && b->start < a->end; } +static void skl_ddb_entry_union(struct skl_ddb_entry *a, + const struct skl_ddb_entry *b) +{ + if (a->end && b->end) { + a->start = min(a->start, b->start); + a->end = max(a->end, b->end); + } else if (b->end) { + a->start = b->start; + a->end = b->end; + } +} + bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, const struct skl_ddb_entry *entries, int num_entries, int ignore_idx) @@ -5781,39 +5719,114 @@ skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state, return 0; } +static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state) +{ + struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev); + u8 enabled_slices; + enum pipe pipe; + + /* + * FIXME: For now we always enable slice S1 as per + * the Bspec display initialization sequence. + */ + enabled_slices = BIT(DBUF_S1); + + for_each_pipe(dev_priv, pipe) + enabled_slices |= dbuf_state->slices[pipe]; + + return enabled_slices; +} + static int skl_compute_ddb(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); const struct intel_dbuf_state *old_dbuf_state; - const struct intel_dbuf_state *new_dbuf_state; + struct intel_dbuf_state *new_dbuf_state = NULL; const struct intel_crtc_state *old_crtc_state; struct intel_crtc_state *new_crtc_state; struct intel_crtc *crtc; int ret, i; - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - ret = skl_allocate_pipe_ddb(new_crtc_state); + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + new_dbuf_state = intel_atomic_get_dbuf_state(state); + if (IS_ERR(new_dbuf_state)) + return PTR_ERR(new_dbuf_state); + + old_dbuf_state = intel_atomic_get_old_dbuf_state(state); + break; + } + + if (!new_dbuf_state) + return 0; + + new_dbuf_state->active_pipes = + intel_calc_active_pipes(state, old_dbuf_state->active_pipes); + + if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) { + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); if (ret) return ret; + } - ret = skl_ddb_add_affected_planes(old_crtc_state, - new_crtc_state); + for_each_intel_crtc(&dev_priv->drm, crtc) { + enum pipe pipe = crtc->pipe; + + new_dbuf_state->slices[pipe] = + skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes); + + if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe]) + continue; + + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); if (ret) return ret; } - old_dbuf_state = intel_atomic_get_old_dbuf_state(state); - new_dbuf_state = intel_atomic_get_new_dbuf_state(state); + new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state); + + if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) { + ret = intel_atomic_serialize_global_state(&new_dbuf_state->base); + if (ret) + return ret; - if (new_dbuf_state && - new_dbuf_state->enabled_slices != old_dbuf_state->enabled_slices) drm_dbg_kms(&dev_priv->drm, "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n", old_dbuf_state->enabled_slices, new_dbuf_state->enabled_slices, INTEL_INFO(dev_priv)->num_supported_dbuf_slices); + } + + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + enum pipe pipe = crtc->pipe; + + new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state); + + if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe]) + continue; + + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); + if (ret) + return ret; + } + + for_each_intel_crtc(&dev_priv->drm, crtc) { + ret = skl_crtc_allocate_ddb(state, crtc); + if (ret) + return ret; + } + + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + new_crtc_state, i) { + ret = skl_allocate_plane_ddb(state, crtc); + if (ret) + return ret; + + ret = skl_ddb_add_affected_planes(old_crtc_state, + new_crtc_state); + if (ret) + return ret; + } return 0; } @@ -5950,83 +5963,6 @@ skl_print_wm_changes(struct intel_atomic_state *state) } } -static int intel_add_affected_pipes(struct intel_atomic_state *state, - u8 pipe_mask) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_crtc *crtc; - - for_each_intel_crtc(&dev_priv->drm, crtc) { - struct intel_crtc_state *crtc_state; - - if ((pipe_mask & BIT(crtc->pipe)) == 0) - continue; - - crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); - } - - return 0; -} - -static int -skl_ddb_add_affected_pipes(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - int i, ret; - - if (dev_priv->wm.distrust_bios_wm) { - /* - * skl_ddb_get_pipe_allocation_limits() currently requires - * all active pipes to be included in the state so that - * it can redistribute the dbuf among them, and it really - * wants to recompute things when distrust_bios_wm is set - * so we add all the pipes to the state. - */ - ret = intel_add_affected_pipes(state, ~0); - if (ret) - return ret; - } - - for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { - struct intel_dbuf_state *new_dbuf_state; - const struct intel_dbuf_state *old_dbuf_state; - - new_dbuf_state = intel_atomic_get_dbuf_state(state); - if (IS_ERR(new_dbuf_state)) - return PTR_ERR(new_dbuf_state); - - old_dbuf_state = intel_atomic_get_old_dbuf_state(state); - - new_dbuf_state->active_pipes = - intel_calc_active_pipes(state, old_dbuf_state->active_pipes); - - if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes) - break; - - ret = intel_atomic_lock_global_state(&new_dbuf_state->base); - if (ret) - return ret; - - /* - * skl_ddb_get_pipe_allocation_limits() currently requires - * all active pipes to be included in the state so that - * it can redistribute the dbuf among them. - */ - ret = intel_add_affected_pipes(state, - new_dbuf_state->active_pipes); - if (ret) - return ret; - - break; - } - - return 0; -} - /* * To make sure the cursor watermark registers are always consistent * with our computed state the following scenario needs special @@ -6092,21 +6028,10 @@ skl_compute_wm(struct intel_atomic_state *state) { struct intel_crtc *crtc; struct intel_crtc_state *new_crtc_state; - struct intel_crtc_state *old_crtc_state; int ret, i; - ret = skl_ddb_add_affected_pipes(state); - if (ret) - return ret; - - /* - * Calculate WM's for all pipes that are part of this transaction. - * Note that skl_ddb_add_affected_pipes may have added more CRTC's that - * weren't otherwise being modified if pipe allocations had to change. - */ - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - ret = skl_build_pipe_wm(new_crtc_state); + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + ret = skl_build_pipe_wm(state, crtc); if (ret) return ret; } @@ -6124,8 +6049,7 @@ skl_compute_wm(struct intel_atomic_state *state) * based on how much ddb is available. Now we can actually * check if the final watermarks changed. */ - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { ret = skl_wm_add_affected_planes(state, crtc); if (ret) return ret; @@ -6240,9 +6164,9 @@ void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, for (level = 0; level <= max_level; level++) { if (plane_id != PLANE_CURSOR) - val = I915_READ(PLANE_WM(pipe, plane_id, level)); + val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level)); else - val = I915_READ(CUR_WM(pipe, level)); + val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level)); skl_wm_level_from_reg_val(val, &wm->wm[level]); } @@ -6251,9 +6175,9 @@ void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, wm->sagv_wm0 = wm->wm[0]; if (plane_id != PLANE_CURSOR) - val = I915_READ(PLANE_WM_TRANS(pipe, plane_id)); + val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id)); else - val = I915_READ(CUR_WM_TRANS(pipe)); + val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe)); skl_wm_level_from_reg_val(val, &wm->trans_wm); } @@ -6264,19 +6188,49 @@ void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, void skl_wm_get_hw_state(struct drm_i915_private *dev_priv) { + struct intel_dbuf_state *dbuf_state = + to_intel_dbuf_state(dev_priv->dbuf.obj.state); struct intel_crtc *crtc; - struct intel_crtc_state *crtc_state; for_each_intel_crtc(&dev_priv->drm, crtc) { - crtc_state = to_intel_crtc_state(crtc->base.state); + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + enum pipe pipe = crtc->pipe; + enum plane_id plane_id; skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal); - } + crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal; - if (dev_priv->active_pipes) { - /* Fully recompute DDB on first atomic commit */ - dev_priv->wm.distrust_bios_wm = true; + memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe])); + + for_each_plane_id_on_crtc(crtc, plane_id) { + struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + struct skl_ddb_entry *ddb_uv = + &crtc_state->wm.skl.plane_ddb_uv[plane_id]; + + skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe, + plane_id, ddb_y, ddb_uv); + + skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y); + skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv); + } + + dbuf_state->slices[pipe] = + skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes); + + dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state); + + crtc_state->wm.skl.ddb = dbuf_state->ddb[pipe]; + + drm_dbg_kms(&dev_priv->drm, + "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n", + crtc->base.base.id, crtc->base.name, + dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start, + dbuf_state->ddb[pipe].end, dbuf_state->active_pipes); } + + dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices; } static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) @@ -6287,13 +6241,8 @@ static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal; enum pipe pipe = crtc->pipe; - static const i915_reg_t wm0_pipe_reg[] = { - [PIPE_A] = WM0_PIPEA_ILK, - [PIPE_B] = WM0_PIPEB_ILK, - [PIPE_C] = WM0_PIPEC_IVB, - }; - hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]); + hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe)); memset(active, 0, sizeof(*active)); @@ -6337,13 +6286,13 @@ static void g4x_read_wm_values(struct drm_i915_private *dev_priv, { u32 tmp; - tmp = I915_READ(DSPFW1); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); wm->sr.plane = _FW_WM(tmp, SR); wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB); wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA); - tmp = I915_READ(DSPFW2); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); wm->fbc_en = tmp & DSPFW_FBC_SR_EN; wm->sr.fbc = _FW_WM(tmp, FBC_SR); wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR); @@ -6351,7 +6300,7 @@ static void g4x_read_wm_values(struct drm_i915_private *dev_priv, wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA); - tmp = I915_READ(DSPFW3); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); wm->hpll_en = tmp & DSPFW_HPLL_SR_EN; wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR); @@ -6365,7 +6314,7 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv, u32 tmp; for_each_pipe(dev_priv, pipe) { - tmp = I915_READ(VLV_DDL(pipe)); + tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe)); wm->ddl[pipe].plane[PLANE_PRIMARY] = (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); @@ -6377,34 +6326,34 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv, (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); } - tmp = I915_READ(DSPFW1); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); wm->sr.plane = _FW_WM(tmp, SR); wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB); wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA); - tmp = I915_READ(DSPFW2); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB); wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA); - tmp = I915_READ(DSPFW3); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); if (IS_CHERRYVIEW(dev_priv)) { - tmp = I915_READ(DSPFW7_CHV); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV); wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); - tmp = I915_READ(DSPFW8_CHV); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV); wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF); wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE); - tmp = I915_READ(DSPFW9_CHV); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV); wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC); wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC); - tmp = I915_READ(DSPHOWM); + tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8; wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8; @@ -6416,11 +6365,11 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv, wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; } else { - tmp = I915_READ(DSPFW7); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7); wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); - tmp = I915_READ(DSPHOWM); + tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; @@ -6441,7 +6390,7 @@ void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) g4x_read_wm_values(dev_priv, wm); - wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; + wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; for_each_intel_crtc(&dev_priv->drm, crtc) { struct intel_crtc_state *crtc_state = @@ -6585,7 +6534,7 @@ void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv) vlv_read_wm_values(dev_priv, wm); - wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; + wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; wm->level = VLV_WM_LEVEL_PM2; if (IS_CHERRYVIEW(dev_priv)) { @@ -6732,9 +6681,9 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv) */ static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) { - I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); - I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); - I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN); + intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN); + intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN); + intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN); /* * Don't touch WM1S_LP_EN here. @@ -6752,25 +6701,25 @@ void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) for_each_intel_crtc(&dev_priv->drm, crtc) ilk_pipe_wm_get_hw_state(crtc); - hw->wm_lp[0] = I915_READ(WM1_LP_ILK); - hw->wm_lp[1] = I915_READ(WM2_LP_ILK); - hw->wm_lp[2] = I915_READ(WM3_LP_ILK); + hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK); + hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK); + hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK); - hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK); + hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK); if (INTEL_GEN(dev_priv) >= 7) { - hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); - hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); + hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB); + hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB); } if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? + hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; else if (IS_IVYBRIDGE(dev_priv)) - hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ? + hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ? INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; hw->enable_fbc_wm = - !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS); + !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS); } /** @@ -6821,14 +6770,14 @@ void intel_enable_ipc(struct drm_i915_private *dev_priv) if (!HAS_IPC(dev_priv)) return; - val = I915_READ(DISP_ARB_CTL2); + val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2); if (dev_priv->ipc_enabled) val |= DISP_IPC_ENABLE; else val &= ~DISP_IPC_ENABLE; - I915_WRITE(DISP_ARB_CTL2, val); + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val); } static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv) @@ -6863,7 +6812,7 @@ static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) * gating for the panel power sequencer or it will fail to * start up when no ports are active. */ - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); + intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); } static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) @@ -6871,12 +6820,12 @@ static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) enum pipe pipe; for_each_pipe(dev_priv, pipe) { - I915_WRITE(DSPCNTR(pipe), - I915_READ(DSPCNTR(pipe)) | + intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe), + intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) | DISPPLANE_TRICKLE_FEED_DISABLE); - I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe))); - POSTING_READ(DSPSURF(pipe)); + intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe))); + intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe)); } } @@ -6892,10 +6841,10 @@ static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) ILK_DPFCUNIT_CLOCK_GATE_DISABLE | ILK_DPFDUNIT_CLOCK_GATE_ENABLE; - I915_WRITE(PCH_3DCGDIS0, + intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0, MARIUNIT_CLOCK_GATE_DISABLE | SVSMUNIT_CLOCK_GATE_DISABLE); - I915_WRITE(PCH_3DCGDIS1, + intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1, VFMUNIT_CLOCK_GATE_DISABLE); /* @@ -6905,12 +6854,12 @@ static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) * The bit 5 of 0x42020 * The bit 15 of 0x45000 */ - I915_WRITE(ILK_DISPLAY_CHICKEN2, - (I915_READ(ILK_DISPLAY_CHICKEN2) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, + (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) | ILK_DPARB_GATE | ILK_VSDPFD_FULL)); dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; - I915_WRITE(DISP_ARB_CTL, - (I915_READ(DISP_ARB_CTL) | + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, + (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | DISP_FBC_WM_DIS)); /* @@ -6922,18 +6871,18 @@ static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) */ if (IS_IRONLAKE_M(dev_priv)) { /* WaFbcAsynchFlipDisableFbcQueue:ilk */ - I915_WRITE(ILK_DISPLAY_CHICKEN1, - I915_READ(ILK_DISPLAY_CHICKEN1) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, + intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) | ILK_FBCQ_DIS); - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, + intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) | ILK_DPARB_GATE); } - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); + intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate); - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, + intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) | ILK_ELPIN_409_SELECT); g4x_disable_trickle_feed(dev_priv); @@ -6951,27 +6900,27 @@ static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) * gating for the panel power sequencer or it will fail to * start up when no ports are active. */ - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | + intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | PCH_DPLUNIT_CLOCK_GATE_DISABLE | PCH_CPUNIT_CLOCK_GATE_DISABLE); - I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | + intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) | DPLS_EDP_PPS_FIX_DIS); /* The below fixes the weird display corruption, a few pixels shifted * downward, on (only) LVDS of some HP laptops with IVY. */ for_each_pipe(dev_priv, pipe) { - val = I915_READ(TRANS_CHICKEN2(pipe)); + val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe)); val |= TRANS_CHICKEN2_TIMING_OVERRIDE; val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; if (dev_priv->vbt.fdi_rx_polarity_inverted) val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; - I915_WRITE(TRANS_CHICKEN2(pipe), val); + intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val); } /* WADP0ClockGatingDisable */ for_each_pipe(dev_priv, pipe) { - I915_WRITE(TRANS_CHICKEN1(pipe), + intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe), TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); } } @@ -6980,7 +6929,7 @@ static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) { u32 tmp; - tmp = I915_READ(MCH_SSKPD); + tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD); if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) drm_dbg_kms(&dev_priv->drm, "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", @@ -6991,14 +6940,14 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) { u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); + intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate); - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, + intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) | ILK_ELPIN_409_SELECT); - I915_WRITE(GEN6_UCGCTL1, - I915_READ(GEN6_UCGCTL1) | + intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, + intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_BLBUNIT_CLOCK_GATE_DISABLE | GEN6_CSUNIT_CLOCK_GATE_DISABLE); @@ -7015,7 +6964,7 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) * WaDisableRCCUnitClockGating:snb * WaDisableRCPBUnitClockGating:snb */ - I915_WRITE(GEN6_UCGCTL2, + intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2, GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | GEN6_RCCUNIT_CLOCK_GATE_DISABLE); @@ -7030,14 +6979,14 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) * * WaFbcAsynchFlipDisableFbcQueue:snb */ - I915_WRITE(ILK_DISPLAY_CHICKEN1, - I915_READ(ILK_DISPLAY_CHICKEN1) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, + intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) | ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, + intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) | ILK_DPARB_GATE | ILK_VSDPFD_FULL); - I915_WRITE(ILK_DSPCLK_GATE_D, - I915_READ(ILK_DSPCLK_GATE_D) | + intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, + intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) | ILK_DPARBUNIT_CLOCK_GATE_ENABLE | ILK_DPFDUNIT_CLOCK_GATE_ENABLE); @@ -7055,23 +7004,23 @@ static void lpt_init_clock_gating(struct drm_i915_private *dev_priv) * disabled when not needed anymore in order to save power. */ if (HAS_PCH_LPT_LP(dev_priv)) - I915_WRITE(SOUTH_DSPCLK_GATE_D, - I915_READ(SOUTH_DSPCLK_GATE_D) | + intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, + intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) | PCH_LP_PARTITION_LEVEL_DISABLE); /* WADPOClockGatingDisable:hsw */ - I915_WRITE(TRANS_CHICKEN1(PIPE_A), - I915_READ(TRANS_CHICKEN1(PIPE_A)) | + intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A), + intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) | TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); } static void lpt_suspend_hw(struct drm_i915_private *dev_priv) { if (HAS_PCH_LPT_LP(dev_priv)) { - u32 val = I915_READ(SOUTH_DSPCLK_GATE_D); + u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D); val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; - I915_WRITE(SOUTH_DSPCLK_GATE_D, val); + intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val); } } @@ -7083,75 +7032,72 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv, u32 val; /* WaTempDisableDOPClkGating:bdw */ - misccpctl = I915_READ(GEN7_MISCCPCTL); - I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); + misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL); + intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); - val = I915_READ(GEN8_L3SQCREG1); + val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1); val &= ~L3_PRIO_CREDITS_MASK; val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits); val |= L3_HIGH_PRIO_CREDITS(high_prio_credits); - I915_WRITE(GEN8_L3SQCREG1, val); + intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val); /* * Wait at least 100 clocks before re-enabling clock gating. * See the definition of L3SQCREG1 in BSpec. */ - POSTING_READ(GEN8_L3SQCREG1); + intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1); udelay(1); - I915_WRITE(GEN7_MISCCPCTL, misccpctl); + intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl); } static void icl_init_clock_gating(struct drm_i915_private *dev_priv) { /* Wa_1409120013:icl,ehl */ - I915_WRITE(ILK_DPFC_CHICKEN, + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL); /* This is not an Wa. Enable to reduce Sampler power */ - I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN, - I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE); + intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN, + intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE); /*Wa_14010594013:icl, ehl */ intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, CNL_DELAY_PMRSP); } -static void tgl_init_clock_gating(struct drm_i915_private *dev_priv) +static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv) { - u32 vd_pg_enable = 0; - unsigned int i; - - /* Wa_1409120013:tgl */ - I915_WRITE(ILK_DPFC_CHICKEN, - ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL); - - /* This is not a WA. Enable VD HCP & MFX_ENC powergate */ - for (i = 0; i < I915_MAX_VCS; i++) { - if (HAS_ENGINE(&dev_priv->gt, _VCS(i))) - vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) | - VDN_MFX_POWERGATE_ENABLE(i); - } - - I915_WRITE(POWERGATE_ENABLE, - I915_READ(POWERGATE_ENABLE) | vd_pg_enable); + /* Wa_1409120013:tgl,rkl,adl_s,dg1 */ + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, + ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL); /* Wa_1409825376:tgl (pre-prod)*/ if (IS_TGL_DISP_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_B1)) - I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) | + intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) | TGL_VRH_GATING_DIS); - /* Wa_14011059788:tgl */ + /* Wa_14011059788:tgl,rkl,adl_s,dg1 */ intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN, 0, DFR_DISABLE); } +static void dg1_init_clock_gating(struct drm_i915_private *dev_priv) +{ + gen12lp_init_clock_gating(dev_priv); + + /* Wa_1409836686:dg1[a0] */ + if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0)) + intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) | + DPT_GATING_DIS); +} + static void cnp_init_clock_gating(struct drm_i915_private *dev_priv) { if (!HAS_PCH_CNP(dev_priv)) return; /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */ - I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) | + intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) | CNP_PWM_CGE_GATING_DISABLE); } @@ -7161,35 +7107,35 @@ static void cnl_init_clock_gating(struct drm_i915_private *dev_priv) cnp_init_clock_gating(dev_priv); /* This is not an Wa. Enable for better image quality */ - I915_WRITE(_3D_CHICKEN3, + intel_uncore_write(&dev_priv->uncore, _3D_CHICKEN3, _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE)); /* WaEnableChickenDCPR:cnl */ - I915_WRITE(GEN8_CHICKEN_DCPR_1, - I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); + intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, + intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); /* * WaFbcWakeMemOn:cnl * Display WA #0859: cnl */ - I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | DISP_FBC_MEMORY_WAKE); - val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE); + val = intel_uncore_read(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE); /* ReadHitWriteOnlyDisable:cnl */ val |= RCCUNIT_CLKGATE_DIS; - I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val); + intel_uncore_write(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE, val); /* Wa_2201832410:cnl */ - val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE); + val = intel_uncore_read(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE); val |= GWUNIT_CLKGATE_DIS; - I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val); + intel_uncore_write(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE, val); /* WaDisableVFclkgate:cnl */ /* WaVFUnitClockGatingDisable:cnl */ - val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE); + val = intel_uncore_read(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE); val |= VFUNIT_CLKGATE_DIS; - I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val); + intel_uncore_write(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, val); } static void cfl_init_clock_gating(struct drm_i915_private *dev_priv) @@ -7197,18 +7143,22 @@ static void cfl_init_clock_gating(struct drm_i915_private *dev_priv) cnp_init_clock_gating(dev_priv); gen9_init_clock_gating(dev_priv); + /* WAC6entrylatency:cfl */ + intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) | + FBC_LLC_FULLY_OPEN); + /* * WaFbcTurnOffFbcWatermark:cfl * Display WA #0562: cfl */ - I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | DISP_FBC_WM_DIS); /* * WaFbcNukeOnHostModify:cfl * Display WA #0873: cfl */ - I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | ILK_DPFC_NUKE_ON_ANY_MODIFICATION); } @@ -7216,28 +7166,32 @@ static void kbl_init_clock_gating(struct drm_i915_private *dev_priv) { gen9_init_clock_gating(dev_priv); + /* WAC6entrylatency:kbl */ + intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) | + FBC_LLC_FULLY_OPEN); + /* WaDisableSDEUnitClockGating:kbl */ if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0)) - I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* WaDisableGamClockGating:kbl */ if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0)) - I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | + intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_GAMUNIT_CLOCK_GATE_DISABLE); /* * WaFbcTurnOffFbcWatermark:kbl * Display WA #0562: kbl */ - I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | DISP_FBC_WM_DIS); /* * WaFbcNukeOnHostModify:kbl * Display WA #0873: kbl */ - I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | ILK_DPFC_NUKE_ON_ANY_MODIFICATION); } @@ -7246,32 +7200,32 @@ static void skl_init_clock_gating(struct drm_i915_private *dev_priv) gen9_init_clock_gating(dev_priv); /* WaDisableDopClockGating:skl */ - I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL) & + intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) & ~GEN7_DOP_CLOCK_GATE_ENABLE); /* WAC6entrylatency:skl */ - I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) | + intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) | FBC_LLC_FULLY_OPEN); /* * WaFbcTurnOffFbcWatermark:skl * Display WA #0562: skl */ - I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | DISP_FBC_WM_DIS); /* * WaFbcNukeOnHostModify:skl * Display WA #0873: skl */ - I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | ILK_DPFC_NUKE_ON_ANY_MODIFICATION); /* * WaFbcHighMemBwCorruptionAvoidance:skl * Display WA #0883: skl */ - I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | ILK_DPFC_DISABLE_DUMMY0); } @@ -7280,42 +7234,42 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) enum pipe pipe; /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */ - I915_WRITE(CHICKEN_PIPESL_1(PIPE_A), - I915_READ(CHICKEN_PIPESL_1(PIPE_A)) | + intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), + intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) | HSW_FBCQ_DIS); /* WaSwitchSolVfFArbitrationPriority:bdw */ - I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); + intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); /* WaPsrDPAMaskVBlankInSRD:bdw */ - I915_WRITE(CHICKEN_PAR1_1, - I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); + intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1, + intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ for_each_pipe(dev_priv, pipe) { - I915_WRITE(CHICKEN_PIPESL_1(pipe), - I915_READ(CHICKEN_PIPESL_1(pipe)) | + intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe), + intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) | BDW_DPRS_MASK_VBLANK_SRD); } /* WaVSRefCountFullforceMissDisable:bdw */ /* WaDSRefCountFullforceMissDisable:bdw */ - I915_WRITE(GEN7_FF_THREAD_MODE, - I915_READ(GEN7_FF_THREAD_MODE) & + intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE, + intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) & ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); - I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, + intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL, _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); /* WaDisableSDEUnitClockGating:bdw */ - I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* WaProgramL3SqcReg1Default:bdw */ gen8_set_l3sqc_credits(dev_priv, 30, 2); /* WaKVMNotificationOnConfigChange:bdw */ - I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1) + intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1) | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT); lpt_init_clock_gating(dev_priv); @@ -7325,24 +7279,24 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP * clock gating. */ - I915_WRITE(GEN6_UCGCTL1, - I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE); + intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, + intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE); } static void hsw_init_clock_gating(struct drm_i915_private *dev_priv) { /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */ - I915_WRITE(CHICKEN_PIPESL_1(PIPE_A), - I915_READ(CHICKEN_PIPESL_1(PIPE_A)) | + intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), + intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) | HSW_FBCQ_DIS); /* This is required by WaCatErrorRejectionIssue:hsw */ - I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, - I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | + intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, + intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); /* WaSwitchSolVfFArbitrationPriority:hsw */ - I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); + intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); lpt_init_clock_gating(dev_priv); } @@ -7351,26 +7305,26 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) { u32 snpcr; - I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); + intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); /* WaFbcAsynchFlipDisableFbcQueue:ivb */ - I915_WRITE(ILK_DISPLAY_CHICKEN1, - I915_READ(ILK_DISPLAY_CHICKEN1) | + intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, + intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) | ILK_FBCQ_DIS); /* WaDisableBackToBackFlipFix:ivb */ - I915_WRITE(IVB_CHICKEN3, + intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3, CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | CHICKEN3_DGMG_DONE_FIX_DISABLE); if (IS_IVB_GT1(dev_priv)) - I915_WRITE(GEN7_ROW_CHICKEN2, + intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); else { /* must write both registers */ - I915_WRITE(GEN7_ROW_CHICKEN2, + intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - I915_WRITE(GEN7_ROW_CHICKEN2_GT2, + intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); } @@ -7378,20 +7332,20 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:ivb workaround. */ - I915_WRITE(GEN6_UCGCTL2, + intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE); /* This is required by WaCatErrorRejectionIssue:ivb */ - I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, - I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | + intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, + intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); g4x_disable_trickle_feed(dev_priv); - snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); + snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR); snpcr &= ~GEN6_MBC_SNPCR_MASK; snpcr |= GEN6_MBC_SNPCR_MED; - I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); + intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr); if (!HAS_PCH_NOP(dev_priv)) cpt_init_clock_gating(dev_priv); @@ -7402,58 +7356,58 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) static void vlv_init_clock_gating(struct drm_i915_private *dev_priv) { /* WaDisableBackToBackFlipFix:vlv */ - I915_WRITE(IVB_CHICKEN3, + intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3, CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | CHICKEN3_DGMG_DONE_FIX_DISABLE); /* WaDisableDopClockGating:vlv */ - I915_WRITE(GEN7_ROW_CHICKEN2, + intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); /* This is required by WaCatErrorRejectionIssue:vlv */ - I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, - I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | + intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, + intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:vlv workaround. */ - I915_WRITE(GEN6_UCGCTL2, + intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE); /* WaDisableL3Bank2xClockGate:vlv * Disabling L3 clock gating- MMIO 940c[25] = 1 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */ - I915_WRITE(GEN7_UCGCTL4, - I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE); + intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4, + intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE); /* * WaDisableVLVClockGating_VBIIssue:vlv * Disable clock gating on th GCFG unit to prevent a delay * in the reporting of vblank events. */ - I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); + intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS); } static void chv_init_clock_gating(struct drm_i915_private *dev_priv) { /* WaVSRefCountFullforceMissDisable:chv */ /* WaDSRefCountFullforceMissDisable:chv */ - I915_WRITE(GEN7_FF_THREAD_MODE, - I915_READ(GEN7_FF_THREAD_MODE) & + intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE, + intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) & ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); /* WaDisableSemaphoreAndSyncFlipWait:chv */ - I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, + intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL, _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); /* WaDisableCSUnitClockGating:chv */ - I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | + intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_CSUNIT_CLOCK_GATE_DISABLE); /* WaDisableSDEUnitClockGating:chv */ - I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* @@ -7468,17 +7422,17 @@ static void g4x_init_clock_gating(struct drm_i915_private *dev_priv) { u32 dspclk_gate; - I915_WRITE(RENCLK_GATE_D1, 0); - I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE | + intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0); + intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE | GS_UNIT_CLOCK_GATE_DISABLE | CL_UNIT_CLOCK_GATE_DISABLE); - I915_WRITE(RAMCLK_GATE_D, 0); + intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0); dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE | OVRUNIT_CLOCK_GATE_DISABLE | OVCUNIT_CLOCK_GATE_DISABLE; if (IS_GM45(dev_priv)) dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE; - I915_WRITE(DSPCLK_GATE_D, dspclk_gate); + intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate); g4x_disable_trickle_feed(dev_priv); } @@ -7499,49 +7453,49 @@ static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv) static void i965g_init_clock_gating(struct drm_i915_private *dev_priv) { - I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | + intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | I965_RCC_CLOCK_GATE_DISABLE | I965_RCPB_CLOCK_GATE_DISABLE | I965_ISC_CLOCK_GATE_DISABLE | I965_FBC_CLOCK_GATE_DISABLE); - I915_WRITE(RENCLK_GATE_D2, 0); - I915_WRITE(MI_ARB_STATE, + intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0); + intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); } static void gen3_init_clock_gating(struct drm_i915_private *dev_priv) { - u32 dstate = I915_READ(D_STATE); + u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE); dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | DSTATE_DOT_CLOCK_GATING; - I915_WRITE(D_STATE, dstate); + intel_uncore_write(&dev_priv->uncore, D_STATE, dstate); if (IS_PINEVIEW(dev_priv)) - I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); + intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); /* IIR "flip pending" means done if this bit is set */ - I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); + intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); /* interrupts should cause a wake up from C3 */ - I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN)); + intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN)); /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ - I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); + intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); - I915_WRITE(MI_ARB_STATE, + intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); } static void i85x_init_clock_gating(struct drm_i915_private *dev_priv) { - I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); + intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); /* interrupts should cause a wake up from C3 */ - I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | + intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE)); - I915_WRITE(MEM_MODE, + intel_uncore_write(&dev_priv->uncore, MEM_MODE, _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); /* @@ -7551,13 +7505,13 @@ static void i85x_init_clock_gating(struct drm_i915_private *dev_priv) * abosultely nothing) would not allow FBC to recompress * until a 2D blit occurs. */ - I915_WRITE(SCPD0, + intel_uncore_write(&dev_priv->uncore, SCPD0, _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D)); } static void i830_init_clock_gating(struct drm_i915_private *dev_priv) { - I915_WRITE(MEM_MODE, + intel_uncore_write(&dev_priv->uncore, MEM_MODE, _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) | _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); } @@ -7590,8 +7544,10 @@ static void nop_init_clock_gating(struct drm_i915_private *dev_priv) */ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) { - if (IS_GEN(dev_priv, 12)) - dev_priv->display.init_clock_gating = tgl_init_clock_gating; + if (IS_DG1(dev_priv)) + dev_priv->display.init_clock_gating = dg1_init_clock_gating; + else if (IS_GEN(dev_priv, 12)) + dev_priv->display.init_clock_gating = gen12lp_init_clock_gating; else if (IS_GEN(dev_priv, 11)) dev_priv->display.init_clock_gating = icl_init_clock_gating; else if (IS_CANNONLAKE(dev_priv)) |