diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_device.c')
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1185 |
1 files changed, 930 insertions, 255 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 66cb10cdc7c3..39bf2ce548c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -25,6 +25,7 @@ * Alex Deucher * Jerome Glisse */ +#include <linux/power_supply.h> #include <linux/kthread.h> #include <linux/console.h> #include <linux/slab.h> @@ -59,6 +60,7 @@ #include "amdgpu_pm.h" MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); +MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); #define AMDGPU_RESUME_MS 2000 @@ -82,11 +84,24 @@ static const char *amdgpu_asic_name[] = { "POLARIS10", "POLARIS11", "POLARIS12", + "VEGAM", "VEGA10", + "VEGA12", + "VEGA20", "RAVEN", "LAST", }; +static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); + +/** + * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control + * + * @dev: drm_device pointer + * + * Returns true if the device is a dGPU with HG/PX power control, + * otherwise return false. + */ bool amdgpu_device_is_px(struct drm_device *dev) { struct amdgpu_device *adev = dev->dev_private; @@ -99,6 +114,15 @@ bool amdgpu_device_is_px(struct drm_device *dev) /* * MMIO register access helper functions. */ +/** + * amdgpu_mm_rreg - read a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * @acc_flags: access flags which require special behavior + * + * Returns the 32 bit value from the offset specified. + */ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t acc_flags) { @@ -121,6 +145,58 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, return ret; } +/* + * MMIO register read with bytes helper functions + * @offset:bytes offset from MMIO start + * +*/ + +/** + * amdgpu_mm_rreg8 - read a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @offset: byte aligned register offset + * + * Returns the 8 bit value from the offset specified. + */ +uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) { + if (offset < adev->rmmio_size) + return (readb(adev->rmmio + offset)); + BUG(); +} + +/* + * MMIO register write with bytes helper functions + * @offset:bytes offset from MMIO start + * @value: the value want to be written to the register + * +*/ +/** + * amdgpu_mm_wreg8 - read a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @offset: byte aligned register offset + * @value: 8 bit value to write + * + * Writes the value specified to the offset specified. + */ +void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) { + if (offset < adev->rmmio_size) + writeb(value, adev->rmmio + offset); + else + BUG(); +} + +/** + * amdgpu_mm_wreg - write to a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * @v: 32 bit value to write to the register + * @acc_flags: access flags which require special behavior + * + * Writes the value specified to the offset specified. + */ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t acc_flags) { @@ -149,6 +225,14 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, } } +/** + * amdgpu_io_rreg - read an IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * + * Returns the 32 bit value from the offset specified. + */ u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg) { if ((reg * 4) < adev->rio_mem_size) @@ -159,6 +243,15 @@ u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg) } } +/** + * amdgpu_io_wreg - write to an IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * @v: 32 bit value to write to the register + * + * Writes the value specified to the offset specified. + */ void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v) { if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { @@ -327,6 +420,14 @@ static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, BUG(); } +/** + * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page + * + * @adev: amdgpu device pointer + * + * Allocates a scratch page of VRAM for use by various things in the + * driver. + */ static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) { return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, @@ -336,6 +437,13 @@ static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) (void **)&adev->vram_scratch.ptr); } +/** + * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page + * + * @adev: amdgpu device pointer + * + * Frees the VRAM scratch page. + */ static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) { amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); @@ -377,6 +485,14 @@ void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, } } +/** + * amdgpu_device_pci_config_reset - reset the GPU + * + * @adev: amdgpu_device pointer + * + * Resets the GPU using the pci config reset sequence. + * Only applicable to asics prior to vega10. + */ void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) { pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); @@ -537,6 +653,7 @@ void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) /** * amdgpu_device_vram_location - try to find VRAM location + * * @adev: amdgpu device structure holding all necessary informations * @mc: memory controller structure holding memory informations * @base: base address at which to put VRAM @@ -545,7 +662,7 @@ void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) * as parameter. */ void amdgpu_device_vram_location(struct amdgpu_device *adev, - struct amdgpu_mc *mc, u64 base) + struct amdgpu_gmc *mc, u64 base) { uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; @@ -559,33 +676,34 @@ void amdgpu_device_vram_location(struct amdgpu_device *adev, } /** - * amdgpu_device_gart_location - try to find GTT location + * amdgpu_device_gart_location - try to find GART location + * * @adev: amdgpu device structure holding all necessary informations * @mc: memory controller structure holding memory informations * - * Function will place try to place GTT before or after VRAM. + * Function will place try to place GART before or after VRAM. * - * If GTT size is bigger than space left then we ajust GTT size. + * If GART size is bigger than space left then we ajust GART size. * Thus function will never fails. - * - * FIXME: when reducing GTT size align new size on power of 2. */ void amdgpu_device_gart_location(struct amdgpu_device *adev, - struct amdgpu_mc *mc) + struct amdgpu_gmc *mc) { u64 size_af, size_bf; - size_af = adev->mc.mc_mask - mc->vram_end; + mc->gart_size += adev->pm.smu_prv_buffer_size; + + size_af = adev->gmc.mc_mask - mc->vram_end; size_bf = mc->vram_start; if (size_bf > size_af) { if (mc->gart_size > size_bf) { - dev_warn(adev->dev, "limiting GTT\n"); + dev_warn(adev->dev, "limiting GART\n"); mc->gart_size = size_bf; } mc->gart_start = 0; } else { if (mc->gart_size > size_af) { - dev_warn(adev->dev, "limiting GTT\n"); + dev_warn(adev->dev, "limiting GART\n"); mc->gart_size = size_af; } /* VCE doesn't like it when BOs cross a 4GB segment, so align @@ -594,7 +712,7 @@ void amdgpu_device_gart_location(struct amdgpu_device *adev, mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL); } mc->gart_end = mc->gart_start + mc->gart_size - 1; - dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n", + dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", mc->gart_size >> 20, mc->gart_start, mc->gart_end); } @@ -609,7 +727,7 @@ void amdgpu_device_gart_location(struct amdgpu_device *adev, */ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) { - u64 space_needed = roundup_pow_of_two(adev->mc.real_vram_size); + u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1; struct pci_bus *root; struct resource *res; @@ -746,6 +864,16 @@ static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state) return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; } +/** + * amdgpu_device_check_block_size - validate the vm block size + * + * @adev: amdgpu_device pointer + * + * Validates the vm block size specified via module parameter. + * The vm block size defines number of bits in page table versus page directory, + * a page is 4KB so we have 12 bits offset, minimum 9 bits in the + * page table and the remaining bits are in the page directory. + */ static void amdgpu_device_check_block_size(struct amdgpu_device *adev) { /* defines number of bits in page table versus page directory, @@ -761,6 +889,14 @@ static void amdgpu_device_check_block_size(struct amdgpu_device *adev) } } +/** + * amdgpu_device_check_vm_size - validate the vm size + * + * @adev: amdgpu_device pointer + * + * Validates the vm size in GB specified via module parameter. + * The VM size is the size of the GPU virtual memory space in GB. + */ static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) { /* no need to check the default value */ @@ -774,6 +910,46 @@ static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) } } +static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) +{ + struct sysinfo si; + bool is_os_64 = (sizeof(void *) == 8) ? true : false; + uint64_t total_memory; + uint64_t dram_size_seven_GB = 0x1B8000000; + uint64_t dram_size_three_GB = 0xB8000000; + + if (amdgpu_smu_memory_pool_size == 0) + return; + + if (!is_os_64) { + DRM_WARN("Not 64-bit OS, feature not supported\n"); + goto def_value; + } + si_meminfo(&si); + total_memory = (uint64_t)si.totalram * si.mem_unit; + + if ((amdgpu_smu_memory_pool_size == 1) || + (amdgpu_smu_memory_pool_size == 2)) { + if (total_memory < dram_size_three_GB) + goto def_value1; + } else if ((amdgpu_smu_memory_pool_size == 4) || + (amdgpu_smu_memory_pool_size == 8)) { + if (total_memory < dram_size_seven_GB) + goto def_value1; + } else { + DRM_WARN("Smu memory pool size not supported\n"); + goto def_value; + } + adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; + + return; + +def_value1: + DRM_WARN("No enough system memory\n"); +def_value: + adev->pm.smu_prv_buffer_size = 0; +} + /** * amdgpu_device_check_arguments - validate module params * @@ -815,6 +991,8 @@ static void amdgpu_device_check_arguments(struct amdgpu_device *adev) amdgpu_vm_fragment_size = -1; } + amdgpu_device_check_smu_prv_buffer_size(adev); + amdgpu_device_check_vm_size(adev); amdgpu_device_check_block_size(adev); @@ -830,6 +1008,8 @@ static void amdgpu_device_check_arguments(struct amdgpu_device *adev) dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n"); amdgpu_lockup_timeout = 10000; } + + adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); } /** @@ -893,10 +1073,22 @@ static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { .can_switch = amdgpu_switcheroo_can_switch, }; -int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev, +/** + * amdgpu_device_ip_set_clockgating_state - set the CG state + * + * @dev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * @state: clockgating state (gate or ungate) + * + * Sets the requested clockgating state for all instances of + * the hardware IP specified. + * Returns the error code from the last instance. + */ +int amdgpu_device_ip_set_clockgating_state(void *dev, enum amd_ip_block_type block_type, enum amd_clockgating_state state) { + struct amdgpu_device *adev = dev; int i, r = 0; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -915,10 +1107,22 @@ int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev, return r; } -int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev, +/** + * amdgpu_device_ip_set_powergating_state - set the PG state + * + * @dev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * @state: powergating state (gate or ungate) + * + * Sets the requested powergating state for all instances of + * the hardware IP specified. + * Returns the error code from the last instance. + */ +int amdgpu_device_ip_set_powergating_state(void *dev, enum amd_ip_block_type block_type, enum amd_powergating_state state) { + struct amdgpu_device *adev = dev; int i, r = 0; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -937,6 +1141,17 @@ int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev, return r; } +/** + * amdgpu_device_ip_get_clockgating_state - get the CG state + * + * @adev: amdgpu_device pointer + * @flags: clockgating feature flags + * + * Walks the list of IPs on the device and updates the clockgating + * flags for each IP. + * Updates @flags with the feature flags for each hardware IP where + * clockgating is enabled. + */ void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, u32 *flags) { @@ -950,6 +1165,15 @@ void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, } } +/** + * amdgpu_device_ip_wait_for_idle - wait for idle + * + * @adev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * + * Waits for the request hardware IP to be idle. + * Returns 0 for success or a negative error code on failure. + */ int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, enum amd_ip_block_type block_type) { @@ -969,6 +1193,15 @@ int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, } +/** + * amdgpu_device_ip_is_idle - is the hardware IP idle + * + * @adev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * + * Check if the hardware IP is idle or not. + * Returns true if it the IP is idle, false if not. + */ bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, enum amd_ip_block_type block_type) { @@ -984,6 +1217,15 @@ bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, } +/** + * amdgpu_device_ip_get_ip_block - get a hw IP pointer + * + * @adev: amdgpu_device pointer + * @type: Type of hardware IP (SMU, GFX, UVD, etc.) + * + * Returns a pointer to the hardware IP block structure + * if it exists for the asic, otherwise NULL. + */ struct amdgpu_ip_block * amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, enum amd_ip_block_type type) @@ -1037,7 +1279,7 @@ int amdgpu_device_ip_block_add(struct amdgpu_device *adev, if (!ip_block_version) return -EINVAL; - DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks, + DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, ip_block_version->funcs->name); adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; @@ -1045,6 +1287,18 @@ int amdgpu_device_ip_block_add(struct amdgpu_device *adev, return 0; } +/** + * amdgpu_device_enable_virtual_display - enable virtual display feature + * + * @adev: amdgpu_device pointer + * + * Enabled the virtual display feature if the user has enabled it via + * the module parameter virtual_display. This feature provides a virtual + * display hardware on headless boards or in virtualized environments. + * This function parses and validates the configuration string specified by + * the user and configues the virtual display configuration (number of + * virtual connectors, crtcs, etc.) specified. + */ static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) { adev->enable_virtual_display = false; @@ -1090,6 +1344,16 @@ static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) } } +/** + * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware + * + * @adev: amdgpu_device pointer + * + * Parses the asic configuration parameters specified in the gpu info + * firmware and makes them availale to the driver for use in configuring + * the asic. + * Returns 0 on success, -EINVAL on failure. + */ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) { const char *chip_name; @@ -1103,9 +1367,10 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) case CHIP_TOPAZ: case CHIP_TONGA: case CHIP_FIJI: - case CHIP_POLARIS11: case CHIP_POLARIS10: + case CHIP_POLARIS11: case CHIP_POLARIS12: + case CHIP_VEGAM: case CHIP_CARRIZO: case CHIP_STONEY: #ifdef CONFIG_DRM_AMDGPU_SI @@ -1122,11 +1387,15 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) case CHIP_KABINI: case CHIP_MULLINS: #endif + case CHIP_VEGA20: default: return 0; case CHIP_VEGA10: chip_name = "vega10"; break; + case CHIP_VEGA12: + chip_name = "vega12"; + break; case CHIP_RAVEN: chip_name = "raven"; break; @@ -1188,6 +1457,16 @@ out: return err; } +/** + * amdgpu_device_ip_early_init - run early init for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Early initialization pass for hardware IPs. The hardware IPs that make + * up each asic are discovered each IP's early_init callback is run. This + * is the first stage in initializing the asic. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) { int i, r; @@ -1198,9 +1477,10 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) case CHIP_TOPAZ: case CHIP_TONGA: case CHIP_FIJI: - case CHIP_POLARIS11: case CHIP_POLARIS10: + case CHIP_POLARIS11: case CHIP_POLARIS12: + case CHIP_VEGAM: case CHIP_CARRIZO: case CHIP_STONEY: if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) @@ -1240,8 +1520,10 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) return r; break; #endif - case CHIP_VEGA10: - case CHIP_RAVEN: + case CHIP_VEGA10: + case CHIP_VEGA12: + case CHIP_VEGA20: + case CHIP_RAVEN: if (adev->asic_type == CHIP_RAVEN) adev->family = AMDGPU_FAMILY_RV; else @@ -1268,6 +1550,8 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) return -EAGAIN; } + adev->powerplay.pp_feature = amdgpu_pp_feature_mask; + for (i = 0; i < adev->num_ip_blocks; i++) { if ((amdgpu_ip_block_mask & (1 << i)) == 0) { DRM_ERROR("disabled ip block: %d <%s>\n", @@ -1297,6 +1581,17 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_ip_init - run init for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main initialization pass for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the sw_init and hw_init callbacks + * are run. sw_init initializes the software state associated with each IP + * and hw_init initializes the hardware associated with each IP. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_init(struct amdgpu_device *adev) { int i, r; @@ -1311,6 +1606,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) return r; } adev->ip_blocks[i].status.sw = true; + /* need to do gmc hw init early so we can allocate gpu mem */ if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { r = amdgpu_device_vram_scratch_init(adev); @@ -1344,8 +1640,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) for (i = 0; i < adev->num_ip_blocks; i++) { if (!adev->ip_blocks[i].status.sw) continue; - /* gmc hw init is done early */ - if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) + if (adev->ip_blocks[i].status.hw) continue; r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); if (r) { @@ -1364,27 +1659,62 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer + * + * @adev: amdgpu_device pointer + * + * Writes a reset magic value to the gart pointer in VRAM. The driver calls + * this function before a GPU reset. If the value is retained after a + * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents. + */ static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) { memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); } +/** + * amdgpu_device_check_vram_lost - check if vram is valid + * + * @adev: amdgpu_device pointer + * + * Checks the reset magic value written to the gart pointer in VRAM. + * The driver calls this after a GPU reset to see if the contents of + * VRAM is lost or now. + * returns true if vram is lost, false if not. + */ static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) { return !!memcmp(adev->gart.ptr, adev->reset_magic, AMDGPU_RESET_MAGIC_NUM); } +/** + * amdgpu_device_ip_late_set_cg_state - late init for clockgating + * + * @adev: amdgpu_device pointer + * + * Late initialization pass enabling clockgating for hardware IPs. + * The list of all the hardware IPs that make up the asic is walked and the + * set_clockgating_state callbacks are run. This stage is run late + * in the init process. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev) { int i = 0, r; + if (amdgpu_emu_mode == 1) + return 0; + for (i = 0; i < adev->num_ip_blocks; i++) { if (!adev->ip_blocks[i].status.valid) continue; /* skip CG for VCE/UVD, it's handled specially */ if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && - adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) { + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && + adev->ip_blocks[i].version->funcs->set_clockgating_state) { /* enable clockgating to save power */ r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, AMD_CG_STATE_GATE); @@ -1395,9 +1725,50 @@ static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev) } } } + return 0; } +static int amdgpu_device_ip_late_set_pg_state(struct amdgpu_device *adev) +{ + int i = 0, r; + + if (amdgpu_emu_mode == 1) + return 0; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + /* skip CG for VCE/UVD, it's handled specially */ + if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && + adev->ip_blocks[i].version->funcs->set_powergating_state) { + /* enable powergating to save power */ + r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, + AMD_PG_STATE_GATE); + if (r) { + DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } + } + return 0; +} + +/** + * amdgpu_device_ip_late_init - run late init for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Late initialization pass for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the late_init callbacks are run. + * late_init covers any special initialization that an IP requires + * after all of the have been initialized or something that needs to happen + * late in the init process. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) { int i = 0, r; @@ -1416,14 +1787,28 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) } } - mod_delayed_work(system_wq, &adev->late_init_work, - msecs_to_jiffies(AMDGPU_RESUME_MS)); + amdgpu_device_ip_late_set_cg_state(adev); + amdgpu_device_ip_late_set_pg_state(adev); + + queue_delayed_work(system_wq, &adev->late_init_work, + msecs_to_jiffies(AMDGPU_RESUME_MS)); amdgpu_device_fill_reset_magic(adev); return 0; } +/** + * amdgpu_device_ip_fini - run fini for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main teardown pass for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks + * are run. hw_fini tears down the hardware associated with each IP + * and sw_fini tears down any software state associated with each IP. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) { int i, r; @@ -1433,7 +1818,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) for (i = 0; i < adev->num_ip_blocks; i++) { if (!adev->ip_blocks[i].status.hw) continue; - if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC && + adev->ip_blocks[i].version->funcs->set_clockgating_state) { /* ungate blocks before hw fini so that we can shutdown the blocks safely */ r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, AMD_CG_STATE_UNGATE); @@ -1442,6 +1828,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) adev->ip_blocks[i].version->funcs->name, r); return r; } + if (adev->powerplay.pp_funcs->set_powergating_by_smu) + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false); r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); /* XXX handle errors */ if (r) { @@ -1458,7 +1846,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) continue; if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && - adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) { + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && + adev->ip_blocks[i].version->funcs->set_clockgating_state) { /* ungate blocks before hw fini so that we can shutdown the blocks safely */ r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, AMD_CG_STATE_UNGATE); @@ -1479,6 +1869,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) adev->ip_blocks[i].status.hw = false; } + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { if (!adev->ip_blocks[i].status.sw) continue; @@ -1514,14 +1905,86 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_ip_late_init_func_handler - work handler for clockgating + * + * @work: work_struct + * + * Work handler for amdgpu_device_ip_late_set_cg_state. We put the + * clockgating setup into a worker thread to speed up driver init and + * resume from suspend. + */ static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work) { struct amdgpu_device *adev = container_of(work, struct amdgpu_device, late_init_work.work); - amdgpu_device_ip_late_set_cg_state(adev); + int r; + + r = amdgpu_ib_ring_tests(adev); + if (r) + DRM_ERROR("ib ring test failed (%d).\n", r); } -int amdgpu_device_ip_suspend(struct amdgpu_device *adev) +/** + * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) + * + * @adev: amdgpu_device pointer + * + * Main suspend function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked, clockgating is disabled and the + * suspend callbacks are run. suspend puts the hardware and software state + * in each IP into a state suitable for suspend. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) +{ + int i, r; + + if (amdgpu_sriov_vf(adev)) + amdgpu_virt_request_full_gpu(adev, false); + + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { + if (!adev->ip_blocks[i].status.valid) + continue; + /* displays are handled separately */ + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { + /* ungate blocks so that suspend can properly shut them down */ + if (adev->ip_blocks[i].version->funcs->set_clockgating_state) { + r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, + AMD_CG_STATE_UNGATE); + if (r) { + DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + } + /* XXX handle errors */ + r = adev->ip_blocks[i].version->funcs->suspend(adev); + /* XXX handle errors */ + if (r) { + DRM_ERROR("suspend of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + } + } + + if (amdgpu_sriov_vf(adev)) + amdgpu_virt_release_full_gpu(adev, false); + + return 0; +} + +/** + * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) + * + * @adev: amdgpu_device pointer + * + * Main suspend function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked, clockgating is disabled and the + * suspend callbacks are run. suspend puts the hardware and software state + * in each IP into a state suitable for suspend. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) { int i, r; @@ -1535,11 +1998,19 @@ int amdgpu_device_ip_suspend(struct amdgpu_device *adev) DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r); } + /* call smu to disable gfx off feature first when suspend */ + if (adev->powerplay.pp_funcs->set_powergating_by_smu) + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false); + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { if (!adev->ip_blocks[i].status.valid) continue; + /* displays are handled in phase1 */ + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) + continue; /* ungate blocks so that suspend can properly shut them down */ - if (i != AMD_IP_BLOCK_TYPE_SMC) { + if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC && + adev->ip_blocks[i].version->funcs->set_clockgating_state) { r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, AMD_CG_STATE_UNGATE); if (r) { @@ -1562,6 +2033,29 @@ int amdgpu_device_ip_suspend(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_ip_suspend - run suspend for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main suspend function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked, clockgating is disabled and the + * suspend callbacks are run. suspend puts the hardware and software state + * in each IP into a state suitable for suspend. + * Returns 0 on success, negative error code on failure. + */ +int amdgpu_device_ip_suspend(struct amdgpu_device *adev) +{ + int r; + + r = amdgpu_device_ip_suspend_phase1(adev); + if (r) + return r; + r = amdgpu_device_ip_suspend_phase2(adev); + + return r; +} + static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) { int i, r; @@ -1569,6 +2063,7 @@ static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) static enum amd_ip_block_type ip_order[] = { AMD_IP_BLOCK_TYPE_GMC, AMD_IP_BLOCK_TYPE_COMMON, + AMD_IP_BLOCK_TYPE_PSP, AMD_IP_BLOCK_TYPE_IH, }; @@ -1584,7 +2079,9 @@ static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) continue; r = block->version->funcs->hw_init(adev); - DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed"); + DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); + if (r) + return r; } } @@ -1597,7 +2094,6 @@ static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) static enum amd_ip_block_type ip_order[] = { AMD_IP_BLOCK_TYPE_SMC, - AMD_IP_BLOCK_TYPE_PSP, AMD_IP_BLOCK_TYPE_DCE, AMD_IP_BLOCK_TYPE_GFX, AMD_IP_BLOCK_TYPE_SDMA, @@ -1617,13 +2113,27 @@ static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) continue; r = block->version->funcs->hw_init(adev); - DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed"); + DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); + if (r) + return r; } } return 0; } +/** + * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs + * + * @adev: amdgpu_device pointer + * + * First resume function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the resume callbacks are run for + * COMMON, GMC, and IH. resume puts the hardware into a functional state + * after a suspend and updates the software state as necessary. This + * function is also used for restoring the GPU after a GPU reset. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) { int i, r; @@ -1632,9 +2142,8 @@ static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) if (!adev->ip_blocks[i].status.valid) continue; if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || - adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || - adev->ip_blocks[i].version->type == - AMD_IP_BLOCK_TYPE_IH) { + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { r = adev->ip_blocks[i].version->funcs->resume(adev); if (r) { DRM_ERROR("resume of IP block <%s> failed %d\n", @@ -1647,6 +2156,19 @@ static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs + * + * @adev: amdgpu_device pointer + * + * First resume function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the resume callbacks are run for + * all blocks except COMMON, GMC, and IH. resume puts the hardware into a + * functional state after a suspend and updates the software state as + * necessary. This function is also used for restoring the GPU after a GPU + * reset. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) { int i, r; @@ -1655,8 +2177,8 @@ static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) if (!adev->ip_blocks[i].status.valid) continue; if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || - adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || - adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ) + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) continue; r = adev->ip_blocks[i].version->funcs->resume(adev); if (r) { @@ -1669,6 +2191,18 @@ static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_ip_resume - run resume for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main resume function for hardware IPs. The hardware IPs + * are split into two resume functions because they are + * are also used in in recovering from a GPU reset and some additional + * steps need to be take between them. In this case (S3/S4) they are + * run sequentially. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_resume(struct amdgpu_device *adev) { int r; @@ -1681,6 +2215,13 @@ static int amdgpu_device_ip_resume(struct amdgpu_device *adev) return r; } +/** + * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV + * + * @adev: amdgpu_device pointer + * + * Query the VBIOS data tables to determine if the board supports SR-IOV. + */ static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) { if (amdgpu_sriov_vf(adev)) { @@ -1697,27 +2238,42 @@ static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) } } +/** + * amdgpu_device_asic_has_dc_support - determine if DC supports the asic + * + * @asic_type: AMD asic type + * + * Check if there is DC (new modesetting infrastructre) support for an asic. + * returns true if DC has support, false if not. + */ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) { switch (asic_type) { #if defined(CONFIG_DRM_AMD_DC) case CHIP_BONAIRE: - case CHIP_HAWAII: case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_MULLINS: + /* + * We have systems in the wild with these ASICs that require + * LVDS and VGA support which is not supported with DC. + * + * Fallback to the non-DC driver here by default so as not to + * cause regressions. + */ + return amdgpu_dc > 0; + case CHIP_HAWAII: case CHIP_CARRIZO: case CHIP_STONEY: - case CHIP_POLARIS11: case CHIP_POLARIS10: + case CHIP_POLARIS11: case CHIP_POLARIS12: + case CHIP_VEGAM: case CHIP_TONGA: case CHIP_FIJI: -#if defined(CONFIG_DRM_AMD_DC_PRE_VEGA) - return amdgpu_dc != 0; -#endif - case CHIP_KABINI: - case CHIP_MULLINS: - return amdgpu_dc > 0; case CHIP_VEGA10: + case CHIP_VEGA12: + case CHIP_VEGA20: #if defined(CONFIG_DRM_AMD_DC_DCN1_0) case CHIP_RAVEN: #endif @@ -1747,7 +2303,7 @@ bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) * amdgpu_device_init - initialize the driver * * @adev: amdgpu_device pointer - * @pdev: drm dev pointer + * @ddev: drm dev pointer * @pdev: pci dev pointer * @flags: driver flags * @@ -1771,14 +2327,16 @@ int amdgpu_device_init(struct amdgpu_device *adev, adev->flags = flags; adev->asic_type = flags & AMD_ASIC_MASK; adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; - adev->mc.gart_size = 512 * 1024 * 1024; + if (amdgpu_emu_mode == 1) + adev->usec_timeout *= 2; + adev->gmc.gart_size = 512 * 1024 * 1024; adev->accel_working = false; adev->num_rings = 0; adev->mman.buffer_funcs = NULL; adev->mman.buffer_funcs_ring = NULL; adev->vm_manager.vm_pte_funcs = NULL; adev->vm_manager.vm_pte_num_rings = 0; - adev->gart.gart_funcs = NULL; + adev->gmc.gmc_funcs = NULL; adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); @@ -1836,6 +2394,8 @@ int amdgpu_device_init(struct amdgpu_device *adev, INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_device_ip_late_init_func_handler); + adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false; + /* Registers mapping */ /* TODO: block userspace mapping of io register */ if (adev->asic_type >= CHIP_BONAIRE) { @@ -1867,6 +2427,8 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (adev->rio_mem == NULL) DRM_INFO("PCI I/O BAR is not found.\n"); + amdgpu_device_get_pcie_info(adev); + /* early init functions */ r = amdgpu_device_ip_early_init(adev); if (r) @@ -1885,6 +2447,12 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (runtime) vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); + if (amdgpu_emu_mode == 1) { + /* post the asic on emulation mode */ + emu_soc_asic_init(adev); + goto fence_driver_init; + } + /* Read BIOS */ if (!amdgpu_get_bios(adev)) { r = -EINVAL; @@ -1937,6 +2505,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, amdgpu_atombios_i2c_init(adev); } +fence_driver_init: /* Fence driver */ r = amdgpu_fence_driver_init(adev); if (r) { @@ -1964,7 +2533,6 @@ int amdgpu_device_init(struct amdgpu_device *adev, } dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); - amdgpu_device_ip_fini(adev); goto failed; } @@ -1987,10 +2555,6 @@ int amdgpu_device_init(struct amdgpu_device *adev, goto failed; } - r = amdgpu_ib_ring_tests(adev); - if (r) - DRM_ERROR("ib ring test failed (%d).\n", r); - if (amdgpu_sriov_vf(adev)) amdgpu_virt_init_data_exchange(adev); @@ -2063,6 +2627,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev) DRM_INFO("amdgpu: finishing device.\n"); adev->shutdown = true; + /* disable all interrupts */ + amdgpu_irq_disable_all(adev); if (adev->mode_info.mode_config_initialized){ if (!amdgpu_device_has_dc_support(adev)) drm_crtc_force_disable_all(adev->ddev); @@ -2071,6 +2637,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) } amdgpu_ib_pool_fini(adev); amdgpu_fence_driver_fini(adev); + amdgpu_pm_sysfs_fini(adev); amdgpu_fbdev_fini(adev); r = amdgpu_device_ip_fini(adev); if (adev->firmware.gpu_info_fw) { @@ -2082,7 +2649,10 @@ void amdgpu_device_fini(struct amdgpu_device *adev) /* free i2c buses */ if (!amdgpu_device_has_dc_support(adev)) amdgpu_i2c_fini(adev); - amdgpu_atombios_fini(adev); + + if (amdgpu_emu_mode != 1) + amdgpu_atombios_fini(adev); + kfree(adev->bios); adev->bios = NULL; if (!pci_is_thunderbolt_attached(adev->pdev)) @@ -2096,7 +2666,6 @@ void amdgpu_device_fini(struct amdgpu_device *adev) iounmap(adev->rmmio); adev->rmmio = NULL; amdgpu_device_doorbell_fini(adev); - amdgpu_pm_sysfs_fini(adev); amdgpu_debugfs_regs_cleanup(adev); } @@ -2107,8 +2676,9 @@ void amdgpu_device_fini(struct amdgpu_device *adev) /** * amdgpu_device_suspend - initiate device suspend * - * @pdev: drm dev pointer - * @state: suspend state + * @dev: drm dev pointer + * @suspend: suspend state + * @fbcon : notify the fbdev of suspend * * Puts the hw in the suspend state (all asics). * Returns 0 for success or an error on failure. @@ -2132,6 +2702,9 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon) drm_kms_helper_poll_disable(dev); + if (fbcon) + amdgpu_fbdev_set_suspend(adev, 1); + if (!amdgpu_device_has_dc_support(adev)) { /* turn off display hw */ drm_modeset_lock_all(dev); @@ -2139,44 +2712,46 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon) drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); } drm_modeset_unlock_all(dev); - } - - amdgpu_amdkfd_suspend(adev); - - /* unpin the front buffers and cursors */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb); - struct amdgpu_bo *robj; - - if (amdgpu_crtc->cursor_bo) { - struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); - r = amdgpu_bo_reserve(aobj, true); - if (r == 0) { - amdgpu_bo_unpin(aobj); - amdgpu_bo_unreserve(aobj); + /* unpin the front buffers and cursors */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_framebuffer *fb = crtc->primary->fb; + struct amdgpu_bo *robj; + + if (amdgpu_crtc->cursor_bo) { + struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); + r = amdgpu_bo_reserve(aobj, true); + if (r == 0) { + amdgpu_bo_unpin(aobj); + amdgpu_bo_unreserve(aobj); + } } - } - if (rfb == NULL || rfb->obj == NULL) { - continue; - } - robj = gem_to_amdgpu_bo(rfb->obj); - /* don't unpin kernel fb objects */ - if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { - r = amdgpu_bo_reserve(robj, true); - if (r == 0) { - amdgpu_bo_unpin(robj); - amdgpu_bo_unreserve(robj); + if (fb == NULL || fb->obj[0] == NULL) { + continue; + } + robj = gem_to_amdgpu_bo(fb->obj[0]); + /* don't unpin kernel fb objects */ + if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { + r = amdgpu_bo_reserve(robj, true); + if (r == 0) { + amdgpu_bo_unpin(robj); + amdgpu_bo_unreserve(robj); + } } } } + + amdgpu_amdkfd_suspend(adev); + + r = amdgpu_device_ip_suspend_phase1(adev); + /* evict vram memory */ amdgpu_bo_evict_vram(adev); amdgpu_fence_driver_suspend(adev); - r = amdgpu_device_ip_suspend(adev); + r = amdgpu_device_ip_suspend_phase2(adev); /* evict remaining vram memory * This second call to evict vram is to evict the gart page table @@ -2195,18 +2770,15 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon) DRM_ERROR("amdgpu asic reset failed\n"); } - if (fbcon) { - console_lock(); - amdgpu_fbdev_set_suspend(adev, 1); - console_unlock(); - } return 0; } /** * amdgpu_device_resume - initiate device resume * - * @pdev: drm dev pointer + * @dev: drm dev pointer + * @resume: resume state + * @fbcon : notify the fbdev of resume * * Bring the hw back to operating state (all asics). * Returns 0 for success or an error on failure. @@ -2222,15 +2794,12 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - if (fbcon) - console_lock(); - if (resume) { pci_set_power_state(dev->pdev, PCI_D0); pci_restore_state(dev->pdev); r = pci_enable_device(dev->pdev); if (r) - goto unlock; + return r; } /* post card */ @@ -2243,34 +2812,30 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) r = amdgpu_device_ip_resume(adev); if (r) { DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r); - goto unlock; + return r; } amdgpu_fence_driver_resume(adev); - if (resume) { - r = amdgpu_ib_ring_tests(adev); - if (r) - DRM_ERROR("ib ring test failed (%d).\n", r); - } r = amdgpu_device_ip_late_init(adev); if (r) - goto unlock; - - /* pin cursors */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - - if (amdgpu_crtc->cursor_bo) { - struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); - r = amdgpu_bo_reserve(aobj, true); - if (r == 0) { - r = amdgpu_bo_pin(aobj, - AMDGPU_GEM_DOMAIN_VRAM, - &amdgpu_crtc->cursor_addr); - if (r != 0) - DRM_ERROR("Failed to pin cursor BO (%d)\n", r); - amdgpu_bo_unreserve(aobj); + return r; + + if (!amdgpu_device_has_dc_support(adev)) { + /* pin cursors */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + + if (amdgpu_crtc->cursor_bo) { + struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); + r = amdgpu_bo_reserve(aobj, true); + if (r == 0) { + r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM); + if (r != 0) + DRM_ERROR("Failed to pin cursor BO (%d)\n", r); + amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj); + amdgpu_bo_unreserve(aobj); + } } } } @@ -2278,6 +2843,9 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) if (r) return r; + /* Make sure IB tests flushed */ + flush_delayed_work(&adev->late_init_work); + /* blat the mode back in */ if (fbcon) { if (!amdgpu_device_has_dc_support(adev)) { @@ -2291,6 +2859,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) } drm_modeset_unlock_all(dev); } + amdgpu_fbdev_set_suspend(adev, 0); } drm_kms_helper_poll_enable(dev); @@ -2314,17 +2883,19 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) #ifdef CONFIG_PM dev->dev->power.disable_depth--; #endif - - if (fbcon) - amdgpu_fbdev_set_suspend(adev, 0); - -unlock: - if (fbcon) - console_unlock(); - - return r; + return 0; } +/** + * amdgpu_device_ip_check_soft_reset - did soft reset succeed + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and + * the check_soft_reset callbacks are run. check_soft_reset determines + * if the asic is still hung or not. + * Returns true if any of the IPs are still in a hung state, false if not. + */ static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) { int i; @@ -2333,6 +2904,9 @@ static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) if (amdgpu_sriov_vf(adev)) return true; + if (amdgpu_asic_need_full_reset(adev)) + return true; + for (i = 0; i < adev->num_ip_blocks; i++) { if (!adev->ip_blocks[i].status.valid) continue; @@ -2347,6 +2921,17 @@ static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) return asic_hang; } +/** + * amdgpu_device_ip_pre_soft_reset - prepare for soft reset + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and the + * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset + * handles any IP specific hardware or software state changes that are + * necessary for a soft reset to succeed. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) { int i, r = 0; @@ -2365,10 +2950,22 @@ static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed + * + * @adev: amdgpu_device pointer + * + * Some hardware IPs cannot be soft reset. If they are hung, a full gpu + * reset is necessary to recover. + * Returns true if a full asic reset is required, false if not. + */ static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) { int i; + if (amdgpu_asic_need_full_reset(adev)) + return true; + for (i = 0; i < adev->num_ip_blocks; i++) { if (!adev->ip_blocks[i].status.valid) continue; @@ -2386,6 +2983,17 @@ static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) return false; } +/** + * amdgpu_device_ip_soft_reset - do a soft reset + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and the + * soft_reset callbacks are run if the block is hung. soft_reset handles any + * IP specific hardware or software state changes that are necessary to soft + * reset the IP. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) { int i, r = 0; @@ -2404,6 +3012,17 @@ static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_ip_post_soft_reset - clean up from soft reset + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and the + * post_soft_reset callbacks are run if the asic was hung. post_soft_reset + * handles any IP specific hardware or software state changes that are + * necessary after the IP has been soft reset. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) { int i, r = 0; @@ -2421,6 +3040,19 @@ static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_device_recover_vram_from_shadow - restore shadowed VRAM buffers + * + * @adev: amdgpu_device pointer + * @ring: amdgpu_ring for the engine handling the buffer operations + * @bo: amdgpu_bo buffer whose shadow is being restored + * @fence: dma_fence associated with the operation + * + * Restores the VRAM buffer contents from the shadow in GTT. Used to + * restore things like GPUVM page tables after a GPU reset where + * the contents of VRAM might be lost. + * Returns 0 on success, negative error code on failure. + */ static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev, struct amdgpu_ring *ring, struct amdgpu_bo *bo, @@ -2456,17 +3088,81 @@ err: return r; } -/* +/** + * amdgpu_device_handle_vram_lost - Handle the loss of VRAM contents + * + * @adev: amdgpu_device pointer + * + * Restores the contents of VRAM buffers from the shadows in GTT. Used to + * restore things like GPUVM page tables after a GPU reset where + * the contents of VRAM might be lost. + * Returns 0 on success, 1 on failure. + */ +static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev) +{ + struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + struct amdgpu_bo *bo, *tmp; + struct dma_fence *fence = NULL, *next = NULL; + long r = 1; + int i = 0; + long tmo; + + if (amdgpu_sriov_runtime(adev)) + tmo = msecs_to_jiffies(8000); + else + tmo = msecs_to_jiffies(100); + + DRM_INFO("recover vram bo from shadow start\n"); + mutex_lock(&adev->shadow_list_lock); + list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) { + next = NULL; + amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next); + if (fence) { + r = dma_fence_wait_timeout(fence, false, tmo); + if (r == 0) + pr_err("wait fence %p[%d] timeout\n", fence, i); + else if (r < 0) + pr_err("wait fence %p[%d] interrupted\n", fence, i); + if (r < 1) { + dma_fence_put(fence); + fence = next; + break; + } + i++; + } + + dma_fence_put(fence); + fence = next; + } + mutex_unlock(&adev->shadow_list_lock); + + if (fence) { + r = dma_fence_wait_timeout(fence, false, tmo); + if (r == 0) + pr_err("wait fence %p[%d] timeout\n", fence, i); + else if (r < 0) + pr_err("wait fence %p[%d] interrupted\n", fence, i); + + } + dma_fence_put(fence); + + if (r > 0) + DRM_INFO("recover vram bo from shadow done\n"); + else + DRM_ERROR("recover vram bo from shadow failed\n"); + + return (r > 0) ? 0 : 1; +} + +/** * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough * * @adev: amdgpu device pointer - * @reset_flags: output param tells caller the reset result * * attempt to do soft-reset or full-reset and reinitialize Asic - * return 0 means successed otherwise failed -*/ -static int amdgpu_device_reset(struct amdgpu_device *adev, - uint64_t* reset_flags) + * return 0 means succeeded otherwise failed + */ +static int amdgpu_device_reset(struct amdgpu_device *adev) { bool need_full_reset, vram_lost = 0; int r; @@ -2481,7 +3177,6 @@ static int amdgpu_device_reset(struct amdgpu_device *adev, DRM_INFO("soft reset failed, will fallback to full reset!\n"); need_full_reset = true; } - } if (need_full_reset) { @@ -2530,28 +3225,22 @@ out: } } - if (reset_flags) { - if (vram_lost) - (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST; - - if (need_full_reset) - (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET; - } + if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost)) + r = amdgpu_device_handle_vram_lost(adev); return r; } -/* +/** * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf * * @adev: amdgpu device pointer - * @reset_flags: output param tells caller the reset result + * @from_hypervisor: request from hypervisor * * do VF FLR and reinitialize Asic - * return 0 means successed otherwise failed -*/ + * return 0 means succeeded otherwise failed + */ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, - uint64_t *reset_flags, bool from_hypervisor) { int r; @@ -2578,21 +3267,12 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, amdgpu_irq_gpu_reset_resume_helper(adev); r = amdgpu_ib_ring_tests(adev); - if (r) - dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r); error: - /* release full control of GPU after ib test */ amdgpu_virt_release_full_gpu(adev, true); - - if (reset_flags) { - if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { - (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST; - atomic_inc(&adev->vram_lost_counter); - } - - /* VF FLR or hotlink reset is always full-reset */ - (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET; + if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { + atomic_inc(&adev->vram_lost_counter); + r = amdgpu_device_handle_vram_lost(adev); } return r; @@ -2603,7 +3283,7 @@ error: * * @adev: amdgpu device pointer * @job: which job trigger hang - * @force forces reset regardless of amdgpu_gpu_recovery + * @force: forces reset regardless of amdgpu_gpu_recovery * * Attempt to reset the GPU if it has hung (all asics). * Returns 0 for success or an error on failure. @@ -2611,8 +3291,6 @@ error: int amdgpu_device_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job *job, bool force) { - struct drm_atomic_state *state = NULL; - uint64_t reset_flags = 0; int i, r, resched; if (!force && !amdgpu_device_ip_check_soft_reset(adev)) { @@ -2632,99 +3310,52 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, atomic_inc(&adev->gpu_reset_counter); adev->in_gpu_reset = 1; + /* Block kfd */ + amdgpu_amdkfd_pre_reset(adev); + /* block TTM */ resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); - /* store modesetting */ - if (amdgpu_device_has_dc_support(adev)) - state = drm_atomic_helper_suspend(adev->ddev); - /* block scheduler */ + /* block all schedulers and reset given job's ring */ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring = adev->rings[i]; if (!ring || !ring->sched.thread) continue; - /* only focus on the ring hit timeout if &job not NULL */ - if (job && job->ring->idx != i) + kthread_park(ring->sched.thread); + + if (job && job->base.sched == &ring->sched) continue; - kthread_park(ring->sched.thread); - drm_sched_hw_job_reset(&ring->sched, &job->base); + drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); } if (amdgpu_sriov_vf(adev)) - r = amdgpu_device_reset_sriov(adev, &reset_flags, job ? false : true); + r = amdgpu_device_reset_sriov(adev, job ? false : true); else - r = amdgpu_device_reset(adev, &reset_flags); - - if (!r) { - if (((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) || - (reset_flags & AMDGPU_RESET_INFO_VRAM_LOST)) { - struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; - struct amdgpu_bo *bo, *tmp; - struct dma_fence *fence = NULL, *next = NULL; - - DRM_INFO("recover vram bo from shadow\n"); - mutex_lock(&adev->shadow_list_lock); - list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) { - next = NULL; - amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next); - if (fence) { - r = dma_fence_wait(fence, false); - if (r) { - WARN(r, "recovery from shadow isn't completed\n"); - break; - } - } + r = amdgpu_device_reset(adev); - dma_fence_put(fence); - fence = next; - } - mutex_unlock(&adev->shadow_list_lock); - if (fence) { - r = dma_fence_wait(fence, false); - if (r) - WARN(r, "recovery from shadow isn't completed\n"); - } - dma_fence_put(fence); - } - - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - struct amdgpu_ring *ring = adev->rings[i]; - - if (!ring || !ring->sched.thread) - continue; + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; - /* only focus on the ring hit timeout if &job not NULL */ - if (job && job->ring->idx != i) - continue; + if (!ring || !ring->sched.thread) + continue; + /* only need recovery sched of the given job's ring + * or all rings (in the case @job is NULL) + * after above amdgpu_reset accomplished + */ + if ((!job || job->base.sched == &ring->sched) && !r) drm_sched_job_recovery(&ring->sched); - kthread_unpark(ring->sched.thread); - } - } else { - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - struct amdgpu_ring *ring = adev->rings[i]; - if (!ring || !ring->sched.thread) - continue; - - /* only focus on the ring hit timeout if &job not NULL */ - if (job && job->ring->idx != i) - continue; - - kthread_unpark(adev->rings[i]->sched.thread); - } + kthread_unpark(ring->sched.thread); } - if (amdgpu_device_has_dc_support(adev)) { - if (drm_atomic_helper_resume(adev->ddev, state)) - dev_info(adev->dev, "drm resume failed:%d\n", r); - } else { + if (!amdgpu_device_has_dc_support(adev)) { drm_helper_resume_force_mode(adev->ddev); } @@ -2735,19 +3366,31 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter)); amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); } else { - dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter)); + dev_info(adev->dev, "GPU reset(%d) succeeded!\n",atomic_read(&adev->gpu_reset_counter)); } + /*unlock kfd */ + amdgpu_amdkfd_post_reset(adev); amdgpu_vf_error_trans_all(adev); adev->in_gpu_reset = 0; mutex_unlock(&adev->lock_reset); return r; } -void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) +/** + * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot + * + * @adev: amdgpu_device pointer + * + * Fetchs and stores in the driver the PCIE capabilities (gen speed + * and lanes) of the slot the device is in. Handles APUs and + * virtualized environments where PCIE config space may not be available. + */ +static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) { - u32 mask; - int ret; + struct pci_dev *pdev; + enum pci_bus_speed speed_cap; + enum pcie_link_width link_width; if (amdgpu_pcie_gen_cap) adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; @@ -2765,27 +3408,61 @@ void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) } if (adev->pm.pcie_gen_mask == 0) { - ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask); - if (!ret) { - adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + /* asic caps */ + pdev = adev->pdev; + speed_cap = pcie_get_speed_cap(pdev); + if (speed_cap == PCI_SPEED_UNKNOWN) { + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); - - if (mask & DRM_PCIE_SPEED_25) - adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; - if (mask & DRM_PCIE_SPEED_50) - adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2; - if (mask & DRM_PCIE_SPEED_80) - adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3; } else { - adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; + if (speed_cap == PCIE_SPEED_16_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); + else if (speed_cap == PCIE_SPEED_8_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); + else if (speed_cap == PCIE_SPEED_5_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); + else + adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; + } + /* platform caps */ + pdev = adev->ddev->pdev->bus->self; + speed_cap = pcie_get_speed_cap(pdev); + if (speed_cap == PCI_SPEED_UNKNOWN) { + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); + } else { + if (speed_cap == PCIE_SPEED_16_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); + else if (speed_cap == PCIE_SPEED_8_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); + else if (speed_cap == PCIE_SPEED_5_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); + else + adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; + } } if (adev->pm.pcie_mlw_mask == 0) { - ret = drm_pcie_get_max_link_width(adev->ddev, &mask); - if (!ret) { - switch (mask) { - case 32: + pdev = adev->ddev->pdev->bus->self; + link_width = pcie_get_width_cap(pdev); + if (link_width == PCIE_LNK_WIDTH_UNKNOWN) { + adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; + } else { + switch (link_width) { + case PCIE_LNK_X32: adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | @@ -2794,7 +3471,7 @@ void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); break; - case 16: + case PCIE_LNK_X16: adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | @@ -2802,36 +3479,34 @@ void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); break; - case 12: + case PCIE_LNK_X12: adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); break; - case 8: + case PCIE_LNK_X8: adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); break; - case 4: + case PCIE_LNK_X4: adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); break; - case 2: + case PCIE_LNK_X2: adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); break; - case 1: + case PCIE_LNK_X1: adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; break; default: break; } - } else { - adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; } } } |